diff --git a/国内党媒/CrawlRenmingonganbao.py b/国内党媒/CrawlRenmingonganbao.py new file mode 100644 index 0000000..a2ac597 --- /dev/null +++ b/国内党媒/CrawlRenmingonganbao.py @@ -0,0 +1,165 @@ +# _*_ coding : UTF-8 _*_ +# @Time : 2024/11/13 02:34 +# @UpdateTime : 2024/11/13 02:34 +# @Author : haochen zhong +# @File : CrawlRenmingonganbao.py +# @Software : PyCharm +# @Comment : 本程序采集人民公安报数据 + +import asyncio +import random +import re +from datetime import datetime, timedelta + +from bs4 import BeautifulSoup +from httpx import AsyncClient +from motor.motor_asyncio import AsyncIOMotorClient + +pattern = r"window\.location\.href=['\"](.*?)['\"]" +"""正则匹配ID""" +start_date = datetime.strptime('2020-04-01', '%Y-%m-%d') +"""人民公安报2020年4月1日开始有数据""" +end_date = datetime.today() +"""截止到今天""" +headers = { + 'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.42'} + +# 链接数据库 +client = AsyncIOMotorClient('mongodb://localhost:27017') +db = client['buweijiguanbao'] +collection = db['renmingonganbao'] + + +async def main(): + collection_names = await db.list_collection_names() + # 判断数据表是否存在 + if "renmingonganbao" not in collection_names: + # 如果不存在，则从2017年9月开始爬取 + print("中国民族数据表不存在，开始采集！") + await getData(start_date, end_date) + else: + # 如果存在，则从数据库中获取最后一条记录的日期 + last_record = await collection.find_one({}, sort=[('release_time', -1)]) + last_date_str = last_record['release_time'] + print("数据库截止时间:", last_date_str) + await getData(last_date_str, end_date) + + +async def getContent(soup: BeautifulSoup) -> str: + """ + :param soup: BeautifulSoup对象 + :return: 文章内容 + """ + content = "" + for p in soup.select("div.content_main > div:nth-child(2) .contentP"): + para = p.text.strip() + if para: + content += para + content += '\n' + return content + + +async def getData(start_date: datetime, end_date: datetime): + """" + :param start_date: 开始日期 + :param end_date: 结束日期 + :return: None + """ + crawl_num = 0 + for i in range((end_date - start_date).days): + date_now = start_date + timedelta(days=i + 1) + date_now_s = date_now.strftime('%Y_%m_%d') + url = f"http://epaper.cpd.com.cn/szb/wwwcpd_9/dzb_16465/rmga/{date_now.year}/{date_now_s}/" + """http://epaper.cpd.com.cn/szb/wwwcpd_9/dzb_16465/rmga/2020/2020_04_05/""" + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url) + try: + async with AsyncClient(headers=headers, timeout=60) as client: + response = await client.get(url) + response.encoding = response.charset_encoding + text = response.text + match = re.search(pattern, text) + if match: + url = url + match.group(1).replace("./", "") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url) + response = await client.get(url) + response.encoding = response.charset_encoding + print(f"一级连接状态：{response.status_code}") + if response.status_code == 200: + soup = BeautifulSoup(response.text, 'lxml') + cont_main = soup.select_one(".tab-title") + if cont_main: + script_tag = cont_main.find("script") + if script_tag: + script_content = script_tag.string + soup1 = BeautifulSoup( + script_content.replace("document.write('", "").replace("')", "").replace(";", + "").replace( + "'+'", "").strip(), "lxml") + for item in soup1.select(".title-item"): + banmianming = item.select_one(".title").text.split(" ")[-1] + banmianhao = item.select_one(".title").text.split(" ")[0] + for item2 in item.select(".title-list a"): + title = item2.text + url2 = item2.get("href") + print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url2) + response2 = await client.get(url2) + response2.encoding = response2.charset_encoding + print(f"二级连接状态：{response2.status_code}") + if response2.status_code == 200: + soup2 = BeautifulSoup(response2.text, 'lxml') + try: + title = soup2.select_one( + ".cont-box > div>div:nth-child(2)").text.strip() + except: + title = title + try: + subtitle = soup2.select_one( + ".cont-box > div>div:nth-child(3)").text.strip() + except: + subtitle = "" + try: + preTitle = soup2.select_one( + ".cont-box > div>div:nth-child(1)").text.strip() + except: + preTitle = "" + content = await getContent(soup2) + await collection.insert_one({ + "title": title, + "subtitle": subtitle, + "preTitle": preTitle, + "author": "", + "banmianming": banmianming, + "banmianhao": banmianhao, + 'keywordlist': "empty", + 'detail_url': url2, + 'release_time': date_now, + 'insert_timestamp': datetime.today(), + 'content': content + }) + crawl_num += 1 + print( + f"人民公安报---{date_now_s}---{banmianming}---{banmianhao}---{title}---采集完成！") + await asyncio.sleep(random.randint(5, 15)) + print(f"人民公安报---{date_now_s}---{banmianming}---{banmianhao}-----采集完成！") + await asyncio.sleep(random.randint(5, 15)) + print(f"人民公安报---{date_now_s}-----采集完成！") + await asyncio.sleep(random.randint(5, 15)) + except Exception as e: + await collection.insert_one( + {'banmianhao': 'empty', + 'banmianming': 'empty', + 'preTitle': 'empty', + 'title': 'empty', + 'subtitle': 'empty', + 'author': 'empty', + 'keywordlist': 'empty', + 'detail_url': url, + 'release_time': date_now, + 'insert_timestamp': datetime.today(), + 'content': 'empty'} + ) + print(e) + print(f"人民公安报采集完毕，共采集{crawl_num}条数据！") + + +asyncio.run(main()) diff --git a/国内党媒/CrawlRenminyoudianbao.py b/国内党媒/CrawlRenminyoudianbao.py index e99ad26..e86e297 100644 --- a/国内党媒/CrawlRenminyoudianbao.py +++ b/国内党媒/CrawlRenminyoudianbao.py @@ -15,8 +15,8 @@ from bs4 import BeautifulSoup from httpx import AsyncClient from motor.motor_asyncio import AsyncIOMotorClient -start_date = datetime.strptime('2017-09', '%Y-%m') -"""中国改革报2017年9月份开始有数据""" +start_date = datetime.strptime('2024-09', '%Y-%m') +"""人民邮电报2024年9月份开始有数据""" end_date = datetime.today() """截止到今天""" headers = {