From 4cb5d4a5d37654353684cbf395f5d97161e7fb8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9A=93=E6=9C=88=E5=BD=92=E5=B0=98?= Date: Sun, 10 Nov 2024 00:17:12 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E4=B8=AD=E5=9B=BD?= =?UTF-8?q?=E6=B0=91=E6=97=8F=E6=8A=A5=EF=BC=8C=E7=A7=91=E6=8A=80=E6=97=A5?= =?UTF-8?q?=E6=8A=A5=EF=BC=8C=E4=BA=BA=E6=B0=91=E9=82=AE=E7=94=B5=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- 国内党媒/CrawlKejiribao.py | 199 +++++++++++++++++++++++++++++ 国内党媒/CrawlRenminyoudianbao.py | 148 +++++++++++++++++++++ 国内党媒/CrawlZhongguogaigebao.py | 4 +- 国内党媒/CrawlZhongguojiaoyubao.py | 9 +- 国内党媒/CrawlZhongguominzubao.py | 142 ++++++++++++++++++++ 5 files changed, 496 insertions(+), 6 deletions(-) create mode 100644 国内党媒/CrawlKejiribao.py create mode 100644 国内党媒/CrawlRenminyoudianbao.py create mode 100644 国内党媒/CrawlZhongguominzubao.py diff --git a/国内党媒/CrawlKejiribao.py b/国内党媒/CrawlKejiribao.py new file mode 100644 index 0000000..7c99046 --- /dev/null +++ b/国内党媒/CrawlKejiribao.py @@ -0,0 +1,199 @@ +# _*_ coding : UTF-8 _*_ +# @Time : 2024/11/10 21:23 +# @UpdateTime : 2024/11/10 21:23 +# @Author : haochen zhong +# @File : CrawlKejiribao.py +# @Software : PyCharm +# @Comment : 本程序采集科技日报数据 + + +import asyncio +import random +from datetime import datetime + +from bs4 import BeautifulSoup +from httpx import AsyncClient +from motor.motor_asyncio import AsyncIOMotorClient + +start_date = datetime.strptime('2011-01', '%Y-%m') +"""科技日报报2011年1月份开始有数据""" +end_date = datetime.today() +"""截止到今天""" +headers = { + 'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.42'} + +# 链接数据库 +client = AsyncIOMotorClient('mongodb://localhost:27017') +db = client['buweijiguanbao'] +collection = db['kejiribao'] + + +async def main(): + collection_names = await db.list_collection_names() + # 判断数据表是否存在 + if "kejiribao" not in collection_names: + # 如果不存在，则从2017年9月开始爬取 + print("科技日报数据表不存在，开始采集！") + await getData(start_date, end_date) + else: + # 如果存在，则从数据库中获取最后一条记录的日期 + last_record = await collection.find_one({}, sort=[('release_time', -1)]) + last_date_str = last_record['release_time'] + print("数据库截止时间:", last_date_str) + await getData(last_date_str, end_date) + + +async def getContent(soup: BeautifulSoup) -> str: + """ + :param soup: BeautifulSoup对象 + :return: 文章内容 + """ + content = "" + for p in soup.select("#ozoom p"): + para = p.text.strip() + if para: + content += para + content += '\n' + return content + + +async def getData(start_date: datetime, end_date: datetime): + """ + :param start_date: 开始日期 + :param end_date: 结束日期 + :return: None + """ + crawl_num = 0 + # 创建一个列表保存月份 + months = [] + # 从开始日期到结束日期，每个月份都添加到列表中 + current_date = start_date + while current_date <= end_date: + months.append(current_date) + # 增加一个月 + if current_date.month == 12: + current_date = current_date.replace(year=current_date.year + 1, month=1) + else: + current_date = current_date.replace(month=current_date.month + 1) + # 遍历月份列表 + for month in months: + # 构造URL + url = f'https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{month.strftime("%Y-%m")}/period.xml' + """https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/2011-10/period.xml""" + print(url) + async with AsyncClient(headers=headers, timeout=60) as client: + response = await client.get(url) + response.encoding = response.charset_encoding + print(f"一级连接状态：{response.status_code}") + if response.status_code == 200: + soup = BeautifulSoup(response.text, 'xml') + for period in soup.select("period"): + period_name = datetime.strptime(period.select_one("period_name").text.strip(), "%Y-%m-%d") + front_page = period.select_one("front_page").text.strip() + try: + url1 = f"https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{period_name.strftime('%Y-%m/%d')}/{front_page}" + """https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/2024-10/30/node_2.htm""" + print(url1) + response2 = await client.get(url1) + response2.encoding = response2.charset_encoding + print(f"二级连接状态：{response2.status_code}") + if response2.status_code == 200: + soup2 = BeautifulSoup(response2.text, 'lxml') + for item in soup2.select("#pageLink"): + banmianming = item.text.split("：")[-1] + banmianhao = item.text.split("：")[0] + url2 = f"https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{period_name.strftime('%Y-%m/%d')}/" + item.get( + "href").replace("./", "").strip() + print(url2) + response3 = await client.get(url2) + response3.encoding = response3.charset_encoding + print(f"三级连接状态：{response3.status_code}") + if response3.status_code == 200: + soup3 = BeautifulSoup(response3.text, 'lxml') + pages = soup3.select(".ul02_l a") + if pages: + titleTag = "body > table:nth-child(2) > tbody > tr:nth-child(1) > td:nth-child(2) > table:nth-child(3) > tbody > tr:nth-child(1) > td > table > tbody > tr > td > strong" + subTitleTag = "body > table:nth-child(2) > tbody > tr:nth-child(1) > td:nth-child(2) > table:nth-child(3) > tbody > tr:nth-child(1) > td > table > tbody > tr > td > span:nth-child(5)" + preTitleTag = "body > table:nth-child(2) > tbody > tr:nth-child(1) > td:nth-child(2) > table:nth-child(3) > tbody > tr:nth-child(1) > td > table > tbody > tr > td > span:nth-child(1)" + authorTag = "body > table:nth-child(2) > tbody > tr:nth-child(1) > td:nth-child(2) > table:nth-child(3) > tbody > tr:nth-child(1) > td > table > tbody > tr > td > span:nth-child(7)" + else: + pages = soup3.select(".title a") + titleTag = ".biaoti" + subTitleTag = ".futi" + preTitleTag = ".yinti" + authorTag = ".autor" + for item2 in pages: + url3 = f"https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{period_name.strftime('%Y-%m/%d')}/" + item2.get( + "href") + """https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/2024-10/30/content_579753.htm?div=-1""" + if await collection.find_one({"detail_url": url3}, {"_id": False}): + continue + title = item2.text.strip() + print(url3) + response4 = await client.get(url3) + response4.encoding = response4.charset_encoding + print(f"四级连接状态：{response4.status_code}") + if response4.status_code == 200: + soup4 = BeautifulSoup(response4.text, 'lxml') + try: + title = soup4.select_one(titleTag).text.strip() + except: + title = title + try: + subTitle = soup4.select(subTitleTag)[0].text.strip() + except: + subTitle = "" + try: + author = soup4.select_one(authorTag).text.strip() + except: + author = "" + try: + perTitle = soup4.select(preTitleTag)[-1].text.strip() + except: + perTitle = "" + try: + keywordlist = soup4.find("founder-keyword").text.strip() + except: + keywordlist = "" + content = await getContent(soup4) + await collection.insert_one({ + "title": title, + "subtitle": subTitle, + "preTitle": perTitle, + "author": author, + "banmianming": banmianming, + "banmianhao": banmianhao, + 'keywordlist': keywordlist, + 'detail_url': url3, + 'release_time': period_name, + 'insert_timestamp': datetime.today(), + 'content': content + }) + crawl_num += 1 + print( + f"科技日报---{period_name.strftime('%Y-%m-%d')}----{banmianming}---{banmianhao}---{title}---采集完成！") + await asyncio.sleep(random.randint(5, 15)) + print( + f"科技日报---{period_name.strftime('%Y-%m-%d')}----{banmianming}---{banmianhao}-----采集完成！") + await asyncio.sleep(random.randint(5, 15)) + print(f"科技日报---{period_name.strftime('%Y-%m-%d')}-----采集完成！") + await asyncio.sleep(random.randint(5, 15)) + except Exception as e: + print(e) + await collection.insert_one( + {'banmianhao': 'empty', + 'banmianming': 'empty', + 'preTitle': 'empty', + 'title': 'empty', + 'subtitle': 'empty', + 'author': 'empty', + 'keywordlist': 'empty', + 'detail_url': url, + 'release_time': period_name, + 'insert_timestamp': datetime.today(), + 'content': 'empty'} + ) + print(f"科技日报采集完毕，共采集{crawl_num}条数据！") + + +asyncio.run(main()) diff --git a/国内党媒/CrawlRenminyoudianbao.py b/国内党媒/CrawlRenminyoudianbao.py new file mode 100644 index 0000000..ebe4a18 --- /dev/null +++ b/国内党媒/CrawlRenminyoudianbao.py @@ -0,0 +1,148 @@ +# _*_ coding : UTF-8 _*_ +# @Time : 2024/11/11 21:41 +# @UpdateTime : 2024/11/11 21:41 +# @Author : haochen zhong +# @File : CrawlRenminyoudianbao.py +# @Software : PyCharm +# @Comment : 本程序采集人民邮电报 + + +import asyncio +import random +from datetime import datetime, timedelta + +from bs4 import BeautifulSoup +from httpx import AsyncClient +from motor.motor_asyncio import AsyncIOMotorClient + +start_date = datetime.strptime('2017-09', '%Y-%m') +"""中国改革报2017年9月份开始有数据""" +end_date = datetime.today() +"""截止到今天""" +headers = { + "connection":'keep-alive', + "host":"rmydb.cnii.com.cn", + 'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.42'} + +# 链接数据库 +client = AsyncIOMotorClient('mongodb://localhost:27017') +db = client['buweijiguanbao'] +collection = db['renminyoudianbao'] + + +async def main(): + collection_names = await db.list_collection_names() + # 判断数据表是否存在 + if "renminyoudianbao" not in collection_names: + # 如果不存在，则从2017年9月开始爬取 + print("人民邮电报数据表不存在，开始采集！") + await getData(start_date, end_date) + else: + # 如果存在，则从数据库中获取最后一条记录的日期 + last_record = await collection.find_one({}, sort=[('release_time', -1)]) + last_date_str = last_record['release_time'] + print("数据库截止时间:", last_date_str) + await getData(last_date_str, end_date) + + +async def getContent(soup: BeautifulSoup) -> str: + """ + :param soup: BeautifulSoup对象 + :return: 文章内容 + """ + content = "" + for p in soup.select("#ozoom p"): + para = p.text.strip() + if para: + content += para + content += '\n' + return content + + +async def getData(start_date: datetime, end_date: datetime): + """ + :param start_date: 开始日期 + :param end_date: 结束日期 + :return: None + """ + crawl_num = 0 + start_date = int(start_date.strftime("%Y%m%d")) + async with AsyncClient(headers=headers, timeout=60) as client: + response = await client.get("https://rmydb.cnii.com.cn/period/yearMonthDay.json") + response.encoding = response.charset_encoding + print(f"一级连接状态：{response.status_code}") + if response.status_code == 200: + data = response.json() + dayList = [] + for value in data.values(): + for item in value.values(): + dayList += item + dayList.sort() + dayList = list(filter(lambda x: x >= start_date, list(map(int, dayList)))) + for day in dayList: + try: + url = f"https://rmydb.cnii.com.cn/html/{day.__str__()[:4]}/{day}/data.json" + print(url) + response = await client.get(url) + response.encoding = response.charset_encoding + print(f"二级连接状态：{response.status_code}") + if response.status_code == 200: + data = response.json() + for item in data: + banmianming = item["pageName"] + banmianhao = f"第{item['pageNo']}版" + for article in item["onePageArticleList"]: + title = article["mainTitle"] + url2 = f"https://rmydb.cnii.com.cn/html/{day.__str__()[:4]}/{day}/{day}_{item['pageNo']}/{article['articleHref']}" + """https://rmydb.cnii.com.cn/html/2024/20240906/20240906_001/20240906_001_01_523.html""" + author = article["articleAuthor"] + if await collection.find_one({"detail_url": url2}, {"_id": False}): + continue + print(url2) + response2 = await client.get(url2) + response2.encoding = response2.charset_encoding + print(f"三级连接状态：{response2.status_code}") + if response2.status_code == 200: + soup = BeautifulSoup(response2.text, "lxml") + preTitle = soup.select_one("#PreTitle").text + title = soup.select_one("#Title").text + subTitle = soup.select_one("#SubTitle").text + content = await getContent(soup) + await collection.insert_one({ + "title": title, + "subtitle": subTitle, + "preTitle": preTitle, + "author": author, + "banmianming": banmianming, + "banmianhao": banmianhao, + 'keywordlist': "empty", + 'detail_url': url2, + 'release_time': datetime.strptime(str(day), "%Y%m%d"), + 'insert_timestamp': datetime.today(), + 'content': content + }) + crawl_num += 1 + print(f"人民邮电报---{day}---{banmianming}---{banmianhao}---{title}---采集完成！") + await asyncio.sleep(random.randint(5, 15)) + print(f"人民邮电报---{day}---{banmianming}---{banmianhao}-----采集完成！") + await asyncio.sleep(random.randint(5, 15)) + print(f"人民邮电报---{day}-----采集完成！") + await asyncio.sleep(random.randint(5, 15)) + except Exception as e: + print(e) + await collection.insert_one( + {'banmianhao': 'empty', + 'banmianming': 'empty', + 'preTitle': 'empty', + 'title': 'empty', + 'subtitle': 'empty', + 'author': 'empty', + 'keywordlist': 'empty', + 'detail_url': url, + 'release_time': datetime.strptime(str(day), "%Y%m%d"), + 'insert_timestamp': datetime.today(), + 'content': 'empty'} + ) + print(f"人民邮电报采集完毕，共采集{crawl_num}条数据！") + +asyncio.run(main()) diff --git a/国内党媒/CrawlZhongguogaigebao.py b/国内党媒/CrawlZhongguogaigebao.py index 503f105..ac5b3b5 100644 --- a/国内党媒/CrawlZhongguogaigebao.py +++ b/国内党媒/CrawlZhongguogaigebao.py @@ -37,7 +37,7 @@ async def main(): # 如果存在，则从数据库中获取最后一条记录的日期 last_record = await collection.find_one({}, sort=[('release_time', -1)]) last_date_str = last_record['release_time'] - print("数据库截止时间:",last_date_str) + print("数据库截止时间:", last_date_str) await getData(last_date_str, end_date) @@ -151,7 +151,7 @@ async def getData(start_date: datetime, end_date: datetime): "banmianhao": banmianhao, 'keywordlist': keyword, 'detail_url': url3, - 'release_time': month + timedelta(days=int(period_id)-1), + 'release_time': month + timedelta(days=int(period_id) - 1), 'insert_timestamp': datetime.today(), 'content': content }) diff --git a/国内党媒/CrawlZhongguojiaoyubao.py b/国内党媒/CrawlZhongguojiaoyubao.py index a25500d..1d30dca 100644 --- a/国内党媒/CrawlZhongguojiaoyubao.py +++ b/国内党媒/CrawlZhongguojiaoyubao.py @@ -8,7 +8,7 @@ import asyncio import random -from datetime import datetime, timedelta +from datetime import datetime from bs4 import BeautifulSoup from httpx import AsyncClient @@ -101,7 +101,7 @@ async def getData(start_date: datetime, end_date: datetime): banmianming = item.text.split("：")[-1] banmianhao = item.text.split("：")[0] url2 = f"http://paper.jyb.cn/zgjyb/html/{period_name.strftime('%Y-%m/%d')}/" + item.get( - "href").replace("./","").strip() + "href").replace("./", "").strip() print(url2) response3 = await client.get(url2) response3.encoding = response3.charset_encoding @@ -201,7 +201,7 @@ async def getData(start_date: datetime, end_date: datetime): banmianming = item.text.split(":")[-1] banmianhao = item.text.split(":")[0] url2 = f"http://paper.jyb.cn/zgjyb/html/{period_name.strftime('%Y-%m/%d')}/" + item.get( - "href").replace("./","").strip() + "href").replace("./", "").strip() print(url2) response3 = await client.get(url2) response3.encoding = response3.charset_encoding @@ -273,10 +273,11 @@ async def getData(start_date: datetime, end_date: datetime): 'author': 'empty', 'keywordlist': 'empty', 'detail_url': url, - 'release_time':period_name , + 'release_time': period_name, 'insert_timestamp': datetime.today(), 'content': 'empty'} ) print(f"中国教育报采集完毕，共采集{crawl_num}条数据！") + asyncio.run(main()) diff --git a/国内党媒/CrawlZhongguominzubao.py b/国内党媒/CrawlZhongguominzubao.py new file mode 100644 index 0000000..57a1348 --- /dev/null +++ b/国内党媒/CrawlZhongguominzubao.py @@ -0,0 +1,142 @@ +# _*_ coding : UTF-8 _*_ +# @Time : 2024/11/12 22:45 +# @UpdateTime : 2024/11/12 22:45 +# @Author : haochen zhong +# @File : CrawlZhongguominzubao.py +# @Software : PyCharm +# @Comment : 本程序采集中国民族报数据 + +import asyncio +import random +from datetime import datetime + +from bs4 import BeautifulSoup +from httpx import AsyncClient +from motor.motor_asyncio import AsyncIOMotorClient + +start_date = datetime.strptime('2022-01', '%Y-%m') +"""中国民族报2022年1月份开始有数据""" +end_date = datetime.today() +"""截止到今天""" +headers = { + 'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.42'} + +# 链接数据库 +client = AsyncIOMotorClient('mongodb://localhost:27017') +db = client['buweijiguanbao'] +collection = db['zhongguominzubao'] + + +async def main(): + collection_names = await db.list_collection_names() + # 判断数据表是否存在 + if "zhongguomizubao" not in collection_names: + # 如果不存在，则从2017年9月开始爬取 + print("中国民族数据表不存在，开始采集！") + await getData(start_date, end_date) + else: + # 如果存在，则从数据库中获取最后一条记录的日期 + last_record = await collection.find_one({}, sort=[('release_time', -1)]) + last_date_str = last_record['release_time'] + print("数据库截止时间:", last_date_str) + await getData(last_date_str, end_date) + + +async def getData(start_date: datetime, end_date: datetime): + """ + :param start_date: 开始日期 + :param end_date: 结束日期 + :return: None + """ + crawl_num = 0 + # 创建一个列表保存月份 + months = [] + # 从开始日期到结束日期，每个月份都添加到列表中 + current_date = start_date + while current_date <= end_date: + months.append(current_date) + # 增加一个月 + if current_date.month == 12: + current_date = current_date.replace(year=current_date.year + 1, month=1) + else: + current_date = current_date.replace(month=current_date.month + 1) + # 遍历月份列表 + async with AsyncClient(headers=headers, timeout=60) as client: + for month in months: + # 构造URL + url = "http://210.12.104.26:81/reader/layout/getSZBDatePub.do" + try: + response = await client.post(url, params={"sj": month.strftime("%Y.%m")}) + response.encoding = response.charset_encoding + print(f"一级连接状态：{response.status_code}") + if response.status_code == 200: + data = response.json() + for item in data: + url2 = "http://210.12.104.26:81/reader/layout/findBmMenuPub.do" + response2 = await client.post(url2, params={"docPubTime": item.replace("/", "")}) + response2.encoding = response2.charset_encoding + print(f"二级连接状态：{response2.status_code}") + if response2.status_code == 200: + data2 = response2.json() + for item2 in data2: + banmianming = item2["BM"] + banmianhao = item2["BC"] + url3 = f"http://210.12.104.26:81/reader/layout/getBmDetailPub.do?bc={item2['IRCATELOG']}&docpubtime={item.replace('/', '')}" + print(url3) + response3 = await client.get(url3) + response3.encoding = response3.charset_encoding + print(f"三级连接状态：{response3.status_code}") + if response3.status_code == 200: + data3 = response3.json() + for item3 in data3: + url4 = "http://210.12.104.26:81/reader/layout/detailData.do" + response4 = await client.post(url4, params={"guid": item3['ZB_GUID']}) + response4.encoding = response4.charset_encoding + print(f"四级连接状态：{response4.status_code}") + if response4.status_code == 200: + data4 = response4.json() + title = BeautifulSoup(data4['docTitle'],"lxml").text + subTitle = BeautifulSoup(data4['fb'],"lxml").text + preTitle = BeautifulSoup(data4['yt'],"lxml").text + author = data4['docAuthor'] + content = BeautifulSoup(data4["content"], "lxml").text + await collection.insert_one({ + "title": title, + "subtitle": subTitle, + "preTitle": preTitle, + "author": author, + "banmianming": banmianming, + "banmianhao": banmianhao, + 'keywordlist': "empty", + 'detail_url': f"http://210.12.104.26:81/epaper/?id={item3['ZB_GUID']}&time={item.replace('/', '')}", + 'release_time': datetime.strptime(data4["docPubTime"], + "%Y/%m/%d %H:%M:%S"), + 'insert_timestamp': datetime.today(), + 'content': content + }) + crawl_num += 1 + print( + f"中国民族报---{item}---{banmianming}---{banmianhao}---{title}---采集完成！") + await asyncio.sleep(random.randint(5, 15)) + print(f"中国民族报---{item}---{banmianming}---{banmianhao}-----采集完成！") + await asyncio.sleep(random.randint(5, 15)) + print(f"中国民族报---{item}----采集完成！") + await asyncio.sleep(random.randint(5, 15)) + except Exception as e: + print(e) + await collection.insert_one( + {'banmianhao': 'empty', + 'banmianming': 'empty', + 'preTitle': 'empty', + 'title': 'empty', + 'subtitle': 'empty', + 'author': 'empty', + 'keywordlist': 'empty', + 'detail_url': url, + 'release_time': month, + 'insert_timestamp': datetime.today(), + 'content': 'empty'} + ) + + +asyncio.run(main())