hygc/guoneimeitishujucaiji
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复科技日报偶尔出现日期匹配失败问题

Browse Source
This commit is contained in:
皓月归尘 2024-11-11 20:30:02 +08:00
parent e134004f2d
commit 5950005bac
1 changed files with 30 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
国内党媒/CrawlKejiribao.py
View File

@ -68,6 +68,7 @@ async def getData(start_date: datetime, end_date: datetime):
months = [] months = []
# 从开始日期到结束日期，每个月份都添加到列表中 # 从开始日期到结束日期，每个月份都添加到列表中
current_date = start_date current_date = start_date
current_date = current_date.replace(day=1)
while current_date <= end_date: while current_date <= end_date:
months.append(current_date) months.append(current_date)
# 增加一个月 # 增加一个月
@ -80,20 +81,20 @@ async def getData(start_date: datetime, end_date: datetime):
# 构造URL # 构造URL
url = f'https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{month.strftime("%Y-%m")}/period.xml' url = f'https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{month.strftime("%Y-%m")}/period.xml'
"""https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/2011-10/period.xml""" """https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/2011-10/period.xml"""
print(url) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url)
async with AsyncClient(headers=headers, timeout=60) as client: try:
response = await client.get(url) async with AsyncClient(headers=headers, timeout=60) as client:
response.encoding = response.charset_encoding response = await client.get(url)
print(f"一级连接状态：{response.status_code}") response.encoding = response.charset_encoding
if response.status_code == 200: print(f"一级连接状态：{response.status_code}")
soup = BeautifulSoup(response.text, 'xml') if response.status_code == 200:
for period in soup.select("period"): soup = BeautifulSoup(response.text, 'xml')
period_name = datetime.strptime(period.select_one("period_name").text.strip(), "%Y-%m-%d") for period in soup.select("period"):
front_page = period.select_one("front_page").text.strip() period_name = datetime.strptime(period.select_one("period_name").text.strip(), "%Y-%m-%d")
try: front_page = period.select_one("front_page").text.strip()
url1 = f"https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{period_name.strftime('%Y-%m/%d')}/{front_page}" url1 = f"https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{period_name.strftime('%Y-%m/%d')}/{front_page}"
"""https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/2024-10/30/node_2.htm""" """https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/2024-10/30/node_2.htm"""
print(url1) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url1)
response2 = await client.get(url1) response2 = await client.get(url1)
response2.encoding = response2.charset_encoding response2.encoding = response2.charset_encoding
print(f"二级连接状态：{response2.status_code}") print(f"二级连接状态：{response2.status_code}")
@ -104,7 +105,7 @@ async def getData(start_date: datetime, end_date: datetime):
banmianhao = item.text.split("：")[0] banmianhao = item.text.split("：")[0]
url2 = f"https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{period_name.strftime('%Y-%m/%d')}/" + item.get( url2 = f"https://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html/{period_name.strftime('%Y-%m/%d')}/" + item.get(
"href").replace("./", "").strip() "href").replace("./", "").strip()
print(url2) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url2)
response3 = await client.get(url2) response3 = await client.get(url2)
response3.encoding = response3.charset_encoding response3.encoding = response3.charset_encoding
print(f"三级连接状态：{response3.status_code}") print(f"三级连接状态：{response3.status_code}")
@ -129,7 +130,7 @@ async def getData(start_date: datetime, end_date: datetime):
if await collection.find_one({"detail_url": url3}, {"_id": False}): if await collection.find_one({"detail_url": url3}, {"_id": False}):
continue continue
title = item2.text.strip() title = item2.text.strip()
print(url3) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url3)
response4 = await client.get(url3) response4 = await client.get(url3)
response4.encoding = response4.charset_encoding response4.encoding = response4.charset_encoding
print(f"四级连接状态：{response4.status_code}") print(f"四级连接状态：{response4.status_code}")
@ -178,21 +179,21 @@ async def getData(start_date: datetime, end_date: datetime):
await asyncio.sleep(random.randint(5, 15)) await asyncio.sleep(random.randint(5, 15))
print(f"科技日报---{period_name.strftime('%Y-%m-%d')}-----采集完成！") print(f"科技日报---{period_name.strftime('%Y-%m-%d')}-----采集完成！")
await asyncio.sleep(random.randint(5, 15)) await asyncio.sleep(random.randint(5, 15))
except Exception as e: except Exception as e:
print(e) print(e)
await collection.insert_one( await collection.insert_one(
{'banmianhao': 'empty', {'banmianhao': 'empty',
'banmianming': 'empty', 'banmianming': 'empty',
'preTitle': 'empty', 'preTitle': 'empty',
'title': 'empty', 'title': 'empty',
'subtitle': 'empty', 'subtitle': 'empty',
'author': 'empty', 'author': 'empty',
'keywordlist': 'empty', 'keywordlist': 'empty',
'detail_url': url, 'detail_url': url,
'release_time': period_name, 'release_time': period_name,
'insert_timestamp': datetime.today(), 'insert_timestamp': datetime.today(),
'content': 'empty'} 'content': 'empty'}
) )
print(f"科技日报采集完毕，共采集{crawl_num}条数据！") print(f"科技日报采集完毕，共采集{crawl_num}条数据！")