hygc/guoneimeitishujucaiji
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复中国改革报偶尔出现日期匹配失败问题

Browse Source
This commit is contained in:
皓月归尘 2024-11-11 20:26:04 +08:00
parent b86a7737cb
commit 8d62f1d2db
1 changed files with 31 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
国内党媒/CrawlZhongguogaigebao.py
View File

@ -66,6 +66,7 @@ async def getData(start_date: datetime, end_date: datetime):
months = [] months = []
# 从开始日期到结束日期，每个月份都添加到列表中 # 从开始日期到结束日期，每个月份都添加到列表中
current_date = start_date current_date = start_date
current_date = current_date.replace(day=1)
while current_date <= end_date: while current_date <= end_date:
months.append(current_date) months.append(current_date)
# 增加一个月 # 增加一个月
@ -78,7 +79,8 @@ async def getData(start_date: datetime, end_date: datetime):
# 构造URL # 构造URL
url = f'http://www.cfgw.net.cn/epaper/{month.strftime("%Y%m")}/period.xml' url = f'http://www.cfgw.net.cn/epaper/{month.strftime("%Y%m")}/period.xml'
"""http://www.cfgw.net.cn/epaper/201709/period.xml""" """http://www.cfgw.net.cn/epaper/201709/period.xml"""
print(url) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url)
try:
async with AsyncClient(headers=headers, timeout=60) as client: async with AsyncClient(headers=headers, timeout=60) as client:
# 发送GET请求 # 发送GET请求
response = await client.get(url) response = await client.get(url)
@ -88,11 +90,10 @@ async def getData(start_date: datetime, end_date: datetime):
# 解析XML # 解析XML
soup = BeautifulSoup(response.text, 'xml') soup = BeautifulSoup(response.text, 'xml')
for period in soup.find_all("period"): for period in soup.find_all("period"):
try:
period_id = period.get("id") period_id = period.get("id")
url1 = f"http://www.cfgw.net.cn/epaper/{month.strftime('%Y%m')}/{period_id}/node_01.htm" url1 = f"http://www.cfgw.net.cn/epaper/{month.strftime('%Y%m')}/{period_id}/node_01.htm"
"""http://www.cfgw.net.cn/epaper/201709/05/node_01.htm""" """http://www.cfgw.net.cn/epaper/201709/05/node_01.htm"""
print(url1) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url1)
response2 = await client.get(url1) response2 = await client.get(url1)
response2.encoding = response2.charset_encoding response2.encoding = response2.charset_encoding
print(f"二级连接状态：{response2.status_code}") print(f"二级连接状态：{response2.status_code}")
@ -104,7 +105,7 @@ async def getData(start_date: datetime, end_date: datetime):
"""http://www.cfgw.net.cn/epaper/201709/05/node_01/node_01.htm""" """http://www.cfgw.net.cn/epaper/201709/05/node_01/node_01.htm"""
banmianming = item.text.split("：")[-1] banmianming = item.text.split("：")[-1]
banmianhao = item.text.split("：")[0] banmianhao = item.text.split("：")[0]
print(url2) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url2)
response3 = await client.get(url2) response3 = await client.get(url2)
response3.encoding = response3.charset_encoding response3.encoding = response3.charset_encoding
print(f"三级连接状态：{response3.status_code}") print(f"三级连接状态：{response3.status_code}")
@ -115,7 +116,7 @@ async def getData(start_date: datetime, end_date: datetime):
if await collection.find_one({"detail_url": url3}, {"_id": False}): if await collection.find_one({"detail_url": url3}, {"_id": False}):
continue continue
title = item2.text.strip() title = item2.text.strip()
print(url3) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url3)
response4 = await client.get(url3) response4 = await client.get(url3)
response4.encoding = response4.charset_encoding response4.encoding = response4.charset_encoding
print(f"四级连接状态：{response4.status_code}") print(f"四级连接状态：{response4.status_code}")
@ -162,8 +163,7 @@ async def getData(start_date: datetime, end_date: datetime):
print( print(
f"中国改革报---{month.strftime('%Y-%m')}-{period_id}---{banmianming}---{banmianhao}----采集完成！") f"中国改革报---{month.strftime('%Y-%m')}-{period_id}---{banmianming}---{banmianhao}----采集完成！")
await asyncio.sleep(random.randint(5, 15)) await asyncio.sleep(random.randint(5, 15))
print( print(f"中国改革报---{month.strftime('%Y-%m')}-{period_id}-----采集完成！")
f"中国改革报---{month.strftime('%Y-%m')}-{period_id}-----采集完成！")
await asyncio.sleep(random.randint(5, 15)) await asyncio.sleep(random.randint(5, 15))
except Exception as e: except Exception as e:
await collection.insert_one( await collection.insert_one(