hygc/guoneimeitishujucaiji
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复中国经济导报偶尔出现日期匹配失败问题

Browse Source
This commit is contained in:
皓月归尘 2024-11-11 20:23:21 +08:00
parent 4cb5d4a5d3
commit b86a7737cb
1 changed files with 34 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
国内党媒/CrawlZhongguojingjidaobao.py
View File

@ -66,6 +66,7 @@ async def getData(start_date: datetime, end_date: datetime):
months = [] months = []
# 从开始日期到结束日期，每个月份都添加到列表中 # 从开始日期到结束日期，每个月份都添加到列表中
current_date = start_date current_date = start_date
current_date = current_date.replace(day=1)
while current_date <= end_date: while current_date <= end_date:
months.append(current_date) months.append(current_date)
# 增加一个月 # 增加一个月
@ -78,23 +79,23 @@ async def getData(start_date: datetime, end_date: datetime):
# 构造URL # 构造URL
url = f'http://www.ceh.com.cn/epaper/uniflows/html/{month.strftime("%Y/%m")}/date.txt' url = f'http://www.ceh.com.cn/epaper/uniflows/html/{month.strftime("%Y/%m")}/date.txt'
"""http://www.ceh.com.cn/epaper/uniflows/html/2012/09/date.txt""" """http://www.ceh.com.cn/epaper/uniflows/html/2012/09/date.txt"""
print(url) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url)
async with AsyncClient(headers=headers, timeout=60) as client: try:
# 发送GET请求 async with AsyncClient(headers=headers, timeout=60) as client:
response = await client.get(url) # 发送GET请求
response.encoding = "gb2312" response = await client.get(url)
print(f"一级连接状态：{response.status_code}") response.encoding = "gb2312"
if response.status_code == 200: print(f"一级连接状态：{response.status_code}")
# 解析XML if response.status_code == 200:
soup = response.text.split("|") # 解析XML
for period in soup: soup = response.text.split("|")
period_id, element = period.split(",") for period in soup:
if len(element) < 5: period_id, element = period.split(",")
continue if len(element) < 5:
try: continue
url1 = f"http://www.ceh.com.cn/epaper/uniflows/html/{month.strftime('%Y/%m')}/{period_id}/boardurl.htm" url1 = f"http://www.ceh.com.cn/epaper/uniflows/html/{month.strftime('%Y/%m')}/{period_id}/boardurl.htm"
"""http://www.ceh.com.cn/epaper/uniflows/html/2012/09/01/boardurl.htm""" """http://www.ceh.com.cn/epaper/uniflows/html/2012/09/01/boardurl.htm"""
print(url1) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url1)
response2 = await client.get(url1) response2 = await client.get(url1)
response2.encoding = "gb2312" response2.encoding = "gb2312"
print(f"二级连接状态：{response2.status_code}") print(f"二级连接状态：{response2.status_code}")
@ -106,7 +107,7 @@ async def getData(start_date: datetime, end_date: datetime):
"""http://www.ceh.com.cn/epaper/uniflows/html/2024/11/07/01/default.htm""" """http://www.ceh.com.cn/epaper/uniflows/html/2024/11/07/01/default.htm"""
banmianming = item.text.split("：")[-1].strip() banmianming = item.text.split("：")[-1].strip()
banmianhao = item.text.split("：")[0].replace("&nbsp;", "").replace(" ", "").strip() banmianhao = item.text.split("：")[0].replace("&nbsp;", "").replace(" ", "").strip()
print(url2) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url2)
response3 = await client.get(url2) response3 = await client.get(url2)
response3.encoding = "gb2312" response3.encoding = "gb2312"
print(f"三级连接状态：{response3.status_code}") print(f"三级连接状态：{response3.status_code}")
@ -118,7 +119,7 @@ async def getData(start_date: datetime, end_date: datetime):
if await collection.find_one({"detail_url": url3}, {"_id": False}): if await collection.find_one({"detail_url": url3}, {"_id": False}):
continue continue
title = item2.text.strip() title = item2.text.strip()
print(url3) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url3)
response4 = await client.get(url3) response4 = await client.get(url3)
response4.encoding = "gb2312" response4.encoding = "gb2312"
print(f"四级连接状态：{response4.status_code}") print(f"四级连接状态：{response4.status_code}")
@ -161,24 +162,23 @@ async def getData(start_date: datetime, end_date: datetime):
print( print(
f"中国经济导报---{month.strftime('%Y-%m')}-{period_id}---{banmianming}---{banmianhao}----采集完成！") f"中国经济导报---{month.strftime('%Y-%m')}-{period_id}---{banmianming}---{banmianhao}----采集完成！")
await asyncio.sleep(random.randint(5, 15)) await asyncio.sleep(random.randint(5, 15))
print( print(f"中国经济导报---{month.strftime('%Y-%m')}-{period_id}-----采集完成！")
f"中国经济导报---{month.strftime('%Y-%m')}-{period_id}-----采集完成！")
await asyncio.sleep(random.randint(5, 15)) await asyncio.sleep(random.randint(5, 15))
except Exception as e: except Exception as e:
await collection.insert_one( await collection.insert_one(
{'banmianhao': 'empty', {'banmianhao': 'empty',
'banmianming': 'empty', 'banmianming': 'empty',
'preTitle': 'empty', 'preTitle': 'empty',
'title': 'empty', 'title': 'empty',
'subtitle': 'empty', 'subtitle': 'empty',
'author': 'empty', 'author': 'empty',
'keywordlist': 'empty', 'keywordlist': 'empty',
'detail_url': url, 'detail_url': url,
'release_time': month + timedelta(days=int(period_id)), 'release_time': month + timedelta(days=int(period_id)),
'insert_timestamp': datetime.today(), 'insert_timestamp': datetime.today(),
'content': 'empty'} 'content': 'empty'}
) )
print(e) print(e)
print(f"中国经济导报采集完毕，共采集{crawl_num}条数据！") print(f"中国经济导报采集完毕，共采集{crawl_num}条数据！")