hygc/guoneimeitishujucaiji
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复中国民族报偶尔出现日期匹配失败问题

Browse Source
This commit is contained in:
皓月归尘 2024-11-11 20:32:53 +08:00
parent 5950005bac
commit b7a9a32601
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
国内党媒/CrawlZhongguominzubao.py
View File

@ -53,6 +53,7 @@ async def getData(start_date: datetime, end_date: datetime):
months = [] months = []
# 从开始日期到结束日期，每个月份都添加到列表中 # 从开始日期到结束日期，每个月份都添加到列表中
current_date = start_date current_date = start_date
current_date = current_date.replace(day=1)
while current_date <= end_date: while current_date <= end_date:
months.append(current_date) months.append(current_date)
# 增加一个月 # 增加一个月
@ -82,7 +83,7 @@ async def getData(start_date: datetime, end_date: datetime):
banmianming = item2["BM"] banmianming = item2["BM"]
banmianhao = item2["BC"] banmianhao = item2["BC"]
url3 = f"http://210.12.104.26:81/reader/layout/getBmDetailPub.do?bc={item2['IRCATELOG']}&docpubtime={item.replace('/', '')}" url3 = f"http://210.12.104.26:81/reader/layout/getBmDetailPub.do?bc={item2['IRCATELOG']}&docpubtime={item.replace('/', '')}"
print(url3) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url3)
response3 = await client.get(url3) response3 = await client.get(url3)
response3.encoding = response3.charset_encoding response3.encoding = response3.charset_encoding
print(f"三级连接状态：{response3.status_code}") print(f"三级连接状态：{response3.status_code}")
@ -95,9 +96,9 @@ async def getData(start_date: datetime, end_date: datetime):
print(f"四级连接状态：{response4.status_code}") print(f"四级连接状态：{response4.status_code}")
if response4.status_code == 200: if response4.status_code == 200:
data4 = response4.json() data4 = response4.json()
title = BeautifulSoup(data4['docTitle'],"lxml").text title = BeautifulSoup(data4['docTitle'], "lxml").text
subTitle = BeautifulSoup(data4['fb'],"lxml").text subTitle = BeautifulSoup(data4['fb'], "lxml").text
preTitle = BeautifulSoup(data4['yt'],"lxml").text preTitle = BeautifulSoup(data4['yt'], "lxml").text
author = data4['docAuthor'] author = data4['docAuthor']
content = BeautifulSoup(data4["content"], "lxml").text content = BeautifulSoup(data4["content"], "lxml").text
await collection.insert_one({ await collection.insert_one({