hygc/guoneimeitishujucaiji
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 调整安徽日报重连逻辑

Browse Source
This commit is contained in:
皓月归尘 2024-11-12 14:03:59 +08:00
parent 5767dfb591
commit 860f128fe6
1 changed files with 74 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
地方政策/报刊/CrawlAnhui.py
View File

@ -79,9 +79,6 @@ async def getData(start_date: datetime, end_date: datetime):
for t in range(5): for t in range(5):
try: try:
response = await client.get(url) response = await client.get(url)
except Exception as e:
await asyncio.sleep(random.randint(8, 20))
print(f"尝试第{t + 1}次重连！")
response.encoding = response.charset_encoding response.encoding = response.charset_encoding
print(f"一级连接状态：{response.status_code}") print(f"一级连接状态：{response.status_code}")
if response.status_code == 200: if response.status_code == 200:
@ -91,12 +88,9 @@ async def getData(start_date: datetime, end_date: datetime):
banmianhao = item.text.split("：")[0].replace(" ", "").replace(" ", "").strip() banmianhao = item.text.split("：")[0].replace(" ", "").replace(" ", "").strip()
url1 = base_url + item.get("href") url1 = base_url + item.get("href")
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url1) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url1)
for t in range(5): for y in range(5):
try: try:
response2 = await client.get(url1) response2 = await client.get(url1)
except Exception as e:
await asyncio.sleep(random.randint(8, 20))
print(f"尝试第{t + 1}次重连！")
response2.encoding = response2.charset_encoding response2.encoding = response2.charset_encoding
print(f"二级连接状态：{response2.status_code}") print(f"二级连接状态：{response2.status_code}")
if response2.status_code == 200: if response2.status_code == 200:
@ -109,15 +103,9 @@ async def getData(start_date: datetime, end_date: datetime):
title = item2.text.strip() title = item2.text.strip()
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url2) print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url2)
# 启用超时重连 # 启用超时重连
for t in range(5): for z in range(5):
try: try:
response3 = await client.get(url2) response3 = await client.get(url2)
if response3.status_code == 200:
break
except:
# 随机等待重连
await asyncio.sleep(random.randint(8, 20))
print(f"尝试第{t + 1}次重连！")
response3.encoding = response3.charset_encoding response3.encoding = response3.charset_encoding
print(f"三级连接状态：{response3.status_code}") print(f"三级连接状态：{response3.status_code}")
if response3.status_code == 200: if response3.status_code == 200:
@ -128,7 +116,8 @@ async def getData(start_date: datetime, end_date: datetime):
except: except:
title = title title = title
try: try:
subTitle = soup3.select(".newsdetatext p")[0].text.strip() subTitle = soup3.select(".newsdetatext p")[
0].text.strip()
except: except:
subTitle = "" subTitle = ""
await collection.insert_one({ await collection.insert_one({
@ -148,8 +137,26 @@ async def getData(start_date: datetime, end_date: datetime):
print( print(
f"安徽日报---{date_now_s}---{banmianming}---{banmianhao}---{title}---采集完成！") f"安徽日报---{date_now_s}---{banmianming}---{banmianhao}---{title}---采集完成！")
await asyncio.sleep(random.randint(8, 20)) await asyncio.sleep(random.randint(8, 20))
break
except Exception as e:
print(e)
# 随机等待重连
await asyncio.sleep(random.randint(8, 20))
print(f"尝试第{z + 1}次重连！")
break
except Exception as e:
print(e)
await asyncio.sleep(random.randint(8, 20))
print(f"尝试第{t + 1}次重连！")
print(f"安徽日报---{date_now_s}---{banmianming}---{banmianhao}-----采集完成！") print(f"安徽日报---{date_now_s}---{banmianming}---{banmianhao}-----采集完成！")
await asyncio.sleep(random.randint(8, 20)) await asyncio.sleep(random.randint(8, 20))
break
except Exception as e:
if t >= 4:
print(f"尝试第{t + 1}次重连失败，请检查网络环境！")
break
await asyncio.sleep(random.randint(8, 20))
print(f"尝试第{t + 1}次重连！")
print(f"安徽日报---{date_now_s}-----采集完成！") print(f"安徽日报---{date_now_s}-----采集完成！")
await asyncio.sleep(random.randint(8, 20)) await asyncio.sleep(random.randint(8, 20))
except Exception as e: except Exception as e: