168 lines
8.1 KiB
Python
168 lines
8.1 KiB
Python
# _*_ coding : UTF-8 _*_
|
|
# @Time : 2022/6/17 8:50
|
|
# @Author : Haochen Zhong
|
|
# @File : 本程序用于抓取上海新民晚报数据
|
|
# @Project : Pytharm
|
|
from bs4 import BeautifulSoup, Comment
|
|
import requests
|
|
from datetime import timedelta, datetime
|
|
import time
|
|
import pymongo
|
|
import random
|
|
|
|
start_date = datetime.strptime('2018-12-31', '%Y-%m-%d') # 抓取上海新民晚报从2019-01-01到至今的数据
|
|
end_date = datetime.today()
|
|
headers = {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36'}
|
|
# 创建数据库
|
|
client = pymongo.MongoClient('localhost', 27017)
|
|
mydb = client.dfdm_qitaguanbao
|
|
shxinminwanbao = mydb.shxinminwanbao
|
|
|
|
|
|
# 设置随机时间
|
|
|
|
|
|
def main():
|
|
# 判断数据库是否存在
|
|
collist = mydb.list_collection_names()
|
|
if "shxinminwanbao" in collist: # 检测集合是否存在
|
|
print("上海新民晚报集合存在，更新数据库")
|
|
# 数据库最新一条内容的时间
|
|
db_time = shxinminwanbao.find_one(sort=[('release_time', -1)])['release_time']
|
|
print('数据库截止时间%s' % db_time)
|
|
# 输入更新数据库时间
|
|
input_time = datetime.today()
|
|
if db_time < input_time:
|
|
getData(db_time, input_time)
|
|
else:
|
|
print('数据库无需更新')
|
|
else:
|
|
# 爬取网页并建立数据库
|
|
print('数据库不存在，建立数据库！')
|
|
getData(start_date, end_date)
|
|
|
|
|
|
def parse_html_text(soup2):
|
|
img_list = soup2.select('.dzb-enter-desc-box p img')
|
|
if img_list:
|
|
img = '图片链接：\n'
|
|
for i in img_list:
|
|
img_url = 'https:' + i.get('src')
|
|
img += img_url
|
|
img += '\n'
|
|
content = img + '正文内容：\n'
|
|
for p in soup2.select('.dzb-enter-desc-box p'):
|
|
para = p.text.split(' ')
|
|
for x in para:
|
|
if x != '' or x != '\\n\\n':
|
|
content += x.strip()
|
|
content += '\n'
|
|
else:
|
|
content = ''
|
|
for p in soup2.select('.dzb-enter-desc-box p'):
|
|
para = p.text.split(' ')
|
|
for x in para:
|
|
if x.strip() != '' or x != '\\n\\n':
|
|
content += x.strip()
|
|
content += '\n'
|
|
return content
|
|
|
|
|
|
def getData(start_date, end_date):
|
|
for i in range((end_date - start_date).days):
|
|
date_now = start_date + timedelta(days=i + 1)
|
|
date_now_s = date_now.strftime('%Y-%m-%d')
|
|
base_url = "https://paper.xinmin.cn/html/xmwb/" + date_now_s + '/'
|
|
url = base_url + '1.html'
|
|
art_base_url = 'https://paper.xinmin.cn'
|
|
# 进入首页
|
|
try:
|
|
try:
|
|
response = requests.get(url=url, headers=headers, timeout=30)
|
|
except:
|
|
time.sleep(10)
|
|
response = requests.get(url=url, headers=headers, timeout=30)
|
|
response.encoding = response.apparent_encoding
|
|
print('一级连接状态：%d' % response.status_code)
|
|
if response.status_code == 200:
|
|
soup = BeautifulSoup(response.text, 'lxml')
|
|
response.close()
|
|
# 提取所有版面信息
|
|
for item in soup.select('.dzb-enter-mulu-wrap-nav a'):
|
|
url1 = art_base_url + item.get('href')
|
|
banmianhao = item.get('title').split(':')[0]
|
|
banmianming = item.get('title').split(':')[-1]
|
|
try:
|
|
response2 = requests.get(url=url1, headers=headers, timeout=30)
|
|
except:
|
|
time.sleep(10)
|
|
response2 = requests.get(url=url1, headers=headers, timeout=30)
|
|
response2.encoding = response2.apparent_encoding
|
|
print('二级连接状态：%d' % response2.status_code)
|
|
if response2.status_code == 200:
|
|
soup1 = BeautifulSoup(response2.text, 'lxml')
|
|
response2.close()
|
|
for item1 in soup1.select('.dzb-enter-benban-wrap div a'):
|
|
url2 = art_base_url + item1.get('href')
|
|
try:
|
|
response3 = requests.get(url=url2, headers=headers, timeout=30)
|
|
except:
|
|
time.sleep(10)
|
|
response3 = requests.get(url=url2, headers=headers, timeout=30)
|
|
response3.encoding = response3.apparent_encoding
|
|
print('三级连接状态：%d' % response3.status_code)
|
|
if response3.status_code == 200:
|
|
soup2 = BeautifulSoup(response3.text, 'lxml')
|
|
response3.close()
|
|
title = soup2.select('.dzb-title-box')[0].text.strip()
|
|
pass_list = ['上海地区今明天气', '上海市今明天气预报', '广告']
|
|
if title in pass_list: # 筛除每天海今明天气和广告
|
|
time.sleep(random.randint(2, 8))
|
|
continue
|
|
subtitle = soup2.select('.dzb-sub-title-box')[0].text.strip()
|
|
# 查找所有注释
|
|
comments = soup2.find_all(string=lambda text: isinstance(text, Comment))
|
|
author = ""
|
|
# 遍历注释，找到包含作者的注释
|
|
for comment in comments:
|
|
if 'dzb-author-box' in comment:
|
|
# 使用 BeautifulSoup 解析注释内容
|
|
author_soup = BeautifulSoup(comment, 'html.parser')
|
|
author = author_soup.find('span', class_='dzb-author-box').text
|
|
pretitle = soup2.select('.dzb-special-title-box')[0].text.strip()
|
|
content = parse_html_text(soup2)
|
|
shxinminwanbao.insert_one({'banmianhao': banmianhao,
|
|
'banmianming': banmianming,
|
|
'pretitle': pretitle,
|
|
'title': title,
|
|
'subtitle': subtitle,
|
|
'author': author,
|
|
'keywordlist': '',
|
|
'detail_url': url2,
|
|
'release_time': date_now,
|
|
'insert_timestamp': datetime.today(),
|
|
'content': content})
|
|
print('上海新民晚报-%s-%s-%s-已完成' % (date_now_s, banmianhao, title))
|
|
time.sleep(random.randint(2, 8))
|
|
print('上海新民晚报-%s-%s-已完成' % (date_now_s, banmianhao))
|
|
print("上海新民晚报-%s-已经完成" % date_now_s)
|
|
except Exception as result:
|
|
shxinminwanbao.insert_one({'banmianhao': 'empty',
|
|
'banmianming': 'empty',
|
|
'title': 'empty',
|
|
'subtitle': 'empty',
|
|
'h3title': 'empty',
|
|
'author': 'empty',
|
|
'keywordlist': 'empty',
|
|
'detail_url': url,
|
|
'release_time': date_now,
|
|
'insert_timestamp': datetime.today(),
|
|
'content': 'empty'})
|
|
print(result)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
print("爬取完毕！")
|