guoneimeitishujucaiji/地方政策/报刊/CrawlXinminwanbao.py

# _*_ coding : UTF-8 _*_
# @Time : 2022/6/17 8:50
# @Author : Haochen Zhong
# @File : 本程序用于抓取上海新民晚报数据
# @Project : Pytharm
import random
import time
from datetime import timedelta, datetime

import pymongo
import requests
from bs4 import BeautifulSoup, Comment

start_date = datetime.strptime('2018-12-31', '%Y-%m-%d')  # 抓取上海新民晚报从2019-01-01到至今的数据
end_date = datetime.today()
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36'}
# 创建数据库
client = pymongo.MongoClient('localhost', 27017)
mydb = client.dfdm_qitaguanbao
shxinminwanbao = mydb.shxinminwanbao


# 设置随机时间


def main():
    # 判断数据库是否存在
    collist = mydb.list_collection_names()
    if "shxinminwanbao" in collist:  # 检测集合是否存在
        print("上海新民晚报集合存在，更新数据库")
        # 数据库最新一条内容的时间
        db_time = shxinminwanbao.find_one(sort=[('release_time', -1)])['release_time']
        print('数据库截止时间%s' % db_time)
        # 输入更新数据库时间
        input_time = datetime.today()
        if db_time < input_time:
            getData(db_time, input_time)
        else:
            print('数据库无需更新')
    else:
        # 爬取网页并建立数据库
        print('数据库不存在，建立数据库！')
        getData(start_date, end_date)


def parse_html_text(soup2):
    img_list = soup2.select('.dzb-enter-desc-box p img')
    if img_list:
        img = '图片链接：\n'
        for i in img_list:
            img_url = 'https:' + i.get('src')
            img += img_url
            img += '\n'
        content = img + '正文内容：\n'
        for p in soup2.select('.dzb-enter-desc-box p'):
            para = p.text.split('    ')
            for x in para:
                if x != '' or x != '\\n\\n':
                    content += x.strip()
                    content += '\n'
    else:
        content = ''
        for p in soup2.select('.dzb-enter-desc-box p'):
            para = p.text.split('    ')
            for x in para:
                if x.strip() != '' or x != '\\n\\n':
                    content += x.strip()
                    content += '\n'
    return content


def getData(start_date, end_date):
    for i in range((end_date - start_date).days):
        date_now = start_date + timedelta(days=i + 1)
        date_now_s = date_now.strftime('%Y-%m-%d')
        base_url = "https://paper.xinmin.cn/html/xmwb/" + date_now_s + '/'
        url = base_url + '1.html'
        art_base_url = 'https://paper.xinmin.cn'
        # 进入首页
        try:
            try:
                response = requests.get(url=url, headers=headers, timeout=30)
            except:
                time.sleep(10)
                response = requests.get(url=url, headers=headers, timeout=30)
            response.encoding = response.apparent_encoding
            print('一级连接状态：%d' % response.status_code)
            if response.status_code == 200:
                soup = BeautifulSoup(response.text, 'lxml')
                response.close()
                # 提取所有版面信息
                for item in soup.select('.dzb-enter-mulu-wrap-nav a'):
                    url1 = art_base_url + item.get('href')
                    banmianhao = item.get('title').split(':')[0]
                    banmianming = item.get('title').split(':')[-1]
                    try:
                        response2 = requests.get(url=url1, headers=headers, timeout=30)
                    except:
                        time.sleep(10)
                        response2 = requests.get(url=url1, headers=headers, timeout=30)
                    response2.encoding = response2.apparent_encoding
                    print('二级连接状态：%d' % response2.status_code)
                    if response2.status_code == 200:
                        soup1 = BeautifulSoup(response2.text, 'lxml')
                        response2.close()
                        for item1 in soup1.select('.dzb-enter-benban-wrap div a'):
                            url2 = art_base_url + item1.get('href')
                            try:
                                response3 = requests.get(url=url2, headers=headers, timeout=30)
                            except:
                                time.sleep(10)
                                response3 = requests.get(url=url2, headers=headers, timeout=30)
                            response3.encoding = response3.apparent_encoding
                            print('三级连接状态：%d' % response3.status_code)
                            if response3.status_code == 200:
                                soup2 = BeautifulSoup(response3.text, 'lxml')
                                response3.close()
                                title = soup2.select('.dzb-title-box')[0].text.strip()
                                pass_list = ['上海地区今明天气', '上海市今明天气预报', '广告']
                                if title in pass_list:  # 筛除每天海今明天气和广告
                                    time.sleep(random.randint(2, 8))
                                    continue
                                subtitle = soup2.select('.dzb-sub-title-box')[0].text.strip()
                                # 查找所有注释
                                comments = soup2.find_all(string=lambda text: isinstance(text, Comment))
                                author = ""
                                # 遍历注释，找到包含作者的注释
                                for comment in comments:
                                    if 'dzb-author-box' in comment:
                                        # 使用 BeautifulSoup 解析注释内容
                                        author_soup = BeautifulSoup(comment, 'html.parser')
                                        author = author_soup.find('span', class_='dzb-author-box').text
                                pretitle = soup2.select('.dzb-special-title-box')[0].text.strip()
                                content = parse_html_text(soup2)
                                shxinminwanbao.insert_one({'banmianhao': banmianhao,
                                                           'banmianming': banmianming,
                                                           'pretitle': pretitle,
                                                           'title': title,
                                                           'subtitle': subtitle,
                                                           'author': author,
                                                           'keywordlist': '',
                                                           'detail_url': url2,
                                                           'release_time': date_now,
                                                           'insert_timestamp': datetime.today(),
                                                           'content': content})
                                print('上海新民晚报-%s-%s-%s-已完成' % (date_now_s, banmianhao, title))
                                time.sleep(random.randint(2, 8))
                        print('上海新民晚报-%s-%s-已完成' % (date_now_s, banmianhao))
                print("上海新民晚报-%s-已经完成" % date_now_s)
        except Exception as result:
            shxinminwanbao.insert_one({'banmianhao': 'empty',
                                       'banmianming': 'empty',
                                       'title': 'empty',
                                       'subtitle': 'empty',
                                       'h3title': 'empty',
                                       'author': 'empty',
                                       'keywordlist': 'empty',
                                       'detail_url': url,
                                       'release_time': date_now,
                                       'insert_timestamp': datetime.today(),
                                       'content': 'empty'})
            print(result)


if __name__ == '__main__':
    main()
    print("爬取完毕！")