Python实现爬取阳光热线问政平台

需求

http://wz.sun0769.com/index.php/question/questionType?type=4

爬取投诉帖子的编号、帖子的url、帖子的标题，和帖子里的内容。

新建Scrapy项目

scrapy startproject Dongguanspider

创建爬虫

scrapy genspider dongguan "wz.sun0769.com"

items.py

# -*- coding: utf-8 -*-

import scrapy


class DongguanspiderItem(scrapy.Item):
    id = scrapy.Field() # 投诉帖子的编号
    url = scrapy.Field() # 投诉帖子的url
    title = scrapy.Field() # 投诉帖子的标题
    content = scrapy.Field() # 投诉帖子的内容

settings.py

ITEM_PIPELINES = {
   'Dongguanspider.pipelines.DongguanspiderPipeline': 300,
}

# 日志文件名和处理等级
LOG_FILE = "dg.log"
LOG_LEVEL = "DEBUG"

pipelines.py

# -*- coding: utf-8 -*-

# 文件处理类库，可以指定编码格式
import codecs
import json


class DongguanspiderPipeline(object):
    def __init__(self):
        # 创建一个只写文件，指定文本编码格式为utf-8
        self.filename = codecs.open('dongguan.json', 'w', encoding='utf-8')

    def process_item(self, item, spider):
        content = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.filename.write(content)
        return item

    def spider_closed(self, spider):
        self.file.close()

main.py

# -*- coding: utf-8 -*-

from scrapy import cmdline
cmdline.execute('scrapy crawl dongguan'.split())

Spider版本

dongguan.py

# -*- coding: utf-8 -*-
import scrapy
from Dongguanspider.items import DongguanspiderItem


class DongguanSpider(scrapy.Spider):
    name = "dongguan"
    allowed_domains = ["wz.sun0769.com"]
    url = 'http://wz.sun0769.com/index.php/question/questionType?type=4&page='
    offset = 0
    start_urls = [url + str(offset)]

    def parse(self, response):
        # 取出每个页面中的帖子链接列表
        links = response.xpath('//*[@class="greyframe"]/table[2]//a[@class="news14"]/@href').extract()
        # 迭代发送每个帖子的请求，调用parse_item方法处理
        for link in links:
            yield scrapy.Request(link, callback=self.parse_item)
        # 设置页码终止条件，并且每次发送新的页面请求调用parse方法处理
        if self.offset <= 71130:
            self.offset += 30
            yield scrapy.Request(self.url + str(self.offset), callback=self.parse)

    # 处理每个帖子里
    def parse_item(self, response):
        item = DongguanspiderItem()
        # 标题
        item['title'] = response.xpath('//div[contains(@class,"pagecenter p3")]//strong/text()').extract()[0].strip()
        # 编号
        item['id'] = item['title'].split(" ")[1].split("：")[1].split(":")[1].strip()
        # 链接
        item['url'] = response.url.strip()
        # 内容
        content = response.xpath('//div[@class="contentext"]/text()').extract()
        if len(content) == 0:
            content = response.xpath('//div[@class="c1 text14_2"]/text()').extract()
            # content为列表，通过join方法拼接为字符串，并去除首尾空格
            item['content'] = "".join(content).strip()
        else:
            item['content'] = "".join(content).strip()
        yield item

CrawlSpider版本