爬虫项目（斗图啦scrapy）

斗图啦网址 https://www.doutula.com/photo/list/ （1）分析网站 得到图片的地址 （2）进入得到的网址分析

（3）编写项目代码 （4）scrapy爬取，源码 items.py

import scrapy class DoutubaItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() name = scrapy.Field() link = scrapy.Field()

settings.py

ROBOTSTXT_OBEY = False ITEM_PIPELINES = { #'doutuba.pipelines.DoutubaPipeline': 300, 'doutuba.pipelines.DoutuImgPipeline': 300, } IMAGES_STORE = 'images' #图片的地址 当前文件夹

pipeline.py

class DoutuImgPipeline(ImagesPipeline): def get_media_requests(self, item, info): yield scrapy.Request(url=item["link"], meta={'item': item}) def file_path(self, request, response=None, info=None): item = request.meta['item'] path = item['name'] + '.jpg' return path

spider.py

import scrapy from doutuba.items import DoutubaItem class DoutuSpider(scrapy.Spider): name = 'doutu' allowed_domains = ['doutula.com'] offset = 1 start_urls = ['https://www.doutula.com/photo/list/'] def parse(self, response): img_srcs = response.xpath('//*[@id="pic-detail"]/div/div[2]/div[2]/ul/li/div/div/a/@href').extract() for link in img_srcs: yield scrapy.Request(link, callback=self.getNew) self.offset += 1 url = "https://www.doutula.com/photo/list/?page=" + str(self.offset) yield scrapy.Request(url, callback=self.parse) def getNew(self, response): item = DoutubaItem() name = response.xpath('//*[@id="detail"]/div/div[2]/li/div[1]/h1/a/text()').extract() img_src = response.xpath('//*[@id="detail"]/div/div[2]/li/div[2]/div/div/div/div/table/tbody/tr[1]/td/img/@src').extract() item["name"] = name[0] item["link"] = img_src[0] yield item

(5)效果图 如有错误，多多指教