scrapy-redis使用例子

可以参考以下代码：

# -*- coding: utf-8 -*-

import scrapy
from scrapy_redis.spiders import RedisSpider

class MySpider(RedisSpider):
name = 'myspider'
redis_key = 'myspider:start_urls'

def parse(self, response):
# 处理响应结果
pass

def make_request_from_data(self, data):
# 从redis队列中获取请求
url = self.decode_request(data)
return scrapy.Request(url, dont_filter=True)

def decode_request(self, data):
# 解码redis队列中的请求
return data.decode('utf-8')

def encode_request(self, request):
# 编码请求，存入redis队列
return request.url.encode('utf-8')

def spider_idle(self):
# 空闲时等待新的请求
self.schedule_next_requests()
self.waiting = True

def schedule_next_requests(self):
# 从redis队列中获取请求并发送
while self.server.rpop(self.redis_key):
url = self.decode_request(data)
yield scrapy.Request(url, dont_filter=True)

def next_request(self):
# 获取下一个请求
request = next(self.spider_idle())
return request

def spider_closed(self, spider):
# 关闭redis连接
self.server.connection_pool.disconnect()