38 lines
1.2 KiB
Python
38 lines
1.2 KiB
Python
# Scrapy settings for example project
|
|
#
|
|
# For simplicity, this file contains only the most important settings by
|
|
# default. All the other settings are documented here:
|
|
#
|
|
# http://doc.scrapy.org/topics/settings.html
|
|
#
|
|
SPIDER_MODULES = ["example.spiders"]
|
|
NEWSPIDER_MODULE = "example.spiders"
|
|
|
|
LOG_LEVEL = "WARNING"
|
|
|
|
USER_AGENT = "scrapy-redis (+https://github.com/rolando/scrapy-redis)"
|
|
|
|
#设置重复过滤器模块
|
|
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
|
|
#设置调度器，scrapy_redis具备与数据库交互的功能
|
|
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
|
|
#设置当爬虫结束时是否保持redis数据库中的去重集合与任务队列
|
|
SCHEDULER_PERSIST = True
|
|
# SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderPriorityQueue"
|
|
# SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderQueue"
|
|
# SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderStack"
|
|
|
|
ITEM_PIPELINES = {
|
|
"example.pipelines.ExamplePipeline": 300,
|
|
#当开启该管道，该管道将会把数据存到redis数据库中
|
|
"scrapy_redis.pipelines.RedisPipeline": 400,
|
|
}
|
|
#设置redis数据库
|
|
REDIS_URL = "redis://127.0.0.1:6379"
|
|
|
|
LOG_LEVEL = "DEBUG"
|
|
|
|
# Introduce an artifical delay to make use of parallelism. to speed up the
|
|
# crawl.
|
|
DOWNLOAD_DELAY = 1
|