A web scraper build to search specific information for a given compound (and its pseudonyms)
dekker.one
1# Scrapy settings for Fourmi project
2#
3# For simplicity, this file contains only the most important settings by
4# default. All the other settings are documented here:
5#
6# http://doc.scrapy.org/en/latest/topics/settings.html
7#
8
9BOT_NAME = 'FourmiCrawler'
10
11SPIDER_MODULES = ['FourmiCrawler']
12NEWSPIDER_MODULE = 'FourmiCrawler'
13ITEM_PIPELINES = {
14 'FourmiCrawler.pipelines.FourmiPipeline': 100
15}
16
17# Crawl responsibly by identifying yourself (and your website) on the
18# user-agent
19
20# USER_AGENT = 'FourmiCrawler (+http://www.yourdomain.com)'