A web scraper build to search specific information for a given compound (and its pseudonyms)
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

Added tests for the Spider

+57
+57
tests/test_spider.py
··· 1 + import unittest 2 + 3 + from scrapy.http import Request 4 + 5 + from FourmiCrawler import spider 6 + from FourmiCrawler.sources.ChemSpider import ChemSpider 7 + from FourmiCrawler.sources.source import Source 8 + 9 + 10 + class TestFoumiSpider(unittest.TestCase): 11 + 12 + def setUp(self): 13 + self.compound = "test_compound" 14 + self.attributes = ["a.*", ".*a"] 15 + self.spi = spider.FourmiSpider(self.compound, self.attributes) 16 + 17 + def test_init(self): 18 + self.assertIn(self.compound, self.spi.synonyms) 19 + for attr in self.attributes: 20 + self.assertIn(attr, self.spi.selected_attributes) 21 + 22 + def test_add_source(self): 23 + src = Source() 24 + self.spi.add_source(src) 25 + self.assertIn(src, self.spi._sources) 26 + 27 + def test_add_sources(self): 28 + srcs = [Source(), Source(), Source()] 29 + self.spi.add_sources(srcs) 30 + 31 + for src in srcs: 32 + self.assertIn(src, self.spi._sources) 33 + 34 + def test_start_requests(self): 35 + self.spi._sources = [] 36 + 37 + src = Source() 38 + self.spi.add_source(src) 39 + self.assertEqual(self.spi.start_requests(), []) 40 + 41 + src2 = ChemSpider() 42 + self.spi.add_source(src2) 43 + self.assertIsNotNone(self.spi.start_requests()) 44 + 45 + def test_synonym_requests(self): 46 + self.spi._sources = [] 47 + 48 + src = Source() 49 + self.spi.add_source(src) 50 + self.assertEqual(self.spi.get_synonym_requests("new_compound"), []) 51 + self.assertIn("new_compound", self.spi.synonyms) 52 + 53 + src2 = ChemSpider() 54 + self.spi.add_source(src2) 55 + self.assertIsInstance(self.spi.get_synonym_requests("other_compound")[0], Request) 56 + self.assertIn("other_compound", self.spi.synonyms) 57 + self.assertEqual(self.spi.get_synonym_requests("other_compound"), [])