dekker.one
A web scraper build to search specific information for a given compound (and its pseudonyms)
Added tests for the Spider
+57
+57
tests/test_spider.py
+57
tests/test_spider.py
···
1
+
import unittest
2
+
3
+
from scrapy.http import Request
4
+
5
+
from FourmiCrawler import spider
6
+
from FourmiCrawler.sources.ChemSpider import ChemSpider
7
+
from FourmiCrawler.sources.source import Source
8
+
9
+
10
+
class TestFoumiSpider(unittest.TestCase):
11
+
12
+
def setUp(self):
13
+
self.compound = "test_compound"
14
+
self.attributes = ["a.*", ".*a"]
15
+
self.spi = spider.FourmiSpider(self.compound, self.attributes)
16
+
17
+
def test_init(self):
18
+
self.assertIn(self.compound, self.spi.synonyms)
19
+
for attr in self.attributes:
20
+
self.assertIn(attr, self.spi.selected_attributes)
21
+
22
+
def test_add_source(self):
23
+
src = Source()
24
+
self.spi.add_source(src)
25
+
self.assertIn(src, self.spi._sources)
26
+
27
+
def test_add_sources(self):
28
+
srcs = [Source(), Source(), Source()]
29
+
self.spi.add_sources(srcs)
30
+
31
+
for src in srcs:
32
+
self.assertIn(src, self.spi._sources)
33
+
34
+
def test_start_requests(self):
35
+
self.spi._sources = []
36
+
37
+
src = Source()
38
+
self.spi.add_source(src)
39
+
self.assertEqual(self.spi.start_requests(), [])
40
+
41
+
src2 = ChemSpider()
42
+
self.spi.add_source(src2)
43
+
self.assertIsNotNone(self.spi.start_requests())
44
+
45
+
def test_synonym_requests(self):
46
+
self.spi._sources = []
47
+
48
+
src = Source()
49
+
self.spi.add_source(src)
50
+
self.assertEqual(self.spi.get_synonym_requests("new_compound"), [])
51
+
self.assertIn("new_compound", self.spi.synonyms)
52
+
53
+
src2 = ChemSpider()
54
+
self.spi.add_source(src2)
55
+
self.assertIsInstance(self.spi.get_synonym_requests("other_compound")[0], Request)
56
+
self.assertIn("other_compound", self.spi.synonyms)
57
+
self.assertEqual(self.spi.get_synonym_requests("other_compound"), [])