+5

.gitignore

··· 4 4 #Python Specific ignores 5 5 *.pyc 6 6 7 7 + #may contain authentication information 8 8 + sources.cfg 9 9 + #Another of our config files 10 10 + GUI.cfg 11 11 + 7 12 #THINGS WE WOULD NEVER EVER WANT! 8 13 #ignore thumbnails created by windows 9 14 Thumbs.db

+10 -2

.travis.yml

··· 3 3 language: python 4 4 python: 2.7 5 5 6 6 + before_install: 7 7 + - "export DISPLAY=:99.0" 8 8 + - "sh -e /etc/init.d/xvfb start" 9 9 + 6 10 # command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors 7 11 install: 8 12 - pip install Scrapy docopt 13 13 + - pip install coveralls 9 14 10 15 # command to run tests, e.g. python setup.py test 11 16 script: 12 12 - - nosetests tests 17 17 + - nosetests --with-coverage --cover-package=FourmiCrawler,utils,GUI tests 13 18 14 19 notifications: 15 15 - slack: descartes2:6sgCzx3PvrO9IIMwKxj12dDM 20 20 + slack: descartes2:6sgCzx3PvrO9IIMwKxj12dDM 21 21 + 22 22 + after_success: 23 23 + coveralls --verbose

+20

Changelog.md

··· 1 1 + ### v0.6.0 2 2 + - Feature: Added a Graphical User interface 3 3 + - Feature: Automatic config file createion from config samples 4 4 + - FIX: The default name of the output files will now consist of the compound name and the file format when using the CLI 5 5 + - FIX: A lot of bugfixes of the PubChem plugin, as is wasn't working as it should 6 6 + - FIX: Using absolute path for configuration files 7 7 + - DEV: General Code cleanup in documentation 8 8 + 9 9 + ### v0.5.3 10 10 + - FIX: It is now again possible to use both verbose and the source inclusion/exclusion options 11 11 + - FIX: Logging is now "actually" disabled if not using the verbose option. 12 12 + - FEATURE: Added support for PubChem 13 13 + 14 14 + ### v0.5.2 15 15 + - FIX: Signatured used to contain untracked and older files, current signature 16 16 + should be correct. 17 17 + 18 18 + ### v0.5.1 19 19 + - UPDATED: Logging functionality from command line 20 20 + - DEV: Code cleanup and extra tests

+1 -2

FourmiCrawler/settings.py

··· 18 18 FEED_URI = 'results.json' 19 19 FEED_FORMAT = 'jsonlines' 20 20 21 21 - 22 21 # Crawl responsibly by identifying yourself (and your website) on the 23 22 # user-agent 24 23 25 25 - # USER_AGENT = 'FourmiCrawler (+http://www.yourdomain.com)' 24 24 + USER_AGENT = 'Fourmi'

+127 -60

FourmiCrawler/sources/ChemSpider.py

··· 10 10 11 11 # [TODO] - Maybe clean up usage of '.extract()[0]', because of possible IndexError exception. 12 12 13 13 - 14 13 class ChemSpider(Source): 15 15 - """ChemSpider scraper for synonyms and properties 16 16 - 14 14 + """ 15 15 + ChemSpider scraper for synonyms and properties 17 16 This parser will manage searching for chemicals through the 18 17 ChemsSpider API, and parsing the resulting ChemSpider page. 19 18 The token required for the API should be in a configuration file 20 19 somewhere. 21 20 """ 22 21 23 23 - def __init__(self): 24 24 - Source.__init__(self) 25 25 - 26 26 - website = 'http://www.chemspider.com/*' 22 22 + website = 'http://www\\.chemspider\\.com/.*' 27 23 28 28 - # [TODO] - Save and access token of specific user. 29 29 - search = ('Search.asmx/SimpleSearch?query=%s&token=' 30 30 - '052bfd06-5ce4-43d6-bf12-89eabefd2338') 24 24 + search = 'Search.asmx/SimpleSearch?query=%s&token=' 31 25 structure = 'Chemical-Structure.%s.html' 32 32 - extendedinfo = ('MassSpecAPI.asmx/GetExtendedCompoundInfo?csid=%s&token=' 33 33 - '052bfd06-5ce4-43d6-bf12-89eabefd2338') 26 26 + extendedinfo = 'MassSpecAPI.asmx/GetExtendedCompoundInfo?csid=%s&token=' 34 27 35 35 - ignore_list = [] 28 28 + def __init__(self, config=None): 29 29 + """ 30 30 + Initialization of ChemSpider scraper 31 31 + :param config: a dictionary of settings for this scraper, must contain 32 32 + 'reliability' key 33 33 + """ 34 34 + Source.__init__(self, config) 35 35 + self.ignore_list = [] 36 36 + if 'token' not in self.cfg or self.cfg['token'] == '': 37 37 + log.msg('ChemSpider token not set or empty, search/MassSpec API ' 38 38 + 'not available', level=log.WARNING) 39 39 + self.cfg['token'] = '' 40 40 + self.search += self.cfg['token'] 41 41 + self.extendedinfo += self.cfg['token'] 36 42 37 43 def parse(self, response): 44 44 + """ 45 45 + This function is called when a Response matching the variable 46 46 + 'website' is available for parsing the Response object. 47 47 + :param response: the Scrapy Response object to be parsed 48 48 + :return: a list of Result items and Request objects 49 49 + """ 38 50 sel = Selector(response) 39 51 requests = [] 40 52 requests_synonyms = self.parse_synonyms(sel) ··· 44 56 45 57 return requests 46 58 47 47 - @staticmethod 48 48 - def parse_properties(sel): 49 49 - """scrape Experimental Data and Predicted ACD/Labs tabs""" 59 59 + def parse_properties(self, sel): 60 60 + """ 61 61 + This function scrapes the Experimental Data and Predicted ACD/Labs tabs 62 62 + :param sel: a Selector object of the whole page 63 63 + :return: a list of Result items 64 64 + """ 50 65 properties = [] 51 66 52 52 - # Predicted - ACD/Labs tab 67 67 + properties.extend(self.parse_acdlabstab(sel)) 68 68 + properties.extend(self.parse_experimentaldatatab(sel)) 69 69 + 70 70 + return properties 71 71 + 72 72 + def parse_acdlabstab(self, sel): 73 73 + """ 74 74 + This function scrapes the 'Predicted ACD/Labs tab' under Properties 75 75 + :param sel: a Selector object of the whole page 76 76 + :return: a list of Request objects 77 77 + """ 78 78 + properties = [] 79 79 + 53 80 td_list = sel.xpath('.//table[@id="acdlabs-table"]//td').xpath( 54 81 'normalize-space(string())') 55 82 prop_names = td_list[::2] ··· 61 88 prop_conditions = '' 62 89 63 90 # Test for properties without values, with one hardcoded exception 64 64 - if not re.match(r'^\d', prop_value) or (prop_name == 'Polarizability' and prop_value == '10-24cm3'): 91 91 + if (not re.match(r'^\d', prop_value) or 92 92 + (prop_name == 'Polarizability' and prop_value == '10-24cm3')): 65 93 continue 66 94 67 67 - # Match for condition in parentheses 68 95 m = re.match(r'(.*) \((.*)\)', prop_name) 69 96 if m: 70 97 prop_name = m.group(1) 71 98 prop_conditions = m.group(2) 72 99 73 73 - # Match for condition in value seperated by an 'at' 74 100 m = re.match(r'(.*) at (.*)', prop_value) 75 101 if m: 76 102 prop_value = m.group(1) 77 103 prop_conditions = m.group(2) 78 104 79 79 - new_prop = Result({ 80 80 - 'attribute': prop_name, 81 81 - 'value': prop_value, 82 82 - 'source': 'ChemSpider Predicted - ACD/Labs Tab', 83 83 - 'reliability': 'Unknown', 84 84 - 'conditions': prop_conditions 85 85 - }) 105 105 + new_prop = self.newresult( 106 106 + attribute=prop_name, 107 107 + value=prop_value, 108 108 + source='ChemSpider Predicted - ACD/Labs Tab', 109 109 + conditions=prop_conditions 110 110 + ) 86 111 properties.append(new_prop) 87 87 - log.msg('CS prop: |%s| |%s| |%s|' % 88 88 - (new_prop['attribute'], new_prop['value'], new_prop['source']), 89 89 - level=log.DEBUG) 112 112 + 113 113 + return properties 114 114 + 115 115 + def parse_experimentaldatatab(self, sel): 116 116 + """ 117 117 + This function scrapes Experimental Data tab, Physico-chemical 118 118 + properties in particular. 119 119 + :param sel: a Selector object of the whole page 120 120 + :return: a list of Result items 121 121 + """ 122 122 + properties = [] 90 123 91 91 - # Experimental Data Tab, Physico-chemical properties in particular 92 124 scraped_list = sel.xpath('.//li[span="Experimental Physico-chemical ' 93 125 'Properties"]//li/table/tr/td') 94 126 if not scraped_list: ··· 100 132 if line.xpath('span/text()'): 101 133 property_name = line.xpath('span/text()').extract()[0].rstrip() 102 134 else: 103 103 - new_prop = Result({ 104 104 - 'attribute': property_name[:-1], 105 105 - 'value': line.xpath('text()').extract()[0].rstrip(), 106 106 - 'source': line.xpath( 107 107 - 'strong/text()').extract()[0].rstrip(), 108 108 - 'reliability': 'Unknown', 109 109 - 'conditions': '' 110 110 - }) 111 111 - properties.append(new_prop) 112 112 - log.msg('CS prop: |%s| |%s| |%s|' % 113 113 - (new_prop['attribute'], new_prop['value'], 114 114 - new_prop['source']), level=log.DEBUG) 135 135 + new_prop = self.newresult( 136 136 + attribute=property_name[:-1], 137 137 + value=line.xpath('text()').extract()[0].rstrip(), 138 138 + source=line.xpath('strong/text()').extract()[0].rstrip(), 139 139 + ) 140 140 + properties.append(new_prop) 115 141 116 142 return properties 117 143 118 144 def parse_synonyms(self, sel): 119 119 - """Scrape list of Names and Identifiers""" 145 145 + """ 146 146 + This function scrapes the list of Names and Identifiers 147 147 + :param sel: a Selector object of the whole page 148 148 + :return: a list of Requests 149 149 + """ 120 150 requests = [] 121 151 synonyms = [] 122 152 ··· 148 178 return requests 149 179 150 180 def new_synonym(self, sel, name, category): 151 151 - """Scrape for a single synonym at a given HTML tag""" 181 181 + """ 182 182 + This function scrapes for a single synonym at a given HTML tag 183 183 + :param sel: a Selector object of the given HTML tag 184 184 + :param name: the name of the synonym in the tag 185 185 + :param category: the name of the category the synonym is labeled as 186 186 + :return: a dictionary containing data on the synonym 187 187 + """ 152 188 self.ignore_list.append(name) 153 189 language = sel.xpath('span[@class="synonym_language"]/text()') 154 190 if language: ··· 183 219 } 184 220 return synonym 185 221 186 186 - @staticmethod 187 187 - def parse_extendedinfo(response): 188 188 - """Scrape data from the ChemSpider GetExtendedCompoundInfo API""" 222 222 + def parse_extendedinfo(self, response): 223 223 + """ 224 224 + This function scrapes data from the ChemSpider GetExtendedCompoundInfo 225 225 + API, if a token is present in the configuration settings 226 226 + :param response: a Response object to be parsed 227 227 + :return: a list of Result items 228 228 + """ 189 229 sel = Selector(response) 190 230 properties = [] 191 231 names = sel.xpath('*').xpath('name()').extract() 192 232 values = sel.xpath('*').xpath('text()').extract() 193 233 for (name, value) in zip(names, values): 194 194 - result = Result({ 195 195 - 'attribute': name, 196 196 - 'value': value, # These values have no unit! 197 197 - 'source': 'ChemSpider ExtendedCompoundInfo', 198 198 - 'reliability': 'Unknown', 199 199 - 'conditions': '' 200 200 - }) 234 234 + result = self.newresult( 235 235 + attribute=name, 236 236 + value=value, # These values have no unit! 237 237 + source='ChemSpider ExtendedCompoundInfo', 238 238 + ) 201 239 if result['value']: 202 240 properties.append(result) 203 241 return properties 204 242 243 243 + def newresult(self, attribute, value, conditions='', source='ChemSpider'): 244 244 + """ 245 245 + This function abstracts from the Result item and provides default 246 246 + values. 247 247 + :param attribute: the name of the attribute 248 248 + :param value: the value of the attribute 249 249 + :param conditions: optional conditions regarding the value 250 250 + :param source: the name of the source if it is not ChemSpider 251 251 + :return: A Result item 252 252 + """ 253 253 + return Result({ 254 254 + 'attribute': attribute, 255 255 + 'value': value, 256 256 + 'source': source, 257 257 + 'reliability': self.cfg['reliability'], 258 258 + 'conditions': conditions 259 259 + }) 260 260 + 205 261 def parse_searchrequest(self, response): 206 206 - """Parse the initial response of the ChemSpider Search API """ 262 262 + """ 263 263 + This function parses the initial response of the ChemSpider Search API 264 264 + Requires a valid token to function. 265 265 + :param response: the Response object to be parsed 266 266 + :return: A Request for the information page and a Request for the 267 267 + extendedinfo API call 268 268 + """ 207 269 sel = Selector(response) 208 270 log.msg('chemspider parse_searchrequest', level=log.DEBUG) 209 271 sel.register_namespace('cs', 'http://www.chemspider.com/') ··· 215 277 log.msg('ChemSpider found multiple substances, taking first ' 216 278 'element', level=log.DEBUG) 217 279 csid = csids[0] 218 218 - structure_url = self.website[:-1] + self.structure % csid 219 219 - extendedinfo_url = self.website[:-1] + self.extendedinfo % csid 280 280 + structure_url = self.website[:-2].replace("\\", "") + self.structure % csid 281 281 + extendedinfo_url = self.website[:-2].replace("\\", "") + self.extendedinfo % csid 220 282 log.msg('chemspider URL: %s' % structure_url, level=log.DEBUG) 221 283 return [Request(url=structure_url, 222 284 callback=self.parse), ··· 224 286 callback=self.parse_extendedinfo)] 225 287 226 288 def new_compound_request(self, compound): 227 227 - if compound in self.ignore_list: # [TODO] - add regular expression 289 289 + """ 290 290 + This function is called when a new synonym is returned to the spider 291 291 + to generate new requests 292 292 + :param compound: the name of the compound to search for 293 293 + """ 294 294 + if compound in self.ignore_list or self.cfg['token'] == '': 228 295 return None 229 229 - searchurl = self.website[:-1] + self.search % compound 296 296 + searchurl = self.website[:-2].replace("\\", "") + self.search % compound 230 297 log.msg('chemspider compound', level=log.DEBUG) 231 298 return Request(url=searchurl, callback=self.parse_searchrequest)

+141 -83

FourmiCrawler/sources/NIST.py

··· 13 13 # Result item, but should be included eventually. 14 14 15 15 class NIST(Source): 16 16 - """NIST Scraper plugin 17 17 - 16 16 + """ 17 17 + NIST Scraper plugin 18 18 This plugin manages searching for a chemical on the NIST website 19 19 and parsing the resulting page if the chemical exists on NIST. 20 20 """ 21 21 - website = "http://webbook.nist.gov/*" 21 21 + website = "http://webbook\\.nist\\.gov/.*" 22 22 23 23 search = 'cgi/cbook.cgi?Name=%s&Units=SI&cTP=on' 24 24 25 25 - ignore_list = set() 26 26 - 27 27 - def __init__(self): 28 28 - Source.__init__(self) 25 25 + def __init__(self, config=None): 26 26 + """ 27 27 + Initialization of NIST scraper 28 28 + :param config: configuration variables for this scraper, must contain 29 29 + 'reliability' key. 30 30 + """ 31 31 + Source.__init__(self, config) 32 32 + self.ignore_list = set() 29 33 30 34 def parse(self, response): 35 35 + """ 36 36 + This function is called when a Response matching the variable 37 37 + 'website' is available for parsing the Response object. 38 38 + :param response: The Scrapy Response object to be parsed 39 39 + :return: a list of Result items and Request objects 40 40 + """ 31 41 sel = Selector(response) 32 42 33 43 title = sel.xpath('head/title/text()').extract()[0] ··· 52 62 log.msg('NIST symbol: |%s|, name: |%s|' % (symbol, name), 53 63 level=log.DEBUG) 54 64 65 65 + requests.extend(self.parse_tables(sel, symbol_table)) 66 66 + 67 67 + return requests 68 68 + 69 69 + def parse_tables(self, sel, symbol_table): 70 70 + """ 71 71 + This function identifies and distributes parsing of tables to other 72 72 + functions below. 73 73 + :param sel: A Selector object of the whole page 74 74 + :param symbol_table: a dictionary containing translations of raw HTML 75 75 + tags to human readable names 76 76 + :return: a list of Result items and Requests 77 77 + """ 78 78 + requests = [] 79 79 + 55 80 for table in sel.xpath('//table[@class="data"]'): 56 81 summary = table.xpath('@summary').extract()[0] 57 82 if summary == 'One dimensional data': ··· 82 107 return requests 83 108 84 109 def parse_generic_info(self, sel): 85 85 - """Parses: synonyms, chemical formula, molecular weight, InChI, 86 86 - InChiKey, CAS number 110 110 + """ 111 111 + This function parses: synonyms, chemical formula, molecular weight, 112 112 + InChI, InChiKey, CAS number 113 113 + :param sel: A Selector object of the entire page in the original 114 114 + response 115 115 + :return: a list of Result items 87 116 """ 88 117 ul = sel.xpath('body/ul[li/strong="IUPAC Standard InChI:"]') 89 89 - li = ul.xpath('li') 90 118 91 119 raw_synonyms = ul.xpath('li[strong="Other names:"]/text()').extract() 92 120 for synonym in raw_synonyms[0].strip().split(';\n'): ··· 114 142 115 143 requests = [] 116 144 for key, value in data.iteritems(): 117 117 - result = Result({ 118 118 - 'attribute': key, 119 119 - 'value': value, 120 120 - 'source': 'NIST', 121 121 - 'reliability': 'Unknown', 122 122 - 'conditions': '' 123 123 - }) 145 145 + result = self.newresult( 146 146 + attribute=key, 147 147 + value=value 148 148 + ) 124 149 requests.append(result) 125 150 126 151 return requests 127 152 128 153 def parse_aggregate_data(self, table, symbol_table): 129 129 - """Parses the table(s) which contain possible links to individual 130 130 - data points 154 154 + """ 155 155 + This function parses the table(s) which contain possible links to 156 156 + individual data points 157 157 + :param table: a Selector object of the table to be parsed 158 158 + :param symbol_table: a dictionary containing translations of raw HTML 159 159 + tags to human readable names 160 160 + :return: a list of Result items and Request objects 131 161 """ 132 162 results = [] 133 163 for tr in table.xpath('tr[td]'): 134 164 extra_data_url = tr.xpath('td[last()][a="Individual data points"]' 135 165 '/a/@href').extract() 136 166 if extra_data_url: 137 137 - request = Request(url=self.website[:-1] + extra_data_url[0], 167 167 + request = Request(url=self.website[:-2].replace("\\", "") + extra_data_url[0], 138 168 callback=self.parse_individual_datapoints) 139 169 results.append(request) 140 170 continue ··· 150 180 name = m.group(1) 151 181 condition = m.group(2) 152 182 153 153 - result = Result({ 154 154 - 'attribute': name, 155 155 - 'value': data[1] + ' ' + data[2], 156 156 - 'source': 'NIST', 157 157 - 'reliability': 'Unknown', 158 158 - 'conditions': condition 159 159 - }) 183 183 + result = self.newresult( 184 184 + attribute=name, 185 185 + value=data[1] + ' ' + data[2], 186 186 + conditions=condition 187 187 + ) 160 188 log.msg('NIST: |%s|' % data, level=log.DEBUG) 161 189 results.append(result) 162 190 return results 163 191 164 164 - @staticmethod 165 165 - def parse_transition_data(table, summary): 166 166 - """Parses the table containing properties regarding phase changes""" 192 192 + def parse_transition_data(self, table, summary): 193 193 + """ 194 194 + This function parses the table containing properties regarding phase 195 195 + changes 196 196 + :param table: a Selector object of the table to be parsed 197 197 + :param summary: the name of the property 198 198 + :return: a list of Result items 199 199 + """ 167 200 results = [] 168 201 169 169 - tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) 170 170 - m = re.search(r'\((.*)\)', tr_unit) 171 171 - unit = '!' 172 172 - if m: 173 173 - unit = m.group(1) 202 202 + unit = self.get_unit(table) 174 203 175 204 for tr in table.xpath('tr[td]'): 176 205 tds = tr.xpath('td/text()').extract() 177 177 - result = Result({ 178 178 - 'attribute': summary, 179 179 - 'value': tds[0] + ' ' + unit, 180 180 - 'source': 'NIST', 181 181 - 'reliability': 'Unknown', 182 182 - 'conditions': '%s K, (%s -> %s)' % (tds[1], tds[2], tds[3]) 183 183 - }) 206 206 + result = self.newresult( 207 207 + attribute=summary, 208 208 + value=tds[0] + ' ' + unit, 209 209 + conditions='%s K, (%s -> %s)' % (tds[1], tds[2], tds[3]) 210 210 + ) 184 211 results.append(result) 185 212 186 213 return results 187 214 188 188 - @staticmethod 189 189 - def parse_generic_data(table, summary): 190 190 - """Parses the common tables of 4 and 5 rows. Assumes they are of the 215 215 + def parse_generic_data(self, table, summary): 216 216 + """ 217 217 + Parses the common tables of 4 and 5 rows. Assumes they are of the 191 218 form: 192 219 Symbol (unit)|Temperature (K)|Method|Reference|Comment 193 220 Symbol (unit)|Temperature (K)|Reference|Comment 221 221 + :param table: a Selector object of the table to be parsed 222 222 + :param summary: the name of the property 223 223 + :return: a list of Result items 194 224 """ 195 225 results = [] 196 226 197 197 - tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) 198 198 - m = re.search(r'\((.*)\)', tr_unit) 199 199 - unit = '!' 200 200 - if m: 201 201 - unit = m.group(1) 227 227 + unit = self.get_unit(table) 202 228 203 229 for tr in table.xpath('tr[td]'): 204 230 tds = tr.xpath('td/text()').extract() 205 205 - result = Result({ 206 206 - 'attribute': summary, 207 207 - 'value': tds[0] + ' ' + unit, 208 208 - 'source': 'NIST', 209 209 - 'reliability': 'Unknown', 210 210 - 'conditions': '%s K' % tds[1] 211 211 - }) 231 231 + result = self.newresult( 232 232 + attribute=summary, 233 233 + value=tds[0] + ' ' + unit, 234 234 + conditions='%s K' % tds[1] 235 235 + ) 212 236 results.append(result) 213 237 return results 214 238 215 215 - @staticmethod 216 216 - def parse_antoine_data(table, summary): 217 217 - """Parse table containing parameters for the Antione equation""" 239 239 + def parse_antoine_data(self, table, summary): 240 240 + """ 241 241 + This function parses the table containing parameters for the Antione 242 242 + equation 243 243 + :param table: a Selector object of the table to be parsed 244 244 + :param summary: the name of the property 245 245 + :return: a list of Result items 246 246 + """ 218 247 results = [] 219 248 220 249 for tr in table.xpath('tr[td]'): 221 250 tds = tr.xpath('td/text()').extract() 222 222 - result = Result({ 223 223 - 'attribute': summary, 224 224 - 'value': 'A=%s, B=%s, C=%s' % (tds[1], tds[2], tds[3]), 225 225 - 'source': 'NIST', 226 226 - 'reliability': 'Unknown', 227 227 - 'conditions': '%s K' % tds[0] 228 228 - }) 251 251 + result = self.newresult( 252 252 + attribute=summary, 253 253 + value='A=%s, B=%s, C=%s' % (tds[1], tds[2], tds[3]), 254 254 + conditions='%s K' % tds[0] 255 255 + ) 229 256 results.append(result) 230 257 231 258 return results 232 259 233 233 - @staticmethod 234 234 - def parse_individual_datapoints(response): 235 235 - """Parses the page linked from aggregate data""" 260 260 + def parse_individual_datapoints(self, response): 261 261 + """ 262 262 + This function parses the 'individual data points' page linked from 263 263 + the aggregate data table(s) 264 264 + :param response: the Scrapy Response object to be parsed 265 265 + :return: a list of Result items 266 266 + """ 236 267 sel = Selector(response) 237 268 table = sel.xpath('//table[@class="data"]')[0] 238 269 ··· 245 276 name = m.group(1) 246 277 condition = m.group(2) 247 278 248 248 - tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) 249 249 - m = re.search(r'\((.*)\)', tr_unit) 250 250 - unit = '!' 251 251 - if m: 252 252 - unit = m.group(1) 279 279 + unit = self.get_unit(table) 253 280 254 281 for tr in table.xpath('tr[td]'): 255 282 tds = tr.xpath('td/text()').extract() ··· 258 285 if m: 259 286 uncertainty = '+- %s ' % m.group(1) 260 287 # [TODO]: get the plusminus sign working in here 261 261 - result = Result({ 262 262 - 'attribute': name, 263 263 - 'value': '%s %s%s' % (tds[0], uncertainty, unit), 264 264 - 'source': 'NIST', 265 265 - 'reliability': 'Unknown', 266 266 - 'conditions': condition 267 267 - }) 288 288 + result = self.newresult( 289 289 + attribute=name, 290 290 + value='%s %s%s' % (tds[0], uncertainty, unit), 291 291 + conditions=condition 292 292 + ) 268 293 results.append(result) 269 294 270 295 return results 271 296 297 297 + @staticmethod 298 298 + def get_unit(table): 299 299 + tr_unit = ''.join(table.xpath('tr[1]/th[1]/node()').extract()) 300 300 + m = re.search(r'\((.*)\)', tr_unit) 301 301 + unit = '!' 302 302 + if m: 303 303 + unit = m.group(1) 304 304 + 305 305 + return unit 306 306 + 307 307 + def newresult(self, attribute, value, conditions=''): 308 308 + """ 309 309 + This function abstracts from the Result item and provides default 310 310 + values 311 311 + :param attribute: the name of the attribute 312 312 + :param value: the value of the attribute 313 313 + :param conditions: optional conditions regarding the value 314 314 + :return: A Result item 315 315 + """ 316 316 + return Result( 317 317 + { 318 318 + 'attribute': attribute, 319 319 + 'value': value, 320 320 + 'source': 'NIST', 321 321 + 'reliability': self.cfg['reliability'], 322 322 + 'conditions': conditions 323 323 + }) 324 324 + 272 325 def new_compound_request(self, compound): 326 326 + """ 327 327 + This function is called when a new synonym is returned to the spider 328 328 + to generate new requests 329 329 + :param compound: the name of the compound to search for 330 330 + """ 273 331 if compound not in self.ignore_list: 274 332 self.ignore_list.update(compound) 275 275 - return Request(url=self.website[:-1] + self.search % compound, 333 333 + return Request(url=self.website[:-2].replace("\\", "") + self.search % compound, 276 334 callback=self.parse)

+149

FourmiCrawler/sources/PubChem.py

··· 1 1 + import re 2 2 + 3 3 + from scrapy.http import Request 4 4 + from scrapy import log 5 5 + from scrapy.selector import Selector 6 6 + 7 7 + from source import Source 8 8 + from FourmiCrawler.items import Result 9 9 + 10 10 + 11 11 + class PubChem(Source): 12 12 + """ PubChem scraper for chemical properties 13 13 + 14 14 + This parser parses the part on PubChem pages that gives Chemical and Physical properties of a substance, 15 15 + including sources of the values of properties. 16 16 + """ 17 17 + 18 18 + # PubChem has its data on compound name, properties and their values on different html pages, so different URLs used 19 19 + website = 'http://.*\\.ncbi\\.nlm\\.nih\\.gov/.*' 20 20 + website_www = 'http://www.ncbi.nlm.nih.gov/*' 21 21 + website_pubchem = 'http://pubchem.ncbi.nlm.nih.gov/.*' 22 22 + search = 'pccompound?term=%s' 23 23 + data_url = 'toc/summary_toc.cgi?tocid=27&cid=%s' 24 24 + 25 25 + __spider = None 26 26 + searched_compounds = set() 27 27 + 28 28 + def __init__(self, config): 29 29 + Source.__init__(self, config) 30 30 + self.cfg = config 31 31 + 32 32 + def parse(self, response): 33 33 + """ 34 34 + Distributes the above described behaviour 35 35 + :param response: The incoming search request 36 36 + :return Returns the found properties if response is unique or returns none if it's already known 37 37 + """ 38 38 + requests = [] 39 39 + log.msg('A response from %s just arrived!' % response.url, level=log.DEBUG) 40 40 + 41 41 + sel = Selector(response) 42 42 + compound = sel.xpath('//h1/text()').extract()[0] 43 43 + if compound in self.searched_compounds: 44 44 + return None 45 45 + 46 46 + self.searched_compounds.update(compound) 47 47 + raw_synonyms = sel.xpath('//div[@class="smalltext"]/text()').extract()[0] 48 48 + for synonym in raw_synonyms.strip().split(', '): 49 49 + log.msg('PubChem synonym found: %s' % synonym, level=log.DEBUG) 50 50 + self.searched_compounds.update(synonym) 51 51 + self._spider.get_synonym_requests(synonym) 52 52 + log.msg('Raw synonyms found: %s' % raw_synonyms, level=log.DEBUG) 53 53 + 54 54 + n = re.search(r'cid=(\d+)', response.url) 55 55 + if n: 56 56 + cid = n.group(1) 57 57 + log.msg('cid: %s' % cid, level=log.DEBUG) # getting the right id of the compound with which it can reach 58 58 + # the seperate html page which contains the properties and their values 59 59 + 60 60 + # using this cid to get the right url and scrape it 61 61 + requests.append( 62 62 + Request(url=self.website_pubchem[:-2].replace("\\", "") + self.data_url % cid, callback=self.parse_data)) 63 63 + return requests 64 64 + 65 65 + def parse_data(self, response): 66 66 + """ 67 67 + Parse data found in 'Chemical and Physical properties' part of a substance page. 68 68 + :param response: The response with the page to parse 69 69 + :return: requests: Returns a list of properties with their values, source, etc. 70 70 + """ 71 71 + log.msg('parsing data', level=log.DEBUG) 72 72 + requests = [] 73 73 + 74 74 + sel = Selector(response) 75 75 + props = sel.xpath('//div') 76 76 + 77 77 + for prop in props: 78 78 + prop_name = ''.join(prop.xpath('b/text()').extract()) # name of property that it is parsing 79 79 + if prop.xpath('a'): # parsing for single value in property 80 80 + prop_source = ''.join(prop.xpath('a/@title').extract()) 81 81 + prop_value = ''.join(prop.xpath('a/text()').extract()) 82 82 + new_prop = Result({ 83 83 + 'attribute': prop_name, 84 84 + 'value': prop_value, 85 85 + 'source': prop_source, 86 86 + 'reliability': self.cfg['reliability'], 87 87 + 'conditions': '' 88 88 + }) 89 89 + log.msg('PubChem prop: |%s| |%s| |%s|' % 90 90 + (new_prop['attribute'], new_prop['value'], 91 91 + new_prop['source']), level=log.DEBUG) 92 92 + requests.append(new_prop) 93 93 + elif prop.xpath('ul'): # parsing for multiple values (list) in property 94 94 + prop_values = prop.xpath('ul//li') 95 95 + for prop_li in prop_values: 96 96 + prop_value = ''.join(prop_li.xpath('a/text()').extract()) 97 97 + prop_source = ''.join(prop_li.xpath('a/@title').extract()) 98 98 + new_prop = Result({ 99 99 + 'attribute': prop_name, 100 100 + 'value': prop_value, 101 101 + 'source': prop_source, 102 102 + 'reliability': self.cfg['reliability'], 103 103 + 'conditions': '' 104 104 + }) 105 105 + log.msg('PubChem prop: |%s| |%s| |%s|' % 106 106 + (new_prop['attribute'], new_prop['value'], 107 107 + new_prop['source']), level=log.DEBUG) 108 108 + requests.append(new_prop) 109 109 + 110 110 + return requests 111 111 + 112 112 + def parse_searchrequest(self, response): 113 113 + """ 114 114 + This function parses the response to the new_compound_request Request 115 115 + :param response: the Response object to be parsed 116 116 + :return: A Request for the compound page or what self.parse returns in 117 117 + case the search request forwarded to the compound page 118 118 + """ 119 119 + 120 120 + # check if pubchem forwarded straight to compound page 121 121 + m = re.match(self.website_pubchem, response.url) 122 122 + if m: 123 123 + log.msg('PubChem search forwarded to compound page', 124 124 + level=log.DEBUG) 125 125 + return self.parse(response) 126 126 + 127 127 + sel = Selector(response) 128 128 + 129 129 + results = sel.xpath('//div[@class="rsltcont"]') 130 130 + if results: 131 131 + url = results[0].xpath('div/p/a[1]/@href') 132 132 + else: 133 133 + log.msg('PubChem search found nothing or xpath failed', 134 134 + level=log.DEBUG) 135 135 + return None 136 136 + 137 137 + if url: 138 138 + url = 'http:' + ''.join(url[0].extract()) 139 139 + log.msg('PubChem compound page: %s' % url, level=log.DEBUG) 140 140 + else: 141 141 + log.msg('PubChem search found results, but no url in first result', 142 142 + level=log.DEBUG) 143 143 + return None 144 144 + 145 145 + return Request(url=url, callback=self.parse) 146 146 + 147 147 + def new_compound_request(self, compound): 148 148 + return Request(url=self.website_www[:-1] + self.search % compound, 149 149 + callback=self.parse_searchrequest)

+93 -43

FourmiCrawler/sources/WikipediaParser.py

··· 12 12 """ Wikipedia scraper for chemical properties 13 13 14 14 This parser parses Wikipedia info boxes (also bordered) to obtain properties and their values. 15 15 - It also returns requests with other external sources which contain information on parsed subject. 15 15 + It also returns requests with other external sources which contain information on parsed subject. 16 16 """ 17 17 18 18 - website = "http://en.wikipedia.org/wiki/*" 18 18 + website = "http://en\\.wikipedia\\.org/wiki/.*" 19 19 __spider = None 20 20 searched_compounds = [] 21 21 22 22 - def __init__(self): 23 23 - Source.__init__(self) 22 22 + def __init__(self, config=None): 23 23 + Source.__init__(self, config) 24 24 25 25 def parse(self, response): 26 26 - """ Distributes the above described behaviour """ 26 26 + """ 27 27 + Distributes the above described behaviour 28 28 + :param response: The incoming search request 29 29 + :return: Returns the found properties if response is unique or returns none if it's already known 30 30 + """ 27 31 log.msg('A response from %s just arrived!' % response.url, level=log.DEBUG) 28 32 sel = Selector(response) 29 33 compound = sel.xpath('//h1[@id="firstHeading"]//span/text()').extract()[0] # makes sure to use main page ··· 35 39 return items 36 40 37 41 def parse_infobox(self, sel): 38 38 - """ scrape data from infobox on wikipedia. """ 42 42 + """ 43 43 + Scrape data from infobox on wikipedia. 44 44 + 45 45 + Data from two types of infoboxes: class="infobox bordered" and class="infobox" is scraped and 46 46 + :param sel: The selector with the html-information of the page to parse 47 47 + :return: item_list: Returns a list of properties with their values, source, etc.. 48 48 + """ 49 49 + 39 50 items = [] 40 51 41 41 - # be sure to get chembox (wikipedia template) 42 42 - tr_list = sel.xpath('.//table[@class="infobox bordered"]//td[not(@colspan)]'). \ 43 43 - xpath('normalize-space(string())') 44 44 - prop_names = tr_list[::2] 45 45 - prop_values = tr_list[1::2] 46 46 - for i, prop_name in enumerate(prop_names): 47 47 - item = Result({ 48 48 - 'attribute': prop_name.extract().encode('utf-8'), 49 49 - 'value': prop_values[i].extract().encode('utf-8'), 50 50 - 'source': "Wikipedia", 51 51 - 'reliability': "Unknown", 52 52 - 'conditions': "" 53 53 - }) 54 54 - items.append(item) 55 55 - log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG) 52 52 + # scrape the chembox (wikipedia template) 53 53 + items = self.parse_chembox(sel, items) 56 54 57 57 - #scrape the drugbox (wikipedia template) 58 58 - tr_list2 = sel.xpath('.//table[@class="infobox"]//tr') 59 59 - log.msg('dit: %s' % tr_list2, level=log.DEBUG) 60 60 - for tablerow in tr_list2: 61 61 - log.msg('item: %s' % tablerow.xpath('./th').xpath('normalize-space(string())'), level=log.DEBUG) 62 62 - if tablerow.xpath('./th').xpath('normalize-space(string())') and tablerow.xpath('./td').xpath( 63 63 - 'normalize-space(string())'): 64 64 - item = Result({ 65 65 - 'attribute': tablerow.xpath('./th').xpath('normalize-space(string())').extract()[0].encode('utf-8'), 66 66 - 'value': tablerow.xpath('./td').xpath('normalize-space(string())').extract()[0].encode('utf-8'), 67 67 - 'source': "Wikipedia", 68 68 - 'reliability': "Unknown", 69 69 - 'conditions': "" 70 70 - }) 71 71 - items.append(item) 72 72 - log.msg( 73 73 - 'Wiki prop: |attribute: %s| |value: %s| |%s|' % (item['attribute'], item['value'], item['source']), 74 74 - level=log.DEBUG) 55 55 + # scrape the drugbox (wikipedia template) 56 56 + items = self.parse_drugbox(sel, items) 75 57 76 58 items = filter(lambda a: a['value'] != '', items) # remove items with an empty value 77 59 item_list = self.clean_items(items) ··· 95 77 96 78 return item_list 97 79 80 80 + def parse_chembox(self, sel, items): 81 81 + """ 82 82 + Scrape data from chembox infobox on wikipedia. 83 83 + 84 84 + :param sel: The selector with the html-information of the page to parse 85 85 + :param items: the list of items where the result have to be stored in 86 86 + :return: items: the list of items with the new found and stored items 87 87 + """ 88 88 + tr_list = sel.xpath('.//table[@class="infobox bordered"]//td[not(@colspan)]'). \ 89 89 + xpath('normalize-space(string())') 90 90 + prop_names = tr_list[::2] 91 91 + prop_values = tr_list[1::2] 92 92 + for i, prop_name in enumerate(prop_names): 93 93 + item = self.newresult( 94 94 + attribute=prop_name.extract().encode('utf-8'), 95 95 + value=prop_values[i].extract().encode('utf-8') 96 96 + ) 97 97 + items.append(item) 98 98 + log.msg('Wiki prop: |%s| |%s| |%s|' % (item['attribute'], item['value'], item['source']), level=log.DEBUG) 99 99 + return items 100 100 + 101 101 + def parse_drugbox(self, sel, items): 102 102 + """ 103 103 + Scrape data from drugbox infobox on wikipedia. 104 104 + 105 105 + :param sel: The selector with the html-information of the page to parse 106 106 + :param items: the list of items where the result have to be stored in 107 107 + :return: items: the list of items with the new found and stored items 108 108 + """ 109 109 + tr_list2 = sel.xpath('.//table[@class="infobox"]//tr') 110 110 + log.msg('dit: %s' % tr_list2, level=log.DEBUG) 111 111 + for tablerow in tr_list2: 112 112 + log.msg('item: %s' % tablerow.xpath('./th').xpath('normalize-space(string())'), level=log.DEBUG) 113 113 + if tablerow.xpath('./th').xpath('normalize-space(string())') and tablerow.xpath('./td').xpath( 114 114 + 'normalize-space(string())'): 115 115 + item = self.newresult( 116 116 + attribute=tablerow.xpath('./th').xpath('normalize-space(string())').extract()[0].encode('utf-8'), 117 117 + value=tablerow.xpath('./td').xpath('normalize-space(string())').extract()[0].encode('utf-8'), 118 118 + ) 119 119 + items.append(item) 120 120 + log.msg( 121 121 + 'Wiki prop: |attribute: %s| |value: %s| |%s|' % (item['attribute'], item['value'], item['source']), 122 122 + level=log.DEBUG) 123 123 + return items 124 124 + 98 125 def new_compound_request(self, compound): 99 99 - return Request(url=self.website[:-1] + compound, callback=self.parse) 126 126 + return Request(url=self.website[:-2].replace("\\", "") + compound, callback=self.parse) 100 127 101 128 @staticmethod 102 129 def clean_items(items): 103 103 - """ clean up properties using regex, makes it possible to split the values from the units """ 130 130 + 131 131 + """ 132 132 + Clean up properties using regex, makes it possible to split the values from the units 133 133 + 134 134 + Almost not in use, only cleans J/K/mol values and boiling/melting points. 135 135 + 136 136 + :param items: List of properties with their values, source, etc.. 137 137 + :return: items: List of now cleaned up items 138 138 + """ 104 139 for item in items: 105 140 value = item['value'] 106 141 m = re.search('F;\s(\d+[\.,]?\d*)', value) # clean up numerical Kelvin value (after F) ··· 113 148 114 149 @staticmethod 115 150 def get_identifiers(sel): 116 116 - """ find external links, named 'Identifiers' to different sources. """ 151 151 + """ 152 152 + Find external links, named 'Identifiers' to different sources. 153 153 + 154 154 + :param sel: The selector with the html-information of the page to parse 155 155 + :return: links: New links which can be used to expand the crawlers search 156 156 + """ 117 157 links = sel.xpath('//span[contains(concat(" ",normalize-space(@class)," "),"reflink")]/a' 118 158 '[contains(concat(" ",normalize-space(@class)," "),"external")]/@href').extract() 119 119 - return links 159 159 + return links 160 160 + 161 161 + def newresult(self, attribute, value): 162 162 + return Result( 163 163 + { 164 164 + 'attribute': attribute, 165 165 + 'value': value, 166 166 + 'source': 'Wikipedia', 167 167 + 'reliability': self.cfg['reliability'], 168 168 + 'conditions': '' 169 169 + })

+6 -3

FourmiCrawler/sources/source.py

··· 3 3 4 4 5 5 class Source: 6 6 - website = "http://something/*" # Regex of URI's the source is able to parse 6 6 + website = "http://something/.*" # Regex of URI's the source is able to parse 7 7 _spider = None 8 8 9 9 - def __init__(self): 9 9 + def __init__(self, config=None): 10 10 """ 11 11 Initiation of a new Source 12 12 """ 13 13 + self.cfg = {} 14 14 + if config is not None: 15 15 + self.cfg = config 13 16 pass 14 17 15 18 def parse(self, response): ··· 27 30 :param compound: A compound name. 28 31 :return: A new Scrapy Request 29 32 """ 30 30 - # return Request(url=self.website[:-1] + compound, callback=self.parse) 33 33 + # return Request(url=self.website[:-2].replace("\\", "") + compound, callback=self.parse) 31 34 pass 32 35 33 36 def set_spider(self, spider):

+12 -8

FourmiCrawler/spider.py

··· 9 9 A spider writen for the Fourmi Project which calls upon all available sources to request and scrape data. 10 10 """ 11 11 name = "FourmiSpider" 12 12 - _sources = [] 13 13 - synonyms = set() 14 12 15 15 - def __init__(self, compound=None, selected_attributes=[".*"], *args, **kwargs): 13 13 + def __init__(self, compound=None, selected_attributes=None, *args, **kwargs): 16 14 """ 17 15 Initiation of the Spider 18 16 :param compound: compound that will be searched. 19 17 :param selected_attributes: A list of regular expressions that the attributes should match. 20 18 """ 19 19 + self._sources = [] 20 20 + self.synonyms = set() 21 21 super(FourmiSpider, self).__init__(*args, **kwargs) 22 22 self.synonyms.add(compound) 23 23 - self.selected_attributes = selected_attributes 23 23 + if selected_attributes is None: 24 24 + self.selected_attributes = [".*"] 25 25 + else: 26 26 + self.selected_attributes = selected_attributes 24 27 25 28 def parse(self, response): 26 29 """ ··· 31 34 """ 32 35 for source in self._sources: 33 36 if re.match(source.website, response.url): 34 34 - log.msg("Url: " + response.url + " -> Source: " + source.website, level=log.DEBUG) 37 37 + log.msg("URL: " + response.url + " -> Source: " + source.website, level=log.DEBUG) 35 38 return source.parse(response) 39 39 + log.msg("URL: " + response.url + " -> No compatible source", level=log.INFO) 36 40 return None 37 41 38 38 - def get_synonym_requests(self, compound): 42 42 + def get_synonym_requests(self, compound, force=False): 39 43 """ 40 44 A function that generates new Scrapy Request for each source given a new synonym of a compound. 41 45 :param compound: A compound name 42 46 :return: A list of Scrapy Request objects 43 47 """ 44 48 requests = [] 45 45 - if compound not in self.synonyms: 49 49 + if force or compound not in self.synonyms: 46 50 self.synonyms.add(compound) 47 51 for parser in self._sources: 48 52 parser_requests = parser.new_compound_request(compound) ··· 57 61 """ 58 62 requests = [] 59 63 for synonym in self.synonyms: 60 60 - requests.extend(self.get_synonym_requests(synonym)) 64 64 + requests.extend(self.get_synonym_requests(synonym, force=True)) 61 65 return requests 62 66 63 67 def add_sources(self, sources):

+1

GUI/__init__.py

··· 1 1 + import gui

+30

GUI/configImporter.py

··· 1 1 + import ConfigParser 2 2 + 3 3 + 4 4 + class ConfigImporter(): 5 5 + def __init__(self, filename): 6 6 + """Read the filename into the parser.""" 7 7 + self.filename = filename 8 8 + self.parser = ConfigParser.ConfigParser() 9 9 + self.parser.read(self.filename) 10 10 + 11 11 + def load_common_attributes(self): 12 12 + """Loads common attributes from the initialized file.""" 13 13 + try: 14 14 + return self.parser.get('GUI', 'CommonParameters') 15 15 + except: 16 16 + return 'One, Two, Three' 17 17 + 18 18 + def load_output_types(self): 19 19 + """Loads output types from the initialized file.""" 20 20 + try: 21 21 + return self.parser.get('GUI', 'OutputTypes') 22 22 + except: 23 23 + return 'csv' 24 24 + 25 25 + def load_always_attributes(self): 26 26 + """Loads attributes that are always searched for from the initialized file.""" 27 27 + try: 28 28 + return self.parser.get('GUI', 'AlwaysParameters') 29 29 + except: 30 30 + return 'Name, Weight'

+196

GUI/gui.py

··· 1 1 + from Tkinter import * 2 2 + import os 3 3 + import shutil 4 4 + from tkFileDialog import asksaveasfilename 5 5 + 6 6 + from configImporter import * 7 7 + 8 8 + 9 9 + class GUI(): 10 10 + def __init__(self, search, config_file='GUI.cfg', sourceloader=None, in_source=True): 11 11 + """Boots the window, configuration.""" 12 12 + if not in_source: 13 13 + current_dir = os.path.dirname(os.path.abspath(__file__)) 14 14 + config_file = current_dir + '../' + config_file 15 15 + if not os.path.isfile(config_file): 16 16 + try: 17 17 + shutil.copyfile(os.path.dirname(os.path.abspath(__file__)) + "/../GUI.cfg.sample", config_file) 18 18 + except IOError: 19 19 + print "GUI configuration couldn't be found and couldn't be created." 20 20 + sys.exit() 21 21 + self.configurator = ConfigImporter(config_file) 22 22 + self.sourceloader = sourceloader 23 23 + self.finish_with_search = False 24 24 + self.values = {} 25 25 + self.required_variables = ['substance'] 26 26 + self.search = search 27 27 + self.window, self.variables = self.generate_window(self.load_common_attributes(), self.load_output_types()) 28 28 + 29 29 + def load_common_attributes(self): 30 30 + """Calls the configuration parser for common attributes.""" 31 31 + return [x.strip() for x in self.configurator.load_common_attributes().split(',')] 32 32 + 33 33 + def load_output_types(self): 34 34 + """Calls the configuration parser for output types.""" 35 35 + return [x.strip() for x in self.configurator.load_output_types().split(',')] 36 36 + 37 37 + def load_always_attributes(self): 38 38 + """Calls the configuration parser for attributes that are always used.""" 39 39 + return ','.join([x.strip() for x in self.configurator.load_always_attributes().split(',')]) 40 40 + 41 41 + def set_output(self): 42 42 + self.variable_output_name.set(asksaveasfilename()) 43 43 + self.button_output_name.config(text=self.variable_output_name.get()) 44 44 + 45 45 + def generate_window(self, common_attributes, output_types): 46 46 + """Creates all widgets and variables in the window.""" 47 47 + window = Tk() 48 48 + window.wm_title("Fourmi Crawler") 49 49 + 50 50 + variables = {} 51 51 + 52 52 + variable_substance = StringVar(window) 53 53 + frame_substance = Frame(window) 54 54 + label_substance = Label(frame_substance, text="Substance: ") 55 55 + input_substance = Entry(frame_substance, font=("Helvetica", 12), width=25, textvariable=variable_substance) 56 56 + variables.update({"substance": variable_substance}) 57 57 + frame_substance.pack(side=TOP) 58 58 + label_substance.pack() 59 59 + input_substance.pack() 60 60 + input_substance.focus() 61 61 + 62 62 + frame_all_attributes = Frame(window) 63 63 + frame_selecting_attributes = Frame(frame_all_attributes) 64 64 + frame_new_attributes = Frame(frame_selecting_attributes) 65 65 + label_new_attributes = Label(frame_new_attributes, text="Parameters: ") 66 66 + input_new_attributes = Text(frame_new_attributes, font=("Helvetica", 8), width=25, height=7, padx=5, pady=5) 67 67 + variables.update({"new_attributes": input_new_attributes}) 68 68 + frame_new_attributes.pack(side=LEFT) 69 69 + label_new_attributes.pack() 70 70 + input_new_attributes.pack() 71 71 + 72 72 + frame_common_attributes = Frame(frame_selecting_attributes) 73 73 + label_common_attributes = Label(frame_common_attributes, text="Common Parameters: ") 74 74 + input_common_attributes = Listbox(frame_common_attributes, selectmode=MULTIPLE, height=7) 75 75 + scrollbar_common_attributes = Scrollbar(frame_common_attributes) 76 76 + input_common_attributes.config(yscrollcommand=scrollbar_common_attributes.set) 77 77 + scrollbar_common_attributes.config(command=input_common_attributes.yview) 78 78 + if common_attributes and len(common_attributes) > 0: 79 79 + input_common_attributes.insert(END, *common_attributes) 80 80 + variables.update({"common_attributes": input_common_attributes}) 81 81 + frame_common_attributes.pack(side=RIGHT) 82 82 + label_common_attributes.pack(side=TOP) 83 83 + input_common_attributes.pack(side=LEFT) 84 84 + scrollbar_common_attributes.pack(side=RIGHT, fill=Y) 85 85 + frame_selecting_attributes.pack() 86 86 + 87 87 + frame_last = Frame(window) 88 88 + search_button = Button(frame_last, text="Start search", command=self.prepare_search) 89 89 + cancel_button = Button(frame_last, text="Cancel", command=window.destroy) 90 90 + frame_last.pack(side=BOTTOM) 91 91 + search_button.pack(side=LEFT) 92 92 + cancel_button.pack(side=RIGHT) 93 93 + 94 94 + frame_name = Frame(window) 95 95 + frame_output_name = Frame(frame_name) 96 96 + label_output_name = Label(frame_output_name, text='Output file:') 97 97 + self.variable_output_name = StringVar() 98 98 + self.variable_output_name.set('results.csv') 99 99 + variables.update({'output_name':self.variable_output_name}) 100 100 + self.button_output_name = Button(frame_output_name, command=self.set_output, text="Select file") 101 101 + frame_output_name.pack(side=LEFT) 102 102 + label_output_name.pack() 103 103 + self.button_output_name.pack() 104 104 + frame_name.pack(side=BOTTOM) 105 105 + 106 106 + 107 107 + frame_checkboxes = Frame(window) 108 108 + frame_checkbox_attributes = Frame(frame_checkboxes) 109 109 + variable_all_attributes = BooleanVar() 110 110 + variable_all_attributes.set(True) 111 111 + input_all_attributes = Checkbutton(frame_checkbox_attributes, text="Search ALL parameters", 112 112 + variable=variable_all_attributes) 113 113 + variables.update({"all_attributes": variable_all_attributes}) 114 114 + frame_checkbox_attributes.pack(side=LEFT) 115 115 + input_all_attributes.pack() 116 116 + 117 117 + frame_logging = Frame(frame_checkboxes) 118 118 + variable_logging = BooleanVar() 119 119 + variable_logging.set(False) 120 120 + input_logging = Checkbutton(frame_logging, text="Verbose logging", variable=variable_logging) 121 121 + variables.update({'logging':variable_logging}) 122 122 + frame_logging.pack(side=RIGHT) 123 123 + frame_checkboxes.pack(side=BOTTOM) 124 124 + input_logging.pack() 125 125 + frame_all_attributes.pack() 126 126 + 127 127 + return window, variables 128 128 + 129 129 + def prepare_search(self): 130 130 + """Saves the values from the window for later retrieval.""" 131 131 + variables = self.variables 132 132 + values = {} 133 133 + 134 134 + values.update({"Always attributes": self.load_always_attributes()}) 135 135 + for name, var in variables.iteritems(): 136 136 + if var.__class__ is StringVar: 137 137 + values.update({name: var.get()}) 138 138 + elif var.__class__ is BooleanVar: 139 139 + values.update({name: var.get()}) 140 140 + elif var.__class__ is Text: 141 141 + values.update({name: str(var.get("1.0", END)).strip()}) 142 142 + elif var.__class__ is Listbox: 143 143 + values.update({name: ", ".join([var.get(int(i)) for i in var.curselection()])}) 144 144 + else: 145 145 + print "No known class, {}, {}".format(name, var) 146 146 + 147 147 + values.update({'output_name':self.variable_output_name.get()}) 148 148 + values.update({'output_type':self.check_output_type(values.get('output_name'))}) 149 149 + 150 150 + self.values = values 151 151 + if all([values.get(i) != '' for i in self.required_variables]): 152 152 + self.finish_with_search = True 153 153 + self.window.destroy() 154 154 + else: 155 155 + self.finish_with_search = False 156 156 + #tkMessageBox.showinfo('Not all required information was entered!') 157 157 + 158 158 + def execute_search(self): 159 159 + """Calls the Fourmi crawler with the values from the GUI""" 160 160 + if self.values.get('all_attributes'): 161 161 + attributes = ".*" 162 162 + else: 163 163 + attribute_types = ['attributes', 'Common attributes', 'Always attributes'] 164 164 + attributes = ','.join([str(self.values.get(attribute)) for attribute in attribute_types]) 165 165 + output_file = "file://" + str(self.values.get('output_name')) #Dealing with absolute paths 166 166 + 167 167 + arguments = {'--attributes': attributes, 168 168 + '--exclude': None, 169 169 + '--format': self.values.get('output_type'), 170 170 + '--help': False, 171 171 + '--include': None, 172 172 + '--log': 'log.txt', 173 173 + '--output': output_file, 174 174 + '-v': 0 if self.values.get('logging') else 3, 175 175 + '--version': False, 176 176 + '<compound>': self.values.get('substance'), 177 177 + 'list': False, 178 178 + 'search': True} 179 179 + 180 180 + self.search(arguments, self.sourceloader) 181 181 + 182 182 + def run(self): 183 183 + """Starts the window and the search.""" 184 184 + self.window.mainloop() 185 185 + if self.finish_with_search: 186 186 + self.execute_search() 187 187 + 188 188 + def check_output_type(self, filename): 189 189 + parts = str(filename).split('.') 190 190 + output_types = self.load_output_types() 191 191 + extension = parts[-1] 192 192 + 193 193 + for type in output_types: 194 194 + if extension==type: 195 195 + return extension 196 196 + return output_types[0]

+10

GUI.cfg.sample

··· 1 1 + [GUI] 2 2 + # Personalize options in your User Interface 3 3 + 4 4 + # Commonly used parameters are listed in the GUI for easy selection 5 5 + CommonParameters = Weight, Polarity, Viscosity, Solubility, Name 6 6 + 7 7 + # Parameters that are always used in the search 8 8 + AlwaysParameters = Name 9 9 + 10 10 + OutputTypes = csv, json, jsonlines, xml

+7 -12

README.md

··· 1 1 # Fourmi 2 2 3 3 - **Master branch**: [](https://travis-ci.org/Recondor/Fourmi) 3 3 + **Master branch**: [](https://travis-ci.org/jjdekker/Fourmi) [](https://coveralls.io/r/jjdekker/Fourmi?branch=master) 4 4 5 5 - **Developing branch**: [](https://travis-ci.org/Recondor/Fourmi) 5 5 + **Developing branch**: [](https://travis-ci.org/jjdekker/Fourmi) [](https://coveralls.io/r/jjdekker/Fourmi?branch=develop) 6 6 7 7 Fourmi is an web scraper for chemical substances. The program is designed to be 8 8 used as a search engine to search multiple chemical databases for a specific ··· 23 23 24 24 ### Installing 25 25 26 26 - If you're installing Fourmi, please take a look at our [installation guide](...) 27 27 - on our wiki. When you've installed the application, make sure to check our 28 28 - [usage guide](...). 26 26 + If you're installing Fourmi, please take a look at our installation guides 27 27 + on our [wiki](https://github.com/jjdekker/Fourmi/wiki). When you've installed the application, make sure to check our 28 28 + usage guide on the [Command Line Interface](https://github.com/jjdekker/Fourmi/wiki/CLI) and on the [Graphical User Interface](https://github.com/jjdekker/Fourmi/wiki/GUI). 29 29 30 30 ### Using the Source 31 31 32 32 To use the Fourmi source code multiple dependencies are required. Take a look at 33 33 - the [wiki page](...) on using the application source code for a step by step 33 33 + our [wiki pages](https://github.com/jjdekker/Fourmi/wiki) on using the application source code in our a step by step 34 34 installation guide. 35 35 36 36 When developing for the Fourmi project keep in mind that code readability is a 37 37 must. To maintain the readability, code should be conform with the 38 38 [PEP-8](http://legacy.python.org/dev/peps/pep-0008/) style guide for Python 39 39 code. More information about the different structures and principles of the 40 40 - Fourmi application can be found on our [wiki](...). 40 40 + Fourmi application can be found on our [wiki](https://github.com/jjdekker/Fourmi/wiki). 41 41 42 42 ### To Do 43 43 ··· 45 45 46 46 __Main goals:__ 47 47 48 48 - - Improve our documentation and guides. (Assignee: Dekker) 49 48 - Build an graphical user interface(GUI) as alternative for the command line 50 49 interface(CLI). (Assignee: Harmen) 51 50 - Compiling the source into an windows executable. (Assignee: Bas) 52 52 - - Create an configuration file to hold logins and API keys. 53 53 - - Determine reliability of our data point. 54 54 - - Create an module to gather data from NIST. (Assignee: Rob) 55 55 - - Create an module to gather data from PubChem. (Assignee: Nout) 56 51 57 52 __Side goals:__ 58 53

+108

SIGNED.md

··· 1 1 + ##### Signed by https://keybase.io/jdekker 2 2 + ``` 3 3 + -----BEGIN PGP SIGNATURE----- 4 4 + Version: GnuPG v1.4.11 (GNU/Linux) 5 5 + 6 6 + iQIcBAABAgAGBQJTpMZAAAoJEJrQ9RIUCT6/Hf8P/AyX9ZD5zj6rBi2CwDOTs5aa 7 7 + flVqw9syvdqTzVfXQaR4UrCSOuyuOeAkiqub0BMjxyCurqAwN/SCPf3uOJ/tGXmt 8 8 + ZPtYVHjevJ4mbojLhZiJ2av8LC9VOh3Zl+reR3L2cLuBD4rVSrfUMJtczbbtNlk+ 9 9 + +mczRcTpzNvHQW6mKqyUoKn8xqNnLC7C+p5ybNZ5EADUfoKIF1xyTN6je6fpYZ1U 10 10 + IHxiUzeOvfX9ohmbfnfkpkuSll1nUJWsTgUPKhthJuxEhwCQ1xMdWhxfcyZJaMT2 11 11 + Pxgo8C8S6lzAk4PxBRBoePjgWAeaFmbr317WXHvw6SSHPIdzToKZgDiDC5LWvKxb 12 12 + RRdLZ6w7tg0/FSUexekrUafGT8Je0oIoLUQlNaEQzrPNhDpma1uHFfZg0vb2m4Hq 13 13 + WHLLKTCr6FMczhP1TmuIEtdjKtymT+rO+Ls4ciw+654R7MtBYcmTr+RqmAd+GadJ 14 14 + vJNmGDod2oPwCydEps8bYAbksqRhMmk3xwco/g6dWYh5/+1GzCr80J7fYpqtoPFH 15 15 + V5qKyDQovF5jPlb/buq4mH8XYVT1z4Sx8azKVctMLig57zRnvN0WyskpT09oY7dK 16 16 + TPvIqwTixekndYLcM3QacVq/NhVOOQPFvD0PwU18eKs4EfD2L7iWd2XjV9Az++aD 17 17 + jUY6EwEuOzDCexWP4eM8 18 18 + =h6TK 19 19 + -----END PGP SIGNATURE----- 20 20 + 21 21 + ``` 22 22 + 23 23 + <!-- END SIGNATURES --> 24 24 + 25 25 + ### Begin signed statement 26 26 + 27 27 + #### Expect 28 28 + 29 29 + ``` 30 30 + size exec file contents 31 31 + ./ 32 32 + 412 .gitignore 25059da2ee328837ece01b979cd5c1083ed1679372f06c14c1c58035d8120614 33 33 + 548 .travis.yml 7f11bc58a8e94276ef949afeb107f9f1e184c0dbb84f821705ea2245902ed546 34 34 + 846 Changelog.md 345f9aea4812b37b1b2714703ea0d5edd27414c0f839ec3e322450ad5ec5c6ed 35 35 + FourmiCrawler/ 36 36 + 0 __init__.py e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 37 37 + 304 items.py b00d49a3d53fa13306c7f8b023adb93ab88423c4fce46600689814f6b02bb806 38 38 + 2178 pipelines.py f9b7b84938060751e15e45de5133dffe50c798bff2a20019206fe7c9d677ad49 39 39 + 677 settings.py f1e7d21b899ffc2523516c0ebe67d967dc62495b90c2fe34651042a3049fcd94 40 40 + sources/ 41 41 + 12103 ChemSpider.py f647d70acf9b3f1ee7bde75586aa45156331f977ca7fe836ceac4477a2c0d4ce 42 42 + 12400 NIST.py cdb4c423355ac8fb1097197a9f8df44f667925a785c6bae7c583820da08908ee 43 43 + 6121 PubChem.py 8f8ad40459090b818a384a202e739fe4696a04154df2b8419aee896b0fa02481 44 44 + 6930 WikipediaParser.py ae9f57bbf2aad9c371abcd143fd2dda5995a196cb700734a5035dd94b1988870 45 45 + 0 __init__.py e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 46 46 + 1281 source.py 7927fda259ff2c8096fa526db1f08586de6e04473a491e19a07b092fdeed81fc 47 47 + 3111 spider.py ec7c946907fea10c17ee6dd88a506f3e3bf2cd748e3eb09200487fcec2ae7ba3 48 48 + GUI/ 49 49 + 11 __init__.py 40567015c415e853210425c1b4f3834dbc2a3165e3713e04dd3424b79bc90aa3 50 50 + 940 configImporter.py 5d731d63a3117b25b7e556a746a1dd5b16e8cbb60e57be46de333c31c8c00271 51 51 + 8776 gui.py 20b2220bc3ca55ebfd6d04e8c0bebbf1ae316c85a54db60b8fc02d22642f19d5 52 52 + 299 GUI.cfg.sample 4ee27f7099d588c21358cd645a21621e631d80712f1b514dad898faa5fee2483 53 53 + 1081 LICENSE 36951e5f1910bad3e008ab7228f35ad8933192e52d3c3ae6a5e875765e27192c 54 54 + 3900 README.md f4a1e3ea1700d2b415acfad661cb45f960fe8e8ffbe98dbecb6c7ed071a101ac 55 55 + 3846 x fourmi.py f0b11f5f153f96f6af2e504cdf369e43c04316752de131a659eb6246fd80212a 56 56 + 261 scrapy.cfg 624c068fd06303daa65b8e0d0d3ef88ac1f123be2694ef5b4f3f9a9dcd983f85 57 57 + 416 sources.cfg.sample 11cd0fc18693da17883c98d25a384ae1b6158adfef13778b6dd02b878f6b8a70 58 58 + tests/ 59 59 + 107 __init__.py ce90e54e58a0912cadbe3adcf5166dc72477bf9ce289bf427f8e2f5b25406670 60 60 + 2870 test_configurator.py 318d542b1cda5075a2a9a6be97e9e7a79372ee58e1ab3014c161534094f7364d 61 61 + 1315 test_gui.py 0fb95d0b542765bf52bcebb037bf2ed1299209beab23448af741a93c9fbb1ca8 62 62 + 1892 test_pipeline.py 387a336b0f36722a20e712aa033e5771c44f9e92561dd73acffd53d622c52031 63 63 + 1260 test_sourceloader.py b108b4b80adcdb7401273a9823b1f1a19eb5178776186eb5a9976aed8b1ee869 64 64 + 2113 test_spider.py 300f280377b522737be0d8e4a80031ab118a4011bdbb92131e9c400fcdab6299 65 65 + utils/ 66 66 + 40 __init__.py f1237ae74693e2ec1b3154e57aec27438a80a735e5ccf2411aecd194ef443b6a 67 67 + 4047 configurator.py 8b566a0435a9f105a8ec616b16c3e21edb9b82f8debe1ef9f1df6bbbf20949d5 68 68 + 2537 sourceloader.py f5a5ac2a6aba0658dbe11361f465caabcf3c06c5c8dc9a631874211cc19d2d37 69 69 + ``` 70 70 + 71 71 + #### Ignore 72 72 + 73 73 + ``` 74 74 + /SIGNED.md 75 75 + ``` 76 76 + 77 77 + #### Presets 78 78 + 79 79 + ``` 80 80 + git # ignore .git and anything as described by .gitignore files 81 81 + dropbox # ignore .dropbox-cache and other Dropbox-related files 82 82 + kb # ignore anything as described by .kbignore files 83 83 + ``` 84 84 + 85 85 + <!-- summarize version = 0.0.9 --> 86 86 + 87 87 + ### End signed statement 88 88 + 89 89 + <hr> 90 90 + 91 91 + #### Notes 92 92 + 93 93 + With keybase you can sign any directory's contents, whether it's a git repo, 94 94 + source code distribution, or a personal documents folder. It aims to replace the drudgery of: 95 95 + 96 96 + 1. comparing a zipped file to a detached statement 97 97 + 2. downloading a public key 98 98 + 3. confirming it is in fact the author's by reviewing public statements they've made, using it 99 99 + 100 100 + All in one simple command: 101 101 + 102 102 + ```bash 103 103 + keybase dir verify 104 104 + ``` 105 105 + 106 106 + There are lots of options, including assertions for automating your checks. 107 107 + 108 108 + For more info, check out https://keybase.io/docs/command_line/code_signing

+23 -51

fourmi.py

··· 1 1 - # !/usr/bin/env python 1 1 + #!/usr/bin/env python 2 2 """ 3 3 - Fourmi, a web scraper build to search specific information for a given compound (and it's pseudonyms). 3 3 + Fourmi, a web scraper build to search specific information for a given compound (and its pseudonyms). 4 4 5 5 Usage: 6 6 + fourmi 6 7 fourmi search <compound> 7 8 fourmi [options] search <compound> 8 8 - fourmi [options] [--include=<sourcename> | --exclude=<sourcename>] search <compound> 9 9 + fourmi [options] [-v | -vv | -vvv] [--include=<sourcename> | --exclude=<sourcename>] search <compound> 9 10 fourmi list 10 11 fourmi [--include=<sourcename> | --exclude=<sourcename>] list 11 12 fourmi -h | --help ··· 15 16 --attributes=<regex> Include only that match these regular expressions split by a comma. [default: .*] 16 17 -h --help Show this screen. 17 18 --version Show version. 18 18 - --verbose Verbose logging output. 19 19 + -v Verbose logging output. (Multiple occurrences increase logging level) 19 20 --log=<file> Save log to an file. 20 20 - -o <file> --output=<file> Output file [default: result.*format*] 21 21 - -f <format> --format=<format> Output formats (supported: csv, json, jsonlines, xml) [default: jsonlines] 21 21 + -o <file> --output=<file> Output file [default: <compound>.*format*] 22 22 + -f <format> --format=<format> Output formats (supported: csv, json, jsonlines, xml) [default: csv] 22 23 --include=<regex> Include only sources that match these regular expressions split by a comma. 23 24 --exclude=<regex> Exclude the sources that match these regular expressions split by a comma. 24 25 """ 25 26 26 27 from twisted.internet import reactor 27 28 from scrapy.crawler import Crawler 28 28 - from scrapy import log, signals 29 29 - from scrapy.utils.project import get_project_settings 29 29 + from scrapy import signals, log 30 30 import docopt 31 31 32 32 from FourmiCrawler.spider import FourmiSpider 33 33 - from sourceloader import SourceLoader 33 33 + from utils.configurator import Configurator 34 34 + from utils.sourceloader import SourceLoader 35 35 + from GUI import gui 34 36 35 37 36 38 def setup_crawler(compound, settings, source_loader, attributes): ··· 50 52 crawler.start() 51 53 52 54 53 53 - def scrapy_settings_manipulation(docopt_arguments): 54 54 - """ 55 55 - This function manipulates the Scrapy settings that normally would be set in the settings file. In the Fourmi 56 56 - project these are command line arguments. 57 57 - :param docopt_arguments: A dictionary generated by docopt containing all CLI arguments. 58 58 - """ 59 59 - settings = get_project_settings() 60 60 - 61 61 - if docopt_arguments["--output"] != 'result.*format*': 62 62 - settings.overrides["FEED_URI"] = docopt_arguments["--output"] 63 63 - elif docopt_arguments["--format"] == "jsonlines": 64 64 - settings.overrides["FEED_URI"] = "results.json" 65 65 - elif docopt_arguments["--format"] is not None: 66 66 - settings.overrides["FEED_URI"] = "results." + docopt_arguments["--format"] 67 67 - 68 68 - if docopt_arguments["--format"] is not None: 69 69 - settings.overrides["FEED_FORMAT"] = docopt_arguments["--format"] 70 70 - 71 71 - return settings 72 72 - 73 73 - 74 74 - def start_log(docopt_arguments): 75 75 - """ 76 76 - This function starts the logging functionality of Scrapy using the settings given by the CLI. 77 77 - :param docopt_arguments: A dictionary generated by docopt containing all CLI arguments. 78 78 - """ 79 79 - if docopt_arguments["--log"] is not None: 80 80 - if docopt_arguments["--verbose"]: 81 81 - log.start(logfile=docopt_arguments["--log"], logstdout=False, loglevel=log.DEBUG) 82 82 - else: 83 83 - log.start(logfile=docopt_arguments["--log"], logstdout=True, loglevel=log.WARNING) 84 84 - else: 85 85 - if docopt_arguments["--verbose"]: 86 86 - log.start(logstdout=False, loglevel=log.DEBUG) 87 87 - else: 88 88 - log.start(logstdout=True, loglevel=log.WARNING) 89 89 - 90 90 - 91 55 def search(docopt_arguments, source_loader): 92 56 """ 93 57 The function that facilitates the search for a specific compound. 94 58 :param docopt_arguments: A dictionary generated by docopt containing all CLI arguments. 95 59 :param source_loader: An initiated SourceLoader object pointed at the directory with the sources. 96 60 """ 97 97 - start_log(docopt_arguments) 98 98 - settings = scrapy_settings_manipulation(docopt_arguments) 99 99 - setup_crawler(docopt_arguments["<compound>"], settings, source_loader, docopt_arguments["--attributes"].split(',')) 61 61 + conf = Configurator() 62 62 + conf.set_logging(docopt_arguments["--log"], docopt_arguments["-v"]) 63 63 + conf.set_output(docopt_arguments["--output"], docopt_arguments["--format"], docopt_arguments["<compound>"]) 64 64 + setup_crawler(docopt_arguments["<compound>"], conf.scrapy_settings, 65 65 + source_loader, docopt_arguments["--attributes"].split(',')) 66 66 + if conf.scrapy_settings.getbool("LOG_ENABLED"): 67 67 + log.start(conf.scrapy_settings.get("LOG_FILE"), 68 68 + conf.scrapy_settings.get("LOG_LEVEL"), conf.scrapy_settings.get("LOG_STDOUT")) 100 69 reactor.run() 101 70 102 71 103 72 # The start for the Fourmi Command Line interface. 104 73 if __name__ == '__main__': 105 105 - arguments = docopt.docopt(__doc__, version='Fourmi - V0.4.1') 74 74 + arguments = docopt.docopt(__doc__, version='Fourmi - V0.6.0') 106 75 loader = SourceLoader() 107 76 108 77 if arguments["--include"]: ··· 115 84 elif arguments["list"]: 116 85 print "-== Available Sources ==-" 117 86 print str(loader) 87 87 + else: 88 88 + gui_window = gui.GUI(search, sourceloader=SourceLoader()) 89 89 + gui_window.run()

-18

setup.py

··· 1 1 - import sys 2 2 - from cx_Freeze import setup, Executable 3 3 - 4 4 - # After running the setup file (python setup.py build) the scrapy/VERSION file has to be manually put into the 5 5 - # library.zip, also the FourmiCrawler map has to be copied to both the library and the exe.win32-2.7 folder. after 6 6 - # putting the files in the library the library has to be zipped and replace the old library. 7 7 - # Dependencies are automatically detected, but it might need fine tuning. 8 8 - build_exe_options = {"packages": ["os", "scrapy", "lxml", "w3lib", "pkg_resources", "zope.interface", "twisted.internet"], "excludes": []} 9 9 - 10 10 - # GUI applications require a different base on Windows (the default is for a 11 11 - # console application). 12 12 - base = None 13 13 - 14 14 - setup( name = "Scrapy", 15 15 - version = "0.1", 16 16 - description = "My GUI application!", 17 17 - options = {"build_exe": build_exe_options}, 18 18 - executables = [Executable("fourmi.py", base=base)])

-60

sourceloader.py

··· 1 1 - import inspect 2 2 - import sys 3 3 - import os 4 4 - import re 5 5 - 6 6 - from FourmiCrawler.sources.source import Source 7 7 - 8 8 - 9 9 - class SourceLoader: 10 10 - sources = [] 11 11 - 12 12 - def __init__(self, rel_dir="FourmiCrawler/sources"): 13 13 - 14 14 - if hasattr(sys,'frozen'): 15 15 - path = os.path.dirname(sys.executable) 16 16 - else: 17 17 - path = os.path.dirname(os.path.abspath(__file__)) 18 18 - 19 19 - path += "/" + rel_dir 20 20 - known_parser = set() 21 21 - 22 22 - for py in [f[:-3] for f in os.listdir(path) if f.endswith('.py') and f != '__init__.py']: 23 23 - mod = __import__('.'.join([rel_dir.replace('/', "."), py]), fromlist=[py]) 24 24 - classes = [getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x))] 25 25 - for cls in classes: 26 26 - if issubclass(cls, Source) and cls not in known_parser: 27 27 - self.sources.append(cls()) # [review] - Would we ever need arguments for the parsers? 28 28 - # known_parser.add(cls) 29 29 - 30 30 - def include(self, source_names): 31 31 - """ 32 32 - This function excludes all sources that don't match the given regular expressions. 33 33 - :param source_names: A list of regular expression (strings) 34 34 - """ 35 35 - new = set() 36 36 - for name in source_names: 37 37 - new.update([src for src in self.sources if re.match(name, src.__class__.__name__)]) 38 38 - self.sources = list(new) 39 39 - 40 40 - def exclude(self, source_names): 41 41 - """ 42 42 - This function excludes all sources that match the given regular expressions. 43 43 - :param source_names: A list of regular expression (strings) 44 44 - """ 45 45 - exclude = [] 46 46 - for name in source_names: 47 47 - exclude.extend([src for src in self.sources if re.match(name, src.__class__.__name__)]) 48 48 - self.sources = [src for src in self.sources if src not in exclude] 49 49 - 50 50 - def __str__(self): 51 51 - """ 52 52 - This function returns a string with all sources currently available in the SourceLoader. 53 53 - :return: a string with all available sources. 54 54 - """ 55 55 - string = "" 56 56 - for src in self.sources: 57 57 - string += "Source: " + src.__class__.__name__ 58 58 - string += " - " 59 59 - string += "URI: " + src.website + "\n" 60 60 - return string

+19

sources.cfg.sample

··· 1 1 + [DEFAULT] 2 2 + reliability = Unknown 3 3 + 4 4 + #For each source listed in FourmiCrawler/sources there should be a section 5 5 + #named exactly as the filename in here. If not present, the DEFAULT value is 6 6 + #used for reliability of that source. 7 7 + 8 8 + [ChemSpider] 9 9 + reliability = High 10 10 + #token=Paste ChemSpider API token here and remove the hashtag 11 11 + 12 12 + [NIST] 13 13 + reliability = High 14 14 + 15 15 + [WikipediaParser] 16 16 + reliability = Medium 17 17 + 18 18 + [PubChem] 19 19 + reliability = High

+5

tests/__init__.py

··· 1 1 + import test_configurator 2 2 + import test_gui 3 3 + import test_pipeline 4 4 + import test_sourceloader 5 5 + import test_spider 1 6

+68

tests/test_configurator.py

··· 1 1 + import unittest 2 2 + import ConfigParser 3 3 + 4 4 + from utils.configurator import Configurator 5 5 + 6 6 + 7 7 + class TestConfigurator(unittest.TestCase): 8 8 + 9 9 + def setUp(self): 10 10 + self.conf = Configurator() 11 11 + 12 12 + def test_set_output(self): 13 13 + self.conf.set_output(filename="test.txt", fileformat="csv", compound="test") 14 14 + self.assertEqual(self.conf.scrapy_settings["FEED_URI"], "test.txt") 15 15 + self.assertEqual(self.conf.scrapy_settings["FEED_FORMAT"], "csv") 16 16 + 17 17 + self.conf.set_output("<compound>.*format*", "jsonlines", "test") 18 18 + self.assertEqual(self.conf.scrapy_settings["FEED_URI"], "test.json") 19 19 + self.assertEqual(self.conf.scrapy_settings["FEED_FORMAT"], "jsonlines") 20 20 + 21 21 + self.conf.set_output("<compound>.*format*", "csv", "test") 22 22 + self.assertEqual(self.conf.scrapy_settings["FEED_URI"], "test.csv") 23 23 + self.assertEqual(self.conf.scrapy_settings["FEED_FORMAT"], "csv") 24 24 + 25 25 + def test_start_log(self): 26 26 + for i in range(0, 3): 27 27 + self.conf.set_logging("TEST", i) 28 28 + self.assertEqual(self.conf.scrapy_settings.get("LOG_FILE"), "TEST") 29 29 + if i > 0: 30 30 + self.assertEqual(self.conf.scrapy_settings.get("LOG_ENABLED"), True) 31 31 + if i > 1: 32 32 + self.assertEqual(self.conf.scrapy_settings.get("LOG_STDOUT"), False) 33 33 + else: 34 34 + self.assertEqual(self.conf.scrapy_settings.get("LOG_STDOUT"), True) 35 35 + else: 36 36 + self.assertEqual(self.conf.scrapy_settings.get("LOG_ENABLED"), False) 37 37 + self.assertEqual(self.conf.scrapy_settings.get("LOG_STDOUT"), True) 38 38 + if i == 1: 39 39 + self.assertEqual(self.conf.scrapy_settings.get("LOG_LEVEL"), "WARNING") 40 40 + elif i == 2: 41 41 + self.assertEqual(self.conf.scrapy_settings.get("LOG_LEVEL"), "INFO") 42 42 + elif i == 3: 43 43 + self.assertEqual(self.conf.scrapy_settings.get("LOG_LEVEL"), "DEBUG") 44 44 + 45 45 + self.conf.set_logging(verbose=i) 46 46 + self.assertEqual(self.conf.scrapy_settings.get("LOG_FILE"), None) 47 47 + 48 48 + def test_read_sourceconfiguration(self): 49 49 + config = self.conf.read_sourceconfiguration() 50 50 + self.assertIsInstance(config, ConfigParser.ConfigParser) 51 51 + 52 52 + def test_get_section(self): 53 53 + config = ConfigParser.ConfigParser() 54 54 + section = self.conf.get_section(config, 'test') 55 55 + self.assertIn('reliability', section) 56 56 + self.assertEquals(section['reliability'], '') 57 57 + 58 58 + config.set('DEFAULT', 'reliability', 'Low') 59 59 + 60 60 + section = self.conf.get_section(config, 'test') 61 61 + self.assertEquals(section['reliability'], 'Low') 62 62 + 63 63 + config.add_section('test') 64 64 + config.set('test', 'var', 'Maybe') 65 65 + 66 66 + section = self.conf.get_section(config, 'test') 67 67 + self.assertEquals(section['reliability'], 'Low') 68 68 + self.assertEqual(section['var'], 'Maybe')

+32

tests/test_gui.py

··· 1 1 + import unittest 2 2 + 3 3 + from GUI import gui 4 4 + 5 5 + class TestGUI(unittest.TestCase): 6 6 + def setUp(self): 7 7 + pass 8 8 + 9 9 + def test_empty_attributes(self): 10 10 + self.test_gui = gui.GUI(None, config_file="../GUI.cfg.sample", in_source=True) 11 11 + self.test_gui.window.after(9, self.test_gui.prepare_search) 12 12 + self.test_gui.window.after(11, self.test_gui.window.destroy) 13 13 + self.test_gui.run() 14 14 + 15 15 + output_type = self.test_gui.configurator.load_output_types().split(',')[0] 16 16 + 17 17 + self.assertEqual(self.test_gui.values.get('substance'), '') 18 18 + self.assertEqual(self.test_gui.values.get('output_type'), output_type) 19 19 + self.assertEqual(self.test_gui.values.get('output_name'), 'results.csv') 20 20 + 21 21 + 22 22 + def test_no_configurations(self): 23 23 + self.test_gui = gui.GUI(None, config_file="../GUI.cfg.sample") 24 24 + self.test_gui.configurator = gui.ConfigImporter('') 25 25 + self.test_gui.finish_with_search = True 26 26 + self.test_gui.window.after(9, self.test_gui.prepare_search) 27 27 + self.test_gui.window.after(11, self.test_gui.window.destroy) 28 28 + self.test_gui.run() 29 29 + 30 30 + self.assertEqual(self.test_gui.values.get('substance'), '') 31 31 + self.assertEqual(self.test_gui.values.get('output_type'), 'csv') 32 32 + self.assertEqual(self.test_gui.values.get('output_name'), 'results.csv')

+1

tests/test_pipeline.py

··· 13 13 def test_none_pipeline(self): 14 14 # Testing the pipeline that replaces the None values in items. 15 15 self.testItem["value"] = "abc" 16 16 + self.testItem["source"] = None 16 17 pipe = pipelines.RemoveNonePipeline() 17 18 processed = pipe.process_item(self.testItem, spider.FourmiSpider()) 18 19

+1 -1

tests/test_sourceloader.py

··· 1 1 import unittest 2 2 3 3 - from sourceloader import SourceLoader 3 3 + from utils.sourceloader import SourceLoader 4 4 5 5 6 6 class TestSourceloader(unittest.TestCase):

+7 -5

tests/test_spider.py

··· 3 3 from scrapy.http import Request 4 4 5 5 from FourmiCrawler import spider 6 6 - from FourmiCrawler.sources.ChemSpider import ChemSpider 6 6 + from FourmiCrawler.sources.NIST import NIST 7 7 from FourmiCrawler.sources.source import Source 8 8 9 9 ··· 41 41 self.spi.add_source(src) 42 42 self.assertEqual(self.spi.start_requests(), []) 43 43 44 44 - src2 = ChemSpider() 44 44 + src2 = NIST() 45 45 self.spi.add_source(src2) 46 46 - self.assertIsNotNone(self.spi.start_requests()) 46 46 + requests = self.spi.start_requests() 47 47 + self.assertGreater(len(requests), 0) 48 48 + self.assertIsInstance(requests[0], Request) 47 49 48 50 def test_synonym_requests(self): 49 51 # A test for the synonym request function ··· 54 56 self.assertEqual(self.spi.get_synonym_requests("new_compound"), []) 55 57 self.assertIn("new_compound", self.spi.synonyms) 56 58 57 57 - src2 = ChemSpider() 59 59 + src2 = NIST() 58 60 self.spi.add_source(src2) 59 61 self.assertIsInstance(self.spi.get_synonym_requests("other_compound")[0], Request) 60 62 self.assertIn("other_compound", self.spi.synonyms) 61 61 - self.assertEqual(self.spi.get_synonym_requests("other_compound"), []) 63 63 + self.assertEqual(self.spi.get_synonym_requests("other_compound"), [])

+2

utils/__init__.py

··· 1 1 + import configurator 2 2 + import sourceloader

+101

utils/configurator.py

··· 1 1 + import ConfigParser 2 2 + import os 3 3 + import shutil 4 4 + 5 5 + from scrapy.utils.project import get_project_settings 6 6 + 7 7 + 8 8 + class Configurator: 9 9 + """ 10 10 + A helper class in the fourmi class. This class is used to process the settings as set 11 11 + from one of the Fourmi applications. 12 12 + """ 13 13 + 14 14 + def __init__(self): 15 15 + self.scrapy_settings = get_project_settings() 16 16 + 17 17 + def set_output(self, filename, fileformat, compound): 18 18 + """ 19 19 + This function manipulates the Scrapy output file settings that normally would be set in the settings file. 20 20 + In the Fourmi project these are command line arguments. 21 21 + :param filename: The filename of the file where the output will be put. 22 22 + :param fileformat: The format in which the output will be. 23 23 + """ 24 24 + 25 25 + if filename != '<compound>.*format*': 26 26 + self.scrapy_settings.overrides["FEED_URI"] = filename 27 27 + elif fileformat == "jsonlines": 28 28 + self.scrapy_settings.overrides["FEED_URI"] = compound + ".json" 29 29 + elif fileformat is not None: 30 30 + self.scrapy_settings.overrides["FEED_URI"] = compound + "." + fileformat 31 31 + 32 32 + if fileformat is not None: 33 33 + self.scrapy_settings.overrides["FEED_FORMAT"] = fileformat 34 34 + 35 35 + def set_logging(self, logfile=None, verbose=0): 36 36 + """ 37 37 + This function changes the default settings of Scapy's logging functionality 38 38 + using the settings given by the CLI. 39 39 + :param logfile: The location where the logfile will be saved. 40 40 + :param verbose: A integer value to switch between loglevels. 41 41 + """ 42 42 + if verbose != 0: 43 43 + self.scrapy_settings.overrides["LOG_ENABLED"] = True 44 44 + else: 45 45 + self.scrapy_settings.overrides["LOG_ENABLED"] = False 46 46 + 47 47 + if verbose == 1: 48 48 + self.scrapy_settings.overrides["LOG_LEVEL"] = "WARNING" 49 49 + elif verbose == 2: 50 50 + self.scrapy_settings.overrides["LOG_LEVEL"] = "INFO" 51 51 + else: 52 52 + self.scrapy_settings.overrides["LOG_LEVEL"] = "DEBUG" 53 53 + 54 54 + if verbose > 1: 55 55 + self.scrapy_settings.overrides["LOG_STDOUT"] = False 56 56 + else: 57 57 + self.scrapy_settings.overrides["LOG_STDOUT"] = True 58 58 + 59 59 + if logfile is not None: 60 60 + self.scrapy_settings.overrides["LOG_FILE"] = logfile 61 61 + else: 62 62 + self.scrapy_settings.overrides["LOG_FILE"] = None 63 63 + 64 64 + @staticmethod 65 65 + def read_sourceconfiguration(): 66 66 + """ 67 67 + This function reads sources.cfg in the main folder for configuration 68 68 + variables for sources 69 69 + :return a ConfigParser object of sources.cfg 70 70 + """ 71 71 + current_dir = os.path.dirname(os.path.abspath(__file__)) 72 72 + config_path = current_dir + '/../sources.cfg' 73 73 + # [TODO]: location of sources.cfg should be softcoded eventually 74 74 + if not os.path.isfile(config_path): 75 75 + try: 76 76 + shutil.copyfile(os.path.dirname(os.path.abspath(__file__)) + "/../sources.cfg.sample", config_path) 77 77 + except IOError: 78 78 + print "WARNING: Source configuration couldn't be found and couldn't be created." 79 79 + config = ConfigParser.ConfigParser() 80 80 + config.read(config_path) 81 81 + return config 82 82 + 83 83 + @staticmethod 84 84 + def get_section(config, sourcename): 85 85 + """ 86 86 + This function reads a config section labeled in variable sourcename and 87 87 + tests whether the reliability variable is set else set to empty string. 88 88 + Return the default section if the labeled config section does not exist 89 89 + :param config: a ConfigParser object 90 90 + :param sourcename: the name of the section to be read 91 91 + :return a dictionary of the section in the config labeled in sourcename 92 92 + """ 93 93 + section = dict() 94 94 + if config.has_section(sourcename): 95 95 + section = dict(config.items(sourcename)) 96 96 + elif config.defaults(): 97 97 + section = config.defaults() 98 98 + if 'reliability' not in section: 99 99 + print 'WARNING: Reliability not set for %s' % sourcename 100 100 + section['reliability'] = '' 101 101 + return section

+64

utils/sourceloader.py

··· 1 1 + import inspect 2 2 + import os 3 3 + import re 4 4 + 5 5 + from FourmiCrawler.sources.source import Source 6 6 + from utils.configurator import Configurator 7 7 + 8 8 + 9 9 + class SourceLoader: 10 10 + sources = [] 11 11 + 12 12 + def __init__(self, rel_dir="../FourmiCrawler/sources"): 13 13 + """ 14 14 + The initiation of a SourceLoader, selects and indexes a directory for usable sources. 15 15 + Also loads a configuration file for Sources and passes the arguments in 16 16 + the named section to the source 17 17 + :param rel_dir: A relative path to a directory. 18 18 + """ 19 19 + path = os.path.dirname(os.path.abspath(__file__)) 20 20 + path += "/" + rel_dir 21 21 + known_parser = set() 22 22 + 23 23 + config = Configurator.read_sourceconfiguration() 24 24 + 25 25 + for py in [f[:-3] for f in os.listdir(path) if f.endswith('.py') and f != '__init__.py']: 26 26 + mod = __import__('.'.join([rel_dir.replace("../", "").replace("/", "."), py]), fromlist=[py]) 27 27 + classes = [getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x))] 28 28 + for cls in classes: 29 29 + if issubclass(cls, Source) and cls not in known_parser: 30 30 + sourcecfg = Configurator.get_section(config, cls.__name__) 31 31 + self.sources.append(cls(sourcecfg)) 32 32 + known_parser.add(cls) 33 33 + 34 34 + def include(self, source_names): 35 35 + """ 36 36 + This function excludes all sources that don't match the given regular expressions. 37 37 + :param source_names: A list of regular expression (strings) 38 38 + """ 39 39 + new = set() 40 40 + for name in source_names: 41 41 + new.update([src for src in self.sources if re.match(name, src.__class__.__name__)]) 42 42 + self.sources = list(new) 43 43 + 44 44 + def exclude(self, source_names): 45 45 + """ 46 46 + This function excludes all sources that match the given regular expressions. 47 47 + :param source_names: A list of regular expression (strings) 48 48 + """ 49 49 + exclude = [] 50 50 + for name in source_names: 51 51 + exclude.extend([src for src in self.sources if re.match(name, src.__class__.__name__)]) 52 52 + self.sources = [src for src in self.sources if src not in exclude] 53 53 + 54 54 + def __str__(self): 55 55 + """ 56 56 + This function returns a string with all sources currently available in the SourceLoader. 57 57 + :return: a string with all available sources. 58 58 + """ 59 59 + string = "" 60 60 + for src in self.sources: 61 61 + string += "Source: " + src.__class__.__name__ 62 62 + string += " - " 63 63 + string += "URI: " + src.website + "\n" 64 64 + return string