Something went wrong. Try again.
Command line interface for accessing the Open Web Index shards and slices.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104import astfrom datetime import datetimefrom typing import List, Dictfrom dateutil import parserimport json as j
class DateConverter: def __init__(self, **kwargs): pass
def __call__(self, jsons:List[Dict])->List[Dict]: """ expect list of dicts and returns a list of dicts that must be JSON Serializable (i.e. no complex data types""" for json in jsons: if "warc_date" in json and isinstance(json["warc_date"],str): json["warc_date"] =parser.parse(json["warc_date"]).strftime('%Y-%m-%dT%H:%M:%SZ') if "day" in json and "month" in json and "year" in json: json["date"] = datetime(json["year"],json["month"],json["day"]).strftime('%Y-%m-%dT%H:%M:%SZ') return jsons
class OURSConverter:
"""Convertor for title based search (i.e. OURS in the OWSEU Dashboard). Makes date conversions and ensures some particular schemas.""" def __init__(self, **kwargs): pass
def __call__(self, jsons:List[Dict])->List[Dict]: """ expect list of dicts and returns a list of dicts that must be JSON Serializable (i.e. no complex data types""" for json in jsons: if "day" in json and "month" in json and "year" in json: try: json["date"] = datetime(int(json["year"]),int(json["month"]),int(json["day"])).strftime('%Y-%m-%dT%H:%M:%SZ') except: json["date"]= None if "warc_date" in json and isinstance(json["warc_date"],str): try: json["warc_date"] =parser.parse(json["warc_date"]).strftime('%Y-%m-%dT%H:%M:%SZ') except: json["warc_date"] = None else: json["warc_date"] = json.get("date","1973-01-01T00:00:00Z") del json["day"],json["month"],json["year"] if json.get("curlielabels", None) and isinstance(json.get("curlielabels", None), str): try: json["curlielabels"] = ast.literal_eval(json.get("curlielabels", "[]")) except Exception as e: import traceback traceback.print_exception() json["curlielabels"] =[] if json.get("curlielabels_en", None) and isinstance(json.get("curlielabels_en", None), str): try: json["curlielabels_en"] = ast.literal_eval(json.get("curlielabels_en", "[]")) except Exception as e: import traceback traceback.print_exception() json["curlielabels_en"] =[] if json.get("url", None): # user url as id json["_id"]=json["url"] if json.get("id", None): # overwrite by internal id json["_id"]=json["id"] # todo: implement proper unique ids. # todo: add main content converter and strip html. # creates unfortunately schema problems. #for f in ["json-ld", "microdata", "opengraph"]: # try: # if not json.get(f,None): # json[f]=[] # continue # json[f]= ast.literal_eval(json.get(f,"[]")) # except Exception as e1: # try: # json[f]= j.loads(json.get(f,"[]")) # except Exception as e: # json[f] = [{"original":str(json.get(f,""))}] # if not isinstance(json[f], list): # json[f] = [json[f]]
return jsons
class MicrodataConverter:
def __init__(self, expand=True, columns="microdata;json-ld", **kwargs): """ param expand: expands the micordata """ self.expand=True self.columns = columns.split(";") self.kwargs = kwargs self._date = DateConverter()
def __call__(self, jsons:List[Dict])->List[Dict]: # lets get proper date formats first. jsons = self._date(jsons) results = [] try: raise NotImplementedError("not implemened yet.") except Exception as e: print("error in parsing json: ", e)