Something went wrong. Try again.
I am attempting to learn web scraping with python by reading information from online.
Something went wrong. Try again.
web-scraping scrapeTools.py
123456789101112131415161718192021222324252627from six.moves.urllib.request import urlopenimport pandas as pdfrom bs4 import BeautifulSoupfrom selenium import webdriverfrom selenium.webdriver.chrome.options import Options
def get_first_table(url): page = urlopen(url) # parse the html using beautiful soup and store in variable `soup` soup = BeautifulSoup(page, 'html.parser') found = True try: return soup.find_all("table")[0] except IndexError: # try: chrome_options = Options() chrome_options.add_argument("--headless") driver = webdriver.Chrome(options=chrome_options) driver.get(url) soup = BeautifulSoup(driver.page_source, 'lxml') driver.close() return soup.find_all("table")[0] # except: # found = False # if not found: # raise ValueError("No table found")