def _get_soup(url, backend='selenium'):
 if (backend == 'requests'):
  req = requests.get(url, headers={'User-Agent': UA})
  html_doc = req.text
  soup = BeautifulSoup(html_doc)
  if (soup.find('div', attrs={'id': 'gs_ab_md'}) is None):
   print 'Falling back on to selenium backend due to captcha.'
   backend = 'selenium'
 if (backend == 'selenium'):
  from selenium import webdriver
  import selenium.webdriver.support.ui as ui
  driver = webdriver.Firefox()
  wait = ui.WebDriverWait(driver, 200)
  driver.get(url)
  wait.until((lambda driver: driver.find_elements_by_id('gs_ab_md')))
  html_doc = driver.page_source
  soup = BeautifulSoup(html_doc)
  driver.close()
 return soup

def _get_soup ( url , backend = 'selenium' ) : if ( backend == 'requests' ) : req = requests . get ( url , headers = { 'User-Agent' : UA } ) html_doc = req . text soup = BeautifulSoup ( html_doc ) if ( soup . find ( 'div' , attrs = { 'id' : 'gs_ab_md' } ) is None ) : print 'Falling back on to selenium backend due to captcha.' backend = 'selenium' if ( backend == 'selenium' ) : from selenium import webdriver import selenium . webdriver . support . ui as ui driver = webdriver . Firefox ( ) wait = ui . WebDriverWait ( driver , 200 ) driver . get ( url ) wait . until ( ( lambda driver : driver . find_elements_by_id ( 'gs_ab_md' ) ) ) html_doc = driver . page_source soup = BeautifulSoup ( html_doc ) driver . close ( ) return soup
get beautifulsoup object from url .