original_docstring: Crawler thread to crawl links from the queue and add to the image queue.
docstring: Crawler thread to crawl links from the queue and add to the image queue.
docstring_tokens: Crawler thread to crawl links from the queue and add to the image queue .
code: def crawler_thread(self):
          """Crawler thread to crawl links from the queue and add to the image queue."""
  
          while True:
  
              try:
  
                  # Get the next item in the queue (index 1 because 0 is the priority #):
                  priority, link = self.link_queue.get()
                  if link in self.crawler.local_explored:
                      continue
                  print('Crawler: Getting', link, priority)
  
                  # Crawl the page:
                  results = self.crawler.get_filtered_links(link)
  
                  # Handle connection errors:
                  if self.crawler.con_err:
                      print('Crawler: Connection error', link)
                      self.crawler.con_err = False
                      self.link_queue.task_done()
  
                      # Demote for conn err
                      try:
                          self.domain_priority[urlparse(link).netloc] += 100
                      except KeyError:
                          self.domain_priority[urlparse(link).netloc] = 48
                      continue
  
                  # Page wasn't HTML:
                  try:
                      if self.crawler.page.response.headers['Content-Type'][0:9] != 'text/html':
                          print('Crawler: Link not HTML', link, self.crawler.page.response.headers['Content-Type'])
                          continue
                  except (KeyError, IndexError):
                      continue
  
                  # Handle bad status codes
                  sc = self.crawler.page.response.status_code
                  if sc != 200:
  
                      # Demote for bad status
                      try:
                          self.domain_priority[urlparse(link).netloc] += 8
                      except KeyError:
                          self.domain_priority[urlparse(link).netloc] = 6
  
                      if sc == 429:
                          print('Crawler: Status code 429, adding back to queue', link)
                          self.link_queue.put((2, link))
                      else:
                          print('Crawler: Bad status code', sc, link)
                      self.link_queue.task_done()
                      continue
  
                  # Demote domain
                  try:
                      self.domain_priority[urlparse(link).netloc] += 4
                  except KeyError:
                      # emphasize new domains
                      self.domain_priority[urlparse(link).netloc] = 0
  
                  # Put the new links on the queue (prioritize urls with 'cat' in them):
                  for new_link in results[0]:
                      if new_link not in self.crawler.local_explored:
                          self.link_queue.put((self.calculate_priority(new_link), new_link))
  
                  # Put any new images on the image queue:
                  for image in results[1]:
  
                      # Promote domain
                      self.domain_priority[urlparse(link).netloc] -= 6
  
                      print('Crawler: Found an image', image)
                      self.image_queue.put((image, link))
  
                  self.link_queue.task_done()
  
              except Exception as e:
                  print('Crawler: EXCEPTION:', e)
                  continue
code_tokens: def crawler_thread ( self ) : while True : try : priority , link = self . link_queue . get ( ) if link in self . crawler . local_explored : continue print ( 'Crawler: Getting' , link , priority ) results = self . crawler . get_filtered_links ( link ) if self . crawler . con_err : print ( 'Crawler: Connection error' , link ) self . crawler . con_err = False self . link_queue . task_done ( ) try : self . domain_priority [ urlparse ( link ) . netloc ] += 100 except KeyError : self . domain_priority [ urlparse ( link ) . netloc ] = 48 continue try : if self . crawler . page . response . headers [ 'Content-Type' ] [ 0 : 9 ] != 'text/html' : print ( 'Crawler: Link not HTML' , link , self . crawler . page . response . headers [ 'Content-Type' ] ) continue except ( KeyError , IndexError ) : continue sc = self . crawler . page . response . status_code if sc != 200 : try : self . domain_priority [ urlparse ( link ) . netloc ] += 8 except KeyError : self . domain_priority [ urlparse ( link ) . netloc ] = 6 if sc == 429 : print ( 'Crawler: Status code 429, adding back to queue' , link ) self . link_queue . put ( ( 2 , link ) ) else : print ( 'Crawler: Bad status code' , sc , link ) self . link_queue . task_done ( ) continue try : self . domain_priority [ urlparse ( link ) . netloc ] += 4 except KeyError : self . domain_priority [ urlparse ( link ) . netloc ] = 0 for new_link in results [ 0 ] : if new_link not in self . crawler . local_explored : self . link_queue . put ( ( self . calculate_priority ( new_link ) , new_link ) ) for image in results [ 1 ] : self . domain_priority [ urlparse ( link ) . netloc ] -= 6 print ( 'Crawler: Found an image' , image ) self . image_queue . put ( ( image , link ) ) self . link_queue . task_done ( ) except Exception as e : print ( 'Crawler: EXCEPTION:' , e ) continue
short_docstring: Crawler thread to crawl links from the queue and add to the image queue.
short_docstring_tokens: Crawler thread to crawl links from the queue and add to the image queue .
comment: """Crawler thread to crawl links from the queue and add to the image queue."""$SEP$# Get the next item in the queue (index 1 because 0 is the priority #):$SEP$# Crawl the page:$SEP$# Handle connection errors:$SEP$# Demote for conn err$SEP$# Page wasn't HTML:$SEP$# Handle bad status codes$SEP$# Demote for bad status$SEP$# Demote domain$SEP$# emphasize new domains$SEP$# Put the new links on the queue (prioritize urls with 'cat' in them):$SEP$# Put any new images on the image queue:$SEP$# Promote domain