original_docstring: Classifier thread to read from the image queue and classify images.
docstring: Classifier thread to read from the image queue and classify images.
docstring_tokens: Classifier thread to read from the image queue and classify images .
code: def classifier_thread(self):
          """Classifier thread to read from the image queue and classify images."""
  
          # Main imaging loop:
          while True:
  
              try:
  
                  # Download the image from a url on the queue:
                  url, page_url = self.image_queue.get()
                  print('Classifier: Downloading', url, 'from', page_url)
                  try:
                      image = requests.get(
                          url,
                          stream=True,
                          headers={
                              'User-Agent': UA
                          },
                          timeout=3,
                          auth=('user', 'pass')
                      )
                      image.raw.decode_content = True
                      # 32 MB limit
                      try:
                          if int(image.headers['Content-Length']) > (1024 ** 2) * 32:
                              print('Classifier: Image too big', url)
                              continue
                      except (KeyError, IndexError):
                          pass
                  except (requests.exceptions.Timeout, requests.exceptions.ConnectionError):
                      print('Classifier: Could not connect', url)
                      self.image_queue.task_done()
                      continue
  
                  # Handle bad status codes
                  if image.status_code != 200:
                      print('Classifier: Bad status code', image.status_code, url)
                      self.image_queue.task_done()
                      continue
  
                  # Classify the image:
                  results = self.classify(image.raw)
                  if results[0]:
  
                      # Promote original domain
                      self.domain_priority[urlparse(page_url).netloc] -= 32
  
                      print('Classifier: Cat found', results[1], url)
                      try:
                          self.server.imag(url)
                      except netexcept.SocketClosed:
                          print('Classifier: Error: Socket closed')
                          self.error_event.set()
                          sys.exit(1)
                      self.cats.add((url, page_url))
                  else:
                      print('Classifier: Not a cat', results[1], url)
  
                      # Demote domain if image is very-not cat or very-cat
                      if not custom:
                          bad = 0
                          for i in results[1]:
                              if i[0] == 'website' or i[0] == 'menu' \
                              or i[0] == 'Band-Aid' or i[0] == 'sunscreen' \
                              or i[0] == 'packet' or i[0] == 'envelope' \
                              or i[0] == 'comic book':
                                      bad += 1
                              elif 'dog' in i[0] or 'Dog' in i[0] \
                              or 'bear' in i[0] or 'fox' in i[0] \
                              or 'Poodle' in i[0] or 'lynx' in i[0] \
                              or i[0] == 'leopard' or i[0] == 'snow leopard' \
                              or i[0] == 'jaguar' or i[0] == 'lion' \
                              or i[0] == 'tiger' or i[0] == 'cheetah':
                                  bad -= 1
                          if bad > 0:
                                  self.domain_priority[urlparse(page_url).netloc] += 10
                          elif bad < 0:
                                  self.domain_priority[urlparse(page_url).netloc] -= 10
  
                  self.image_queue.task_done()
  
              except Exception as e:
                  print('Classifier: EXCEPTION:', e, 'url:', url)
                  continue
code_tokens: def classifier_thread ( self ) : while True : try : url , page_url = self . image_queue . get ( ) print ( 'Classifier: Downloading' , url , 'from' , page_url ) try : image = requests . get ( url , stream = True , headers = { 'User-Agent' : UA } , timeout = 3 , auth = ( 'user' , 'pass' ) ) image . raw . decode_content = True try : if int ( image . headers [ 'Content-Length' ] ) > ( 1024 ** 2 ) * 32 : print ( 'Classifier: Image too big' , url ) continue except ( KeyError , IndexError ) : pass except ( requests . exceptions . Timeout , requests . exceptions . ConnectionError ) : print ( 'Classifier: Could not connect' , url ) self . image_queue . task_done ( ) continue if image . status_code != 200 : print ( 'Classifier: Bad status code' , image . status_code , url ) self . image_queue . task_done ( ) continue results = self . classify ( image . raw ) if results [ 0 ] : self . domain_priority [ urlparse ( page_url ) . netloc ] -= 32 print ( 'Classifier: Cat found' , results [ 1 ] , url ) try : self . server . imag ( url ) except netexcept . SocketClosed : print ( 'Classifier: Error: Socket closed' ) self . error_event . set ( ) sys . exit ( 1 ) self . cats . add ( ( url , page_url ) ) else : print ( 'Classifier: Not a cat' , results [ 1 ] , url ) if not custom : bad = 0 for i in results [ 1 ] : if i [ 0 ] == 'website' or i [ 0 ] == 'menu' or i [ 0 ] == 'Band-Aid' or i [ 0 ] == 'sunscreen' or i [ 0 ] == 'packet' or i [ 0 ] == 'envelope' or i [ 0 ] == 'comic book' : bad += 1 elif 'dog' in i [ 0 ] or 'Dog' in i [ 0 ] or 'bear' in i [ 0 ] or 'fox' in i [ 0 ] or 'Poodle' in i [ 0 ] or 'lynx' in i [ 0 ] or i [ 0 ] == 'leopard' or i [ 0 ] == 'snow leopard' or i [ 0 ] == 'jaguar' or i [ 0 ] == 'lion' or i [ 0 ] == 'tiger' or i [ 0 ] == 'cheetah' : bad -= 1 if bad > 0 : self . domain_priority [ urlparse ( page_url ) . netloc ] += 10 elif bad < 0 : self . domain_priority [ urlparse ( page_url ) . netloc ] -= 10 self . image_queue . task_done ( ) except Exception as e : print ( 'Classifier: EXCEPTION:' , e , 'url:' , url ) continue
short_docstring: Classifier thread to read from the image queue and classify images.
short_docstring_tokens: Classifier thread to read from the image queue and classify images .
comment: """Classifier thread to read from the image queue and classify images."""$SEP$# Main imaging loop:$SEP$# Download the image from a url on the queue:$SEP$# 32 MB limit$SEP$# Handle bad status codes$SEP$# Classify the image:$SEP$# Promote original domain$SEP$# Demote domain if image is very-not cat or very-cat