original_docstring: 
      Function to add documents to the index
      :param writer: the index writer
      :param path: path of the file to add to index
      
docstring: Function to add documents to the index
docstring_tokens: Function to add documents to the index
code: def add_doc(writer, path):
      import codecs
      """
      Function to add documents to the index
      :param writer: the index writer
      :param path: path of the file to add to index
      """
      #myjob = []
      myjob = open("H:/myproject/apache2/htdocs/stop-words_english_1_en.txt").read().split('\n')
      stopwords = set()
      for line in myjob:
         stopwords.add(line.strip())
      stopwords = dict.fromkeys(stopwords)    
      #myjob.close()
      print '<p>Opening ', path + '\n'
      try:
          content = ''
          
          f_link = open(path)
           
          soup = BeautifulSoup(f_link,"html.parser")
  
          titles = soup.findAll('title')
          for title in titles:
              title = title.get_text()
             
         
          #path = soup.findAll('a')
          for link in soup.findAll('a'):
              path = link.get('href')
            
          
              
          #for docs in f_link:  
          content = soup.get_text()      
          content = content.lower()
          content = (' '.join(w for w in content.split() if w not in stopwords)) 
            
          f_link.close()
      except IOError:
          print 'Unable to read %s file' % path
      else:
          try:
  
              writer.add_document(path=unicode(path), content=unicode(content), title=unicode(title))
              
          except Exception, e:
              print e
code_tokens: def add_doc ( writer , path ) : import codecs myjob = open ( "H:/myproject/apache2/htdocs/stop-words_english_1_en.txt" ) . read ( ) . split ( '\n' ) stopwords = set ( ) for line in myjob : stopwords . add ( line . strip ( ) ) stopwords = dict . fromkeys ( stopwords ) print '<p>Opening ' , path + '\n' try : content = '' f_link = open ( path ) soup = BeautifulSoup ( f_link , "html.parser" ) titles = soup . findAll ( 'title' ) for title in titles : title = title . get_text ( ) for link in soup . findAll ( 'a' ) : path = link . get ( 'href' ) content = soup . get_text ( ) content = content . lower ( ) content = ( ' ' . join ( w for w in content . split ( ) if w not in stopwords ) ) f_link . close ( ) except IOError : print 'Unable to read %s file' % path else : try : writer . add_document ( path = unicode ( path ) , content = unicode ( content ) , title = unicode ( title ) ) except Exception , e : print e
short_docstring: Function to add documents to the index
short_docstring_tokens: Function to add documents to the index
comment: """
      Function to add documents to the index
      :param writer: the index writer
      :param path: path of the file to add to index
      """$SEP$#myjob = []$SEP$#myjob.close()$SEP$#path = soup.findAll('a')$SEP$#for docs in f_link:  