title: Recursion after 499-500 times of code running... why? All help appreciated. Novel web-scrape
body: <p>Using chrome 90 and python 3.9. All imports are fully updated as I just installed them.</p>
  <p>As I have a bad ISP I made this script to copy novels from the internet to text files for offline viewing while my internet is out. This script pretty much works until the recursion error pops up then I have to manually go in and change the chapter after setting it. My expected results from the code is to run until the novel is completely copied (from chapter 1 to ######) to text files no matter how many chapters there is.</p>
  <p>Always get a recursion error after I hit 499 or 500 chapters copied. I am not sure why it is this low nor how it even gets this error. I have read that recursion errors are usually after 999 iterations.</p>
  <p>Error :: (first 2 lines repeat for quite a while)</p>
  <pre><code>  File &quot;C:\Users\james\Documents\Novels\PEERLESS MARTIAL GOD\novel.py&quot;, line 42, in CopyChapter
      NextChapter()
    File &quot;C:\Users\james\Documents\Novels\PEERLESS MARTIAL GOD\novel.py&quot;, line 49, in NextChapter
      link = driver.find_element_by_link_text(cLink)
    File &quot;C:\Program Files\Python39\lib\site-packages\selenium\webdriver\remote\webdriver.py&quot;, line 428, in find_element_by_link_text
      return self.find_element(by=By.LINK_TEXT, value=link_text)
    File &quot;C:\Program Files\Python39\lib\site-packages\selenium\webdriver\remote\webdriver.py&quot;, line 976, in find_element
      return self.execute(Command.FIND_ELEMENT, {
    File &quot;C:\Program Files\Python39\lib\site-packages\selenium\webdriver\remote\webdriver.py&quot;, line 319, in execute
      response = self.command_executor.execute(driver_command, params)
    File &quot;C:\Program Files\Python39\lib\site-packages\selenium\webdriver\remote\remote_connection.py&quot;, line 374, in execute
      return self._request(command_info[0], url, body=data)
    File &quot;C:\Program Files\Python39\lib\site-packages\selenium\webdriver\remote\remote_connection.py&quot;, line 397, in _request
      resp = self._conn.request(method, url, body=body, headers=headers)
    File &quot;C:\Program Files\Python39\lib\site-packages\urllib3\request.py&quot;, line 78, in request
      return self.request_encode_body(
    File &quot;C:\Program Files\Python39\lib\site-packages\urllib3\request.py&quot;, line 170, in request_encode_body
      return self.urlopen(method, url, **extra_kw)
    File &quot;C:\Program Files\Python39\lib\site-packages\urllib3\poolmanager.py&quot;, line 375, in urlopen
      response = conn.urlopen(method, u.request_uri, **kw)
    File &quot;C:\Program Files\Python39\lib\site-packages\urllib3\connectionpool.py&quot;, line 699, in urlopen
      httplib_response = self._make_request(
    File &quot;C:\Program Files\Python39\lib\site-packages\urllib3\connectionpool.py&quot;, line 445, in _make_request
      six.raise_from(e, None)
    File &quot;&lt;string&gt;&quot;, line 3, in raise_from
    File &quot;C:\Program Files\Python39\lib\site-packages\urllib3\connectionpool.py&quot;, line 440, in _make_request
      httplib_response = conn.getresponse()
    File &quot;C:\Program Files\Python39\lib\http\client.py&quot;, line 1347, in getresponse
      response.begin()
    File &quot;C:\Program Files\Python39\lib\http\client.py&quot;, line 331, in begin
      self.headers = self.msg = parse_headers(self.fp)
    File &quot;C:\Program Files\Python39\lib\http\client.py&quot;, line 225, in parse_headers
      return email.parser.Parser(_class=_class).parsestr(hstring)
    File &quot;C:\Program Files\Python39\lib\email\parser.py&quot;, line 67, in parsestr
      return self.parse(StringIO(text), headersonly=headersonly)
    File &quot;C:\Program Files\Python39\lib\email\parser.py&quot;, line 56, in parse
      feedparser.feed(data)
    File &quot;C:\Program Files\Python39\lib\email\feedparser.py&quot;, line 176, in feed
      self._call_parse()
    File &quot;C:\Program Files\Python39\lib\email\feedparser.py&quot;, line 180, in _call_parse
      self._parse()
    File &quot;C:\Program Files\Python39\lib\email\feedparser.py&quot;, line 295, in _parsegen
      if self._cur.get_content_maintype() == 'message':
    File &quot;C:\Program Files\Python39\lib\email\message.py&quot;, line 594, in get_content_maintype
      ctype = self.get_content_type()
    File &quot;C:\Program Files\Python39\lib\email\message.py&quot;, line 578, in get_content_type
      value = self.get('content-type', missing)
    File &quot;C:\Program Files\Python39\lib\email\message.py&quot;, line 471, in get
      return self.policy.header_fetch_parse(k, v)
    File &quot;C:\Program Files\Python39\lib\email\_policybase.py&quot;, line 316, in header_fetch_parse
      return self._sanitize_header(name, value)
    File &quot;C:\Program Files\Python39\lib\email\_policybase.py&quot;, line 287, in _sanitize_header
      if _has_surrogates(value):
    File &quot;C:\Program Files\Python39\lib\email\utils.py&quot;, line 57, in _has_surrogates
      s.encode()
  RecursionError: maximum recursion depth exceeded while calling a Python object
  </code></pre>
  <p>Code ::</p>
  <pre><code>#! python3
  import requests
  import bs4 as BeautifulSoup
  from selenium import webdriver
  from selenium.webdriver.support.wait import WebDriverWait
  from selenium.webdriver.chrome.options import Options
  from unidecode import unidecode
  
  CHROMEDRIVER_PATH = 'C:\Program Files\Python39\chromedriver.exe'
  
  NovelChapter = 'peerless-martial-god/chapter-1-spirit-awakening.html'
  BaseURL = 'https://novelfull.com'
  url = '%(U)s/%(N)s' % {'U': BaseURL, &quot;N&quot;: NovelChapter}
  
  options = Options()
  options.add_argument(&quot;--headless&quot;) # Runs Chrome in headless mode.
  driver = webdriver.Chrome(CHROMEDRIVER_PATH, options=options)
  driver.get(url)
  
  def Close():
      driver.stop_client()
      driver.close()
      driver.quit()
  
  # start copy of chapter and add to a file
  def CopyChapter():
      global soup
      soup = BeautifulSoup.BeautifulSoup(driver.page_source, 'html.parser')
      readables = soup.find(id='chapter-content')
      name = driver.title
      filename = name.replace('&lt;',' ').replace('&quot;',' ').replace('&gt;',' ').replace('/',' ').replace(&quot;|&quot;,' ').replace(&quot;?&quot;,' ').replace(&quot;*&quot;,' ').replace(&quot;:&quot;, ' -').replace('Read ',&quot;&quot;).replace(' online free from your Mobile, Table, PC... Novel Updates Daily ',&quot;&quot;).replace(' online free - Novel Full',&quot;&quot;)
      file_name = (filename + '.txt')
      print(file_name)
      data = ''
      for data in soup.find_all(&quot;p&quot;):
          myfile = open(file_name, 'a+')
          myfile.write(unidecode(data.get_text())+'\n'+'\n')
          myfile.close()
      global lastURL
      lastURL = driver.current_url
      print('**********Chapter Copied!**********')
      NextChapter()
  # end copy of chapter and add to a file
  
  # start goto next chapter if exists then return to copy chapter else Close()
  def NextChapter():
      bLink = soup.find(id = &quot;next_chap&quot;)
      cLink = 'Next Chapter'
      link = driver.find_element_by_link_text(cLink)
      link.click()
      global currentURL
      currentURL = driver.current_url
      if currentURL != lastURL:
          CopyChapter()
      else:
          print('Finished!!!')
          Close()
  # end goto next chapter if exists then return to copy chapter else Close()
  
  CopyChapter()
  #EOF
  </code></pre>