title: Problem with Crawling Mechanism of Twitter Python Crawler
body: <p>Below is a small snippet of code I have for my twitter crawler mechanism: </p>
  
  <pre><code>from BeautifulSoup import BeautifulSoup
  import re
  import urllib2
  
  url = 'http://mobile.twitter.com/NYTimesKrugman'
  
  def gettweets(soup):
      tags = soup.findAll('div', {'class' : "list-tweet"})#to obtain tweet of a follower
      for tag in tags: 
          print tag.renderContents()
          print ('\n\n')
  
  def are_more_tweets(soup):#to check whether there is more than one page on mobile   twitter 
      links = soup.findAll('a', {'href': True}, {id: 'more_link'})
      for link in links:
          b = link.renderContents()
          test_b = str(b)
          if test_b.find('more'):
              return True
          else:
              return False
  
  def getnewlink(soup): #to get the link to go to the next page of tweets on twitter 
      links = soup.findAll('a', {'href': True}, {id : 'more_link'})
      for link in links:
          b = link.renderContents()
          if str(b) == 'more':
              c = link['href']
              d = 'http://mobile.twitter.com' +c
              return d
  
  def checkforstamp(soup): # the parser scans a webpage to check if any of the tweets are older than 3 months
      times = soup.findAll('a', {'href': True}, {'class': 'status_link'})
      for time in times:
          stamp = time.renderContents()
          test_stamp = str(stamp)
          if test_stamp == '3 months ago':  
              print test_stamp
              return True
          else:
              return False
  
  
  response = urllib2.urlopen(url)
  html = response.read()
  soup = BeautifulSoup(html)
  gettweets(soup)
  stamp = checkforstamp(soup)
  tweets = are_more_tweets(soup)
  print 'stamp' + str(stamp)
  print 'tweets' +str (tweets)
  while (stamp is False) and (tweets is True): 
      b = getnewlink(soup)
      print b
      red = urllib2.urlopen(b)
      html = red.read()
      soup = BeautifulSoup(html)
      gettweets(soup)
      stamp = checkforstamp(soup)
      tweets = are_more_tweets(soup)
  print 'done' 
  </code></pre>
  
  <p>The problem is, after my twitter crawler hits about 3 months of tweets, I would like it to stop going to the next page of a user.  However, it does not appear to be doing that. It seems to continually going searching for the next page of tweets. I believe this is due to the fact that checkstamp keeps evaluating to False. Does anyone have any suggestions as to how I can modify the code so that the crawler keeps looking for the next page of tweets as long as there are more tweets (verified by are_more_tweets mechanism) and it hasn't hit 3 months of tweets yet??? Thanks! </p>
  
  <p><strong>EDIT</strong> - Please see below: </p>
  
  <pre><code>from BeautifulSoup import BeautifulSoup
  import re
  import urllib
  
  url = 'http://mobile.twitter.com/cleversallie'
  output = open(r'C:\Python28\testrecursion.txt', 'a') 
  
  def gettweets(soup):
      tags = soup.findAll('div', {'class' : "list-tweet"})#to obtain tweet of a follower
      for tag in tags: 
          a = tag.renderContents()
          b = str (a)
          print(b)
          print('\n\n')
  
  def are_more_tweets(soup):#to check whether there is more than one page on mobile twitter 
      links = soup.findAll('a', {'href': True}, {id: 'more_link'})
      for link in links:
          b = link.renderContents()
          test_b = str(b)
          if test_b.find('more'):
              return True
          else:
              return False
  
  def getnewlink(soup): #to get the link to go to the next page of tweets on twitter 
      links = soup.findAll('a', {'href': True}, {id : 'more_link'})
      for link in links:
          b = link.renderContents()
          if str(b) == 'more':
              c = link['href']
              d = 'http://mobile.twitter.com' +c
              return d
  
   def checkforstamp(soup): # the parser scans a webpage to check if any of the tweets are older than 3 months
      times = soup.findAll('a', {'href': True}, {'class': 'status_link'})
      for time in times:
          stamp = time.renderContents()
          test_stamp = str(stamp)
          if not (test_stamp[0]) in '0123456789':
              continue
          if test_stamp == '3 months ago':
              print test_stamp
              return True
          else:
              return False
  
  
  response = urllib.urlopen(url)
  html = response.read()
  soup = BeautifulSoup(html)
  gettweets(soup)
  stamp = checkforstamp(soup)
  tweets = are_more_tweets(soup)
  while (not stamp) and (tweets): 
      b = getnewlink(soup)
      print b
      red = urllib.urlopen(b)
      html = red.read()
      soup = BeautifulSoup(html)
      gettweets(soup)
      stamp = checkforstamp(soup)
      tweets = are_more_tweets(soup)
   print 'done' 
  </code></pre>