title: Creating a dataframe from a loop from a scraper
body: <p>My dataframe "df" either returns an empty dataframe or just a one or two  column dataframe and not all columns related to the amount of days for which i'm running it for.</p>
  
  <pre><code>from bs4 import BeautifulSoup
  import requests
  from datetime import datetime, timedelta 
  import pandas as pd
  import numpy as np
  import matplotlib as plt
  import pandas as pd
  import numpy as np
  import matplotlib as plt
  from matplotlib import pyplot as plt
  from matplotlib.pyplot import figure
  from matplotlib.patches import Ellipse
  import numpy as np
  from matplotlib.text import OffsetFrom
  import re
  
  df=pd.DataFrame()
  today=datetime.today().strftime('%Y-%b-%d')
  end_date=datetime.strptime(today,'%Y-%b-%d')
  run_date=datetime.strptime('2019-Jul-1', '%Y-%b-%d')
  ref_date=datetime.strptime('2000-May-25', '%Y-%b-%d')
  
  while (run_date-ref_date)&lt;(end_date-ref_date):
    run_date=run_date+timedelta(1)
    url = f'https://www.nasdaq.com/earnings/earnings-calendar.aspx?date={run_date}'
    r =requests.get(url)
    soup = BeautifulSoup(r.text, 'html.parser')
    table = soup.find("table", { "class" : "USMN_EarningsCalendar" })
    if table is None:
      continue
    else:
      rows = table.find_all("tr")
      for i in rows:   
        datas = i.find_all('td')   
        if len(datas)&gt;0:        
          for j in datas:
            a.append(j.text.strip())
            for k in a:
              b.append(re.findall('\(.*?\)',k))
              b = [x for x in b if x != []]
              b_set = set(map(tuple,b))  
              b=list(b_set)
              b.strip('(')
              df[run_date]=b
  </code></pre>
  
  <p>Is this due to the line run_date=run_date+timedelta(1) within the loop? </p>