title: how to make this code run many times based on "page_num" varibale, to scrape all pages? using BeautifulSoup
body: <p>I'm trying to scrape the website<code>https://www.bayut.sa/en/riyadh-region/villas-for-sale-in-riyadh/page-2/</code>, the code succeeded to scrape the first page only which is <code>page-2</code> here, but it does not work to scrape the rest of the pages to the last page in the website.
  I want to scrape all pages like the first page by increasing the <code>page_num</code> variable</p>
  <pre><code>import requests
  from bs4 import BeautifulSoup
  import csv
  from itertools import zip_longest
  import lxml
  
  
  page_num = 2
  
  
  while page_num &lt; 288:
      # 1 lists
      district_Name = []
      property_size = []
      property_price = []
      links = []
      dates = []
      # beds = []
      # paths = []
  
      # 2 the link to the website
      # here I included the `page_num` in the link
      try:
          result = requests.get(f&quot;https://www.bayut.sa/en/riyadh-region/villas-for-sale/page-{page_num}/&quot;)
          src = result.content
  
          # 4 create soup
          soup = BeautifulSoup(src, &quot;lxml&quot;)
  
          # 5 titles we need: districtName, property Age, size, rooms, price
          districtName = soup.findAll(&quot;div&quot;, {&quot;aria-label&quot;: &quot;Location&quot;})
          size = soup.findAll(&quot;span&quot;, {&quot;aria-label&quot;: &quot;Area&quot;})
          price = soup.findAll(&quot;span&quot;,{&quot;aria-label&quot;: &quot;Price&quot;})
          listing_link = soup.findAll(&quot;a&quot;, {&quot;aria-label&quot;: &quot;Listing link&quot;})
          bed = soup.findAll(&quot;span&quot;, {&quot;aria-label&quot;: &quot;Beds&quot;}, {&quot;class&quot;: &quot;b6a29bc0&quot;})
          path = soup.findAll(&quot;span&quot;, {&quot;aria-label&quot;: &quot;Beds&quot;}, {&quot;class&quot;: &quot;b6a29bc0&quot;})
  
  
          main_url= 'https://www.bayut.sa'
  
          # 6 for loop to get text and append it to a list
          for i in range(len(districtName)):
              district_Name.append(districtName[i].text)
              links.append(main_url+listing_link[i].attrs[&quot;href&quot;])
              property_size.append(size[i].text)
              property_price.append(price[i].text)
              # beds.append(bed[i].text)
              # paths.append(path[i].text)
  
          # 7 extract post date from inner page
          for link in (links):
              result = requests.get(link)
              src = result.content
              soup = BeautifulSoup(src, &quot;lxml&quot;)
              date = soup.find(&quot;span&quot;, {&quot;aria-label&quot;:&quot;Reactivated date&quot;})
              dates.append(date.text)
  
          page_num += 1
      except Exception as e:
          print(e)
  
  
      file_list = [district_Name, property_size, property_price, dates, links]
      exported = zip_longest(*file_list)
      # 8 create a csv file and fill it with values
      with open(&quot;C:/Users/Manso/Desktop/files\Data Analysis\Riyadh_Homes_english_beta.csv&quot;, &quot;w&quot;) as homes_file:
          wr = csv.writer(homes_file, lineterminator=&quot;\n&quot;)
          wr.writerow(['district_Name', 'property_size', 'property_price', 'dates', 'links'])
          wr.writerows(exported)
  </code></pre>