title: Give these error twisted.python.failure.Failure twisted.internet
body: <p>It will give the error of <code>twisted.python.failure.Failure twisted.internet</code> I will use <code>download_delay</code> in setting   but some time they work and  give output but sometime they do not work I want a permanent solution how I avoid these error <code>twisted.python.failure.Failure twisted.internet</code> is there any solution for it kindly share it I am searching but not find any appropiate solution  these is page link <a href="https://www.benrishi-navi.com/english/english1_2.php" rel="nofollow noreferrer">https://www.benrishi-navi.com/english/english1_2.php</a></p>
  <pre><code>import scrapy
  from scrapy import FormRequest
  from scrapy.crawler import CrawlerProcess
  from scrapy.http import Request
  
  
  class TestSpider(scrapy.Spider):
      name = 'test'
      url = 'https://www.benrishi-navi.com/english/english1_2.php'
      cookies = {
      'CAKEPHP': 'u6u40lefkqnm45j49a5i0h6bs3',
      '__utma': '42336182.871903078.1657200864.1657200864.1657200864.1',
      '__utmz': '42336182.1657200864.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)',
      }
  
      headers = {
          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
          'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8,pt;q=0.7',
          'Cache-Control': 'max-age=0',
          'Connection': 'keep-alive',
          # Requests sorts cookies= alphabetically
          # 'Cookie': 'CAKEPHP=u6u40lefkqnm45j49a5i0h6bs3; __utma=42336182.871903078.1657200864.1657200864.1657200864.1; __utmz=42336182.1657200864.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)',
          'Origin': 'https://www.benrishi-navi.com',
          'Referer': 'https://www.benrishi-navi.com/english/english1_2.php',
          'Sec-Fetch-Dest': 'document',
          'Sec-Fetch-Mode': 'navigate',
          'Sec-Fetch-Site': 'same-origin',
          'Sec-Fetch-User': '?1',
          'Upgrade-Insecure-Requests': '1',
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
          'sec-ch-ua': '&quot;.Not/A)Brand&quot;;v=&quot;99&quot;, &quot;Google Chrome&quot;;v=&quot;103&quot;, &quot;Chromium&quot;;v=&quot;103&quot;',
          'sec-ch-ua-mobile': '?0',
          'sec-ch-ua-platform': '&quot;Windows&quot;',
      }
  
      formdata = [
          ('tuusan_year', ''),
          ('tuusan_month', ''),
          ('tuusan_chk', ''),
          ('methodAndOr1', ''),
          ('methodAndOr2', ''),
          ('methodAndOr3', ''),
          ('text_sen', ''),
          ('text_skill', ''),
          ('text_business', ''),
          ('tokkyo_data', ''),
          ('fuki_day_chk', ''),
          ('shuju', ''),
          ('kensyuu_bunya', ''),
          ('text_kensyuu', ''),
          ('methodAndOr_kensyuu', ''),
          ('keitai_kikan', ''),
          ('keitai_hisu', ''),
          ('display_flag', '1'),
          ('search', '2'),
          ('text', ''),
          ('method', ''),
          ('methodAndOr', ''),
          ('area', ''),
          ('pref', ''),
          ('name', ''),
          ('kana', ''),
          ('id', ''),
          ('year', ''),
          ('month', ''),
          ('day', ''),
          ('day_chk', ''),
          ('exp01', ''),
          ('exp02', ''),
          ('exp03', ''),
          ('trip', ''),
          ('venture_support', ''),
          ('venture_flag', ''),
          ('university_support', ''),
          ('university_flag', ''),
          ('university1', ''),
          ('university2', ''),
          ('university', ''),
          ('college', ''),
          ('high_pref', ''),
          ('junior_pref', ''),
          ('elementary_pref', ''),
          ('tyosaku', ''),
          ('hp', ''),
          ('jukoureki', ''),
          ('experience1', ''),
          ('experience2', ''),
          ('experience3', ''),
          ('experience4', ''),
          ('sort', ''),
          ('fuki_year', ''),
          ('fuki_month', ''),
          ('fuki_day', ''),
          ('fuki_day_chk', ''),
          ('id_chk', ''),
          ('shugyou', ''),
          ('fuki', ''),
          ('address1', ''),
          ('address2', ''),
          ('trip_pref', ''),
          ('expref', ''),
          ('office', ''),
          ('max_count', '1438'),
          ('search_count', '1438'),
          ('start_count', '1'),
          ('search_default', '1000'),
      ]
     
      
      def start_requests(self):
          yield scrapy.FormRequest(
              url=self.url,
              method='POST',
              formdata=self.formdata,
              cookies=self.cookies,
              headers=self.headers,
              callback=self.parse_item,
              )
          
          
      def parse_item(self, response):
          base_url=&quot;https://www.benrishi-navi.com/english/&quot;
          links =response.xpath(&quot;//table[4]//tr&quot;)
          for link in links[1:]:
              t=link.xpath(&quot;//form//@action&quot;).get()
              u=link.xpath(&quot;.//input[@name='serial']//@value&quot;).get()
              product=base_url+t+&quot;?serial=&quot;+u+&quot;&amp;office_serial=&amp;submit2=Details&quot;
              yield Request(product,callback=self.parse_book)
                      
      def parse_book(self,response):
          name=response.xpath(&quot;normalize-space(//td[text()[contains(.,'Name')]]/following-sibling::td//text())&quot;).get()
                  
          telephone=response.xpath(&quot;normalize-space(//td[text()[contains(.,'TEL')]]/following-sibling::td//text())&quot;).get()
          
          fax=response.xpath(&quot;normalize-space(//td[text()[contains(.,'FAX')]]/following-sibling::td//text())&quot;).get()
          
          email=response.xpath(&quot;normalize-space(//td[text()[contains(.,'Email')]]/following-sibling::td//text())&quot;).get()
          
          website=response.xpath(&quot;//td[text()[contains(.,'Website')]]/following-sibling::td//a[starts-with(@href, 'http')]/@href&quot;).get()
          
          
          registration_date=response.xpath(&quot;normalize-space(//td[text()[contains(.,'Registration date')]]/following-sibling::td//text())&quot;).get()
          
          
          firm=response.xpath(&quot;normalize-space(//td[text()[contains(.,'Firm Name')]]/following-sibling::td//text())&quot;).get()
          
          
          address=response.xpath(&quot;normalize-space(//td[text()[contains(.,'Address (Prefecture)')]]/following-sibling::td//text())&quot;).get()
          
          
          
          spec=response.xpath(&quot;normalize-space(//td[text()[contains(.,'Specialization')]]/following-sibling::td//text())&quot;).get().replace(&quot; ï½œ&quot;,&quot;|&quot;)
          
          
          tech=response.xpath(&quot;normalize-space(//td[text()[contains(.,'Technical field')]]/following-sibling::td//text())&quot;).get().replace(&quot; ï½œ&quot;,&quot;|&quot;)
          
          
          yield{
          &quot;name&quot;:name,
          &quot;Telephone&quot;:telephone,
          &quot;Fax&quot;:fax,
          &quot;Email&quot;:email,
          &quot;website&quot;:website,
          &quot;Registration_date&quot;:registration_date,
          &quot;Firm_name&quot;:firm,
          &quot;Address&quot;:address,
          &quot;Specialization&quot;:spec,
          &quot;Technical_field&quot;:tech
          }
  </code></pre>