..

scrapping/dosen_scrap.py

@@ -11,12 +11,12 @@ class DosenSpider(scrapy.Spider):
     start_urls = ['https://sipeg.pnp.ac.id/']
     
     custom_settings = {
-        'DOWNLOAD_DELAY': 2,
+        'DOWNLOAD_DELAY': 1,
         'USER_AGENT': 'PNPBot/1.0',
         'ROBOTSTXT_OBEY': True,
         'LOG_LEVEL': 'INFO',
         'CONCURRENT_REQUESTS': 1,
-        'DOWNLOAD_TIMEOUT': 100,
+        'HTTPCACHE_ENABLED': False,
         'RETRY_TIMES': 3
     }
     

scrapping/jadwal_scrap.py

@@ -348,12 +348,12 @@ class PnpSpider(scrapy.Spider):
 
 if __name__ == "__main__":
     process = CrawlerProcess(settings={
-        'DOWNLOAD_DELAY': 2,
+        'DOWNLOAD_DELAY': 1,
         'USER_AGENT': 'PNPBot/1.0',
         'ROBOTSTXT_OBEY': True,
         'LOG_LEVEL': 'INFO',
+        'HTTPCACHE_ENABLED': False,
         'CONCURRENT_REQUESTS': 1,
-        'DOWNLOAD_TIMEOUT': 100,
         'RETRY_TIMES': 3
     })
     process.crawl(PnpSpider)

scrapping/jurusan_scrap.py

@@ -23,12 +23,12 @@ def is_valid_prodi(nama):
 class JurusanSpider(scrapy.Spider):
     name = "jurusan"
     custom_settings = {
-        'DOWNLOAD_DELAY': 0.5,
+        'DOWNLOAD_DELAY': 1,
         'USER_AGENT': 'PNPBot/1.0',
         'ROBOTSTXT_OBEY': True,
         'LOG_LEVEL': 'INFO',
-        'CONCURRENT_REQUESTS': 4,
-        'DOWNLOAD_TIMEOUT': 100,
+        'HTTPCACHE_ENABLED': False,
+        'CONCURRENT_REQUESTS': 1,
         'RETRY_TIMES': 3
     }
 

scrapping/pnp_scrap.py

@@ -27,9 +27,12 @@ class PNPContentSpider(scrapy.Spider):
     ]
 
     custom_settings = {
-        'DOWNLOAD_DELAY': 2,
+        'DOWNLOAD_DELAY': 1,
         'RETRY_TIMES': 3,
         'HTTPCACHE_ENABLED': False,
+        'ROBOTSTXT_OBEY': True,
+        'CONCURRENT_REQUESTS': 1,
+        'RETRY_ENABLED': True,  
         'USER_AGENT': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
     }