..

scrapping/jurusan_scrap.py

@@ -6,7 +6,7 @@ from supabase import create_client
 from datetime import datetime
 import os, re, tempfile
 
-# Load environment variables from .env
+# Load environment variables
 load_dotenv()
 SUPABASE_URL = os.getenv("NEXT_PUBLIC_SUPABASE_URL")
 SUPABASE_KEY = os.getenv("NEXT_PUBLIC_SUPABASE_SERVICE_KEY")
@@ -14,99 +14,105 @@ SUPABASE_BUCKET = os.getenv("NEXT_PUBLIC_SUPABASE_STORAGE_BUCKET", "pnp-bot-stor
 
 class JurusanSpider(scrapy.Spider):
     name = "jurusan"
-    start_urls = [
-        f"https://{url}/" for url in {
-            'akt.pnp.ac.id': 'Akuntansi',
-            'an.pnp.ac.id': 'Administrasi_Niaga',
-            'bing.pnp.ac.id': 'Bahasa_Inggris',
-            'elektro.pnp.ac.id': 'Teknik_Elektro',
-            'me.pnp.ac.id': 'Teknik_Mesin',
-            'sipil.pnp.ac.id': 'Teknik_Sipil',
-            'ti.pnp.ac.id': 'Teknologi_Informasi',
-        }.keys()
-    ]
-
-    custom_settings = {
-        "LOG_LEVEL": "INFO",
-        "USER_AGENT": "Mozilla/5.0",
+    custom_settings = {"LOG_LEVEL": "INFO", "USER_AGENT": "Mozilla/5.0"}
+
+    domain_to_name = {
+        'akt.pnp.ac.id': 'Akuntansi',
+        'an.pnp.ac.id': 'Administrasi_Niaga',
+        'bing.pnp.ac.id': 'Bahasa_Inggris',
+        'elektro.pnp.ac.id': 'Teknik_Elektro',
+        'me.pnp.ac.id': 'Teknik_Mesin',
+        'sipil.pnp.ac.id': 'Teknik_Sipil',
+        'ti.pnp.ac.id': 'Teknologi_Informasi',
     }
 
+    start_urls = [f"https://{d}/" for d in domain_to_name.keys()]
+
     def __init__(self):
         self.supabase = create_client(SUPABASE_URL, SUPABASE_KEY)
         self.bucket = SUPABASE_BUCKET
         self.timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        self.collected_data = []
-        self.domain_to_name = {
-            'akt.pnp.ac.id': 'Akuntansi',
-            'an.pnp.ac.id': 'Administrasi_Niaga',
-            'bing.pnp.ac.id': 'Bahasa_Inggris',
-            'elektro.pnp.ac.id': 'Teknik_Elektro',
-            'me.pnp.ac.id': 'Teknik_Mesin',
-            'sipil.pnp.ac.id': 'Teknik_Sipil',
-            'ti.pnp.ac.id': 'Teknologi_Informasi',
-        }
+        self.rekap_prodi = {}
+        self.per_jurusan_pages = {}
 
     def parse(self, response):
         domain = response.url.split("//")[1].split("/")[0]
         jurusan = self.domain_to_name.get(domain, domain)
-        url = response.url
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        # Cari link penting
+        all_links = set()
+        for a in soup.find_all("a", href=True):
+            href = a["href"]
+            if href.startswith("http") and domain in href:
+                all_links.add(href)
+            elif href.startswith("/"):
+                all_links.add(response.urljoin(href))
+
+        # Ambil link Program Studi
+        program_studi = []
+        menu_elements = soup.find_all("a", string=re.compile("program studi", re.I))
+        for menu in menu_elements:
+            ul = menu.find_next("ul")
+            if ul:
+                for li in ul.find_all("li"):
+                    item = li.get_text(strip=True)
+                    if item and item not in program_studi:
+                        program_studi.append(item)
+
+        self.rekap_prodi[jurusan] = program_studi
+
+        for link in all_links:
+            yield scrapy.Request(link, callback=self.parse_detail, meta={"jurusan": jurusan, "url": link})
+
+    def parse_detail(self, response):
+        jurusan = response.meta["jurusan"]
+        url = response.meta["url"]
 
         soup = BeautifulSoup(response.text, "html.parser")
         for tag in soup(["script", "style", "noscript"]):
             tag.decompose()
         visible_text = soup.get_text(separator="\n")
-
         lines = [line.strip() for line in visible_text.splitlines()]
         lines = [line for line in lines if line and not re.match(r'^\W+$', line)]
         text_cleaned = "\n".join(lines)[:8000]
 
-        program_studi = []
-        menu_elements = soup.find_all("a", string=re.compile("program studi", re.I))
-        for menu in menu_elements:
-            program_studi.append(menu.get_text(strip=True))
-
-        self.collected_data.append({
-            "jurusan": jurusan,
+        self.per_jurusan_pages.setdefault(jurusan, []).append({
             "url": url,
-            "prodi": program_studi,
-            "profil": text_cleaned,
+            "content": text_cleaned
         })
 
-        filename = f"{jurusan.replace(' ', '_').upper()}_{self.timestamp}.txt"
-        try:
-            with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False, suffix=".txt") as f:
-                f.write(f"[Jurusan: {jurusan.replace('_', ' ')}]\n\n")
-                f.write(f"URL: {url}\n\n")
-                f.write("Program Studi:\n")
-                for i, p in enumerate(program_studi, 1):
-                    f.write(f"{i}. {p}\n")
-                f.write(f"\nTotal Program Studi: {len(program_studi)}\n\n")
-                f.write("Profil Jurusan:\n")
-                f.write(text_cleaned + "\n\n")
-                temp_path = f.name
+    def closed(self, reason):
+        # Upload file per jurusan
+        for jurusan, pages in self.per_jurusan_pages.items():
+            filename = f"{jurusan.replace(' ', '_').upper()}_{self.timestamp}.txt"
+            try:
+                with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False, suffix=".txt") as f:
+                    for page in pages:
+                        f.write(f"=== [JURUSAN: {jurusan.replace('_', ' ').upper()}] ===\n")
+                        f.write(f"=== [HALAMAN: {page['url']}] ===\n\n")
+                        f.write(page["content"] + "\n\n")
+                    temp_path = f.name
 
-            self.supabase.storage.from_(self.bucket).upload(
-                path=filename,
-                file=temp_path,
-                file_options={"content-type": "text/plain"}
-            )
-            self.logger.info(f"✅ Uploaded file jurusan: {filename}")
-        except Exception as e:
-            self.logger.error(f"❌ Gagal upload {filename}: {e}")
-        finally:
-            if os.path.exists(temp_path):
-                os.remove(temp_path)
+                self.supabase.storage.from_(self.bucket).upload(
+                    path=filename,
+                    file=temp_path,
+                    file_options={"content-type": "text/plain"}
+                )
+                self.logger.info(f"✅ Uploaded file jurusan: {filename}")
+            except Exception as e:
+                self.logger.error(f"❌ Gagal upload {filename}: {e}")
+            finally:
+                if os.path.exists(temp_path):
+                    os.remove(temp_path)
 
-    def closed(self, reason):
-        """Dipanggil saat spider selesai — membuat dan upload file rekap"""
+        # Upload file rekap program studi
         filename = f"REKAP_PROGRAM_STUDI_{self.timestamp}.txt"
         try:
             with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", delete=False, suffix=".txt") as f:
                 f.write(f"# REKAP PROGRAM STUDI PNP\nDiperbarui pada: {datetime.now().strftime('%d %B %Y %H:%M')}\n\n")
                 total = 0
-                for item in self.collected_data:
-                    jurusan = item["jurusan"]
-                    daftar = item["prodi"]
+                for jurusan, daftar in self.rekap_prodi.items():
                     f.write(f"{jurusan.replace('_', ' ')}:\n")
                     for p in daftar:
                         f.write(f"- {p}\n")