update

run_job.py

@@ -27,11 +27,13 @@ COPY ({query}) to 'tmp' (FORMAT PARQUET, ROW_GROUP_SIZE_BYTES '100MB', ROW_GROUP
 CMD_SRC_DRY_RUN = CMD_SRC[:-1] + " LIMIT 5;"
 CMD_DST_DRY_RUN = "{query};"
 
+DATA_CARD = "# Dataset Card for {dst}\n\nDataset prepared from {src} using\n\n```\n{query}\n```\n"
+
 def sql(src: str, dst: str, query: str, config: str = "default", split: str = "train", private: bool = False, dry_run: bool = False):
     import os
     import duckdb
     from contextlib import nullcontext
-    from huggingface_hub import CommitScheduler
+    from huggingface_hub import CommitScheduler, DatasetCard
 
     class CommitAndCleanScheduler(CommitScheduler):
 
@@ -58,10 +60,11 @@ def sql(src: str, dst: str, query: str, config: str = "default", split: str = "t
             con.sql("PRAGMA enable_progress_bar;")
 
         result = con.sql((CMD_DST_DRY_RUN if dry_run else CMD_DST).format(query=query.rstrip("\n ;")))
-        if dry_run:
-            print(result.df().to_markdown())
-        else:
-            print("done")
+        DatasetCard(DATA_CARD.format(src=src, dst=dst, query=query)).push_to_hub(repo_id=dst, repo_type="dataset")
+    if dry_run:
+        print(result.df().to_markdown())
+    else:
+        print("done")
     
 
 if __name__ == '__main__':

start_app.py

@@ -79,12 +79,18 @@ def run(src, config, split, dst, query, oauth_token: gr.OAuthToken | None, profi
         job_id = resp.json()["metadata"]["job_id"]
         resp = requests.get(
             f"https://huggingface.co/api/jobs/{username}/{job_id}/logs-stream",
-            headers={"Authorization": f"Bearer {token}"}
         )
         for line in iter(resp.raw.readline, b""):
             logs += parse_log(line.decode(), pbars=pbars)
             yield {output_markdown: logs, progress_labels: gr.Label(pbars, visible=bool(pbars))}
-        pbars = {"Finished" + (" ✅" if process.returncode == 0 else " with an error ❌"): 1.0}
+        job_status = requests.get(
+            f"https://huggingface.co/api/jobs/{username}/{job_id}",
+        ).json()
+        if job_status["status"]["stage"] == "COMPLETED":
+            pbars = {"Finished ✅": 1.0}
+        else:
+            logs += f'{job_status["status"]["message"]} ({job_status["status"]["error"]})'
+            pbars = {"Finished with an error ❌": 1.0}
     yield {output_markdown: logs, progress_labels: gr.Label(pbars, visible=bool(pbars))}
 
 READ_FUNCTIONS = ("pl.read_parquet", "pl.read_csv", "pl.read_json")