add dataset card

app.py

@@ -13,7 +13,7 @@ import requests
 from datasets import Dataset, Features, Value, Sequence
 from datasets.fingerprint import Hasher
 from gradio_huggingfacehub_search import HuggingfaceHubSearch
-from huggingface_hub import InferenceClient
+from huggingface_hub import DatasetCard, InferenceClient
 
 from utils import StringIteratorIO
 
@@ -26,6 +26,7 @@ session = requests.Session()
 empty_dataframe = pd.DataFrame({"1": [], "2": [], "3": []})
 
 NAMESPACE = "dataset-rewriter"
+URL = "https://huggingface.co/spaces/dataset-rewrite/dataset-rewrite"
 
 NUM_ROWS_PREVIEW = 3
 MAX_NUM_ROWS_TO_REWRITE = 1000
@@ -55,6 +56,26 @@ FIND_NEW_NAME = (
     "Here is your first job: rephrase the sentence 'Take this dataset and apply the instruction \"{prompt}\"'"
 )
 
+
+DATASET_CARD_CONTENT = """
+---
+license: mit
+tags:
+- dataset-rewriter
+- synthetic
+---
+
+# {new_dataset}
+
+_Note: This is an AI-generated dataset so its content may be inaccurate or false_
+
+**Source of the data:**
+The dataset was generated using the [Dataset ReWriter]({url}) and {model_id} from the dataset {dataset} and using the prompt '{prompt}':
+- **Original Dataset**: https://huggingface.co/datasets/{dataset}
+- **Model**: https://huggingface.co/{model_id}
+- **More Datasets**: https://huggingface.co/datasets?other=dataset-rewriter
+"""
+
 css = """
 .settings {
     background: transparent;
@@ -163,9 +184,7 @@ with gr.Blocks(css=css) as demo:
             prompt=prompt,
         )}]
         response_format = {"type": "json", "value": {"properties": {"data": {"type": "array", "items": format}}, "required": ["data"]}}
-        print(f"Streaming preview of {dataset} with instruction '{prompt}'")
         yield from ijson.items(StringIteratorIO(stream_reponse(messages, response_format=response_format)), "data.item", buf_size=4)
-        print(f"Done streaming preview of {dataset} with instruction '{prompt}'")
 
 
     def stream_rewrite_dataset_row_by_row(dataset: str, rows: list[dict[str, str]], prompt: str, format: str, input_preview_rows: list[dict[str, str]], output_preview_rows: list[dict[str, str]]) -> Iterator[dict[str, str]]:
@@ -305,7 +324,7 @@ with gr.Blocks(css=css) as demo:
         format = output_format_df.to_dict(orient="records")
         format = {"properties": {x["column"]: json.loads(x["type"]) for x in format}, "required": [x["column"] for x in format]}
         output_rows = []
-        print(f"ReWriting {dataset} preview with instruction '{prompt}'")
+        print(f"(preview) ReWriting {dataset} with instruction '{prompt}'")
         yield {rewrite_full_dataset_button: gr.Button(interactive=False), full_dataset_generation_label: gr.Label(visible=False)}
         yield {
             pretty_output_preview: gr.DataFrame(visible=True),
@@ -315,6 +334,7 @@ with gr.Blocks(css=css) as demo:
             output_rows.append({k: json.dumps(row[k], ensure_ascii=False) for k in output_format_df["column"]})
             yield {pretty_output_preview: gr.DataFrame(pd.DataFrame(output_rows))}
         yield {rewrite_full_dataset_button: gr.Button(interactive=True)}
+        print(f"(preview) Done ReWriting {dataset} with instruction '{prompt}'")
 
 
     @rewrite_full_dataset_button.click(inputs=[dataset_search, subset_dropdown, split_dropdown, pretty_input_preview, pretty_output_preview, input_prompt, output_format_dataframe, dataset_info_json, select_namespace_dropdown], outputs=[full_dataset_generation_label, full_dataset_generation_success_markdown, pretty_output_preview, pretty_full_dataset_generation_output])
@@ -325,7 +345,7 @@ with gr.Blocks(css=css) as demo:
         format = {"properties": {x["column"]: json.loads(x["type"]) for x in format}, "required": [x["column"] for x in format]}
         num_examples = dataset_info["splits"][split]["num_examples"]
         total = min(num_examples, MAX_NUM_ROWS_TO_REWRITE)
-        print(f"ReWriting {dataset} (full dataset) with instruction '{prompt}'")
+        print(f"ReWriting {dataset} with instruction '{prompt}'")
         yield {full_dataset_generation_label: gr.Label({f"⚙️ ReWriting {dataset}": 0.}, visible=True)}
         yield {pretty_full_dataset_generation_output: empty_dataframe}
         yield {
@@ -357,15 +377,17 @@ with gr.Blocks(css=css) as demo:
             full_dataset_generation_label: gr.Label({f"⚙️ ReWriting {dataset}": current / total}),
             pretty_full_dataset_generation_output: gr.DataFrame(pd.DataFrame([row for rows in parallel_output_rows for row in rows]))
         }
-        print(f"Done ReWriting {dataset} (full dataset) with instruction '{prompt}'")
+        print(f"Done ReWriting {dataset} with instruction '{prompt}'")
 
         output_rows = [{k: json.loads(row[k]) for k in output_format_df["column"]} for rows in parallel_output_rows for row in rows]
-        repo_id = namespace + "/" + find_new_name(dataset + (PARTIAL_SUFFIX if num_examples > total else ""), prompt)
+        new_dataset = find_new_name(dataset + (PARTIAL_SUFFIX if num_examples > total else ""), prompt)
+        repo_id = namespace + "/" + new_dataset
         yield {full_dataset_generation_label: gr.Label({f"✅ ReWriting {dataset}": len(output_rows) / total, f"⚙️ Saving to {repo_id}": 0.})}
         token = oauth_token.token if oauth_token else save_dataset_hf_token
         print(f"Saving {repo_id}")
         ds = Dataset.from_list(output_rows)
         ds.push_to_hub(repo_id, config_name=subset, split=split, token=token)
+        DatasetCard(DATASET_CARD_CONTENT.format(new_dataset=new_dataset, dataset=dataset, model_id=model_id, prompt=prompt, url=URL)).push_to_hub(repo_id=repo_id, repo_type="dataset", token=token)
         yield {full_dataset_generation_label: gr.Label({f"✅ ReWriting {dataset}": len(output_rows) / total, f"✅ Saving to {repo_id}": 1.})}
         yield {full_dataset_generation_success_markdown: f"# Open the ReWriten dataset in a new tab: [{repo_id}](https://huggingface.co/datasets/{repo_id})"}
         print(f"Saved {repo_id}")