rename + parse json

README.md

@@ -1,8 +1,8 @@
 ---
-title: Dataset Spreadsheets
-emoji: 🚀
+title: DuckDB Spreadsheets
+emoji: 🐥📝
 colorFrom: green
-colorTo: indigo
+colorTo: yellow
 sdk: gradio
 sdk_version: 5.6.0
 app_file: app.py

app.py

@@ -2,14 +2,17 @@ from functools import partial, lru_cache
 
 import duckdb
 import gradio as gr
+import json
 import pandas as pd
 import pyarrow as pa
+import pyarrow.compute as pc
 import requests
 from huggingface_hub import HfApi
 
 READ_PARQUET_FUNCTIONS = ("dd.read_parquet", "pd.read_parquet")
 EMPTY_TABLE = pa.Table.from_pylist([{str(i): "" for i in range(4)}] * 10)
 EMPTY_DF: pd.DataFrame = EMPTY_TABLE.to_pandas()
+NUM_ROWS = 10
 MAX_NUM_COLUMNS = 20
 NUM_TRENDING_DATASETS = 10
 NUM_USER_DATASETS = 10
@@ -102,6 +105,14 @@ def get_prepared_functions_from_table(table: pa.Table) -> dict[str, list[str]]:
             prepared_functions[field.name] = [prepare_function(numeric_func, ["x"], field.name) for numeric_func in numeric_functions_df.Name]
         elif pa.types.is_string(field.type):
             prepared_functions[field.name] = [prepare_function(text_func, ["string"], field.name) for text_func in text_functions_df.Name]
+            # try parsing json
+            if pc.all(pc.starts_with(table[field.name], "{")).as_py() or pc.all(pc.starts_with(table[field.name], "[")).as_py():
+                try:
+                    json_parsed_table = pa.Table.from_pylist([{field.name: json.loads(row)} for row in table[field.name].to_pylist()])
+                    parsed_type = str(duckdb.from_arrow(json_parsed_table).dtypes[0])
+                    prepared_functions[field.name] = [f"CAST({field.name} as {parsed_type})"] + prepared_functions[field.name]
+                except Exception:
+                    pass
         elif pa.types.is_date(field.type):
             prepared_functions[field.name] = [prepare_function(date_func, ["startdate", "date"], field.name) for date_func in date_functions_df.Name]
         elif pa.types.is_list(field.type):
@@ -122,7 +133,7 @@ with gr.Blocks(css=css, js=js) as demo:
     dataset_subset_split_textbox = gr.Textbox(visible=False)
     input_table_state = gr.State()
     run_button = gr.Button(visible=False, elem_id="run_button")
-    gr.Markdown("# Dataset Spreadsheets\n\nEdit any dataset on Hugging Face (full list [here](https://huggingface.co/datasets)) using DuckDB functions (documentation [here](https://duckdb.org/docs/sql/functions/overview))")
+    gr.Markdown("# DuckDB Spreadsheets\n\nEdit any dataset on Hugging Face (full list [here](https://huggingface.co/datasets)) using DuckDB functions (documentation [here](https://duckdb.org/docs/sql/functions/overview))")
     with gr.Group():
         with gr.Row():
             dataset_dropdown = gr.Dropdown(label="Dataset", allow_custom_value=True, scale=10)
@@ -133,7 +144,7 @@ with gr.Blocks(css=css, js=js) as demo:
             transform_dropdowns = [gr.Dropdown(choices=[column_name] + [prepare_function(text_func, "string", column_name) for text_func in text_functions_df.Name if "string" in text_func], value=column_name, container=False, interactive=True, allow_custom_value=True, visible=True, elem_classes="transform_dropdown") for column_name in EMPTY_DF.columns]
             transform_dropdowns += [gr.Dropdown(choices=[None], value=None, container=False, interactive=True, allow_custom_value=True, visible=False, elem_classes="transform_dropdown") for _ in range(MAX_NUM_COLUMNS - len(transform_dropdowns))]
         dataframe = gr.DataFrame(EMPTY_DF, column_widths=[f"{1/len(EMPTY_DF.columns):.0%}"] * len(EMPTY_DF.columns), interactive=True, elem_classes="readonly-dataframe")
-        with gr.Accordion("Show SQL command", open=False, elem_classes="transparent-accordion"):
+        with gr.Accordion("Show DuckDB SQL command", open=False, elem_classes="transparent-accordion"):
             code_markdown = gr.Markdown()
 
     def show_subset_dropdown(dataset: str):
@@ -153,7 +164,7 @@ with gr.Blocks(css=css, js=js) as demo:
     def show_input_dataframe(dataset: str, subset: str, split: str, loading_codes: list[dict]):
         pattern = ([loading_code["arguments"]["splits"][split] for loading_code in loading_codes if loading_code["config_name"] == subset] or [None])[0]
         if dataset and subset and split and pattern:
-            table = duckdb_sql(f"SELECT * FROM 'hf://datasets/{dataset}/{pattern}' LIMIT 10").arrow()
+            table = duckdb_sql(f"SELECT * FROM 'hf://datasets/{dataset}/{pattern}' LIMIT {NUM_ROWS}").arrow()
         else:
             table = EMPTY_TABLE
         prepared_functions = get_prepared_functions_from_table(table)
@@ -181,7 +192,7 @@ with gr.Blocks(css=css, js=js) as demo:
             code_markdown: (
                 "```sql\n"
                 + f"SELECT {', '.join(new_transform_dropdown['value'] for new_transform_dropdown in new_transform_dropdowns if new_transform_dropdown['value'])} "
-                + f"FROM 'hf://datasets/{dataset}/{pattern}';"
+                + f"FROM 'hf://datasets/{dataset}/{pattern}' LIMIT {NUM_ROWS};"
                 + "\n```"
             ) if pattern else "",
         }
@@ -213,7 +224,7 @@ with gr.Blocks(css=css, js=js) as demo:
             code_markdown: (
                 "```sql\n"
                 + f"SELECT {', '.join(new_transform_dropdown['value'] for new_transform_dropdown in new_transform_dropdowns if new_transform_dropdown['value'])} "
-                + f"FROM 'hf://datasets/{dataset}/{pattern}';"
+                + f"FROM 'hf://datasets/{dataset}/{pattern}' LIMIT {NUM_ROWS};"
                 + "\n```"
             ) if pattern else "",
         }
@@ -234,7 +245,7 @@ with gr.Blocks(css=css, js=js) as demo:
             code_markdown: (
                 "```sql\n"
                 + f"SELECT {', '.join(new_transform_dropdown['value'] for new_transform_dropdown in new_transform_dropdowns if new_transform_dropdown['value'])} "
-                + f"FROM 'hf://datasets/{dataset}/{pattern}';"
+                + f"FROM 'hf://datasets/{dataset}/{pattern}' LIMIT {NUM_ROWS};"
                 + "\n```"
             ) if pattern else "",
         }
@@ -252,7 +263,7 @@ with gr.Blocks(css=css, js=js) as demo:
             code_markdown: (
                 "```sql\n"
                 + f"SELECT {', '.join(new_transform_dropdown['value'] for new_transform_dropdown in new_transform_dropdowns if new_transform_dropdown['value'])} "
-                + f"FROM 'hf://datasets/{dataset}/{pattern}';"
+                + f"FROM 'hf://datasets/{dataset}/{pattern}' LIMIT {NUM_ROWS};"
                 + "\n```"
             ) if pattern else "",
         }
@@ -268,7 +279,7 @@ with gr.Blocks(css=css, js=js) as demo:
             code_markdown: (
                 "```sql\n"
                 + f"SELECT {', '.join(new_transform_dropdown['value'] for new_transform_dropdown in new_transform_dropdowns if new_transform_dropdown['value'])} "
-                + f"FROM 'hf://datasets/{dataset}/{pattern}';"
+                + f"FROM 'hf://datasets/{dataset}/{pattern}' LIMIT {NUM_ROWS};"
                 + "\n```"
             ) if pattern else "",
         }