Upload dataset.py with huggingface_hub

dataset.py

@@ -2,14 +2,15 @@ import os
 
 import datasets
 
-from .artifact import Artifact, UnitxtArtifactNotFoundError
+from .api import __file__ as _
 from .artifact import __file__ as _
-from .artifact import fetch_artifact
 from .blocks import __file__ as _
 from .card import __file__ as _
 from .catalog import __file__ as _
 from .collections import __file__ as _
 from .dataclass import __file__ as _
+from .dataset_utils import __file__ as _
+from .dataset_utils import get_dataset_artifact
 from .dict_utils import __file__ as _
 from .file_utils import __file__ as _
 from .formats import __file__ as _
@@ -17,11 +18,11 @@ from .fusion import __file__ as _
 from .generator_utils import __file__ as _
 from .hf_utils import __file__ as _
 from .instructions import __file__ as _
-from .load import __file__ as _
 from .loaders import __file__ as _
 from .logging_utils import __file__ as _
 from .logging_utils import get_logger
 from .metric import __file__ as _
+from .metric_utils import __file__ as _
 from .metrics import __file__ as _
 from .normalizers import __file__ as _
 from .operator import __file__ as _
@@ -30,7 +31,6 @@ from .processors import __file__ as _
 from .random_utils import __file__ as _
 from .recipe import __file__ as _
 from .register import __file__ as _
-from .register import _reset_env_local_catalogs, register_all_artifacts
 from .schema import __file__ as _
 from .split_utils import __file__ as _
 from .splitters import __file__ as _
@@ -47,57 +47,6 @@ from .version import version
 
 logger = get_logger()
 
-__default_recipe__ = "standard_recipe"
-
-
-def fetch(artifact_name):
-    try:
-        artifact, _ = fetch_artifact(artifact_name)
-        return artifact
-    except UnitxtArtifactNotFoundError:
-        return None
-
-
-def parse(query: str):
-    """Parses a query of the form 'key1=value1,key2=value2,...' into a dictionary."""
-    result = {}
-    kvs = query.split(",")
-    if len(kvs) == 0:
-        raise ValueError(
-            'Illegal query: "{query}" should contain at least one assignment of the form: key1=value1,key2=value2'
-        )
-    for kv in kvs:
-        key_val = kv.split("=")
-        if (
-            len(key_val) != 2
-            or len(key_val[0].strip()) == 0
-            or len(key_val[1].strip()) == 0
-        ):
-            raise ValueError(
-                f'Illegal query: "{query}" with wrong assignment "{kv}" should be of the form: key=value.'
-            )
-        key, val = key_val
-        if val.isdigit():
-            result[key] = int(val)
-        elif val.replace(".", "", 1).isdigit():
-            result[key] = float(val)
-        else:
-            result[key] = val
-
-    return result
-
-
-def get_dataset_artifact(dataset_str):
-    _reset_env_local_catalogs()
-    register_all_artifacts()
-    recipe = fetch(dataset_str)
-    if recipe is None:
-        args = parse(dataset_str)
-        if "type" not in args:
-            args["type"] = os.environ.get("UNITXT_DEFAULT_RECIPE", __default_recipe__)
-        recipe = Artifact.from_dict(args)
-    return recipe
-
 
 class Dataset(datasets.GeneratorBasedBuilder):
     """TODO: Short description of my dataset."""
@@ -108,7 +57,7 @@ class Dataset(datasets.GeneratorBasedBuilder):
     def generators(self):
         if not hasattr(self, "_generators") or self._generators is None:
             try:
-                from unitxt.dataset import \
+                from unitxt.dataset_utils import \
                     get_dataset_artifact as get_dataset_artifact_installed
 
                 unitxt_installed = True