Upload split_utils.py with huggingface_hub

split_utils.py

@@ -1,12 +1,14 @@
 import itertools
-import logging
 import re
 from typing import Dict
 
 from .generator_utils import ReusableGenerator
-from .random_utils import nested_seed
+from .logging_utils import get_logger
+from .random_utils import new_random_generator
 from .stream import Stream
 
+logger = get_logger()
+
 
 def parse_random_mix_string(input_str):
     """Parses a string of format "source1[percentage1%]+source2[value2]+..." and returns a dictionary.
@@ -179,7 +181,7 @@ def build_stream_routing(mapping):
                 }
             }
             stream_mapping = build_stream_mapping(mapping)
-            logging.info(stream_mapping)
+            logger.info(stream_mapping)
             # Output: {'my_old_stream1': (['my_new_stream', 'my_new_stream2'], [0.6, 0.4]),
             #          'my_old_stream2': (['my_new_stream', 'my_new_stream2'], [0.2, 0.8])}
     """
@@ -230,14 +232,14 @@ def random_mix_generator(
 ):
     for old_stream_name in new_stream_sources:
         optinal_streams, weights = stream_routing[old_stream_name]
-        with nested_seed(old_stream_name) as rand:
-            assert (
-                old_stream_name in input_streams
-            ), f"'{old_stream_name}' split not found.  Possibles options: {input_streams.keys()}"
-            for item in input_streams[old_stream_name]:
-                choice = rand.choices(optinal_streams, weights=weights, k=1)[0]
-                if choice == new_stream_name:
-                    yield item
+        random_generator = new_random_generator(sub_seed=old_stream_name)
+        assert (
+            old_stream_name in input_streams
+        ), f"'{old_stream_name}' split not found.  Possibles options: {input_streams.keys()}"
+        for item in input_streams[old_stream_name]:
+            choice = random_generator.choices(optinal_streams, weights=weights, k=1)[0]
+            if choice == new_stream_name:
+                yield item
 
 
 def random_mix_streams(input_streams, mapping):
@@ -276,30 +278,29 @@ def random_mix_streams(input_streams, mapping):
             }
             new_streams = create_streams(input_streams, mapping)
             for new_stream_name, new_stream in new_streams.items():
-                logging.info(f"{new_stream_name}:")
+                logger.info(f"{new_stream_name}:")
                 for _, item in zip(range(10), new_stream):
-                    logging.info(item)
+                    logger.info(item)
     """
     new_streams = {}
 
     # Build stream routing
     stream_routing = build_stream_routing(mapping)
 
-    with nested_seed():
-        # Create new stream generators
-        for new_stream_name, new_stream_sources in mapping.items():
-            new_streams[new_stream_name] = ReusableGenerator(
-                random_mix_generator,
-                gen_kwargs={
-                    "new_stream_name": new_stream_name,
-                    "new_stream_sources": new_stream_sources,
-                    "stream_routing": stream_routing,
-                    "input_streams": input_streams,
-                },
-            )
+    # Create new stream generators
+    for new_stream_name, new_stream_sources in mapping.items():
+        new_streams[new_stream_name] = ReusableGenerator(
+            random_mix_generator,
+            gen_kwargs={
+                "new_stream_name": new_stream_name,
+                "new_stream_sources": new_stream_sources,
+                "stream_routing": stream_routing,
+                "input_streams": input_streams,
+            },
+        )
 
     return new_streams
 
 
 if __name__ == "__main__":
-    logging.info(parse_random_mix_string("dale[90%]+oren[0.7]+mike"))
+    logger.info(parse_random_mix_string("dale[90%]+oren[0.7]+mike"))