Upload operator.py with huggingface_hub

operator.py

@@ -150,6 +150,11 @@ class MultiStreamOperator(StreamingOperator):
     def process(self, multi_stream: MultiStream) -> MultiStream:
         pass
 
+    def process_instance(self, instance, stream_name="tmp"):
+        multi_stream = {stream_name: [instance]}
+        processed_multi_stream = self(multi_stream)
+        return next(iter(processed_multi_stream[stream_name]))
+
 
 class SingleStreamOperator(MultiStreamOperator):
     """A class representing a single-stream operator in the streaming system.
@@ -216,6 +221,10 @@ class SingleStreamOperator(MultiStreamOperator):
     def process(self, stream: Stream, stream_name: Optional[str] = None) -> Generator:
         pass
 
+    def process_instance(self, instance, stream_name="tmp"):
+        processed_stream = self._process_single_stream([instance], stream_name)
+        return next(iter(processed_stream))
+
 
 class PagedStreamOperator(SingleStreamOperator):
     """A class representing a paged-stream operator in the streaming system.
@@ -240,12 +249,21 @@ class PagedStreamOperator(SingleStreamOperator):
             if len(page) >= self.page_size:
                 yield from self.process(page, stream_name)
                 page = []
+        yield from self._process_page(page, stream_name)
+
+    def _process_page(
+        self, page: List[Dict], stream_name: Optional[str] = None
+    ) -> Generator:
         yield from self.process(page, stream_name)
 
     @abstractmethod
     def process(self, page: List[Dict], stream_name: Optional[str] = None) -> Generator:
         pass
 
+    def process_instance(self, instance, stream_name="tmp"):
+        processed_stream = self._process_page([instance], stream_name)
+        return next(iter(processed_stream))
+
 
 class SingleStreamReducer(StreamingOperator):
     """A class representing a single-stream reducer in the streaming system.
@@ -298,6 +316,9 @@ class StreamInstanceOperator(SingleStreamOperator):
     ) -> Dict[str, Any]:
         pass
 
+    def process_instance(self, instance, stream_name="tmp"):
+        return self._process_instance(instance, stream_name)
+
 
 class StreamInstanceOperatorValidator(StreamInstanceOperator):
     """A class representing a stream instance operator validator in the streaming system.
@@ -316,7 +337,7 @@ class StreamInstanceOperatorValidator(StreamInstanceOperator):
         try:
             first_instance = next(iterator)
         except StopIteration as e:
-            raise StopIteration(f"Strem '{stream_name}' is empty") from e
+            raise StopIteration(f"Stream '{stream_name}' is empty") from e
         result = self._process_instance(first_instance, stream_name)
         self.validate(result)
         yield result