Upload operators.py with huggingface_hub
Browse files- operators.py +16 -0
operators.py
CHANGED
|
@@ -136,6 +136,22 @@ class AddFields(StreamInstanceOperator):
|
|
| 136 |
return instance
|
| 137 |
|
| 138 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
class FieldOperator(StreamInstanceOperator):
|
| 140 |
"""
|
| 141 |
A general stream that processes the values of a field (or multiple ones
|
|
|
|
| 136 |
return instance
|
| 137 |
|
| 138 |
|
| 139 |
+
class RemoveFields(StreamInstanceOperator):
|
| 140 |
+
"""
|
| 141 |
+
Adds specified fields to each instance in a stream.
|
| 142 |
+
|
| 143 |
+
Args:
|
| 144 |
+
fields (Dict[str, object]): The fields to add to each instance.
|
| 145 |
+
"""
|
| 146 |
+
|
| 147 |
+
fields: List[str]
|
| 148 |
+
|
| 149 |
+
def process(self, instance: Dict[str, Any], stream_name: str = None) -> Dict[str, Any]:
|
| 150 |
+
for field in self.fields:
|
| 151 |
+
del instance[field]
|
| 152 |
+
return instance
|
| 153 |
+
|
| 154 |
+
|
| 155 |
class FieldOperator(StreamInstanceOperator):
|
| 156 |
"""
|
| 157 |
A general stream that processes the values of a field (or multiple ones
|