_ASSET_KEY_COLLECTION = 'tft_asset_key_collection'
_ASSET_FILENAME_COLLECTION = 'tft_asset_filename_collection'
_OBJECT_TRACKER = None
VOCABULARY_SIZE_BY_NAME_COLLECTION = 'tft_vocabulary_size_by_name_collection'
  _OBJECT_TRACKER = object_tracker
    _OBJECT_TRACKER = None
class ObjectTracker:
  def __init__(self):
  def trackable_objects(self) -> List[base.Trackable]:
  def add_trackable_object(self, trackable_object: base.Trackable,
                           name: Optional[str]):
def object_tracker_scope(object_tracker: ObjectTracker):
def _get_object(name: str) -> Optional[base.Trackable]:
def track_object(trackable: base.Trackable, name: Optional[str]):
def make_and_track_object(trackable_factory_callable: Callable[[],
                                                               base.Trackable],
                          name: Optional[str] = None) -> base.Trackable:
def get_asset_annotations(graph: tf.Graph):
def clear_asset_annotations(graph: tf.Graph):
def annotate_asset(asset_key: str, asset_filename: str):
def annotate_vocab_size(vocab_filename: str, vocab_size: tf.Tensor): ANALYZER_COLLECTION = 'tft_analyzer_use'
MAPPER_COLLECTION = 'tft_mapper_use'
ANNOTATION_PREFIX_URL = 'type.googleapis.com'
  IS_ANNOTATIONS_PB_AVAILABLE = True
  IS_ANNOTATIONS_PB_AVAILABLE = False
def logging_context() -> Generator[None, None, None]:
def log_api_use(
    collection_name: str) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
  def decorator(fn):
    def wrapped_fn(*args, **kwargs): def is_ragged_feature_available() -> bool:
def is_ragged_feature(spec: FeatureSpecType) -> bool: class TFGraphContext:
  class _State(
      tfx_namedtuple.namedtuple('_State', [
          'module_to_export',
          'temp_dir',
          'evaluated_replacements',
      ])):
    def make_empty(cls):
  def __init__(self,
               module_to_export: tf.Module,
               temp_dir: Optional[str] = None,
               evaluated_replacements: Optional[Dict[str, Any]] = None):
  def __enter__(self):
  def __exit__(self, *exn_info):
  def module_to_export(self):
  def _get_current_state(cls) -> 'TFGraphContext._State':
  def get_or_create_temp_dir(cls) -> Optional[str]:
  def get_evaluated_replacements(cls) -> Optional[Dict[str, Any]]:
  def get_module_to_export(cls) -> Optional[tf.Module]:
  _TEMP_SUBDIR = 'analyzer_temporary_assets' _INITIALIZABLE_TABLE_OP_TYPES = [
    'CuckooTable',
    'CuckooTableV2',
    'HashTable',
    'HashTableV2',
    'IndexTable',
    'IndexTableV2',
]
_TABLE_INIT_OP_TYPES = [
    'InitializeTable',
    'InitializeTableV2',
    'InitializeTableFromTextFile',
    'InitializeTableFromTextFileV2',
    'InitializeTableFromDataset',
    'LookupTableImport',
    'LookupTableImportV2',
    # If a TF 2 SavedModel/Hub module with tables is loaded inside the
    # pre-processing fn, a StatefulPartitionedCall is added to the
    # TABLE_INITIALIZERS collection.
    'StatefulPartitionedCall',
]
def _decompose_tensor_or_op(tensor_or_op):
def retrieve_sources(sinks, ignore_control_dependencies=False):
def get_func_graph_for_name(graph, func_name):
  def __init__(self, op, func_graph_name):
  def __init__(self, op, func_graph_name):
def _reraise_unexpected_error(func):
  def wrapper(self, tensor_or_op):
  def __init__(self, source_info_dict, translate_path_fn, graph):
  def _get_parents(self, tensor_or_op):
  def _compute_analysis_results_for_func_attributes(self, tensor_or_op,
                                                    parent_analysis_results):
  def _compute_analysis_result(self, tensor_or_op, parent_analysis_results):
  def analyze_tensor(self, tensor_or_op):
  def ready_to_run(self, tensor_or_op):
  def get_unique_path(self, tensor):
def _set_unique_value_in_dict(input_dict, key, value):
  def __init__(self,
               graph,
               input_signature,
               replaced_tensors_ready,
               translate_path_fn=None):
  def _log_warning(self, message: str):
  def _get_table_init_op_and_inputs(self, table_init_op_or_tensor):
  def _make_source_infos_dict(self, input_signature, replaced_tensors_ready):
  def _get_table_init_op_source_info(self, table_init_op, graph_analyzer,
                                     translate_path_fn):
  def ready_table_initializers(self):
  def ready_to_run(self, tensor_or_op):
  def get_unique_path(self, tensor):
  def get_dependent_inputs(self, tensor_or_op):
  def _log_warning(self, message: str):
def get_dependent_inputs(graph, input_tensors, output_tensors):
def _serialize_op_attr(op_attr):
def describe_path_as_analyzer_cache_hash(
    x: Optional[Union[tf.Operation, tf.Tensor, str]],
    parents: Optional[List[bytes]] = None) -> Optional[bytes]:
  def __init__(self):
  def visit(self, operation_def, input_values):
  def validate_value(self, value):
def _retrieve_source_keys(
    sourced_tensors: Iterable[tf.Tensor],
    structured_inputs: Mapping[str, common_types.TensorType]) -> Set[str]:
def get_analyzers_fingerprint(
    graph: tf.Graph, structured_inputs: Mapping[str, common_types.TensorType]
) -> Mapping[str, AnalyzersFingerprint]:
class _UnexpectedPlaceholderError(Exception):
class _UnexpectedTableError(Exception):
class _GraphAnalyzer:
class InitializableGraphAnalyzer:
class _QuietInitializableGraphAnalyzer(InitializableGraphAnalyzer):
class SourcedTensorsVisitor(nodes.Visitor): _CACHED_EMPTY_ARRAY_BY_DTYPE = {}
_VALID_SCOPE_REGEX = re.compile('^[A-Za-z0-9]*$')
_INVALID_SCOPE_CHAR = re.compile('[^A-Za-z0-9_.\\-/>]')
METADATA_DIR_NAME = '.tft_metadata'
def _get_empty_array(dtype):
def batched_placeholders_from_specs(specs):
def _is_feature_spec(spec):
def _sanitize_scope_name(name):
def _batched_placeholder_from_typespec(name, typespec):
def _batched_placeholder_from_feature_spec(name, feature_spec):
def _extract_sparse_components(
    sparse_value: common_types.SparseTensorValueType
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
def _get_num_values_per_instance_in_sparse_batch(batch_indices: np.ndarray,
                                                 batch_size: int) -> List[int]:
def _decompose_sparse_batch(
    sparse_value: common_types.SparseTensorValueType
) -> Tuple[List[_CompositeComponentType], _CompositeComponentType]:
def _decompose_varlen_batch(
    sparse_value: common_types.SparseTensorValueType
) -> Tuple[_CompositeComponentType, _CompositeComponentType]:
def _handle_varlen_batch(tensor_or_value: common_types.SparseTensorValueType,
                         name: str) -> _CompositeComponentType:
def _handle_sparse_batch(
    tensor_or_value: common_types.SparseTensorValueType,
    spec: common_types.FeatureSpecType, name: str
) -> Dict[str, Union[List[_CompositeComponentType], _CompositeComponentType]]:
def _get_ragged_instance_component(
    component_batch: _CompositeInstanceComponentType, batch_splits: np.ndarray,
    instance_idx: int) -> _CompositeInstanceComponentType:
def _get_num_inner_uniform_elements(spec: common_types.RaggedFeature,
                                    name: str) -> int:
def _handle_ragged_batch(tensor_or_value: common_types.RaggedTensorValueType,
                         spec: common_types.FeatureSpecType,
                         name: str) -> Dict[str, _CompositeComponentType]:
def to_instance_dicts(schema, fetches):
def _tf_dtype_to_arrow_type(dtype: tf.DType) -> pa.DataType:
def get_type_specs_from_feature_specs(
    feature_specs: Dict[str, common_types.FeatureSpecType]
) -> Dict[str, tf.TypeSpec]:
def make_tensor_to_arrow_converter(
    schema: schema_pb2.Schema) -> tensor_to_arrow.TensorsToRecordBatchConverter:
def convert_to_arrow(
    schema: schema_pb2.Schema,
    converter: tensor_to_arrow.TensorsToRecordBatchConverter,
    fetches: Dict[str, common_types.TensorValueType]
) -> Tuple[List[pa.Array], pa.Schema]:
def _check_valid_sparse_tensor(indices: Union[_CompositeComponentType,
                                              List[_CompositeComponentType]],
                               values: _CompositeComponentType,
                               size: Union[int, List[int]], name: str):
def get_traced_transform_fn(
    preprocessing_fn: Callable[[Mapping[str, common_types.TensorType]],
                               Mapping[str, common_types.TensorType]],
    input_signature: Mapping[str, tf.TypeSpec],
    tf_graph_context: graph_context.TFGraphContext,
    output_keys_to_name_map: Optional[Dict[str,
                                           str]] = None) -> function.Function:
  def transform_fn(inputs):
def _trace_preprocessing_fn_v1(preprocessing_fn, specs):
def _trace_preprocessing_fn_v2(preprocessing_fn, specs, base_temp_dir):
def trace_preprocessing_function(preprocessing_fn,
                                 input_specs,
                                 use_tf_compat_v1,
                                 base_temp_dir=None):
def _trace_and_write_transform_fn(
    saved_model_dir: str,
    preprocessing_fn: Callable[[Mapping[str, common_types.TensorType]],
                               Mapping[str, common_types.TensorType]],
    input_signature: Mapping[str, tf.TypeSpec], base_temp_dir: Optional[str],
    tensor_replacement_map: Optional[Dict[str, tf.Tensor]],
    output_keys_to_name_map: Optional[Dict[str,
                                           str]]) -> function.ConcreteFunction:
def _trace_and_get_metadata(
    concrete_transform_fn: function.ConcreteFunction,
    structured_inputs: Mapping[str, common_types.TensorType],
    preprocessing_fn: Callable[[Mapping[str, common_types.TensorType]],
                               Mapping[str, common_types.TensorType]],
    base_temp_dir: Optional[str],
    tensor_replacement_map: Optional[Dict[str, tf.Tensor]]
) -> dataset_metadata.DatasetMetadata:
def _validate_analyzers_fingerprint(
    baseline_analyzers_fingerprint: Mapping[str,
                                            graph_tools.AnalyzersFingerprint],
    graph: tf.Graph, structured_inputs: Mapping[str, common_types.TensorType]):
def trace_and_write_v2_saved_model(
    saved_model_dir: str,
    preprocessing_fn: Callable[[Mapping[str, common_types.TensorType]],
                               Mapping[str, common_types.TensorType]],
    input_signature: Mapping[str, tf.TypeSpec], base_temp_dir: Optional[str],
    baseline_analyzers_fingerprint: Mapping[str,
                                            graph_tools.AnalyzersFingerprint],
    tensor_replacement_map: Optional[Dict[str, tf.Tensor]],
    output_keys_to_name_map: Optional[Dict[str, str]]):
def _assert_no_analyzers_in_graph(graph):
def analyze_in_place(preprocessing_fn, force_tf_compat_v1, feature_specs,
                     type_specs, transform_output_path): class ValueNode(
    tfx_namedtuple.namedtuple('ValueNode',
                              ['parent_operation', 'value_index'])):
class OperationDef(metaclass=abc.ABCMeta):
class OperationNode:
class Visitor(metaclass=abc.ABCMeta):
class Traverser:
class _PrintGraphVisitor(Visitor):
  def __init__(self, parent_operation, value_index: int):
  def __iter__(self):
  def num_outputs(self) -> int:
  def label(self) -> str:
  def get_field_str(self, field_name: str) -> str:
  def is_partitionable(self) -> bool:
  def cache_coder(self) -> Optional[object]:
  def __init__(self, operation_def, inputs):
  def __repr__(self):
  def operation_def(self):
  def inputs(self):
  def outputs(self):
def apply_operation(operation_def_cls, *args, **kwargs):
def apply_multi_output_operation(operation_def_cls, *args, **kwargs):
  def validate_value(self, value):
  def visit(self, operation_def, input_values):
  def __init__(self, visitor: Visitor):
  def visit_value_node(self, value_node: ValueNode):
  def _maybe_visit_value_node(self, value_node: ValueNode):
  def _visit_operation(self, operation: OperationNode):
def _escape(line: str) -> str:
  def __init__(self):
  def get_dot_graph(self) -> pydot.Dot:
  def visit(self, operation_def, input_nodes) -> Tuple[pydot.Node, ...]:
  def validate_value(self, value: pydot.Node):
def get_dot_graph(leaf_nodes: Collection[ValueNode]) -> pydot.Dot: def _ragged_feature_spec_from_batched_tensor(
    name: str, tensor: tf.RaggedTensor
) -> Union[tf.io.VarLenFeature, common_types.RaggedFeature]:
def _feature_spec_from_batched_tensors(
    tensors: Mapping[str, common_types.TensorType],
    is_evaluation_complete: bool) -> Mapping[str, common_types.FeatureSpecType]:
def infer_feature_schema(
    features: Mapping[str, common_types.TensorType],
    graph: tf.compat.v1.Graph,
    session: Optional[tf.compat.v1.Session] = None) -> schema_pb2.Schema:
def infer_feature_schema_v2(
    features: Mapping[str, common_types.TensorType],
    concrete_metadata_fn: function.ConcreteFunction,
    evaluate_schema_overrides: bool) -> schema_pb2.Schema:
def _infer_feature_schema_common(
    features: Mapping[str, common_types.TensorType],
    tensor_ranges: Mapping[str, Tuple[int, int]],
    feature_annotations: Mapping[str, List[any_pb2.Any]],
    global_annotations: List[any_pb2.Any],
    is_evaluation_complete: bool) -> schema_pb2.Schema:
def set_tensor_schema_override(tensor, min_value, max_value):
def _get_tensor_ranges(graph):
def _get_tensor_ranges_v2(metadata):
def get_tensor_schema_override(
    tensor: common_types.TensorType) -> Tuple[tf.Tensor, tf.Tensor]:
def annotate(type_url, proto_message, tensor=None):
def _get_schema_annotations(graph, session):
def _get_schema_annotations_v2(metadata):
def _get_schema_annotations_common(tensor_annotation_keys, type_urls,
                                   proto_values):
def _get_tensor_value_to_key_map(features_dict):
def _get_schema_overrides(graph,
                          tensor_name_to_key_map,
                          tensor_collection_key,
                          overrides_keys,
                          default_tensor_name=None):
def get_traced_metadata_fn(
    preprocessing_fn: Callable[[Mapping[str, common_types.TensorType]],
                               Mapping[str, common_types.TensorType]],
    structured_inputs: Mapping[str, common_types.TensorType],
    tf_graph_context: graph_context.TFGraphContext,
    evaluate_schema_overrides: bool) -> function.Function:
  def metadata_fn():
_TF_METADATA_TENSOR_COLLECTION = 'tft_schema_override_tensor'
_TF_METADATA_TENSOR_MIN_COLLECTION = 'tft_schema_override_min'
_TF_METADATA_TENSOR_MAX_COLLECTION = 'tft_schema_override_max'
_TF_METADATA_EXTRA_ANNOTATION = 'tft_schema_override_annotation_tensor'
_TF_METADATA_EXTRA_ANNOTATION_TYPE_URL = 'tft_schema_override_annotation_type'
_TF_METADATA_EXTRA_ANNOTATION_PROTO = 'tft_schema_override_annotation_proto'
_TF_METADATA_EXTRA_ANNOTATION_GLOBAL = 'tft_schema_override_global_sentinel' _ANALYSIS_GRAPH = None
  _ANALYSIS_GRAPH = optimized_saved_model_future
def _tensor_name(tensor):
  def __init__(self, graph_analyzer):
  def _validate_operation_label_uniqueness(self, operation_def):
  def visit(self, operation_def, input_values):
  def validate_value(self, value):
  def __init__(self):
  def visit(self, operation_def, input_values):
  def validate_value(self, value):
  def __init__(self, prefer_fine_grained_view, flattened_view,
               fine_grained_view, hashed_path):
  def _validate_flattened_view(self, view):
  def _validate_fine_grained_view(self, view):
  def __init__(self, dataset_keys, cache_dict, tensor_keys_to_paths,
               cache_output_nodes):
  def _validate_operation_def(self, operation_def):
  def _make_next_hashed_path(self, parent_hashed_paths, operation_def):
  def visit(self, operation_def, input_values):
  def _visit_partitionable_operation(self, operation_def, upstream_views):
  def _apply_operation_on_fine_grained_view(self, operation_def,
                                            fine_grained_views,
                                            next_hashed_path):
  def _visit_apply_savedmodel_operation(self, operation_def, upstream_views):
  def validate_value(self, value):
def _perform_cache_optimization(saved_model_future, dataset_keys,
                                tensor_keys_to_paths, cache_dict):
  def __init__(self, required_dataset_keys_output):
  def visit(self, operation_def, input_values):
  def validate_value(self, value):
def _build_analysis_graph_for_inspection(preprocessing_fn, specs, dataset_keys,
                                         input_cache, force_tf_compat_v1):
def get_analysis_dataset_keys(preprocessing_fn,
                              specs,
                              dataset_keys,
                              input_cache,
                              force_tf_compat_v1):
def get_analysis_cache_entry_keys(preprocessing_fn,
                                  specs,
                                  dataset_keys,
                                  force_tf_compat_v1):
def build(graph,
          input_signature,
          output_signature,
          dataset_keys=None,
          cache_dict=None):
class _ReadyVisitor(nodes.Visitor):
class _TranslateVisitor(nodes.Visitor):
class _OptimizationView(
    tfx_namedtuple.namedtuple('_OptimizationView', [
        'prefer_fine_grained_view', 'flattened_view', 'fine_grained_view',
        'hashed_path'
    ])):
class _OptimizeVisitor(nodes.Visitor):
class _InspectVisitor(nodes.Visitor): _CACHE_VERSION_NUMBER = 1
_PYTHON_VERSION = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)
_CACHE_VERSION = tf.compat.as_bytes('__v{}__{}_'.format(_CACHE_VERSION_NUMBER,
                                                        _PYTHON_VERSION))
_CACHE_COMPONENT_CHARACTER_REPLACEMENTS = (
    ('/', '-'),
    ('\\', '-'),
    ('*', 'STAR'),
    ('@', 'AT'),
    ('[', '--'),
    (']', '--'),
    (':', 'P'),
    ('=', 'E'),
)
  _FLATTENED_DATASET_KEY = object()
  _MANIFEST_FILE_NAME = 'MANIFEST'
def _make_valid_cache_component(name):
  def __new__(cls, dataset_key):
  def __str__(self):
  def __hash__(self):
  def __eq__(self, other):
  def is_flattened_dataset_key(self):
def _make_flattened_dataset_key():
def _get_dataset_cache_path(base_dir, dataset_key):
  def __init__(self, base_path):
  def _open(self):
  def _close(self):
  def _delete(self):
  def __enter__(self):
  def __exit__(self, *exn_info):
  def _get_manifest_contents(self, manifest_file_handle):
  def read(self):
  def write(self, manifest):
  def __init__(self, file_path_prefix):
  def __init__(self, pipeline, cache_base_dir, dataset_keys=None, sink=None):
  def _write_cache(self, manifest_file, dataset_key_index, dataset_key_dir,
                   cache_dict):
  def expand(self, dataset_cache_dict):
  def __init__(self,
               cache_base_dir,
               dataset_keys,
               cache_entry_keys=None,
               source=None):
  def _should_read_cache_entry_key(self, key):
  def expand(self, pvalue):
def validate_dataset_keys(dataset_keys):
def make_cache_entry_key(cache_key):
class DatasetKey(tfx_namedtuple.namedtuple('DatasetKey', ['key'])):
class _ManifestFile:
class _WriteToTFRecordGzip(beam.io.WriteToTFRecord):
class WriteAnalysisCacheToFS(beam.PTransform):
class ReadAnalysisCacheFromFS(beam.PTransform): class CreateTensorBinding(
    tfx_namedtuple.namedtuple(
        'CreateTensorBinding',
        ['tensor_name', 'dtype_enum', 'is_asset_filepath', 'label']),
    nodes.OperationDef):
class CreateSavedModel(
    tfx_namedtuple.namedtuple(
        'CreateSavedModel',
        ['table_initializers', 'output_signature', 'label']),
    nodes.OperationDef):
class ExtractInputForSavedModel(
    tfx_namedtuple.namedtuple('ExtractInputForSavedModel',
                              ['dataset_key', 'label']), nodes.OperationDef):
class ApplySavedModel(
    tfx_namedtuple.namedtuple('ApplySavedModel', ['phase', 'label']),
    nodes.OperationDef):
class ExtractFromDict(
    tfx_namedtuple.namedtuple('ExtractFromDict', ['keys', 'label']),
    nodes.OperationDef):
class Flatten(
    tfx_namedtuple.namedtuple('Flatten', ['label']), nodes.OperationDef):
  def _get_tensor_type_name(self, tensor):
  def get_field_str(self, field_name):
  def is_partitionable(self):
  def is_partitionable(self):
  def is_partitionable(self): NUMERIC_TYPE = Union[float, int]
PRIMITIVE_TYPE = Union[NUMERIC_TYPE, str, bytes]
METRICS_NAMESPACE = util.MakeTfxNamespace(['Transform'])
  TF_COMPAT_V1 = 'tf_compat_v1'
  TF_V2_ONLY = 'tf_v2_only'
_ALLOWED_PTRANSFORM_TAGS = [tag.value for tag in EnvironmentTags]
  _GENERAL_ENVIRONMENT_TAG = object()
_PTRANSFORM_BY_OPERATION_DEF_SUBCLASS = (
    collections.defaultdict(_PtransformWrapper))
class EnvironmentTags(enum.Enum):
class _PtransformWrapper:
class ConstructBeamPipelineVisitor(nodes.Visitor):
class IncrementCounter(beam.PTransform):
def get_unique_temp_path(base_temp_dir):
  def __init__(self):
  def add_ptransform(self, ptransform_class, tags):
  def get_ptransform(self, tag):
def register_ptransform(operation_def_subclass, tags=None):
  def register(ptransform_class):
  def __init__(self, extra_args):
  def visit(self, operation, inputs):
  def validate_value(self, value):
  def __init__(self, counter_name):
  def _make_and_increment_counter(self, unused_element):
  def expand(self, pcoll): class Context:
  class _State(
      tfx_namedtuple.namedtuple('_State', [
          'temp_dir',
          'desired_batch_size',
          'passthrough_keys',
          'use_deep_copy_optimization',
          'force_tf_compat_v1',
      ])):
  class _StateStack:
    def make_empty(cls):
    def __init__(self):
  def __init__(self,
               temp_dir: Optional[str] = None,
               desired_batch_size: Optional[int] = None,
               passthrough_keys: Optional[Iterable[str]] = None,
               use_deep_copy_optimization: Optional[bool] = None,
               force_tf_compat_v1: Optional[bool] = None):
  def __enter__(self):
  def __exit__(self, *exn_info):
  def _get_topmost_state_frame(cls) -> 'Context._State':
  def create_base_temp_dir(cls) -> str:
  def get_desired_batch_size(cls) -> Optional[int]:
  def get_passthrough_keys(cls) -> Iterable[str]:
  def get_use_deep_copy_optimization(cls) -> bool:
  def _get_force_tf_compat_v1(cls) -> bool:
  def get_use_tf_compat_v1(cls) -> bool:
  _TEMP_SUBDIR = 'tftransform_tmp' def deep_copy(pcollection):
  """Create a deep copy of a PCollection up to materialization boundaries."""
  if not isinstance(pcollection, pvalue.PCollection):
    raise ValueError('Input to deep_copy must be a PCollection.')

  # AppliedPTransform.update_input_refcounts() is a vestigial method that
  # uses an incorrect heuristic; it will be removed in a future version of
  # Beam, since its results aren't used anyway.  Until then, we work around
  # this (see https://issues.apache.org/jira/browse/BEAM-4593).
  if getattr(beam_pipeline.AppliedPTransform,
             'update_input_refcounts', None) is not None:
    beam_pipeline.AppliedPTransform.update_input_refcounts = lambda _: None

  to_clone = _get_items_to_clone(pcollection)
  pcollection_replacements = _clone_items(pcollection.pipeline, to_clone)

  return pcollection_replacements[pcollection] class BeamDatasetMetadata(
    tfx_namedtuple.namedtuple(
        'BeamDatasetMetadata',
        ['dataset_metadata', 'deferred_metadata', 'asset_map'])):
class WriteMetadata(beam.PTransform):
  def schema(self):
  def __init__(self, path, pipeline, write_to_unique_subdirectory=False):
  def _extract_input_pvalues(self, metadata):
  def expand(self, metadata):
    def write_metadata_output(metadata): def _make_cast_fn(np_dtype):
  def identity(x):
  def utf8(s):
  def string_cast(x):
def _make_feature_value_fn(dtype):
  def __init__(self, name, feature_spec):
  def name(self):
  def initialize_encode_cache(self, example):
  def encode_value(self, values):
  def __init__(self, name, dtype):
  def name(self):
  def initialize_encode_cache(self, example):
  def encode_value(self, values):
  def __init__(self, schema, serialized=True):
  def __reduce__(self):
  def encode(self, instance):
class _FixedLenFeatureHandler:
class _VarLenFeatureHandler:
class ExampleProtoCoder: _MANGLED_TENSOR_NAME_RE = re.compile(
    r'(.*)\$(indices|values|dense_shape|dense_tensor)$')
_PARTITIONED_VARIABLE_NAME_RE = re.compile(r'^(.*)/part_(\d*)$')
def _update_legacy_signature(signature):
def _load_transform_saved_model(transform_savedmodel_dir):
def _expand_input_map(logical_input_map, input_signature):
def _maybe_register_addon_ops():
  def _try_import(name):
def _partially_apply_saved_transform_impl(saved_model_dir,
                                          logical_input_map,
                                          tensor_replacement_map=None):
  def lookup_remapped_tensor(tensor_name):
  def lookup_tensor_or_sparse_or_composite_tensor(tensor_info):
def partially_apply_saved_transform_internal(saved_model_dir,
                                             logical_input_map,
                                             tensor_replacement_map=None):
def write_saved_transform_from_session(
    session, inputs, outputs, export_path, as_text=False):
def exported_as_v1(transform_savedmodel_dir): def _restore_from_v1_saved_model(
    restored_function: function.ConcreteFunction, saved_model_dir: str
) -> Tuple[function.ConcreteFunction, Mapping[str, Any], Mapping[
def _as_operation(op_or_tensor: Union[tf.Operation, tf.Tensor]) -> tf.Operation:
def _get_component_tensors(
    tensor: Union[tf.Tensor, composite_tensor.CompositeTensor]
) -> Iterable[tf.Tensor]:
def _get_output_to_inputs_map(
    output_signature: Mapping[str, common_types.TensorType]
) -> Dict[str, Iterable[tf.Tensor]]:
  def __init__(self, saved_model_dir: str):
  def _initialize(self, load_v2_in_compat, imported, wrapped, structured_inputs,
                  structured_outputs, outputs_to_inputs_map):
  def load_v2_in_compat(self):
  def structured_outputs(self):
  def _get_feeds(self, unfed_input_keys: Iterable[str]) -> Iterable[tf.Tensor]:
  def _get_unfed_input_keys(self,
                            input_tensor_keys: Iterable[str]) -> Iterable[str]:
  def _get_fetches(
      self, feeds: Iterable[tf.Tensor]) -> Dict[str, common_types.TensorType]:
  def _get_fetches_keys(self, feeds: Iterable[tf.Tensor]) -> Iterable[str]:
  def _get_missing_inputs(
      self, unfed_input_keys: Iterable[str],
      batch_size: int) -> Dict[str, common_types.TensorType]:
  def _apply_v1_transform_model_in_v2(
      self, logical_input_map: Mapping[str, common_types.TensorType]
  ) -> Dict[str, common_types.TensorType]:
  def _format_input_map_as_tensors(self, input_map):
  def _apply_v2_transform_model_finalized(
      self, logical_input_map: Mapping[str, common_types.TensorType]
  ) -> Dict[str, common_types.TensorType]:
  def _apply_v2_transform_model(
      self, logical_input_map: Mapping[str, common_types.TensorType]
  ) -> Dict[str, common_types.TensorType]:
  def apply_transform_model(
      self, logical_input_map: Mapping[str, common_types.TensorType]
  ) -> Dict[str, common_types.TensorType]:
  def _finalize_wrapped_function(
      self, unfed_input_keys: Iterable[str],
      fetches_keys: Iterable[str]) -> function.ConcreteFunction:
    def wrapped_finalized(inputs):
  def finalize(self, input_tensor_keys: Iterable[str],
               output_tensor_keys: Iterable[str]):
def optimize_concrete_function(
    concrete_function: function.ConcreteFunction,
    strip_control_dependencies: bool) -> wrap_function.WrappedFunction:
def trace_and_update_module(
    module: tf.Module, tf_function: function.Function, name: str,
    strip_control_dependencies: bool) -> function.ConcreteFunction:
  def _variable_creator(next_creator, **kwargs):
def write_v2_saved_model(module: tf.Module, tf_function: function.Function,
                         name: str,
                         saved_model_dir: str) -> function.ConcreteFunction:
class SavedModelLoader: class DatasetMetadata:
  def __init__(self, schema: schema_pb2.Schema):
  def from_feature_spec(
      cls: Type[_DatasetMetadataType],
      feature_spec: Mapping[str, common_types.FeatureSpecType],
      domains: Optional[Mapping[str, common_types.DomainType]] = None
  ) -> _DatasetMetadataType:
  def schema(self) -> schema_pb2.Schema:
  def __eq__(self, other):
  def __ne__(self, other):
  def __repr__(self): def read_metadata(path):
def _parse_schema_json(schema_json):
def _column_schema_from_json(feature_dict):
def _domain_from_json(domain):
    def maybe_to_int(s):
def _dtype_from_json(domain):
def write_metadata(metadata, path):
def _convert_scalar_or_list(fn, scalar_or_list): TENSOR_REPRESENTATION_GROUP = ''
RAGGED_TENSOR_TAG = 'ragged_tensor'
_DEPRECATED_LIFECYCLE_STAGES = [
    schema_pb2.DEPRECATED, schema_pb2.DISABLED, schema_pb2.PLANNED,
    schema_pb2.ALPHA, schema_pb2.DEBUG_ONLY
]
_LEGACY_DEFAULT_VALUE_FOR_DTYPE = {tf.string: '', tf.int64: -1, tf.float32: -1}
def schema_from_feature_spec(
    feature_spec: Mapping[str, common_types.FeatureSpecType],
    domains: Optional[Mapping[str, common_types.DomainType]] = None
) -> schema_pb2.Schema:
def _ragged_tensor_representation_from_feature_spec(
    spec: common_types.RaggedFeature, name: str,
    domains: Dict[str, common_types.DomainType]
) -> Tuple[schema_pb2.Feature, List[schema_pb2.Feature],
def _sparse_feature_from_feature_spec(spec, name, domains):
def _feature_from_feature_spec(spec, name, domains):
def _set_type(name, feature, dtype):
def _set_domain(name, feature, domain):
def schema_as_feature_spec(
    schema_proto: schema_pb2.Schema) -> SchemaAsFeatureSpecResult:
def _get_string_domains(schema):
def _get_domain(feature, string_domains):
def pop_ragged_source_columns(
    name: str, tensor_representation: schema_pb2.TensorRepresentation,
    feature_by_name: Dict[str, schema_pb2.Feature]) -> schema_pb2.Feature:
def _ragged_tensor_representation_as_feature_spec(
    name: str, tensor_representation: schema_pb2.TensorRepresentation,
    feature_by_name: Dict[str, schema_pb2.Feature],
    string_domains: Dict[str, common_types.DomainType]
) -> Tuple[common_types.RaggedFeature, Optional[common_types.DomainType]]:
def _sparse_feature_as_feature_spec(feature, feature_by_name, string_domains):
def _feature_as_feature_spec(feature, string_domains):
def _feature_dtype(feature):
def _fixed_shape_as_tf_shape(fixed_shape):
def _include_in_parsing_spec(feature):
def _legacy_schema_from_feature_spec(feature_spec, domains=None):
def _legacy_schema_as_feature_spec(schema_proto):
def _legacy_feature_as_feature_spec(feature):
def _legacy_infer_default_value(feature_proto, dtype):
import copy
import datetime
import os
import apache_beam as beam
import numpy as np
import pyarrow as pa
import tensorflow as tf
from apache_beam.runners.portability import fn_api_runner
from apache_beam.typehints import Any
from apache_beam.typehints import Dict
from apache_beam.typehints import Iterable
from apache_beam.typehints import List
from apache_beam.typehints import Set
from apache_beam.typehints import Tuple
from apache_beam.typehints import Union
from apache_beam.utils import shared
from tensorflow_transform import annotators
from tensorflow_transform import common
from tensorflow_transform import common_types
from tensorflow_transform import graph_context
from tensorflow_transform import graph_tools
from tensorflow_transform import impl_helper
from tensorflow_transform import nodes
from tensorflow_transform import schema_inference
from tensorflow_transform.beam import analysis_graph_builder
from tensorflow_transform.beam import analyzer_cache
from tensorflow_transform.beam import beam_nodes
from tensorflow_transform.beam import common as beam_common
from tensorflow_transform.beam import context
from tensorflow_transform.beam import deep_copy
from tensorflow_transform.beam.tft_beam_io import beam_metadata_io
from tensorflow_transform.coders import example_proto_coder
from tensorflow_transform.saved import saved_transform_io
from tensorflow_transform.saved import saved_transform_io_v2
from tensorflow_transform.tf_metadata import dataset_metadata
from tensorflow_transform.tf_metadata import metadata_io
from tensorflow_transform.tf_metadata import schema_utils
from tfx_bsl.telemetry import collection as telemetry
from tfx_bsl.tfxio import tensor_representation_util
from tfx_bsl.tfxio import tensor_to_arrow
from tfx_bsl.tfxio import tf_example_record
from tfx_bsl.tfxio.tensor_adapter import TensorAdapter
from tfx_bsl.tfxio.tensor_adapter import TensorAdapterConfig
from tfx_bsl.types import tfx_namedtuple
from tensorflow.python.framework import ops  # pylint: disable=g-direct-tensorflow-import
from tensorflow_metadata.proto.v0 import schema_pb2
# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""An implementation of tf.Transform using Beam.

The beam implementation takes a user defined preprocessing function (see
examples for how to define a preprocessing function) and implements it as a
Beam PTransform.

The AnalyzeDataset takes the user's preprocessing function and converts into
a TensorFlow function that can be applied to each row of a dataset.  For
example if the user's preprocessing function describes normalizing a column by
subtracting its mean, the tensorflow function will contain the mean of the
column as a constant, and will subtract this value from each value of the
column.  We refer to the result of AnalyzeDataset as a "transform function".

Since AnalyzeDataset is implemented with beam, it accepts a PCollection that
represents the dataset (see below for the exact format) and returns a singleton
PCollection containing the transform function (as a serialized TF graph).

The TransformDataset PTransform takes a dataset and a transform function, and
returns the transformed dataset where the transform function is applied to each
row of the original dataset.

There is also an AnalyzeAndTransformDataset PTransform that applies
AnalyzeDataset and TransformDataset to the same input dataset, possibly with
optimizations.
"""




# TODO(https://issues.apache.org/jira/browse/SPARK-22674): Switch to
# `collections.namedtuple` or `typing.NamedTuple` once the Spark issue is
# resolved.


# TODO(b/123325923): Fix the key type here to agree with the actual keys.
_DatasetElementType = Dict[Any,  # Any -> str?

                           Union[beam_common.PRIMITIVE_TYPE,