_ZIP_NAME = 'WikiTables-WithLinks'
_CONFIG_NAME = 'hybridqa_rc_config.json'
  NONE = 'none'
  FULL_RANDOM = 'full'
  SELECTIVE_RANDOM = 'selective'
  COLUMN_CONFINED_RANDOM = 'col_confined'
class Prediction:
class NoisyCellStrategy(enum.Enum):
class HybridQaRcConfig:
  def example_id(self):
  def interaction_id(self):
def get_predictions(prediction_file):
def get_table_dimensions(
    table):
  def from_dict(cls, json_object):
  def from_json(cls, json_str):
  def hybridqa_prediction_filepaths(self):
  def interactions_dirs(self):
def _parse_answer_cell(cell,
                       text,
                       links,
                       descriptions,
                       *,
                       url_unquote = True):
def _init_answer_table(idx, title,
                       url):
def get_texts_and_links(table_dict, coord):
def _parse_answer_table(
    json_dict,
    descriptions,
    coordinates,
):
def _parse_answer_tables(
    json_dict,
    descriptions,
    coordinates,
):
def _create_answer_table(
    original_table,
    descriptions,
    answer_coordinates,
):
def generate_sample_space(
    strategy, n_rows, n_cols,
    true_coordinates):
def sample_noisy_coordinates(
    n_rows, n_cols, answer_coordinates,
    strategy, n_samples,
    random_state):
def _parse_answer_interactions(
    table_json,
    descriptions,
    example,
    *,
    single_cell_examples = False,
    use_original_coordinates = True,
    noisy_cell_strategy = NoisyCellStrategy.NONE
):
def get_referred_annotation_keys(table):
def remove_unreferred_annotation_descriptions(
    interaction):
def _create_eval_answer_interaction(
    interaction,
    answer_table,
    i_table = 0):
def _create_eval_answer_interactions(
    interaction,
    predicted_coordinates,
    *,
    single_cell_examples = False
):
def _convert_split(
    split,
    main_path,
    table_jsons,
    descriptions_dict,
    config,
):
def create_interactions_from_hybridqa_predictions(
    output_dir,
):
def convert(
    input_dir,
    output_dir,
): _ZIP_NAME = 'WikiTables-WithLinks'
_TRANSLATION_TABLE = str.maketrans(string.punctuation,
                                   ' ' * len(string.punctuation))
_ARTICLES = ('a', 'an', 'the')  # List from HybridQA/evaluate_script.py
_RECALL_KS = (1, 3, 5)
  TRAIN = 'train'
  DEV = 'dev'
  TEST = 'test'
  ALL = 'all'
  ONE_IN_TABLE = '1_in_table'
  ONE_IN_TEXT = '1_in_text'
  MANY_IN_TABLE = '2+_in_table'
  MANY_IN_TEXT = '2+_in_text'
  TABLE_AND_TEXT = 'table_&_text'
  NO_ANSWER = 'no_answer'
class Split(enum.Enum):
class AnswerType(enum.Enum):
class CellSelectionMetrics:
def get_hybridqa_references(
    references_path):
def _get_answer_type(question):
def get_best_cells(
    probabilities,):
def get_cell_selection_metrics(
    interactions_file,
    predictions_file,
):
def eval_cell_selection(
    questions,
    predictions_file,
):
def _parse_cell(
    cell,
    text,
    links,
    descriptions,
):
def parse_table(
    json_dict,
    descriptions,
):
def _normalize(phrase):
def find_answer_coordinates(
    answer_text, table, desc_map
):
def find_dataset_coordinates(
    example_json_or_question
):
def _parse_interaction(
    table,
    descriptions,
    example,
    counters,
):
def extract_zip_archive(zip_path):
def read_json_directory(path):
def _convert_split(
    split, main_path, table_dict,
    descriptions_dict
):
def convert(
    input_dir): class SupervisionMode(enum.Enum):
  NONE = 0
  REMOVE_ALL = 2
  REMOVE_ALL_STRICT = 3
_CLEAR_FIELDS = frozendict.frozendict({
    SupervisionMode.REMOVE_ALL: [
        "answer_coordinates", "float_value", "aggregation_function"
    ],
    SupervisionMode.REMOVE_ALL_STRICT: [
        "answer_coordinates", "float_value", "aggregation_function"
    ]
})
def _find_matching_coordinates(table, answer_text,
                               normalize):
def _compute_cost_matrix_inner(
    table,
    answer,
    normalize,
    discard_ambiguous_examples,
):
def _compute_cost_matrix(
    table,
    answer,
    discard_ambiguous_examples,
):
def _parse_answer_coordinates(table,
                              answer,
                              discard_ambiguous_examples):
def _parse_answer_float(answer):
def _has_single_float_answer_equal_to(question,
                                      target):
def _parse_question(
    table,
    original_question,
    clear_fields,
    discard_ambiguous_examples,
):
def parse_question(table,
                   question, mode): _NS = "main"
  COLUMN = 1
  CELL = 2
def get_column_texts(
    table,
    column_index,
):
def _get_column_cost(
    tokenizer,
    table,
    column_index,
):
def _get_question_cost(
    tokenizer,
    question,
):
def get_cleaned_seq_tokens(str_tokens):
def from_selected_columns_to_selected_tokens(
    interaction,
    selected_columns,
    tokenizer):
  def _add_cell_tokens(column_index, row_index, cell_text, selected_tokens):
def _get_question_column_similarity(column_tokens,
                                    question_tokens,
                                    epsilon = 0.0):
  def __init__(self, vocab_file, use_previous_questions):
  def prepare(self):
  def select_tokens(
      self,
      interaction,
      question,
  ):
  def annotated_interaction(
      self,
      interaction,
  ):
  def __init__(self, vocab_file, max_nb_tokens,
               selection_level, use_previous_answer,
               use_previous_questions):
  def _get_column_tokens(self, interaction,
                         column_index):
  def _get_columns_tokens(
      self, interaction):
  def _get_columns_similarity_scores(
      self, interaction,
      question_tokens):
  def _get_headers_tokens(
      self, interaction):
  def _get_headers_similarity_scores(
      self, interaction,
      question_tokens):
  def _get_cells(
      self, interaction
  ):
  def _get_row_tokens(self, interaction,
                      row_index):
  def _get_header_tokens(
      self, interaction):
  def _get_row_similarity_scores(
      self, interaction,
      question_tokens):
  def _update_priority_queue_from_previous_answer(
      self,
      selection_type,
      priority_queue,
      interaction,
      question,
  ):
  def _get_columns_from_priority_queue(
      self,
      columns_queue,
      interaction,
      question,
  ):
  def _get_tokens_from_priority_queue(
      self,
      tokens_queue,
      interaction,
      question,
  ):
  def _select_columns(
      self,
      interaction,
      question,
  ):
  def _select_cells(
      self,
      interaction,
      question,
  ):
  def _select_columns(
      self,
      interaction,
      question,
  ):
  def _select_cells(
      self,
      interaction,
      question,
  ):
  def select_tokens(self, interaction,
                    question):
class SelectionType(enum.Enum):
class TokenScoreDebugInfo:
class TokenSelection:
class TokenSelector:
class MaxTokenSelector(TokenSelector):
class HeuristicExactMatchTokenSelector(MaxTokenSelector):
class SelectAllTokensFn(TokenSelector): class Type(enum.Enum):
class Version(enum.Enum):
  V1 = 1
  V2 = 2
def _format(text):
def _to_type_index(type_name):
def _index_to_type(index):
def _fill_proto_table(
    doc_id,
    table,
    table_proto,
    version = Version.V2,
):
def _process_evidence(
    statement_id,
    cell,
    evidence,
    table,
    question,
):
def _process_table(
    doc_id,
    table,
    version = Version.V2,
):
def _process_doc(
    filepath,
    version = Version.V2,
):
def _process_docs(
    filepaths,
    name,
    version = Version.V2,
):
def convert(
    input_dir,
    skip_auto_train = True,
    version = Version.V2,
):
def _get_doc_id(filepath):
def _get_table_id(doc_id, table_id):
def _get_interaction_id(table_id, statement_id):
def _get_question_id(interaction_id):
def _get_majority_label(prediction_list):
def create_outputs(
    input_dir,
    prediction_files,
    output_dir,
): def _read_interactions(input_dir):
def _add_tables(input_dir,
                interaction_dict):
def _write_report(
    report_filename,
    supervision_modes,
    counters,
):
def _parse_questions(interaction_dict,
                     supervision_modes,
                     report_filename):
def _write_tfrecord(
    interactions,
    filepath,
    token_selector,
):
def _get_output_filename(output_dir, input_file):
def create_interactions(
    supervision_modes,
    input_dir,
    output_dir,
    token_selector,
): _TABLE_DIR_NAME = 'table_csv'  # Name that the table folder has in SQA.
FEATURE_ALL = 'ALL'
FEATURE_TRUE = 'TRUE'
FEATURE_FALSE = 'FALSE'
FEATURE_SUPERLATIVE = 'SUPER'
FEATURE_COMPARATIVE = 'COMP'
FEATURE_NEGATIVE = 'NEG'
FEATURE_PERCENTAGE = 'PERC'
FEATURE_AGGREGATION = 'AGG'
_SUPERLATIVE_WORDS = {
    'lowest', 'highest', 'best', 'worst', 'least', 'most', 'oldest', 'newest',
    'fewest', 'last', 'first', 'earliest', 'latest', 'greatest', 'smallest'
}
_COMPARATIVE_WORDS = {
    'than', 'less', 'more', 'better', 'worse', 'higher', 'lower', 'shorter',
    'same'
}
_NEGATIVE_WORDS = {'not', 'any', 'none', 'no', 'never'}
_AGGREGATION_WORDS = {
    'only', 'there', 'total', 'count', 'average', 'sum', 'amount'
}
_PERCENTAGE_RE = re.compile(r'\d[\d.]* ?%')
_DEFAULT_FEATURE_LIST = [
    FEATURE_TRUE, FEATURE_FALSE, FEATURE_SUPERLATIVE, FEATURE_COMPARATIVE,
    FEATURE_NEGATIVE, FEATURE_PERCENTAGE, FEATURE_AGGREGATION
]
def get_features_single(question, label):
def get_features(questions):
def get_interesting_features(accuracy,
                             heardoom,
                             num_features):
  def _get_feature_score(feature):
def _log_stats(counter, file_name):
def _convert_data(
    all_questions,
    input_file,
    tables,
):
def read_all_tables(input_dir):
def _convert_table(table_id, table_text):
def _convert_tables(input_dir):
def read_questions(input_dir):
def convert(
    input_dir,): class Task(enum.Enum):
  SQA = 0
  WTQ = 1
  WIKISQL = 2
  WIKISQL_SUPERVISED = 3
  TABFACT = 4
  NQ_RETRIEVAL = 7
  HYBRIDQA_RC = 8  # Reading comprehension (RC) of HybridQA.
  HYBRIDQA_E2E = 9  # HybridQA RC using preds of HybridQA Cell Selection.
  HYBRIDQA = 6
  SEM_TAB_FACT = 10  # https://sites.google.com/corp/view/sem-tab-facts _TABLE_DIR_NAME = 'table_csv'  # Name that the table folder has in SQA.
_TYPE_CONVERTER = {
    'text': lambda x: x,
    'real': text_utils.convert_to_float,
}
_TOKENIZER = re.compile(r'\w+|[^\w\s]+', re.UNICODE | re.MULTILINE | re.DOTALL)
_DATASETS = ['train', 'test', 'dev']
_NAN = float('nan')
  NONE = 0
  MAX = 1
  MIN = 2
  COUNT = 3
  SUM = 4
  AVERAGE = 5
  EQUALS = 0
  GREATER = 1
  LESSER = 2
class _Aggregation(enum.Enum):
class _Operator(enum.Enum):
class _Condition:
def _iterate_jsonl_file(
    input_dir,
    suffix):
def _convert_tables(input_dir, output_dir):
def _parse_value(table, column,
                 cell_value):
def _parse_table(table):
def _load_tables(input_dir):
def _compare(operator, src, tgt):
def _is_string(x):
def _normalize_for_match(x):
def _respect_conditions(table, row,
                        conditions):
def _get_answer_coordinates(
    table,
    example):
def _get_float_answer(table,
                      answer_coordinates,
                      aggregation_op):
def _get_aggregation_name(aggregation):
def _get_answer_text(table,
                     answer_coordinates,
                     float_answer):
def _convert_questions(input_dir, output_dir):
def _create_dirs(output_dir):
def convert(input_dir, output_dir): _TABLE_DIR_NAME = 'table_csv'  # Name that the table folder has in SQA.
  V_02 = 1
  V_10 = 2
class Version(enum.Enum):
def _get_train_test(
    split_number,
    version,
):
def _get_random_split_name(
    split_number,
    version,
):
def _get_reader(file_path):
def _get_sqa_file_path(input_dir, file_name):
def _get_sqa_table_id(wtq_table_id):
def _read_wtq_table(input_dir, wtq_table_id):
def _export_table(table, output_dir,
                  sqa_table_id):
def _log_stats(counter, file_name):
def _iterate_examples(
    file_in,
    version,
):
def _convert_data(
    table_cache,
    input_dir,
    output_dir,
    file_name,
    version,
):
def _create_dirs(output_dir):
def convert(
    input_dir,
    output_dir,
    version = Version.V_10,
):
import collections
import os
import tensorflow.compat.v1 as tf
from typing import Iterable, Mapping, Optional, Text
from tapas.protos import interaction_pb2
from tapas.utils import hybridqa_rc_utils
from tapas.utils import hybridqa_utils
from tapas.utils import interaction_utils_parser
from tapas.utils import pruning_utils
from tapas.utils import sem_tab_fact_utils
from tapas.utils import sqa_utils
from tapas.utils import tabfact_utils
from tapas.utils import tasks
from tapas.utils import wikisql_utils
from tapas.utils import wtq_utils
# coding=utf-8
# Copyright 2019 The Google AI Language Team Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Lint as: python3
"""Utilities for data conversions for different tasks / datasets."""



_Mode = interaction_utils_parser.SupervisionMode


def get_interaction_dir(output_dir):
  return os.path.join(output_dir, 'interactions')


def _to_tfrecord(
    interactions,
    output_dir,
    token_selector,
):
  """Dump interactions to TFRecord files."""
  interactions_dir = get_interaction_dir(output_dir)
  tf.io.gfile.makedirs(interactions_dir)
  for name, examples in interactions.items():
    tfrecord_file = os.path.join(interactions_dir, f'{name}.tfrecord')
    with tf.io.TFRecordWriter(tfrecord_file) as writer:
      for interaction in examples:
        if token_selector is not None:
          interaction = token_selector.annotated_interaction(interaction)
        writer.write(interaction.SerializeToString())


def get_train_filename(task):
  """Gets the name of the file with training data."""
  if task in [

      tasks.Task.WIKISQL,