A utility function for reading in text2sql data. The blob is
    the result of loading the json from a file produced by the script
    ``scripts/reformat_text2sql_data.py``.

    Parameters
    ----------
    data : ``JsonDict``
    use_all_sql : ``bool``, optional (default = False)
        Whether to use all of the sql queries which have identical semantics,
        or whether to just use the first one.
    use_all_queries : ``bool``, (default = False)
        Whether or not to enforce query sentence uniqueness. If false,
        duplicated queries will occur in the dataset as separate instances,
        as for a given SQL query, not only are there multiple queries with
        the same template, but there are also duplicate queries.
    remove_unneeded_aliases : ``bool``, (default = False)
        The text2sql data by default creates alias names for `all` tables,
        regardless of whether the table is derived or if it is identical to
        the original (e.g SELECT TABLEalias0.COLUMN FROM TABLE AS TABLEalias0).
        This is not necessary and makes the action sequence and grammar manipulation
        much harder in a grammar based decoder. Note that this does not
        remove aliases which are legitimately required, such as when a new
        table is formed by performing operations on the original table.
    schema : ``Dict[str, List[TableColumn]]``, optional, (default = None)
        A schema to resolve primary keys against. Converts 'ID' column names
        to their actual name with respect to the Primary Key for the table
        in the schema.

def process_sql_data(data: List[JsonDict],
                     use_all_sql: bool = False,
                     use_all_queries: bool = False,
                     remove_unneeded_aliases: bool = False,
                     schema: Dict[str, List[TableColumn]] = None) -> Iterable[SqlData]:
    """
    A utility function for reading in text2sql data. The blob is
    the result of loading the json from a file produced by the script
    ``scripts/reformat_text2sql_data.py``.

    Parameters
    ----------
    data : ``JsonDict``
    use_all_sql : ``bool``, optional (default = False)
        Whether to use all of the sql queries which have identical semantics,
        or whether to just use the first one.
    use_all_queries : ``bool``, (default = False)
        Whether or not to enforce query sentence uniqueness. If false,
        duplicated queries will occur in the dataset as separate instances,
        as for a given SQL query, not only are there multiple queries with
        the same template, but there are also duplicate queries.
    remove_unneeded_aliases : ``bool``, (default = False)
        The text2sql data by default creates alias names for `all` tables,
        regardless of whether the table is derived or if it is identical to
        the original (e.g SELECT TABLEalias0.COLUMN FROM TABLE AS TABLEalias0).
        This is not necessary and makes the action sequence and grammar manipulation
        much harder in a grammar based decoder. Note that this does not
        remove aliases which are legitimately required, such as when a new
        table is formed by performing operations on the original table.
    schema : ``Dict[str, List[TableColumn]]``, optional, (default = None)
        A schema to resolve primary keys against. Converts 'ID' column names
        to their actual name with respect to the Primary Key for the table
        in the schema.
    """
    for example in data:
        seen_sentences: Set[str] = set()
        for sent_info in example['sentences']:
            # Loop over the different sql statements with "equivalent" semantics
            for sql in example["sql"]:
                text_with_variables = sent_info['text'].strip().split()
                text_vars = sent_info['variables']

                query_tokens, tags = replace_variables(text_with_variables, text_vars)
                if not use_all_queries:
                    key = " ".join(query_tokens)
                    if key in seen_sentences:
                        continue
                    else:
                        seen_sentences.add(key)

                sql_tokens = clean_and_split_sql(sql)
                if remove_unneeded_aliases:
                    sql_tokens = clean_unneeded_aliases(sql_tokens)
                if schema is not None:
                    sql_tokens = resolve_primary_keys_in_schema(sql_tokens, schema)

                sql_variables = {}
                for variable in example['variables']:
                    sql_variables[variable['name']] = {'text': variable['example'], 'type': variable['type']}

                sql_data = SqlData(text=query_tokens,
                                   text_with_variables=text_with_variables,
                                   variable_tags=tags,
                                   sql=sql_tokens,
                                   text_variables=text_vars,
                                   sql_variables=sql_variables)
                yield sql_data

                # Some questions might have multiple equivalent SQL statements.
                # By default, we just use the first one. TODO(Mark): Use the shortest?
                if not use_all_sql:
                    break