Update the frontier for all given files, up to the given revision.

        Built for quick continuous _forward_ updating of large sets
        of files of TUIDs. Backward updating should be done through
        get_tuids(files, revision). If we cannot find a frontier, we will
        stop looking after max_csets_proc and update all files at the given
        revision.

        :param frontier_list: list of files to update
        :param revision: revision to update files to
        :param max_csets_proc: maximum number of changeset logs to look through
                               to find past frontiers.
        :param going_forward: If we know the requested revision is in front
                              of the latest revision use this flag. Used when
                              the frontier is too far away. If this is not set and
                              a frontier is too far, the latest revision will not
                              be updated.
        :return: list of (file, list(tuids)) tuples

def _update_file_frontiers(
            self,
            frontier_list,
            revision,
            max_csets_proc=30,
            going_forward=False
        ):
        '''
        Update the frontier for all given files, up to the given revision.

        Built for quick continuous _forward_ updating of large sets
        of files of TUIDs. Backward updating should be done through
        get_tuids(files, revision). If we cannot find a frontier, we will
        stop looking after max_csets_proc and update all files at the given
        revision.

        :param frontier_list: list of files to update
        :param revision: revision to update files to
        :param max_csets_proc: maximum number of changeset logs to look through
                               to find past frontiers.
        :param going_forward: If we know the requested revision is in front
                              of the latest revision use this flag. Used when
                              the frontier is too far away. If this is not set and
                              a frontier is too far, the latest revision will not
                              be updated.
        :return: list of (file, list(tuids)) tuples
        '''

        # Get the changelogs and revisions until we find the
        # last one we've seen, and get the modified files in
        # each one.

        # Holds the files modified up to the last frontiers.
        files_to_process = {}

        # Holds all frontiers to find
        remaining_frontiers = {cset for cset in list(set([frontier for _, frontier in frontier_list]))}

        if len(remaining_frontiers) <= 1 and frontier_list[0][1] == revision:
            # If the latest revision is the requested revision,
            # and there is only one frontier requested
            # continue to the tuid querys.
            remaining_frontiers = {}

        # Revision we are searching from
        final_rev = revision

        # If this exceeds max_csets_proc,
        # all frontiers not found are considered lost
        csets_proced = 0

        # Holds info on how to apply the diffs onto each frontier,
        # and all known frontiers.
        diffs_to_frontier = {cset: [] for cset in remaining_frontiers}

        Log.note("Searching for frontier(s): {{frontier}} ", frontier=str(list(remaining_frontiers)))
        Log.note(
            "Running on revision with HG URL: {{url}}",
            url=self.hg_url / self.config.hg.branch / 'rev' / revision
        )
        while remaining_frontiers:
            # Get a changelog
            clog_url = self.hg_url / self.config.hg.branch / 'json-log' / final_rev
            try:
                Log.note("Searching through changelog {{url}}", url=clog_url)
                clog_obj = http.get_json(clog_url, retry=RETRY)
                if isinstance(clog_obj, (text_type, str)):
                    Log.error(
                        "Revision {{cset}} does not exist in the {{branch}} branch",
                        cset=final_rev, branch=self.config.hg.branch
                    )
            except Exception as e:
                Log.error(
                    "Unexpected error getting changset-log for {{url}}: {{error}}",
                    url=clog_url,
                    error=e
                )

            # For each changeset in the log (except the last one
            # which is duplicated on the next log page requested.
            clog_obj_list = list(clog_obj['changesets'])
            for clog_cset in clog_obj_list[:-1]:
                nodes_cset = clog_cset['node'][:12]

                if remaining_frontiers:
                    if nodes_cset in remaining_frontiers:
                        # Found a frontier, remove it from search list.
                        remaining_frontiers.remove(nodes_cset)

                        if not remaining_frontiers:
                            # Found all frontiers, get out of the loop before
                            # we add the diff to a frontier update list.
                            break

                    # Add this diff to the processing list
                    # for each remaining frontier
                    for cset in diffs_to_frontier:
                        if cset in remaining_frontiers:
                            diffs_to_frontier[cset].append(nodes_cset)

            csets_proced += 1
            if not remaining_frontiers:
                # End searching
                break
            elif csets_proced >= max_csets_proc:
                # In this case, all files need to be updated to this revision to ensure
                # line ordering consistency (between past, and future) when a revision
                # that is in the past is asked for.
                files_to_process = {file: [revision] for file, _ in frontier_list}
                break
            else:
                # Go to the next log page
                last_entry = clog_obj_list[-1]
                final_rev = last_entry['node'][:12]

        if not remaining_frontiers:
            Log.note("Found all frontiers: {{frontiers_list}}", frontiers_list=str(list(diffs_to_frontier.keys())))
        else:
            found_frontiers = [
                frontier for frontier in diffs_to_frontier if frontier not in remaining_frontiers
            ]
            Log.note("Found frontiers: {{found}}", found=str(found_frontiers))
            Log.note("Did not find frontiers: {{not_found}}", not_found=str(list(remaining_frontiers)))

        added_files = {}
        removed_files = {}
        parsed_diffs = {}

        # This list is used to determine what files
        file_to_frontier = {file: frontier for file, frontier in frontier_list}
        if len(remaining_frontiers) != len(diffs_to_frontier.keys()):
            # If there is at least one frontier that was found
            # Only get diffs that are needed (if any frontiers were not found)
            diffs_cache = []
            for cset in diffs_to_frontier:
                if cset not in remaining_frontiers:
                    diffs_cache.extend(diffs_to_frontier[cset])

            Log.note("Gathering diffs for: {{csets}}", csets=str(diffs_cache))
            all_diffs = self.get_diffs(diffs_cache)

            # Build a dict for faster access to the diffs,
            # to be used later when applying them.
            parsed_diffs = {diff_entry['cset']: diff_entry['diff'] for diff_entry in all_diffs}

            # In case the file name changes, this will map
            # the requested file to the new file name so
            # diffs can all be gathered.
            filenames_to_seek = {}

            # Parse diffs for files to process and store diffs to
            # apply for each file in files_to_process.
            added_and_removed_counts = {file: 1 for file in file_to_frontier}
            for csets_diff in all_diffs:
                cset_len12 = csets_diff['cset']
                parsed_diff = csets_diff['diff']['diffs']

                for f_added in parsed_diff:
                    # Get new entries for removed files.
                    new_name = f_added['new'].name.lstrip('/')
                    old_name = f_added['old'].name.lstrip('/')

                    # If we don't need this file, skip it
                    if new_name not in file_to_frontier and \
                       new_name not in filenames_to_seek:
                        if old_name not in file_to_frontier and \
                           old_name not in filenames_to_seek:
                            # File not requested
                            continue
                        if new_name == 'dev/null':
                            frontier_filename = old_name
                            while frontier_filename in filenames_to_seek:
                                frontier_filename = filenames_to_seek[frontier_filename]

                            if frontier_filename not in removed_files:
                                removed_files[frontier_filename] = 0
                            removed_files[frontier_filename] = added_and_removed_counts[frontier_filename]
                            added_and_removed_counts[frontier_filename] += 1
                            continue

                    if old_name == 'dev/null':
                        frontier_filename = new_name
                        while frontier_filename in filenames_to_seek:
                            frontier_filename = filenames_to_seek[frontier_filename]

                        if frontier_filename not in added_files:
                            added_files[frontier_filename] = 0
                        added_files[frontier_filename] = added_and_removed_counts[frontier_filename]
                        added_and_removed_counts[frontier_filename] += 1
                        continue
                    if new_name != old_name:
                        # File name was changed, keep the diff anyway
                        # to add any changes it makes.
                        filenames_to_seek[new_name] = old_name

                    # Get the originally requested file name
                    # by following filenames_to_seek entries
                    frontier_filename = new_name
                    while frontier_filename in filenames_to_seek:
                        frontier_filename = filenames_to_seek[frontier_filename]

                    # If we are past the frontier for this file,
                    # or if we are at the frontier skip it.
                    if file_to_frontier[frontier_filename] == '':
                        # Previously found frontier, skip
                        continue

                    # At this point, file is in the database, is
                    # asked to be processed, and we are still
                    # searching for the last frontier.
                    if file_to_frontier[frontier_filename] == cset_len12:
                        file_to_frontier[frontier_filename] = ''
                        # Found the frontier, skip
                        continue

                    if old_name != new_name:
                        Log.note(
                            "{{cset}} changes a requested file's name: {{file}} from {{oldfile}}. ",
                            file=new_name,
                            oldfile=old_name,
                            cset=cset
                        )

                    # Store the diff as it needs to be applied
                    if frontier_filename in files_to_process:
                        files_to_process[frontier_filename].append(cset_len12)
                    else:
                        files_to_process[frontier_filename] = [cset_len12]

        # Process each file that needs it based on the
        # files_to_process list.
        result = []
        ann_inserts = []
        latestFileMod_inserts = {}
        anns_to_get = []
        total = len(frontier_list)
        tmp_results = {}

        with self.conn.transaction() as transaction:
            for count, (file, old_frontier) in enumerate(frontier_list):
                if old_frontier in remaining_frontiers:
                    # If we were still looking for the frontier by the end, get a new
                    # annotation for this file.
                    anns_to_get.append(file)

                    if going_forward:
                        # If we are always going forward, update the frontier
                        latestFileMod_inserts[file] = (file, revision)

                    Log.note(
                        "Frontier update - can't find frontier {{lost_frontier}}: "
                        "{{count}}/{{total}} - {{percent|percent(decimal=0)}} | {{rev}}|{{file}} ",
                        count=count,
                        total=total,
                        file=file,
                        rev=revision,
                        percent=count / total,
                        lost_frontier=old_frontier
                    )
                    continue
                elif file in removed_files or file in added_files:
                    if file not in removed_files:
                        removed_files[file] = 0
                    if file not in added_files:
                        added_files[file] = 0

                    if removed_files[file] <= added_files[file]:
                        # For it to still exist it has to be
                        # added last (to give it a larger count)
                        anns_to_get.append(file)
                        Log.note(
                            "Frontier update - adding: "
                            "{{count}}/{{total}} - {{percent|percent(decimal=0)}} | {{rev}}|{{file}} ",
                            count=count,
                            total=total,
                            file=file,
                            rev=revision,
                            percent=count / total,
                            lost_frontier=old_frontier
                        )
                    else:
                        Log.note(
                            "Frontier update - deleting: "
                            "{{count}}/{{total}} - {{percent|percent(decimal=0)}} | {{rev}}|{{file}} ",
                            count=count,
                            total=total,
                            file=file,
                            rev=revision,
                            percent=count / total,
                            lost_frontier=old_frontier
                        )
                        tmp_results[file] = []
                    if going_forward:
                        # If we are always going forward, update the frontier
                        latestFileMod_inserts[file] = (file, revision)

                    continue

                # If the file was modified, get it's newest
                # annotation and update the file.
                tmp_res = None
                if file in files_to_process:
                    # Process this file using the diffs found
                    tmp_ann = self._get_annotation(old_frontier, file, transaction)
                    if tmp_ann is None or tmp_ann == '' or self.destringify_tuids(tmp_ann) is None:
                        Log.warning(
                            "{{file}} has frontier but can't find old annotation for it in {{rev}}, "
                            "restarting it's frontier.",
                            rev=old_frontier,
                            file=file
                        )
                        anns_to_get.append(file)
                    else:
                        # File was modified, apply it's diffs
                        # Reverse the diff list, we always find the newest diff first
                        csets_to_proc = files_to_process[file][::-1]
                        tmp_res = self.destringify_tuids(tmp_ann)
                        new_fname = file
                        for i in csets_to_proc:
                            tmp_res, new_fname = self._apply_diff(transaction, tmp_res, parsed_diffs[i], i, new_fname)

                        ann_inserts.append((revision, file, self.stringify_tuids(tmp_res)))
                        Log.note(
                            "Frontier update - modified: {{count}}/{{total}} - {{percent|percent(decimal=0)}} "
                            "| {{rev}}|{{file}} ",
                            count=count,
                            total=total,
                            file=file,
                            rev=revision,
                            percent=count / total
                        )
                else:
                    old_ann = self._get_annotation(old_frontier, file, transaction)
                    if old_ann is None or (old_ann == '' and file in added_files):
                        # File is new (likely from an error), or re-added - we need to create
                        # a new initial entry for this file.
                        anns_to_get.append(file)
                        Log.note(
                            "Frontier update - readded: {{count}}/{{total}} - {{percent|percent(decimal=0)}} "
                            "| {{rev}}|{{file}} ",
                            count=count,
                            total=total,
                            file=file,
                            rev=revision,
                            percent=count / total
                        )
                    else:
                        # File was not modified since last
                        # known revision
                        tmp_res = self.destringify_tuids(old_ann) if old_ann != '' else []
                        ann_inserts.append((revision, file, old_ann))
                        Log.note(
                            "Frontier update - not modified: {{count}}/{{total}} - {{percent|percent(decimal=0)}} "
                            "| {{rev}}|{{file}} ",
                            count=count,
                            total=total,
                            file=file,
                            rev=revision,
                            percent=count / total
                        )

                if tmp_res:
                    tmp_results[file] = tmp_res
                else:
                    Log.note(
                        "Error occured for file {{file}} in revision {{revision}}",
                        file=file,
                        revision=revision
                    )
                    tmp_results[file] = []

                # If we have found all frontiers, update to the
                # latest revision. Otherwise, the requested
                # revision is too far away (can't be sure
                # if it's past). Unless we are told that we are
                # going forward.
                if going_forward or not remaining_frontiers:
                    latest_rev = revision
                else:
                    latest_rev = old_frontier
                latestFileMod_inserts[file] = (file, latest_rev)

            Log.note("Updating DB tables `latestFileMod` and `annotations`...")

            # No need to double-check if latesteFileMods has been updated before,
            # we perform an insert or replace any way.
            if len(latestFileMod_inserts) > 0:
                for _, inserts_list in jx.groupby(latestFileMod_inserts.values(), size=SQL_BATCH_SIZE):
                    transaction.execute(
                        "INSERT OR REPLACE INTO latestFileMod (file, revision) VALUES " +
                        sql_list(quote_list(i) for i in inserts_list)
                    )

            anns_added_by_other_thread = {}
            if len(ann_inserts) > 0:
                ann_inserts = list(set(ann_inserts))
                for _, tmp_inserts in jx.groupby(ann_inserts, size=SQL_ANN_BATCH_SIZE):
                    # Check if any were added in the mean time by another thread
                    recomputed_inserts = []
                    for rev, filename, string_tuids in tmp_inserts:
                        tmp_ann = self._get_annotation(rev, filename, transaction)
                        if not tmp_ann or tmp_ann == '':
                            recomputed_inserts.append((rev, filename, string_tuids))
                        else:
                            anns_added_by_other_thread[filename] = self.destringify_tuids(tmp_ann)

                    if len(recomputed_inserts) <= 0:
                        continue

                    try:
                        for rev, filename, tuids_ann in recomputed_inserts:
                            tmp_ann = self.destringify_tuids(tuids_ann)
                            for tuid_map in tmp_ann:
                                if tuid_map is None or tuid_map.tuid is None or tuid_map.line is None:
                                    Log.warning(
                                        "None value encountered in annotation insertion in {{rev}} for {{file}}: {{tuids}}" ,
                                        rev=rev, file=filename, tuids=str(tuid_map)
                                    )
                        self.insert_annotations(transaction, recomputed_inserts)
                    except Exception as e:
                        Log.error("Error inserting into annotations table: {{inserting}}", inserting=recomputed_inserts, cause=e)

        if len(anns_to_get) > 0:
            result.extend(self.get_tuids(anns_to_get, revision, commit=False))

        for f in tmp_results:
            tuids = tmp_results[f]
            if f in anns_added_by_other_thread:
                tuids = anns_added_by_other_thread[f]
            result.append((f, tuids))
        return result