public List<JsonMessage> processDatasets(List<String> datasetIncludeList,
                                             List<String> datasetExcludeList,
                                             List<String> tableExcludeList,
                                             String dataRegionId) throws IOException, InterruptedException, NonRetryableApplicationException {
        List<String> tablesIncludeList = new ArrayList<>();
        for (String dataset : datasetIncludeList) {
            try {
                if (!datasetExcludeList.contains(dataset)) {
                    List<String> tokens = Utils.tokenize(dataset, ".", true);
                    String projectId = tokens.get(0);
                    String datasetId = tokens.get(1);
                    String datasetLocation = bqService.getDatasetLocation(projectId, datasetId);
                /*
                 TODO: Support tagging in multiple locations
                 to support all locations:
                 1- Taxonomies/PolicyTags have to be created in each required location
                 2- Update the Tagger Cloud Function to read one mapping per location
                 For now, we don't submit tasks for tables in other locations than the PolicyTag location
                 */
                    if (!datasetLocation.toLowerCase().equals(dataRegionId.toLowerCase())) {
                        logger.logWarnWithTracker(runId,
                                String.format(
                                        "Ignoring dataset %s in location %s. Only location %s is configured",
                                        dataset,
                                        datasetLocation,
                                        dataRegionId)
                        );
                        continue;
                    }
                    // get all tables that have DLP findings
                    List<String> datasetTables = scanner.listChildren(projectId, datasetId);
                    tablesIncludeList.addAll(datasetTables);
                    if (datasetTables.isEmpty()) {
                        String msg = String.format(
                                "No Tables found under dataset '%s'",
                                dataset);
                        logger.logWarnWithTracker(runId, msg);
                    } else {
                        logger.logInfoWithTracker(runId, String.format("Tables found in dataset %s : %s", dataset, datasetTables));
                    }
                }
            } catch (Exception exception) {
                // log and continue
                logger.logFailedDispatcherEntityId(runId, dataset, exception);
            }
        }
        return processTables(tablesIncludeList, tableExcludeList);
    }
public List<JsonMessage> processDatasets(List<String> datasetIncludeList,
                                             List<String> datasetExcludeList,
                                             List<String> tableExcludeList,
                                             String dataRegionId) throws IOException, InterruptedException, NonRetryableApplicationException {
        List<String> tablesIncludeList = new ArrayList<>();
        for (String dataset : datasetIncludeList) {
            try {
                if (!datasetExcludeList.contains(dataset)) {
                    List<String> tokens = Utils.tokenize(dataset, ".", true);
                    String projectId = tokens.get(0);
                    String datasetId = tokens.get(1);
                    String datasetLocation = bqService.getDatasetLocation(projectId, datasetId);
               
                    if (!datasetLocation.toLowerCase().equals(dataRegionId.toLowerCase())) {
                        logger.logWarnWithTracker(runId,
                                String.format(
                                        "Ignoring dataset %s in location %s. Only location %s is configured",
                                        dataset,
                                        datasetLocation,
                                        dataRegionId)
                        );
                        continue;
                    }
                   
                    List<String> datasetTables = scanner.listChildren(projectId, datasetId);
                    tablesIncludeList.addAll(datasetTables);
                    if (datasetTables.isEmpty()) {
                        String msg = String.format(
                                "No Tables found under dataset '%s'",
                                dataset);
                        logger.logWarnWithTracker(runId, msg);
                    } else {
                        logger.logInfoWithTracker(runId, String.format("Tables found in dataset %s : %s", dataset, datasetTables));
                    }
                }
            } catch (Exception exception) {
               
                logger.logFailedDispatcherEntityId(runId, dataset, exception);
            }
        }
        return processTables(tablesIncludeList, tableExcludeList);
    }
public list<jsonmessage> processdatasets(list<string> datasetincludelist, list<string> datasetexcludelist, list<string> tableexcludelist, string dataregionid) throws ioexception, interruptedexception, nonretryableapplicationexception { list<string> tablesincludelist = new arraylist<>(); for (string dataset : datasetincludelist) { try { if (!datasetexcludelist.contains(dataset)) { list<string> tokens = utils.tokenize(dataset, ".", true); string projectid = tokens.get(0); string datasetid = tokens.get(1); string datasetlocation = bqservice.getdatasetlocation(projectid, datasetid); if (!datasetlocation.tolowercase().equals(dataregionid.tolowercase())) { logger.logwarnwithtracker(runid, string.format( "ignoring dataset %s in location %s. only location %s is configured", dataset, datasetlocation, dataregionid) ); continue; } list<string> datasettables = scanner.listchildren(projectid, datasetid); tablesincludelist.addall(datasettables); if (datasettables.isempty()) { string msg = string.format( "no tables found under dataset '%s'", dataset); logger.logwarnwithtracker(runid, msg); } else { logger.loginfowithtracker(runid, string.format("tables found in dataset %s : %s", dataset, datasettables)); } } } catch (exception exception) { logger.logfaileddispatcherentityid(runid, dataset, exception); } } return processtables(tablesincludelist, tableexcludelist); }