indexing-service/src/main/java/org/apache/druid/indexing/input/InputRowSchemas.java
/**
   * Build a {@link ColumnsFilter} that can filter down the list of columns that must be read after flattening.
   *
   * @see InputRowSchema#getColumnsFilter()
   */
Build a  ColumnsFilter that can filter down the list of columns that must be read after flattening.
Build a ColumnsFilter that can filter down the list of columns that must be read after flattening .
@VisibleForTesting
  static ColumnsFilter createColumnsFilter(
      final TimestampSpec timestampSpec,
      final DimensionsSpec dimensionsSpec,
      final TransformSpec transformSpec,
      final AggregatorFactory[] aggregators
  )
  {
    final Set<String> transformOutputNames =
        transformSpec.getTransforms().stream().map(Transform::getName).collect(Collectors.toSet());
    if (dimensionsSpec.hasCustomDimensions()) {
      final Set<String> inclusions = new HashSet<>();
      inclusions.add(timestampSpec.getTimestampColumn());
      inclusions.addAll(transformSpec.getRequiredColumns());
      for (String column : dimensionsSpec.getDimensionNames()) {
        if (!transformOutputNames.contains(column)) {
          inclusions.add(column);
        }
      }
      for (AggregatorFactory aggregator : aggregators) {
        for (String column : aggregator.requiredFields()) {
          if (!transformOutputNames.contains(column)) {
            inclusions.add(column);
          }
        }
      }
      return ColumnsFilter.inclusionBased(inclusions);
    } else {
      final Set<String> exclusions = new HashSet<>(dimensionsSpec.getDimensionExclusions());
      exclusions.remove(timestampSpec.getTimestampColumn());
      exclusions.removeAll(transformSpec.getRequiredColumns());
      for (AggregatorFactory aggregator : aggregators) {
        for (String column : aggregator.requiredFields()) {
          if (!transformOutputNames.contains(column)) {
            exclusions.remove(column);
          }
        }
      }
      return ColumnsFilter.exclusionBased(exclusions);
    }
  }
@ VisibleForTesting static ColumnsFilter createColumnsFilter ( final TimestampSpec timestampSpec , final DimensionsSpec dimensionsSpec , final TransformSpec transformSpec , final AggregatorFactory [ ] aggregators ) { final Set < String > transformOutputNames = transformSpec . getTransforms ( ) . stream ( ) . map ( Transform :: getName ) . collect ( Collectors . toSet ( ) ) ; if ( dimensionsSpec . hasCustomDimensions ( ) ) { final Set < String > inclusions = new HashSet < > ( ) ; inclusions . add ( timestampSpec . getTimestampColumn ( ) ) ; inclusions . addAll ( transformSpec . getRequiredColumns ( ) ) ; for ( String column : dimensionsSpec . getDimensionNames ( ) ) { if ( ! transformOutputNames . contains ( column ) ) { inclusions . add ( column ) ; } } for ( AggregatorFactory aggregator : aggregators ) { for ( String column : aggregator . requiredFields ( ) ) { if ( ! transformOutputNames . contains ( column ) ) { inclusions . add ( column ) ; } } } return ColumnsFilter . inclusionBased ( inclusions ) ; } else { final Set < String > exclusions = new HashSet < > ( dimensionsSpec . getDimensionExclusions ( ) ) ; exclusions . remove ( timestampSpec . getTimestampColumn ( ) ) ; exclusions . removeAll ( transformSpec . getRequiredColumns ( ) ) ; for ( AggregatorFactory aggregator : aggregators ) { for ( String column : aggregator . requiredFields ( ) ) { if ( ! transformOutputNames . contains ( column ) ) { exclusions . remove ( column ) ; } } } return ColumnsFilter . exclusionBased ( exclusions ) ; } }
Build a {@link ColumnsFilter} that can filter down the list of columns that must be read after flattening.
Build a { @link ColumnsFilter } that can filter down the list of columns that must be read after flattening .
// We'll need to know what fields are generated from transforms, vs. expected from the raw data. // We need an inclusion-based filter. // Add timestamp column. // Add all transform inputs. // Add all dimension inputs that are *not* transform outputs. // Add all aggregator inputs that are *not* transform outputs. // Schemaless dimensions mode: we need an exclusion-based filter. // Start from the list of dimension exclusions. // Remove (un-exclude) timestamp column. // Remove (un-exclude) all transform inputs. // Remove (un-exclude) all aggregator inputs that are *not* transform outputs.