<p>After some struggling I got fluentd to forward Openshift audit log files to Splunk. However this resulted in a huge number of events, so I applied a filter to exclude &quot;get&quot; and &quot;watch&quot;. I would  like to include the get secrets.</p>
<p>My question, how to change the filter to exclude &quot;get&quot; but include &quot;get secret&quot;?</p>
<pre><code>apiVersion: v1
kind: ConfigMap
metadata:
  name: splunk-kubernetes-audit
  namespace: splunk-logging
  labels:
    app: splunk-kubernetes-audit
data:
  fluent.conf: |-
    &lt;system&gt;
      log_level info
    &lt;/system&gt;
    @include source.audit.conf
    @include output.conf
  output.conf: |-
    &lt;label @SPLUNK&gt;
      # = filters for non-container log files =
      # extract sourcetype
       &lt;filter tail.file.**&gt;
          @type grep
          &lt;exclude&gt;
             key verb
             pattern /watch/
          &lt;/exclude&gt;
          &lt;and&gt;
            &lt;exclude&gt;
               key verb
               pattern /get/
            &lt;/exclude&gt;
          &lt;/and&gt;
  &lt;/filter&gt;
      &lt;filter tail.file.**&gt;
        @type jq_transformer
        jq '.record.sourcetype = (.tag | ltrimstr(&quot;tail.file.&quot;)) | .record.cluster_name = &quot;opcdev&quot; | .record.splunk_index = &quot;openshift_audit_n&quot; | .record'
      &lt;/filter&gt;
      # = custom filters specified by users =

      # = output =
      &lt;match **&gt;
        @type splunk_hec
        protocol https
        hec_host &quot;splunk-heavyforwarder.linux.rabobank.nl&quot;
        hec_port 8088
        hec_token &quot;#{ENV['SPLUNK_HEC_TOKEN']}&quot;
        index_key splunk_index
        insecure_ssl false
        ca_file /fluentd/etc/splunk/hec_ca_file
        host &quot;#{ENV['K8S_NODE_NAME']}&quot;
        source_key source
        sourcetype_key sourcetype
        &lt;fields&gt;
          # currently CRI does not produce log paths with all the necessary
          # metadata to parse out pod, namespace, container_name, container_id.
          # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
          pod
          namespace
          container_name
          cluster_name
          container_id
        &lt;/fields&gt;
        app_name splunk-kubernetes-audit
        app_version 1.4.7
        &lt;buffer&gt;
          @type memory
          chunk_limit_records 100000
          chunk_limit_size 10m
          flush_interval 10s
          flush_thread_count 1
          overflow_action block
          retry_max_times 5
          retry_type exponential_backoff
          retry_wait 2
          retry_max_interval 300
          total_limit_size 600m
        &lt;/buffer&gt;
        &lt;format&gt;
          @type &quot;json&quot;
        &lt;/format&gt;
      &lt;/match&gt;
    &lt;/label&gt;
  source.audit.conf: |-
    # This fluentd conf file contains sources for log files other than container logs.
    &lt;source&gt;
      @id tail.file.kube-api-audit
      @type tail
      @label @SPLUNK
      tag tail.file.kube-api-audit
      path /var/log/kube-apiserver/audit.log
      pos_file /var/log/splunk-fluentd-audit-kube-api-audit.pos
      read_from_head true
      path_key source
      &lt;parse&gt;
        @type json
      &lt;/parse&gt;
    &lt;/source&gt;
    &lt;source&gt;
      @id tail.file.oauth-api-audit
      @type tail
      @label @SPLUNK
      tag tail.file.oauth-api-audit
      path /var/log/oauth-apiserver/audit.log
      pos_file /var/log/splunk-fluentd-audit-oauth-api-audit.pos
      read_from_head true
      path_key source
      &lt;parse&gt;
        @type json
      &lt;/parse&gt;
    &lt;/source&gt;
    &lt;source&gt;
      @id tail.file.openshift-api-audit
      @type tail
      @label @SPLUNK
      tag tail.file.openshift-api-audit
      path /var/log/openshift-apiserver/audit.log
      pos_file /var/log/splunk-fluentd-audit-openshift-api-audit.pos
      read_from_head true
      path_key source
      &lt;parse&gt;
        @type json
      &lt;/parse&gt;
    &lt;/source&gt;
</code></pre>
<p>the secrets</p>
<pre><code>apiVersion: v1
kind: Secret
metadata:
  labels:
    app: splunk-kubernetes-audit
  name: splunk-kubernetes-audit
  namespace: splunk-logging
type: Opaque
data:
  hec_ca_file: {{ base64 encoded CA certificate }}
  splunk_hec_token: {{ base64 encoded Get_token_for_index }}
</code></pre>
<p>and the daemonset</p>
<pre><code>apiVersion: apps/v1
kind: DaemonSet
metadata:
  annotations:
    configmap.update: &quot;1&quot;
    deprecated.daemonset.template.generation: &quot;34&quot;
  generation: 34
  labels:
    app: splunk-kubernetes-audit
    engine: fluentd
  name: splunk-kubernetes-audit
  namespace: splunk-logging
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: splunk-kubernetes-audit
      release: rabo-splunk
  template:
    metadata:
      annotations:
        checksum/config: 0574cfe32baa34dcb02d7e3293f7c5ac0379ffb45cf4b7e455eb6975e6102320
        configmap.update.trigger: &quot;1&quot;
        prometheus.io/port: &quot;24231&quot;
        prometheus.io/scrape: &quot;true&quot;
      creationTimestamp: null
      labels:
        app: splunk-kubernetes-audit
        release: rabo-splunk
    spec:
      containers:
      - env:
        - name: K8S_NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: MY_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        - name: MY_POD_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.name
        - name: SPLUNK_HEC_TOKEN
          valueFrom:
            secretKeyRef:
              key: splunk_hec_token
              name: splunk-kubernetes-audit
        - name: SSL_CERT_FILE
          value: /fluentd/etc/splunk/hec_ca_file
        image: docker.io/splunk/fluentd-hec:1.2.6
        imagePullPolicy: Always
        name: splunk-fluentd-k8s-audit
        ports:
        - containerPort: 24231
          name: metrics
          protocol: TCP
        resources:
          requests:
            cpu: 500m
            memory: 600Mi
        securityContext:
          privileged: true
          runAsUser: 0
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /var/log
          name: varlog
        - mountPath: /var/log/kube-apiserver
          name: varlogkube
          readOnly: true
        - mountPath: /var/log/oauth-apiserver
          name: varlogoauth
          readOnly: true
        - mountPath: /var/log/openshift-apiserver
          name: varlogopenshift
          readOnly: true
        - mountPath: /fluentd/etc
          name: conf-configmap
        - mountPath: /fluentd/etc/splunk
          name: secrets
          readOnly: true
      dnsPolicy: ClusterFirst
      imagePullSecrets:
      - name: acr-secret
      nodeSelector:
        node-role.kubernetes.io/master: ''
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      serviceAccount: splunk-logging
      serviceAccountName: splunk-logging
      terminationGracePeriodSeconds: 30
      tolerations:
      - effect: NoSchedule
        key: node-role.kubernetes.io/master
      volumes:
      - hostPath:
          path: /var/log
          type: &quot;&quot;
        name: varlog
      - hostPath:
          path: /var/log/kube-apiserver
          type: &quot;&quot;
        name: varlogkube
      - hostPath:
          path: /var/log/oauth-apiserver
          type: &quot;&quot;
        name: varlogoauth
      - hostPath:
          path: /var/log/openshift-apiserver
          type: &quot;&quot;
        name: varlogopenshift
      - configMap:
          defaultMode: 420
          name: splunk-kubernetes-audit
        name: conf-configmap
      - name: secrets
        secret:
          defaultMode: 420
          secretName: splunk-kubernetes-audit
  updateStrategy:
    rollingUpdate:
      maxUnavailable: 1
    type: RollingUpdate
</code></pre>

<p>Instead of excluding all get, list and watch actions except for secrets I opted to exclude the objects that cause the most events, like namespaces, pods and configmaps. This resulted in the extra filters below. This gives a reduction in Splunk events of about 65%. An Openshift cluster in rest generates without filtering about 12 GB audit logging a day.</p>
<pre><code>      # reduce the number of events by removing get, watch and list api calls
      &lt;filter tail.file.**&gt;
        @type grep
        &lt;and&gt;
          &lt;exclude&gt;
             key verb
             pattern /list/
          &lt;/exclude&gt;

          &lt;exclude&gt;
             key $.objectRef.resource
             pattern /namespaces/
          &lt;/exclude&gt;
        &lt;/and&gt;
      &lt;/filter&gt;
      &lt;filter tail.file.**&gt;
        @type grep
        &lt;and&gt;
          &lt;exclude&gt;
             key verb
             pattern /list/
          &lt;/exclude&gt;

          &lt;exclude&gt;
             key $.objectRef.resource
             pattern /pods/
          &lt;/exclude&gt;
        &lt;/and&gt;
      &lt;/filter&gt;
      &lt;filter tail.file.**&gt;
        @type grep
        &lt;and&gt;
          &lt;exclude&gt;
             key verb
             pattern /watch/
          &lt;/exclude&gt;

          &lt;exclude&gt;
             key $.objectRef.resource
             pattern /namespaces/
          &lt;/exclude&gt;
        &lt;/and&gt;
      &lt;/filter&gt;
      &lt;filter tail.file.**&gt;
        @type grep
        &lt;and&gt;
          &lt;exclude&gt;
             key verb
             pattern /watch/
          &lt;/exclude&gt;

          &lt;exclude&gt;
             key $.objectRef.resource
             pattern /pods/
          &lt;/exclude&gt;
        &lt;/and&gt;
      &lt;/filter&gt;
      &lt;filter tail.file.**&gt;
        @type grep
        &lt;and&gt;
          &lt;exclude&gt;
             key verb
             pattern /watch/
          &lt;/exclude&gt;

          &lt;exclude&gt;
             key $.objectRef.resource
             pattern /configmaps/
          &lt;/exclude&gt;
        &lt;/and&gt;
      &lt;/filter&gt;
      &lt;filter tail.file.**&gt;
        @type grep
        &lt;and&gt;
          &lt;exclude&gt;
             key verb
             pattern /get/
          &lt;/exclude&gt;

          &lt;exclude&gt;
             key $.objectRef.resource
             pattern /configmaps/
          &lt;/exclude&gt;
        &lt;/and&gt;
      &lt;/filter&gt;
      &lt;filter tail.file.**&gt;
        @type grep
        &lt;and&gt;
          &lt;exclude&gt;
             key verb
             pattern /get/
          &lt;/exclude&gt;

          &lt;exclude&gt;
             key $.objectRef.resource
             pattern /namespaces/
          &lt;/exclude&gt;
        &lt;/and&gt;
      &lt;/filter&gt;
      &lt;filter tail.file.**&gt;
        @type grep
        &lt;and&gt;
          &lt;exclude&gt;
             key verb
             pattern /get/
          &lt;/exclude&gt;

          &lt;exclude&gt;
             key $.objectRef.resource
             pattern /clusterrolebindings/
          &lt;/exclude&gt;
         &lt;/and&gt;
      &lt;/filter&gt;
</code></pre>
