title: Pods are evicted due to Container airflow-worker exceeded its local ephemeral storage limit "10137Mi" in google composer
body: <p>I'm using <strong>google Composer version</strong> <strong>1.18.0</strong> which is having <strong>Airflow version 2.2.3</strong> and the worker node count is set to 4 with a Disk Size of 100 GB and machine type <strong>n1-standard-2</strong> and</p>
  <pre><code>Web server machine type=composer-n1-webserver-2 (2 vCPU, 1.6 GB memory)
  
  Cloud SQL machine type=db-n1-standard-2 (2 vCPU, 7.5 GB memory)
  </code></pre>
  <p>number of schedulers is 1 and the rest of the other configs are almost basic.</p>
  <p>I have just moved 4 jobs into this and I'm already seeing POD evicted due to this below error</p>
  <p><strong>Container airflow-worker exceeded its local ephemeral storage limit &quot;10137Mi&quot;.</strong></p>
  <p>I have tried contacting google support team, they told to create a new environment with enough resources and this one newly created as autopilot in another older environment is not working as expected and also pods are all evicted with the same error, so we creatd this above config environment with enough resources but still seeing this error and not sure how to get rid of this.</p>
  <p>PFA screenshots for reference
  <a href="https://i.stack.imgur.com/PGLmL.png" rel="nofollow noreferrer"><img src="https://i.stack.imgur.com/PGLmL.png" alt="enter image description here" /></a></p>
  <p><a href="https://i.stack.imgur.com/bDHDu.png" rel="nofollow noreferrer"><img src="https://i.stack.imgur.com/bDHDu.png" alt="enter image description here" /></a></p>
  <p>Can anyone help me here</p>
  <p>YAML FILE</p>
  <pre><code>    apiVersion: v1
      kind: Pod
      metadata:
      annotations:
            composer.cloud.google.com/running-task: &quot;true&quot;
            composer.cloud.google.com/template-version: 
             919cc331f4a25a332a1e0b6989f5fb56
        seccomp.security.alpha.kubernetes.io/pod: runtime/default
        creationTimestamp: &quot;2022-01-25T17:02:00Z&quot;
       generateName: airflow-worker-
       labels:
      run: airflow-worker
      managedFields:
     - apiVersion: v1
     fieldsType: FieldsV1
     fieldsV1:
     f:metadata:
       f:annotations:
        .: {}
        f:composer.cloud.google.com/template-version: {}
      f:generateName: {}
      f:labels:
        .: {}
        f:run: {}
      f:ownerReferences:
        .: {}
        k:{&quot;uid&quot;:&quot;7ece9e6c-e252-43d9-b989-239d765d375b&quot;}:
          .: {}
          f:apiVersion: {}
          f:blockOwnerDeletion: {}
          f:controller: {}
          f:kind: {}
          f:name: {}
          f:uid: {}
    f:spec:
      f:containers:
        k:{&quot;name&quot;:&quot;airflow-worker&quot;}:
          .: {}
          f:args: {}
          f:env:
            .: {}
            k:{&quot;name&quot;:&quot;AIRFLOW__CORE__FERNET_KEY&quot;}:
              .: {}
              f:name: {}
              f:valueFrom:
                .: {}
                f:secretKeyRef:
                  .: {}
                  f:key: {}
                  f:name: {}
            k:{&quot;name&quot;:&quot;AIRFLOW__CORE__SQL_ALCHEMY_CONN&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;AIRFLOW__WEBSERVER__BASE_URL&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;AIRFLOW_DATABASE_VERSION&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;AIRFLOW_HOME&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;AUTOGKE&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;C_FORCE_ROOT&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;CLOUDSDK_METRICS_ENVIRONMENT&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_ENVIRONMENT&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_GKE_LOCATION&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_GKE_NAME&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_GKE_ZONE&quot;}:
              .: {}
              f:name: {}
            k:{&quot;name&quot;:&quot;COMPOSER_LOCATION&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_PYTHON_VERSION&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_VERSION&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_VERSIONED_NAMESPACE&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;DAGS_FOLDER&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;GCP_PROJECT&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;GCS_BUCKET&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;GCSFUSE_EXTRACTED&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;GRPC_POLL_STRATEGY&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;SQL_DATABASE&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;SQL_HOST&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;SQL_PASSWORD&quot;}:
              .: {}
              f:name: {}
              f:valueFrom:
                .: {}
                f:secretKeyRef:
                  .: {}
                  f:key: {}
                  f:name: {}
            k:{&quot;name&quot;:&quot;SQL_SUBNET&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;SQL_USER&quot;}:
              .: {}
              f:name: {}
              f:value: {}
          f:image: {}
          f:imagePullPolicy: {}
          f:livenessProbe:
            .: {}
            f:exec:
              .: {}
              f:command: {}
            f:failureThreshold: {}
            f:initialDelaySeconds: {}
            f:periodSeconds: {}
            f:successThreshold: {}
            f:timeoutSeconds: {}
          f:name: {}
          f:ports:
            .: {}
            k:{&quot;containerPort&quot;:8793,&quot;protocol&quot;:&quot;TCP&quot;}:
              .: {}
              f:containerPort: {}
              f:protocol: {}
          f:resources:
            .: {}
            f:limits:
              .: {}
              f:cpu: {}
              f:ephemeral-storage: {}
              f:memory: {}
            f:requests:
              .: {}
              f:cpu: {}
              f:ephemeral-storage: {}
              f:memory: {}
          f:terminationMessagePath: {}
          f:terminationMessagePolicy: {}
          f:volumeMounts:
            .: {}
            k:{&quot;mountPath&quot;:&quot;/etc/airflow/airflow_cfg&quot;}:
              .: {}
              f:mountPath: {}
              f:name: {}
            k:{&quot;mountPath&quot;:&quot;/home/airflow/container-comms&quot;}:
              .: {}
              f:mountPath: {}
              f:name: {}
            k:{&quot;mountPath&quot;:&quot;/home/airflow/gcs&quot;}:
              .: {}
              f:mountPath: {}
              f:name: {}
            k:{&quot;mountPath&quot;:&quot;/home/airflow/gcsfuse&quot;}:
              .: {}
              f:mountPath: {}
              f:mountPropagation: {}
              f:name: {}
        k:{&quot;name&quot;:&quot;gcs-syncd&quot;}:
          .: {}
          f:args: {}
          f:env:
            .: {}
            k:{&quot;name&quot;:&quot;AUTOGKE&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_GKE_LOCATION&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_GKE_NAME&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;COMPOSER_GKE_ZONE&quot;}:
              .: {}
              f:name: {}
            k:{&quot;name&quot;:&quot;GCS_BUCKET&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;SQL_DATABASE&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;SQL_PASSWORD&quot;}:
              .: {}
              f:name: {}
              f:valueFrom:
                .: {}
                f:secretKeyRef:
                  .: {}
                  f:key: {}
                  f:name: {}
            k:{&quot;name&quot;:&quot;SQL_SUBNET&quot;}:
              .: {}
              f:name: {}
              f:value: {}
            k:{&quot;name&quot;:&quot;SQL_USER&quot;}:
              .: {}
              f:name: {}
              f:value: {}
          f:image: {}
          f:imagePullPolicy: {}
          f:name: {}
          f:resources:
            .: {}
            f:limits:
              .: {}
              f:cpu: {}
              f:ephemeral-storage: {}
              f:memory: {}
            f:requests:
              .: {}
              f:cpu: {}
              f:ephemeral-storage: {}
              f:memory: {}
          f:terminationMessagePath: {}
          f:terminationMessagePolicy: {}
          f:volumeMounts:
            .: {}
            k:{&quot;mountPath&quot;:&quot;/home/airflow/gcs&quot;}:
              .: {}
              f:mountPath: {}
              f:name: {}
      f:dnsPolicy: {}
      f:enableServiceLinks: {}
      f:restartPolicy: {}
      f:schedulerName: {}
      f:securityContext: {}
      f:terminationGracePeriodSeconds: {}
      f:volumes:
        .: {}
        k:{&quot;name&quot;:&quot;airflow-config&quot;}:
          .: {}
          f:configMap:
            .: {}
            f:defaultMode: {}
            f:name: {}
          f:name: {}
        k:{&quot;name&quot;:&quot;container-comms&quot;}:
          .: {}
          f:hostPath:
            .: {}
            f:path: {}
            f:type: {}
          f:name: {}
        k:{&quot;name&quot;:&quot;gcsdir&quot;}:
          .: {}
          f:emptyDir: {}
          f:name: {}
        k:{&quot;name&quot;:&quot;gcsfuse&quot;}:
          .: {}
          f:hostPath:
            .: {}
            f:path: {}
            f:type: {}
          f:name: {}
     manager: manager
     operation: Update
     time: &quot;2022-01-25T17:02:00Z&quot;
     - apiVersion: v1
     fieldsType: FieldsV1
     fieldsV1:
     f:metadata:
      f:annotations:
        f:composer.cloud.google.com/running-task: {}
      manager: OpenAPI-Generator
      operation: Update
      time: &quot;2022-01-25T17:02:33Z&quot;
     - apiVersion: v1
      fieldsType: FieldsV1
     fieldsV1:
       f:status:
      f:message: {}
      f:phase: {}
      f:reason: {}
      f:startTime: {}
      manager: kubelet
     operation: Update
     time: &quot;2022-01-26T10:01:06Z&quot;
     name: airflow-worker-nlzdq
     namespace: composer-2-0-0-airflow-2-1-4-2884adda
     ownerReferences:
    - apiVersion: composer.cloud.google.com/v1beta1
     blockOwnerDeletion: true
     controller: true
     kind: AirflowWorkerSet
     name: airflow-worker
     uid: 7ece9e6c-e252-43d9-b989-239d765d375b
     resourceVersion: &quot;27047147&quot;
    uid: b904b45d-0001-4dc4-8f6c-88420d286973
    spec:
       containers:
    - args:
      - worker
      env:
      - name: GRPC_POLL_STRATEGY
      value: epoll1
     - name: CLOUDSDK_METRICS_ENVIRONMENT
      value: 2.1.4+composer
     - name: GCS_BUCKET
     value: us-east4-XYZ-gcp-airflow-p-2884adda-bucket
    - name: GCP_PROJECT
    value: XYZ-259410
  - name: COMPOSER_LOCATION
    value: us-east4
  - name: COMPOSER_GKE_ZONE
  - name: COMPOSER_GKE_NAME
    value: us-east4-XYZ-gcp-airflow-p-2884adda-gke
  - name: AUTOGKE
    value: &quot;TRUE&quot;
  - name: COMPOSER_GKE_LOCATION
    value: us-east4
  - name: COMPOSER_PYTHON_VERSION
    value: &quot;3&quot;
  - name: COMPOSER_ENVIRONMENT
    value: XYZ-gcp-airflow-prod-shared-vpc
  - name: COMPOSER_VERSIONED_NAMESPACE
    value: composer-2-0-0-airflow-2-1-4-2884adda
  - name: AIRFLOW_HOME
    value: /etc/airflow
  - name: DAGS_FOLDER
    value: /home/airflow/gcs/dags
   - name: SQL_HOST
    value: airflow-sqlproxy-service.composer- 
   system.svc.cluster.local
  - name: SQL_DATABASE
    value: composer-2-0-0-airflow-2-1-4-2884adda
  - name: SQL_USER
    value: root
  - name: SQL_PASSWORD
    valueFrom:
      secretKeyRef:
        key: sql_password
        name: airflow-secrets
  - name: GCSFUSE_EXTRACTED
    value: &quot;TRUE&quot;
  - name: COMPOSER_VERSION
    value: 2.0.0
  - name: AIRFLOW__WEBSERVER__BASE_URL
    value: https://XYZ-dot-us-east4.composer.googleusercontent.com
  - name: SQL_SUBNET
    value: 172.16.2.0/29
  - name: AIRFLOW_DATABASE_VERSION
    value: POSTGRES_13
  - name: AIRFLOW__CORE__SQL_ALCHEMY_CONN
    value: postgresql+psycopg2://$(SQL_USER):$(SQL_PASSWORD)@airflow-sqlproxy-service.composer-system.svc.cluster.local:3306/$(SQL_DATABASE)
  - name: AIRFLOW__CORE__FERNET_KEY
    valueFrom:
      secretKeyRef:
        key: fernet_key
        name: airflow-secrets
  - name: C_FORCE_ROOT
    value: &quot;TRUE&quot;
  image: us-east4-docker.pkg.dev/XYZ-259410/composer-images-us-east4-XYZ-gcp-airflow-p-2884adda-gke/e77f16de-356f-40b5-b42b-f5482f02f793
  imagePullPolicy: IfNotPresent
  livenessProbe:
    exec:
      command:
      - /var/local/worker_checker.py
    failureThreshold: 6
    initialDelaySeconds: 120
    periodSeconds: 90
    successThreshold: 1
    timeoutSeconds: 30
  name: airflow-worker
  ports:
  - containerPort: 8793
    protocol: TCP
  resources:
    limits:
      cpu: 1700m
      ephemeral-storage: 4Gi
      memory: 6963Mi
    requests:
      cpu: 1700m
      ephemeral-storage: 2Gi
      memory: 6963Mi
  securityContext:
    capabilities:
      drop:
      - NET_RAW
  terminationMessagePath: /dev/termination-log
  terminationMessagePolicy: File
  volumeMounts:
  - mountPath: /etc/airflow/airflow_cfg
    name: airflow-config
  - mountPath: /home/airflow/gcs
    name: gcsdir
  - mountPath: /home/airflow/container-comms
    name: container-comms
  - mountPath: /home/airflow/gcsfuse
    mountPropagation: HostToContainer
    name: gcsfuse
  - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
    name: kube-api-access-59j75
    readOnly: true
   - args:
   - /home/airflow/gcs
   env:
   - name: GCS_BUCKET
     value: us-east4-XYZ-gcp-airflow-p-2884adda-bucket
   - name: SQL_DATABASE
    value: composer-2-0-0-airflow-2-1-4-2884adda
  - name: SQL_USER
    value: root
  - name: SQL_PASSWORD
    valueFrom:
      secretKeyRef:
        key: sql_password
        name: airflow-secrets
  - name: COMPOSER_GKE_ZONE
  - name: COMPOSER_GKE_NAME
    value: us-east4-XYZ-gcp-airflow-p-2884adda-gke
  - name: SQL_SUBNET
    value: 172.16.2.0/29
  - name: AUTOGKE
    value: &quot;TRUE&quot;
  - name: COMPOSER_GKE_LOCATION
    value: us-east4
  image: us-docker.pkg.dev/cloud-airflow-releaser/gcs-syncd/gcs-syncd:cloud_composer_service_2021-12-08-RC1
  imagePullPolicy: IfNotPresent
  name: gcs-syncd
  resources:
    limits:
      cpu: 300m
      ephemeral-storage: 102Mi
      memory: 1228Mi
    requests:
      cpu: 300m
      ephemeral-storage: 102Mi
      memory: 1228Mi
  securityContext:
    capabilities:
      drop:
      - NET_RAW
  terminationMessagePath: /dev/termination-log
  terminationMessagePolicy: File
  volumeMounts:
  - mountPath: /home/airflow/gcs
    name: gcsdir
  - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
    name: kube-api-access-59j75
    readOnly: true
   dnsPolicy: ClusterFirst
   enableServiceLinks: true
   nodeName: gk3-us-east4-XYZ-gcp-ai-nap-meqa92wv-4be2b93f-5kdv
   preemptionPolicy: PreemptLowerPriority
   priority: 0
   restartPolicy: Always
   schedulerName: gke.io/optimize-utilization-scheduler
   securityContext:
   seccompProfile:
    type: RuntimeDefault
    serviceAccount: default
    serviceAccountName: default
    terminationGracePeriodSeconds: 30
    tolerations:
   - effect: NoExecute
   key: node.kubernetes.io/not-ready
    operator: Exists
    tolerationSeconds: 300
   - effect: NoExecute
     key: node.kubernetes.io/unreachable
     operator: Exists
     tolerationSeconds: 300
    volumes:
    - configMap:
     defaultMode: 420
     name: airflow-configmap
     name: airflow-config
    - emptyDir: {}
    name: gcsdir
   - hostPath:
    path: /var/composer/gcs_mount_status
    type: &quot;&quot;
   name: container-comms
  - hostPath:
    path: /var/composer/gcs_mount
    type: &quot;&quot;
    name: gcsfuse
   - name: kube-api-access-59j75
    projected:
     defaultMode: 420
    sources:
    - serviceAccountToken:
        expirationSeconds: 3607
        path: token
    - configMap:
        items:
        - key: ca.crt
          path: ca.crt
        name: kube-root-ca.crt
    - downwardAPI:
        items:
        - fieldRef:
            apiVersion: v1
            fieldPath: metadata.namespace
          path: namespace
  status:
    message: 'Container airflow-worker exceeded its local ephemeral 
    storage limit &quot;10137Mi&quot;. '
    phase: Failed
   reason: Evicted
   startTime: &quot;2022-01-25T17:02:01Z&quot;
  </code></pre>