airflow helm 설치

1. helm repo 추가 및 values.yaml 다운로드

# helm repo add
helm repo add apache-airflow https://airflow.apache.org
 
# values.yaml 다운로드
helm show values apache-airflow/airflow > values.yaml

 

2. values.yaml 수정

2-1. ingress 설정

# Ingress configuration
ingress:
  # Enable all ingress resources (deprecated - use ingress.web.enabled and ingress.flower.enabled)
  enabled: false

  # Configs for the Ingress of the web Service
  web:
    # Enable web ingress resource
    enabled: true

    # Annotations for the web Ingress
    annotations: {}

    # The path for the web Ingress
    path: "/"

    # The pathType for the above path (used only with Kubernetes v1.19 and above)
    pathType: "ImplementationSpecific"

    # The hostname for the web Ingress (Deprecated - renamed to `ingress.web.hosts`)
    host: ""

    # The hostnames or hosts configuration for the web Ingress
    hosts: ["airflow.dd.io"]
    #   # The hostname for the web Ingress (can be templated)
    # - name: ""
    #   # configs for web Ingress TLS
    #   tls:
    #     # Enable TLS termination for the web Ingress
    #     enabled: false
    #     # the name of a pre-created Secret containing a TLS private key and certificate
    #     secretName: ""

    # The Ingress Class for the web Ingress (used only with Kubernetes v1.19 and above)
    ingressClassName: "nginx"

 

2-2. airflow.cfg 설정

아래 내용을 수정 및 추가한다.

config:
  core:
    default_timezone: 'Asia/Seoul'
  logging:
    colored_console_log: 'True'
  webserver:
    expose_config: 'True'
    default_ui_timezone: 'Asia/Seoul'

 

2-3. git-credentials secret 생성

git-sync 하기 위한 git-credentials secret을 생성한다. base64 인코딩된 문자를 입력해야 한다.

shell에서 다음 커맨드를 사용해 문자열을 base64로 인코딩할 수 있다.

`echo 'kyengseooh' | tr -d '\n' | base64`

extraSecrets:
  git-credentials:
    data: |
      GIT_SYNC_USERNAME: "a3llbmdzZW9vaA=="
      GIT_SYNC_PASSWORD: "Z2hwXZZyNXBDdW1FbUdoZXXXXXXXXXXXXXXXXXXX"
      GITSYNC_USERNAME: "a3llbmdzZW9vaA=="
      GITSYNC_PASSWORD: "Z2hwXZZyNXBDdW1FbUdoZXXXXXXXXXXXXXXXXXXX=="

 

2-4. executor 설정

# Airflow executor
# One of: LocalExecutor, LocalKubernetesExecutor, CeleryExecutor, KubernetesExecutor, CeleryKubernetesExecutor
executor: "KubernetesExecutor"

 

2-5. fernet, webserver key 생성

fernet key 생성방법 : 

python3 -c 'from cryptography.fernet import Fernet ; fernet_key = Fernet.generate_key() ; print(fernet_key.decode())'

 

webserver key 생성방법:

python3 -c 'import secrets; print(secrets.token_hex(16))'

 

생성된 값을 아래에 입력한다.

# Fernet key settings
# Note: fernetKey can only be set during install, not upgrade
fernetKey: FDLyJe1N004_S5eTyZswSKqKm361wDFasO6l8q0mvPs=
fernetKeySecretName: ~

# Flask secret key for Airflow Webserver: `[webserver] secret_key` in airflow.cfg
webserverSecretKey: 22552516a26abdcedb0200e8ae73dd2c
webserverSecretKeySecretName: ~

 

2-6. log를 pv에 저장하도록 설정

logs:
  # Configuration for empty dir volume (if logs.persistence.enabled == false)
  # emptyDirConfig:
  #   sizeLimit: 1Gi
  #   medium: Memory

  persistence:
    # Enable persistent volume for storing logs
    enabled: true
    # Volume size for logs
    size: 10Gi
    # Annotations for the logs PVC
    annotations: {}
    # If using a custom storageClass, pass name here
    storageClassName: nfs-client
    ## the name of an existing PVC to use
    existingClaim:

 

2-7. dag를 git에서 sync하도록 git-sync 설정

dags:
  # Where dags volume will be mounted. Works for both persistence and gitSync.
  # If not specified, dags mount path will be set to $AIRFLOW_HOME/dags
  mountPath: ~
  persistence:
    # Annotations for dags PVC
    annotations: {}
    # Enable persistent volume for storing dags
    enabled: false
    # Volume size for dags
    size: 1Gi
    # If using a custom storageClass, pass name here
    storageClassName:
    # access mode of the persistent volume
    accessMode: ReadWriteOnce
    ## the name of an existing PVC to use
    existingClaim:
    ## optional subpath for dag volume mount
    subPath: ~
  gitSync:
    enabled: true

    # git repo clone url
    # ssh example: git@github.com:apache/airflow.git
    # https example: https://github.com/apache/airflow.git
    repo: https://github.com/kyeongseooh/airflow_dag.git
    branch: main
    rev: HEAD
    # The git revision (branch, tag, or hash) to check out, v4 only
    depth: 1
    # the number of consecutive failures allowed before aborting
    maxFailures: 0
    # subpath within the repo where dags are located
    # should be "" if dags are at repo root
    subPath: "dags"
    # if your repo needs a user name password
    # you can load them to a k8s secret like the one below
    #   ---
    #   apiVersion: v1
    #   kind: Secret
    #   metadata:
    #     name: git-credentials
    #   data:
    #     # For git-sync v3
    #     GIT_SYNC_USERNAME: <base64_encoded_git_username>
    #     GIT_SYNC_PASSWORD: <base64_encoded_git_password>
    #     # For git-sync v4
    #     GITSYNC_USERNAME: <base64_encoded_git_username>
    #     GITSYNC_PASSWORD: <base64_encoded_git_password>
    # and specify the name of the secret below
    #
    credentialsSecret: git-credentials

 

3. 설치

helm upgrade --install airflow apache-airflow/airflow --namespace airflow --create-namespace -f values.yaml