From 1fddd01ca94de3df49a29c15fc27af1e56db8bde Mon Sep 17 00:00:00 2001 From: Dheeraj Turaga Date: Thu, 24 Apr 2025 00:02:52 -0500 Subject: [PATCH 1/5] Mount airflow project dir to containers airflow home --- airflow-core/docs/howto/docker-compose/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index cb99c6085db39..d7e3079906b1d 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -78,6 +78,7 @@ x-airflow-common: - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins + - ${AIRFLOW_PROJ_DIR:-.}:/opt/airflow user: "${AIRFLOW_UID:-50000}:0" depends_on: &airflow-common-depends-on From 6874db5d26311fe70bb2ccae532bc3f4bb4ac9f1 Mon Sep 17 00:00:00 2001 From: Dheeraj Turaga Date: Thu, 24 Apr 2025 10:25:29 -0500 Subject: [PATCH 2/5] Remove dags/logs/config/plugin mounts They are no longer needed as we are directly mounting AIRFLOW_HOME --- airflow-core/docs/howto/docker-compose/docker-compose.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index d7e3079906b1d..ef29932b1edd7 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -74,10 +74,6 @@ x-airflow-common: # If you want to use it, outcomment it and replace airflow.cfg with the name of your config file # AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' volumes: - - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags - - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs - - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config - - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins - ${AIRFLOW_PROJ_DIR:-.}:/opt/airflow user: "${AIRFLOW_UID:-50000}:0" depends_on: From 37cf75962a5c013f7cf475cd942264e88205d9fe Mon Sep 17 00:00:00 2001 From: Dheeraj Turaga Date: Thu, 24 Apr 2025 10:41:56 -0500 Subject: [PATCH 3/5] Set AIRFLOW_CFG in common env --- .../docs/howto/docker-compose/docker-compose.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index ef29932b1edd7..008c9e0313feb 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -71,10 +71,12 @@ x-airflow-common: # for other purpose (development, test and especially production usage) build/extend Airflow image. _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} # The following line can be used to set a custom config file, stored in the local config folder - # If you want to use it, outcomment it and replace airflow.cfg with the name of your config file - # AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' + AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg' volumes: - - ${AIRFLOW_PROJ_DIR:-.}:/opt/airflow + - ${AIRFLOW_PROJ_DIR:-.}/dags:/opt/airflow/dags + - ${AIRFLOW_PROJ_DIR:-.}/logs:/opt/airflow/logs + - ${AIRFLOW_PROJ_DIR:-.}/config:/opt/airflow/config + - ${AIRFLOW_PROJ_DIR:-.}/plugins:/opt/airflow/plugins user: "${AIRFLOW_UID:-50000}:0" depends_on: &airflow-common-depends-on From f25184fbb29b3d0c65cdc69bc2ad5f0efc0ac49d Mon Sep 17 00:00:00 2001 From: Dheeraj Turaga Date: Thu, 24 Apr 2025 15:14:36 -0500 Subject: [PATCH 4/5] Update docs and airflow init step to align with common-env --- .../howto/docker-compose/docker-compose.yaml | 6 ++--- .../docs/howto/docker-compose/index.rst | 22 +++++++++++++------ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/airflow-core/docs/howto/docker-compose/docker-compose.yaml b/airflow-core/docs/howto/docker-compose/docker-compose.yaml index 008c9e0313feb..603d373e4faa3 100644 --- a/airflow-core/docs/howto/docker-compose/docker-compose.yaml +++ b/airflow-core/docs/howto/docker-compose/docker-compose.yaml @@ -251,8 +251,8 @@ services: echo " https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin" echo fi - mkdir -p /sources/logs /sources/dags /sources/plugins - chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins} + mkdir -p /opts/airflow/{logs,dags,plugins,config} + chown -R "${AIRFLOW_UID}:0" /opts/airflow/{logs,dags,plugins,config} exec /entrypoint airflow version # yamllint enable rule:line-length environment: @@ -263,8 +263,6 @@ services: _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} _PIP_ADDITIONAL_REQUIREMENTS: '' user: "0:0" - volumes: - - ${AIRFLOW_PROJ_DIR:-.}:/sources airflow-cli: <<: *airflow-common diff --git a/airflow-core/docs/howto/docker-compose/index.rst b/airflow-core/docs/howto/docker-compose/index.rst index 1e60262da14a0..03f8773e0255c 100644 --- a/airflow-core/docs/howto/docker-compose/index.rst +++ b/airflow-core/docs/howto/docker-compose/index.rst @@ -125,8 +125,8 @@ Setting the right Airflow user ------------------------------ On **Linux**, the quick-start needs to know your host user id and needs to have group id set to ``0``. -Otherwise the files created in ``dags``, ``logs`` and ``plugins`` will be created with ``root`` user ownership. -You have to make sure to configure them for the docker-compose: +Otherwise the files created in ``dags``, ``logs``, ``config`` and ``plugins`` will be created with +``root`` user ownership. You have to make sure to configure them for the docker-compose: .. code-block:: bash @@ -143,6 +143,17 @@ safely ignore it. You can also manually create an ``.env`` file in the same fold AIRFLOW_UID=50000 +Initialize airflow.cfg (Optional) +--------------------------------- + +If you want to initialize ``airflow.cfg`` with default values before lauching the airflow service, run. + +.. code-block:: bash + + docker compose run airflow-cli airflow config list + +This will seed ``airflow.cfg`` with default values in ``config`` folder. + Initialize the database ----------------------- @@ -346,12 +357,9 @@ Special case - Adding a custom config file If you have a custom config file and wish to use it in your Airflow instance, you need to perform the following steps: -1) Remove comment from the ``AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'`` line - in the ``docker-compose.yaml`` file. - -2) Place your custom ``airflow.cfg`` file in the local config folder. +1) Replace the auto-generated ``airflow.cfg`` file in the local config folder with your custom config file. -3) If your config file has a different name than ``airflow.cfg``, adjust the filename in +2) If your config file has a different name than ``airflow.cfg``, adjust the filename in ``AIRFLOW_CONFIG: '/opt/airflow/config/airflow.cfg'`` Networking From 8f4951d4c29a86c45782feeea54f8a48c34e20c0 Mon Sep 17 00:00:00 2001 From: Dheeraj Turaga Date: Thu, 24 Apr 2025 16:47:13 -0500 Subject: [PATCH 5/5] Fix spell check and static checks --- airflow-core/docs/howto/docker-compose/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow-core/docs/howto/docker-compose/index.rst b/airflow-core/docs/howto/docker-compose/index.rst index 03f8773e0255c..270dd70e935a0 100644 --- a/airflow-core/docs/howto/docker-compose/index.rst +++ b/airflow-core/docs/howto/docker-compose/index.rst @@ -125,7 +125,7 @@ Setting the right Airflow user ------------------------------ On **Linux**, the quick-start needs to know your host user id and needs to have group id set to ``0``. -Otherwise the files created in ``dags``, ``logs``, ``config`` and ``plugins`` will be created with +Otherwise the files created in ``dags``, ``logs``, ``config`` and ``plugins`` will be created with ``root`` user ownership. You have to make sure to configure them for the docker-compose: .. code-block:: bash @@ -146,7 +146,7 @@ safely ignore it. You can also manually create an ``.env`` file in the same fold Initialize airflow.cfg (Optional) --------------------------------- -If you want to initialize ``airflow.cfg`` with default values before lauching the airflow service, run. +If you want to initialize ``airflow.cfg`` with default values before launching the airflow service, run. .. code-block:: bash