From 1f368ab1e69335c9f65ba08af19f3e853be2ad98 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 19:38:44 +0100
Subject: [PATCH 01/63] feat: add parent images build

---
 .github/workflows/pi_build.yml | 40 ++++++++++++++++++++++++++++++++++
 pi/build.sh                    | 31 ++++++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 .github/workflows/pi_build.yml
 create mode 100644 pi/build.sh

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
new file mode 100644
index 0000000..5a4655d
--- /dev/null
+++ b/.github/workflows/pi_build.yml
@@ -0,0 +1,40 @@
+name: 'Parent Images: Build'
+
+on:
+  workflow_call:
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+env:
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+jobs:
+  pi-build:
+    name: 'Parent Images: Build Requirements'
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref_name }}
+    
+      - name: GitHub Configuration
+        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+
+      - name: Clone cicd-deployment-scripts
+        run: git clone https://github.com/code-kern-ai/cicd-deployment-scripts.git
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      
+      - name: Run build
+        run: |
+          bash cicd-deployment-scripts/pi/build.sh -p "${{ github.event.pull_request.number }}"
\ No newline at end of file
diff --git a/pi/build.sh b/pi/build.sh
new file mode 100644
index 0000000..0e4fee8
--- /dev/null
+++ b/pi/build.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+PR_NUMBER=""
+
+while getopts p: flag
+do
+    case "${flag}" in
+        p) PR_NUMBER=${OPTARG};;
+    esac
+done
+
+UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only)
+while IFS= read -r file; do
+    if [[ $file != requirements/* ]] || [[ $file != *.in ]]; then
+        continue
+    fi
+    
+    parent_image_type=$(basename $file | sed 's|-requirements.in||g')
+
+    echo "::group::Compiling $parent_image_type-requirements.in"
+    pip-compile requirements/$parent_image_type-requirements.in
+
+    echo "Running pip install for $parent_image_type"
+    python -m venv ./venv/
+    source ./venv/bin/activate
+    pip install -r requirements/$parent_image_type-requirements.txt
+    echo "::endgroup::"
+
+done <<< "$UPDATED_FILES"

From 08dc28cabdb658fb418f50c9c5e35d399fd0257c Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 19:43:55 +0100
Subject: [PATCH 02/63] test: cicd-deployment-scripts checkout parent-images
 test: git diff instead of gh pr diff

---
 .github/workflows/pi_build.yml | 2 +-
 pi/build.sh                    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 5a4655d..fd55fca 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -28,7 +28,7 @@ jobs:
         run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
 
       - name: Clone cicd-deployment-scripts
-        run: git clone https://github.com/code-kern-ai/cicd-deployment-scripts.git
+        run: git clone https://github.com/code-kern-ai/cicd-deployment-scripts.git && git checkout parent-images
       
       - name: Set up Python
         uses: actions/setup-python@v5
diff --git a/pi/build.sh b/pi/build.sh
index 0e4fee8..da29bf8 100644
--- a/pi/build.sh
+++ b/pi/build.sh
@@ -11,7 +11,7 @@ do
     esac
 done
 
-UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only)
+UPDATED_FILES=$(git diff --name-only)
 while IFS= read -r file; do
     if [[ $file != requirements/* ]] || [[ $file != *.in ]]; then
         continue

From cee4878bd9f5d1a364718de6bd3a58a9cef8e695 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 19:47:19 +0100
Subject: [PATCH 03/63] test: git clone from parent-images

---
 .github/workflows/pi_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index fd55fca..7a9e8a4 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -28,7 +28,7 @@ jobs:
         run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
 
       - name: Clone cicd-deployment-scripts
-        run: git clone https://github.com/code-kern-ai/cicd-deployment-scripts.git && git checkout parent-images
+        run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
       
       - name: Set up Python
         uses: actions/setup-python@v5

From a1a8a61ca2e15e14f696e3c2d4fab0d99eb1acde Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 19:51:24 +0100
Subject: [PATCH 04/63] perf: add diff ref param to build

---
 .github/workflows/pi_build.yml | 4 +++-
 pi/build.sh                    | 7 +++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 7a9e8a4..b93c5ef 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -37,4 +37,6 @@ jobs:
       
       - name: Run build
         run: |
-          bash cicd-deployment-scripts/pi/build.sh -p "${{ github.event.pull_request.number }}"
\ No newline at end of file
+          bash cicd-deployment-scripts/pi/build.sh \
+            -p "${{ github.event.pull_request.number }}" \
+            -r "${{ github.repository.default_branch }}" \
\ No newline at end of file
diff --git a/pi/build.sh b/pi/build.sh
index da29bf8..89c7102 100644
--- a/pi/build.sh
+++ b/pi/build.sh
@@ -3,15 +3,18 @@
 set -e
 
 PR_NUMBER=""
+DIFF_REF="dev"
 
-while getopts p: flag
+while getopts p:r: flag
 do
     case "${flag}" in
         p) PR_NUMBER=${OPTARG};;
+        r) DIFF_REF=${OPTARG};;
     esac
 done
 
-UPDATED_FILES=$(git diff --name-only)
+git diff $DIFF_REF --color
+UPDATED_FILES=$(git diff $DIFF_REF --name-only)
 while IFS= read -r file; do
     if [[ $file != requirements/* ]] || [[ $file != *.in ]]; then
         continue

From 66853ec004dd1c7af9a907bad7f5c71f607ab90c Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 19:55:59 +0100
Subject: [PATCH 05/63] debug: add print statements

---
 .github/workflows/pi_build.yml | 12 ++++++------
 pi/build.sh                    |  1 +
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index b93c5ef..f2d540d 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -3,11 +3,11 @@ name: 'Parent Images: Build'
 on:
   workflow_call:
 
-# Special permissions required for OIDC authentication
-permissions:
-  id-token: write
-  contents: read
-  actions: read
+# # Special permissions required for OIDC authentication
+# permissions:
+#   id-token: write
+#   contents: read
+#   actions: read
 
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
@@ -39,4 +39,4 @@ jobs:
         run: |
           bash cicd-deployment-scripts/pi/build.sh \
             -p "${{ github.event.pull_request.number }}" \
-            -r "${{ github.repository.default_branch }}" \
\ No newline at end of file
+            -r "${{ github.event.repository.default_branch }}"
\ No newline at end of file
diff --git a/pi/build.sh b/pi/build.sh
index 89c7102..87851a2 100644
--- a/pi/build.sh
+++ b/pi/build.sh
@@ -13,6 +13,7 @@ do
     esac
 done
 
+echo "Printing diff"
 git diff $DIFF_REF --color
 UPDATED_FILES=$(git diff $DIFF_REF --name-only)
 while IFS= read -r file; do

From 84a3e10434039575ab563c60fda143d33bcd99b1 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 20:03:12 +0100
Subject: [PATCH 06/63] perf: update checkout config

---
 .github/workflows/pi_build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index f2d540d..7a43494 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -22,7 +22,8 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.ref_name }}
+          ref: ${{ github.sha }}
+          fetch-depth: 0
     
       - name: GitHub Configuration
         run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com

From 551fe230142b3e43c1258dbece723128ac6ef147 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 20:07:37 +0100
Subject: [PATCH 07/63] perf: update build

---
 .github/workflows/pi_build.yml | 3 +--
 pi/build.sh                    | 8 ++------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 7a43494..cd9eaaa 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -39,5 +39,4 @@ jobs:
       - name: Run build
         run: |
           bash cicd-deployment-scripts/pi/build.sh \
-            -p "${{ github.event.pull_request.number }}" \
-            -r "${{ github.event.repository.default_branch }}"
\ No newline at end of file
+            -p "${{ github.event.pull_request.number }}"
\ No newline at end of file
diff --git a/pi/build.sh b/pi/build.sh
index 87851a2..0e4fee8 100644
--- a/pi/build.sh
+++ b/pi/build.sh
@@ -3,19 +3,15 @@
 set -e
 
 PR_NUMBER=""
-DIFF_REF="dev"
 
-while getopts p:r: flag
+while getopts p: flag
 do
     case "${flag}" in
         p) PR_NUMBER=${OPTARG};;
-        r) DIFF_REF=${OPTARG};;
     esac
 done
 
-echo "Printing diff"
-git diff $DIFF_REF --color
-UPDATED_FILES=$(git diff $DIFF_REF --name-only)
+UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only)
 while IFS= read -r file; do
     if [[ $file != requirements/* ]] || [[ $file != *.in ]]; then
         continue

From 7cdd8cd2b274f642fb32c75afafbc0ceb0f50c10 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 20:10:03 +0100
Subject: [PATCH 08/63] perf: install pip dependencies

---
 .github/workflows/pi_build.yml | 3 +++
 pi/build.sh                    | 1 +
 2 files changed, 4 insertions(+)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index cd9eaaa..2317242 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -36,6 +36,9 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       
+      - name: Install dependencies
+        run: python -m pip install pip-tools
+        
       - name: Run build
         run: |
           bash cicd-deployment-scripts/pi/build.sh \
diff --git a/pi/build.sh b/pi/build.sh
index 0e4fee8..01cac8f 100644
--- a/pi/build.sh
+++ b/pi/build.sh
@@ -26,6 +26,7 @@ while IFS= read -r file; do
     python -m venv ./venv/
     source ./venv/bin/activate
     pip install -r requirements/$parent_image_type-requirements.txt
+    rm -rf ./venv/
     echo "::endgroup::"
 
 done <<< "$UPDATED_FILES"

From ec86e8e390d0b85863ef7bc4f3291e7ae4029e71 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 21:16:39 +0100
Subject: [PATCH 09/63] feat: add pi docker build

---
 .github/workflows/pi_build.yml | 83 ++++++++++++++++++++++++++++++----
 pi/build.sh                    | 15 +++---
 2 files changed, 83 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 2317242..30d8956 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -13,33 +13,98 @@ env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
 
 jobs:
-  pi-build:
-    name: 'Parent Images: Build Requirements'
+  pi-compile:
+    name: 'Parent Images: Compile Requirements'
     runs-on: ubuntu-latest
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    outputs:
+      updated_parent_types: ${{ steps.run-build.outputs.updated_parent_types }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-        with:
-          ref: ${{ github.sha }}
-          fetch-depth: 0
     
       - name: GitHub Configuration
         run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
 
       - name: Clone cicd-deployment-scripts
         run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
-      
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-      
+
       - name: Install dependencies
         run: python -m pip install pip-tools
-        
+
       - name: Run build
+        id: run-build
         run: |
           bash cicd-deployment-scripts/pi/build.sh \
-            -p "${{ github.event.pull_request.number }}"
\ No newline at end of file
+            -p "${{ github.event.pull_request.number }}"
+
+  pi-build:
+    name: 'Parent Images: Build'
+    runs-on: ubuntu-latest
+    needs: pi-compile
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+      DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
+      DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
+      DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
+      DOCKERHUB_CONTAINER_REGISTRY: ${{ vars.DOCKERHUB_CONTAINER_REGISTRY }}
+      DOCKERHUB_LOGIN_USERNAME: ${{ secrets.DOCKERHUB_LOGIN_USERNAME }}
+      DOCKERHUB_LOGIN_PASSWORD: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }}
+      IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
+    strategy:
+      matrix:
+        parent_image_type: ${{ toJson(needs.pi-compile.outputs.updated_parent_types) }}
+    steps:
+      - name: Dump needs context
+        env:
+          NEEDS_CONTEXT: ${{ toJson(needs) }}
+        run: echo "$NEEDS_CONTEXT"
+      
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          submodules: 'true'
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64,arm
+
+      - name: Log into DEV registry
+        uses: docker/login-action@v3
+        with:
+          registry: "${{ env.DEV_CONTAINER_REGISTRY }}"
+          username: "${{ env.DEV_LOGIN_USERNAME }}"
+          password: "${{ env.DEV_LOGIN_PASSWORD }}"
+
+      - name: Log into Docker Hub registry
+        uses: docker/login-action@v3
+        with:
+          username: "${{ env.DOCKERHUB_LOGIN_USERNAME }}"
+          password: "${{ env.DOCKERHUB_LOGIN_PASSWORD }}"
+    
+      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:amd64
+        if: ${{ github.event_name != 'release' }}
+        uses: docker/build-push-action@v5
+        with:
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-image:dev-${{ matrix.parent_image_type }}-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-image:dev-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+          platforms: linux/amd64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-image:dev-${{ matrix.parent_image_type }}
+          push: true
+          build-args: |
+            platform=linux/amd64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
diff --git a/pi/build.sh b/pi/build.sh
index 01cac8f..ceb0ee5 100644
--- a/pi/build.sh
+++ b/pi/build.sh
@@ -12,21 +12,24 @@ do
 done
 
 UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only)
+UPDATED_PARENT_TYPES=()
 while IFS= read -r file; do
     if [[ $file != requirements/* ]] || [[ $file != *.in ]]; then
         continue
     fi
     
     parent_image_type=$(basename $file | sed 's|-requirements.in||g')
+    UPDATED_PARENT_TYPES+=($parent_image_type)
 
     echo "::group::Compiling $parent_image_type-requirements.in"
     pip-compile requirements/$parent_image_type-requirements.in
-
-    echo "Running pip install for $parent_image_type"
-    python -m venv ./venv/
-    source ./venv/bin/activate
-    pip install -r requirements/$parent_image_type-requirements.txt
-    rm -rf ./venv/
     echo "::endgroup::"
 
 done <<< "$UPDATED_FILES"
+
+JSON=""
+for parent_image_type in "${UPDATED_PARENT_TYPES[@]}"; do
+    JSON+="\"$parent_image_type\","
+done
+JSON="[${JSON::-1}]"
+echo "updated_parent_types=$JSON" >> $GITHUB_OUTPUT
\ No newline at end of file

From 78a9c9f61b0e5bc8c12b0b11f3de1f37d92ed314 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 4 Dec 2024 21:20:04 +0100
Subject: [PATCH 10/63] chore: rename build to compile

---
 .github/workflows/pi_build.yml | 20 ++++++++++----------
 pi/{build.sh => compile.sh}    |  0
 2 files changed, 10 insertions(+), 10 deletions(-)
 rename pi/{build.sh => compile.sh} (100%)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 30d8956..b7c347d 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -19,7 +19,7 @@ jobs:
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     outputs:
-      updated_parent_types: ${{ steps.run-build.outputs.updated_parent_types }}
+      updated_parent_types: ${{ steps.run-compile.outputs.updated_parent_types }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -38,12 +38,17 @@ jobs:
       - name: Install dependencies
         run: python -m pip install pip-tools
 
-      - name: Run build
-        id: run-build
+      - name: Run compile
+        id: run-compile
         run: |
-          bash cicd-deployment-scripts/pi/build.sh \
+          bash cicd-deployment-scripts/pi/compile.sh \
             -p "${{ github.event.pull_request.number }}"
 
+      - name: Dump job context
+        env:
+          JOB_CONTEXT: ${{ toJson(job) }}
+        run: echo "$JOB_CONTEXT"
+
   pi-build:
     name: 'Parent Images: Build'
     runs-on: ubuntu-latest
@@ -59,13 +64,8 @@ jobs:
       IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
     strategy:
       matrix:
-        parent_image_type: ${{ toJson(needs.pi-compile.outputs.updated_parent_types) }}
+        parent_image_type: ${{ fromJson(needs.pi-compile.outputs.updated_parent_types) }}
     steps:
-      - name: Dump needs context
-        env:
-          NEEDS_CONTEXT: ${{ toJson(needs) }}
-        run: echo "$NEEDS_CONTEXT"
-      
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
diff --git a/pi/build.sh b/pi/compile.sh
similarity index 100%
rename from pi/build.sh
rename to pi/compile.sh

From 2d6edc66476891ea5692a34a33d4b05616598bb7 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 10:53:48 +0100
Subject: [PATCH 11/63] fix: use environment secrets instead of repo

---
 .github/workflows/pi_build.yml | 64 ++++++++++++++++++++++++----------
 1 file changed, 45 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index b7c347d..c4e387f 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -16,6 +16,7 @@ jobs:
   pi-compile:
     name: 'Parent Images: Compile Requirements'
     runs-on: ubuntu-latest
+    environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     outputs:
@@ -44,23 +45,16 @@ jobs:
           bash cicd-deployment-scripts/pi/compile.sh \
             -p "${{ github.event.pull_request.number }}"
 
-      - name: Dump job context
-        env:
-          JOB_CONTEXT: ${{ toJson(job) }}
-        run: echo "$JOB_CONTEXT"
-
   pi-build:
-    name: 'Parent Images: Build'
+    name: 'Parent Images: Docker Build'
     runs-on: ubuntu-latest
     needs: pi-compile
+    environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
       DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
       DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
       DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
-      DOCKERHUB_CONTAINER_REGISTRY: ${{ vars.DOCKERHUB_CONTAINER_REGISTRY }}
-      DOCKERHUB_LOGIN_USERNAME: ${{ secrets.DOCKERHUB_LOGIN_USERNAME }}
-      DOCKERHUB_LOGIN_PASSWORD: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }}
       IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
     strategy:
       matrix:
@@ -89,22 +83,54 @@ jobs:
           username: "${{ env.DEV_LOGIN_USERNAME }}"
           password: "${{ env.DEV_LOGIN_PASSWORD }}"
 
-      - name: Log into Docker Hub registry
-        uses: docker/login-action@v3
+      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev
+        uses: docker/build-push-action@v5
         with:
-          username: "${{ env.DOCKERHUB_LOGIN_USERNAME }}"
-          password: "${{ env.DOCKERHUB_LOGIN_PASSWORD }}"
-    
-      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:amd64
-        if: ${{ github.event_name != 'release' }}
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+          platforms: linux/amd64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}
+          push: true
+          build-args: |
+            platform=linux/amd64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev-arm64
         uses: docker/build-push-action@v5
         with:
-          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-image:dev-${{ matrix.parent_image_type }}-cache
-          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-image:dev-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+          platforms: linux/arm64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64
+          push: true
+          build-args: |
+            platform=linux/arm64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha
+        uses: docker/build-push-action@v5
+        with:
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
           platforms: linux/amd64
           file: Dockerfile
-          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-image:dev-${{ matrix.parent_image_type }}
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
           push: true
           build-args: |
             platform=linux/amd64
             label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha-arm64
+        uses: docker/build-push-action@v5
+        with:
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+          platforms: linux/arm64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
+          push: true
+          build-args: |
+            platform=linux/arm64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile

From 3dd4dbed3db918ce37685c7d966c745c16401860 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:04:39 +0100
Subject: [PATCH 12/63] fix: docker build context

---
 .github/workflows/pi_build.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index c4e387f..05e9eca 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -24,6 +24,9 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
+          ref: ${{ github.event.pull_request.head.ref }}
     
       - name: GitHub Configuration
         run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
@@ -65,6 +68,13 @@ jobs:
         with:
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
           submodules: 'true'
+      
+      - name: Update submodules
+        run: |
+          ls -l
+          cd ..
+          ls -l
+
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -86,6 +96,7 @@ jobs:
       - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev
         uses: docker/build-push-action@v5
         with:
+          context: .
           cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-cache
           cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
           platforms: linux/amd64
@@ -99,6 +110,7 @@ jobs:
       - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev-arm64
         uses: docker/build-push-action@v5
         with:
+          context: .
           cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache
           cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
           platforms: linux/arm64
@@ -112,6 +124,7 @@ jobs:
       - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha
         uses: docker/build-push-action@v5
         with:
+          context: .
           cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
           cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
           platforms: linux/amd64
@@ -125,6 +138,7 @@ jobs:
       - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha-arm64
         uses: docker/build-push-action@v5
         with:
+          context: .
           cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
           cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
           platforms: linux/arm64

From 938804e11175e892c45e1df86b7f7b26c406c31f Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:23:43 +0100
Subject: [PATCH 13/63] test: upload requirments artifacts

---
 .github/workflows/pi_build.yml | 51 +++++++++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 05e9eca..e7b148e 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -48,6 +48,41 @@ jobs:
           bash cicd-deployment-scripts/pi/compile.sh \
             -p "${{ github.event.pull_request.number }}"
 
+  pi-upload-artifacts:
+    name: 'Parent Images: Upload Artifacts'
+    runs-on: ubuntu-latest
+    needs: pi-compile
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    strategy:
+      matrix:
+        parent_image_type: ${{ fromJson(needs.pi-compile.outputs.updated_parent_types) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
+          ref: ${{ github.event.pull_request.head.ref }}
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: python -m pip install pip-tools
+      
+      - name: Compile Artifacts
+        run: pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
+      
+      # Save plan to artifacts  
+      - name: Upload ${{ matrix.parent_image_type }}-requirements
+        uses: actions/upload-artifact@v4
+        with:
+          name: requirements/${{ matrix.parent_image_type }}-requirements.txt
+          path: ${{ matrix.parent_image_type }}-requirements.txt
+
   pi-build:
     name: 'Parent Images: Docker Build'
     runs-on: ubuntu-latest
@@ -69,13 +104,27 @@ jobs:
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
           submodules: 'true'
       
+      # Download requirements from artifacts  
+      - name: Download ${{ matrix.parent_image_type }}-requirements
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ matrix.parent_image_type }}-requirements.txt
+          path: ${{ github.workspace }}/submodules/requirements/${{ matrix.parent_image_type }}-requirements.txt
+
+      - name: GitHub Configuration
+        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+
+      - name: Clone cicd-deployment-scripts
+        run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
+
       - name: Update submodules
         run: |
+          echo "ls -l: $(pwd)"
           ls -l
           cd ..
+          echo "ls -l: $(pwd)"
           ls -l
 
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:

From 69f83a97a348fa29444eccb2d977595796b9b4a7 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:24:43 +0100
Subject: [PATCH 14/63] test: skip docker builds to speed up execution

---
 .github/workflows/pi_build.yml | 82 +++++++++++++++++-----------------
 1 file changed, 41 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index e7b148e..5b62974 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -156,44 +156,44 @@ jobs:
             platform=linux/amd64
             label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
 
-      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev-arm64
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache
-          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
-          platforms: linux/arm64
-          file: Dockerfile
-          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64
-          push: true
-          build-args: |
-            platform=linux/arm64
-            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
-
-      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
-          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
-          platforms: linux/amd64
-          file: Dockerfile
-          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
-          push: true
-          build-args: |
-            platform=linux/amd64
-            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
-
-      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha-arm64
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
-          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
-          platforms: linux/arm64
-          file: Dockerfile
-          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
-          push: true
-          build-args: |
-            platform=linux/arm64
-            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+      # - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev-arm64
+      #   uses: docker/build-push-action@v5
+      #   with:
+      #     context: .
+      #     cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache
+      #     cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+      #     platforms: linux/arm64
+      #     file: Dockerfile
+      #     tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64
+      #     push: true
+      #     build-args: |
+      #       platform=linux/arm64
+      #       label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      # - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha
+      #   uses: docker/build-push-action@v5
+      #   with:
+      #     context: .
+      #     cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
+      #     cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+      #     platforms: linux/amd64
+      #     file: Dockerfile
+      #     tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
+      #     push: true
+      #     build-args: |
+      #       platform=linux/amd64
+      #       label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      # - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha-arm64
+      #   uses: docker/build-push-action@v5
+      #   with:
+      #     context: .
+      #     cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
+      #     cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+      #     platforms: linux/arm64
+      #     file: Dockerfile
+      #     tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
+      #     push: true
+      #     build-args: |
+      #       platform=linux/arm64
+      #       label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile

From 54a3eb835356608d5fe0fdcf53423bc29fd27fbe Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:25:39 +0100
Subject: [PATCH 15/63] fix: update pi-build needs

---
 .github/workflows/pi_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 5b62974..2f101a5 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -86,7 +86,7 @@ jobs:
   pi-build:
     name: 'Parent Images: Docker Build'
     runs-on: ubuntu-latest
-    needs: pi-compile
+    needs: [pi-compile, pi-upload-artifacts]
     environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}

From 7b70c97b5626d4b5de16e82be50e5dd38cf8cf8e Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:27:48 +0100
Subject: [PATCH 16/63] fix: upload artifacts settings

---
 .github/workflows/pi_build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 2f101a5..09e06c8 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -80,8 +80,8 @@ jobs:
       - name: Upload ${{ matrix.parent_image_type }}-requirements
         uses: actions/upload-artifact@v4
         with:
-          name: requirements/${{ matrix.parent_image_type }}-requirements.txt
-          path: ${{ matrix.parent_image_type }}-requirements.txt
+          name: ${{ matrix.parent_image_type }}-requirements.txt
+          path: requirements/${{ matrix.parent_image_type }}-requirements.txt
 
   pi-build:
     name: 'Parent Images: Docker Build'

From 5362d09a638a1da1d446ceb92861e8f76f5c58da Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:31:08 +0100
Subject: [PATCH 17/63] test: update submodules

---
 .github/workflows/pi_build.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 09e06c8..9cca612 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -82,6 +82,7 @@ jobs:
         with:
           name: ${{ matrix.parent_image_type }}-requirements.txt
           path: requirements/${{ matrix.parent_image_type }}-requirements.txt
+          if-no-files-found: error
 
   pi-build:
     name: 'Parent Images: Docker Build'
@@ -109,7 +110,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: ${{ matrix.parent_image_type }}-requirements.txt
-          path: ${{ github.workspace }}/submodules/requirements/${{ matrix.parent_image_type }}-requirements.txt
+          path: ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
 
       - name: GitHub Configuration
         run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
@@ -121,6 +122,7 @@ jobs:
         run: |
           echo "ls -l: $(pwd)"
           ls -l
+          cat ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
           cd ..
           echo "ls -l: $(pwd)"
           ls -l

From 796c0fb05af2604894de1441cc62d1cf04946753 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:36:21 +0100
Subject: [PATCH 18/63] fix: purge downloadable artifact

---
 .github/workflows/pi_build.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 9cca612..708a5dd 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -105,6 +105,9 @@ jobs:
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
           submodules: 'true'
       
+      - name: Purge stale requirements
+        run: rm ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
+      
       # Download requirements from artifacts  
       - name: Download ${{ matrix.parent_image_type }}-requirements
         uses: actions/download-artifact@v4

From fdcdece9a7979e90a486437cf67f6bf499bfaa9c Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:38:35 +0100
Subject: [PATCH 19/63] fix: purge downloadable artifact

---
 .github/workflows/pi_build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 708a5dd..a9be98b 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -105,15 +105,15 @@ jobs:
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
           submodules: 'true'
       
-      - name: Purge stale requirements
-        run: rm ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
+      # - name: Purge stale requirements
+      #   run: rm ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
       
       # Download requirements from artifacts  
       - name: Download ${{ matrix.parent_image_type }}-requirements
         uses: actions/download-artifact@v4
         with:
           name: ${{ matrix.parent_image_type }}-requirements.txt
-          path: ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
+          path: ${{ github.workspace }}/submodules/parent-images/requirements
 
       - name: GitHub Configuration
         run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com

From 9d4e5f01c6e7935ae7b153b04342698b0148f07f Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:46:06 +0100
Subject: [PATCH 20/63] test: checkout submodules

---
 .github/workflows/pi_build.yml | 99 +++++++++++++++++++---------------
 1 file changed, 55 insertions(+), 44 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index a9be98b..0d81bac 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -48,46 +48,46 @@ jobs:
           bash cicd-deployment-scripts/pi/compile.sh \
             -p "${{ github.event.pull_request.number }}"
 
-  pi-upload-artifacts:
-    name: 'Parent Images: Upload Artifacts'
-    runs-on: ubuntu-latest
-    needs: pi-compile
-    environment: dev
-    env:
-      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
-    strategy:
-      matrix:
-        parent_image_type: ${{ fromJson(needs.pi-compile.outputs.updated_parent_types) }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
-          ref: ${{ github.event.pull_request.head.ref }}
+  # pi-upload-artifacts:
+  #   name: 'Parent Images: Upload Artifacts'
+  #   runs-on: ubuntu-latest
+  #   needs: pi-compile
+  #   environment: dev
+  #   env:
+  #     PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+  #   strategy:
+  #     matrix:
+  #       parent_image_type: ${{ fromJson(needs.pi-compile.outputs.updated_parent_types) }}
+  #   steps:
+  #     - name: Checkout repository
+  #       uses: actions/checkout@v4
+  #       with:
+  #         repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
+  #         ref: ${{ github.event.pull_request.head.ref }}
       
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v5
+  #       with:
+  #         python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Install dependencies
-        run: python -m pip install pip-tools
+  #     - name: Install dependencies
+  #       run: python -m pip install pip-tools
       
-      - name: Compile Artifacts
-        run: pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
+  #     - name: Compile Artifacts
+  #       run: pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
       
-      # Save plan to artifacts  
-      - name: Upload ${{ matrix.parent_image_type }}-requirements
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ matrix.parent_image_type }}-requirements.txt
-          path: requirements/${{ matrix.parent_image_type }}-requirements.txt
-          if-no-files-found: error
+  #     # Save plan to artifacts  
+  #     - name: Upload ${{ matrix.parent_image_type }}-requirements
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: ${{ matrix.parent_image_type }}-requirements.txt
+  #         path: requirements/${{ matrix.parent_image_type }}-requirements.txt
+  #         if-no-files-found: error
 
   pi-build:
     name: 'Parent Images: Docker Build'
     runs-on: ubuntu-latest
-    needs: [pi-compile, pi-upload-artifacts]
+    needs: [pi-compile]
     environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
@@ -104,22 +104,33 @@ jobs:
         with:
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
           submodules: 'true'
+        
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: python -m pip install pip-tools
       
-      # - name: Purge stale requirements
-      #   run: rm ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
+      - name: Compile Artifacts
+        run: |
+          cd submodules/parent-images
+          git fetch origin && git checkout ${{ github.event.pull_request.head.ref }}
+          pip-compile submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
       
-      # Download requirements from artifacts  
-      - name: Download ${{ matrix.parent_image_type }}-requirements
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ matrix.parent_image_type }}-requirements.txt
-          path: ${{ github.workspace }}/submodules/parent-images/requirements
+      # # Download requirements from artifacts  
+      # - name: Download ${{ matrix.parent_image_type }}-requirements
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     name: ${{ matrix.parent_image_type }}-requirements.txt
+      #     path: ${{ github.workspace }}/submodules/parent-images/requirements
 
-      - name: GitHub Configuration
-        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+      # - name: GitHub Configuration
+      #   run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
 
-      - name: Clone cicd-deployment-scripts
-        run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
+      # - name: Clone cicd-deployment-scripts
+      #   run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
 
       - name: Update submodules
         run: |

From b62abded9251cf256472567db99f7c49727d9219 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:49:39 +0100
Subject: [PATCH 21/63] test: checkout submodules

---
 .github/workflows/pi_build.yml | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 0d81bac..9d98123 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -113,6 +113,18 @@ jobs:
       - name: Install dependencies
         run: python -m pip install pip-tools
       
+      - name: Update submodules
+        run: |
+          echo "ls -l: $(pwd)"
+          ls -l
+          cat ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
+          cd submodules
+          echo "ls -l: $(pwd)"
+          ls -l
+          cd parent-images
+          echo "ls -l: $(pwd)"
+          ls -l
+
       - name: Compile Artifacts
         run: |
           cd submodules/parent-images
@@ -132,15 +144,6 @@ jobs:
       # - name: Clone cicd-deployment-scripts
       #   run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
 
-      - name: Update submodules
-        run: |
-          echo "ls -l: $(pwd)"
-          ls -l
-          cat ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
-          cd ..
-          echo "ls -l: $(pwd)"
-          ls -l
-
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
         with:

From 818f996d633d52d81c965a6005eeaac854263bfe Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:52:30 +0100
Subject: [PATCH 22/63] test: checkout submodules

---
 .github/workflows/pi_build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 9d98123..f531224 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -117,7 +117,7 @@ jobs:
         run: |
           echo "ls -l: $(pwd)"
           ls -l
-          cat ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
+          # cat ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
           cd submodules
           echo "ls -l: $(pwd)"
           ls -l
@@ -127,7 +127,7 @@ jobs:
 
       - name: Compile Artifacts
         run: |
-          cd submodules/parent-images
+          cd ${{ github.workspace }}/submodules/parent-images
           git fetch origin && git checkout ${{ github.event.pull_request.head.ref }}
           pip-compile submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
       

From 385d07a450b14cd5ea89ba2f7a8ea995d4f21022 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 11:55:13 +0100
Subject: [PATCH 23/63] test: checkout submodules

---
 .github/workflows/pi_build.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index f531224..959e7f8 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -128,7 +128,9 @@ jobs:
       - name: Compile Artifacts
         run: |
           cd ${{ github.workspace }}/submodules/parent-images
-          git fetch origin && git checkout ${{ github.event.pull_request.head.ref }}
+          ls -l
+          git log --oneline -5
+          git fetch origin && git checkout ${{ github.event.pull_request.head.sha }}
           pip-compile submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
       
       # # Download requirements from artifacts  

From d8d8be3136aaad786977d4df239c35bb05dfc0fa Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:00:44 +0100
Subject: [PATCH 24/63] test: checkout submodules

---
 .github/workflows/pi_build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 959e7f8..dd031a7 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -130,7 +130,8 @@ jobs:
           cd ${{ github.workspace }}/submodules/parent-images
           ls -l
           git log --oneline -5
-          git fetch origin && git checkout ${{ github.event.pull_request.head.sha }}
+          git config --list --show-origin
+          git fetch && git checkout ${{ github.event.pull_request.head.sha }}
           pip-compile submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
       
       # # Download requirements from artifacts  

From 0e8eab901fc852203ed0dd1a667aff37e9f1f263 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:03:33 +0100
Subject: [PATCH 25/63] test: checkout submodules

---
 .github/workflows/pi_build.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index dd031a7..d8207a9 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -103,7 +103,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
-          submodules: 'true'
+          submodules: 'recursive'
         
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -130,7 +130,6 @@ jobs:
           cd ${{ github.workspace }}/submodules/parent-images
           ls -l
           git log --oneline -5
-          git config --list --show-origin
           git fetch && git checkout ${{ github.event.pull_request.head.sha }}
           pip-compile submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
       

From 6d0ad7451fc3fc8d93116d93ceb7c3e28034d94f Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:06:04 +0100
Subject: [PATCH 26/63] test: checkout submodules

---
 .github/workflows/pi_build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index d8207a9..64c995f 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -103,7 +103,8 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
-          submodules: 'recursive'
+          submodules: 'true'
+          fetch-depth: 0
         
       - name: Set up Python
         uses: actions/setup-python@v5

From 6f3a1c1aa619fe71f32c384480618a7cd3b73388 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:07:51 +0100
Subject: [PATCH 27/63] test: checkout submodules

---
 .github/workflows/pi_build.yml | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 64c995f..03fd57e 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -114,25 +114,23 @@ jobs:
       - name: Install dependencies
         run: python -m pip install pip-tools
       
-      - name: Update submodules
-        run: |
-          echo "ls -l: $(pwd)"
-          ls -l
-          # cat ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
-          cd submodules
-          echo "ls -l: $(pwd)"
-          ls -l
-          cd parent-images
-          echo "ls -l: $(pwd)"
-          ls -l
+      # - name: Update submodules
+      #   run: |
+      #     echo "ls -l: $(pwd)"
+      #     ls -l
+      #     # cat ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
+      #     cd submodules
+      #     echo "ls -l: $(pwd)"
+      #     ls -l
+      #     cd parent-images
+      #     echo "ls -l: $(pwd)"
+      #     ls -l
 
       - name: Compile Artifacts
         run: |
           cd ${{ github.workspace }}/submodules/parent-images
-          ls -l
-          git log --oneline -5
-          git fetch && git checkout ${{ github.event.pull_request.head.sha }}
-          pip-compile submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
+          git checkout ${{ github.event.pull_request.head.sha }}
+          pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
       
       # # Download requirements from artifacts  
       # - name: Download ${{ matrix.parent_image_type }}-requirements

From 276d7d4b705ce27d1f51662c8cb0497a6304cdda Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:18:26 +0100
Subject: [PATCH 28/63] perf: compile artifacts & update child repo

---
 .github/workflows/pi_build.yml | 104 +++++++++------------------------
 pi/{compile.sh => diff.sh}     |   4 --
 2 files changed, 26 insertions(+), 82 deletions(-)
 rename pi/{compile.sh => diff.sh} (80%)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 03fd57e..b6c1eab 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -13,16 +13,16 @@ env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
 
 jobs:
-  pi-compile:
-    name: 'Parent Images: Compile Requirements'
+  pi-diff:
+    name: 'Parent Images: Diff PI Types'
     runs-on: ubuntu-latest
     environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     outputs:
-      updated_parent_types: ${{ steps.run-compile.outputs.updated_parent_types }}
+      updated_parent_types: ${{ steps.get-diff.outputs.updated_parent_types }}
     steps:
-      - name: Checkout repository
+      - name: Checkout Repository
         uses: actions/checkout@v4
         with:
           repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
@@ -34,60 +34,16 @@ jobs:
       - name: Clone cicd-deployment-scripts
         run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install dependencies
-        run: python -m pip install pip-tools
-
-      - name: Run compile
-        id: run-compile
+      - name: Get Diff Types
+        id: get-diff
         run: |
-          bash cicd-deployment-scripts/pi/compile.sh \
+          bash cicd-deployment-scripts/pi/diff.sh \
             -p "${{ github.event.pull_request.number }}"
 
-  # pi-upload-artifacts:
-  #   name: 'Parent Images: Upload Artifacts'
-  #   runs-on: ubuntu-latest
-  #   needs: pi-compile
-  #   environment: dev
-  #   env:
-  #     PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
-  #   strategy:
-  #     matrix:
-  #       parent_image_type: ${{ fromJson(needs.pi-compile.outputs.updated_parent_types) }}
-  #   steps:
-  #     - name: Checkout repository
-  #       uses: actions/checkout@v4
-  #       with:
-  #         repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
-  #         ref: ${{ github.event.pull_request.head.ref }}
-      
-  #     - name: Set up Python
-  #       uses: actions/setup-python@v5
-  #       with:
-  #         python-version: ${{ env.PYTHON_VERSION }}
-
-  #     - name: Install dependencies
-  #       run: python -m pip install pip-tools
-      
-  #     - name: Compile Artifacts
-  #       run: pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
-      
-  #     # Save plan to artifacts  
-  #     - name: Upload ${{ matrix.parent_image_type }}-requirements
-  #       uses: actions/upload-artifact@v4
-  #       with:
-  #         name: ${{ matrix.parent_image_type }}-requirements.txt
-  #         path: requirements/${{ matrix.parent_image_type }}-requirements.txt
-  #         if-no-files-found: error
-
   pi-build:
     name: 'Parent Images: Docker Build'
     runs-on: ubuntu-latest
-    needs: [pi-compile]
+    needs: [pi-diff]
     environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
@@ -97,7 +53,7 @@ jobs:
       IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
     strategy:
       matrix:
-        parent_image_type: ${{ fromJson(needs.pi-compile.outputs.updated_parent_types) }}
+        parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -105,42 +61,34 @@ jobs:
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
           submodules: 'true'
           fetch-depth: 0
-        
+
+      - name: GitHub Configuration
+        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
-      - name: Install dependencies
+      - name: Install Dependencies
         run: python -m pip install pip-tools
       
-      # - name: Update submodules
-      #   run: |
-      #     echo "ls -l: $(pwd)"
-      #     ls -l
-      #     # cat ${{ github.workspace }}/submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.txt
-      #     cd submodules
-      #     echo "ls -l: $(pwd)"
-      #     ls -l
-      #     cd parent-images
-      #     echo "ls -l: $(pwd)"
-      #     ls -l
-
       - name: Compile Artifacts
         run: |
           cd ${{ github.workspace }}/submodules/parent-images
-          git checkout ${{ github.event.pull_request.head.sha }}
+          git checkout ${{ github.event.pull_request.head.ref }}
           pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
-      
-      # # Download requirements from artifacts  
-      # - name: Download ${{ matrix.parent_image_type }}-requirements
-      #   uses: actions/download-artifact@v4
-      #   with:
-      #     name: ${{ matrix.parent_image_type }}-requirements.txt
-      #     path: ${{ github.workspace }}/submodules/parent-images/requirements
-
-      # - name: GitHub Configuration
-      #   run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+          cd ${{ github.workspace }}
+          git checkout -b ${{ github.event.pull_request.head.ref }}
+          git add submodules
+          git commit -m "${{ github.event.pull_request.title }}"
+          git push origin ${{ github.event.pull_request.head.ref }}
+          gh pr create \
+            --title "${{ github.event.pull_request.title }}" \
+            --body "${{ github.event.pull_request.body }}" \
+            --base dev \
+            --head ${{ github.event.pull_request.head.ref }} \
+            --repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image
 
       # - name: Clone cicd-deployment-scripts
       #   run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
diff --git a/pi/compile.sh b/pi/diff.sh
similarity index 80%
rename from pi/compile.sh
rename to pi/diff.sh
index ceb0ee5..30aedc4 100644
--- a/pi/compile.sh
+++ b/pi/diff.sh
@@ -21,10 +21,6 @@ while IFS= read -r file; do
     parent_image_type=$(basename $file | sed 's|-requirements.in||g')
     UPDATED_PARENT_TYPES+=($parent_image_type)
 
-    echo "::group::Compiling $parent_image_type-requirements.in"
-    pip-compile requirements/$parent_image_type-requirements.in
-    echo "::endgroup::"
-
 done <<< "$UPDATED_FILES"
 
 JSON=""

From 7d5bf954d7c276ced7e494a00b84ddc9ac4d171b Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:20:48 +0100
Subject: [PATCH 29/63] fix: add git config

---
 .github/workflows/pi_build.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index b6c1eab..67ef021 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -78,6 +78,10 @@ jobs:
           cd ${{ github.workspace }}/submodules/parent-images
           git checkout ${{ github.event.pull_request.head.ref }}
           pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
+
+          git config --global user.email "devtools@kern.ai"
+          git config --global user.name "GitHub Actions"
+          
           cd ${{ github.workspace }}
           git checkout -b ${{ github.event.pull_request.head.ref }}
           git add submodules

From 53718ef5716bd26b0f605fce8fa096fed34a68e7 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:21:20 +0100
Subject: [PATCH 30/63] perf: update git config

---
 .github/workflows/pi_build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 67ef021..de62df8 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -78,11 +78,11 @@ jobs:
           cd ${{ github.workspace }}/submodules/parent-images
           git checkout ${{ github.event.pull_request.head.ref }}
           pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
-
-          git config --global user.email "devtools@kern.ai"
-          git config --global user.name "GitHub Actions"
           
           cd ${{ github.workspace }}
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
           git checkout -b ${{ github.event.pull_request.head.ref }}
           git add submodules
           git commit -m "${{ github.event.pull_request.title }}"

From b1224698868fa17bff677447d9fa1262dd2f7db5 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:24:14 +0100
Subject: [PATCH 31/63] test: update git config

---
 .github/workflows/pi_build.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index de62df8..9b8e829 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -63,7 +63,10 @@ jobs:
           fetch-depth: 0
 
       - name: GitHub Configuration
-        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+        run: |
+          git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+          git config --global user.email "devtools@kern.ai"
+          git config --global user.name "GitHub Actions"
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -80,9 +83,6 @@ jobs:
           pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
           
           cd ${{ github.workspace }}
-          git config user.email "devtools@kern.ai"
-          git config user.name "GitHub Actions"
-
           git checkout -b ${{ github.event.pull_request.head.ref }}
           git add submodules
           git commit -m "${{ github.event.pull_request.title }}"

From 88bc8e9c85939fbbbf20d9900ff6ba953b7ce635 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:26:32 +0100
Subject: [PATCH 32/63] test: update permission scope

---
 .github/workflows/pi_build.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 9b8e829..addaeff 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -3,11 +3,11 @@ name: 'Parent Images: Build'
 on:
   workflow_call:
 
-# # Special permissions required for OIDC authentication
-# permissions:
-#   id-token: write
-#   contents: read
-#   actions: read
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
 
 env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}

From c4badbe494f6045305e4b34e4f9a468dd9ab1bfd Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 12:44:34 +0100
Subject: [PATCH 33/63] test: update permission scope

---
 .github/workflows/pi_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index addaeff..d0208f2 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -6,7 +6,7 @@ on:
 # Special permissions required for OIDC authentication
 permissions:
   id-token: write
-  contents: read
+  contents: write
   actions: read
 
 env:

From 765a3aa427fc311848ca5f714340f8e6d7a6d7a6 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 15:11:11 +0100
Subject: [PATCH 34/63] test: git push

---
 .github/workflows/pi_build.yml | 76 +++++++++++++++++++++++++++++-----
 1 file changed, 66 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index d0208f2..6151fe9 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -6,7 +6,7 @@ on:
 # Special permissions required for OIDC authentication
 permissions:
   id-token: write
-  contents: write
+  contents: read
   actions: read
 
 env:
@@ -14,7 +14,7 @@ env:
 
 jobs:
   pi-diff:
-    name: 'Parent Images: Diff PI Types'
+    name: 'Parent Images: Diff Types'
     runs-on: ubuntu-latest
     environment: dev
     env:
@@ -40,17 +40,13 @@ jobs:
           bash cicd-deployment-scripts/pi/diff.sh \
             -p "${{ github.event.pull_request.number }}"
 
-  pi-build:
-    name: 'Parent Images: Docker Build'
+  pi-compile:
+    name: 'Parent Images: Compile Requirements'
     runs-on: ubuntu-latest
     needs: [pi-diff]
     environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
-      DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
-      DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
-      DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
-      IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
     strategy:
       matrix:
         parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
@@ -83,10 +79,70 @@ jobs:
           pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
           
           cd ${{ github.workspace }}
-          git checkout -b ${{ github.event.pull_request.head.ref }}
+          git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
+          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
+
           git add submodules
-          git commit -m "${{ github.event.pull_request.title }}"
+          git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
           git push origin ${{ github.event.pull_request.head.ref }}
+
+          gh pr create \
+            --title "${{ github.event.pull_request.title }}" \
+            --body "${{ github.event.pull_request.body }}" \
+            --base dev \
+            --head ${{ github.event.pull_request.head.ref }} \
+            --repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image
+          
+
+  pi-build:
+    name: 'Parent Images: Docker Build'
+    runs-on: ubuntu-latest
+    needs: [pi-diff, pi-compile]
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+      DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
+      DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
+      DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
+      IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
+    strategy:
+      matrix:
+        parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          submodules: 'true'
+
+      - name: GitHub Configuration
+        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Dependencies
+        run: python -m pip install pip-tools
+      
+      - name: Compile Artifacts
+        run: |
+          cd ${{ github.workspace }}/submodules/parent-images
+          git checkout ${{ github.event.pull_request.head.ref }}
+          pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
+          
+          cd ${{ github.workspace }}
+          git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
+          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
+
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
+          git add submodules
+          git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
+          git push origin ${{ github.event.pull_request.head.ref }}
+
           gh pr create \
             --title "${{ github.event.pull_request.title }}" \
             --body "${{ github.event.pull_request.body }}" \

From bc64d92d61726086a1542d65484cc89d5f2b6d70 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 15:12:45 +0100
Subject: [PATCH 35/63] test: git push

---
 .github/workflows/pi_build.yml | 56 +---------------------------------
 1 file changed, 1 insertion(+), 55 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 6151fe9..62d0380 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -40,64 +40,10 @@ jobs:
           bash cicd-deployment-scripts/pi/diff.sh \
             -p "${{ github.event.pull_request.number }}"
 
-  pi-compile:
-    name: 'Parent Images: Compile Requirements'
-    runs-on: ubuntu-latest
-    needs: [pi-diff]
-    environment: dev
-    env:
-      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
-    strategy:
-      matrix:
-        parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
-          submodules: 'true'
-          fetch-depth: 0
-
-      - name: GitHub Configuration
-        run: |
-          git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
-          git config --global user.email "devtools@kern.ai"
-          git config --global user.name "GitHub Actions"
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install Dependencies
-        run: python -m pip install pip-tools
-      
-      - name: Compile Artifacts
-        run: |
-          cd ${{ github.workspace }}/submodules/parent-images
-          git checkout ${{ github.event.pull_request.head.ref }}
-          pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
-          
-          cd ${{ github.workspace }}
-          git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
-          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
-
-          git add submodules
-          git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
-          git push origin ${{ github.event.pull_request.head.ref }}
-
-          gh pr create \
-            --title "${{ github.event.pull_request.title }}" \
-            --body "${{ github.event.pull_request.body }}" \
-            --base dev \
-            --head ${{ github.event.pull_request.head.ref }} \
-            --repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image
-          
-
   pi-build:
     name: 'Parent Images: Docker Build'
     runs-on: ubuntu-latest
-    needs: [pi-diff, pi-compile]
+    needs: [pi-diff]
     environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}

From 81840ccc81dd689ce865269b965762ecbf1344dd Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 15:14:31 +0100
Subject: [PATCH 36/63] test: git push

---
 .github/workflows/pi_build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 62d0380..108481f 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -60,6 +60,7 @@ jobs:
         with:
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
           submodules: 'true'
+          fetch-depth: 0
 
       - name: GitHub Configuration
         run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com

From e78533b4188cc176a8d2bef82562ebfe2f6203fa Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 15:16:20 +0100
Subject: [PATCH 37/63] test: git push

---
 .github/workflows/pi_build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 108481f..2c5c408 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -58,9 +58,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
+          token: ${{ secrets.GH_TOKEN }}
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
-          submodules: 'true'
           fetch-depth: 0
+          submodules: 'true'
 
       - name: GitHub Configuration
         run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com

From a805760e76a1b51af365e2d6c249d0f10423517a Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 15:29:34 +0100
Subject: [PATCH 38/63] perf: separate build from smodules merge

---
 .github/workflows/pi_build.yml          | 116 ++++++++++--------------
 .github/workflows/pi_smodules_merge.yml |  89 ++++++++++++++++++
 2 files changed, 135 insertions(+), 70 deletions(-)
 create mode 100644 .github/workflows/pi_smodules_merge.yml

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 2c5c408..f20f521 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -63,9 +63,6 @@ jobs:
           fetch-depth: 0
           submodules: 'true'
 
-      - name: GitHub Configuration
-        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
-
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
@@ -74,32 +71,11 @@ jobs:
       - name: Install Dependencies
         run: python -m pip install pip-tools
       
-      - name: Compile Artifacts
+      - name: Compile Requirements
         run: |
           cd ${{ github.workspace }}/submodules/parent-images
           git checkout ${{ github.event.pull_request.head.ref }}
           pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
-          
-          cd ${{ github.workspace }}
-          git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
-          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
-
-          git config user.email "devtools@kern.ai"
-          git config user.name "GitHub Actions"
-
-          git add submodules
-          git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
-          git push origin ${{ github.event.pull_request.head.ref }}
-
-          gh pr create \
-            --title "${{ github.event.pull_request.title }}" \
-            --body "${{ github.event.pull_request.body }}" \
-            --base dev \
-            --head ${{ github.event.pull_request.head.ref }} \
-            --repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image
-
-      # - name: Clone cicd-deployment-scripts
-      #   run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -118,58 +94,58 @@ jobs:
           username: "${{ env.DEV_LOGIN_USERNAME }}"
           password: "${{ env.DEV_LOGIN_PASSWORD }}"
 
-      - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev
+      - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+          platforms: linux/amd64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
+          push: true
+          build-args: |
+            platform=linux/amd64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+          platforms: linux/arm64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
+          push: true
+          build-args: |
+            platform=linux/arm64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}
         uses: docker/build-push-action@v5
         with:
           context: .
-          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-cache
-          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
           platforms: linux/amd64
           file: Dockerfile
-          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
           push: true
           build-args: |
             platform=linux/amd64
             label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
 
-      # - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:dev-arm64
-      #   uses: docker/build-push-action@v5
-      #   with:
-      #     context: .
-      #     cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache
-      #     cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
-      #     platforms: linux/arm64
-      #     file: Dockerfile
-      #     tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:dev-${{ matrix.parent_image_type }}-arm64
-      #     push: true
-      #     build-args: |
-      #       platform=linux/arm64
-      #       label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
-
-      # - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha
-      #   uses: docker/build-push-action@v5
-      #   with:
-      #     context: .
-      #     cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
-      #     cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
-      #     platforms: linux/amd64
-      #     file: Dockerfile
-      #     tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
-      #     push: true
-      #     build-args: |
-      #       platform=linux/amd64
-      #       label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
-
-      # - name: Build & Push refinery-${{ matrix.parent_image_type }}-parent-image:sha-arm64
-      #   uses: docker/build-push-action@v5
-      #   with:
-      #     context: .
-      #     cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
-      #     cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
-      #     platforms: linux/arm64
-      #     file: Dockerfile
-      #     tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
-      #     push: true
-      #     build-args: |
-      #       platform=linux/arm64
-      #       label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+      - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}-arm64
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+          platforms: linux/arm64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
+          push: true
+          build-args: |
+            platform=linux/arm64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
diff --git a/.github/workflows/pi_smodules_merge.yml b/.github/workflows/pi_smodules_merge.yml
new file mode 100644
index 0000000..cf00cf7
--- /dev/null
+++ b/.github/workflows/pi_smodules_merge.yml
@@ -0,0 +1,89 @@
+name: 'Parent Images: Submodules Merge'
+
+on:
+  workflow_call:
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+env:
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+jobs:
+  pi-diff:
+    name: 'Parent Images: Diff Types'
+    runs-on: ubuntu-latest
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    outputs:
+      updated_parent_types: ${{ steps.get-diff.outputs.updated_parent_types }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
+          ref: ${{ github.event.pull_request.head.ref }}
+
+      - name: Clone cicd-deployment-scripts
+        run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
+
+      - name: Get Diff Types
+        id: get-diff
+        run: |
+          bash cicd-deployment-scripts/pi/diff.sh \
+            -p "${{ github.event.pull_request.number }}"
+
+  pi-smodules-merge:
+    name: 'Parent Images: Submodules Merge'
+    runs-on: ubuntu-latest
+    needs: [pi-diff]
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    strategy:
+      matrix:
+        parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_TOKEN }}
+          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Dependencies
+        run: python -m pip install pip-tools
+      
+      - name: Perform Edit/Git Operations
+        run: |
+          cd ${{ github.workspace }}/submodules/parent-images
+          git checkout ${{ github.event.pull_request.base.ref }}
+          
+          cd ${{ github.workspace }}
+          git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
+          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
+
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
+          git add submodules
+          git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
+          git push origin ${{ github.event.pull_request.head.ref }}
+          echo "::notice::${{ github.event.repository.name }} updated to origin/${{ github.event.pull_request.head.ref }}"
+
+          gh pr create \
+            --title "${{ github.event.pull_request.title }}" \
+            --body "${{ github.event.pull_request.body }}" \
+            --base dev \
+            --head ${{ github.event.pull_request.head.ref }} \
+            --repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image

From 4e466b087c057225cb0796f6747f274d10bc48b7 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:08:59 +0100
Subject: [PATCH 39/63] perf: add strategy includes

---
 .github/workflows/pi_build.yml | 249 ++++++++++++++++++++-------------
 1 file changed, 150 insertions(+), 99 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index f20f521..3982ea3 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -40,112 +40,163 @@ jobs:
           bash cicd-deployment-scripts/pi/diff.sh \
             -p "${{ github.event.pull_request.number }}"
 
-  pi-build:
-    name: 'Parent Images: Docker Build'
+  pi-matrix-test:
+    name: 'Parent Images: Matrix Test'
     runs-on: ubuntu-latest
     needs: [pi-diff]
     environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
-      DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
-      DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
-      DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
-      IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
     strategy:
       matrix:
         parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
+        include:
+          - parent_image_type: common
+            app_repo: refinery-gateway
+          - parent_image_type: common
+            app_repo: refinery-neural-search
+          - parent_image_type: common
+            app_repo: refinery-tokenizer
+          - parent_image_type: common
+            app_repo: refinery-updater
+          - parent_image_type: common
+            app_repo: refinery-weak-supervisor
+          - parent_image_type: common
+            app_repo: refinery-model-provider
+          - parent_image_type: common
+            app_repo: cognition-gateway
+
+          - parent_image_type: exec-env
+            app_repo: refinery-ac-exec-env
+          - parent_image_type: exec-env
+            app_repo: refinery-lf-exec-env
+          - parent_image_type: exec-env
+            app_repo: cognition-exec-env
+        
+          - parent_image_type: torch-cpu
+            app_repo: refinery-embedder
+          - parent_image_type: torch-cpu
+            app_repo: refinery-ml-exec-env
+          
+          - parent_image_type: torch-cuda
+            app_repo: refinery-embedder
+
+          - parent_image_type: next
+            app_repo: admin-dashboard
+          - parent_image_type: next
+            app_repo: cognition-ui
+          - parent_image_type: next
+            app_repo: refinery-ui
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GH_TOKEN }}
-          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
-          fetch-depth: 0
-          submodules: 'true'
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install Dependencies
-        run: python -m pip install pip-tools
+      - name: Print matrix values - ${{ matrix.parent_image_type }}
+        run: echo ${{ matrix }}
+
+  # pi-build:
+  #   name: 'Parent Images: Docker Build'
+  #   runs-on: ubuntu-latest
+  #   needs: [pi-diff]
+  #   environment: dev
+  #   env:
+  #     PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+  #     DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
+  #     DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
+  #     DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
+  #     IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
+  #   strategy:
+  #     matrix:
+  #       parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
+  #   steps:
+  #     - name: Checkout repository
+  #       uses: actions/checkout@v4
+  #       with:
+  #         token: ${{ secrets.GH_TOKEN }}
+  #         repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+  #         fetch-depth: 0
+  #         submodules: 'true'
+
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v5
+  #       with:
+  #         python-version: ${{ env.PYTHON_VERSION }}
+
+  #     - name: Install Dependencies
+  #       run: python -m pip install pip-tools
       
-      - name: Compile Requirements
-        run: |
-          cd ${{ github.workspace }}/submodules/parent-images
-          git checkout ${{ github.event.pull_request.head.ref }}
-          pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          platforms: linux/amd64,linux/arm64
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: arm64,arm
-
-      - name: Log into DEV registry
-        uses: docker/login-action@v3
-        with:
-          registry: "${{ env.DEV_CONTAINER_REGISTRY }}"
-          username: "${{ env.DEV_LOGIN_USERNAME }}"
-          password: "${{ env.DEV_LOGIN_PASSWORD }}"
-
-      - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache
-          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
-          platforms: linux/amd64
-          file: Dockerfile
-          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
-          push: true
-          build-args: |
-            platform=linux/amd64
-            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
-
-      - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache
-          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
-          platforms: linux/arm64
-          file: Dockerfile
-          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
-          push: true
-          build-args: |
-            platform=linux/arm64
-            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
-
-      - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
-          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
-          platforms: linux/amd64
-          file: Dockerfile
-          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
-          push: true
-          build-args: |
-            platform=linux/amd64
-            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
-
-      - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}-arm64
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
-          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
-          platforms: linux/arm64
-          file: Dockerfile
-          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
-          push: true
-          build-args: |
-            platform=linux/arm64
-            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+  #     - name: Compile Requirements
+  #       run: |
+  #         cd ${{ github.workspace }}/submodules/parent-images
+  #         git checkout ${{ github.event.pull_request.head.ref }}
+  #         pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
+
+  #     - name: Set up Docker Buildx
+  #       uses: docker/setup-buildx-action@v3
+  #       with:
+  #         platforms: linux/amd64,linux/arm64
+
+  #     - name: Set up QEMU
+  #       uses: docker/setup-qemu-action@v3
+  #       with:
+  #         platforms: arm64,arm
+
+  #     - name: Log into DEV registry
+  #       uses: docker/login-action@v3
+  #       with:
+  #         registry: "${{ env.DEV_CONTAINER_REGISTRY }}"
+  #         username: "${{ env.DEV_LOGIN_USERNAME }}"
+  #         password: "${{ env.DEV_LOGIN_PASSWORD }}"
+
+  #     - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
+  #       uses: docker/build-push-action@v5
+  #       with:
+  #         context: .
+  #         cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache
+  #         cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+  #         platforms: linux/amd64
+  #         file: Dockerfile
+  #         tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
+  #         push: true
+  #         build-args: |
+  #           platform=linux/amd64
+  #           label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+  #     - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
+  #       uses: docker/build-push-action@v5
+  #       with:
+  #         context: .
+  #         cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache
+  #         cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+  #         platforms: linux/arm64
+  #         file: Dockerfile
+  #         tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
+  #         push: true
+  #         build-args: |
+  #           platform=linux/arm64
+  #           label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+  #     - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}
+  #       uses: docker/build-push-action@v5
+  #       with:
+  #         context: .
+  #         cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
+  #         cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+  #         platforms: linux/amd64
+  #         file: Dockerfile
+  #         tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
+  #         push: true
+  #         build-args: |
+  #           platform=linux/amd64
+  #           label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+  #     - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}-arm64
+  #       uses: docker/build-push-action@v5
+  #       with:
+  #         context: .
+  #         cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
+  #         cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+  #         platforms: linux/arm64
+  #         file: Dockerfile
+  #         tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
+  #         push: true
+  #         build-args: |
+  #           platform=linux/arm64
+  #           label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile

From 15164fe253a3be19f0d19e031783c7535340aca5 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:10:25 +0100
Subject: [PATCH 40/63] test: matrix executions

---
 .github/workflows/pi_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 3982ea3..c111929 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -89,7 +89,7 @@ jobs:
             app_repo: refinery-ui
     steps:
       - name: Print matrix values - ${{ matrix.parent_image_type }}
-        run: echo ${{ matrix }}
+        run: echo ${{ toJson(matrix) }}
 
   # pi-build:
   #   name: 'Parent Images: Docker Build'

From e7a1810e8d2342be7027183a16c4938017b84eeb Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:13:04 +0100
Subject: [PATCH 41/63] test: matrix executions

---
 .github/workflows/pi_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index c111929..cc8d5c9 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -89,7 +89,7 @@ jobs:
             app_repo: refinery-ui
     steps:
       - name: Print matrix values - ${{ matrix.parent_image_type }}
-        run: echo ${{ toJson(matrix) }}
+        run: echo "${{ toJson(matrix) }}"
 
   # pi-build:
   #   name: 'Parent Images: Docker Build'

From 26a5b4437991ac29c084ca56fc83e8dc4a9f329c Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:24:54 +0100
Subject: [PATCH 42/63] test: matrix executions

---
 .github/workflows/pi_build.yml | 42 ++--------------------------------
 1 file changed, 2 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index cc8d5c9..c39bde6 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -20,7 +20,7 @@ jobs:
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     outputs:
-      updated_parent_types: ${{ steps.get-diff.outputs.updated_parent_types }}
+      matrix: ${{ steps.get-diff.outputs.matrix }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -48,45 +48,7 @@ jobs:
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     strategy:
-      matrix:
-        parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
-        include:
-          - parent_image_type: common
-            app_repo: refinery-gateway
-          - parent_image_type: common
-            app_repo: refinery-neural-search
-          - parent_image_type: common
-            app_repo: refinery-tokenizer
-          - parent_image_type: common
-            app_repo: refinery-updater
-          - parent_image_type: common
-            app_repo: refinery-weak-supervisor
-          - parent_image_type: common
-            app_repo: refinery-model-provider
-          - parent_image_type: common
-            app_repo: cognition-gateway
-
-          - parent_image_type: exec-env
-            app_repo: refinery-ac-exec-env
-          - parent_image_type: exec-env
-            app_repo: refinery-lf-exec-env
-          - parent_image_type: exec-env
-            app_repo: cognition-exec-env
-        
-          - parent_image_type: torch-cpu
-            app_repo: refinery-embedder
-          - parent_image_type: torch-cpu
-            app_repo: refinery-ml-exec-env
-          
-          - parent_image_type: torch-cuda
-            app_repo: refinery-embedder
-
-          - parent_image_type: next
-            app_repo: admin-dashboard
-          - parent_image_type: next
-            app_repo: cognition-ui
-          - parent_image_type: next
-            app_repo: refinery-ui
+      matrix: ${{ fromJson(needs.pi-diff.outputs.matrix) }}
     steps:
       - name: Print matrix values - ${{ matrix.parent_image_type }}
         run: echo "${{ toJson(matrix) }}"

From 45d598bdea7279ae2fca72d0c83286b4c53031ef Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:25:34 +0100
Subject: [PATCH 43/63] test: matrix executions

---
 pi/diff.sh | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/pi/diff.sh b/pi/diff.sh
index 30aedc4..babe423 100644
--- a/pi/diff.sh
+++ b/pi/diff.sh
@@ -23,9 +23,21 @@ while IFS= read -r file; do
 
 done <<< "$UPDATED_FILES"
 
-JSON=""
+PARENT_IMAGE_TYPES=""
+INCLUDES=""
 for parent_image_type in "${UPDATED_PARENT_TYPES[@]}"; do
-    JSON+="\"$parent_image_type\","
+    PARENT_IMAGE_TYPES+='"'$parent_image_type'",'
+    eval 'APP_REPOS=( "${'$(echo ${parent_image_type} | sed "s|-|_|g")'[@]}" )'
+    for app in "${APP_REPOS[@]}"; do
+        INCLUDES+='{ "parent_image_type": "'${parent_image_type}'", "app": "'${app}'" },'
+    done
 done
-JSON="[${JSON::-1}]"
-echo "updated_parent_types=$JSON" >> $GITHUB_OUTPUT
\ No newline at end of file
+
+MATRIX=$(cat <<EOF
+{
+    "parent_image_type": [${PARENT_IMAGE_TYPES::-1}],
+    "include": [${INCLUDES::-1}]
+}
+EOF
+)
+echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
\ No newline at end of file

From 577a7e67cb4b5369282d8aee7b67eae3e10f8c84 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:27:24 +0100
Subject: [PATCH 44/63] test: matrix executions

---
 .github/workflows/pi_build.yml | 3 ++-
 pi/diff.sh                     | 6 +++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index c39bde6..2debca1 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -38,7 +38,8 @@ jobs:
         id: get-diff
         run: |
           bash cicd-deployment-scripts/pi/diff.sh \
-            -p "${{ github.event.pull_request.number }}"
+            -p "${{ github.event.pull_request.number }}" \
+            -s cicd-deployment-scripts/pi/settings.sh
 
   pi-matrix-test:
     name: 'Parent Images: Matrix Test'
diff --git a/pi/diff.sh b/pi/diff.sh
index babe423..0a09f7e 100644
--- a/pi/diff.sh
+++ b/pi/diff.sh
@@ -3,14 +3,18 @@
 set -e
 
 PR_NUMBER=""
+SOURCE_SCRIPT="pi/settings.sh"
 
-while getopts p: flag
+while getopts p:s: flag
 do
     case "${flag}" in
         p) PR_NUMBER=${OPTARG};;
+        s) SOURCE_SCRIPT=${OPTARG};;
     esac
 done
 
+source $SOURCE_SCRIPT
+
 UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only)
 UPDATED_PARENT_TYPES=()
 while IFS= read -r file; do

From 642a99c2ba95ebb102e700e1a29251016d652fe7 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:28:40 +0100
Subject: [PATCH 45/63] perf: add settings.sh

---
 pi/settings.sh | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 pi/settings.sh

diff --git a/pi/settings.sh b/pi/settings.sh
new file mode 100644
index 0000000..38d6720
--- /dev/null
+++ b/pi/settings.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+export PARENT_IMAGE_TYPES=(
+    mini
+    common
+    exec_env
+    torch_cpu
+    torch_cuda
+    next
+)
+
+export mini=(
+    "refinery-authorizer"
+    # "refinery-config"
+    # "refinery-doc-ock"
+    "refinery-gateway-proxy"
+    # "platform-monitoring"
+)
+
+export common=(
+    "refinery-gateway"
+    "refinery-neural-search"
+    "refinery-tokenizer"
+    "refinery-updater"
+    "refinery-weak-supervisor"
+    "refinery-model-provider"
+    # "refinery-commercial-proxy"
+    # "gates-gateway"
+    # "chat-gateway"
+    "cognition-gateway"
+)
+
+export exec_env=(
+    "refinery-ac-exec-env"
+    "refinery-lf-exec-env"
+    # "refinery-record-ide-env"
+    # "gates-runtime"
+    # "chat-exec-env"
+    "cognition-exec-env"
+)
+
+export torch_cpu=(
+    "refinery-embedder"
+    "refinery-ml-exec-env"
+    # "refinery-zero-shot"
+    # "hosted-inference-api"
+)
+
+export torch_gpu=(
+    "refinery-embedder"
+    # "refinery-zero-shot"
+)
+
+export next=(
+    # "gates-ui"
+    "admin-dashboard"
+    # "chat-ui"
+    "cognition-ui"
+    "refinery-ui"
+)
+
+export ALL_SERVICES=( "${mini[@]}" "${common[@]}" "${exec_env[@]}" "${torch_cpu[@]}" "${torch_gpu[@]}" )
\ No newline at end of file

From 1c2f8d9a7ddd45ee8b7b61975c8ff72680ba3168 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:32:58 +0100
Subject: [PATCH 46/63] fix: outputting matrix

---
 pi/diff.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pi/diff.sh b/pi/diff.sh
index 0a09f7e..8fbacc2 100644
--- a/pi/diff.sh
+++ b/pi/diff.sh
@@ -44,4 +44,5 @@ MATRIX=$(cat <<EOF
 }
 EOF
 )
-echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
\ No newline at end of file
+echo $MATRIX | jq -c --indent 2 '.'
+echo "matrix='$MATRIX'" >> $GITHUB_OUTPUT
\ No newline at end of file

From f0eee7ea9d32f22f004017333e8cb9407a38310c Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:34:32 +0100
Subject: [PATCH 47/63] fix: outputting matrix

---
 pi/diff.sh | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/pi/diff.sh b/pi/diff.sh
index 8fbacc2..f6fa4f3 100644
--- a/pi/diff.sh
+++ b/pi/diff.sh
@@ -37,12 +37,6 @@ for parent_image_type in "${UPDATED_PARENT_TYPES[@]}"; do
     done
 done
 
-MATRIX=$(cat <<EOF
-{
-    "parent_image_type": [${PARENT_IMAGE_TYPES::-1}],
-    "include": [${INCLUDES::-1}]
-}
-EOF
-)
+MATRIX='{"parent_image_type": ['${PARENT_IMAGE_TYPES::-1}'],"include": ['${INCLUDES::-1}']}'
 echo $MATRIX | jq -c --indent 2 '.'
 echo "matrix='$MATRIX'" >> $GITHUB_OUTPUT
\ No newline at end of file

From 502e7de0b0ca5c5a92742b0a97fb9bebd5221e64 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:38:20 +0100
Subject: [PATCH 48/63] fix: outputting matrix

---
 .github/workflows/pi_build.yml | 7 +++++--
 pi/diff.sh                     | 5 +++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 2debca1..1f285e7 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -20,7 +20,8 @@ jobs:
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     outputs:
-      matrix: ${{ steps.get-diff.outputs.matrix }}
+      parent_image_type: ${{ steps.get-diff.outputs.parent_image_type }}
+      include: ${{ steps.get-diff.outputs.include }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -49,7 +50,9 @@ jobs:
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     strategy:
-      matrix: ${{ fromJson(needs.pi-diff.outputs.matrix) }}
+      matrix: 
+        parent_image_type: ${{ fromJson(needs.pi-diff.outputs.parent_image_type) }}
+        include: ${{ fromJson(needs.pi-diff.outputs.include) }}
     steps:
       - name: Print matrix values - ${{ matrix.parent_image_type }}
         run: echo "${{ toJson(matrix) }}"
diff --git a/pi/diff.sh b/pi/diff.sh
index f6fa4f3..195bb78 100644
--- a/pi/diff.sh
+++ b/pi/diff.sh
@@ -38,5 +38,6 @@ for parent_image_type in "${UPDATED_PARENT_TYPES[@]}"; do
 done
 
 MATRIX='{"parent_image_type": ['${PARENT_IMAGE_TYPES::-1}'],"include": ['${INCLUDES::-1}']}'
-echo $MATRIX | jq -c --indent 2 '.'
-echo "matrix='$MATRIX'" >> $GITHUB_OUTPUT
\ No newline at end of file
+echo $MATRIX | jq -C --indent 2 '.'
+echo "parent_image_type='[${PARENT_IMAGE_TYPES::-1}]'" >> $GITHUB_OUTPUT
+echo "include='[${INCLUDES::-1}]'" >> $GITHUB_OUTPUT
\ No newline at end of file

From ec5f83090c0cf4cc2b8ed3a48f229150f96d73bb Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:39:18 +0100
Subject: [PATCH 49/63] fix: outputting matrix

---
 pi/diff.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pi/diff.sh b/pi/diff.sh
index 195bb78..bfce2b0 100644
--- a/pi/diff.sh
+++ b/pi/diff.sh
@@ -39,5 +39,5 @@ done
 
 MATRIX='{"parent_image_type": ['${PARENT_IMAGE_TYPES::-1}'],"include": ['${INCLUDES::-1}']}'
 echo $MATRIX | jq -C --indent 2 '.'
-echo "parent_image_type='[${PARENT_IMAGE_TYPES::-1}]'" >> $GITHUB_OUTPUT
-echo "include='[${INCLUDES::-1}]'" >> $GITHUB_OUTPUT
\ No newline at end of file
+echo "parent_image_type=[${PARENT_IMAGE_TYPES::-1}]" >> $GITHUB_OUTPUT
+echo "include=[${INCLUDES::-1}]" >> $GITHUB_OUTPUT
\ No newline at end of file

From 72063ac0e2a9763425c957a36f49d279ac42d85b Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 16:44:53 +0100
Subject: [PATCH 50/63] fix: outputting matrix

---
 .github/workflows/pi_build.yml | 2 --
 pi/diff.sh                     | 5 +----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 1f285e7..320a532 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -20,7 +20,6 @@ jobs:
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     outputs:
-      parent_image_type: ${{ steps.get-diff.outputs.parent_image_type }}
       include: ${{ steps.get-diff.outputs.include }}
     steps:
       - name: Checkout Repository
@@ -51,7 +50,6 @@ jobs:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     strategy:
       matrix: 
-        parent_image_type: ${{ fromJson(needs.pi-diff.outputs.parent_image_type) }}
         include: ${{ fromJson(needs.pi-diff.outputs.include) }}
     steps:
       - name: Print matrix values - ${{ matrix.parent_image_type }}
diff --git a/pi/diff.sh b/pi/diff.sh
index bfce2b0..f63041c 100644
--- a/pi/diff.sh
+++ b/pi/diff.sh
@@ -27,17 +27,14 @@ while IFS= read -r file; do
 
 done <<< "$UPDATED_FILES"
 
-PARENT_IMAGE_TYPES=""
 INCLUDES=""
 for parent_image_type in "${UPDATED_PARENT_TYPES[@]}"; do
-    PARENT_IMAGE_TYPES+='"'$parent_image_type'",'
     eval 'APP_REPOS=( "${'$(echo ${parent_image_type} | sed "s|-|_|g")'[@]}" )'
     for app in "${APP_REPOS[@]}"; do
         INCLUDES+='{ "parent_image_type": "'${parent_image_type}'", "app": "'${app}'" },'
     done
 done
 
-MATRIX='{"parent_image_type": ['${PARENT_IMAGE_TYPES::-1}'],"include": ['${INCLUDES::-1}']}'
+MATRIX='{"include": ['${INCLUDES::-1}']}'
 echo $MATRIX | jq -C --indent 2 '.'
-echo "parent_image_type=[${PARENT_IMAGE_TYPES::-1}]" >> $GITHUB_OUTPUT
 echo "include=[${INCLUDES::-1}]" >> $GITHUB_OUTPUT
\ No newline at end of file

From 2adb2302f6e9b16b233aa30cb7197437e8f0d57c Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 17:18:18 +0100
Subject: [PATCH 51/63] perf: create PR for app requirements

---
 .github/workflows/pi_build.yml | 41 ++++++++++++++++++++++++++++++++--
 1 file changed, 39 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 320a532..28c592c 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -52,8 +52,45 @@ jobs:
       matrix: 
         include: ${{ fromJson(needs.pi-diff.outputs.include) }}
     steps:
-      - name: Print matrix values - ${{ matrix.parent_image_type }}
-        run: echo "${{ toJson(matrix) }}"
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          submodules: 'true'
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Dependencies
+        run: python -m pip install pip-tools
+      
+      - name: Compile Requirements
+        run: pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
+
+      - name: Create PR - ${{ matrix.app }}
+        run: |
+          git clone https://oauth2:${{ secrets.GH_TOKEN }}@github.com/${{ github.repository_owner }}/${{ matrix.app }}.git
+          cd ${{ matrix.app }}
+
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
+          git checkout -b ${{ github.event.pull_request.head.ref }} || git checkout ${{ github.event.pull_request.head.ref }}
+          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
+
+          git add requirements/${{ matrix.parent_image_type }}-requirements.txt
+          git commit -m "ci: update ${{ matrix.parent_image_type }}-requirements.txt"
+          git push origin ${{ github.event.pull_request.head.ref }}
+          echo "::notice::${{ matrix.app }} updated to origin/${{ github.event.pull_request.head.ref }}"
+
+          gh pr create \
+            --title "${{ github.event.pull_request.title }}" \
+            --body "${{ github.event.pull_request.body }}" \
+            --base dev \
+            --head ${{ github.event.pull_request.head.ref }} \
+            --repo ${{ github.repository_owner }}/${{ matrix.app }}
 
   # pi-build:
   #   name: 'Parent Images: Docker Build'

From 205d0a1075b8ba24dacf552fdea8a8947a766608 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 17:20:09 +0100
Subject: [PATCH 52/63] fix: pip-compile path

---
 .github/workflows/pi_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 28c592c..2502326 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -67,7 +67,7 @@ jobs:
         run: python -m pip install pip-tools
       
       - name: Compile Requirements
-        run: pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
+        run: pip-compile submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
 
       - name: Create PR - ${{ matrix.app }}
         run: |

From 844731fcaa91ebd49151ef9a30d83c80e76c4dec Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 17:25:00 +0100
Subject: [PATCH 53/63] fix: pi build order of operations

---
 .github/workflows/pi_build.yml | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 2502326..3574eda 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -62,16 +62,21 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-
+    
       - name: Install Dependencies
         run: python -m pip install pip-tools
       
+      - name: Clone ${{ matrix.app }}
+        run: git clone https://oauth2:${{ secrets.GH_TOKEN }}@github.com/${{ github.repository_owner }}/${{ matrix.app }}.git
+      
       - name: Compile Requirements
-        run: pip-compile submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
+        run: |
+          pip-compile \
+            --output-file ${{ matrix.app }}/requirements/${{ matrix.parent_image_type }}-requirements.txt \
+            submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
 
-      - name: Create PR - ${{ matrix.app }}
+      - name: Perform Edit/Git Operations
         run: |
-          git clone https://oauth2:${{ secrets.GH_TOKEN }}@github.com/${{ github.repository_owner }}/${{ matrix.app }}.git
           cd ${{ matrix.app }}
 
           git config user.email "devtools@kern.ai"

From b61ae1c1a50893fcd6a7ca44c06bfcf0878fb61b Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 17:27:36 +0100
Subject: [PATCH 54/63] style: rename jobs and steps

---
 .github/workflows/pi_build.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 3574eda..5f40b54 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -13,14 +13,14 @@ env:
   GH_TOKEN: ${{ secrets.GH_TOKEN }}
 
 jobs:
-  pi-diff:
-    name: 'Parent Images: Diff Types'
+  pi-matrix:
+    name: 'Parent Images: Generate Matrix'
     runs-on: ubuntu-latest
     environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     outputs:
-      include: ${{ steps.get-diff.outputs.include }}
+      include: ${{ steps.gen-matrix.outputs.include }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -34,23 +34,23 @@ jobs:
       - name: Clone cicd-deployment-scripts
         run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
 
-      - name: Get Diff Types
-        id: get-diff
+      - name: Generate Matrix
+        id: gen-matrix
         run: |
-          bash cicd-deployment-scripts/pi/diff.sh \
+          bash cicd-deployment-scripts/pi/generate_matrix.sh \
             -p "${{ github.event.pull_request.number }}" \
             -s cicd-deployment-scripts/pi/settings.sh
 
-  pi-matrix-test:
-    name: 'Parent Images: Matrix Test'
+  pi-update-app:
+    name: 'Parent Images: Update'
     runs-on: ubuntu-latest
-    needs: [pi-diff]
+    needs: [pi-matrix]
     environment: dev
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     strategy:
       matrix: 
-        include: ${{ fromJson(needs.pi-diff.outputs.include) }}
+        include: ${{ fromJson(needs.pi-matrix.outputs.include) }}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -100,7 +100,7 @@ jobs:
   # pi-build:
   #   name: 'Parent Images: Docker Build'
   #   runs-on: ubuntu-latest
-  #   needs: [pi-diff]
+  #   needs: [pi-matrix]
   #   environment: dev
   #   env:
   #     PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
@@ -110,7 +110,7 @@ jobs:
   #     IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
   #   strategy:
   #     matrix:
-  #       parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
+  #       parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.updated_parent_types) }}
   #   steps:
   #     - name: Checkout repository
   #       uses: actions/checkout@v4

From a71c996457eb1212388d744e05dea7a049c81b23 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 17:32:58 +0100
Subject: [PATCH 55/63] feat: rename diff.sh

---
 pi/{diff.sh => generate_matrix.sh} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename pi/{diff.sh => generate_matrix.sh} (100%)

diff --git a/pi/diff.sh b/pi/generate_matrix.sh
similarity index 100%
rename from pi/diff.sh
rename to pi/generate_matrix.sh

From 634331e343415e21979c8e5b7056cae4ccede196 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 17:37:13 +0100
Subject: [PATCH 56/63] perf: requirements compilation

---
 .github/workflows/pi_build.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 5f40b54..1e62107 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -74,6 +74,10 @@ jobs:
           pip-compile \
             --output-file ${{ matrix.app }}/requirements/${{ matrix.parent_image_type }}-requirements.txt \
             submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
+          
+          pip-compile \
+            --output-file ${{ matrix.app }}/requirements.txt \
+            ${{ matrix.app }}/requirements/requirements.in
 
       - name: Perform Edit/Git Operations
         run: |
@@ -85,7 +89,7 @@ jobs:
           git checkout -b ${{ github.event.pull_request.head.ref }} || git checkout ${{ github.event.pull_request.head.ref }}
           git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
 
-          git add requirements/${{ matrix.parent_image_type }}-requirements.txt
+          git add requirements*
           git commit -m "ci: update ${{ matrix.parent_image_type }}-requirements.txt"
           git push origin ${{ github.event.pull_request.head.ref }}
           echo "::notice::${{ matrix.app }} updated to origin/${{ github.event.pull_request.head.ref }}"

From 3a2ac3fdd445e48197df764d86156defeda2a8ec Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 17:39:19 +0100
Subject: [PATCH 57/63] test: log outputs

---
 .github/workflows/pi_build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 1e62107..780f8b7 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -73,11 +73,11 @@ jobs:
         run: |
           pip-compile \
             --output-file ${{ matrix.app }}/requirements/${{ matrix.parent_image_type }}-requirements.txt \
-            submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
+            submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in 1> /dev/null
           
           pip-compile \
             --output-file ${{ matrix.app }}/requirements.txt \
-            ${{ matrix.app }}/requirements/requirements.in
+            ${{ matrix.app }}/requirements/requirements.in 1> /dev/null
 
       - name: Perform Edit/Git Operations
         run: |

From 994c221d9312b0d8f7d3fb88efd8ab7a3d8efc70 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 17:41:59 +0100
Subject: [PATCH 58/63] chore: continue-on-error

---
 .github/workflows/pi_build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 780f8b7..3f86649 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -46,6 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [pi-matrix]
     environment: dev
+    continue-on-error: true
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
     strategy:

From b7006b61c9ff3e5e87593bf08eff111b40fed8b6 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Thu, 5 Dec 2024 17:44:45 +0100
Subject: [PATCH 59/63] chore: quiet pip-compile

---
 .github/workflows/pi_build.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index 3f86649..fc360bd 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -72,13 +72,13 @@ jobs:
       
       - name: Compile Requirements
         run: |
-          pip-compile \
+          pip-compile --quiet \
             --output-file ${{ matrix.app }}/requirements/${{ matrix.parent_image_type }}-requirements.txt \
-            submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in 1> /dev/null
+            submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
           
-          pip-compile \
+          pip-compile --quiet \
             --output-file ${{ matrix.app }}/requirements.txt \
-            ${{ matrix.app }}/requirements/requirements.in 1> /dev/null
+            ${{ matrix.app }}/requirements/requirements.in
 
       - name: Perform Edit/Git Operations
         run: |

From 51a26ee980b1fb69a4f56c8779bdbdaa48c4a6c5 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Fri, 6 Dec 2024 01:41:00 +0100
Subject: [PATCH 60/63] feat: make pi-matrix reusable

---
 .github/workflows/pi_matrix.yml      | 48 ++++++++++++++++++++++++++++
 pi/{generate_matrix.sh => matrix.sh} | 13 ++++++--
 2 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/pi_matrix.yml
 rename pi/{generate_matrix.sh => matrix.sh} (71%)

diff --git a/.github/workflows/pi_matrix.yml b/.github/workflows/pi_matrix.yml
new file mode 100644
index 0000000..1b73e1b
--- /dev/null
+++ b/.github/workflows/pi_matrix.yml
@@ -0,0 +1,48 @@
+name: 'Parent Images: Matrix'
+
+on:
+  workflow_call:
+    inputs:
+      parent_image_type:
+        description: 'If specified, "include" only outputs apps associated to this parent image type'
+        required: false
+        type: string
+        default: ''
+    outputs:
+      parent_image_type:
+        description: 'List[str] of parent image types'
+        value: ${{ jobs.pi-matrix.outputs.parent_image_type }}
+      include:
+        description: 'List[Dict] of apps associated to parent image types'
+        value: ${{ jobs.pi-matrix.outputs.include }}
+
+jobs:
+  pi-matrix:
+    name: 'Parent Images: Generate Matrix'
+    runs-on: ubuntu-latest
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    outputs:
+      parent_image_type: ${{ steps.generate-matrix.outputs.parent_image_type }}
+      include: ${{ steps.generate-matrix.outputs.include }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
+          ref: ${{ github.event.pull_request.head.ref }}
+    
+      - name: GitHub Configuration
+        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+
+      - name: Clone cicd-deployment-scripts
+        run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
+
+      - name: Generate Matrix
+        id: generate-matrix
+        run: |
+          bash cicd-deployment-scripts/pi/matrix.sh \
+            -p "${{ github.event.pull_request.number }}" \
+            -s cicd-deployment-scripts/pi/settings.sh \
+            -t "${{ inputs.parent_image_type }}"
diff --git a/pi/generate_matrix.sh b/pi/matrix.sh
similarity index 71%
rename from pi/generate_matrix.sh
rename to pi/matrix.sh
index f63041c..82237e4 100644
--- a/pi/generate_matrix.sh
+++ b/pi/matrix.sh
@@ -2,12 +2,14 @@
 
 set -e
 
+PARENT_IMAGE_TYPE=""
 PR_NUMBER=""
 SOURCE_SCRIPT="pi/settings.sh"
 
-while getopts p:s: flag
+while getopts t:p:s: flag
 do
     case "${flag}" in
+        t) PARENT_IMAGE_TYPE=${OPTARG};;
         p) PR_NUMBER=${OPTARG};;
         s) SOURCE_SCRIPT=${OPTARG};;
     esac
@@ -27,8 +29,14 @@ while IFS= read -r file; do
 
 done <<< "$UPDATED_FILES"
 
+if [ -n $PARENT_IMAGE_TYPE ]; then
+    UPDATED_PARENT_TYPES=( $PARENT_IMAGE_TYPE )
+fi
+
+PARENT_IMAGE_TYPES=""
 INCLUDES=""
 for parent_image_type in "${UPDATED_PARENT_TYPES[@]}"; do
+    PARENT_IMAGE_TYPES+="\"$parent_image_type\","
     eval 'APP_REPOS=( "${'$(echo ${parent_image_type} | sed "s|-|_|g")'[@]}" )'
     for app in "${APP_REPOS[@]}"; do
         INCLUDES+='{ "parent_image_type": "'${parent_image_type}'", "app": "'${app}'" },'
@@ -37,4 +45,5 @@ done
 
 MATRIX='{"include": ['${INCLUDES::-1}']}'
 echo $MATRIX | jq -C --indent 2 '.'
-echo "include=[${INCLUDES::-1}]" >> $GITHUB_OUTPUT
\ No newline at end of file
+echo "include=[${INCLUDES::-1}]" >> $GITHUB_OUTPUT
+echo "parent_image_type=[${PARENT_IMAGE_TYPES::-1}]" >> $GITHUB_OUTPUT
\ No newline at end of file

From 8307ba755fbc352ee952e9ed6c89b9e43314053a Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Fri, 6 Dec 2024 01:43:42 +0100
Subject: [PATCH 61/63] feat: add merge, release workflows

---
 .github/workflows/pi_build.yml          | 259 ++++++++----------------
 .github/workflows/pi_merge.yml          | 131 ++++++++++++
 .github/workflows/pi_release.yml        |  80 ++++++++
 .github/workflows/pi_smodules_merge.yml |  89 --------
 pi/edit_dockerfile.sh                   |  27 +++
 pi/pr_create.sh                         |  58 ++++++
 6 files changed, 383 insertions(+), 261 deletions(-)
 create mode 100644 .github/workflows/pi_merge.yml
 create mode 100644 .github/workflows/pi_release.yml
 delete mode 100644 .github/workflows/pi_smodules_merge.yml
 create mode 100644 pi/edit_dockerfile.sh
 create mode 100644 pi/pr_create.sh

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index fc360bd..f2a5769 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -14,200 +14,115 @@ env:
 
 jobs:
   pi-matrix:
-    name: 'Parent Images: Generate Matrix'
-    runs-on: ubuntu-latest
-    environment: dev
-    env:
-      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
-    outputs:
-      include: ${{ steps.gen-matrix.outputs.include }}
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
-          ref: ${{ github.event.pull_request.head.ref }}
-    
-      - name: GitHub Configuration
-        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
+    secrets: inherit
 
-      - name: Clone cicd-deployment-scripts
-        run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
-
-      - name: Generate Matrix
-        id: gen-matrix
-        run: |
-          bash cicd-deployment-scripts/pi/generate_matrix.sh \
-            -p "${{ github.event.pull_request.number }}" \
-            -s cicd-deployment-scripts/pi/settings.sh
-
-  pi-update-app:
-    name: 'Parent Images: Update'
+  pi-build:
+    name: 'Parent Images: Docker Build'
     runs-on: ubuntu-latest
     needs: [pi-matrix]
     environment: dev
-    continue-on-error: true
     env:
       PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+      DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
+      DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
+      DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
+      IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
     strategy:
-      matrix: 
-        include: ${{ fromJson(needs.pi-matrix.outputs.include) }}
+      matrix:
+        parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }}
     steps:
-      - name: Checkout Repository
+      - name: Checkout repository
         uses: actions/checkout@v4
         with:
+          token: ${{ secrets.GH_TOKEN }}
           repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          fetch-depth: 0
           submodules: 'true'
-      
+
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-    
+
       - name: Install Dependencies
         run: python -m pip install pip-tools
       
-      - name: Clone ${{ matrix.app }}
-        run: git clone https://oauth2:${{ secrets.GH_TOKEN }}@github.com/${{ github.repository_owner }}/${{ matrix.app }}.git
-      
       - name: Compile Requirements
         run: |
-          pip-compile --quiet \
-            --output-file ${{ matrix.app }}/requirements/${{ matrix.parent_image_type }}-requirements.txt \
-            submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
-          
-          pip-compile --quiet \
-            --output-file ${{ matrix.app }}/requirements.txt \
-            ${{ matrix.app }}/requirements/requirements.in
-
-      - name: Perform Edit/Git Operations
-        run: |
-          cd ${{ matrix.app }}
-
-          git config user.email "devtools@kern.ai"
-          git config user.name "GitHub Actions"
+          cd ${{ github.workspace }}/submodules/parent-images
+          git checkout ${{ github.event.pull_request.head.ref }}
+          pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
 
-          git checkout -b ${{ github.event.pull_request.head.ref }} || git checkout ${{ github.event.pull_request.head.ref }}
-          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
-
-          git add requirements*
-          git commit -m "ci: update ${{ matrix.parent_image_type }}-requirements.txt"
-          git push origin ${{ github.event.pull_request.head.ref }}
-          echo "::notice::${{ matrix.app }} updated to origin/${{ github.event.pull_request.head.ref }}"
-
-          gh pr create \
-            --title "${{ github.event.pull_request.title }}" \
-            --body "${{ github.event.pull_request.body }}" \
-            --base dev \
-            --head ${{ github.event.pull_request.head.ref }} \
-            --repo ${{ github.repository_owner }}/${{ matrix.app }}
-
-  # pi-build:
-  #   name: 'Parent Images: Docker Build'
-  #   runs-on: ubuntu-latest
-  #   needs: [pi-matrix]
-  #   environment: dev
-  #   env:
-  #     PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
-  #     DEV_CONTAINER_REGISTRY: ${{ vars.DEV_CONTAINER_REGISTRY }}
-  #     DEV_LOGIN_USERNAME: ${{ secrets.DEV_LOGIN_USERNAME }}
-  #     DEV_LOGIN_PASSWORD: ${{ secrets.DEV_LOGIN_PASSWORD }}
-  #     IMAGE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event.pull_request.head.ref }}
-  #   strategy:
-  #     matrix:
-  #       parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.updated_parent_types) }}
-  #   steps:
-  #     - name: Checkout repository
-  #       uses: actions/checkout@v4
-  #       with:
-  #         token: ${{ secrets.GH_TOKEN }}
-  #         repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
-  #         fetch-depth: 0
-  #         submodules: 'true'
-
-  #     - name: Set up Python
-  #       uses: actions/setup-python@v5
-  #       with:
-  #         python-version: ${{ env.PYTHON_VERSION }}
-
-  #     - name: Install Dependencies
-  #       run: python -m pip install pip-tools
-      
-  #     - name: Compile Requirements
-  #       run: |
-  #         cd ${{ github.workspace }}/submodules/parent-images
-  #         git checkout ${{ github.event.pull_request.head.ref }}
-  #         pip-compile requirements/${{ matrix.parent_image_type }}-requirements.in
-
-  #     - name: Set up Docker Buildx
-  #       uses: docker/setup-buildx-action@v3
-  #       with:
-  #         platforms: linux/amd64,linux/arm64
-
-  #     - name: Set up QEMU
-  #       uses: docker/setup-qemu-action@v3
-  #       with:
-  #         platforms: arm64,arm
-
-  #     - name: Log into DEV registry
-  #       uses: docker/login-action@v3
-  #       with:
-  #         registry: "${{ env.DEV_CONTAINER_REGISTRY }}"
-  #         username: "${{ env.DEV_LOGIN_USERNAME }}"
-  #         password: "${{ env.DEV_LOGIN_PASSWORD }}"
-
-  #     - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
-  #       uses: docker/build-push-action@v5
-  #       with:
-  #         context: .
-  #         cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache
-  #         cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
-  #         platforms: linux/amd64
-  #         file: Dockerfile
-  #         tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
-  #         push: true
-  #         build-args: |
-  #           platform=linux/amd64
-  #           label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
 
-  #     - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
-  #       uses: docker/build-push-action@v5
-  #       with:
-  #         context: .
-  #         cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache
-  #         cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
-  #         platforms: linux/arm64
-  #         file: Dockerfile
-  #         tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
-  #         push: true
-  #         build-args: |
-  #           platform=linux/arm64
-  #           label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64,arm
 
-  #     - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}
-  #       uses: docker/build-push-action@v5
-  #       with:
-  #         context: .
-  #         cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
-  #         cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
-  #         platforms: linux/amd64
-  #         file: Dockerfile
-  #         tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
-  #         push: true
-  #         build-args: |
-  #           platform=linux/amd64
-  #           label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+      - name: Log into DEV registry
+        uses: docker/login-action@v3
+        with:
+          registry: "${{ env.DEV_CONTAINER_REGISTRY }}"
+          username: "${{ env.DEV_LOGIN_USERNAME }}"
+          password: "${{ env.DEV_LOGIN_PASSWORD }}"
 
-  #     - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}-arm64
-  #       uses: docker/build-push-action@v5
-  #       with:
-  #         context: .
-  #         cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
-  #         cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
-  #         platforms: linux/arm64
-  #         file: Dockerfile
-  #         tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
-  #         push: true
-  #         build-args: |
-  #           platform=linux/arm64
-  #           label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+      - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+          platforms: linux/amd64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}
+          push: true
+          build-args: |
+            platform=linux/amd64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+          platforms: linux/arm64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.event.pull_request.head.ref }}-${{ matrix.parent_image_type }}-arm64
+          push: true
+          build-args: |
+            platform=linux/arm64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-cache,mode=max,image-manifest=true
+          platforms: linux/amd64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}
+          push: true
+          build-args: |
+            platform=linux/amd64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
+
+      - name: Build & Push refinery-parent-images:sha-${{ matrix.parent_image_type }}-arm64
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache
+          cache-to: type=registry,ref=${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64-cache,mode=max,image-manifest=true
+          platforms: linux/arm64
+          file: Dockerfile
+          tags: ${{ env.DEV_CONTAINER_REGISTRY }}/refinery-parent-images:${{ github.sha }}-${{ matrix.parent_image_type }}-arm64
+          push: true
+          build-args: |
+            platform=linux/arm64
+            label=dockerfile-path=https://github.com/refinery-${{ matrix.parent_image_type }}-parent-image/blob/${{ github.sha }}/Dockerfile
diff --git a/.github/workflows/pi_merge.yml b/.github/workflows/pi_merge.yml
new file mode 100644
index 0000000..ae9b6ae
--- /dev/null
+++ b/.github/workflows/pi_merge.yml
@@ -0,0 +1,131 @@
+name: 'Parent Images: Submodules Merge'
+
+on:
+  workflow_call:
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+env:
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+jobs:
+  pi-matrix:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
+    secrets: inherit
+
+  pi-update-submodule:
+    name: 'Parent Images: Submodules'
+    runs-on: ubuntu-latest
+    needs: [pi-matrix]
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    strategy:
+      matrix:
+        parent_image_type: ${{ fromJson(needs.pi-matrix.outputs.parent_image_type) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_TOKEN }}
+          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Dependencies
+        run: python -m pip install pip-tools
+      
+      - name: Perform Edit/Git Operations
+        run: |
+          cd ${{ github.workspace }}/submodules/parent-images
+          git checkout ${{ github.event.pull_request.base.ref }}
+          
+          cd ${{ github.workspace }}
+          git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
+          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
+
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
+          git add submodules
+          git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
+          git push origin ${{ github.event.pull_request.head.ref }}
+          echo "::notice::${{ github.event.repository.name }} updated to origin/${{ github.event.pull_request.head.ref }}"
+
+          gh pr create --draft \
+            --title "${{ github.event.pull_request.title }}" \
+            --body "${{ github.event.pull_request.body }}" \
+            --base dev \
+            --head ${{ github.event.pull_request.head.ref }} \
+            --repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image
+
+  pi-update-app:
+    name: 'Parent Images: ' # suffix populated by matrix
+    runs-on: ubuntu-latest
+    needs: [pi-matrix]
+    environment: dev
+    continue-on-error: true
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+    strategy:
+      matrix: 
+        include: ${{ fromJson(needs.pi-matrix.outputs.include) }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
+          submodules: 'true'
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+    
+      - name: Install Dependencies
+        run: python -m pip install pip-tools
+      
+      - name: Clone ${{ matrix.app }}
+        run: git clone https://oauth2:${{ secrets.GH_TOKEN }}@github.com/${{ github.repository_owner }}/${{ matrix.app }}.git
+      
+      - name: Compile Requirements
+        run: |
+          pip-compile --quiet \
+            --output-file ${{ matrix.app }}/requirements/${{ matrix.parent_image_type }}-requirements.txt \
+            submodules/parent-images/requirements/${{ matrix.parent_image_type }}-requirements.in
+          
+          pip-compile --quiet \
+            --output-file ${{ matrix.app }}/requirements.txt \
+            ${{ matrix.app }}/requirements/requirements.in
+
+      - name: Perform Edit/Git Operations
+        run: |
+          cd ${{ matrix.app }}
+
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
+          git checkout -b ${{ github.event.pull_request.head.ref }} || git checkout ${{ github.event.pull_request.head.ref }}
+          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
+
+          git add requirements*
+          git commit -m "ci: update ${{ matrix.parent_image_type }}-requirements.txt"
+          git push origin ${{ github.event.pull_request.head.ref }}
+          echo "::notice::${{ matrix.app }} updated to origin/${{ github.event.pull_request.head.ref }}"
+
+          gh pr create --draft \
+            --title "${{ github.event.pull_request.title }}" \
+            --body "${{ github.event.pull_request.body }}" \
+            --base dev \
+            --head ${{ github.event.pull_request.head.ref }} \
+            --repo ${{ github.repository_owner }}/${{ matrix.app }}
+          
\ No newline at end of file
diff --git a/.github/workflows/pi_release.yml b/.github/workflows/pi_release.yml
new file mode 100644
index 0000000..79f9692
--- /dev/null
+++ b/.github/workflows/pi_release.yml
@@ -0,0 +1,80 @@
+name: 'Parent Images: Release'
+
+on:
+  workflow_call:
+
+# Special permissions required for OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
+env:
+  GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+jobs:
+  pi-matrix:
+    uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
+    secrets: inherit
+    with:
+      parent_image_type: ${{ vars.PARENT_IMAGE_TYPE }}
+
+  pi-edit:
+    name: 'Parent Images: Dockerfile ' # suffix populated by matrix
+    runs-on: ubuntu-latest
+    needs: [pi-matrix]
+    environment: dev
+    env:
+      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
+      DOCKERHUB_CONTAINER_REGISTRY: ${{ vars.DOCKERHUB_CONTAINER_REGISTRY }}
+      DOCKERHUB_LOGIN_USERNAME: ${{ secrets.DOCKERHUB_LOGIN_USERNAME }}
+      DOCKERHUB_LOGIN_PASSWORD: ${{ secrets.DOCKERHUB_LOGIN_PASSWORD }}
+      PARENT_IMAGE_NAME: ${{ vars.PARENT_IMAGE_NAME }}
+      PARENT_IMAGE_TYPE: ${{ vars.PARENT_IMAGE_TYPE }}
+      DOCKERFILE: ${{ vars.DOCKERFILE }}
+    strategy:
+      matrix:
+        include: ${{ fromJson(needs.pi-matrix.outputs.include) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GH_TOKEN }}
+          repository: ${{ github.repository_owner }}/${{ matrix.app }}
+        
+      - name: Clone cicd-deployment-scripts
+        run: git clone --branch parent-images https://oauth2:${{ secrets.GH_TOKEN }}@github.com/code-kern-ai/cicd-deployment-scripts.git
+
+      - name: Perform Edit/Git Operations
+        run: |
+          bash cicd-deployment-scripts/pi/edit_dockerfile.sh \
+            -i ${{ env.PARENT_IMAGE_NAME }} \
+            -t ${{ matrix.parent_image_type }} \
+            -l ${{ github.event.release.tag_name }} \
+            -r ${{ env.DOCKERHUB_CONTAINER_REGISTRY }} \
+            -d ${{ env.DOCKERFILE }}
+
+          LATEST_IMAGE_TAG="${{ env.DOCKERHUB_CONTAINER_REGISTRY }}/${{ matrix.parent_image_type }}:${{ github.event.release.tag_name }}"
+          BASE_REF="dev"
+          HEAD_REF="parent-images"
+          PR_TITLE="ci(pi): update to $LATEST_IMAGE_TAG"
+
+          git config user.email "devtools@kern.ai"
+          git config user.name "GitHub Actions"
+
+          git checkout -b parent-images || git checkout parent-images
+          git push origin parent-images && git pull origin parent-images
+
+          git add ${{ env.DOCKERFILE }}
+          git commit -m "$PR_TITLE" || true
+          git push origin parent-images
+          echo "::notice::${{ matrix.app }} updated to $LATEST_IMAGE_TAG"
+
+          bash cicd-deployment-scripts/pi/pr_create.sh \
+            -b "$BASE_REF" \
+            -h "$HEAD_REF" \
+            -t "ci(pi): update to $LATEST_IMAGE_TAG" \
+            -o ${{ github.repository_owner }} \}}
+            -r ${{ github.event.repository.name }} \}}
+            -n ${{ github.event.release.tag_name }} \}}
+            -a ${{ matrix.app }} 
\ No newline at end of file
diff --git a/.github/workflows/pi_smodules_merge.yml b/.github/workflows/pi_smodules_merge.yml
deleted file mode 100644
index cf00cf7..0000000
--- a/.github/workflows/pi_smodules_merge.yml
+++ /dev/null
@@ -1,89 +0,0 @@
-name: 'Parent Images: Submodules Merge'
-
-on:
-  workflow_call:
-
-# Special permissions required for OIDC authentication
-permissions:
-  id-token: write
-  contents: read
-  actions: read
-
-env:
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
-
-jobs:
-  pi-diff:
-    name: 'Parent Images: Diff Types'
-    runs-on: ubuntu-latest
-    environment: dev
-    env:
-      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
-    outputs:
-      updated_parent_types: ${{ steps.get-diff.outputs.updated_parent_types }}
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
-          ref: ${{ github.event.pull_request.head.ref }}
-
-      - name: Clone cicd-deployment-scripts
-        run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
-
-      - name: Get Diff Types
-        id: get-diff
-        run: |
-          bash cicd-deployment-scripts/pi/diff.sh \
-            -p "${{ github.event.pull_request.number }}"
-
-  pi-smodules-merge:
-    name: 'Parent Images: Submodules Merge'
-    runs-on: ubuntu-latest
-    needs: [pi-diff]
-    environment: dev
-    env:
-      PYTHON_VERSION: ${{ vars.PYTHON_VERSION }}
-    strategy:
-      matrix:
-        parent_image_type: ${{ fromJson(needs.pi-diff.outputs.updated_parent_types) }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          token: ${{ secrets.GH_TOKEN }}
-          repository: '${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image'
-          fetch-depth: 0
-          submodules: 'true'
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install Dependencies
-        run: python -m pip install pip-tools
-      
-      - name: Perform Edit/Git Operations
-        run: |
-          cd ${{ github.workspace }}/submodules/parent-images
-          git checkout ${{ github.event.pull_request.base.ref }}
-          
-          cd ${{ github.workspace }}
-          git checkout ${{ github.event.pull_request.head.ref }} || git checkout -b ${{ github.event.pull_request.head.ref }}
-          git push origin ${{ github.event.pull_request.head.ref }} && git pull origin ${{ github.event.pull_request.head.ref }}
-
-          git config user.email "devtools@kern.ai"
-          git config user.name "GitHub Actions"
-
-          git add submodules
-          git commit -m "ci: update submodules to origin/${{ github.event.pull_request.head.ref }}" || true
-          git push origin ${{ github.event.pull_request.head.ref }}
-          echo "::notice::${{ github.event.repository.name }} updated to origin/${{ github.event.pull_request.head.ref }}"
-
-          gh pr create \
-            --title "${{ github.event.pull_request.title }}" \
-            --body "${{ github.event.pull_request.body }}" \
-            --base dev \
-            --head ${{ github.event.pull_request.head.ref }} \
-            --repo ${{ github.repository_owner }}/refinery-${{ matrix.parent_image_type }}-parent-image
diff --git a/pi/edit_dockerfile.sh b/pi/edit_dockerfile.sh
new file mode 100644
index 0000000..3a19319
--- /dev/null
+++ b/pi/edit_dockerfile.sh
@@ -0,0 +1,27 @@
+# !/bin/bash
+
+set -e
+
+PARENT_IMAGE_NAME="refinery-parent-images"
+PARENT_IMAGE_TYPE=""
+RELEASE_TAG=""
+DOCKER_REGISTRY="kernai"
+DOCKERFILE_PATH="Dockerfile"
+
+while getopts i:t:l:r:d: flag
+do
+    case "${flag}" in
+        i) PARENT_IMAGE_NAME=${OPTARG};;
+        t) PARENT_IMAGE_TYPE=$(echo ${OPTARG} | sed 's|_|-|g');;
+        l) RELEASE_TAG=${OPTARG};;
+        r) DOCKER_REGISTRY=${OPTARG};;
+        d) DOCKERFILE_PATH=${OPTARG};;
+    esac
+done
+
+PI_EXISTING_TAG=$(grep "${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:v.*-${PARENT_IMAGE_TYPE}" $DOCKERFILE_PATH | sed 's|FROM ||g' | cut -d ':' -f 2)
+PI_EXISTING_IMAGE="${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:${PI_EXISTING_TAG}"
+PI_NEW_IMAGE="${DOCKER_REGISTRY}/${PARENT_IMAGE_NAME}:${RELEASE_TAG}"
+
+echo "$(sed "s|${PI_EXISTING_IMAGE}|${PI_NEW_IMAGE}|g" ${DOCKERFILE_PATH})" > $DOCKERFILE_PATH
+echo "::notice::Dockerfile updated with new image: ${PI_NEW_IMAGE}"
\ No newline at end of file
diff --git a/pi/pr_create.sh b/pi/pr_create.sh
new file mode 100644
index 0000000..d49ec70
--- /dev/null
+++ b/pi/pr_create.sh
@@ -0,0 +1,58 @@
+# !/bin/bash
+set -e
+
+BASE_REF="dev"
+HEAD_REF="parent-images"
+PR_TITLE="ci(pi): update parent image"
+REPOSITORY_OWNER="code-kern-ai"
+REPOSITORY_NAME=""
+RELEASE_TAG=""
+APP=""
+
+while getopts b:h:t:o:r:n:a: flag
+do
+    case "${flag}" in
+        b) BASE_REF=${OPTARG};;
+        h) HEAD_REF=${OPTARG};;
+        t) PR_TITLE=${OPTARG};;
+        o) REPOSITORY_OWNER=${OPTARG};;
+        r) REPOSITORY_NAME=${OPTARG};;
+        n) RELEASE_TAG=${OPTARG};;
+        a) APP=${OPTARG};;
+    esac
+done
+
+EXISTING_PR_NUMBER=""
+EXISTING_PR_BODY=$(gh pr list --base $BASE_REF --head $HEAD_REF --json body --jq '.[].body')
+
+if [ -z "$EXISTING_PR_BODY" ]; then
+    PR_BODY=$(cat <<EOF
+Automated parent image release for:
+- https://github.com/$REPOSITORY_OWNER/$REPOSITORY_NAME/releases/tag/$RELEASE_TAG
+EOF
+)
+    gh pr create \
+        --base $BASE_REF \
+        --head $HEAD_REF \
+        --title "$PR_TITLE" \
+        --body "$PR_BODY" \
+        --draft \
+        --repo $REPOSITORY_OWNER/$APP
+        # --reviewer $REPOSITORY_OWNER/devops-admin \
+
+else
+    EXISTING_PR_NUMBER=$(gh pr list \
+        --base $BASE_REF \
+        --head $HEAD_REF \
+        --repo $REPOSITORY_OWNER/$APP \
+        --json number --jq '.[].number')
+    PR_BODY=$(cat <<EOF
+$EXISTING_PR_BODY
+- https://github.com/$REPOSITORY_OWNER/$REPOSITORY_NAME/pull/$RELEASE_TAG
+EOF
+)
+    gh pr edit $EXISTING_PR_NUMBER \
+        --body "$PR_BODY" \
+        --repo $REPOSITORY_OWNER/$APP || true
+fi
+

From 458a1c7ea3b68642d5146ce03f03b847a8e0b3ed Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Fri, 6 Dec 2024 01:52:32 +0100
Subject: [PATCH 62/63] fix: pr number input param for pi-matrix

---
 .github/workflows/pi_matrix.yml |  2 +-
 pi/matrix.sh                    | 25 ++++++++++++++-----------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/pi_matrix.yml b/.github/workflows/pi_matrix.yml
index 1b73e1b..49f7b1f 100644
--- a/.github/workflows/pi_matrix.yml
+++ b/.github/workflows/pi_matrix.yml
@@ -43,6 +43,6 @@ jobs:
         id: generate-matrix
         run: |
           bash cicd-deployment-scripts/pi/matrix.sh \
-            -p "${{ github.event.pull_request.number }}" \
+            -p "${{ github.event.pull_request.number || '' }}" \
             -s cicd-deployment-scripts/pi/settings.sh \
             -t "${{ inputs.parent_image_type }}"
diff --git a/pi/matrix.sh b/pi/matrix.sh
index 82237e4..189862f 100644
--- a/pi/matrix.sh
+++ b/pi/matrix.sh
@@ -17,19 +17,22 @@ done
 
 source $SOURCE_SCRIPT
 
-UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only)
 UPDATED_PARENT_TYPES=()
-while IFS= read -r file; do
-    if [[ $file != requirements/* ]] || [[ $file != *.in ]]; then
-        continue
-    fi
-    
-    parent_image_type=$(basename $file | sed 's|-requirements.in||g')
-    UPDATED_PARENT_TYPES+=($parent_image_type)
 
-done <<< "$UPDATED_FILES"
-
-if [ -n $PARENT_IMAGE_TYPE ]; then
+if [ -n $PR_NUMBER ] && [ -z $PARENT_IMAGE_TYPE ]; then
+    UPDATED_FILES=$(gh pr diff $PR_NUMBER --name-only)
+    while IFS= read -r file; do
+        if [[ $file != requirements/* ]] || [[ $file != *.in ]]; then
+            continue
+        fi
+        
+        parent_image_type=$(basename $file | sed 's|-requirements.in||g')
+        UPDATED_PARENT_TYPES+=($parent_image_type)
+
+    done <<< "$UPDATED_FILES"
+    echo "::notice::Exporting matrix for parent image types: $UPDATED_PARENT_TYPES"
+elif [ -z $PR_NUMBER ] && [ -n $PARENT_IMAGE_TYPE ]; then
+    echo "::notice::Exporting matrix for parent image type: $PARENT_IMAGE_TYPE"
     UPDATED_PARENT_TYPES=( $PARENT_IMAGE_TYPE )
 fi
 

From 9e194ff02a4d66fcd27a317e3cfc322409a336f7 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Fri, 6 Dec 2024 01:59:22 +0100
Subject: [PATCH 63/63] perf: include repository input

---
 .github/workflows/pi_build.yml   |  2 ++
 .github/workflows/pi_matrix.yml  | 14 ++++++++------
 .github/workflows/pi_merge.yml   |  2 ++
 .github/workflows/pi_release.yml |  1 +
 4 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pi_build.yml b/.github/workflows/pi_build.yml
index f2a5769..2874579 100644
--- a/.github/workflows/pi_build.yml
+++ b/.github/workflows/pi_build.yml
@@ -16,6 +16,8 @@ jobs:
   pi-matrix:
     uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
     secrets: inherit
+    with:
+      repository: "${{ github.repository_owner }}/refinery-submodule-parent-images"
 
   pi-build:
     name: 'Parent Images: Docker Build'
diff --git a/.github/workflows/pi_matrix.yml b/.github/workflows/pi_matrix.yml
index 49f7b1f..3d5f767 100644
--- a/.github/workflows/pi_matrix.yml
+++ b/.github/workflows/pi_matrix.yml
@@ -3,6 +3,11 @@ name: 'Parent Images: Matrix'
 on:
   workflow_call:
     inputs:
+      repository:
+        description: 'Repository for actions/checkout'
+        required: false
+        type: string
+        default: ${{ github.repository }}
       parent_image_type:
         description: 'If specified, "include" only outputs apps associated to this parent image type'
         required: false
@@ -30,14 +35,11 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
         with:
-          repository: ${{ github.repository_owner }}/refinery-submodule-parent-images
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ inputs.repository }}/refinery-submodule-parent-images
+          ref: ${{ github.event.pull_request.head.ref || github.event.repository.default_branch }}
     
-      - name: GitHub Configuration
-        run: git config --global url."https://oauth2:${{ secrets.GH_TOKEN }}@github.com".insteadOf https://github.com
-
       - name: Clone cicd-deployment-scripts
-        run: git clone --branch parent-images https://github.com/code-kern-ai/cicd-deployment-scripts.git
+        run: git clone --branch parent-images https://oauth2:${{ secrets.GH_TOKEN }}@github.com/code-kern-ai/cicd-deployment-scripts.git
 
       - name: Generate Matrix
         id: generate-matrix
diff --git a/.github/workflows/pi_merge.yml b/.github/workflows/pi_merge.yml
index ae9b6ae..347854f 100644
--- a/.github/workflows/pi_merge.yml
+++ b/.github/workflows/pi_merge.yml
@@ -16,6 +16,8 @@ jobs:
   pi-matrix:
     uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
     secrets: inherit
+    with:
+      repository: "${{ github.repository_owner }}/refinery-submodule-parent-images"
 
   pi-update-submodule:
     name: 'Parent Images: Submodules'
diff --git a/.github/workflows/pi_release.yml b/.github/workflows/pi_release.yml
index 79f9692..5409a2a 100644
--- a/.github/workflows/pi_release.yml
+++ b/.github/workflows/pi_release.yml
@@ -17,6 +17,7 @@ jobs:
     uses: code-kern-ai/cicd-deployment-scripts/.github/workflows/pi_matrix.yml@parent-images
     secrets: inherit
     with:
+      repository: ${{ github.repository }}
       parent_image_type: ${{ vars.PARENT_IMAGE_TYPE }}
 
   pi-edit: