Merge pull request #103 from kbst/buildimages

Distribute images to speed up automation II

Author: pst

.github/actions/build_artifacts/dist.py

@@ -9,11 +9,29 @@
 SRCDIR = 'quickstart/src'
 DISTDIR = 'quickstart/_dist'
 
+
+def replace_version(dist_path, file_name, context):
+    # Replace templated variable with version in clusters.tf
+    jinja = Environment(loader=FileSystemLoader(dist_path))
+    template = jinja.get_template(file_name)
+    data = template.render(context)
+
+    with open(f'{dist_path}/{file_name}', 'w') as f:
+        f.write(data)
+        # always include newline at end of file
+        f.write('\n')
+
+
 # Use tag as version, fallback to commit sha
 version = environ.get('GITHUB_SHA')
+# Non tagged images go to a different image repository
+image_name = 'kubestack/framework-dev'
+
 gitref = environ.get('GITHUB_REF')
 if gitref.startswith('refs/tags/'):
     version = gitref.replace('refs/tags/', '')
+    # Tagged releases go to main image repository
+    image_name = 'kubestack/framework'
 
 # Clean DISTDIR
 if isdir(DISTDIR):
@@ -38,14 +56,12 @@
     copytree(manifests_src, manifests_dist)
     copytree(cicd_src, cicd_dist)
 
-    # Replace templated variable with version in clusters.tf
-    jinja = Environment(loader=FileSystemLoader(configuration_dist))
-    template = jinja.get_template('clusters.tf')
-    data = template.render(version=version)
+    # Replace templated version variable in clusters.tf
+    replace_version(configuration_dist, 'clusters.tf', {'version': version})
 
-    with open(f'{configuration_dist}/clusters.tf', 'w') as f:
-        f.write(data)
-        # always include newline at end of file
-        f.write('\n')
+    # Replace templated variables in Dockerfile
+    replace_version(cicd_dist,
+                    'Dockerfile',
+                    {'image_name': image_name, 'image_tag': version})
 
     make_archive(archive_name, 'zip', f'{DISTDIR}', configuration_name)

.github/workflows/main.yml

@@ -8,7 +8,7 @@ on:
     - "*"  # run for tags
 
 jobs:
-  build:
+  build-artifacts:
     runs-on: ubuntu-latest
 
     steps:
@@ -18,15 +18,50 @@ jobs:
     - name: Build artifacts
       uses: ./.github/actions/build_artifacts
 
+    # Docker actions run as root, fix ownership of _dist
+    - name: Fix owner
+      run: |
+        sudo chown -R `id -u`:`id -g` ./quickstart/_dist
+
     - name: 'Upload artifacts'
       uses: actions/upload-artifact@v1
       with:
         name: quickstart-artifacts
         path: ./quickstart/_dist
 
+  build-image:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: 'Checkout'
+      uses: actions/checkout@v1
+
+    - name: 'Docker login'
+      run: |
+        echo ${{ secrets.DOCKER_AUTH }} | docker login --username kbstci --password-stdin
+
+    - name: Build image
+      env:
+        DOCKER_BUILDKIT: 1
+      run: |
+        docker run \
+          --rm \
+          --privileged \
+          -v `pwd`/oci:/tmp/work \
+          -v $HOME/.docker:/root/.docker \
+          --entrypoint buildctl-daemonless.sh \
+          moby/buildkit:master \
+              build \
+              --frontend dockerfile.v0 \
+              --local context=/tmp/work \
+              --local dockerfile=/tmp/work \
+              --output type=image,name=kubestack/framework-dev:test-${{ github.sha }},push=true \
+              --export-cache type=registry,ref=kubestack/framework-dev:buildcache,push=true \
+              --import-cache type=registry,ref=kubestack/framework-dev:buildcache
+
   test:
     runs-on: ubuntu-latest
-    needs: [build]
+    needs: [build-artifacts, build-image]
     strategy:
       matrix:
         test-for: ["aks", "eks", "gke"]
@@ -41,6 +76,13 @@ jobs:
       run: |
         unzip quickstart-artifacts/infra-quickstart-${{ matrix.test-for }}-*.zip
 
+    # Align image test tag and tag in artifact's Dockerfile
+    - name: 'Docker retag'
+      run: |
+        FROM_IMAGE=$(cat ./infra-quickstart-${{ matrix.test-for }}/ci-cd/Dockerfile | sed 's/FROM //')
+        docker pull kubestack/framework-dev:test-${{ github.sha }}
+        docker tag kubestack/framework-dev:test-${{ github.sha }} $FROM_IMAGE
+
     - name: 'Docker build'
       env:
         DOCKER_BUILDKIT: 1
@@ -84,9 +126,38 @@ jobs:
           kbst-infra-automation:test \
           terraform validate
 
-  publish:
+  publish-image:
     runs-on: ubuntu-latest
-    needs: [build, test]
+    needs: [test]
+
+    steps:
+    - name: 'Download artifacts'
+      uses: actions/download-artifact@v1
+      with:
+        name: quickstart-artifacts
+
+    - name: 'Docker retag'
+      run: |
+        FROM_IMAGE=$(cat quickstart-artifacts/infra-quickstart-gke/ci-cd/Dockerfile | sed 's/FROM //')
+        docker pull kubestack/framework-dev:test-${{ github.sha }}
+        docker tag kubestack/framework-dev:test-${{ github.sha }} $FROM_IMAGE
+
+    - name: 'Docker login'
+      run: |
+        echo ${{ secrets.DOCKER_AUTH }} | docker login --username kbstci --password-stdin
+
+    - name: 'Docker push'
+      run: |
+        IMAGE=$(cat quickstart-artifacts/infra-quickstart-gke/ci-cd/Dockerfile | sed 's/FROM //')
+        docker push $IMAGE
+
+  publish-artifacts:
+    runs-on: ubuntu-latest
+    needs: [test]
+
+    strategy:
+      matrix:
+        test-for: ["aks", "eks", "gke"]
 
     steps:
     - name: 'Download artifacts'
@@ -99,5 +170,13 @@ jobs:
       with:
         service_account_key: ${{ secrets.GCLOUD_AUTH }}
 
+    - name: 'Publish dev archives'
+      if: startsWith(github.ref, 'refs/tags/v') == false
+      run: |
+        gsutil -m cp quickstart-artifacts/infra-quickstart-${{ matrix.test-for }}-${{ github.sha }}.zip gs://dev.quickstart.kubestack.com
+
     - name: 'Publish archives'
-      run: gsutil -m cp quickstart-artifacts/*.zip gs://dev.quickstart.kubestack.com
+      if: startsWith(github.ref, 'refs/tags/v')
+      run: |
+        VERSION=$(echo $GITHUB_REF | sed -e "s#^refs/tags/##")
+        gsutil -m cp quickstart-artifacts/infra-quickstart-${{ matrix.test-for }}-${VERSION}.zip gs://quickstart.kubestack.com

.github/workflows/promote.yml

@@ -0,0 +1,148 @@
+#
+#
+# Image versions
+ARG BASE_BUILDER=python:3.8
+ARG BASE_IMAGE=python:3.8-slim
+
+
+#
+#
+# Base builder
+FROM ${BASE_BUILDER} AS builder
+
+RUN apt-get update && apt-get install -y \
+    curl \
+    gcc \
+    unzip
+
+RUN mkdir -p /opt/bin
+
+
+#
+#
+# Common builder
+FROM builder AS common
+
+ARG KUBECTL_VERSION=v1.18.1
+ARG KUSTOMIZE_VERSION=v3.5.4
+ARG TERRAFORM_VERSION=0.12.24
+ARG TERRAFORM_PROVIDER_KUSTOMIZE_VERSION=v0.1.0-beta.3
+
+RUN echo "KUBECTL_VERSION: ${KUBECTL_VERSION}" \
+    && curl -Lo /opt/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
+    && chmod +x /opt/bin/kubectl \
+    && /opt/bin/kubectl version --client=true
+
+RUN echo "KUSTOMIZE_VERSION: ${KUSTOMIZE_VERSION}" \
+    && curl -LO https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz \
+    && tar -xf kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz \
+    && mv kustomize /opt/bin/kustomize \
+    && /opt/bin/kustomize version
+
+RUN echo "TERRAFORM_VERSION: ${TERRAFORM_VERSION}" \
+    && curl -LO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \
+    && unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /opt/bin \
+    && chmod +x /opt/bin/terraform \
+    && /opt/bin/terraform version
+
+RUN echo "TERRAFORM_PROVIDER_KUSTOMIZE_VERSION: ${TERRAFORM_PROVIDER_KUSTOMIZE_VERSION}" \
+    && curl -LO https://github.com/kbst/terraform-provider-kustomize/releases/download/${TERRAFORM_PROVIDER_KUSTOMIZE_VERSION}/terraform-provider-kustomization-${TERRAFORM_PROVIDER_KUSTOMIZE_VERSION}-linux-amd64 \
+    && mv terraform-provider-kustomization-${TERRAFORM_PROVIDER_KUSTOMIZE_VERSION}-linux-amd64 /opt/bin/terraform-provider-kustomization \
+    && chmod +x /opt/bin/terraform-provider-kustomization
+
+COPY entrypoint /opt/bin/entrypoint
+
+
+#
+#
+# AWS builder
+FROM builder AS aws
+
+ARG AWS_IAM_AUTHENTICATOR_VERSION=0.5.0
+
+RUN mkdir -p /opt/aws/bin
+
+RUN echo "AWS_IAM_AUTHENTICATOR_VERSION: ${AWS_IAM_AUTHENTICATOR_VERSION}" \
+    && curl -Lo /opt/aws/bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v${AWS_IAM_AUTHENTICATOR_VERSION}/aws-iam-authenticator_${AWS_IAM_AUTHENTICATOR_VERSION}_linux_amd64 \
+    && chmod +x /opt/aws/bin/aws-iam-authenticator \
+    && /opt/aws/bin/aws-iam-authenticator
+
+RUN echo "AWS_CLI_VERSION: N/A" \
+    && curl "https://s3.amazonaws.com/aws-cli/awscli-bundle.zip" -o "awscli-bundle.zip" \
+    && unzip awscli-bundle.zip \
+    && ./awscli-bundle/install -i /opt/aws \
+    && /opt/aws/bin/aws --version
+
+
+#
+#
+# Google builder
+FROM builder AS google
+
+ARG GOOGLE_CLOUD_SDK_VERSION=289.0.0
+
+RUN echo "GOOGLE_CLOUD_SDK_VERSION: ${GOOGLE_CLOUD_SDK_VERSION}" \
+    && curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz \
+    && tar zxvf google-cloud-sdk-${GOOGLE_CLOUD_SDK_VERSION}-linux-x86_64.tar.gz google-cloud-sdk \
+    && mv google-cloud-sdk /opt/google \
+    && /opt/google/bin/gcloud --version
+
+
+#
+#
+# Azure builder
+FROM builder AS azure
+
+ARG AZURE_CLI_VERSION=2.3.1
+
+RUN apt-get update && apt-get install -y \
+    libffi-dev
+
+RUN echo "AZURE_CLI_VERSION: ${AZURE_CLI_VERSION}" \
+    && python -m venv /opt/azure/ \
+    && /opt/azure/bin/pip install --no-cache-dir \
+        azure-cli==${AZURE_CLI_VERSION} \
+    && echo '#!/usr/bin/env bash\n/opt/azure/bin/python -m azure.cli "$@"' \
+        > /opt/bin/az \
+    && chmod +x /opt/bin/az \
+    && /opt/bin/az --version
+
+
+#
+#
+# Final
+FROM ${BASE_IMAGE} AS final
+
+RUN apt-get update && apt-get install -y \
+      ca-certificates \
+      git \
+      jq \
+      wget \
+      openssh-client \
+      dnsutils \
+      libnss-wrapper \
+      unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV HOME=/infra/.user
+
+# Common
+COPY --from=common /opt/bin /opt/bin
+ENV PATH=/opt/bin:$PATH
+
+# AWS
+COPY --from=aws /opt/aws /opt/aws
+ENV PATH=$PATH:/opt/aws/bin
+
+# Google
+COPY --from=google /opt/google /opt/google
+ENV PATH=$PATH:/opt/google/bin
+
+# Azure
+COPY --from=azure /opt/azure /opt/azure
+ENV PATH=$PATH:/opt/azure/bin
+
+WORKDIR /infra
+
+ENTRYPOINT ["/opt/bin/entrypoint"]
+CMD bash