Merge pull request #428 from boromir674/dev #2140

Workflow file for this run

	name: CI/CD Pipeline
	# Continuous Integration / Continuous Delivery

	# Triggers on all Branches and v* Tags

	### Stress-Testing, with Multi-Factor Job Matrix, on: ###
	# - tags v*
	# - the 'stress-test' branch (GITHUB_REF_NAME == 'stress-test')


	### Production PyPI Publish, pypi.org, on: ###
	# - v* tags on 'master' branch only

	### Staging/Test PyPI Publish, test.pypi.org, on: ###
	## Test PyPI publish on: ##
	# - v*-rc 'pre-release' tags on 'release' branch

	### Dockerhub publish on ###
	# - all branches and tags

	on:
	push:
	tags:
	- v*

	env:
	### STRESS TEST Job MATRIX ###
	# slowest: windows 3.12 (8 mins), slowest python: 3.12 !
	# TODO: add python 3.11 !!!
	FULL_MATRIX_STRATEGY: "{\"platform\": [\"ubuntu-latest\", \"macos-latest\"], \"python-version\": [\"3.8\", \"3.9\", \"3.10\", \"3.11\", \"3.12\"]}"
	# Python 3.7 has reached End of Life (EOL) on June 27th, 2023
	# Python 3.12 is in bugfix mode, same as 3.11 -> can start supporting 3.12 it
	UBUNTU_PY310_STRATEGY: "{\"platform\": [\"ubuntu-latest\"], \"python-version\": [\"3.10\"]}"
	TEST_STRATEGY: "{\"platform\": [\"ubuntu-latest\", \"macos-latest\", \"windows-latest\"], \"python-version\":[\"3.10\"]}"

	##### JOB ON/OFF SWITCHES - Top/1st level overrides #####
	RUN_UNIT_TESTS: "true"
	RUN_LINT_CHECKS: "true"
	DOCKER_JOB_ON: "true"
	PUBLISH_ON_PYPI: "true"
	DOCS_JOB_ON: "true"
	DRAW_DEPENDENCIES: "true"
	##########################

	### DOCKER Job Policy ####
	# Override Docker Policy-dependent decision-making and
	# Accept any ALL (branch/build) to Publish to Dockerhub
	# if true, it will push image and ignore DOCKER_JOB_POLICY
	ALWAYS_BUILD_N_PUBLISH_DOCKER: "false"

	DOCKER_JOB_POLICY: "CDeployment"
	# - CDeployment : '2' Builds and Publishes only if Tests ran and passed
	# - CDelivery : '3' Builds and Publishes if Tests Passed or if Tests were Skipped
	############################

	#### DOCS Job ####
	# 2nd level override
	ALWAYS_DOCS: "false"
	DOCS_JOB_POLICY: '2' # {2, 3}
	PY_VERSION: "3.11"

	#### STATIC CODE ANALYSIS Job ####
	ALWAYS_LINT: "false"
	LINT_JOB_POLICY: '2' # {2, 3}
	## Python Runtime version to set the Job runner with ##
	STATIC_ANALYSIS_PY: "3.10"
	## Pylint Minimum Acceptance Rating/Score ##
	PYLINT_SCORE_THRESHOLD: "8.2"

	#### CODE VISUALIZATION Job ####
	ALWAYS_CODE_VIZ: "false"
	CODE_VIZ_POLICY: '2' # {2, 3}
	##########################

	jobs:
	# we use the below to read the workflow env vars and be able to use in "- if:" Job conditionals
	# now we can do -> if: ${{ needs.set_github_outputs.outputs.TESTS_ENABLED == 'true' }}
	# github does not have a way to simply do "- if: ${{ env.RUN_UNIT_TESTS == 'true' }} " !!
	set_github_outputs:
	name: Read Workflow Env Section Vars and set Github Outputs
	runs-on: ubuntu-latest
	steps:
	- name: Pass 'env' section variables to GITHUB_OUTPUT
	id: pass-env-to-output
	env:
	BOARDING_EVENT: 'do Boarding CI Tests'
	BOARDING_MSG: "Auto Merging '[^']+' carrying '([^']+)' Changes"
	run: \|
	BRANCH_NAME=${GITHUB_REF_NAME}
	PIPE_DOCS_POLICY="${{ (env.DOCS_JOB_ON != 'true' && '0') \|\| (env.ALWAYS_DOCS == 'true' && '1') \|\| env.DOCS_JOB_POLICY }}"

	# set the matrix strategy to Full Matrix Stress Test if on master/main or stress-test branch or any tag
	if [[ $BRANCH_NAME == "stress-test" \|\| $GITHUB_REF == refs/tags/* ]]; then
	echo "matrix=$FULL_MATRIX_STRATEGY" >> $GITHUB_OUTPUT

	# github.event.head_commit.message has 'do Boarding CI Tests' string
	elif [[ "${{ contains(github.event.head_commit.message, env.BOARDING_EVENT) }}" == 'true' ]]; then
	echo "matrix=$TEST_STRATEGY" >> $GITHUB_OUTPUT

	# interpret Boarding Message: if head_commit.message matches BOARDING_MSG regex
	elif [[ "${{ github.event.head_commit.message }}" =~ ${BOARDING_MSG} ]]; then

	affected_components="${BASH_REMATCH[1]}"
	echo "--> Detected Affected Components: $affected_components <--"

	if [[ "$affected_components" =~ "Distro" ]]; then
	echo "matrix=$TEST_STRATEGY" >> $GITHUB_OUTPUT
	else
	echo "matrix=$UBUNTU_PY310_STRATEGY" >> $GITHUB_OUTPUT
	fi
	if [[ "$affected_components" =~ "Docs" ]]; then
	# set policy to 1 to trigger Docs Build (higher level override might be in place)
	PIPE_DOCS_POLICY="${{ (env.DOCS_JOB_ON != 'true' && '0') \|\| (env.ALWAYS_DOCS == 'true' && '1') \|\| '1' }}"
	fi

	else
	# Fall back to the default strategy
	echo "matrix=$UBUNTU_PY310_STRATEGY" >> $GITHUB_OUTPUT
	fi

	## we skip ci, on pushes to 'release' branch ##
	# echo "TESTS_ENABLED=${{ env.RUN_UNIT_TESTS == 'true' && github.ref_name != 'release' }}" >> $GITHUB_OUTPUT

	echo "TESTS_ENABLED=${{ env.RUN_UNIT_TESTS == 'true' }}" >> $GITHUB_OUTPUT

	echo "PUBLISH_ON_PYPI=$PUBLISH_ON_PYPI" >> $GITHUB_OUTPUT
	echo "PIPE_DOCS_POLICY=$PIPE_DOCS_POLICY" >> $GITHUB_OUTPUT
	echo "PIPE_DOCS_PY=$PY_VERSION" >> $GITHUB_OUTPUT
	## Docker - Pipeline Settings ##
	- id: derive_docker_policy
	run: echo "POL=${{ (env.DOCKER_JOB_ON != 'true' && '0') \|\| (env.ALWAYS_BUILD_N_PUBLISH_DOCKER == 'true' && '1') \|\| (env.DOCKER_JOB_POLICY == 'CDeployment' && '2') \|\| (env.DOCKER_JOB_POLICY == 'CDelivery' && '3') }}" >> $GITHUB_OUTPUT
	## Static Code Analysis - Pipeline Settings ##
	- id: derive_sqa_policy
	run: echo "POL=${{ (env.RUN_LINT_CHECKS != 'true' && '0') \|\| (env.ALWAYS_LINT == 'true' && '1') \|\| env.LINT_JOB_POLICY }}" >> $GITHUB_OUTPUT
	- id: read_sqa_py
	run: echo SQA_PY=${{ env.STATIC_ANALYSIS_PY }} >> $GITHUB_OUTPUT
	- id: read_pylint_score_threshold
	run: echo PYLINT_SCORE_THRESHOLD=${{ env.PYLINT_SCORE_THRESHOLD }} >> $GITHUB_OUTPUT
	## Code Visualization - Pipeline Settings ##
	- id: derive_code_viz_policy
	run: echo "POL=${{ (env.DRAW_DEPENDENCIES != 'true' && '0') \|\| (env.ALWAYS_CODE_VIZ == 'true' && '1') \|\| env.CODE_VIZ_POLICY }}" >> $GITHUB_OUTPUT
	outputs:
	matrix: ${{ steps.pass-env-to-output.outputs.matrix }}
	TESTS_ENABLED: ${{ steps.pass-env-to-output.outputs.TESTS_ENABLED }}
	PUBLISH_ON_PYPI: ${{ steps.pass-env-to-output.outputs.PUBLISH_ON_PYPI }}
	## DOCS - Pipeline Settings ##
	PIPE_DOCS_POLICY: ${{ steps.pass-env-to-output.outputs.PIPE_DOCS_POLICY }}
	PIPE_DOCS_PY: ${{ steps.pass-env-to-output.outputs.PIPE_DOCS_PY }}
	## Docker - Pipeline Settings ##
	PIPE_DOCKER_POLICY: ${{ steps.derive_docker_policy.outputs.POL }}
	## Static Code Analysis - Pipeline Settings ##
	PIPE_SQA_POLICY: ${{ steps.derive_sqa_policy.outputs.POL }}
	PIPE_SQA_PY: ${{ steps.read_sqa_py.outputs.SQA_PY }}
	PIPE_SQA_PYLINT_PASS_SCORE: ${{ steps.read_pylint_score_threshold.outputs.PYLINT_SCORE_THRESHOLD }}
	## Code Visualization - Pipeline Settings ##
	PIPE_CODE_VIZ_POLICY: ${{ steps.derive_code_viz_policy.outputs.POL }}

	# RUN TEST SUITE ON ALL PLATFORMS
	test:
	needs: set_github_outputs
	if: ${{ needs.set_github_outputs.outputs.TESTS_ENABLED == 'true' }}
	uses: ./.github/workflows/test-job.yml
	with:
	job_matrix: "{\"platform\": [\"ubuntu-latest\", \"macos-latest\"], \"python-version\": [\"3.8\", \"3.9\", \"3.10\", \"3.11\", \"3.12\"]}"
	# job_matrix: "{\"platform\": [\"ubuntu-latest\", \"macos-latest\", \"windows-latest\"], \"python-version\": [\"3.11\"]}"
	# job_matrix: "{\"platform\": [\"windows-latest\"], \"python-version\": [\"3.11\"]}"
	# artifact_name: 'dist'

	codecov_coverage_host:
	needs: test
	if: always() && contains(fromJSON('["success", "failure"]'), needs.test.result)
	uses: ./.github/workflows/codecov-job.yml
	secrets:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	## STATIC CODE ANALYSIS: mypy, ruff, isort, black, bandit, mccabe, prospector, etc
	sca:
	uses: ./.github/workflows/sca-job.yml
	with:
	python_version: '3.10'
	allow_failure: false
	force_styles: true
	bandit: '{"h": "0", "m": "2", "l": "4"}' # Automated Acceptance Criteria for Bandit

	### DOCS BUILD/TEST - DOCUMENTATION SITE ###
	docs:
	uses: ./.github/workflows/docs-job.yml
	with:
	default_trigger: true
	override: '${{ vars.OV_DOCS }}'
	python_version: '3.11'

	### DRAW CODE DEPENDENCY GRAPHS ###
	pydeps:
	uses: ./.github/workflows/pydeps-job.yml
	with:
	default_trigger: true
	override: '${{ vars.OV_PYDEPS }}'
	python_version: '3.10'


	############# Continuous Delivery -> PyPI, Docker, Github Release #############


	## DOCKER BUILD and PUBLISH ON DOCKERHUB ##
	# Ref Page: https://automated-workflows.readthedocs.io/en/main/ref_docker/
	docker_build:
	needs: [test, set_github_outputs]
	uses: boromir674/automated-workflows/.github/workflows/[email protected]
	if: always()
	with:
	acceptance_policy: ${{ needs.set_github_outputs.outputs.PIPE_DOCKER_POLICY }}
	image_slug: "generate-python"
	# target_stage: "some_stage_alias" # no stage, means no `--target` flag, on build
	tests_pass: ${{ needs.test.result == 'success' }}
	tests_run: ${{ !contains(fromJSON('["skipped", "cancelled"]'), needs.test.result) }}
	DOCKER_USER: ${{ vars.DOCKER_USER }}
	secrets:
	DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

	## JOB: Signal for Automated PyPI Upload ##
	check_which_git_branch_we_are_on:
	runs-on: ubuntu-latest
	needs: [test, docs, sca, pydeps]
	if: ${{ startsWith(github.event.ref, 'refs/tags/v') }}
	env:
	RELEASE_BR: 'release'
	MAIN_BR: 'master'
	steps:
	# Fetch 'master' and 'release' branches
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- run: git branch --track "${{ env.RELEASE_BR }}" "origin/${{ env.RELEASE_BR }}"
	- run: git branch --track "${{ env.MAIN_BR }}" "origin/${{ env.MAIN_BR }}"

	- name: "Check if '${{ github.ref }}' tag is on '${{ env.MAIN_BR }}' branch"
	uses: rickstaa/action-contains-tag@v1
	id: main_contains_tag
	with:
	reference: ${{ env.MAIN_BR }}
	tag: "${{ github.ref }}"
	- name: "Check if '${{ github.ref }}' tag is on '${{ env.RELEASE_BR }}' branch"
	uses: rickstaa/action-contains-tag@v1
	id: release_contains_tag
	with:
	reference: ${{ env.RELEASE_BR }}
	tag: "${{ github.ref }}"

	- name: Pick Production or Test Environment, if tag on master or release branch respectively
	id: set_environment_name
	run: \|
	DEPLOY=true
	if [[ "${{ steps.main_contains_tag.outputs.retval }}" == "true" ]]; then
	echo "ENVIRONMENT_NAME=PROD_DEPLOYMENT" >> $GITHUB_OUTPUT
	elif [[ "${{ steps.release_contains_tag.outputs.retval }}" == "true" ]]; then
	echo "ENVIRONMENT_NAME=TEST_DEPLOYMENT" >> $GITHUB_OUTPUT
	else
	echo "A tag was pushed but not on master or release branch. No deployment will be done."
	DEPLOY=false
	fi
	echo "AUTOMATED_DEPLOY=$DEPLOY" >> $GITHUB_OUTPUT
	outputs:
	ENVIRONMENT_NAME: ${{ steps.set_environment_name.outputs.ENVIRONMENT_NAME }}
	AUTOMATED_DEPLOY: ${{ steps.set_environment_name.outputs.AUTOMATED_DEPLOY }}

	## JOB: PYPI UPLOAD ##
	pypi_publish:
	needs: [check_which_git_branch_we_are_on, test]
	uses: boromir674/automated-workflows/.github/workflows/pypi_env.yml@test
	with:
	should_trigger: ${{ needs.check_which_git_branch_we_are_on.outputs.AUTOMATED_DEPLOY == 'true' }}
	distro_name: cookiecutter_python
	distro_version: ${{ needs.test.outputs.PEP_VERSION }} # TODO remove this
	pypi_env: '${{ needs.check_which_git_branch_we_are_on.outputs.ENVIRONMENT_NAME }}'
	artifacts_path: downloaded-artifacts

	require_wheel: true
	allow_existing: ${{ needs.check_which_git_branch_we_are_on.outputs.ENVIRONMENT_NAME == 'TEST_DEPLOYMENT' }}
	secrets:
	TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}

	### Make a Github Release ###
	gh_release:
	needs: check_which_git_branch_we_are_on
	if: ${{ needs.check_which_git_branch_we_are_on.outputs.AUTOMATED_DEPLOY == 'true' }}
	uses: boromir674/automated-workflows/.github/workflows/[email protected]
	name: 'GH Release'
	with:
	tag: ${{ github.ref_name }}
	draft: ${{ needs.check_which_git_branch_we_are_on.outputs.ENVIRONMENT_NAME == 'TEST_DEPLOYMENT' }}
	secrets:
	# pass a Github PAT "sign" Release (ie render in UI as 'boromir674 (instead of github-actions) released this yesterday'
	gh_token: ${{ secrets.PYTHON_GEN_GH_RELEASE_RW }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Merge pull request #428 from boromir674/dev #2140

Workflow file

Merge pull request #428 from boromir674/dev #2140

Uh oh!

Jobs

Run details

Workflow file for this run