diff --git a/.envrc b/.envrc index 43eb1bc0a4..58ad27ad4d 100644 --- a/.envrc +++ b/.envrc @@ -5,39 +5,54 @@ export HAB_STUDIO_SUP="--auto-update" export HAB_ORIGIN_KEYS export HAB_ORIGIN="habitat" +# This script originally had two large repeated code blocks with the difference +# being that one ran using '~/.hab' and the other used '/hab' as the root for +# the operations within. This eliminates the duplication in favor of +# $hab_root. if [ -d ~/.hab ]; then - HAB_CONFIG=~/.hab/etc/cli.toml - if [ -e "$HAB_CONFIG" ]; then - HAB_AUTH_TOKEN=$(cat $HAB_CONFIG | grep "auth_token" | awk '{print $3}' | tr -d '"') - HAB_ORIGIN=$(cat $HAB_CONFIG | grep "origin" | awk '{print $3}' | tr -d '"') - export HAB_AUTH_TOKEN HAB_ORIGIN - fi - if ls ~/.hab/cache/keys/*.sig.key 1>/dev/null 2>&1; then - HAB_ORIGIN_KEY_NAMES=$(find ~/.hab/cache/keys/*.sig.key -type f -exec basename {} \;) - for key in $HAB_ORIGIN_KEY_NAMES; do - HAB_ORIGIN_KEYS="$HAB_ORIGIN_KEYS ${key%-*}" - done - HAB_ORIGIN_KEYS=$(echo "$HAB_ORIGIN_KEYS" | tr ' ', ',') - fi + hab_root=~/.hab else - HAB_CONFIG=/hab/etc/cli.toml - if [ -e "$HAB_CONFIG" ]; then - HAB_AUTH_TOKEN=$(cat $HAB_CONFIG | grep "auth_token" | awk '{print $3}' | tr -d '"') - HAB_ORIGIN=$(cat $HAB_CONFIG | grep "origin" | awk '{print $3}' | tr -d '"') - export HAB_AUTH_TOKEN HAB_ORIGIN - fi - if ls /hab/cache/keys/*.sig.key 1>/dev/null 2>&1; then - HAB_ORIGIN_KEY_NAMES=$(find /hab/cache/keys/*.sig.key -type f -exec basename {} \;) - for key in $HAB_ORIGIN_KEY_NAMES; do - HAB_ORIGIN_KEYS="$HAB_ORIGIN_KEYS ${key%-*}" - done - HAB_ORIGIN_KEYS=$(echo "$HAB_ORIGIN_KEYS" | tr ' ', ',') - fi + hab_root=/hab fi +HAB_CONFIG="$hab_root/etc/cli.toml" + +# This really only ever execute if the executing user has configured the +# cli.toml via 'hab cli setup'. Also, I think this will set the variables +# within to the empty string if they aren't present but that's a problem for +# later as it's been that way for a long time now and I'm not aware of an +# issues that its's causing at this time. +if [ -e "$HAB_CONFIG" ]; then + HAB_AUTH_TOKEN=$(cat $HAB_CONFIG | grep "auth_token" | awk '{print $3}' | tr -d '"') + HAB_ORIGIN=$(cat $HAB_CONFIG | grep "origin" | awk '{print $3}' | tr -d '"') + export HAB_AUTH_TOKEN HAB_ORIGIN +fi + +# This is the problem I really came to address in reworking this script. I have +# a lot of different keys for the same origin and this variable was a mess with +# 10s of the same key and it was causing me issues often enough that I started +# unsetting it as a regular practice. This creates a list without dups and +# only adds the HAB_ORIGIN to the list if it doesn't already exist in the list. +declare -A hash +for key in "$hab_root"/cache/keys/*.sig.key; do + x=${key##*/} + x=${x%-*} + hash["$x"]="$x" +done +unset x + +for x in "${!hash[@]}"; do + if [[ -n $HAB_ORIGIN_KEYS ]]; then + HAB_ORIGIN_KEYS+=",$x" + else + HAB_ORIGIN_KEYS+="$x" + fi +done +unset x -if [ -n "$HAB_ORIGIN" ]; then +if [[ ! -v hash["$HAB_ORIGIN"] ]]; then HAB_ORIGIN_KEYS="${HAB_ORIGIN_KEYS},${HAB_ORIGIN}" fi +unset hash if [ -z "${HAB_AUTH_TOKEN:-}" ]; then echo "WARNING: No auth token set. Please run hab setup before running builder." diff --git a/.expeditor/scripts/post_habitat_release/cargo_update.sh b/.expeditor/scripts/post_habitat_release/cargo_update.sh index f9287ba218..e20ddf373a 100755 --- a/.expeditor/scripts/post_habitat_release/cargo_update.sh +++ b/.expeditor/scripts/post_habitat_release/cargo_update.sh @@ -1,9 +1,9 @@ #!/bin/bash -set -euo pipefail +set -euo pipefail -# shellcheck source=.expeditor/scripts/shared.sh -source .expeditor/scripts/post_habitat_release/shared.sh +# shellcheck source=shared.sh +source .expeditor/scripts/post_habitat_release/shared.sh branch="expeditor/cargo-update-$(date +"%Y%m%d%H%M%S")" git checkout -b "$branch" @@ -13,21 +13,8 @@ toolchain="$(get_toolchain)" install_hub echo "--- :habicat: Installing and configuring build dependencies" -hab pkg install core/rust/"$toolchain" \ - core/libarchive \ - core/openssl \ - core/pkg-config \ - core/postgresql \ - core/protobuf \ - core/zeromq \ - core/cmake - -export OPENSSL_NO_VENDOR=1 -export LD_RUN_PATH -LD_RUN_PATH="$(hab pkg path core/glibc)/lib:$(hab pkg path core/gcc-libs)/lib:$(hab pkg path core/openssl)/lib:$(hab pkg path core/postgresql)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/libarchive)/lib" -export PKG_CONFIG_PATH -PKG_CONFIG_PATH="$(hab pkg path core/zeromq)/lib/pkgconfig:$(hab pkg path core/libarchive)/lib/pkgconfig:$(hab pkg path core/postgresql)/lib/pkgconfig:$(hab pkg path core/openssl)/lib/pkgconfig" -eval "$(hab pkg env core/rust/"$toolchain"):$(hab pkg path core/protobuf)/bin:$(hab pkg path core/pkg-config)/bin:$(hab pkg path core/postgresql)/bin:$(hab pkg path core/cmake)/bin:$PATH" +# shellcheck source=../../../support/ci/shared_build_environment.sh +source support/ci/shared_build_environment.sh echo "--- :rust: Cargo Update" cargo clean @@ -43,14 +30,14 @@ git commit -s -m "Update Cargo.lock" pr_labels="" pr_message="" -if [ "$update_status" -ne 0 ]; then +if [ "$update_status" -ne 0 ]; then pr_labels="T-DO-NOT-MERGE" - # read will exit 1 if it can't find a delimeter. - # -d '' will always trigger this case as there is no delimeter to find, - # but this is required in order to write the entire message into a single PR + # read will exit 1 if it can't find a delimiter. + # -d '' will always trigger this case as there is no delimiter to find, + # but this is required in order to write the entire message into a single PR # preserving newlines. - read -r -d '' pr_message < logs/sup.log 2>&1 & +if pgrep hab-sup >/dev/null 2>&1; then + echo "Before attempting to start a hab-sup a hab-sup process was found, exiting" + exit 1 +fi +env HAB_FUNC_TEST=1 hab sup run >logs/sup.log 2>&1 & -until hab svc status >/dev/null 2>&1; - do echo "waiting for hab sup to start" +until hab svc status >/dev/null 2>&1; do + echo "waiting for hab sup to start" sleep 1 done echo "--- Starting builder" start-builder -while ! [ -f "/hab/svc/builder-api/files/builder-github-app.pem" ]; -do +while ! [ -f "/hab/svc/builder-api/files/builder-github-app.pem" ]; do echo "Waiting for builder-github-app.pem" ls /hab/svc/builder-api/files sleep 10 @@ -40,20 +43,22 @@ done # Redirect the output into a file that is automatically uploaded # to buildkite so we can inspect if necessary echo "--- Building changed builder components" + +# NOTE: While building builder-api there is repeating error that can be +# recognized by "Crypto error: No revisions found for bldr". We might be able +# prevent it but things will self-correct and the script will continue. echo "--- Building builder-api" echo "Redirecting log output; See build artifact 'builder-api.build.log'" -build-builder api > logs/builder-api.build.log 2>&1 +build-builder api >logs/builder-api.build.log 2>&1 echo "--- Waiting for services to start" -while hab svc status | grep --quiet down; -do +while hab svc status | grep --quiet down; do echo "Waiting for services to start..." sleep 10 done echo "--- Waiting for builder-github-app.pem to arrive" -while ! [ -f "/hab/svc/builder-api/files/builder-github-app.pem" ]; -do +while ! [ -f "/hab/svc/builder-api/files/builder-github-app.pem" ]; do echo "Waiting for builder-github-app.pem" ls /hab/svc/builder-api/files sleep 10 diff --git a/.expeditor/verify.pipeline.yml b/.expeditor/verify.pipeline.yml index 8fec0ee3cf..c3a972a243 100644 --- a/.expeditor/verify.pipeline.yml +++ b/.expeditor/verify.pipeline.yml @@ -3,9 +3,12 @@ expeditor: buildkite: timeout_in_minutes: 30 env: + HAB_BLDR_CHANNEL: LTS-2024 + HAB_REFRESH_CHANNEL: LTS-2024 + HAB_FALLBACK_CHANNEL: LTS-2024 HAB_STUDIO_SECRET_HAB_BLDR_CHANNEL: LTS-2024 HAB_STUDIO_SECRET_HAB_REFRESH_CHANNEL: LTS-2024 - HAB_STUDIO_SECRET_HAB_FALLBACK_CHANNEL: "LTS-2024" + HAB_STUDIO_SECRET_HAB_FALLBACK_CHANNEL: LTS-2024 steps: ####################################################################### @@ -142,4 +145,3 @@ steps: # `studio` should not matter. # test test - HAB_STUDIO_SECRET_NODE_OPTIONS="--dns-result-order=ipv4first" - diff --git a/.studiorc b/.studiorc index 361b1a471e..67d6aa03a6 100755 --- a/.studiorc +++ b/.studiorc @@ -75,7 +75,6 @@ install-packages() { core/procps-ng \ core/sccache \ core/shadow \ - --channel LTS-2024 \ --binlink } @@ -181,7 +180,7 @@ _load-if-not-loaded() { if hab svc status "$pkg_ident" >/dev/null 2>&1; then echo "$pkg_ident is already loaded" else - hab svc load --channel "${HAB_BLDR_CHANNEL:-LTS-2024}" "$@" + hab svc load "$@" fi } load-if-not-loaded() { stop-on-failure _load-if-not-loaded "$@"; } @@ -192,7 +191,7 @@ start-datastore() { echo "habitat/builder-datastore is already loaded" else init-datastore - hab svc load --channel "${HAB_BLDR_CHANNEL:-LTS-2024}" habitat/builder-datastore + hab svc load habitat/builder-datastore fi } @@ -251,8 +250,7 @@ generate_bldr_keys() { } load_package() { - # JAH: Should there really be a --channel option here at all? - hab pkg upload --url http://localhost --auth "${HAB_AUTH_TOKEN}" "$@" --channel "${HAB_BLDR_CHANNEL:-LTS-2024}" + hab pkg upload --url http://localhost --auth "${HAB_AUTH_TOKEN}" "$@" } load_packages() { @@ -310,7 +308,7 @@ test-builder() { fi echo "Starting supervisor in test mode. Logs saved to ${logs}" - HAB_BLDR_CHANNEL="${HAB_BLDR_CHANNEL:-LTS-2024}" HAB_FUNC_TEST=1 RUST_LOG=debug HAB_NONINTERACTIVE=true HAB_NOCOLORING=true hab sup run --no-color >${logs} 2>&1 & + HAB_FUNC_TEST=1 RUST_LOG=debug HAB_NONINTERACTIVE=true HAB_NOCOLORING=true hab sup run --no-color >${logs} 2>&1 & sleep 8 start-builder @@ -327,11 +325,33 @@ test-builder() { fi } +PSQL_POSTGRES_IDENT='' +export PSQL_POSTGRES_IDENT +set_postgres_ident() { + if [[ -z $PSQL_POSTGRES_IDENT ]]; then + s="$(hab pkg list core/postgresql17-client)" + if [[ -n $s ]]; then + PSQL_POSTGRES_IDENT='core/postgresql17-client' + else + s="$(hab pkg list core/postgresql17)" + if [[ -n $s ]]; then + PSQL_POSTGRES_IDENT='core/postgresql17' + else + hab pkg install core/postgresql17-client + PSQL_POSTGRES_IDENT='core/postgresql17-client' + fi + fi + fi +} + +export -f set_postgres_ident + function psql() { local config_dir port config_dir="/hab/svc/builder-datastore/config" port=$(grep port $config_dir/postgresql.conf | grep -oE '[[:digit:]]+') - PGPASSWORD=$(cat $config_dir/pwfile) hab pkg exec core/postgresql17-client psql -U hab -h 127.0.0.1 -p "$port" "$@" + set_postgres_ident + PGPASSWORD=$(cat $config_dir/pwfile) hab pkg exec "$PSQL_POSTGRES_IDENT" psql -U hab -h 127.0.0.1 -p "$port" "$@" } export -f psql diff --git a/Makefile b/Makefile index 77a58c7a2f..768884373a 100644 --- a/Makefile +++ b/Makefile @@ -57,9 +57,9 @@ clean-lib: $(addprefix clean-,$(LIB)) ## cleans the library components' project clean-srv: $(addprefix clean-,$(SRV)) ## cleans the service components' project trees .PHONY: clean-srv -fmt: +fmt: bash ./support/ci/rustfmt.sh -.PHONY: fmt +.PHONY: fmt help: @perl -nle'print $& if m{^[a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -82,7 +82,7 @@ $(foreach component,$(ALL),$(eval $(call UNIT,$(component)))) TOOLCHAIN := $(shell tail -n 1 rust-toolchain | cut -d'"' -f 2) lint: - $(run) test/run_clippy.sh $(TOOLCHAIN) test/unexamined_lints.txt \ + test/run_clippy.sh $(TOOLCHAIN) test/unexamined_lints.txt \ test/allowed_lints.txt \ test/lints_to_fix.txt \ test/denied_lints.txt diff --git a/build.sh b/build.sh index afb1872576..cebbc5e936 100755 --- a/build.sh +++ b/build.sh @@ -2,31 +2,9 @@ set -eou pipefail -source ./support/ci/shared.sh - -toolchain=$(get_toolchain) - component=${1?component argument required} -# Accept hab license -sudo hab pkg install core/rust/"$toolchain" --channel LTS-2024 -sudo hab pkg install core/libarchive --channel LTS-2024 -sudo hab pkg install core/openssl --channel LTS-2024 -sudo hab pkg install core/zeromq --channel LTS-2024 -sudo hab pkg install core/pkg-config --channel LTS-2024 -sudo hab pkg install core/protobuf --channel LTS-2024 -sudo hab pkg install core/postgresql15 --channel LTS-2024 -sudo hab pkg install core/cmake --channel LTS-2024 -# It is important NOT to use a vendored openssl from openssl-sys -# pg-sys does not use openssl-sys. So for components that use -# diesel's postgres feature, you wil end up with 2 versions of openssl -# which can lead to segmentation faults when connecting to postgres -export OPENSSL_NO_VENDOR=1 -export LD_RUN_PATH -LD_RUN_PATH="$(hab pkg path core/glibc)/lib:$(hab pkg path core/gcc-libs)/lib:$(hab pkg path core/openssl)/lib:$(hab pkg path core/postgresql15)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/libarchive)/lib" -export PKG_CONFIG_PATH -PKG_CONFIG_PATH="$(hab pkg path core/zeromq)/lib/pkgconfig:$(hab pkg path core/libarchive)/lib/pkgconfig:$(hab pkg path core/postgresql15)/lib/pkgconfig:$(hab pkg path core/openssl)/lib/pkgconfig" -eval "$(hab pkg env core/rust/"$toolchain"):$(hab pkg path core/protobuf)/bin:$(hab pkg path core/pkg-config)/bin:$(hab pkg path core/postgresql15)/bin:$(hab pkg path core/cmake)/bin:$PATH" +source support/ci/shared_build_environment.sh cd "components/$component" cargo build diff --git a/components/builder-api/habitat/default.toml b/components/builder-api/habitat/default.toml index b08f7baeb5..4e42d56c25 100644 --- a/components/builder-api/habitat/default.toml +++ b/components/builder-api/habitat/default.toml @@ -48,6 +48,5 @@ password = "" database = "builder" connection_retry_ms = 300 connection_timeout_sec = 3600 -db_workers = 4 host = "127.0.0.1" port = 5432 diff --git a/components/builder-api/habitat/hooks/init b/components/builder-api/habitat/hooks/init index cdda80f2ec..d7a287fa91 100755 --- a/components/builder-api/habitat/hooks/init +++ b/components/builder-api/habitat/hooks/init @@ -1,16 +1,20 @@ #!/bin/bash +# shellcheck disable=SC1083 set -euo pipefail -PGHOST="{{cfg.datastore.host}}" -PGPORT="{{cfg.datastore.port}}" -PGUSER="{{cfg.datastore.user}}" -PGPASSWORD="{{cfg.datastore.password}}" -PGDATABASE="{{cfg.datastore.database}}" +PGHOST={{cfg.datastore.host}} +PGPORT={{cfg.datastore.port}} +PGUSER={{cfg.datastore.user}} +PGPASSWORD={{cfg.datastore.password}} +PGDATABASE={{cfg.datastore.database}} export PGHOST PGPORT PGUSER PGPASSWORD PGDATABASE -# Check that the DB is exists and is reachable or create it -# NOTE: Builder's .studiorc has a psql function necessitating the use of command -if ! command psql --no-password --command=";"; then - createdb +# confirm that the builder db is accessible +if command psql --no-password -c ";"; then + echo The builder-datastore is available to the builder-api + exit 0 +else + echo The builder-datastore is UNAVAILABLE to the builder-api + exit 3 fi diff --git a/components/builder-api/habitat/plan.sh b/components/builder-api/habitat/plan.sh index 0e1375e318..0004713e1d 100644 --- a/components/builder-api/habitat/plan.sh +++ b/components/builder-api/habitat/plan.sh @@ -7,7 +7,6 @@ pkg_name=builder-api pkg_origin=habitat pkg_maintainer="The Habitat Maintainers " pkg_license=('Apache-2.0') -pkg_bin_dirs=(bin) pkg_deps=( core/coreutils diff --git a/components/builder-datastore/config/postgresql.conf b/components/builder-datastore/config/postgresql.conf index e7711e6c38..d0e5d37f27 100644 --- a/components/builder-datastore/config/postgresql.conf +++ b/components/builder-datastore/config/postgresql.conf @@ -5,7 +5,6 @@ external_pid_file = '{{pkg.svc_var_path}}/postgresql.pid' ssl_cert_file = '{{pkg.svc_config_path}}/server.crt' ssl_key_file = '{{pkg.svc_config_path}}/server.key' log_directory = '{{pkg.svc_var_path}}/pg_log' -stats_temp_directory = '{{pkg.svc_var_path}}/pg_stat_tmp' include_dir = '{{pkg.svc_config_path}}/conf.d' listen_addresses = '{{cfg.listen_addresses}}' port = {{cfg.port}} @@ -24,7 +23,6 @@ ssl_ecdh_curve = '{{cfg.ssl_ecdh_curve}}' ssl_ca_file = '{{cfg.ssl_ca_file}}' ssl_crl_file = '{{cfg.ssl_crl_file}}' password_encryption = {{cfg.password_encryption}} -db_user_namespace = {{cfg.db_user_namespace}} row_security = {{cfg.row_security}} krb_server_keyfile = '{{cfg.krb_server_keyfile}}' krb_caseins_users = {{cfg.krb_caseins_users}} @@ -73,12 +71,10 @@ archive_mode = {{cfg.archive_mode}} archive_command = '{{cfg.archive_command}}' archive_timeout = {{cfg.archive_timeout}} max_wal_senders = {{cfg.max_wal_senders}} -wal_keep_segments = {{cfg.wal_keep_segments}} wal_sender_timeout = '{{cfg.wal_sender_timeout}}' max_replication_slots = {{cfg.max_replication_slots}} track_commit_timestamp = {{cfg.track_commit_timestamp}} synchronous_standby_names = '{{cfg.synchronous_standby_names}}' -vacuum_defer_cleanup_age = {{cfg.vacuum_defer_cleanup_age}} hot_standby = {{cfg.hot_standby}} max_standby_archive_delay = '{{cfg.max_standby_archive_delay}}' max_standby_streaming_delay = '{{cfg.max_standby_streaming_delay}}' @@ -209,7 +205,6 @@ backslash_quote = '{{cfg.backslash_quote}}' default_with_oids = {{cfg.default_with_oids}} escape_string_warning = {{cfg.escape_string_warning}} lo_compat_privileges = {{cfg.lo_compat_privileges}} -operator_precedence_warning = {{cfg.operator_precedence_warning}} quote_all_identifiers = {{cfg.quote_all_identifiers}} standard_conforming_strings = {{cfg.standard_conforming_strings}} synchronize_seqscans = {{cfg.synchronize_seqscans}} diff --git a/components/builder-datastore/default.toml b/components/builder-datastore/default.toml index f34b526354..4e20b27650 100644 --- a/components/builder-datastore/default.toml +++ b/components/builder-datastore/default.toml @@ -1,5 +1,5 @@ # For postgresql.conf settings, see -# http://www.postgresql.org/docs/9.5/static/config-setting.html +# https://www.postgresql.org/docs/17/config-setting.html # listen_addresses = "*" port = 5432 @@ -17,8 +17,7 @@ ssl_prefer_server_ciphers = "on" ssl_ecdh_curve = "prime256v1" ssl_ca_file = "" ssl_crl_file = "" -password_encryption = "on" -db_user_namespace = "off" +password_encryption = "md5" row_security = "on" krb_server_keyfile = "" krb_caseins_users = "off" @@ -67,12 +66,10 @@ archive_mode = "on" archive_command = "cd ." archive_timeout = 0 max_wal_senders = 20 -wal_keep_segments = 0 wal_sender_timeout = "60s" max_replication_slots = 20 track_commit_timestamp = "off" synchronous_standby_names = "" -vacuum_defer_cleanup_age = 0 hot_standby = "on" max_standby_archive_delay = "30s" max_standby_streaming_delay = "30s" @@ -215,3 +212,9 @@ restart_after_crash = "on" encoding = "UTF8" # If this isn't set, the package will create a random password at startup superuser_password = "" + +[datastore] +host = "127.0.0.1" +port = 5432 +user = "hab" +database = "builder" diff --git a/components/builder-datastore/hooks/init b/components/builder-datastore/hooks/init deleted file mode 100644 index 6313773caa..0000000000 --- a/components/builder-datastore/hooks/init +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh -# shellcheck disable=1083 -set -e - -exec 2>&1 - -export RANDFILE="{{pkg.svc_var_path}}/.rnd" -DB_PASSWORD="{{cfg.init.superuser_password}}" - -mkdir -p {{pkg.svc_config_path}}/conf.d -mkdir -p {{pkg.svc_var_path}}/pg_stat_tmp - -chown -R hab:hab {{pkg.svc_var_path}} -touch $RANDFILE - -if [ ! -f "{{pkg.svc_data_path}}/PG_VERSION" ]; then - echo " Database does not exist, creating with 'initdb'" - if [ -z "${DB_PASSWORD}" ]; then - openssl rand -base64 32 > {{pkg.svc_config_path}}/pwfile - else - echo "${DB_PASSWORD}" > {{pkg.svc_config_path}}/pwfile - fi - initdb -U hab \ - -E {{cfg.init.encoding}} \ - -D {{pkg.svc_data_path}} \ - --pwfile {{pkg.svc_config_path}}/pwfile \ - --no-locale -fi - -chmod 400 {{pkg.svc_config_path}}/pwfile diff --git a/components/builder-datastore/hooks/install b/components/builder-datastore/hooks/install new file mode 100644 index 0000000000..e79eea84d9 --- /dev/null +++ b/components/builder-datastore/hooks/install @@ -0,0 +1,496 @@ +#!/bin/bash +# shellcheck disable=SC1083 +# handlebars templating triggers SC1083 + +exec 2>&1 +set -uo pipefail + +mkdir -p {{pkg.svc_path}} || exit 10 +cd {{pkg.svc_path}} || exit 11 + +readonly OIFS=$IFS +IFS='.' +readonly PG_VERSION_FILE={{pkg.svc_data_path}}/PG_VERSION +MOMENT="$(date '+%Y%m%d%H%M%S')" +readonly MOMENT +readonly PWFILE="{{pkg.svc_config_path}}/pwfile" +readonly PACKAGED_DATASTORE_IDENT={{pkg.ident}} +ACTIVE_IDENT=$PACKAGED_DATASTORE_IDENT +readonly PG_TIMEOUT=30 + +set_installed_pg_ident_and_active_ident() { + if [[ -f $PG_VERSION_FILE ]]; then + local installed_pg_version + installed_pg_version=$(cat $PG_VERSION_FILE | sed 's/\.//') + INSTALLED_PG_IDENT="core/postgresql${installed_pg_version}" + ACTIVE_IDENT=$INSTALLED_PG_IDENT + else + INSTALLED_PG_IDENT="" + ACTIVE_IDENT=$PACKAGED_DATASTORE_IDENT + fi + # The pkg ident of the currently installed version in upgrade scenarios + readonly INSTALLED_PG_IDENT +} + +postgres_running() { + + sudo -u hab hab pkg exec "$ACTIVE_IDENT" pg_ctl -- status \ + --pgdata={{pkg.svc_data_path}} \ + --silent + + local -ri exit_status=$? + local what_happened="" + case $exit_status in + 0) + what_happened="success (server running in specified directory)" + ;; + 3) + what_happened="server is not running" + ;; + 4) + what_happened="accessible data directory not specified" + ;; + *) + what_happened="unexpected exit status" + ;; + esac + printf "%s: exit status: %s (%s)\n" "${FUNCNAME[0]}" $exit_status "$what_happened" + return "$exit_status" +} + +postgres_ready() { + + sudo -u hab hab pkg exec "$ACTIVE_IDENT" pg_isready -- \ + --host={{cfg.datastore.host}} \ + --port={{cfg.datastore.port}} \ + --username={{cfg.datastore.user}} \ + --dbname={{cfg.datastore.database}} \ + --quiet + + local -ri exit_status=$? + local what_happened="" + case $exit_status in + 0) + what_happened="success" + ;; + 1) + what_happened="connection rejected" + ;; + 2) + what_happened="no response to connection attempt" + ;; + 3) + what_happened="no attempt made" + ;; + *) + what_happened="unexpected exit status" + ;; + esac + printf "%s: exit status: %s (%s)\n" "${FUNCNAME[0]}" $exit_status "$what_happened" + return $exit_status +} + +builder_db_exists() { + + sudo -E hab pkg exec "$ACTIVE_IDENT" psql -- \ + --host={{cfg.datastore.host}} \ + --port={{cfg.datastore.port}} \ + --username={{cfg.datastore.user}} \ + --dbname={{cfg.datastore.database}} \ + --no-password \ + --command ";" \ + --quiet + + local -ri exit_status=$? + local what_happened="" + case $exit_status in + 0) + what_happened="success" + ;; + 1) + what_happened="fatal psql error" + ;; + 2) + what_happened="bad server connection, session not interactive" + ;; + 3) + what_happened="sql script error" + ;; + *) + what_happened="unexpected exit status" + ;; + esac + printf "%s: exit status: %s (%s)\n" "${FUNCNAME[0]}" $exit_status "$what_happened" + return $exit_status +} + +postgres_running_ours() { + if [[ -f {{pkg.svc_pid_file}} ]]; then + pgrep -P "$(cat {{pkg.svc_pid_file}})" -f postgres >/dev/null + fi +} + +readonly PG_NOT_RUNNING=0 +readonly PG_RUNNING_OURS_HAS_BLDR=1 +readonly PG_RUNNING_OURS_NO_BLDR=2 +readonly PG_RUNNING_NOT_OURS=3 +PG_STATUS=-1 + +postgres_status() { + if postgres_ready; then + if builder_db_exists; then + PG_STATUS=$PG_RUNNING_OURS_HAS_BLDR + elif postgres_running_ours; then + PG_STATUS=$PG_RUNNING_OURS_NO_BLDR + else + PG_STATUS=$PG_RUNNING_NOT_OURS + fi + else + PG_STATUS=$PG_NOT_RUNNING + fi +} + +postgres_installed() { + if [[ ! -f "$PG_VERSION_FILE" ]]; then + echo PostgreSQL is not installed + return 1 + fi + echo PostgreSQL has been installed previously + echo "Detected PostgreSQL $(cat "$PG_VERSION_FILE")" +} + +create_postgres_cluster() { + + echo Creating PostgreSQL Cluster + + export randfile="{{pkg.svc_var_path}}/.rnd" + local -r DB_PASSWORD="{{cfg.init.superuser_password}}" + + mkdir -p {{pkg.svc_config_path}}/conf.d + mkdir -p {{pkg.svc_var_path}}/pg_stat_tmp + + sudo chown -R hab:hab {{pkg.svc_var_path}} + sudo touch $randfile + + if [[ ! -f "$PWFILE" ]]; then + if [[ -z "$DB_PASSWORD" ]]; then + openssl rand -base64 32 >"$PWFILE" + else + echo "${DB_PASSWORD}" | sudo -u hab tee "$PWFILE" >/dev/null + fi + fi + + sudo mkdir -p {{pkg.svc_data_path}} + sudo chown hab:hab -R {{pkg.svc_data_path}} + + # JAH: do I need to take some sort of collation into account here because of that one plugin? + output=$(sudo -u hab hab pkg exec "$ACTIVE_IDENT" initdb -- \ + --username {{cfg.datastore.user}} \ + --encoding {{cfg.init.encoding}} \ + --pgdata {{pkg.svc_data_path}} \ + --pwfile $PWFILE \ + --no-locale 2>&1) + + local exit_code=$? + if [[ $exit_code -eq 0 ]]; then + echo "Created PostgreSQL cluster" + else + echo "Failure while creating PostgreSQL cluster" + echo "$output" + fi + + chmod 400 "$PWFILE" +} + +postgres_healthy() { + echo Allowing ~${PG_TIMEOUT}s for PostgreSQL to become healthy + for ((i = 0; i < PG_TIMEOUT; i++)); do + if postgres_running && postgres_ready; then + echo PostgreSQL is healthy + return + elif ((i >= PG_TIMEOUT)); then + echo "PostgreSQL failed to become healthy within ${PG_TIMEOUT}s" + exit 12 + else + echo "Waiting for PostgreSQL to be healthy ($((i + 1))/$PG_TIMEOUT)" + sleep 1 + fi + done +} + +start_postgres() { + if postgres_running; then + echo PostgreSQL is already running, will not try to start + return + fi + + echo "Starting PostgreSQL (waiting ~${PG_TIMEOUT}s, current time: $(date +"%H:%M:%S"))" + sudo -u hab hab pkg exec "$ACTIVE_IDENT" pg_ctl -- start \ + --pgdata={{pkg.svc_data_path}} \ + --timeout=$PG_TIMEOUT \ + -w \ + --silent + + if postgres_healthy; then + echo PostgreSQL started successfully + return + else + echo PostgreSQL failed to started + return 1 + fi +} + +create_builder_database() { + echo Checking that the builder database exists in the PostgreSQL cluster + if builder_db_exists; then + echo The builder database already exists + else + echo Creating the builder database + sudo -u hab hab pkg exec "$ACTIVE_IDENT" createdb \ + --host={{cfg.datastore.host}} \ + --port={{cfg.datastore.port}} \ + --username={{cfg.datastore.user}} \ + --no-password \ + --echo \ + {{cfg.datastore.database}} + fi +} + +stop_postgres() { + + if ! postgres_running; then + echo PostgreSQL not running, will not issue stop + return + fi + + echo "Stopping PostgreSQL (waiting ~${PG_TIMEOUT}s, current time: $(date +"%H:%M:%S"))" + sudo -u hab hab pkg exec "$ACTIVE_IDENT" pg_ctl -- stop \ + --pgdata {{pkg.svc_data_path}} \ + --mode=fast \ + --timeout=$PG_TIMEOUT \ + -w \ + --silent + + echo Allowing ~${PG_TIMEOUT}s for PostgreSQL to stop + for ((i = 0; i < PG_TIMEOUT; i++)); do + if postgres_running; then + echo "Waiting for PostgreSQL to stop ($((i + 1))/$PG_TIMEOUT)" + sleep 1 + elif ((i >= PG_TIMEOUT)); then + echo "PostgreSQL failed to stop in ${PG_TIMEOUT}s" + echo "$output" + exit 13 + else + echo PostgreSQL successfully stopped + return + fi + done +} + +install_postgres() { + echo Installing PostgreSQL + create_postgres_cluster + start_postgres + create_builder_database + stop_postgres +} + +manage_upgrade_readme() { + local -r readme="$1/README.txt" + if [[ ! -f "$readme" ]]; then + cat <<-EOF >"$readme" + This directory contains subdirectories used when upgrading PostgreSQL from + one version to another. The subdirectories are named using the date and time + of the upgrades. When you feel comfortable with the success of the upgrade + you may delete these files. + EOF + fi + echo "Created $readme, please review at your convenience" +} + +postgres_needs_upgraded() { + + echo Checking if PostgreSQL needs to be upgraded + + # Prior to PostgreSQL 10 the version scheme was 3 parts (9.6.24, for example). + # Starting with PostgreSQL 10 the versioning scheme PostgreSQL versions are 2 + # parts (for example, 17.4). We do have some 3 part 9.x.x PostgreSQL packages + # to account for migration from but given that 9.6.24 has been EOL since Nov + # 11th 2021 hopefully no one has a reason to continue using such packages in a + # context where they would be might migrate from from x.x.x to y.y.y. Even if + # some did I think this will likely work acceptably + # + # https://www.postgresql.org/support/versioning/ + # https://endoflife.date/postgresql + + read -ra installed_version <<<"$(sudo -u hab hab pkg exec "$INSTALLED_PG_IDENT" pg_config -- --version | awk '{print $2}')" + if [[ ! (${installed_version[0]} =~ [[:digit:]]+ && ${installed_version[1]} =~ [[:digit:]]+) ]]; then + echo "Failed to parse the version of the installed PostgreSQL: We detected ${installed_version[*]}" + exit 14 + fi + + read -ra packaged_version <<<"$(sudo -u hab hab pkg exec "$PACKAGED_DATASTORE_IDENT" pg_config -- --version | awk '{print $2}')" + if [[ ! (${packaged_version[0]} =~ [[:digit:]]+ && ${packaged_version[1]} =~ [[:digit:]]+) ]]; then + echo "Failed to parse the version of the packaged PostgreSQL: We detected ${packaged_version[*]}" + exit 15 + fi + + declare -i retval=1 + if ((installed_version[0] < packaged_version[0])); then + echo "PostgreSQL upgrade needed: ${installed_version[*]} < ${packaged_version[*]}" + retval=0 + else + if ((installed_version[0] == packaged_version[0])); then + if ((installed_version[1] < packaged_version[1])); then + echo "PostgreSQL upgrade needed: ${installed_version[*]} < ${packaged_version[*]}" + retval=0 + fi + fi + fi + return $retval +} + +dump_postgres_database() { + + echo Dumping the builder database to a file via pg_dump + + # https://www.postgresql.org/docs/17/upgrading.html#UPGRADING-VIA-PGDUMPALL + # "It is recommended that you use the pg_dump and pg_dumpall programs from the + # newer version of PostgreSQL, to take advantage of enhancements that might + # have been made in these programs. Current releases of the dump programs can + # read data from any server version back to 9.2." + sudo -E hab pkg exec $PACKAGED_DATASTORE_IDENT pg_dump -- \ + --host={{cfg.datastore.host}} \ + --port={{cfg.datastore.port}} \ + --username={{cfg.datastore.user}} \ + --dbname={{cfg.datastore.database}} \ + --no-password \ + --quote-all-identifiers \ + --file="$pg_dump_file" + + # shellcheck disable=SC2181 + if [[ $? == 0 ]]; then + echo "pg_dump was successful" + else + echo "pg_dump FAILED" + exit 16 + fi +} + +load_postgres_database() { + + echo Loading the builder database from file generated via pg_dump using psql + + sudo -E hab pkg exec "$ACTIVE_IDENT" psql -- \ + --host={{cfg.datastore.host}} \ + --port={{cfg.datastore.port}} \ + --username={{cfg.datastore.user}} \ + --dbname={{cfg.datastore.database}} \ + --no-password \ + --file="$pg_dump_file" + + local -ri exit_status=$? + local what_happened="" + case $exit_status in + 0) + what_happened="success" + ;; + 1) + what_happened="fatal psql error" + ;; + 2) + what_happened="bad server connection, session not interactive" + ;; + 3) + what_happened="sql script error" + ;; + *) + what_happened="unexpected exit status" + ;; + esac + printf "%s: exit status: %s (%s)\n" "${FUNCNAME[0]}" $exit_status "$what_happened" + if [[ $exit_status -eq 0 ]]; then + echo "Succeeded in loading the builder database from the pg_dump file" + else + echo "Failed to load the pg_dump file" + exit 17 + fi + return $exit_status +} + +upgrade_postgres() { + + echo Upgrading PostgreSQL + + local -r work_dir="{{pkg.svc_var_path}}/tmp/upgrades/$MOMENT" + local -r pg_dump_file="$work_dir/pg_dump.sql" + local -r previous_dir="$work_dir/previous" + mkdir -p "$work_dir" + + PGPASSWORD="$(cat "$PWFILE")" + export PGPASSWORD + + manage_upgrade_readme "$work_dir" + + ACTIVE_IDENT=$INSTALLED_PG_IDENT + if ! hab pkg list "$INSTALLED_PG_IDENT" >/dev/null ; then + sudo hab pkg install "$INSTALLED_PG_IDENT" + fi + + start_postgres + + # https://www.postgresql.org/docs/17/upgrading.html + # STEP 1) Making a backup, making sure that your database is not being updated + dump_postgres_database + + # STEP 2) Shut down the old server + stop_postgres + + # STEP 3) Rename the old installation directory + sudo mv "{{pkg.svc_data_path}}" "$previous_dir" + + # STEP 4) Install the new version of PostgreSQL (as outlined in Chapter 17) + # This is a mostly a noop for us as this is accomplished as part of the hab + # svc but we do need to manage the active_ident + ACTIVE_IDENT=$PACKAGED_DATASTORE_IDENT + + # STEP 5) If needed, create a new database cluster. Remember you must be logged in as the special database user account + install_postgres + + # STEP 6) Restore your previous pg_hba.conf and any postgresql.conf modifications + + # STEP 7) Start the database server (again using the special database user account) + start_postgres + + # step 8) finally, restore your data from backup + load_postgres_database + stop_postgres # because the run method is going to start it again + + # JAH: password upgrade would go here but would we even need it? + # JAH: will this method of upgrade address it? + # JAH: scram-sha-256 isn't supported until 10.x +} + +#------------------------------------------------------------------------------- +# BEGIN "main" +#------------------------------------------------------------------------------- + +echo The builder-datastore install hook is starting + +set_installed_pg_ident_and_active_ident + +postgres_status +if [[ "$PG_STATUS" -eq "$PG_RUNNING_NOT_OURS" ]]; then + echo "The builder-datastore install hook is exiting because a PostgreSQL instance is running but it's not managed by us" + exit 19 +fi + +if ! postgres_installed; then + install_postgres +elif postgres_needs_upgraded; then + upgrade_postgres +else + echo PostgreSQL is installed and no upgrade is needed +fi + +IFS="$OIFS" diff --git a/components/builder-datastore/hooks/run b/components/builder-datastore/hooks/run index 67e976ff67..b8b9847a9a 100644 --- a/components/builder-datastore/hooks/run +++ b/components/builder-datastore/hooks/run @@ -1,7 +1,4 @@ -#!/bin/sh -# - -export PGDATA="{{pkg.svc_data_path}}" +#!/bin/bash exec 2>&1 @@ -10,8 +7,7 @@ chmod 0700 "{{pkg.svc_data_path}}" title="Service Start" text="builder-datastore starting" # shellcheck disable=SC2039,SC3025 -echo "_e{${#title},${#text}}:$title|$text|#datastore" >/dev/udp/localhost/8125 +echo "_e{${#title},${#text}}:$title|$text|#datastore" >/dev/udp/localhost/8125 # shellcheck disable=SC1083 -exec postmaster \ - -c config_file={{pkg.svc_config_path}}/postgresql.conf +exec postgres -c config_file={{pkg.svc_config_path}}/postgresql.conf diff --git a/components/builder-datastore/plan.sh b/components/builder-datastore/plan.sh index bb8f4d440e..0ca5458137 100644 --- a/components/builder-datastore/plan.sh +++ b/components/builder-datastore/plan.sh @@ -3,31 +3,14 @@ pkg_origin=habitat pkg_name=builder-datastore -pkg_internal_version=11.2 -pkg_internal_name=postgresql11 -pkg_description="Datastore service for a Habitat Builder service" pkg_maintainer="The Habitat Maintainers " -pkg_license=("PostgreSQL") -pkg_source="https://ftp.postgresql.org/pub/source/v${pkg_internal_version}/postgresql-${pkg_internal_version}.tar.bz2" -pkg_shasum="2676b9ce09c21978032070b6794696e0aa5a476e3d21d60afc036dc0a9c09405" -pkg_dirname="postgresql-${pkg_internal_version}" +pkg_license=('Apache-2.0') -pkg_deps=( - core/bash - core/glibc - core/openssl - core/perl - core/readline - core/zlib - core/libossp-uuid -) +pkg_description="Datastore service for a Habitat Builder service" -pkg_build_deps=( - core/coreutils - core/gcc - core/make - core/git -) +pkg_deps=(core/postgresql17) + +pkg_build_deps=(core/git) pkg_bin_dirs=(bin) pkg_include_dirs=(include) @@ -43,67 +26,15 @@ pkg_version() { echo "$(($(git rev-list HEAD --count) + 5000))" } -ext_semver_version=0.17.0 -ext_semver_source=https://github.com/theory/pg-semver/archive/v${ext_semver_version}.tar.gz -ext_semver_filename=pg-semver-${ext_semver_version}.tar.gz -ext_semver_shasum=031046695b143eb545a2856c5d139ebf61ae4e2f68cccb1f21b700ce65d0cd60 - do_before() { git config --global --add safe.directory /src update_pkg_version - ext_semver_dirname="pg-semver-${ext_semver_version}" - ext_semver_cache_path="$HAB_CACHE_SRC_PATH/${ext_semver_dirname}" -} - -do_download() { - do_default_download - download_file $ext_semver_source $ext_semver_filename $ext_semver_shasum -} - -do_verify() { - do_default_verify - verify_file $ext_semver_filename $ext_semver_shasum -} - -do_clean() { - do_default_clean - rm -rf "$ext_semver_cache_path" -} - -do_unpack() { - do_default_unpack - unpack_file $ext_semver_filename } do_build() { - # shellcheck disable=2154 - # ld manpage: "If -rpath is not used when linking an ELF - # executable, the contents of the environment variable LD_RUN_PATH - # will be used if it is defined" - ./configure --disable-rpath \ - --with-openssl \ - --prefix="$pkg_prefix" \ - --with-uuid=ossp \ - --with-includes="$LD_INCLUDE_PATH" \ - --with-libraries="$LD_LIBRARY_PATH" \ - --sysconfdir="$pkg_svc_config_path" \ - --localstatedir="$pkg_svc_var_path" - make world - - # semver can't be built until after postgresql is installed to $pkg_prefix + return 0 } do_install() { - make install-world - - # make and install semver extension - export PATH="${PATH}:${pkg_prefix}/bin" - build_line "Added postgresql binaries to PATH: ${pkg_prefix}/bin" - - pushd "$ext_semver_cache_path" >/dev/null || exit - build_line "Building ${ext_semver_dirname}" - make - build_line "Installing ${ext_semver_dirname}" - make install - popd >/dev/null || exit + return 0 } diff --git a/components/builder-db/src/test.rs b/components/builder-db/src/test.rs index ec5d88989c..79b6103721 100644 --- a/components/builder-db/src/test.rs +++ b/components/builder-db/src/test.rs @@ -47,7 +47,7 @@ pub mod postgres { .env("DB_TEST_DIR", root_path) .current_dir("/tmp") .spawn() - .expect("Failed to launch core/postgresql"); + .expect("Failed to launch core/postgresql17"); Postgres { inner: child } } } diff --git a/components/builder-db/tests/db/start.sh b/components/builder-db/tests/db/start.sh index 248d83fc86..816bb093c5 100755 --- a/components/builder-db/tests/db/start.sh +++ b/components/builder-db/tests/db/start.sh @@ -21,7 +21,7 @@ mkdir -p /hab/svc/postgresql mkdir -p /hab/user/postgresql cp "$DB_TEST_DIR"/pg_hba.conf /hab/svc/postgresql cp "$DB_TEST_DIR"/user.toml /hab/user/postgresql -hab sup run core/postgresql & +hab sup run core/postgresql17 & hab_pid=$! read -r sudo_ppid < <(ps -p $$ -o 'ppid=') @@ -29,7 +29,7 @@ read -r original_gpid < <(ps -p "$sudo_ppid" -o 'ppid=') while true; do read -r current_gpid < <(ps -p "$sudo_ppid" -o 'ppid=') if [ "$original_gpid" != "$current_gpid" ]; then - echo "Stopping core/postgresql" + echo "Stopping core/postgresql17" kill $hab_pid exit 0 fi diff --git a/support/builder/datastore.toml b/support/builder/datastore.toml index 22028948a1..8d078f84cb 100644 --- a/support/builder/datastore.toml +++ b/support/builder/datastore.toml @@ -1,6 +1,6 @@ max_connections = 200 max_locks_per_transaction = 128 -dynamic_shared_memory_type = 'none' +dynamic_shared_memory_type = 'mmap' port = 5433 # To avoid conflict with travis' own postgres instance [superuser] diff --git a/support/builder/init-datastore.sh b/support/builder/init-datastore.sh index dd7a73da7b..ea30985c2e 100755 --- a/support/builder/init-datastore.sh +++ b/support/builder/init-datastore.sh @@ -14,7 +14,7 @@ while [ $running -eq 0 ]; do if [ -f $pwfile ]; then PGPASSWORD=$(cat $pwfile) export PGPASSWORD - if hab pkg exec core/postgresql psql -w -lqt --host 127.0.0.1 -U hab; then + if hab pkg exec core/postgresql17 psql -w -lqt --host 127.0.0.1 -U hab; then running=1 fi fi diff --git a/support/ci/platform-inspection.sh b/support/ci/platform-inspection.sh new file mode 100644 index 0000000000..90a43d0cac --- /dev/null +++ b/support/ci/platform-inspection.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +# This was written to inspect the context that run_clippy was executing under +# during verify pipeline while debugging segfaults due to version misalignments +# with glibc and other libraries +# +# One word of caution, the ldd --version statement is dependent on the runtime +# environment at the time it's called. If you're managing that environment you +# will want to do another ldd --version call prior to the environment mgmt. +# +# Also, it's still purpose built for the run_clippy case but it can be extended +# in the future if needed. + +echo "--- cat /etc/os-release" +if [[ -f /etc/os-release ]]; then + cat /etc/os-release +else + echo "/etc/os-release doesn't exist" +fi +echo "" + +echo "--- cat /etc/system-release" +if [[ -f /etc/system-release ]]; then + cat /etc/system-release +else + echo "/etc/system-release doesn't exist" +fi +echo "" + +echo "--- docker --version" +if command -v docker >/dev/null 2>&1; then + docker --version +else + echo "docker command unavailable" +fi +echo "" + +echo "--- [[ -f /.dockerenv ]]" +if [[ -f /.dockerenv ]]; then + echo "Running inside Docker, catting /.dockerenv" + cat /.dockerenv + echo "after the catting .dockerenv, if no line before this and after 'Running inside' file was empty" +else + echo "[[-f /.dockerenv]] failed " +fi +echo "" + +echo "--- grepping /proc/1/cgroup " +if grep -qE '(docker|kubepods)' /proc/1/cgroup 2>&1/dev/null; then + echo "Running inside Docker based on grepping /proc/1/cgroup, running the command again" + grep -qE '(docker|kubepods)' /proc/1/cgroup + echo "end output" +else + echo "Not running inside Docker" +fi + +echo "--- uname -a" +if command -v uname >/dev/null 2>&1; then + uname -a +else + echo "uname command unavailable" +fi +echo "" + +echo "--- lsb_release -a" +if command -v lsb_release >/dev/null 2>&1; then + lsb_release -a +else + echo "lsb_release command unavailable" +fi +echo "" + +# shellcheck source=./shared.sh +source ./support/ci/shared.s +toolchain=$(get_toolchain) +CMD="$(hab pkg path core/rust/"$toolchain")/bin/cargo-clippy" +readonly CMD +if command -v ldd >/dev/null 2>&1; then + + echo "--- ldd --version" + ldd --version + echo "" + + LDD_TARGET="$(hab pkg path core/glibc)/lib/libc.so.6" + echo "--- ldd $LDD_TARGET" + ldd "$LDD_TARGET" || true + echo "" + + echo "--- ldd $CMD" + ldd "$CMD" || true + echo "" + +else + echo "ldd command unavailable" +fi +echo "" diff --git a/support/ci/shared.sh b/support/ci/shared.sh index dd050552da..b71a857661 100755 --- a/support/ci/shared.sh +++ b/support/ci/shared.sh @@ -20,18 +20,7 @@ get_rustfmt_toolchain() { } get_toolchain() { - tail -n 1 "$dir/../../rust-toolchain" | cut -d'"' -f 2 -} - -install_rustup() { - if command -v rustup && command -v cargo &>/dev/null; then - echo "--- :rust: rustup is currently installed." - else - echo "--- :rust: Installing rustup." - curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path -y - # shellcheck disable=SC1090 - source "$HOME"/.cargo/env - fi + tail -n 1 "$dir/../../rust-toolchain" | cut -d'"' -f 2 } install_rustup() { @@ -41,6 +30,7 @@ install_rustup() { echo "--- :rust: Installing rustup." curl https://sh.rustup.rs -sSf | sh -s -- --no-modify-path -y fi + # shellcheck disable=SC1091 source "$HOME"/.cargo/env } @@ -64,9 +54,8 @@ install_rustfmt() { install_hab_pkg() { for ident; do installed_pkgs=$(hab pkg list "$ident") - if [[ -z $installed_pkgs ]]; then - sudo hab pkg install "$ident" + sudo -E hab pkg install "$ident" else echo "$ident already installed" fi diff --git a/support/ci/shared_build_environment.sh b/support/ci/shared_build_environment.sh new file mode 100755 index 0000000000..0b6e504999 --- /dev/null +++ b/support/ci/shared_build_environment.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +sudo hab license accept + +# shellcheck source=./shared.sh +source ./support/ci/shared.sh + +readonly OG_PATH=$PATH +toolchain=$(get_toolchain) +sudo -E hab pkg install core/rust/"$toolchain" --force +eval "$(hab pkg env core/rust/"$toolchain")" +PATH=$PATH:$OG_PATH + +install_hab_pkg core/glibc +install_hab_pkg core/gcc-base +install_hab_pkg core/binutils +install_hab_pkg core/cmake +install_hab_pkg core/bash +install_hab_pkg core/coreutils +install_hab_pkg core/curl +install_hab_pkg core/diffutils +install_hab_pkg core/gawk +install_hab_pkg core/git +install_hab_pkg core/grep +install_hab_pkg core/hab-ld-wrapper +install_hab_pkg core/libarchive +install_hab_pkg core/libb2 +install_hab_pkg core/libsodium +install_hab_pkg core/make +install_hab_pkg core/openssl +install_hab_pkg core/pkg-config +install_hab_pkg core/postgresql17 +install_hab_pkg core/protobuf +install_hab_pkg core/sed +install_hab_pkg core/sudo +install_hab_pkg core/zeromq +install_hab_pkg core/zlib + +# It is important NOT to use a vendored openssl from openssl-sys pg-sys does +# not use openssl-sys. So for components that use diesel's postgres feature, +# you wil end up with 2 versions of openssl which can lead to segmentation +# faults when connecting to postgres +export OPENSSL_NO_VENDOR=1 + +export OPENSSL_LIB_DIR +OPENSSL_LIB_DIR="$(hab pkg path core/openssl)/lib64" +export OPENSSL_INCLUDE_DIR +OPENSSL_INCLUDE_DIR="$(hab pkg path core/openssl)/include" + +export SODIUM_USE_PKG_CONFIG=1 + +unset LD_RUN_PATH +export LD_RUN_PATH +LD_RUN_PATH="$(hab pkg path core/glibc)/lib" +LD_RUN_PATH+=":$(hab pkg path core/gcc-base)/lib64" +LD_RUN_PATH+=":$(hab pkg path core/binutils)/lib" +LD_RUN_PATH+=":$(hab pkg path core/libarchive)/lib" +LD_RUN_PATH+=":$(hab pkg path core/libb2)/lib" +LD_RUN_PATH+=":$(hab pkg path core/libsodium)/lib" +LD_RUN_PATH+=":$(hab pkg path core/openssl)/lib64" +LD_RUN_PATH+=":$(hab pkg path core/postgresql17)/lib" +LD_RUN_PATH+=":$(hab pkg path core/zeromq)/lib" +LD_RUN_PATH+=":$(hab pkg path core/zlib)/lib" +printf "\nLD_RUN_PATH='%s'\n" "${LD_RUN_PATH:-UNSET}" + +unset LD_LIBRARY_PATH +export LD_LIBRARY_PATH +LD_LIBRARY_PATH="$(hab pkg path core/gcc-base)/lib64" +LD_LIBRARY_PATH+=":$(hab pkg path core/zlib)/lib" +printf "\nLD_LIBRARY_PATH='%s'\n" "${LD_LIBRARY_PATH:-UNSET}" + +unset PKG_CONFIG_PATH +export PKG_CONFIG_PATH +PKG_CONFIG_PATH="$(hab pkg path core/libarchive)/lib/pkgconfig" +PKG_CONFIG_PATH+=":$(hab pkg path core/bash)/lib/pkgconfig" +PKG_CONFIG_PATH+=":$(hab pkg path core/libb2)/lib/pkgconfig" +PKG_CONFIG_PATH+=":$(hab pkg path core/libsodium)/lib/pkgconfig" +PKG_CONFIG_PATH+=":$(hab pkg path core/openssl)/lib64/pkgconfig" +PKG_CONFIG_PATH+=":$(hab pkg path core/postgresql17)/lib/pkgconfig" +PKG_CONFIG_PATH+=":$(hab pkg path core/zeromq)/lib/pkgconfig" +PKG_CONFIG_PATH+=":$(hab pkg path core/zlib)/lib/pkgconfig" +printf "\nPKG_CONFIG_PATH='%s'\n" "${PKG_CONFIG_PATH:-UNSET}" + +prepend_path="$(hab pkg path core/glibc)/bin" +prepend_path+=":$(hab pkg path core/gcc-base)/bin" +prepend_path+=":$(hab pkg path core/binutils)/bin" +prepend_path+=":$(hab pkg path core/bash)/bin" +prepend_path+=":$(hab pkg path core/coreutils)/bin" +prepend_path+=":$(hab pkg path core/curl)/bin" +prepend_path+=":$(hab pkg path core/diffutils)/bin" +prepend_path+=":$(hab pkg path core/gawk)/bin" +prepend_path+=":$(hab pkg path core/git)/bin" +prepend_path+=":$(hab pkg path core/grep)/bin" +prepend_path+=":$(hab pkg path core/hab-ld-wrapper)/bin" +prepend_path+=":$(hab pkg path core/libarchive)/bin" +prepend_path+=":$(hab pkg path core/make)/bin" +prepend_path+=":$(hab pkg path core/cmake)/bin" +prepend_path+=":$(hab pkg path core/openssl)/bin" +prepend_path+=":$(hab pkg path core/pkg-config)/bin" +prepend_path+=":$(hab pkg path core/postgresql17)/bin" +prepend_path+=":$(hab pkg path core/protobuf)/bin" +prepend_path+=":$(hab pkg path core/rust/"$toolchain")/bin" +prepend_path+=":$(hab pkg path core/sed)/bin" +prepend_path+=":$(hab pkg path core/sudo)/bin" +prepend_path+=":$(hab pkg path core/zeromq)/bin" +PATH=$prepend_path:$PATH +export PATH +unset prepend_path +printf "\nPATH='%s'\n\n" "${PATH:-UNSET}" diff --git a/terraform/files/db_connect.sh b/terraform/files/db_connect.sh index 8f70ca8728..dcf6757b3b 100644 --- a/terraform/files/db_connect.sh +++ b/terraform/files/db_connect.sh @@ -4,4 +4,4 @@ export PAGER=less PGHOST=$(sudo -E grep rds /hab/svc/builder-api/config/config.toml | awk -F\" '{print $2}') PGPASSWORD=$(sudo -E grep password /hab/svc/builder-api/config/config.toml | awk -F\" '{print $2}') -hab pkg exec core/postgresql psql -U hab -h "${PGHOST}" builder +hab pkg exec core/postgresql17 psql -U hab -h "${PGHOST}" builder diff --git a/test/builder-api/bin/cleanup-integration-tests.sh b/test/builder-api/bin/cleanup-integration-tests.sh index b98c2be50c..59a6e92313 100755 --- a/test/builder-api/bin/cleanup-integration-tests.sh +++ b/test/builder-api/bin/cleanup-integration-tests.sh @@ -1,5 +1,11 @@ #!/bin/bash +# This cleanup-integration-tests.sh file seems to have been OBE as the comment +# below seems to date to Nov 21st 2017 (based on git blame) but it references +# stuff from our Makefile that seems to have been was removed on Sept 7th 2018 +# (based on looking at our git history). However, this I think this file may be +# worth rehabbing at some point. -- Jason Heath + # You might be asking yourself "Why does this file even exist?" The answer to that question lies # in the amount of time it takes to run 'test.sh'. Since test.sh is designed to be run in CI, # and requires a full compilation of the entire builder cluster, plus a spin-up of a temporary @@ -27,13 +33,13 @@ set -eu base_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" dir="$base_dir/target/debug" depot=/root/habitat/tmp/depot -origins=( neurosis xmen ) -users=( bobo mystique ) +origins=(neurosis xmen) +users=(bobo mystique) # cleanup origins -for origin in "${origins[@]}" -do - sql=$(cat <