Skip to content

Commit 31d5687

Browse files
author
Jason Heath
committed
Use LTS-2024 pervasively and address segfaults
This commit also overhauls several scripts in order to deal with glibc conflicts that led to most of the verify.pipeline failing with segfaults in the compliation stage. It happens that core/rust/1.79.0 was built against core/glibc/2.35 while the LTS-2024 packages are built against core/glibc/2.36. This commit provides a shared build environment that solves the issue everywhere while drying up several inconsistent variations of building out the environment. Signed-off-by: Jason Heath <[email protected]>
1 parent 2ac65a8 commit 31d5687

File tree

15 files changed

+245
-140
lines changed

15 files changed

+245
-140
lines changed

‎.expeditor/scripts/post_habitat_release/cargo_update.sh

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
#!/bin/bash
22

3-
set -euo pipefail
3+
set -euo pipefail
44

5-
# shellcheck source=.expeditor/scripts/shared.sh
6-
source .expeditor/scripts/post_habitat_release/shared.sh
5+
# shellcheck source=.expeditor/scripts/post_habitat_release/shared.sh
6+
source .expeditor/scripts/post_habitat_release/shared.sh
77

88
branch="expeditor/cargo-update-$(date +"%Y%m%d%H%M%S")"
99
git checkout -b "$branch"
@@ -13,21 +13,7 @@ toolchain="$(get_toolchain)"
1313
install_hub
1414

1515
echo "--- :habicat: Installing and configuring build dependencies"
16-
hab pkg install core/rust/"$toolchain" \
17-
core/libarchive \
18-
core/openssl \
19-
core/pkg-config \
20-
core/postgresql17 \
21-
core/protobuf \
22-
core/zeromq \
23-
core/cmake
24-
25-
export OPENSSL_NO_VENDOR=1
26-
export LD_RUN_PATH
27-
LD_RUN_PATH="$(hab pkg path core/glibc)/lib:$(hab pkg path core/gcc-libs)/lib:$(hab pkg path core/openssl)/lib:$(hab pkg path core/postgresql17)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/libarchive)/lib"
28-
export PKG_CONFIG_PATH
29-
PKG_CONFIG_PATH="$(hab pkg path core/zeromq)/lib/pkgconfig:$(hab pkg path core/libarchive)/lib/pkgconfig:$(hab pkg path core/postgresql17)/lib/pkgconfig:$(hab pkg path core/openssl)/lib/pkgconfig"
30-
eval "$(hab pkg env core/rust/"$toolchain"):$(hab pkg path core/protobuf)/bin:$(hab pkg path core/pkg-config)/bin:$(hab pkg path core/postgresql17)/bin:$(hab pkg path core/cmake)/bin:$PATH"
16+
source support/ci/shared_build_environment.sh
3117

3218
echo "--- :rust: Cargo Update"
3319
cargo clean
@@ -43,14 +29,14 @@ git commit -s -m "Update Cargo.lock"
4329

4430
pr_labels=""
4531
pr_message=""
46-
if [ "$update_status" -ne 0 ]; then
32+
if [ "$update_status" -ne 0 ]; then
4733
pr_labels="T-DO-NOT-MERGE"
4834

4935
# read will exit 1 if it can't find a delimeter.
50-
# -d '' will always trigger this case as there is no delimeter to find,
51-
# but this is required in order to write the entire message into a single PR
36+
# -d '' will always trigger this case as there is no delimeter to find,
37+
# but this is required in order to write the entire message into a single PR
5238
# preserving newlines.
53-
read -r -d '' pr_message <<EOM || true
39+
read -r -d '' pr_message <<EOM || true
5440
Unable to update Cargo.lock!
5541
5642
For details on the failure, please visit ${BUILDKITE_BUILD_URL:-No Buildkite url}#${BUILDKITE_JOB_ID:-No Buildkite job id}
@@ -63,7 +49,7 @@ fi
6349
# the latter requires multiple curl commands and parsing json responses and error handling at each step.
6450
push_current_branch
6551

66-
# We have to use --force to open the PR. We're specifying where to push, rather than using a remote, in
52+
# We have to use --force to open the PR. We're specifying where to push, rather than using a remote, in
6753
# the previous command to avoid writing secrets to disk, so hub isn't able to read that information from
6854
# the git configuration
6955
hub pull-request --force --no-edit --labels "$pr_labels" --file - <<EOF

‎.expeditor/scripts/release_builder/update_api_docs.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ if [ -z "$hub_check" ]; then
1515
install_hub
1616
fi
1717

18-
hab pkg install core/node
18+
hab pkg install --channel LTS-2024 core/node
1919

2020
npm install [email protected]
2121
npm install [email protected]
@@ -26,7 +26,7 @@ cd "${tempdir}"
2626
git clone "https://github.com/habitat-sh/habitat.git"
2727
cd ..
2828

29-
# Generate the api docs file.
29+
# Generate the api docs file.
3030
input_file=components/${component}/doc/api.raml
3131
output_file=${tempdir}/${component}".json"
3232

‎.expeditor/templates/studiorc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
cd /src || exit
33

44
echo "--- Installing prerequisites"
5-
hab pkg install core/openssl
6-
hab pkg install core/node --binlink
7-
hab pkg install core/coreutils
5+
hab pkg install --channel LTS-2024 core/openssl
6+
hab pkg install --channel LTS-2024 core/node --binlink
7+
hab pkg install --channel LTS-2024 core/coreutils
88
hab pkg binlink core/coreutils env -d /usr/bin
99
(
1010
# At the point studiorc is evaluated, our cwd is /

‎.studiorc

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,6 @@ generate_bldr_keys() {
251251
}
252252

253253
load_package() {
254-
# JAH: Should there really be a --channel option here at all?
255254
hab pkg upload --url http://localhost --auth "${HAB_AUTH_TOKEN}" "$@" --channel "${HAB_BLDR_CHANNEL:-LTS-2024}"
256255
}
257256

‎build.sh

Lines changed: 1 addition & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,31 +2,9 @@
22

33
set -eou pipefail
44

5-
source ./support/ci/shared.sh
6-
7-
toolchain=$(get_toolchain)
8-
95
component=${1?component argument required}
106

11-
# Accept hab license
12-
sudo hab pkg install core/rust/"$toolchain" --channel LTS-2024
13-
sudo hab pkg install core/libarchive --channel LTS-2024
14-
sudo hab pkg install core/openssl --channel LTS-2024
15-
sudo hab pkg install core/zeromq --channel LTS-2024
16-
sudo hab pkg install core/pkg-config --channel LTS-2024
17-
sudo hab pkg install core/protobuf --channel LTS-2024
18-
sudo hab pkg install core/postgresql17 --channel LTS-2024
19-
sudo hab pkg install core/cmake --channel LTS-2024
20-
# It is important NOT to use a vendored openssl from openssl-sys
21-
# pg-sys does not use openssl-sys. So for components that use
22-
# diesel's postgres feature, you wil end up with 2 versions of openssl
23-
# which can lead to segmentation faults when connecting to postgres
24-
export OPENSSL_NO_VENDOR=1
25-
export LD_RUN_PATH
26-
LD_RUN_PATH="$(hab pkg path core/glibc)/lib:$(hab pkg path core/gcc-libs)/lib:$(hab pkg path core/openssl)/lib:$(hab pkg path core/postgresql17)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/libarchive)/lib"
27-
export PKG_CONFIG_PATH
28-
PKG_CONFIG_PATH="$(hab pkg path core/zeromq)/lib/pkgconfig:$(hab pkg path core/libarchive)/lib/pkgconfig:$(hab pkg path core/postgresql17)/lib/pkgconfig:$(hab pkg path core/openssl)/lib/pkgconfig"
29-
eval "$(hab pkg env core/rust/"$toolchain"):$(hab pkg path core/protobuf)/bin:$(hab pkg path core/pkg-config)/bin:$(hab pkg path core/postgresql17)/bin:$(hab pkg path core/cmake)/bin:$PATH"
7+
source support/ci/shared_build_environment.sh
308

319
cd "components/$component"
3210
cargo build

‎components/builder-db/tests/db/start.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ mkdir -p /hab/svc/postgresql
2121
mkdir -p /hab/user/postgresql
2222
cp "$DB_TEST_DIR"/pg_hba.conf /hab/svc/postgresql
2323
cp "$DB_TEST_DIR"/user.toml /hab/user/postgresql
24-
hab sup run core/postgresql17 &
24+
hab sup run --channel LTS-2024 core/postgresql17 &
2525
hab_pid=$!
2626

2727
read -r sudo_ppid < <(ps -p $$ -o 'ppid=')
@@ -35,4 +35,3 @@ while true; do
3535
fi
3636
sleep 1
3737
done
38-

‎support/builder/build_dep_populate.sh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ installDeps() (
120120
configMsg
121121

122122
for bin in "${pdeps[@]}"; do
123-
hab pkg install -b "${bin}"
123+
hab pkg install --channel LTS-2024 --binlink "${bin}"
124124
done
125125
)
126126

@@ -154,7 +154,7 @@ setBucket() {
154154
echo ""
155155
echo "Configured bucket: ${bucket_name} has been verified!"
156156
echo ""
157-
if promptUser "Are you sure you would like to use this bucket?"; then
157+
if promptUser "Are you sure you would like to use this bucket?"; then
158158
echo "Using specified bucket."
159159
S3_BUCKET=${bucket_name}
160160
else
@@ -352,7 +352,7 @@ welcome() {
352352
echo "${HAB_AUTH_TOKEN}"
353353
if promptUser "Would you like to use these credentials?"; then
354354
echo "Setting detected credentials"
355-
else
355+
else
356356
genBldrCreds
357357
fi
358358
else

‎support/builder/s3migrate.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ installDeps() {
7777
configMsg
7878

7979
for bin in "${!PDEPS[@]}"; do
80-
hab pkg install -b "${PDEPS[$bin]}"
80+
hab pkg install --channel=LTS-2024 --binlink "${PDEPS[$bin]}"
8181
done
8282
}
8383

‎support/ci/install_protobuf.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
#!/bin/sh
22

33
export HAB_LICENSE="accept-no-persist"
4-
sudo -E hab pkg install core/protobuf-cpp -b
4+
sudo hab pkg install --channel=LTS-2024 --binlink core/protobuf-cpp

‎support/ci/shared.sh

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ get_rustfmt_toolchain() {
2020
}
2121

2222
get_toolchain() {
23-
tail -n 1 "$dir/../../rust-toolchain" | cut -d'"' -f 2
23+
tail -n 1 "$dir/../../rust-toolchain" | cut -d'"' -f 2
2424
}
2525

2626
install_rustup() {
@@ -63,12 +63,6 @@ install_rustfmt() {
6363

6464
install_hab_pkg() {
6565
for ident; do
66-
installed_pkgs=$(hab pkg list "$ident")
67-
68-
if [[ -z $installed_pkgs ]]; then
69-
sudo hab pkg install "$ident"
70-
else
71-
echo "$ident already installed"
72-
fi
66+
sudo hab pkg install "$ident" --channel=LTS-2024 --ignore-local
7367
done
7468
}

‎support/ci/shared_build_environment.sh

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
#!/bin/bash
2+
3+
sudo hab license accept
4+
5+
# shellcheck source=./shared.sh
6+
source ./support/ci/shared.sh
7+
8+
OG_PATH=$PATH
9+
toolchain=$(get_toolchain)
10+
eval "$(hab pkg env core/rust/"$toolchain")"
11+
PATH=$PATH:$OG_PATH
12+
HAB_PATH="$(hab pkg path core/hab)/bin"
13+
14+
install_hab_pkg core/glibc
15+
install_hab_pkg core/gcc-base
16+
install_hab_pkg core/binutils
17+
install_hab_pkg core/cmake
18+
install_hab_pkg core/bash
19+
install_hab_pkg core/coreutils
20+
install_hab_pkg core/curl
21+
install_hab_pkg core/diffutils
22+
install_hab_pkg core/gawk
23+
install_hab_pkg core/git
24+
install_hab_pkg core/grep
25+
install_hab_pkg core/libarchive
26+
install_hab_pkg core/libb2
27+
install_hab_pkg core/libsodium
28+
install_hab_pkg core/make
29+
install_hab_pkg core/openssl
30+
install_hab_pkg core/pkg-config
31+
install_hab_pkg core/postgresql17
32+
install_hab_pkg core/protobuf
33+
install_hab_pkg core/rust/"$toolchain"
34+
install_hab_pkg core/sed
35+
install_hab_pkg core/sudo
36+
install_hab_pkg core/zeromq
37+
38+
# It is important NOT to use a vendored openssl from openssl-sys pg-sys does
39+
# not use openssl-sys. So for components that use diesel's postgres feature,
40+
# you wil end up with 2 versions of openssl which can lead to segmentation
41+
# faults when connecting to postgres
42+
export OPENSSL_NO_VENDOR=1
43+
44+
export OPENSSL_LIB_DIR
45+
OPENSSL_LIB_DIR="$(hab pkg path core/openssl)/lib64"
46+
export OPENSSL_INCLUDE_DIR
47+
OPENSSL_INCLUDE_DIR="$(hab pkg path core/openssl)/include"
48+
49+
export SODIUM_USE_PKG_CONFIG=1
50+
51+
unset LD_RUN_PATH
52+
export LD_RUN_PATH
53+
LD_RUN_PATH="$(hab pkg path core/glibc)/lib"
54+
LD_RUN_PATH+=":$(hab pkg path core/gcc-base)/lib64"
55+
LD_RUN_PATH+=":$(hab pkg path core/binutils)/lib"
56+
LD_RUN_PATH+=":$(hab pkg path core/libarchive)/lib"
57+
LD_RUN_PATH+=":$(hab pkg path core/libb2)/lib"
58+
LD_RUN_PATH+=":$(hab pkg path core/libsodium)/lib"
59+
LD_RUN_PATH+=":$(hab pkg path core/openssl)/lib64"
60+
LD_RUN_PATH+=":$(hab pkg path core/postgresql17)/lib"
61+
LD_RUN_PATH+=":$(hab pkg path core/zeromq)/lib"
62+
printf "\nLD_RUN_PATH='%s'\n" "${LD_RUN_PATH:-UNSET}"
63+
64+
if ${BUILDKITE:-false}; then
65+
unset LD_LIBRARY_PATH
66+
export LD_LIBRARY_PATH
67+
LD_LIBRARY_PATH="$(hab pkg path core/gcc-base)/lib64"
68+
# LD_LIBRARY_PATH="$(hab pkg path core/glibc)/lib"
69+
# LD_LIBRARY_PATH+=":$(hab pkg path core/binutils)/lib"
70+
# LD_LIBRARY_PATH+=":$(hab pkg path core/zeromq)/lib"
71+
72+
# LD_PRELOAD="$(hab pkg path core/glibc)/lib/libc.so.6"
73+
# export LD_PRELOAD
74+
fi
75+
printf "\nLD_LIBRARY_PATH='%s'\n" "${LD_LIBRARY_PATH:-UNSET}"
76+
77+
unset PKG_CONFIG_PATH
78+
export PKG_CONFIG_PATH
79+
PKG_CONFIG_PATH="$(hab pkg path core/libarchive)/lib/pkgconfig"
80+
PKG_CONFIG_PATH+=":$(hab pkg path core/bash)/lib/pkgconfig"
81+
PKG_CONFIG_PATH+=":$(hab pkg path core/libb2)/lib/pkgconfig"
82+
PKG_CONFIG_PATH+=":$(hab pkg path core/libsodium)/lib/pkgconfig"
83+
PKG_CONFIG_PATH+=":$(hab pkg path core/openssl)/lib64/pkgconfig"
84+
PKG_CONFIG_PATH+=":$(hab pkg path core/postgresql17)/lib/pkgconfig"
85+
PKG_CONFIG_PATH+=":$(hab pkg path core/zeromq)/lib/pkgconfig"
86+
printf "\nPKG_CONFIG_PATH='%s'\n" "${PKG_CONFIG_PATH:-UNSET}"
87+
88+
PATH=$HAB_PATH
89+
export PATH
90+
PATH+=":$(hab pkg path core/glibc)/bin"
91+
PATH+=":$(hab pkg path core/gcc-base)/bin"
92+
PATH+=":$(hab pkg path core/binutils)/bin"
93+
PATH+=":$(hab pkg path core/bash)/bin"
94+
PATH+=":$(hab pkg path core/coreutils)/bin"
95+
PATH+=":$(hab pkg path core/curl)/bin"
96+
PATH+=":$(hab pkg path core/diffutils)/bin"
97+
PATH+=":$(hab pkg path core/gawk)/bin"
98+
PATH+=":$(hab pkg path core/git)/bin"
99+
PATH+=":$(hab pkg path core/grep)/bin"
100+
PATH+=":$(hab pkg path core/libarchive)/bin"
101+
PATH+=":$(hab pkg path core/make)/bin"
102+
PATH+=":$(hab pkg path core/cmake)/bin"
103+
PATH+=":$(hab pkg path core/openssl)/bin"
104+
PATH+=":$(hab pkg path core/pkg-config)/bin"
105+
PATH+=":$(hab pkg path core/postgresql17)/bin"
106+
PATH+=":$(hab pkg path core/protobuf)/bin"
107+
PATH+=":$(hab pkg path core/rust/"$toolchain")/bin"
108+
PATH+=":$(hab pkg path core/sed)/bin"
109+
PATH+=":$(hab pkg path core/sudo)/bin"
110+
PATH+=":$(hab pkg path core/zeromq)/bin"
111+
export PATH
112+
printf "\nPATH='%s'\n\n" "${PATH:-UNSET}"

‎support/linux/provision.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
set -eux
33

44
curl https://raw.githubusercontent.com/habitat-sh/habitat/master/components/hab/install.sh | sudo bash
5-
sudo hab install core/busybox-static core/hab-studio
6-
sudo hab install \
7-
core/direnv \
5+
sudo hab install --channel stable --binlink core/direnv core/hab-studio
6+
sudo hab install --channel LTS-2024 --binlink \
7+
core/busybox-static \
88
core/wget \
99
core/docker \
10-
core/curl -b
10+
core/curl
1111
# shellcheck disable=SC2016
1212
echo 'eval "$(direnv hook bash)"' >> ~/.bashrc

‎test/builder-api/test.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ clean_test_artifacts() {
5353
wait_for_migrations() {
5454
echo "Waiting for migrations to finish"
5555
local count=0
56-
# while ! command with set -e fails on the first loop, so we get this slightly
56+
# while ! command with set -e fails on the first loop, so we get this slightly
5757
# more complex implementation
5858
while true; do
5959
# The status endpoint won't become available until migrations are finished
@@ -76,7 +76,7 @@ wait_for_migrations
7676
clean_test_artifacts
7777

7878
if ! command -v npm >/dev/null 2>&1; then
79-
hab pkg install core/node -b
79+
hab pkg install --channel LTS-2024 --binlink core/node
8080
fi
8181

8282
if ! [ -f /usr/bin/env ]; then

‎test/run_cargo_test.sh

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,32 +2,9 @@
22

33
set -eou pipefail
44

5-
source ./support/ci/shared.sh
6-
7-
toolchain=$(get_toolchain)
8-
95
component=${1?component argument required}
106

11-
# Accept hab license
12-
sudo hab license accept
13-
sudo hab pkg install core/rust/"$toolchain"
14-
sudo hab pkg install core/libarchive
15-
sudo hab pkg install core/openssl
16-
sudo hab pkg install core/zeromq
17-
sudo hab pkg install core/pkg-config
18-
sudo hab pkg install core/protobuf
19-
sudo hab pkg install core/postgresql17
20-
sudo hab pkg install core/cmake
21-
# It is important NOT to use a vendored openssl from openssl-sys
22-
# pg-sys does not use openssl-sys. So for components that use
23-
# diesel's postgres feature, you wil end up with 2 versions of openssl
24-
# which can lead to segmentation faults when connecting to postgres
25-
export OPENSSL_NO_VENDOR=1
26-
export LD_RUN_PATH
27-
LD_RUN_PATH="$(hab pkg path core/glibc)/lib:$(hab pkg path core/gcc-libs)/lib:$(hab pkg path core/openssl)/lib:$(hab pkg path core/postgresql17)/lib:$(hab pkg path core/zeromq)/lib:$(hab pkg path core/libarchive)/lib"
28-
export PKG_CONFIG_PATH
29-
PKG_CONFIG_PATH="$(hab pkg path core/zeromq)/lib/pkgconfig:$(hab pkg path core/libarchive)/lib/pkgconfig:$(hab pkg path core/postgresql17)/lib/pkgconfig:$(hab pkg path core/openssl)/lib/pkgconfig"
30-
eval "$(hab pkg env core/rust/"$toolchain"):$(hab pkg path core/protobuf)/bin:$(hab pkg path core/pkg-config)/bin:$(hab pkg path core/postgresql17)/bin:$(hab pkg path core/cmake)/bin:$PATH"
7+
source support/ci/shared_build_environment.sh
318

329
# Set testing filesystem root
3310
export TESTING_FS_ROOT

0 commit comments

Comments
 (0)