MONGOCRYPT-777 update scripts and release instructions for SilkBomb 2.0 (#964)

* Remove silk-create-asset-group.sh
* Remove Earthly sbom-download
* Update Earthly targets and commands for SilkBomb 2.0
* Update release instructions for SilkBomb 2.0

Author: eramongodb

.evergreen/config.yml

@@ -388,6 +388,55 @@ functions:
           # Earthly hardcodes use of docker/dockerfile-copy:v0.1.9 for the CACHE command.
           bash ${workdir}/libmongocrypt/.evergreen/earthly.sh ${args} --persist-build=false
 
+  sbom:
+    - command: ec2.assume_role
+      type: setup
+      params:
+        role_arn: ${kondukto_role_arn}
+    - command: subprocess.exec
+      type: setup
+      params:
+        binary: bash
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+        args:
+          - -c
+          - |
+            set -o errexit
+            set -o pipefail
+            kondukto_token="$(aws secretsmanager get-secret-value --secret-id "kondukto-token" --region "us-east-1" --query 'SecretString' --output text)"
+            printf "KONDUKTO_TOKEN: %s\n" "$kondukto_token" >|expansions.kondukto.yml
+    - command: expansions.update
+      type: setup
+      params:
+        file: expansions.kondukto.yml
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: libmongocrypt
+        include_expansions_in_env:
+          - artifactory_password
+          - artifactory_username
+          - branch_name
+          - KONDUKTO_TOKEN
+        args:
+          - -c
+          - .evergreen/sbom.sh
+    - command: s3.put
+      type: test
+      params:
+        display_name: Augmented SBOM
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        bucket: mciuploads
+        content_type: application/json
+        local_file: libmongocrypt/cyclonedx.augmented.sbom.json
+        permissions: public-read
+        remote_file: ${project}/${build_variant}/${branch_name}/${libmongocrypt_s3_suffix}/sbom/cyclonedx.augmented.sbom.json
+
 tasks:
 - name: build-and-test-and-upload
   commands:
@@ -994,6 +1043,11 @@ tasks:
       params:
         file: libmongocrypt/bindings/python/results.json
 
+- name: sbom
+  commands:
+    - func: fetch source
+    - func: sbom
+
 pre:
   # Update the evergreen expansion to dynamically set the ${libmongocrypt_s3_suffix} and ${libmongocrypt_s3_suffix_copy} expansions.
   - command: "shell.exec"
@@ -1586,3 +1640,10 @@ buildvariants:
   - name: publish-deb-packages-with-earthly
     # Use a distro suitable for running curator to publish .deb packages.
     run_on: ubuntu2004-small
+
+- name: sbom
+  display_name: SBOM
+  tasks:
+    - name: sbom
+      # Just for high host availability. Not platform-specific.
+      run_on: rhel80

.evergreen/sbom.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+: "${artifactory_username:?}"
+: "${artifactory_password:?}"
+: "${branch_name:?}"
+: "${KONDUKTO_TOKEN:?}"
+
+command -v podman >/dev/null || {
+  echo "missing required program podman" 1>&2
+  exit 1
+}
+
+podman login --password-stdin --username "${artifactory_username:?}" artifactory.corp.mongodb.com <<<"${artifactory_password:?}"
+
+silkbomb="artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0"
+
+# Ensure latest version of SilkBomb is being used.
+podman pull "${silkbomb:?}"
+
+silkbomb_augment_flags=(
+  --repo mongodb/libmongocrypt
+  --branch "${branch_name:?}"
+  --sbom-in /pwd/etc/cyclonedx.sbom.json
+  --sbom-out /pwd/cyclonedx.augmented.sbom.json
+
+  # Any notable updates to the Augmented SBOM version should be done manually after careful inspection.
+  # Otherwise, it should be equal to the SBOM Lite version, which should normally be `1`.
+  --no-update-sbom-version
+)
+
+# First validate the SBOM Lite.
+podman run -it --rm -v "$(pwd):/pwd" "${silkbomb:?}" \
+  validate --purls /pwd/etc/purls.txt --sbom-in /pwd/etc/cyclonedx.sbom.json --exclude jira
+
+# Then download the Augmented SBOM. Allow the timestamp to be updated.
+podman run -it --rm -v "$(pwd):/pwd" --env 'KONDUKTO_TOKEN' "${silkbomb:?}" \
+  augment "${silkbomb_augment_flags[@]:?}"
+
+[[ -f ./cyclonedx.augmented.sbom.json ]] || {
+  echo "failed to download Augmented SBOM" 1>&2
+  exit 1
+}

Earthfile

@@ -462,62 +462,60 @@ sign:
 #
 # See https://docs.devprod.prod.corp.mongodb.com/mms/python/src/sbom/silkbomb/ for documentation of silkbomb.
 silkbomb:
-    FROM artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:1.0
-    # Alias the silkbom executable to a simpler name:
+    FROM artifactory.corp.mongodb.com/release-tools-container-registry-public-local/silkbomb:2.0
+    # Alias the silkbomb executable to a simpler name:
     RUN ln -s /python/src/sbom/silkbomb/bin /usr/local/bin/silkbomb
 
 # sbom-generate:
 #   Generate/update the etc/cyclonedx.sbom.json file from the etc/purls.txt file.
 #
 # This target will update the existing etc/cyclonedx.sbom.json file in-place based
-# on the content of etc/purls.txt.
-#
+# on the content of etc/purls.txt and etc/cyclonedx.sbom.json.
 sbom-generate:
     FROM +silkbomb
     # Copy in the relevant files:
     WORKDIR /s
     COPY etc/purls.txt etc/cyclonedx.sbom.json /s/
     # Update the SBOM file:
     RUN silkbomb update \
+        --refresh \
+        --no-update-sbom-version \
         --purls purls.txt \
         --sbom-in cyclonedx.sbom.json \
         --sbom-out cyclonedx.sbom.json
     # Save the result back to the host:
     SAVE ARTIFACT /s/cyclonedx.sbom.json AS LOCAL etc/cyclonedx.sbom.json
 
-# sbom-download:
-#   Download the Augmented SBOM file from Silk.
-#
-# See https://wiki.corp.mongodb.com/display/DRIVERS/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets for instructions to get secrets from AWS Secrets Manager. Secrets are available under `drivers/libmongocrypt`.
+# sbom-generate-new-serial-number:
+#   Equivalent to +sbom-generate but includes the --generate-new-serial-number
+#   flag to generate a new unique serial number and reset the SBOM version to 1.
 #
-sbom-download:
-    ARG --required out
-    ARG --required branch
+# This target will update the existing etc/cyclonedx.sbom.json file in-place based
+# on the content of etc/purls.txt and etc/cyclonedx.sbom.json.
+sbom-generate-new-serial-number:
     FROM +silkbomb
+    # Copy in the relevant files:
     WORKDIR /s
-    # Download the Augmented SBOM file:
-    RUN --no-cache --secret silk_client_id --secret silk_client_secret \
-        SILK_CLIENT_ID=${silk_client_id} \
-        SILK_CLIENT_SECRET=${silk_client_secret} \
-        silkbomb download \
-        --sbom-out cyclonedx.augmented.sbom.json \
-        --silk-asset-group libmongocrypt-${branch}
+    COPY etc/purls.txt etc/cyclonedx.sbom.json /s/
+    # Update the SBOM file:
+    RUN silkbomb update \
+        --refresh \
+        --generate-new-serial-number \
+        --purls purls.txt \
+        --sbom-in cyclonedx.sbom.json \
+        --sbom-out cyclonedx.sbom.json
     # Save the result back to the host:
-    SAVE ARTIFACT /s/cyclonedx.augmented.sbom.json AS LOCAL ${out}
-    RUN echo "Augmented SBOM saved to ${out}"
+    SAVE ARTIFACT /s/cyclonedx.sbom.json AS LOCAL etc/cyclonedx.sbom.json
 
-# silk-create-asset-group:
-#   Create an asset group for Silk.
-#
-# See https://wiki.corp.mongodb.com/display/DRIVERS/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets for instructions to get secrets from AWS Secrets Manager. Secrets are available under `drivers/libmongocrypt`.
-#
-silk-create-asset-group:
-    ARG --required branch
-    FROM +env.alpine
-    RUN __install curl jq
-    COPY etc/silk-create-asset-group.sh /s/silk-create-asset-group.sh
-    RUN --no-cache --secret silk_client_id --secret silk_client_secret \
-        silk_client_id=${silk_client_id} \
-        silk_client_secret=${silk_client_secret} \
-        branch=${branch} \
-            /s/silk-create-asset-group.sh
+# sbom-validate:
+#   Validate the SBOM Lite for the given branch.
+sbom-validate:
+    FROM +silkbomb
+    # Copy in the relevant files:
+    WORKDIR /s
+    COPY etc/purls.txt etc/cyclonedx.sbom.json /s/
+    # Run the SilkBomb tool to download the artifact that matches the requested branch
+    RUN silkbomb validate \
+        --purls purls.txt \
+        --sbom-in cyclonedx.sbom.json \
+        --exclude jira

doc/releasing.md

@@ -44,18 +44,15 @@ snyk monitor \
 
 Check the updated reference targets in Snyk for detected vulnerabilities.
 
-#### Check Silk
+#### Check the Augmented SBOM
 
-Get credentials for Silk from the `drivers/libmongocrypt` vault in [AWS Secrets Manager](https://wiki.corp.mongodb.com/display/DRIVERS/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets).
+Examine the Augmented SBOM from a recent execution of the `sbom` task in an Evergreen patch or commit build.
+
+Evergreen CLI may be used to schedule only the `sbom` task:
 
-Download the Augmented SBOM using:
 ```bash
-./.evergreen/earthly.sh \
-   --secret silk_client_id=${silk_client_id} \
-   --secret silk_client_secret=${silk_client_secret} \
-   +sbom-download \
-   --out cyclonedx.augmented.sbom.json \
-   --branch <branch>
+# Ensure `-p` matches the correct Evergreen project for the current branch!
+evergreen patch -y -p libmongocrypt -t all -v sbom -f
 ```
 
 Check the contents of the "vulnerabilities" field (if present) in the Augmented SBOM.
@@ -83,29 +80,16 @@ Do the following when releasing:
       - If the `publish-packages` tasks fail with an error like `[curator] 2024/01/02 13:56:17 [p=emergency]: problem submitting repobuilder job: 404 (Not Found)`, this suggests the published path does not yet exist. Barque (the Linux package publishing service) has protection to avoid unintentional publishes. File a DEVPROD ticket ([example](https://jira.mongodb.org/browse/DEVPROD-4053)) and assign to the team called Release Infrastructure to request the path be created. Then re-run the failing `publish-packages` task. Ask in the slack channel `#devprod-release-tools` for further help with `Barque` or `curator`.
 - Create the release from the GitHub releases page from the new tag.
    - Attach the tarball and signature file from the Files tab of the `windows-upload-release` task. [Example](https://github.com/mongodb/libmongocrypt/releases/tag/1.10.0).
-   - Attach the Augmented SBOM file. Download the Augmented SBOM using:
-     ```bash
-     ./.evergreen/earthly.sh \
-        --secret silk_client_id=${silk_client_id} \
-        --secret silk_client_secret=${silk_client_secret} \
-        +sbom-download \
-        --out cyclonedx.augmented.sbom.json \
-        --branch <branch>
-     ```
-     For a new minor release, use `master` for `--branch`. For a patch release, use the release branch (e.g. `rx.y`).
-     Secrets can be obtained from [AWS Secrets Manager](https://wiki.corp.mongodb.com/display/DRIVERS/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets) under `drivers/libmongocrypt`.
+   - Attach the Augmented SBOM file to the release as `cyclonedx.augmented.sbom.json`.
+     Download the Augmented SBOM from a recent execution of the `sbom` task in an Evergreen patch or commit build.
    - Attach `etc/third_party_vulnerabilities.md` to the release.
    - Attach `etc/ssdlc_compliance_report.md` to the release.
 
 - If this is a new minor release (e.g. `x.y.0`):
    - File a DOCSP ticket to update the installation instructions on [Install libmongocrypt](https://www.mongodb.com/docs/manual/core/csfle/reference/libmongocrypt/). ([Example](https://jira.mongodb.org/browse/DOCSP-36863))
-   - Create a new Silk asset group. Use the newly created release branch (e.g. `rx.y`) as the `--branch` argument:
+   - Generate a new unique SBOM serial number for the next release:
      ```bash
-     ./.evergreen/earthly.sh \
-        --secret silk_client_id=${silk_client_id} \
-        --secret silk_client_secret=${silk_client_secret} \
-        +silk-create-asset-group \
-        --branch <branch>
+     ./.evergreen/earthly.sh +sbom-generate-new-serial-number
      ```
    - Create a new Snyk reference target. The following instructions use the example branch `rx.y`:
 
@@ -164,4 +148,3 @@ index 609dc0b..f7530a9 100644
      Architectures: amd64 arm64
    suites:
 ```
-