feat: Store log references in structured fields

Author: jamesmistry

.github/workflows/ci.yaml

@@ -65,11 +65,11 @@ jobs:
     - name: Setup | Rust Nightly
       run: "rustup install nightly && rustup default nightly"
     - name: Setup | LLVM Tooling
-      run: "rustup component add llvm-tools-preview"
+      run: "rustup component add llvm-tools-preview --toolchain nightly"
     - name: Setup | LLVM helper
-      run: "cargo install cargo-binutils && rustup component add llvm-tools-preview"
+      run: "cargo install cargo-binutils && cargo install cargo-llvm-cov"
     - name: Tests | App
-      run: './.github/workflows/test_with_coverage.bash "$(pwd)" &> ./small_test_coverage_report.txt && cat ./small_test_coverage_report.txt'
+      run: './.github/workflows/test_with_coverage.bash "$(pwd)" &> ./test_coverage_report.txt && cat ./test_coverage_report.txt'
     - name: Clean artifacts from test | App
       run: 'rm -rf ./target/*'
     - name: Create build environment | App
@@ -79,8 +79,8 @@ jobs:
     - name: Store code coverage report | App
       uses: actions/upload-artifact@v3
       with:
-        name: small_test_coverage_report
-        path: small_test_coverage_report.txt
+        name: test_coverage_report
+        path: test_coverage_report.txt
         retention-days: 7
     - name: Store debug output | App
       uses: actions/upload-artifact@v3

.github/workflows/test_with_coverage.bash

@@ -25,25 +25,8 @@ trap on_exit EXIT
 
 cd "${PROJ_DIR}" || exit 2
 
-scratchpad="$(mktemp -d)"
-printf "%s\n" "Profiling artefacts in ${scratchpad}"
-
-RUSTFLAGS="-C instrument-coverage" LLVM_PROFILE_FILE="${scratchpad}/profiler-%m.profraw" cargo test --tests || exit 3
-
-prof_out_file="${scratchpad}/profiler.profdata"
-rust-profdata merge -sparse ${scratchpad}/*.profraw -o "${prof_out_file}" || exit 4
-
-rust-cov report \
-    $( \
-      for file in \
-        $( \
-          RUSTFLAGS="-C instrument-coverage" \
-            cargo test --tests --no-run --message-format=json \
-              | jq -r "select(.profile.test == true) | .filenames[]" \
-              | grep -v dSYM - \
-        ); \
-      do \
-        printf "%s %s " -object $file; \
-      done \
-    ) \
-    --instr-profile="${prof_out_file}" --summary-only --ignore-filename-regex='/.cargo/registry' || exit 5
+# Generate a plain text report for easy consumption in stdout
+cargo llvm-cov || exit 3
+
+# Generate formatted HTML report (it will be stored in target/llvm-cov/html
+cargo llvm-cov report --html || exit 4

Breadlog.lock

@@ -2,4 +2,4 @@
 # If you would like to recalculate the next reference from your code, delete this file and
 # run Breadlog.
 
-next_reference_id: 35
+next_reference_id: 36

docs/index.rst

@@ -15,7 +15,7 @@ complex text parsing required.
    source/getting-started
    source/configuration
    source/running-breadlog
-   source/excluding-statements
+   source/directives
    source/known-limitations
    source/using-log-references
 

docs/source/configuration.rst

@@ -27,6 +27,14 @@ configuration options.
    # Required. Configuration stanza for Rust code.
    rust:
 
+     # Optional, default = false. If true, causes Breadlog to look for and 
+     # insert references in the "structured" arguments to log statements.
+     # Note that generated code requires use of the "kv" feature with the log
+     # crate.
+     #
+     # See https://docs.rs/log/latest/log/kv/index.html for more details.
+     structured: false
+
      # Required. Detail about the macros and their containing modules used in your 
      # code for logging. Breadlog assumes use of the log crate 
      # (https://docs.rs/log/latest/log/), and this example configuration specifies

docs/source/directives.rst

@@ -0,0 +1,37 @@
+Directives
+==========
+
+Directives provide ways to modify Breadlog behaviour from within your code 
+using comments.
+
+Disable use of structured logging
+---------------------------------
+
+When using structured logging there may be some situations where you still
+want a log statement to contain a reference in its log message, rather than
+as a key-value pair.
+
+In these scenarios, add a comment to the line before the corresponding 
+statement with the text ``breadlog:no-kvp``.
+
+For example:
+
+.. code-block:: rust
+
+   // breadlog:no-kvp
+   info!("[ref: 123] This log message will contain the reference, even when structured logging is on.");
+
+Ignore log statements
+---------------------
+
+If you'd like Breadlog to ignore particular log statements, add a comment to 
+the line before the statement with the text ``breadlog:ignore``.
+
+For example:
+
+.. code-block:: rust
+
+   // breadlog:ignore
+   info!("This log statement will be ignored by Breadlog.");
+
+

docs/source/excluding-statements.rst

@@ -40,6 +40,7 @@ Step 2: Configuring a repository
       ---
       source_dir: <RELATIVE SOURCE DIRECTORY>
       rust:
+        structured: false
         log_macros:
           - module: log
             name: info
@@ -58,6 +59,10 @@ Step 2: Configuring a repository
 This configuration assumes you're using the `Rust log crate <https://crates.io/crates/log>`_
 for logging in your code.
 
+If you're using structured logging (the "kv" feature), set ``structured`` to 
+``true`` and Breadlog will maintain references as key-value pairs rather than 
+message text.
+
 Step 3: Running Breadlog for the first time
 -------------------------------------------
 
@@ -104,7 +109,7 @@ references in check mode.
 
    If you'd like Breadlog to ignore a particular log statement, add a comment 
    to the line before the statement with the text ``breadlog:ignore``. For
-   more details, see :doc:`excluding-statements`.
+   more details, see :doc:`directives`.
 
 2. Once you're happy with the output, you can run Breadlog in code generation
    mode (without the ``--check`` flag). This will modify your code, inserting 

docs/source/getting-started.rst

@@ -4,6 +4,10 @@ Using Log References
 Extracting references from log messages
 ---------------------------------------
 
+If you're not using structured logging, you'll need to extract references
+from log message text through configuration of a log ingestion tool
+(such as `Vector <https://vector.dev/guides/level-up/transformation/>`_).
+
 References can be extracted from log messages using the following regular
 expression: