Merge branch 'main' into breaking

Author: penelopeysm

.github/workflows/Benchmarking.yml

@@ -0,0 +1,76 @@
+name: Benchmarking
+
+on:
+  pull_request:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Julia
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+
+      - name: Install Dependencies
+        run: julia --project=benchmarks/ -e 'using Pkg; Pkg.instantiate()'
+
+      - name: Run Benchmarks
+        id: run_benchmarks
+        run: |
+          # Capture version info into a variable, print it, and set it as an env var for later steps
+          version_info=$(julia -e 'using InteractiveUtils; versioninfo()')
+          echo "$version_info"
+          echo "VERSION_INFO<<EOF" >> $GITHUB_ENV
+          echo "$version_info" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+          # Capture benchmark output into a variable
+          echo "Running Benchmarks..."
+          benchmark_output=$(julia --project=benchmarks benchmarks/benchmarks.jl)
+          
+          # Print benchmark results directly to the workflow log
+          echo "Benchmark Results:"
+          echo "$benchmark_output"
+          
+          # Set the benchmark output as an env var for later steps
+          echo "BENCHMARK_OUTPUT<<EOF" >> $GITHUB_ENV
+          echo "$benchmark_output" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+          # Get the current commit SHA of DynamicPPL
+          DPPL_COMMIT_SHA=$(git rev-parse HEAD)
+          echo "DPPL_COMMIT_SHA=$DPPL_COMMIT_SHA" >> $GITHUB_ENV
+
+          COMMIT_URL="https://github.com/${{ github.repository }}/commit/$DPPL_COMMIT_SHA"
+          echo "DPPL_COMMIT_URL=$COMMIT_URL" >> $GITHUB_ENV
+
+      - name: Find Existing Comment
+        uses: peter-evans/find-comment@v3
+        id: find_comment
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: github-actions[bot]
+
+      - name: Post Benchmark Results as PR Comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            ## Benchmark Report for Commit [`${{ env.DPPL_COMMIT_SHA }}`](${{ env.DPPL_COMMIT_URL }})
+            ### Computer Information
+            ```
+            ${{ env.VERSION_INFO }}
+            ```
+            ### Benchmark Results
+            ```
+            ${{ env.BENCHMARK_OUTPUT }}
+            ```
+          comment-id: ${{ steps.find_comment.outputs.comment-id }}
+          edit-mode: replace

.github/workflows/CI.yml

@@ -70,6 +70,8 @@ jobs:
         env:
           GROUP: ${{ matrix.test_group }}
           JULIA_NUM_THREADS: ${{ matrix.runner.num_threads }}
+          # Only run Aqua tests on latest version
+          AQUA: ${{ matrix.runner.version == '1' && 'true' || 'false' }}
 
       - uses: julia-actions/julia-processcoverage@v1
 

.github/workflows/Format.yml

@@ -17,20 +17,7 @@ concurrency:
 jobs:
   format:
     runs-on: ubuntu-latest
+
     steps:
-      - uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@v2
-        with:
-          version: 1
       - name: Format code
-        run: |
-          using Pkg
-          Pkg.add(; name="JuliaFormatter", uuid="98e50ef6-434e-11e9-1051-2b60c6c9e899")
-          using JuliaFormatter
-          format("."; verbose=true)
-        shell: julia --color=yes {0}
-      - uses: reviewdog/action-suggester@v1
-        if: github.event_name == 'pull_request'
-        with:
-          tool_name: JuliaFormatter
-          fail_on_error: true
+        uses: TuringLang/actions/Format@main

HISTORY.md

@@ -1,5 +1,28 @@
 # DynamicPPL Changelog
 
+## 0.35.4
+
+Fixed a type instability in an implementation of `with_logabsdet_jacobian`, which resulted in the log-jacobian returned being an Int in some cases and a Float in others.
+This resolves an Enzyme.jl error on a number of models.
+More generally, this version also changes the type of various log probabilities to be more consistent with one another.
+Although we aren't fully there yet, our eventual aim is that log probabilities will generally default to Float64 on 64-bit systems, and Float32 on 32-bit systems.
+If you run into any issues with these types, please get in touch.
+
+## 0.35.3
+
+`model | (@varname(x) => 1.0, @varname(y) => 2.0)` now works.
+Previously, this would throw a `MethodError` if the tuple had more than one element.
+
+## 0.35.2
+
+`unflatten(::VarInfo, params)` now works with params that have non-float types (such as Int or Bool).
+
+## 0.35.1
+
+`subset(::AbstractVarInfo, ::AbstractVector{<:VarName})` now preserves the ordering of the varnames in the original varinfo argument.
+Previously, this would select the varnames according to their order in the second argument.
+This fixes an upstream Turing.jl issue with Gibbs sampling when a component sampler was assigned multiple variables.
+
 ## 0.35.0
 
 **Breaking changes**

Project.toml

@@ -16,7 +16,6 @@ DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
-KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LogDensityProblems = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c"
 MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
@@ -30,6 +29,7 @@ ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
+KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
 MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d"
 Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6"
 
@@ -56,6 +56,7 @@ Distributions = "0.25"
 DocStringExtensions = "0.9"
 EnzymeCore = "0.6 - 0.8"
 ForwardDiff = "0.10.12"
+InteractiveUtils = "1"
 JET = "0.9"
 KernelAbstractions = "0.9.33"
 LinearAlgebra = "1.6"

benchmarks/Project.toml

@@ -3,11 +3,23 @@ uuid = "d94a1522-c11e-44a7-981a-42bf5dc1a001"
 version = "0.1.0"
 
 [deps]
+ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
-DiffUtils = "8294860b-85a6-42f8-8c35-d911f667b5f6"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DynamicPPL = "366bfd00-2699-11ea-058f-f148b4cae6d8"
-LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
-Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
-Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-Weave = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+LogDensityProblems = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c"
+Mooncake = "da2b9cff-9c12-43a0-ae48-6db2b0edb7d6"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267"
+StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
+
+[compat]
+ADTypes = "1.14.0"
+BenchmarkTools = "1.6.0"
+Distributions = "0.25.117"
+ForwardDiff = "0.10.38"
+LogDensityProblems = "2.1.2"
+PrettyTables = "2.4.0"
+ReverseDiff = "1.15.3"

benchmarks/README.md

@@ -1,27 +1,5 @@
-To run the benchmarks, simply do:
+To run the benchmarks, run this from the root directory of the repository:
 
 ```sh
-julia --project -e 'using DynamicPPLBenchmarks; weave_benchmarks();'
-```
-
-```julia
-julia> @doc weave_benchmarks
-  weave_benchmarks(input="benchmarks.jmd"; kwargs...)
-
-  Weave benchmarks present in benchmarks.jmd into a single file.
-
-  Keyword arguments
-  ≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡≡
-
-    •  benchmarkbody: JMD-file to be rendered for each model.
-
-    •  include_commit_id=false: specify whether to include commit-id in the default name.
-
-    •  name: the name of directory in results/ to use as output directory.
-
-    •  name_old=nothing: if specified, comparisons of current run vs. the run pinted to by name_old will be included in the generated document.
-
-    •  include_typed_code=false: if true, output of code_typed for the evaluator of the model will be included in the weaved document.
-
-    •  Rest of the passed kwargs will be passed on to Weave.weave.
+julia --project=benchmarks benchmarks/benchmarks.jl
 ```

benchmarks/benchmark_body.jmd

@@ -0,0 +1,103 @@
+using Pkg
+# To ensure we benchmark the local version of DynamicPPL, dev the folder above.
+Pkg.develop(; path=joinpath(@__DIR__, ".."))
+
+using DynamicPPLBenchmarks: Models, make_suite, model_dimension
+using BenchmarkTools: @benchmark, median, run
+using PrettyTables: PrettyTables, ft_printf
+using StableRNGs: StableRNG
+
+rng = StableRNG(23)
+
+# Create DynamicPPL.Model instances to run benchmarks on.
+smorgasbord_instance = Models.smorgasbord(randn(rng, 100), randn(rng, 100))
+loop_univariate1k, multivariate1k = begin
+    data_1k = randn(rng, 1_000)
+    loop = Models.loop_univariate(length(data_1k)) | (; o=data_1k)
+    multi = Models.multivariate(length(data_1k)) | (; o=data_1k)
+    loop, multi
+end
+loop_univariate10k, multivariate10k = begin
+    data_10k = randn(rng, 10_000)
+    loop = Models.loop_univariate(length(data_10k)) | (; o=data_10k)
+    multi = Models.multivariate(length(data_10k)) | (; o=data_10k)
+    loop, multi
+end
+lda_instance = begin
+    w = [1, 2, 3, 2, 1, 1]
+    d = [1, 1, 1, 2, 2, 2]
+    Models.lda(2, d, w)
+end
+
+# Specify the combinations to test:
+# (Model Name, model instance, VarInfo choice, AD backend, linked)
+chosen_combinations = [
+    (
+        "Simple assume observe",
+        Models.simple_assume_observe(randn(rng)),
+        :typed,
+        :forwarddiff,
+        false,
+    ),
+    ("Smorgasbord", smorgasbord_instance, :typed, :forwarddiff, false),
+    ("Smorgasbord", smorgasbord_instance, :simple_namedtuple, :forwarddiff, true),
+    ("Smorgasbord", smorgasbord_instance, :untyped, :forwarddiff, true),
+    ("Smorgasbord", smorgasbord_instance, :simple_dict, :forwarddiff, true),
+    ("Smorgasbord", smorgasbord_instance, :typed, :reversediff, true),
+    ("Smorgasbord", smorgasbord_instance, :typed, :mooncake, true),
+    ("Loop univariate 1k", loop_univariate1k, :typed, :mooncake, true),
+    ("Multivariate 1k", multivariate1k, :typed, :mooncake, true),
+    ("Loop univariate 10k", loop_univariate10k, :typed, :mooncake, true),
+    ("Multivariate 10k", multivariate10k, :typed, :mooncake, true),
+    ("Dynamic", Models.dynamic(), :typed, :mooncake, true),
+    ("Submodel", Models.parent(randn(rng)), :typed, :mooncake, true),
+    ("LDA", lda_instance, :typed, :reversediff, true),
+]
+
+# Time running a model-like function that does not use DynamicPPL, as a reference point.
+# Eval timings will be relative to this.
+reference_time = begin
+    obs = randn(rng)
+    median(@benchmark Models.simple_assume_observe_non_model(obs)).time
+end
+
+results_table = Tuple{String,Int,String,String,Bool,Float64,Float64}[]
+
+for (model_name, model, varinfo_choice, adbackend, islinked) in chosen_combinations
+    @info "Running benchmark for $model_name"
+    suite = make_suite(model, varinfo_choice, adbackend, islinked)
+    results = run(suite)
+    eval_time = median(results["evaluation"]).time
+    relative_eval_time = eval_time / reference_time
+    ad_eval_time = median(results["gradient"]).time
+    relative_ad_eval_time = ad_eval_time / eval_time
+    push!(
+        results_table,
+        (
+            model_name,
+            model_dimension(model, islinked),
+            string(adbackend),
+            string(varinfo_choice),
+            islinked,
+            relative_eval_time,
+            relative_ad_eval_time,
+        ),
+    )
+end
+
+table_matrix = hcat(Iterators.map(collect, zip(results_table...))...)
+header = [
+    "Model",
+    "Dimension",
+    "AD Backend",
+    "VarInfo Type",
+    "Linked",
+    "Eval Time / Ref Time",
+    "AD Time / Eval Time",
+]
+PrettyTables.pretty_table(
+    table_matrix;
+    header=header,
+    tf=PrettyTables.tf_markdown,
+    formatters=ft_printf("%.1f", [6, 7]),
+)