updates from internal repository

Author: JanSellner

.dockerignore

@@ -567,8 +567,8 @@ migrated
 **/*.lprof
 
 # Stuff only needed to run Docker
-base.Dockerfile
-Dockerfile
-docker-compose.yml
-docker-compose.override.yml
+dependencies/base.Dockerfile
+dependencies/Dockerfile
+dependencies/docker-compose.yml
+dependencies/docker-compose.override.yml
 run_docker.py

.github/workflows/dataset.yml

@@ -37,7 +37,7 @@ jobs:
         with:
           # Include requirements.txt so that the caching works
           sparse-checkout: |
-            requirements.txt
+            dependencies/requirements.txt
             tutorials/website_example.py
           sparse-checkout-cone-mode: false
 

.github/workflows/tests.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -r requirements.txt
+          pip install -r dependencies/requirements.txt
           pip install pytest pytest-xdist wheel
 
       - name: Install htc

.gitlab-ci.yml

@@ -12,30 +12,30 @@ stages:
 build-docker:
   stage: build
   script:
-    - docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml build $BUILD_OPT
-    - docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc conda list --export
+    - docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml build $BUILD_OPT
+    - docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc conda list --export
 
 code-checks:
   stage: test
   script:
     # We trigger the build process again before running the actual checks in case the code changed
     # This may happen if jobs from multiple branches run at the same time
     # If the code did not change, the build cache will be used
-    - docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml build
-    - docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc bash run_hooks.sh
+    - docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml build
+    - docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc bash run_hooks.sh
 
 tests:
   stage: test
   script:
-    - docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml build
-    - docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc htc tests --parallel 4 $TEST_OPT
+    - docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml build
+    - docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc htc tests --parallel 4 $TEST_OPT
 
 test-package:
   stage: test
   script:
-    - docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml build
+    - docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml build
     - >
-      docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc bash -c '
+      docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc bash -c '
       CIBW_BUILD="cp312-*"
       CIBW_TEST_COMMAND=""
       cibuildwheel --platform linux --output-dir dist &&
@@ -50,9 +50,9 @@ package:
   stage: deploy
   script:
     # cibuildwheel is configured via pyproject.toml
-    - docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml build
+    - docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml build
     - >
-      docker compose -f docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc bash -c "
+      docker compose -f dependencies/docker-compose.yml -f gitlab_runner/docker-compose.yml run --rm htc bash -c "
       pip install --upgrade cibuildwheel twine &&
       CIBW_ENVIRONMENT=CI_COMMIT_TAG=${CI_COMMIT_TAG}
       python -m cibuildwheel --platform linux --output-dir dist &&

.pre-commit-config.yaml

@@ -1,10 +1,15 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.3
     hooks:
       - id: ruff
         args: [--fix]
       - id: ruff-format
+  - repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v19.1.7
+    hooks:
+      - id: clang-format
+        types_or: [c++, c, cuda]
   # Avoid potential problems with py.test if __init__.py files are missing
   - repo: https://github.com/lk16/detect-missing-init
     rev: v0.1.6

CITATION.cff

@@ -10,5 +10,5 @@ authors:
 doi: 10.5281/zenodo.6577614
 repository-code: "https://github.com/IMSY-DKFZ/htc"
 license: MIT
-version: v0.0.18
-date-released: "2025-01-06"
+version: v0.0.19
+date-released: "2025-02-01"

README.md

@@ -39,10 +39,10 @@ If you use the `htc` framework, please consider citing the [corresponding papers
   author    = {Sellner, Jan and Seidlitz, Silvia},
   publisher = {Zenodo},
   url       = {https://github.com/IMSY-DKFZ/htc},
-  date      = {2025-01-06},
+  date      = {2025-02-01},
   doi       = {10.5281/zenodo.6577614},
   title     = {Hyperspectral Tissue Classification},
-  version   = {v0.0.18},
+  version   = {v0.0.19},
 }
 ```
 
@@ -58,7 +58,7 @@ This package can be installed via pip:
 pip install imsy-htc
 ```
 
-This installs all the required dependencies defined in [`requirements.txt`](./requirements.txt). The requirements include [PyTorch](https://pytorch.org/), so you may want to install it manually before installing the package in case you have specific needs (e.g. CUDA version).
+This installs all the required dependencies defined in [`requirements.txt`](./dependencies/requirements.txt). The requirements include [PyTorch](https://pytorch.org/), so you may want to install it manually before installing the package in case you have specific needs (e.g. CUDA version).
 
 > ⚠️ This framework was developed and tested using the Ubuntu 20.04+ Linux distribution. Despite we do provide wheels for Windows and macOS as well, they are not tested.
 
@@ -82,6 +82,7 @@ We cannot provide wheels for all PyTorch versions. Hence, a version of `imsy-htc
 | 0.0.16     | 2.4     |
 | 0.0.17     | 2.5     |
 | 0.0.18     | 2.5     |
+| 0.0.19     | 2.6     |
 
 However, we do not make explicit version constraints in the dependencies of the `imsy-htc` package because a future version of PyTorch may still work and we don't want to break the installation if it is not necessary.
 
@@ -105,7 +106,7 @@ or by adding the following lines to your `requirements.txt`
 imsy-htc[extra]
 ```
 
-This installs the optional dependencies defined in [`requirements-extra.txt`](./requirements-extra.txt), including for example our Python wrapper for the [challengeR toolkit](https://github.com/wiesenfa/challengeR).
+This installs the optional dependencies defined in [`requirements-extra.txt`](./dependencies/requirements-extra.txt), including for example our Python wrapper for the [challengeR toolkit](https://github.com/wiesenfa/challengeR).
 
 </details>
 
@@ -129,7 +130,7 @@ export PATH_Tivita_HeiPorSPECTRAL="/path/to/the/dataset"
 python run_docker.py bash
 ```
 
-You can now run any commands you like. All datasets you provided via an environment variable that starts with `PATH_Tivita` will be accessible in your container (you can also check the generated `docker-compose.override.yml` file for details). Please note that the Docker container is meant for small testing only and not for development. This is also reflected by the fact that per default all results are stored inside the container and hence will also be deleted after exiting the container. If you want to keep your results, let the environment variable `PATH_HTC_DOCKER_RESULTS` point to the directory where you want to store the results.
+You can now run any commands you like. All datasets you provided via an environment variable that starts with `PATH_Tivita` will be accessible in your container (you can also check the generated `dependencies/docker-compose.override.yml` file for details). Please note that the Docker container is meant for small testing only and not for development. This is also reflected by the fact that per default all results are stored inside the container and hence will also be deleted after exiting the container. If you want to keep your results, let the environment variable `PATH_HTC_DOCKER_RESULTS` point to the directory where you want to store the results.
 
 </details>
 
@@ -144,7 +145,7 @@ conda create --yes --name htc python=3.12
 conda activate htc
 
 # Install the htc package and its requirements
-pip install -r requirements-dev.txt
+pip install -r dependencies/requirements-dev.txt
 pip install --no-use-pep517 -e .
 ```
 

Dockerfile renamed to dependencies/Dockerfile

@@ -21,6 +21,9 @@ RUN apt install -y libmagickwand-dev \
  && pip install Wand \
  && sed -i '/disable ghostscript format types/,+6d' /etc/ImageMagick-6/policy.xml
 
+# Install chromium for kaleido, as it is explicitly required now
+RUN apt update && apt install -y chromium-browser
+
 # Install and cache the static code checks
 # We need a git repo for pre-commit to work but we don't want to copy the main repo to the Docker container as it unnecessarily slows down the build process
 # Instead, we create a new empty git repo and make sure that files are staged before the hooks run (because pre-commit does not consider unstaged files)
@@ -33,4 +36,4 @@ RUN git init \
 COPY . /home/src
 RUN pip install --no-use-pep517 -e /home/src
 
-ENTRYPOINT ["python", "docker_startup.py"]
+ENTRYPOINT ["python", "dependencies/docker_startup.py"]

dependencies/README.md

@@ -0,0 +1,7 @@
+# Dependencies
+
+This folder contains all dependency files to get the htc project running.
+
+The `requirements*.txt` list all the Python dependencies and are installed automatically if you followed the installation instructions. Despite the general dependencies (`requirements.txt`, 'requirements-extra.txt' and 'requirements-dev.txt') there may also be additional project-dependent dependencies which are not installed automatically but which may be required to run the project (and hence need to be installed manually.).
+
+The Docker files define how to run the htc project in an isolated Docker container. From a user perspective, it is usually enough to run `htc docker bash` to get an isolated container (with mounted read-only results folder and datasets). The Docker files are mainly used for testing.

base.Dockerfile renamed to dependencies/base.Dockerfile

@@ -21,15 +21,17 @@ ENV PATH="/opt/conda/bin:${PATH}"
 
 RUN curl -L -O https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh \
  && bash Miniforge3-Linux-x86_64.sh -b -p "/opt/conda" \
+ && conda update -y --all \
+ && conda install -y python=3.12 \
  && rm -f Miniforge3-Linux-x86_64.sh
 
 # Cache common pretrained models
 RUN curl -L https://github.com/lukemelas/EfficientNet-PyTorch/releases/download/1.0/efficientnet-b5-b6417697.pth --create-dirs -o /root/.cache/torch/hub/checkpoints/efficientnet-b5-b6417697.pth
 
 # Install all requirements separately so that this step can be cached
-COPY requirements.txt /requirements.txt
-COPY requirements-extra.txt /requirements-extra.txt
-COPY requirements-dev.txt /requirements-dev.txt
+COPY dependencies/requirements.txt /requirements.txt
+COPY dependencies/requirements-extra.txt /requirements-extra.txt
+COPY dependencies/requirements-dev.txt /requirements-dev.txt
 RUN python -m pip install -U pip \
  && pip install -r /requirements-dev.txt
 

docker-compose.yml renamed to dependencies/docker-compose.yml

@@ -1,11 +1,13 @@
 services:
   htc-base:
     build:
-      context: .
-      dockerfile: base.Dockerfile
+      context: ..
+      dockerfile: dependencies/base.Dockerfile
     image: htc-base
   htc:
-    build: .
+    build:
+      context: ..
+      dockerfile: dependencies/Dockerfile
     depends_on:
       - htc-base
     image: htc

docker_startup.py renamed to dependencies/docker_startup.py

@@ -1,5 +1,4 @@
 -r requirements.txt
-matplotlib-venn
 pynrrd
 pypdf
 ipywidgets

requirements-dev.txt renamed to dependencies/requirements-dev.txt

@@ -0,0 +1,2 @@
+--index-url https://read_package:[email protected]/api/v4/projects/90/packages/pypi/simple
+susi

requirements-extra.txt renamed to dependencies/requirements-extra.txt

@@ -123,6 +123,7 @@
     "utils.LabelMapping": ["LabelMapping"],
     "utils.LDA": ["LDA"],
     "utils.MeasureTime": ["MeasureTime"],
+    "utils.mitk.mitk_masks": ["nrrd_mask", "segmentation_to_nrrd"],
     "utils.MultiPath": ["MultiPath"],
     "utils.parallel": ["p_map"],
     "utils.SpectrometerReader": ["SpectrometerReader"],
@@ -253,6 +254,7 @@
     from htc.utils.LabelMapping import LabelMapping
     from htc.utils.LDA import LDA
     from htc.utils.MeasureTime import MeasureTime
+    from htc.utils.mitk.mitk_masks import nrrd_mask, segmentation_to_nrrd
     from htc.utils.MultiPath import MultiPath
     from htc.utils.parallel import p_map
     from htc.utils.SpectrometerReader import SpectrometerReader

dependencies/requirements-organ-clamping.txt

@@ -76,7 +76,7 @@ def update_container_files(self) -> str:
 
         # Build docker and apptainer container
         self.run_command(
-            command="docker build --tag htc-base --file base.Dockerfile .",
+            command="docker build --tag htc-base --file dependencies/base.Dockerfile .",
             task="building the htc-base container",
         )
         self.run_command(