Merge pull request #1 from int-brain-lab/develop

Improved the packaging and logging.

Author: rai-pranav

.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: Build and Test
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+    - uses: actions/checkout@v3
+    
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    
+    - name: Install package in development mode
+      run: |
+        python -m pip install --upgrade pip
+        pip install -e .
+
+    - name: Test imports
+      run: |
+        python -c "import ephysatlas"

.gitignore

@@ -130,6 +130,7 @@ ENV/
 env.bak/
 venv.bak/
 .vscode/
+load_venv.sh
 
 # Spyder project settings
 .spyderproject
@@ -170,4 +171,10 @@ cython_debug/
 #  version on PyPI doesn't break anything, then do not submit the pdm.lock file.
 pdm.lock
 
-Todos.md
+# Configuration files
+config.yaml
+
+# Scratchpad
+Todos.md
+Issues.md
+testing.ipynb

README.md

@@ -29,8 +29,14 @@ python main.py --config config.yaml
 
 ### Configuration File
 
-The `config.yaml` file contains all the necessary parameters for running the analysis. Here's an example configuration:
+The configuration is managed through a YAML file. To avoid committing local changes, the actual configuration file (`config.yaml`) is ignored by git. Instead, a template file (`config_template.yaml`) is provided. To use the tool:
 
+1. Copy the template file to create your local configuration:
+```bash
+cp config_template.yaml config.yaml
+```
+
+2. Edit `config.yaml` with your specific settings:
 ```yaml
 # Required parameters
 pid: "5246af08-0730-40f7-83de-29b5d62b9b6d"  # Probe ID

config_template.yaml

@@ -0,0 +1,34 @@
+# config file
+pid: "YOUR_PROBE_ID"  # Probe ID
+t_start: 300.0  # Start time in seconds
+duration: 1  # Duration in seconds
+
+# Logging configuration
+log_path: "/path/to/logs/ibleatools.log"  # Absolute path for log file. If not provided, no file logging will be done.
+
+# Big .cbin files
+# ap_file: "/path/to/your/ap/file.cbin"
+# lf_file: "/path/to/your/lf/file.cbin"
+
+
+# Trajectory information for xyz target computation
+# traj_dict:
+#   x: -2243.1  # x coordinate
+#   y: -1999.8  # y coordinate
+#   z: -361.0  # z coordinate
+#   depth: 4000.0  # insertion depth
+#   theta: 15.0  # insertion angle theta
+#   phi: 180.0  # insertion angle phi
+
+# Operation mode: 'features' for feature computation only,
+# 'inference' for region inference only, or 'both' for both operations
+mode: "both"
+
+# Path to save/load features file. If not provided,
+# features will be saved as 'features_{pid}.parquet' in the current directory
+features_path: "/path/to/your/features.parquet"
+
+# Path to the model directory for region inference
+model_path: "/path/to/your/model/directory/"
+
+# Usage: python main.py --config config.yaml 

examples/compute_features_raw_NP1.py

@@ -0,0 +1,17 @@
+# %%
+import spikeglx
+import ephysatlas.feature_computation
+
+
+file_ap = "/mnt/s1/spikesorting/raw_data/mrsicflogellab/Subjects/SWC_038/2020-07-30/001/raw_ephys_data/probe01/_spikeglx_ephysData_g0_t0.imec1.ap.cbin"
+file_lf = "/mnt/s1/spikesorting/raw_data/mrsicflogellab/Subjects/SWC_038/2020-07-30/001/raw_ephys_data/probe01/_spikeglx_ephysData_g0_t0.imec1.lf.cbin"
+
+t0 = 485
+
+sr_ap = spikeglx.Reader(file_ap)
+sr_lf = spikeglx.Reader(file_lf)
+duration = 3
+
+df_features = ephysatlas.feature_computation.online_feature_computation(
+    sr_lf, sr_ap, t0, duration, channel_labels=None
+)

main.py

@@ -1,146 +1,160 @@
-import os
 import argparse
+import random
+import string
 from typing import List, Optional, Dict, Any
-from src.feature_computation import compute_features
-from src.region_inference import infer_regions
-from one.api import ONE
+from pathlib import Path
+
 import numpy as np
 import yaml
-from pathlib import Path
 import pandas as pd
-from src.logger_config import setup_logger
-from src.plots import plot_results
-from src import decoding
-import random
-import string
 
-# Set up logger
-logger = setup_logger(__name__)
+from iblutil.util import setup_logger
+from one.api import ONE
+
+from ephysatlas.feature_computation import compute_features
+from ephysatlas.region_inference import infer_regions
+from ephysatlas.plots import plot_results
+from ephysatlas import decoding
+
 
 def load_config(config_path: str) -> Dict[str, Any]:
     """Load configuration from YAML file."""
-    logger.info(f"Loading configuration from {config_path}")
-    with open(config_path, 'r') as f:
+    with open(config_path, "r") as f:
         return yaml.safe_load(f)
 
 
 def parse_arguments(args: List[str]) -> argparse.Namespace:
     """Parse command line arguments."""
-    logger.debug("Parsing command line arguments")
-    parser = argparse.ArgumentParser(description="Electrophysiology feature computation and region inference")
-    parser.add_argument("--config", required=True, help="Path to YAML configuration file")
+    parser = argparse.ArgumentParser(
+        description="Electrophysiology feature computation and region inference"
+    )
+    parser.add_argument(
+        "--config", required=True, help="Path to YAML configuration file"
+    )
     return parser.parse_args(args)
 
 
 def get_parameters(args: argparse.Namespace) -> Dict[str, Any]:
     """Get parameters from config file."""
-    logger.info("Loading configuration from YAML file")
     config = load_config(args.config)
-    
+
     # Validate required parameters
-    if 'pid' in config:
+    if "pid" in config:
         # PID-based configuration
-        required_params = ['pid', 't_start', 'duration']
+        required_params = ["pid", "t_start", "duration"]
         missing_params = [param for param in required_params if param not in config]
         if missing_params:
-            raise ValueError(f"Missing required parameters in config file: {', '.join(missing_params)}")
+            raise ValueError(
+                f"Missing required parameters in config file: {', '.join(missing_params)}"
+            )
     else:
         # File-based configuration
-        required_params = ['ap_file', 'lf_file']
+        required_params = ["ap_file", "lf_file"]
         missing_params = [param for param in required_params if param not in config]
         if missing_params:
-            raise ValueError(f"Missing required parameters in config file: {', '.join(missing_params)}")
-    
+            raise ValueError(
+                f"Missing required parameters in config file: {', '.join(missing_params)}"
+            )
+
     return {
-        'pid': config.get('pid'),
-        'ap_file': config.get('ap_file'),
-        'lf_file': config.get('lf_file'),
-        't_start': config.get('t_start', 0.0),  # Default to 0.0 if not specified
-        'duration': config.get('duration'),  # Default to None if not specified
-        'mode': config.get('mode', 'both'),
-        'features_path': config.get('features_path'),
-        'model_path': config.get('model_path'),
-        'traj_dict': config.get('traj_dict')
+        "pid": config.get("pid"),
+        "ap_file": config.get("ap_file"),
+        "lf_file": config.get("lf_file"),
+        "t_start": config.get("t_start", 0.0),  # Default to 0.0 if not specified
+        "duration": config.get("duration"),  # Default to None if not specified
+        "mode": config.get("mode", "both"),
+        "features_path": config.get("features_path"),
+        "model_path": config.get("model_path"),
+        "traj_dict": config.get("traj_dict"),
+        "log_path": config.get("log_path"),  # Get log path from config
     }
 
 
 def main(args: Optional[List[str]] = None) -> int:
     """Main function that can be called with arguments or use command line arguments."""
-    logger.info("Starting main function")
     if args is None:
         import sys
+
         args = sys.argv[1:]
-    
+
     # Parse arguments
     parsed_args = parse_arguments(args)
-    
+
     # Get parameters from config file
     params = get_parameters(parsed_args)
-    logger.info(f"Processing probe ID: {params['pid']}")
-    
+
+    # Set up logger with config path
+    logger = setup_logger(__name__, file=params.get("log_path"))
+    logger.info("Starting main function")
+
     # Initialize ONE if using PID
     one = ONE()
-    if params['pid'] is not None:
+    if params["pid"] is not None:
         logger.info("ONE client initialized")
         logger.info(f"Processing probe ID: {params['pid']}")
     else:
         logger.info(f"Processing files: AP={params['ap_file']}, LF={params['lf_file']}")
-    
+
     df_features = None
     # Determine features file path
-    features_path = params.get('features_path')
+    features_path = params.get("features_path")
     if features_path is None:
-        if params['pid'] is not None:
+        if params["pid"] is not None:
             features_path = Path(f"features_{params['pid']}.parquet")
         else:
             # Generate 8 character alphanumeric filename
-            filename = ''.join(random.choices(string.ascii_letters + string.digits, k=8))
+            filename = "".join(
+                random.choices(string.ascii_letters + string.digits, k=8)
+            )
             features_path = Path(f"features_{filename}.parquet")
             logger.info(f"Generated features filename: {features_path}")
     else:
         features_path = Path(features_path)
         # Ensure the file has .parquet extension
-        if features_path.suffix != '.parquet':
-            features_path = features_path.with_suffix('.parquet')
-    
+        if features_path.suffix != ".parquet":
+            features_path = features_path.with_suffix(".parquet")
+
     # Compute features if mode is 'features' or 'both'
-    if params['mode'] in ['features', 'both']:
+    if params["mode"] in ["features", "both"]:
         logger.info("Starting feature computation")
         df_features = compute_features(
-            pid=params.get('pid'),
-            t_start=params['t_start'],
-            duration=params['duration'],
+            pid=params.get("pid"),
+            t_start=params["t_start"],
+            duration=params["duration"],
             one=one,
-            ap_file=params.get('ap_file'),
-            lf_file=params.get('lf_file'),
-            traj_dict=params.get('traj_dict')
+            ap_file=params.get("ap_file"),
+            lf_file=params.get("lf_file"),
+            traj_dict=params.get("traj_dict"),
         )
         logger.info(f"Feature computation completed. Shape: {df_features.shape}")
-        
+
         # Save features to parquet file
         logger.info(f"Saving features to {features_path}")
         df_features.to_parquet(features_path, index=True)
-    
+
     # Infer regions if mode is 'inference' or 'both'
-    if params['mode'] in ['inference', 'both']:
+    if params["mode"] in ["inference", "both"]:
         logger.info("Starting region inference")
         # Get model path from parameters or use default
-        model_path = params.get('model_path')
+        model_path = params.get("model_path")
         if model_path is None:
-            model_path = Path("/Users/pranavrai/Downloads/models/2024_W50_Cosmos_voter-snap-pudding/")
+            model_path = Path(
+                "/Users/pranavrai/Downloads/models/2024_W50_Cosmos_voter-snap-pudding/"
+            )
         else:
             model_path = Path(model_path)
-        
+
         # If df_features is None, load from file
         if df_features is None:
             # This should only happen in inference mode
-            assert params['mode'] == 'inference'
+            assert params["mode"] == "inference"
             if not features_path.exists():
-                raise ValueError(f"Features file not found at {features_path}. Please compute features first.")
+                raise ValueError(
+                    f"Features file not found at {features_path}. Please compute features first."
+                )
             logger.info(f"Loading features from {features_path}")
             df_features = pd.read_parquet(features_path)
-        
-        
+
         predicted_probas, predicted_regions = infer_regions(df_features, model_path)
         logger.info(f"Predicted regions shape: {predicted_regions.shape}")
         logger.info(f"Prediction probabilities shape: {predicted_probas.shape}")
@@ -149,24 +163,28 @@ def main(args: Optional[List[str]] = None) -> int:
         output_dir = features_path.parent
         np_probas_path = output_dir / f"probas_{params['pid']}.npy"
         np_regions_path = output_dir / f"regions_{params['pid']}.npy"
-        
-        logger.info(f"Saving prediction probabilities as numpy array to {np_probas_path}")
+
+        logger.info(
+            f"Saving prediction probabilities as numpy array to {np_probas_path}"
+        )
         np.save(np_probas_path, predicted_probas)
-        
+
         logger.info(f"Saving predicted regions as numpy array to {np_regions_path}")
         np.save(np_regions_path, predicted_regions)
 
-        #Plot the results
-        #Todo need to have better interface than calling dict_model here just for plotting.
-        dict_model = decoding.load_model(model_path.joinpath(f'FOLD04'))
+        # Plot the results
+        # Todo need to have better interface than calling dict_model here just for plotting.
+        dict_model = decoding.load_model(model_path.joinpath("FOLD04"))
         fig, ax = plot_results(df_features, predicted_probas, dict_model)
         import matplotlib.pyplot as plt
+
         plt.savefig(output_dir / f"results_{params['pid']}.png")
-    
+
     return 0
 
 
 if __name__ == "__main__":
     exit_code = main()
     import sys
-    sys.exit(exit_code)
+
+    sys.exit(exit_code)