Merge pull request #110 from cokelaer/main

add a download utility function

Author: cokelaer

README.rst

@@ -65,7 +65,6 @@ pipelines, and pipeline management tools into a single library (Sequana) as illu
 in **Fig 1** below.
 
 .. figure:: https://raw.githubusercontent.com/sequana/sequana_pipetools/main/doc/veryold.png
-    :scale: 40%
 
     **Figure 1** Old Sequana framework will all pipelines and Sequana library in the same
     place including pipetools (this library).
@@ -74,7 +73,6 @@ Despite maintaining an 80% test coverage, whenever changes were introduced to th
 
 
 .. figure:: https://raw.githubusercontent.com/sequana/sequana_pipetools/main/doc/old.png
-    :scale: 40%
 
     **Figure 2** v0.8 of Sequana moved the Snakemake pipelines in independent
     repositories. A `cookie cutter <https://github.com/sequana/sequana_pipeline_template>`_
@@ -86,7 +84,6 @@ Nevertheless, certain tools, including those utilized for user interface and inp
 
 
 .. figure:: https://raw.githubusercontent.com/sequana/sequana_pipetools/main/doc/new.png
-    :scale: 40%
 
     **Figure 3** New Sequana framework. The new Sequana framework comprises the core library
     and bioinformatics tools, which are now separate from the pipelines. Moreover, the
@@ -96,7 +93,6 @@ Nevertheless, certain tools, including those utilized for user interface and inp
 As a final step, we separated the rules originally available in Sequana to create an independent package featuring a collection of Snakemake wrappers. These wrappers can be accessed at https://github.com/sequana/sequana-wrappers and offer the added benefit of being rigorously tested through continuous integration.
 
 .. figure:: https://raw.githubusercontent.com/sequana/sequana_pipetools/main/doc/wrappers.png
-    :scale: 40%
 
     **Figure 3** New Sequana framework 2021. The library itself with the core, the
     bioinformatics tools is now fully independent of the pipelines.
@@ -202,9 +198,9 @@ Python module (the last two lines is where the magic happens)::
         # create a function for a given option (here --method)
         def fill_method():
             # any extra sanity checks
-            cfg['method'] = options['method']
+            cfg["method"] = options["method"]
 
-        if options['from-project']:
+        if options["from-project"]:
             # in --from-project, we fill the method is --method is provided only (since already pre-filled)
             if "--method" in sys.argv
                 fill_method()
@@ -235,9 +231,9 @@ For FastQ files (paired ot not), The config file should look like::
 
     sequana_wrappers: "v0.15.1"
 
-    input_directory: '.'
+    input_directory: "."
     input_readtag: "_R[12]_"
-    input_pattern: '*fastq.gz'
+    input_pattern: "*fastq.gz"
 
 
     apptainers:
@@ -317,6 +313,7 @@ Changelog
 ========= ======================================================================
 Version   Description
 ========= ======================================================================
+1.0.4     * add utility function to download and untar a tar.gz file
 1.0.3     * add levenshtein function. some typo corrections.
 1.0.2     * add the dot2png command. pin docutils <0.21 due to pip error
 1.0.1     * hot fix in the profile creation (regression)
@@ -355,7 +352,7 @@ Version   Description
           * --from-project not funtcional (example in multitax pipeline)
           * Click checks that input-directoyr is a directory indeed
 0.16.1    * Fix/rename error_report into onerror to be included in the Snakemake
-            onerror section. added 'slurm' in slurm output log file in the
+            onerror section. added *slurm* in slurm output log file in the
             profile
 0.16.0    * scripts now use click instead of argparse
           * All Options classes have now an equivalent using click.
@@ -379,8 +376,8 @@ Version   Description
 0.14.X    * Module now returns the list of requirements. SequanaManager
             creates a txt file with all standalones from the requirements.
 0.13.0    * switch to pyproject and fixes #64
-0.12.X    * automatically populater 'wrappers' in PipelineManager' based on the
-            config entry 'sequana_wrappers'.
+0.12.X    * automatically populater *wrappers* in PipelineManager based on the
+            config entry *sequana_wrappers*.
           * Fix the singularity arguments by (i) adding -e and (ii) bind the
             /home. Indeed, snakemake sets --home to the current directory.
             Somehow the /home is lost. Removed deprecated function
@@ -413,7 +410,7 @@ Version   Description
           * Move all modules related to pipelines from sequana into
             sequana_pipetools
 0.5.X     * feature removed in sequana to deal with adapter removal and
-            changes updated in the package (removed the 'design' option
+            changes updated in the package (removed the *design* option
             from the cutadapt rules and needed); add TrimmingOptions.
 0.4.X     * add FeatureCounts options and slurm status utility
 0.4.0     * stable version

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 #maintainer ?#maintainer email
 [tool.poetry]
 name = "sequana_pipetools"
-version = "1.0.3"
+version = "1.0.4"
 description = "A set of tools to help building or using Sequana pipelines"
 authors = ["Sequana Team"]
 license = "BSD-3"

sequana_pipetools/__init__.py

@@ -20,7 +20,7 @@ def get_package_version(package_name):
 
 logger = colorlog.getLogger(logger.name)
 
-from .misc import url2hash, levenshtein_distance
+from .misc import download_and_extract_tar_gz, levenshtein_distance, url2hash
 from .sequana_manager import SequanaManager  # , get_pipeline_location
 from .snaketools import (
     Pipeline,

sequana_pipetools/misc.py

@@ -11,11 +11,70 @@
 #  Contributors:  https://github.com/sequana/sequana/graphs/contributors
 ##############################################################################
 import hashlib
+import os
 import sys
+import tarfile
+
+import colorlog
+import requests
+from tqdm import tqdm
 
 from sequana_pipetools import get_package_version
 
-__all__ = ["Colors", "print_version", "error", "url2hash", "levenshtein_distance"]
+logger = colorlog.getLogger(__name__)
+
+
+__all__ = ["Colors", "print_version", "error", "url2hash", "levenshtein_distance", "download_and_extract_tar_gz"]
+
+
+def download_and_extract_tar_gz(url, extract_to):
+    """
+    Downloads a .tar.gz file from a given URL and extracts it to the specified directory.
+
+    :param url: URL of the .tar.gz file
+    :param extract_to: Directory where the contents will be extracted
+    """
+    # Get the file name from the URL
+    filename = url.split("/")[-1]
+    file_path = os.path.join(extract_to, filename)
+
+    # create the directory
+    os.makedirs(extract_to, exist_ok=True)
+
+    # Download the file
+    logger.info(f"Downloading {filename}...")
+    response = requests.get(url, stream=True)
+    response.raise_for_status()  # Raise an exception for HTTP errors
+    total_size = int(response.headers.get("content-length", 0))
+
+    # Write the downloaded content to a file
+    # Download with a progress bar
+    with open(file_path, "wb") as file, tqdm(
+        desc=f"Downloading {filename}",
+        total=total_size,
+        unit="B",
+        unit_scale=True,
+        unit_divisor=1024,
+    ) as bar:
+        for chunk in response.iter_content(chunk_size=8192):
+            file.write(chunk)
+            bar.update(len(chunk))
+
+    logger.info(f"Downloaded {filename} to {file_path}")
+
+    # Extract the tar.gz file
+    if tarfile.is_tarfile(file_path):
+        logger.info(f"Extracting {filename}...")
+
+        with tarfile.open(file_path, "r:gz") as tar:
+            tar.extractall(path=extract_to)
+        logger.info(f"Extracted to {extract_to}")
+    else:
+        logger.info(f"{file_path} is not a valid tar.gz file.")
+
+    # Optionally, you can delete the .tar.gz file after extraction
+    os.remove(file_path)
+    logger.info("Process completed.")
 
 
 def levenshtein_distance(token1: str, token2: str) -> int:
@@ -59,14 +118,18 @@ def levenshtein_distance(token1: str, token2: str) -> int:
             if token1[t1 - 1] == token2[t2 - 1]:
                 distances[t1][t2] = distances[t1 - 1][t2 - 1]
             else:
-                distances[t1][t2] = min(
-                    distances[t1][t2 - 1],  # Insertion
-                    distances[t1 - 1][t2],  # Deletion
-                    distances[t1 - 1][t2 - 1]  # Substitution
-                ) + 1
+                distances[t1][t2] = (
+                    min(
+                        distances[t1][t2 - 1],  # Insertion
+                        distances[t1 - 1][t2],  # Deletion
+                        distances[t1 - 1][t2 - 1],  # Substitution
+                    )
+                    + 1
+                )
 
     return distances[len1][len2]
 
+
 def url2hash(url):
     md5hash = hashlib.md5()
     md5hash.update(url.encode())

sequana_pipetools/sequana_manager.py

@@ -473,8 +473,7 @@ def teardown(self, check_schema=True, check_input_files=True):
                         msg = (
                             "The version {} of your completion file for the {} pipeline seems older than the installed"
                             " pipeline itself ({}). Please, consider updating the completion file {}"
-                            " using the following command: \n\t sequana_pipetools --completion {}\n"
-                            "available in the sequana_pipetools package (pip install sequana_completion)"
+                            " using the following command: \n\t sequana_pipetools --completion {}\n\n"
                         )
                         msg = msg.format(version, self.name, self._get_package_version(), completion, self.name)
                         logger.info(msg)

tests/test_misc.py

@@ -1,6 +1,18 @@
-from sequana_pipetools.misc import Colors, print_version, error, url2hash, levenshtein_distance
+from sequana_pipetools.misc import (
+    Colors,
+    download_and_extract_tar_gz,
+    error,
+    levenshtein_distance,
+    print_version,
+    url2hash,
+)
 
 
+def test_download_and_extract_tar_gz(tmpdir):
+
+    url = "https://github.com/sequana/sequana_pipetools/archive/refs/tags/v1.0.3.tar.gz"
+    download_and_extract_tar_gz(url, tmpdir)
+
 
 def test_levenshtein():
     assert levenshtein_distance("kitten", "sitting") == 3
@@ -9,7 +21,7 @@ def test_levenshtein():
 
 def test_url2hash():
     md5 = url2hash("https://zenodo.org/record/7822910/files/samtools_1.17_minimap2_2.24.0.img")
-    assert md5 == 'c3e4a8244ce7b65fa873ebda134fea7f'
+    assert md5 == "c3e4a8244ce7b65fa873ebda134fea7f"
 
 
 def test_colors():