Merge pull request #356 from rustprooflabs/cleanup

Cleanup docblock, type hinting, remove mdbook-variables

Author: rustprooflabs

.github/workflows/deploy-book.yml

@@ -18,8 +18,7 @@ jobs:
         echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV
     - name: Install mdbook
       run: |
-        cargo install mdbook --version 0.4.32
-        cargo install mdbook-variables --version 0.2.1
+        cargo install mdbook
     - name: Deploy GitHub Pages
       run: |
         # This assumes your book is in the root of your repository.

docker/db.py

@@ -1,4 +1,4 @@
-"""Interacts with Postgres
+"""Module to interact with Postgres database.
 """
 import logging
 import os
@@ -13,7 +13,7 @@
 LOGGER = logging.getLogger('pgosm-flex')
 
 
-def connection_string(admin=False):
+def connection_string(admin: bool=False) -> str:
     """Returns connection string to `db_name`.
 
     Env vars for user/password defined by Postgres docker image.
@@ -63,7 +63,8 @@ def connection_string(admin=False):
 
 
 def pg_conn_parts() -> dict:
-    """Retrieves username/password from environment variables if they exist.
+    """Returns dictionary of connection parts based on environment variables
+    if they exist.
 
     Returns
     --------------------------
@@ -129,7 +130,8 @@ def wait_for_postgres():
     """Ensures Postgres service is reliably ready for use.
 
     Required b/c Postgres process in Docker gets restarted shortly
-    after starting.
+    after starting.  Calls `sys.exit()` after `max_loops` reached
+    indicating failure due to inability to connect.
     """
     logger = logging.getLogger('pgosm-flex')
     logger.info('Checking for Postgres service to be available')
@@ -161,7 +163,7 @@ def wait_for_postgres():
     logger.info('Postgres instance ready')
 
 
-def pg_isready():
+def pg_isready() -> bool:
     """Checks for Postgres to be available.
 
     Uses pg_version_check() for simple approach.
@@ -184,7 +186,7 @@ def pg_isready():
 
 
 def log_pg_details():
-    """Logs non-sensitive Postgres connection details.
+    """Logs non-sensitive Postgres connection details to LOGGER.
     """
     conn_parts = pg_conn_parts()
     pg_host = conn_parts['pg_host']

docker/geofabrik.py

@@ -1,4 +1,4 @@
-"""This module handles the auto-file using Geofabrik's download service.
+"""This module handles the auto-file handling using Geofabrik's download service.
 """
 import logging
 import os
@@ -8,8 +8,9 @@
 import helpers
 
 
-def get_region_filename():
-    """Returns the filename needed to download/manage PBF files.
+def get_region_filename() -> str:
+    """Returns the filename needed to download/manage PBF files based on the
+    region/subregion.
 
     Returns
     ----------------------
@@ -27,7 +28,7 @@ def get_region_filename():
     return filename
 
 
-def prepare_data(out_path):
+def prepare_data(out_path: str) -> str:
     """Ensures the PBF file is available.
 
     Checks if it already exists locally, download if needed,
@@ -70,13 +71,15 @@ def prepare_data(out_path):
     return pbf_file
 
 
-def pbf_download_needed(pbf_file_with_date, md5_file_with_date, pgosm_date):
+def pbf_download_needed(pbf_file_with_date: str, md5_file_with_date: str,
+                        pgosm_date: str) -> bool:
     """Decides if the PBF/MD5 files need to be downloaded.
 
     Parameters
     -------------------------------
     pbf_file_with_date : str
     md5_file_with_date : str
+    pgosm_date : str
 
     Returns
     --------------------------
@@ -110,7 +113,7 @@ def pbf_download_needed(pbf_file_with_date, md5_file_with_date, pgosm_date):
     return download_needed
 
 
-def get_pbf_url(region, subregion):
+def get_pbf_url(region: str, subregion: str) -> str:
     """Returns the URL to the PBF for the region / subregion.
 
     Parameters
@@ -132,7 +135,7 @@ def get_pbf_url(region, subregion):
     return pbf_url
 
 
-def download_data(region, subregion, pbf_file, md5_file):
+def download_data(region: str, subregion: str, pbf_file: str, md5_file: str):
     """Downloads PBF and MD5 file using wget.
 
     Parameters
@@ -166,7 +169,8 @@ def download_data(region, subregion, pbf_file, md5_file):
     )
 
 
-def archive_data(pbf_file, md5_file, pbf_file_with_date, md5_file_with_date):
+def archive_data(pbf_file: str, md5_file: str, pbf_file_with_date: str,
+                 md5_file_with_date: str):
     """Copies `pbf_file` and `md5_file` to `pbf_file_with_date` and
     `md5_file_with_date`.
 
@@ -190,8 +194,8 @@ def archive_data(pbf_file, md5_file, pbf_file_with_date, md5_file_with_date):
         shutil.copy2(md5_file, md5_file_with_date)
 
 
-
-def unarchive_data(pbf_file, md5_file, pbf_file_with_date, md5_file_with_date):
+def unarchive_data(pbf_file: str, md5_file: str, pbf_file_with_date: str,
+                   md5_file_with_date: str):
     """Copies `pbf_file_with_date` and `md5_file_with_date`
     to `pbf_file` and `md5_file`.
 
@@ -218,7 +222,7 @@ def unarchive_data(pbf_file, md5_file, pbf_file_with_date, md5_file_with_date):
     shutil.copy2(md5_file_with_date, md5_file)
 
 
-def remove_latest_files(out_path):
+def remove_latest_files(out_path: str):
     """Removes the PBF and MD5 file with -latest in the name.
 
     Files are archived via prepare_data() before processing starts

docker/helpers.py

@@ -1,4 +1,4 @@
-"""Generic functions used in multiple modules.
+"""Generic functions and attributes used in multiple modules of PgOSM Flex.
 """
 import datetime
 import logging
@@ -14,19 +14,20 @@
 DEFAULT_SRID = '3857'
 
 
-def get_today():
+def get_today() -> str:
     """Returns yyyy-mm-dd formatted string for today.
 
-    Retunrs
+    Returns
     -------------------------
     today : str
     """
     today = datetime.datetime.today().strftime('%Y-%m-%d')
     return today
 
 
-def run_command_via_subprocess(cmd, cwd, output_lines=[], print=False):
-    """Wraps around subprocess.Popen to run commands outside of Python. Prints
+def run_command_via_subprocess(cmd: list, cwd: str, output_lines: list=[],
+                               print: bool=False) -> int:
+    """Wraps around subprocess.Popen() to run commands outside of Python. Prints
     output as it goes, returns the status code from the command.
 
     Parameters
@@ -66,12 +67,15 @@ def run_command_via_subprocess(cmd, cwd, output_lines=[], print=False):
     return status
 
 
-def verify_checksum(md5_file, path):
-    """If verfication fails calls `sys.exit()`
+def verify_checksum(md5_file: str, path: str):
+    """Verifies checksum of osm pbf file.
+
+    If verification fails calls `sys.exit()`
 
     Parameters
     ---------------------
     md5_file : str
+        Filename of the MD5 file to verify the osm.pbf file.
     path : str
         Path to directory with `md5_file` to validate
     """
@@ -146,7 +150,7 @@ def set_env_vars(region, subregion, srid, language, pgosm_date, layerset,
 
 
 
-def get_region_combined(region, subregion):
+def get_region_combined(region: str, subregion: str) -> str:
     """Returns combined region with optional subregion.
 
     Parameters
@@ -167,7 +171,7 @@ def get_region_combined(region, subregion):
     return pgosm_region
 
 
-def get_git_info():
+def get_git_info() -> str:
     """Provides git info in the form of the latest tag and most recent short sha
 
     Sends info to logger and returns string.

docker/import_mode.py

@@ -66,6 +66,10 @@ def okay_to_run(self, prior_import: dict) -> bool:
 
             An empty dictionary (len==0) indicates no prior import.
             Only the replication key is specifically used
+
+        Returns
+        -------------------
+        okay_to_run : bool
         """
         self.logger.debug(f'Checking if it is okay to run...')
         if self.force:
@@ -132,8 +136,13 @@ def set_run_post_sql(self):
             if self.update == 'append':
                 self.run_post_sql = False
 
-    def as_json(self):
-        """Returns key details as a dictionary.
+    def as_json(self) -> str:
+        """Packs key details as a dictionary passed through `json.dumps()`
+
+        Returns
+        ------------------------
+        json_text : str
+            Text representation of JSON object built using class attributes.
         """
         self_as_dict = {'update': self.update,
                 'replication': self.replication,

docker/osm2pgsql_recommendation.py

@@ -1,17 +1,19 @@
-"""Used by PgOSM-Flex Docker image to get osm2pgsql command to run from
-the osm2pgsql-tuner API.
+"""Determine the appropriate osm2pgsql command to run from the osm2pgsql-tuner
+project.
 """
 import logging
 import os
 import osm2pgsql_tuner as tuner
 
-import db
+import db, import_mode
 
 LOGGER = logging.getLogger('pgosm-flex')
 
 
-def osm2pgsql_recommendation(ram, pbf_filename, out_path, import_mode):
-    """Returns recommended osm2pgsql command.
+def osm2pgsql_recommendation(ram: float, pbf_filename: str, out_path: str,
+                             import_mode: import_mode.ImportMode) -> str:
+    """Returns recommended osm2pgsql command from the osm2pgsql-tuner
+    Python module: https://pypi.org/project/osm2pgsql-tuner/
 
     Recommendation from Python project.
     Public API available at https://osm2pgsql-tuner.com
@@ -45,8 +47,11 @@ def osm2pgsql_recommendation(ram, pbf_filename, out_path, import_mode):
                                            out_path)
     return osm2pgsql_cmd
 
-def get_recommended_script(system_ram_gb, osm_pbf_gb, import_mode, pbf_filename,
-                           output_path):
+def get_recommended_script(system_ram_gb: float,
+                           osm_pbf_gb: float,
+                           import_mode:import_mode.ImportMode,
+                           pbf_filename: str,
+                           output_path: str) -> str:
     """Generates recommended osm2pgsql command from osm2pgsql-tuner.
 
     Parameters

docker/pgosm_flex.py

@@ -1,11 +1,10 @@
 #!/usr/bin/env python3
-"""Python script to run PgOSM Flex within Docker container.
+"""Runs PgOSM Flex within Docker container.
 
 Docker image available on Docker Hub
     https://hub.docker.com/r/rustprooflabs/pgosm-flex
 
-Usage instructions:
-    https://github.com/rustprooflabs/pgosm-flex/blob/main/docs/DOCKER-RUN.md
+Documentation available at https://pgosm-flex.com/
 """
 import configparser
 import logging
@@ -39,7 +38,7 @@
 @click.option('--input-file',
               required=False,
               default=None,
-              help='Set filename or absolute filepath to input osm.pbf file. Overrides default file handling, archiving, and MD5 checksum validation. Filename is assumed under /app/output unless absolute path is used.')
+              help='Set filename or absolute filepath to input osm.pbf file. Overrides default file handling, archiving, and MD5 checksum validation. Filename is assumed to reference a file under /app/output unless absolute path is used.')
 @click.option('--layerset', required=True,
               default='default',
               help='Layerset to load. Defines name of included layerset unless --layerset-path is defined.')

docs/src/docker-build.md

@@ -66,14 +66,14 @@ docker build -t rustprooflabs/pgosm-flex .
 Tag with version.
 
 ```bash
-docker build -t rustprooflabs/pgosm-flex:{{ pgosm_flex_version }} .
+docker build -t rustprooflabs/pgosm-flex:0.10.0 .
 ```
 
 Push to Docker Hub.
 
 ```bash
 docker push rustprooflabs/pgosm-flex:latest
-docker push rustprooflabs/pgosm-flex:{{ pgosm_flex_version }}
+docker push rustprooflabs/pgosm-flex:0.10.0
 ```
 
 

docs/src/replication.md

@@ -54,7 +54,7 @@ docker run --name pgosm -d --rm \
     -v /etc/localtime:/etc/localtime:ro \
     -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
     -p 5433:5432 \
-    -d rustprooflabs/pgosm-flex:{{ pgosm_flex_version }} \
+    -d rustprooflabs/pgosm-flex:0.10.0 \
         -c max_connections=300
 ```
 

docs/src/troubleshooting.md

@@ -72,7 +72,7 @@ docker run --name pgosm -d --rm \
     -v ~/pgosm-data:/app/output \
     -v /etc/localtime:/etc/localtime:ro \
     -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
-    -p 5433:5432 -d rustprooflabs/pgosm-flex:{{ pgosm_flex_version }} \
+    -p 5433:5432 -d rustprooflabs/pgosm-flex \
     -c shared_buffers=1GB \
     -c work_mem=50MB \
     -c maintenance_work_mem=10GB \