Merge pull request #147 from rustprooflabs/bash-to-python

Rewrite Docker runtime in Python from Bash

Author: rustprooflabs

.dockerignore

@@ -0,0 +1,8 @@
+tests/
+**/.coverage
+*.md
+.github
+.gitignore
+Makefile
+.dockerignore
+Dockerfile

.gitignore

@@ -1 +1,4 @@
-tests/tmp/*
+tests/tmp/*
+output/
+**/.coverage
+**/__pycache__

DOCKER-BUILD.md

@@ -17,13 +17,13 @@ docker build -t rustprooflabs/pgosm-flex .
 
 Tag with version.
 
-```
+```bash
 docker build -t rustprooflabs/pgosm-flex:0.1.2 .
 ```
 
 Push to Docker Hub.
 
-```
+```bash
 docker push rustprooflabs/pgosm-flex:0.1.2
 docker push rustprooflabs/pgosm-flex:latest
 ```

DOCKER-RUN.md

@@ -1,7 +1,7 @@
 # Using PgOSM-Flex within Docker
 
 This README provides details about running PgOSM-Flex using the image defined
-in `Dockerfile` and the script loaded from `docker/run_pgosm_flex.sh`.
+in `Dockerfile` and the script loaded from `docker/pgosm_flex.py`.
 
 
 ## Setup and Run Container
@@ -12,6 +12,15 @@ Create directory for the `.osm.pbf` file and output `.sql` file.
 mkdir ~/pgosm-data
 ```
 
+
+Set environment variables for the temporary Postgres connection in Docker.
+
+```bash
+export POSTGRES_USER=postgres
+export POSTGRES_PASSWORD=mysecretpassword
+```
+
+
 Start the `pgosm` Docker container to make PostgreSQL/PostGIS available.
 This command exposes Postgres inside Docker on port 5433 and establishes links
 to the local directory created above (`~/pgosm-data`).
@@ -20,10 +29,10 @@ your the host machine's timezone, important when for archiving PBF & MD5 files b
 
 
 ```bash
-docker run --name pgosm -d \
+docker run --name pgosm -d --rm \
     -v ~/pgosm-data:/app/output \
     -v /etc/localtime:/etc/localtime:ro \
-    -e POSTGRES_PASSWORD=mysecretpassword \
+    -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
     -p 5433:5432 -d rustprooflabs/pgosm-flex
 ```
 
@@ -32,84 +41,103 @@ docker run --name pgosm -d \
 The following `docker exec` command runs PgOSM Flex to load the District of Columbia
 region
 
-The command  `bash docker/run_pgosm_flex.sh` runs the full process. The
+The command  `python3 docker/pgosm_flex.py` runs the full process. The
 script uses a region (`north-america/us`) and sub-region (`district-of-columbia`)
 that must match values in URLs from the Geofabrik download server.
 The 3rd parameter tells the script the server has 8 GB RAM available for osm2pgsql, Postgres, and the OS.  The PgOSM-Flex layer set is defined (`run-all`).
 
 
 ```bash
 docker exec -it \
-    -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_USER=postgres \
-    pgosm bash docker/run_pgosm_flex.sh \
-    north-america/us \
-    district-of-columbia \
-    8 \
-    run-all
+    -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD -e POSTGRES_USER=$POSTGRES_USER \
+    pgosm python3 docker/pgosm_flex.py \
+    --layerset=run-all --ram=8 \
+    --region=north-america/us \
+    --subregion=district-of-columbia
 ```
 
 
 ## Customize PgOSM-Flex
 
-The following command sets the four (4) main env vars used to customize PgOSM-Flex.
-
-* `PGOSM_SRID` - Set custom SRID, must be in `public.spatial_ref_sys`.  Default `3857`
-* `PGOSM_DATA_SCHEMA_NAME` - Final schema name for the OpenStreetMap data. Default `osm`
-* `PGOSM_DATA_SCHEMA_ONLY` - When `false` (default) the QGIS styles and `pgosm` schema are exported along with the `PGOSM_DATA_SCHEMA_NAME` schema
-* `PGOSM_DATE` - Used to document data loaded to DB in `osm.pgosm_flex.pgosm_date`, and for archiving PBF/MD5 files.  Defaults to today.
-* `PGOSM_LANGUAGE` - Used to prefer specific language when it exists.
-
+See full set of options via `--help`.
 
 ```bash
-docker exec -it \
-    -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_USER=postgres \
-    -e PGOSM_SRID=4326 \
-    -e PGOSM_DATA_SCHEMA_ONLY=true \
-    -e PGOSM_DATA_SCHEMA_NAME=osm_dc \
-    -e PGOSM_DATE='2021-03-11' \
-    -e PGOSM_LANGUAGE=en \
-    pgosm bash docker/run_pgosm_flex.sh \
-    north-america/us \
-    district-of-columbia \
-    8 \
-    run-all
+docker exec -it pgosm python3 docker/pgosm_flex.py --help
 ```
 
-## Skip nested polygon calculation
+```bash
+Usage: pgosm_flex.py [OPTIONS]
+
+  Logic to run PgOSM Flex within Docker.
+
+Options:
+  --layerset TEXT     Layer set from PgOSM Flex to load. e.g. run-all
+                      [default: (run-all);required]
+  --ram INTEGER       Amount of RAM in GB available on the server running this
+                      process.  [default: 4;required]
+  --region TEXT       Region name matching the filename for data sourced from
+                      Geofabrik. e.g. north-america/us  [default: (north-
+                      america/us);required]
+  --subregion TEXT    Sub-region name matching the filename for data sourced
+                      from Geofabrik. e.g. district-of-columbia  [default:
+                      (district-of-columbia)]
+  --srid TEXT         SRID for data in PostGIS.
+  --pgosm-date TEXT   Date of the data in YYYY-MM-DD format. Set to historic
+                      date to load locally archived PBF/MD5 file, will fail if
+                      both files do not exist.
+  --language TEXT     Set default language in loaded OpenStreetMap data when
+                      available.  e.g. 'en' or 'kn'.
+  --schema-name TEXT  Coming soon
+  --skip-nested       When True, skips calculating nested admin polygons. Can
+                      be time consuming on large regions.
+  --data-only         When True, skips running Sqitch and importing QGIS
+                      Styles.
+  --debug             Enables additional log output
+  --basepath TEXT     Debugging option. Used when testing locally and not
+                      within Docker
+  --help              Show this message and exit.
+```
 
-The default is to run the nested polygon calculation. This can take considerable time on larger regions or may
-be otherwise unwanted.  Define the env var `PGOSM_SKIP_NESTED_POLYGON` with the `docker exec` command
-to skip this process.
+An example of running with all current options, except `--basepath` which is only
+used during development.
 
 ```bash
- -e PGOSM_SKIP_NESTED_POLYGON=anything
+docker exec -it \
+    -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD  -e POSTGRES_USER=$POSTGRES_USER \
+    pgosm python3 docker/pgosm_flex.py \
+    --layerset=run-all \
+    --ram=8 \
+    --region=north-america/us \
+    --subregion=district-of-columbia \
+    --schema-name=osm_dc \
+    --pgosm-date="2021-03-11" \
+    --language="en" \
+    --srid="4326" \
+    --data-only \
+    --skip-nested \
+    --debug
 ```
 
 
-## Always download
+## Skip nested polygon calculation
 
-To force the processing to remove existing files and re-download the latest PBF and MD5 files from Geofabrik, set the `PGOSM_ALWAYS_DOWNLOAD` env var when running the Docker container.
+Use `--skip-nested` to bypass the calculation of nested admin polygons.
+
+The default is to run the nested polygon calculation. This can take considerable time on larger regions or may
+be otherwise unwanted.
 
-```bash
-docker run --name pgosm -d \
-    -v ~/pgosm-data:/app/output \
-    -e POSTGRES_PASSWORD=mysecretpassword \
-    -e PGOSM_ALWAYS_DOWNLOAD=1 \
-    -p 5433:5432 -d rustprooflabs/pgosm
-```
 
 ## Configure Postgres in Docker
 
 Add customizations with the `-c` switch, e.g. `-c shared_buffers=1GB`,
 to customize Postgres' configuration at run-time in Docker.
 
 
-
 ```bash
-docker run --name pgosm -d \
+docker run --name pgosm -d --rm \
     -v ~/pgosm-data:/app/output \
     -v /etc/localtime:/etc/localtime:ro \
-    -e POSTGRES_PASSWORD=mysecretpassword \
+    -e POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
     -p 5433:5432 -d rustprooflabs/pgosm-flex \
     -c shared_buffers=1GB \
     -c maintenance_work_mem=1GB \

Dockerfile

@@ -9,23 +9,20 @@ RUN apt-get update \
         libboost-dev libboost-system-dev \
         libboost-filesystem-dev libexpat1-dev zlib1g-dev \
         libbz2-dev libpq-dev libproj-dev lua5.2 liblua5.2-dev \
-        python3 python3-distutils \
+        python3 python3-distutils python3-psycopg2 \
         curl \
     && rm -rf /var/lib/apt/lists/*
 
 RUN curl -o /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py \
     && python3 /tmp/get-pip.py \
     && rm /tmp/get-pip.py
 
-RUN pip install requests click
-
-
 WORKDIR /tmp
 RUN git clone git://github.com/openstreetmap/osm2pgsql.git \
     && mkdir osm2pgsql/build \
     && cd osm2pgsql/build \
     && cmake .. \
-    && make \
+    && make -j$(nproc) \
     && make install \
     && apt remove -y \
         make cmake g++ \
@@ -36,6 +33,9 @@ RUN git clone git://github.com/openstreetmap/osm2pgsql.git \
     && apt autoremove -y \
     && cd /tmp && rm -r /tmp/osm2pgsql
 
+COPY ./sqitch.conf /etc/sqitch/sqitch.conf
 
 WORKDIR /app
 COPY . ./
+
+RUN pip install -r requirements.txt

MANUAL-STEPS-RUN.md

@@ -3,7 +3,7 @@
 These instructions show how to manually run the PgOSM-Flex process.
 This is the best option for scaling to larger regions (North America, Europe, etc.)
 due to the need to customize a number of configurations.  Review the
-`docker/run_pgosm_flex.sh` for a starting point to automating the process.
+`python3 docker/pgosm_flex.py` for a starting point to automating the process.
 
 This basic working example uses Washington D.C. for a small, fast test of the
 process.
@@ -253,3 +253,47 @@ psql -d pgosm -f data/roads-us.sql
 
 Currently only U.S. region drafted, more regions with local `maxspeed` are welcome via PR!
 
+
+## Customize PgOSM Flex
+
+Track additional details in the `osm.pgosm_meta` table (see more below)
+and customize behavior with the use of environment variables.
+
+* `OSM_DATE`
+* `PGOSM_SRID`
+* `PGOSM_REGION`
+* `PGOSM_LANGUAGE`
+
+
+### Custom SRID
+
+To use `SRID 4326` instead of the default `SRID 3857`, set the `PGOSM_SRID`
+environment variable before running osm2pgsql.
+
+```bash
+export PGOSM_SRID=4326
+```
+
+Changes to the SRID are reflected in output printed.
+
+```bash
+2021-01-08 15:01:15  osm2pgsql version 1.4.0 (1.4.0-72-gc3eb0fb6)
+2021-01-08 15:01:15  Database version: 13.1 (Ubuntu 13.1-1.pgdg20.10+1)
+2021-01-08 15:01:15  Node-cache: cache=800MB, maxblocks=12800*65536, allocation method=11
+Custom SRID: 4326
+...
+```
+
+### Preferred Language
+
+The `name` column throughout PgOSM-Flex defaults to using the highest priority
+name tag according to the [OSM Wiki](https://wiki.openstreetmap.org/wiki/Names). Setting `PGOSM_LANGUAGE` allows giving preference to name tags with the
+given language.
+The value of `PGOSM_LANGUAGE` should match the codes used by OSM:
+
+> where code is a lowercase language's ISO 639-1 alpha2 code, or a lowercase ISO 639-2 code if an ISO 639-1 code doesn't exist." -- [Multilingual names on OSM Wiki](https://wiki.openstreetmap.org/wiki/Multilingual_names)
+
+
+```bash
+export PGOSM_LANGUAGE=kn
+```

Makefile

@@ -0,0 +1,85 @@
+## ----------------------------------------------------------------------
+## This Makefile builds and tests the PgOSM Flex Docker image.
+##
+## For full build/test use:
+##    make
+##
+## To cleanup after you are done:
+##    make docker-clean
+## ----------------------------------------------------------------------
+CURRENT_UID := $(shell id -u)
+CURRENT_GID := $(shell id -g)
+TODAY := $(shell date +'%Y-%m-%d')
+
+.PHONY: all
+all: docker-clean build-run-docker unit-tests
+
+.PHONY: docker-clean
+docker-clean: ## Stops pgosm Docker container and removes local pgosm-data directory
+	@docker stop pgosm > /dev/null 2>&1 && echo "pgosm container removed"|| echo "pgosm container not present, nothing to remove"
+	rm -rvf pgosm-data|| echo "folder pgosm-data did not exist"
+
+
+.PHONY: build-run-docker
+build-run-docker: ## Builds and runs PgOSM Flex with D.C. test file
+	docker build -t rustprooflabs/pgosm-flex .
+	docker run --name pgosm \
+		--rm \
+		-v $(shell pwd)/pgosm-data:/app/output \
+		-v /etc/localtime:/etc/localtime:ro \
+		-e POSTGRES_PASSWORD=mysecretpassword \
+		-p 5433:5432 \
+		-d \
+		rustprooflabs/pgosm-flex
+	# copy the test data pretending it's latest to avoid downloading each time
+	docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf \
+		pgosm:/app/output/district-of-columbia-$(TODAY).osm.pbf
+	docker cp tests/data/district-of-columbia-2021-01-13.osm.pbf.md5 \
+		pgosm:/app/output/district-of-columbia-$(TODAY).osm.pbf.md5
+
+	# allow files created in later step to be created
+	docker exec -it pgosm \
+		chown $(CURRENT_UID):$(CURRENT_GID) /app/output/
+	# Needed for unit-tests
+	docker exec -it pgosm \
+		chown $(CURRENT_UID):$(CURRENT_GID) /app/docker/
+
+	docker exec -it \
+		-e POSTGRES_PASSWORD=mysecretpassword \
+		-e POSTGRES_USER=postgres \
+		-u $(CURRENT_UID):$(CURRENT_GID) \
+		pgosm python3 docker/pgosm_flex.py  \
+		--layerset=run-all \
+		--ram=1 \
+		--region=north-america/us \
+		--subregion=district-of-columbia \
+		--debug
+
+
+.PHONY: unit-tests
+unit-tests: ## Runs Python unit tests and data import tests
+	# Unit tests covering Python runtime
+	docker exec -it \
+		-e POSTGRES_PASSWORD=mysecretpassword \
+		-e POSTGRES_USER=postgres \
+		-u $(CURRENT_UID):$(CURRENT_GID) \
+		pgosm /bin/bash -c "cd docker && coverage run -m unittest tests/*.py"
+
+	# Data import tests
+	docker cp tests \
+		pgosm:/app/tests
+	docker exec -it pgosm \
+		chown $(CURRENT_UID):$(CURRENT_GID) /app/tests/
+
+	# Detailed results from tests currently get buried in the Docker container
+	#   Error such as: FAILED TEST: sql/amenity_point_osm_type_count.sql - See tmp/amenity_point_osm_type_count.diff
+	#   Use command (changing file at end): docker exec -it     -e POSTGRES_PASSWORD=mysecretpassword -e POSTGRES_USER=postgres     pgosm /bin/cat /app/tests/tmp/amenity_point_osm_type_count.diff
+	docker exec -it \
+		-e POSTGRES_PASSWORD=mysecretpassword \
+		-e POSTGRES_USER=postgres \
+		-u $(CURRENT_UID):$(CURRENT_GID) \
+		pgosm /bin/bash -c "cd tests && ./run-output-tests.sh"
+
+
+help: ## Show this help
+	@egrep -h '\s##\s' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m  %-30s\033[0m %s\n", $$1, $$2}'