Merge pull request #421 from jupyter-naas/407-storage---pullpush-triple-store-from-local-to-prod

fix: Rename target into triplestore-prod-*

Author: Dr0p42

Makefile

@@ -44,14 +44,17 @@ storage-push: .venv storage-pull
 	@ echo "Pushing storage..."
 	@ docker compose run --rm --remove-orphans  abi bash -c 'poetry run python scripts/storage_push.py | sh'
 
-storage-prod-remove: .venv
+triplestore-prod-remove: .venv
 	@ echo "Removing storage..."
-	@ docker compose run --rm -it --remove-orphans  abi bash -c 'poetry run python scripts/storage_remove_prod.py'
+	@ docker compose run --rm -it --remove-orphans  abi bash -c 'poetry run python scripts/triplestore_prod_remove.py'
 
-storage-prod-override: .venv
+triplestore-prod-override: .venv
 	@ echo "Overriding storage..."
-	@ make storage-push
-	@ docker compose run -it --rm --remove-orphans  abi bash -c 'poetry run python scripts/storage_override_prod.py'
+	@ docker compose run -it --rm --remove-orphans  abi bash -c 'poetry run python scripts/triplestore_prod_override.py'
+
+triplestore-prod-pull: .venv
+	@ echo "Pulling storage..."
+	@ docker compose run --rm --remove-orphans abi bash -c 'poetry run python scripts/triplestore_prod_pull.py'
 
 clean:
 	@echo "Cleaning up build artifacts..."

scripts/storage_override_prod.py renamed to scripts/triplestore_prod_override.py

@@ -17,4 +17,5 @@
     backup_timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
     os.system(f'AWS_ACCESS_KEY_ID={access_key_id} AWS_SECRET_ACCESS_KEY={secret_access_key} AWS_SESSION_TOKEN={session_token} aws s3 mv --recursive {bucket}/ontologies {bucket}/ontologies-backups/{backup_timestamp}')
 
-    os.system(f'AWS_ACCESS_KEY_ID={access_key_id} AWS_SECRET_ACCESS_KEY={secret_access_key} AWS_SESSION_TOKEN={session_token} aws s3 cp --recursive {bucket}/storage/triplestore {bucket}/ontologies')
+    # os.system(f'AWS_ACCESS_KEY_ID={access_key_id} AWS_SECRET_ACCESS_KEY={secret_access_key} AWS_SESSION_TOKEN={session_token} aws s3 cp --recursive {bucket}/storage/triplestore {bucket}/ontologies')
+    os.system(f'AWS_ACCESS_KEY_ID={access_key_id} AWS_SECRET_ACCESS_KEY={secret_access_key} AWS_SESSION_TOKEN={session_token} aws s3 sync --follow-symlinks --delete  storage/triplestore {bucket}/ontologies')

scripts/triplestore_prod_pull.py

@@ -0,0 +1,9 @@
+from common import get_config, get_storage_credentials
+import os
+from datetime import datetime
+if __name__ == "__main__":
+    
+    naas_api_key, workspace_id, storage_name = get_config()
+    bucket, access_key_id, secret_access_key, session_token = get_storage_credentials(naas_api_key, workspace_id, storage_name)
+    
+    os.system(f'AWS_ACCESS_KEY_ID={access_key_id} AWS_SECRET_ACCESS_KEY={secret_access_key} AWS_SESSION_TOKEN={session_token} aws s3 sync --follow-symlinks --delete {bucket}/ontologies storage/triplestore')