Merge branch 'master' into dependabot/pip/neuro-auth-client-24.8.0

Author: asvetlov

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: 'v4.4.0'
+  rev: 'v5.0.0'
   hooks:
   - id: check-merge-conflict
     exclude: "rst$"
 - repo: https://github.com/asottile/yesqa
-  rev: v1.4.0
+  rev: v1.5.0
   hooks:
   - id: yesqa
 - repo: https://github.com/sondrelg/pep585-upgrade
@@ -15,22 +15,21 @@ repos:
     args:
     - --futures=true
 - repo: https://github.com/Zac-HD/shed
-  rev: 2023.3.1
+  rev: 2024.10.1
   hooks:
   - id: shed
     args:
     - --refactor
-    - --py39-plus
     types_or:
     - python
     - markdown
     - rst
 - repo: https://github.com/PyCQA/flake8
-  rev: 6.0.0
+  rev: 7.1.1
   hooks:
   - id: flake8
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: 'v4.4.0'
+  rev: 'v5.0.0'
   hooks:
   - id: check-case-conflict
   - id: check-json
@@ -44,7 +43,7 @@ repos:
   - id: debug-statements
 # Another entry is required to apply file-contents-sorter to another file
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: 'v4.4.0'
+  rev: 'v5.0.0'
   hooks:
   - id: file-contents-sorter
     files: |
@@ -62,7 +61,7 @@ repos:
 #     - -ignore
 #     - 'SC1004:'
 - repo: https://github.com/sirosen/check-jsonschema
-  rev: 0.22.0
+  rev: 0.31.0
   hooks:
   - id: check-github-actions
   - id: check-github-workflows

minikube.sh

@@ -14,7 +14,7 @@ function minikube::start {
 
 function minikube::load_images {
     echo "Loading images to minikube..."
-    make gke_docker_pull_test_images
+    make docker_pull_test_images
 }
 
 function minikube::apply_all_configurations {

platform_secrets/api.py

@@ -12,6 +12,7 @@
     HTTPInternalServerError,
     HTTPNoContent,
     HTTPNotFound,
+    HTTPUnprocessableEntity,
     Request,
     Response,
     StreamResponse,
@@ -34,12 +35,13 @@
 from .config_factory import EnvironConfigFactory
 from .identity import untrusted_user
 from .kube_client import KubeClient
-from .service import Secret, SecretNotFound, Service
+from .service import CopyScopeMissingError, Secret, SecretNotFound, Service
 from .validators import (
     secret_key_validator,
     secret_list_response_validator,
     secret_request_validator,
     secret_response_validator,
+    secret_unwrap_validator,
 )
 
 logger = logging.getLogger(__name__)
@@ -80,6 +82,7 @@ def register(self, app: aiohttp.web.Application) -> None:
                 aiohttp.web.post("", self.handle_post),
                 aiohttp.web.get("", self.handle_get_all),
                 aiohttp.web.delete("/{key}", self.handle_delete),
+                aiohttp.web.post("/copy", self.handle_copy),
             ]
         )
 
@@ -199,6 +202,37 @@ async def handle_delete(self, request: Request) -> Response:
             return json_response(resp_payload, status=HTTPNotFound.status_code)
         raise HTTPNoContent()
 
+    async def handle_copy(self, request: Request) -> Response:
+        payload = await request.json()
+        payload = secret_unwrap_validator.check(payload)
+        user = await self._get_untrusted_user(request)
+
+        org_name = payload["org_name"]
+        project_name = payload["project_name"] or user.name
+
+        await check_permissions(
+            request,
+            [self._get_secrets_write_perm(project_name, org_name)],
+        )
+
+        target_namespace = payload["target_namespace"]
+        secret_names = payload["secret_names"]
+
+        try:
+            await self._service.copy_to_namespace(
+                org_name=org_name,
+                project_name=project_name,
+                target_namespace=target_namespace,
+                secret_names=secret_names,
+            )
+        except CopyScopeMissingError as e:
+            resp_payload = {"error": str(e)}
+            return json_response(
+                resp_payload, status=HTTPUnprocessableEntity.status_code
+            )
+
+        return Response(status=HTTPCreated.status_code)
+
 
 @middleware
 async def handle_exceptions(

platform_secrets/kube_client.py

@@ -2,6 +2,7 @@
 import json
 import logging
 import ssl
+import typing
 from contextlib import suppress
 from io import BytesIO
 from pathlib import Path
@@ -107,7 +108,7 @@ async def init(self) -> None:
             limit=self._conn_pool_size, ssl=self._create_ssl_context()
         )
         if self._token_path:
-            self._token = Path(self._token_path).read_text()
+            self._token = self._token_from_path()
             self._token_updater_task = asyncio.create_task(self._start_token_updater())
         timeout = aiohttp.ClientTimeout(
             connect=self._conn_timeout_s, total=self._read_timeout_s
@@ -123,7 +124,7 @@ async def _start_token_updater(self) -> None:
             return
         while True:
             try:
-                token = Path(self._token_path).read_text()
+                token = self._token_from_path()
                 if token != self._token:
                     self._token = token
                     logger.info("Kube token was refreshed")
@@ -133,6 +134,10 @@ async def _start_token_updater(self) -> None:
                 logger.exception("Failed to update kube token: %s", exc)
             await asyncio.sleep(self._token_update_interval_s)
 
+    def _token_from_path(self) -> str:
+        token_path = typing.cast(str, self._token_path)
+        return Path(token_path).read_text().strip()
+
     @property
     def namespace(self) -> str:
         return self._namespace
@@ -158,9 +163,13 @@ async def __aexit__(self, *args: Any) -> None:
     def _api_v1_url(self) -> str:
         return f"{self._base_url}/api/v1"
 
+    @property
+    def _namespaces_url(self) -> str:
+        return f"{self._api_v1_url}/namespaces"
+
     def _generate_namespace_url(self, namespace_name: Optional[str] = None) -> str:
         namespace_name = namespace_name or self._namespace
-        return f"{self._api_v1_url}/namespaces/{namespace_name}"
+        return f"{self._namespaces_url}/{namespace_name}"
 
     def _generate_all_secrets_url(self, namespace_name: Optional[str] = None) -> str:
         namespace_url = self._generate_namespace_url(namespace_name)
@@ -210,24 +219,34 @@ def _raise_for_status(self, payload: dict[str, Any]) -> None:
     async def create_secret(
         self,
         secret_name: str,
-        data: dict[str, str],
+        data: Union[str, dict[str, str]],
         labels: dict[str, str],
         *,
         namespace_name: Optional[str] = None,
+        replace_on_conflict: bool = False,
     ) -> None:
         url = self._generate_all_secrets_url(namespace_name)
-        data = data.copy()
-        data[self._dummy_secret_key] = ""
+        if isinstance(data, dict):
+            data_payload = data.copy()
+            data_payload[self._dummy_secret_key] = ""
+        else:
+            data_payload = {secret_name: data}
+
         payload = {
             "apiVersion": "v1",
             "kind": "Secret",
             "metadata": {"name": secret_name, "labels": labels},
-            "data": data,
+            "data": data_payload,
             "type": "Opaque",
         }
-        headers = {"Content-Type": "application/json"}
-        req_data = BytesIO(json.dumps(payload).encode())
-        await self._request(method="POST", url=url, headers=headers, data=req_data)
+        try:
+            await self._request(method="POST", url=url, json=payload)
+        except ResourceConflict as e:
+            if not replace_on_conflict:
+                raise e
+            # replace a secret
+            url = f"{url}/{secret_name}"
+            await self._request(method="PUT", url=url, json=payload)
 
     async def add_secret_key(
         self,
@@ -281,3 +300,17 @@ async def list_secrets(
         for item in items:
             self._cleanup_secret_payload(item)
         return items
+
+    async def create_namespace(self, name: str) -> None:
+        """Creates a namespace. Ignores conflict errors"""
+        url = URL(self._namespaces_url)
+        payload = {
+            "apiVersion": "v1",
+            "kind": "Namespace",
+            "metadata": {"name": name},
+        }
+        try:
+            await self._request(method="POST", url=url, json=payload)
+        except ResourceConflict:
+            # ignore on conflict
+            return

platform_secrets/service.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 import re
 from dataclasses import dataclass, field
@@ -14,16 +16,23 @@
 logger = logging.getLogger()
 
 SECRET_API_ORG_LABEL = "platform.neuromation.io/secret-api-org-name"
+APPS_SECRET_NAME = "apps-secrets"
 
 NO_ORG = "NO_ORG"
 
 
 class SecretNotFound(Exception):
     @classmethod
-    def create(cls, secret_key: str) -> "SecretNotFound":
+    def create(cls, secret_key: str) -> SecretNotFound:
         return cls(f"Secret {secret_key!r} not found")
 
 
+class CopyScopeMissingError(Exception):
+    @classmethod
+    def create(cls, missing_keys: set[str]) -> CopyScopeMissingError:
+        return cls(f"Missing secrets: {', '.join(missing_keys)}")
+
+
 @dataclass(frozen=True)
 class Secret:
     key: str
@@ -118,6 +127,44 @@ async def get_all_secrets(
             ]
         return result
 
+    async def copy_to_namespace(
+        self,
+        org_name: str,
+        project_name: str,
+        target_namespace: str,
+        secret_names: list[str],
+    ) -> None:
+        """
+        Unwraps secrets from a dict and extracts them to a dedicated namespace.
+        """
+        secrets = await self.get_all_secrets(
+            with_values=True,
+            org_name=org_name,
+            project_name=project_name,
+        )
+        secrets_scope = set(secret_names)
+
+        missing_secret_names = secrets_scope - {secret.key for secret in secrets}
+        if missing_secret_names:
+            raise CopyScopeMissingError.create(missing_secret_names)
+
+        # ensure namespace is there
+        await self._kube_client.create_namespace(name=target_namespace)
+
+        data = {
+            secret.key: secret.value
+            for secret in secrets
+            if secret.key in secrets_scope
+        }
+
+        await self._kube_client.create_secret(
+            APPS_SECRET_NAME,
+            data=data,
+            labels={},
+            namespace_name=target_namespace,
+            replace_on_conflict=True,
+        )
+
     async def migrate_user_to_project_secrets(self) -> None:
         # TODO: remove migration after deploy to prod
         user_secrets = [

platform_secrets/validators.py

@@ -43,3 +43,11 @@ def check_base64(value: str) -> str:
     }
 )
 secret_list_response_validator = t.List(secret_response_validator)
+secret_unwrap_validator = t.Dict(
+    {
+        t.Key("org_name"): t.String(min_length=1, max_length=253) | t.Null(),
+        t.Key("project_name"): t.String(min_length=1, max_length=253) | t.Null(),
+        t.Key("target_namespace"): t.String(min_length=1, max_length=253),
+        t.Key("secret_names"): t.List(t.String(), min_length=1),
+    }
+)

setup.cfg

@@ -16,22 +16,22 @@ platforms = any
 include_package_data = True
 install_requires =
     aiohttp==3.10.10
-    yarl==1.17.1
+    yarl==1.18.3
     multidict==6.0.5
     neuro-auth-client==24.8.0
     trafaret==2.1.1
-    neuro-logging==24.4.0
+    neuro-logging==25.1.0
 
 [options.entry_points]
 console_scripts =
     platform-secrets = platform_secrets.api:main
 
 [options.extras_require]
 dev =
-    mypy==1.13.0
-    pre-commit==4.0.1
+    mypy==1.14.1
+    pre-commit==4.1.0
     pytest==8.3.3
-    pytest-asyncio==0.23.8
+    pytest-asyncio==0.25.2
     pytest-cov==6.0.0
 
 [flake8]

tests/integration/conftest.py

@@ -37,12 +37,12 @@ def config_factory(
     auth_config: PlatformAuthConfig, kube_config: KubeConfig, cluster_name: str
 ) -> Callable[..., Config]:
     def _f(**kwargs: Any) -> Config:
-        defaults = dict(
-            server=ServerConfig(host="0.0.0.0", port=8080),
-            platform_auth=auth_config,
-            kube=kube_config,
-            cluster_name=cluster_name,
-        )
+        defaults = {
+            "server": ServerConfig(host="0.0.0.0", port=8080),
+            "platform_auth": auth_config,
+            "kube": kube_config,
+            "cluster_name": cluster_name,
+        }
         kwargs = {**defaults, **kwargs}
         return Config(**kwargs)
 

tests/integration/conftest_auth.py

@@ -9,7 +9,6 @@
 from yarl import URL
 
 from platform_secrets.config import PlatformAuthConfig
-
 from tests.integration.conftest import get_service_url, random_name