feat(settings): really plug settings object

Author: tepelbaum

.github/workflows/code-quality.yml

@@ -51,6 +51,7 @@ jobs:
             -e app_port=8000
             -v ${{github.workspace}}/tests:/app/tests
             -v ${{github.workspace}}/ecodev_core:/app/app
+            -v ${{github.workspace}}/config:/app/app/assets
             -v ${{github.workspace}}:/app/
             -v ${{github.workspace}}/.coveragerc:/app/.coveragerc
           run: |

config/local.yaml

@@ -0,0 +1,30 @@
+---
+ authentication:
+   algorithm: null
+   access_token_expire_minutes: null
+   secret_key: null
+
+ database:
+   db_port: null
+   db_name: null
+   db_host: null
+   db_username: null
+   db_password: null
+
+ elastic_search:
+   port: null
+   host: null
+   user: null
+   password: null
+   index: null
+
+ backup:
+   backup_username: null
+   backup_password: null
+   backup_url: null
+
+ smtp:
+    email_smtp: null
+    email_port: null
+    email_sender: null
+    email_password: null

ecodev_core/auth_configuration.py

@@ -4,6 +4,8 @@
 from pydantic_settings import BaseSettings
 from pydantic_settings import SettingsConfigDict
 
+from ecodev_core.settings import SETTINGS
+
 
 class AuthenticationConfiguration(BaseSettings):
     """
@@ -16,3 +18,7 @@ class AuthenticationConfiguration(BaseSettings):
 
 
 AUTH = AuthenticationConfiguration()
+SETTINGS_AUTH = SETTINGS.authentication  # type: ignore[attr-defined]
+SECRET_KEY = SETTINGS_AUTH.secret_key or AUTH.secret_key
+ALGO = SETTINGS_AUTH.algorithm or AUTH.algorithm
+EXPIRATION_LENGTH = SETTINGS_AUTH.access_token_expire_minutes or AUTH.access_token_expire_minutes

ecodev_core/authentication.py

@@ -26,7 +26,9 @@
 from starlette.responses import RedirectResponse
 
 from ecodev_core.app_user import AppUser
-from ecodev_core.auth_configuration import AUTH
+from ecodev_core.auth_configuration import ALGO
+from ecodev_core.auth_configuration import EXPIRATION_LENGTH
+from ecodev_core.auth_configuration import SECRET_KEY
 from ecodev_core.db_connection import engine
 from ecodev_core.logger import logger_get
 from ecodev_core.permissions import Permission
@@ -235,11 +237,11 @@ def _create_access_token(data: Dict, tfa_value: Optional[str] = None) -> str:
     Create an access token out of the passed data. Only called if credentials are valid
     """
     to_encode = data.copy()
-    expire = datetime.now(timezone.utc) + timedelta(minutes=AUTH.access_token_expire_minutes)
+    expire = datetime.now(timezone.utc) + timedelta(minutes=EXPIRATION_LENGTH)
     to_encode['exp'] = expire
     if tfa_value:
         to_encode['tfa'] = _hash_password(tfa_value)
-    return jwt.encode(to_encode, AUTH.secret_key, algorithm=AUTH.algorithm)
+    return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGO)
 
 
 def _verify_access_token(token: str,
@@ -249,7 +251,7 @@ def _verify_access_token(token: str,
     Retrieves the token data associated to the passed token if it contains valid credential info.
     """
     try:
-        payload = jwt.decode(token, AUTH.secret_key, algorithms=[AUTH.algorithm])
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGO])
         if tfa_check and (not tfa_value or not _check_password(tfa_value, payload.get('tfa'))):
             raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail=INVALID_TFA,
                                 headers={'WWW-Authenticate': 'Bearer'})

ecodev_core/backup.py

@@ -15,7 +15,7 @@
 
 from ecodev_core.db_connection import DB_URL
 from ecodev_core.logger import logger_get
-
+from ecodev_core.settings import SETTINGS
 
 log = logger_get(__name__)
 
@@ -30,8 +30,11 @@ class BackUpSettings(BaseSettings):
     model_config = SettingsConfigDict(env_file='.env')
 
 
-BCK = BackUpSettings()
-BACKUP_URL = f'ftp://{BCK.backup_username}:{BCK.backup_password}@{BCK.backup_url}'
+BCK, SETTINGS_BCK = BackUpSettings(), SETTINGS.backup  # type: ignore[attr-defined]
+_USER = SETTINGS_BCK.backup_username or BCK.backup_username
+_PASSWD = SETTINGS_BCK.backup_password or BCK.backup_password
+_URL = SETTINGS_BCK.backup_url or BCK.backup_url
+BACKUP_URL = f'ftp://{_USER}:{_PASSWD}@{_URL}'
 
 
 def backup(backed_folder: Path, nb_saves: int = 5, additional_id: str = 'default') -> None:

ecodev_core/db_connection.py

@@ -14,6 +14,7 @@
 from sqlmodel import SQLModel
 
 from ecodev_core.logger import logger_get
+from ecodev_core.settings import SETTINGS
 
 log = logger_get(__name__)
 
@@ -30,9 +31,11 @@ class DbSettings(BaseSettings):
     model_config = SettingsConfigDict(env_file='.env')
 
 
-DB = DbSettings()
-_PASSWORD = quote(DB.db_password, safe='')
-DB_URL = f'postgresql://{DB.db_username}:{_PASSWORD}@{DB.db_host}:{DB.db_port}/{DB.db_name}'
+DB, SETTINGS_DB = DbSettings(), SETTINGS.database  # type: ignore[attr-defined]
+_PASSWORD = quote(SETTINGS_DB.db_password or DB.db_password, safe='')
+_USER, _HOST = SETTINGS_DB.db_username or DB.db_username, SETTINGS_DB.db_host or DB.db_host
+_PORT, _NAME = SETTINGS_DB.db_port or DB.db_port, SETTINGS_DB.db_name or DB.db_name
+DB_URL = f'postgresql://{_USER}:{_PASSWORD}@{_HOST}:{_PORT}/{_NAME}'
 engine = create_engine(DB_URL)
 
 

ecodev_core/email_sender.py

@@ -11,6 +11,8 @@
 from pydantic_settings import BaseSettings
 from pydantic_settings import SettingsConfigDict
 
+from ecodev_core.settings import SETTINGS
+
 
 class EmailAuth(BaseSettings):
     """
@@ -19,10 +21,15 @@ class EmailAuth(BaseSettings):
     email_smtp: str = ''
     email_sender: str = ''
     email_password: str = ''
+    email_port: int = 587
     model_config = SettingsConfigDict(env_file='.env')
 
 
-EMAIL_AUTH = EmailAuth()
+EMAIL_AUTH, EMAIL_SETTINGS = EmailAuth(), SETTINGS.smtp  # type: ignore[attr-defined]
+_SENDER = EMAIL_SETTINGS.email_sender or EMAIL_AUTH.email_sender
+_SMTP = EMAIL_SETTINGS.email_smtp or EMAIL_AUTH.email_smtp
+_PASSWD = EMAIL_SETTINGS.email_password or EMAIL_AUTH.email_password
+_PORT = EMAIL_SETTINGS.email_port or EMAIL_AUTH.email_port
 
 
 def send_email(email: str, body: str, topic: str, images: dict[str, Path] | None = None) -> None:
@@ -36,7 +43,7 @@ def send_email(email: str, body: str, topic: str, images: dict[str, Path] | None
         - images: if any, the Dict of image tags:image paths to incorporate in the email
     """
     em = MIMEMultipart('related')
-    em['From'] = EMAIL_AUTH.email_sender
+    em['From'] = _SENDER
     em['To'] = email
     em['Subject'] = topic
     em.attach(MIMEText(body, 'html'))
@@ -46,8 +53,8 @@ def send_email(email: str, body: str, topic: str, images: dict[str, Path] | None
         img.add_header('Content-ID', f'<{tag}>')
         em.attach(img)
 
-    with SMTP(EMAIL_AUTH.email_smtp, 587) as server:
+    with SMTP(_SMTP, _PORT) as server:
         server.ehlo()
         server.starttls(context=create_default_context())
-        server.login(EMAIL_AUTH.email_sender, EMAIL_AUTH.email_password)
-        server.sendmail(EMAIL_AUTH.email_sender, email, em.as_string())
+        server.login(_SENDER, _PASSWD)
+        server.sendmail(_SENDER, email, em.as_string())

ecodev_core/es_connection.py

@@ -11,6 +11,7 @@
 from pydantic_settings import SettingsConfigDict
 
 from ecodev_core.logger import logger_get
+from ecodev_core.settings import SETTINGS
 
 ES_CLIENT: Union[Elasticsearch, None] = None
 log = logger_get(__name__)
@@ -21,15 +22,18 @@ class ESAuth(BaseSettings):
     """
     Simple ES authentication configuration class
     """
-    host: str
-    user: str
-    password: str
-    port: int
-    index: str
+    host: str = ''
+    user: str = ''
+    password: str = ''
+    port: int = 9200
+    index: str = ''
     model_config = SettingsConfigDict(env_file='.env', env_prefix='ES_')
 
 
-ES_AUTH = ESAuth()  # type: ignore
+ES_AUTH, ES_SETTINGS = ESAuth(), SETTINGS.elastic_search  # type: ignore[attr-defined]
+_HOST, _PORT = ES_SETTINGS.host or ES_AUTH.host,  ES_SETTINGS.port or ES_AUTH.port
+_USER, _PASSWD = ES_SETTINGS.user or ES_AUTH.user, ES_SETTINGS.password or ES_AUTH.password
+_INDEX = ES_SETTINGS.index or ES_AUTH.index
 
 
 def get_es_client():
@@ -39,8 +43,7 @@ def get_es_client():
     global ES_CLIENT
 
     if ES_CLIENT is None:
-        ES_CLIENT = Elasticsearch(f'http://{ES_AUTH.host}:{ES_AUTH.port}/',
-                                  basic_auth=[ES_AUTH.user, ES_AUTH.password])
+        ES_CLIENT = Elasticsearch(f'http://{_HOST}:{_PORT}/', basic_auth=[_USER, _PASSWD])
 
     return ES_CLIENT
 
@@ -51,11 +54,11 @@ def create_es_index(body: dict) -> None:
     """
     client = get_es_client()
     try:
-        client.indices.delete(index=ES_AUTH.index)
+        client.indices.delete(index=_INDEX)
     except Exception:
         pass
-    client.indices.create(index=ES_AUTH.index, body=body)
-    log.info(f'index {ES_AUTH.index} created')
+    client.indices.create(index=_INDEX, body=body)
+    log.info(f'index {_INDEX} created')
 
 
 def insert_es_fields(operations: list[dict], batch_size: int = ES_BATCH_SIZE) -> None:
@@ -66,11 +69,11 @@ def insert_es_fields(operations: list[dict], batch_size: int = ES_BATCH_SIZE) ->
     batches = [list(operations)[i:i + batch_size] for i in range(0, len(operations), batch_size)]
     log.info('indexing fields')
     for batch in progressbar.progressbar(batches, redirect_stdout=False):
-        helpers.bulk(client, batch, index=ES_AUTH.index)
+        helpers.bulk(client, batch, index=_INDEX)
 
 
 def retrieve_es_fields(body: dict[str, Any]) -> list[dict]:
     """
     Core call to the elasticsearch index
     """
-    return get_es_client().search(index=ES_AUTH.index, body=body)
+    return get_es_client().search(index=_INDEX, body=body)

ecodev_core/settings.py

@@ -1,6 +1,7 @@
 """
 Module defining a dynamic setting class
 """
+from contextlib import suppress
 from pathlib import Path
 
 from pydantic.v1.utils import deep_update
@@ -9,18 +10,24 @@
 
 from ecodev_core.deployment import Deployment
 from ecodev_core.list_utils import dict_to_class
+from ecodev_core.logger import logger_get
 from ecodev_core.read_write import load_yaml_file
 
+log = logger_get(__name__)
+
 
 class DeploymentSetting(BaseSettings):
     """
     Settings class used to load the deployment type from environment variables.
     """
     environment: str = 'local'
+    base_path: str = '/app/app/assets'
     model_config = SettingsConfigDict(env_file='.env')
 
 
-DEPLOYMENT = Deployment(DeploymentSetting().environment.lower())
+DEPLOYMENT_SETTINGS = DeploymentSetting()
+DEPLOYMENT = Deployment(DEPLOYMENT_SETTINGS.environment.lower())
+BASE_PATH = Path(DEPLOYMENT_SETTINGS.base_path)
 
 
 class Settings:
@@ -29,15 +36,21 @@ class Settings:
     this configuration with additional information coming from a secret file.
     """
 
-    def __init__(self, base_path: Path = Path('/app'), deployment: Deployment = DEPLOYMENT):
+    def __init__(self, base_path: Path = BASE_PATH, deployment: Deployment = DEPLOYMENT):
         """
         Dynamically setting Settings attributes, doing so recursively. Attributes are loaded
          from config file, possibly overwriting some of this configuration with additional
          information coming from a secret file.
         """
         self.deployment = deployment
-        data = load_yaml_file(base_path / 'config' / f'{deployment.value}.yaml')
-        if (secrets_file := base_path / 'secrets' / f'{deployment.value}.yaml').exists():
-            data = deep_update(data, load_yaml_file(secrets_file))
-        for k, v in dict_to_class(data).items():
-            setattr(self, k, v)
+
+        with suppress(FileNotFoundError):
+            log.info((base_path / 'config' / f'{deployment.value}.yaml').exists())
+            data = load_yaml_file(base_path / 'config' / f'{deployment.value}.yaml')
+            if (secrets_file := base_path / 'secrets' / f'{deployment.value}.yaml').exists():
+                data = deep_update(data, load_yaml_file(secrets_file))
+            for k, v in dict_to_class(data).items():
+                setattr(self, k, v)
+
+
+SETTINGS = Settings()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ecodev-core"
-version = "0.0.49"
+version = "0.0.50"
 description = "Low level sqlmodel/fastapi/pydantic building blocks"
 authors = ["Thomas Epelbaum <[email protected]>",
 					 "Olivier Gabriel <[email protected]>",