Merge branch 'development' of github.com:hotosm/fmtm into fix/offline-bugs

Author: NSUWAL123

.env.example

@@ -61,3 +61,11 @@ OTEL_AUTH_TOKEN=${OTEL_AUTH_TOKEN}
 # Monitoring (Sentry)
 SENTRY_DSN=${SENTRY_DSN}
 
+## SMTP CONFIG ##
+SMTP_TLS=${SMTP_TLS:-True}
+SMTP_SSL=${SMTP_SSL:-False}
+SMTP_PORT=${SMTP_PORT:-587}
+SMTP_HOST=${SMTP_HOST}
+SMTP_USER=${SMTP_USER}
+SMTP_PASSWORD=${SMTP_PASSWORD}
+EMAILS_FROM_EMAIL=${EMAILS_FROM_EMAIL}

compose.yaml

@@ -214,6 +214,7 @@ services:
       - /app/.svelte-kit/
       # - ../ui:/app/node_modules/@hotosm/ui:ro
     environment:
+      - TEST_PWA=${TEST_PWA:-false}
       - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}
       - VITE_SYNC_URL=http://sync.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}
       - VITE_S3_URL=http://s3.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}
@@ -222,6 +223,8 @@ services:
     networks:
       - fmtm-net
     restart: "unless-stopped"
+    command: >
+      sh -c "if [ \"$TEST_PWA\" = \"true\" ]; then pnpm run build && pnpm run preview; else pnpm run dev; fi"
 
   central:
     profiles: ["", "central"]
@@ -343,7 +346,7 @@ services:
   fmtm-db:
     # Temp workaround until https://github.com/postgis/docker-postgis/issues/216
     image: "ghcr.io/hotosm/postgis:${POSTGIS_TAG:-14-3.5-alpine}"
-    command: -c 'max_connections=300' -c 'wal_level=logical'
+    command: -c 'wal_level=logical'
     volumes:
       - fmtm_db_data:/var/lib/postgresql/data/
     environment:
@@ -363,7 +366,7 @@ services:
       retries: 3
 
   electric:
-    image: "electricsql/electric:${ELECTRIC_TAG:-1.0.11}"
+    image: "electricsql/electric:${ELECTRIC_TAG:-1.0.12}"
     depends_on:
       proxy:
         condition: service_started
@@ -385,6 +388,7 @@ services:
   central-db:
     profiles: ["", "central"]
     image: "ghcr.io/hotosm/postgis:${POSTGIS_TAG:-14-3.5-alpine}"
+    command: -c 'max_connections=300' -c 'wal_level=logical'
     volumes:
       - central_db_data:/var/lib/postgresql/data/
     environment:

deploy/compose.development.yaml

@@ -309,7 +309,7 @@ services:
       retries: 3
 
   electric:
-    image: "electricsql/electric:${ELECTRIC_TAG:-1.0.11}"
+    image: "electricsql/electric:${ELECTRIC_TAG:-1.0.12}"
     depends_on:
       proxy:
         condition: service_started
@@ -331,6 +331,7 @@ services:
 
   central-db:
     image: "ghcr.io/hotosm/postgis:${POSTGIS_TAG:-14-3.5-alpine}"
+    command: -c 'max_connections=300'
     volumes:
       - central_db_data:/var/lib/postgresql/data/
     environment:

deploy/compose.sub.yaml

@@ -309,7 +309,7 @@ services:
       retries: 3
 
   electric:
-    image: "electricsql/electric:${ELECTRIC_TAG:-1.0.11}"
+    image: "electricsql/electric:${ELECTRIC_TAG:-1.0.12}"
     depends_on:
       proxy:
         condition: service_started
@@ -331,6 +331,7 @@ services:
 
   central-db:
     image: "ghcr.io/hotosm/postgis:${POSTGIS_TAG:-14-3.5-alpine}"
+    command: -c 'max_connections=300'
     volumes:
       - central_db_data:/var/lib/postgresql/data/
     environment:

docs/manuals/mapping.md

@@ -220,6 +220,30 @@ Sometimes the feature does not exist on the map yet!
       In a future release, this process will be handled seamlessly
       without user interaction.
 
+## Working Offline
+
+It is possible to work entirely offline, once FieldTM has been loaded
+while internet is available.
+
+To achieve this:
+
+- Visit [https://mapper.fmtm.hotosm.org](https://mapper.fmtm.hotosm.org)
+  once from the phone you wish to map with.
+  - This will cache the 12 most recent projects on your device.
+  - To cache a different set of projects, use the search or
+    next/previous page functionality.
+- For best results, 'Install' the webpage on your device, creating
+  a shortcut on your home screen.
+- Click on the project you wish to map offline.
+  - The project details will be stored entirely offline.
+  - You may also go to the 'Offline' tab, to download additional
+    offline data such as base map imagery.
+- Upon opening the installed app, you should be able to browse
+  the cached versions of your projects (note they will not update
+  while offline).
+- Submissions can be made while offline & they will since when
+  connectivity is restored.
+
 ## Upcoming improvements
 
 Refer the milestone: <https://github.com/hotosm/fmtm/milestone/49>

src/Dockerfile.ui.debug

@@ -18,7 +18,7 @@ RUN npm install -g [email protected]
 # FIXME delete this line after fixed upstream
 RUN corepack enable && corepack install
 RUN pnpm install
-ENTRYPOINT ["pnpm", "run", "dev"]
+CMD ["pnpm", "run", "dev"]
 
 
 # Test code for initialising PGLite db and creating tar dump

src/backend/app/config.py

@@ -367,6 +367,22 @@ def monitoring_config(self) -> Optional[OpenObserveSettings | SentrySettings]:
             return OpenObserveSettings()
         return None
 
+    # SMTP Configurations
+    SMTP_TLS: bool = True
+    SMTP_SSL: bool = False
+    SMTP_PORT: int = 587
+    SMTP_HOST: Optional[str] = None
+    SMTP_USER: Optional[str] = None
+    SMTP_PASSWORD: Optional[str] = None
+    EMAILS_FROM_EMAIL: Optional[str] = None
+    EMAILS_FROM_NAME: Optional[str] = "Field-TM"
+
+    @computed_field
+    @property
+    def emails_enabled(self) -> bool:
+        """Check if email settings are configured."""
+        return bool(self.SMTP_HOST and self.EMAILS_FROM_EMAIL)
+
 
 @lru_cache
 def get_settings():

src/backend/app/helpers/helper_routes.py

@@ -19,12 +19,14 @@
 
 import csv
 import json
+from email.message import EmailMessage
 from io import BytesIO, StringIO
 from pathlib import Path
 from textwrap import dedent
 from typing import Annotated
 from uuid import uuid4
 
+import aiosmtplib
 import requests
 from fastapi import (
     APIRouter,
@@ -333,3 +335,32 @@ async def send_test_osm_message(
         return HTTPException(status_code=HTTPStatus.CONFLICT, detail=msg)
 
     return Response(status_code=HTTPStatus.OK)
+
+
+@router.post("/send-test-email")
+async def send_test_email():
+    """Sends a test email using real SMTP settings."""
+    if not settings.emails_enabled:
+        raise HTTPException(
+            status_code=HTTPStatus.NOT_IMPLEMENTED,
+            detail="An SMTP server has not been configured.",
+        )
+    message = EmailMessage()
+    message["Subject"] = "Test email from Field-TM"
+    message.set_content("This is a test email sent from Field-TM.")
+
+    log.info("Sending test email to recipients")
+    await aiosmtplib.send(
+        message,
+        sender=settings.SMTP_USER,
+        # NOTE this is a test email, so use your own email address to receive it
+        recipients=[],  # List of recipient email addresses
+        hostname=settings.SMTP_HOST,
+        port=settings.SMTP_PORT,
+        username=settings.SMTP_USER,
+        password=settings.SMTP_PASSWORD,
+    )
+
+    return Response(
+        status_code=HTTPStatus.OK,
+    )

src/backend/app/users/user_crud.py

@@ -19,9 +19,12 @@
 
 import os
 from datetime import datetime, timedelta, timezone
+from email.message import EmailMessage
 from textwrap import dedent
 from typing import Literal, Optional
 
+import aiosmtplib
+import markdown
 from fastapi import Request
 from fastapi.exceptions import HTTPException
 from loguru import logger as log
@@ -217,41 +220,78 @@ async def send_warning_email_or_osm(
     log.info(f"Sent warning to {username}: {days_remaining} days remaining.")
 
 
-async def send_invitation_osm_message(
+async def send_mail(
+    user_email: str,
+    title: str,
+    message_content: str,
+):
+    """Sends an email."""
+    if not settings.emails_enabled:
+        log.info("An SMTP server has not been configured.")
+        return
+    message = EmailMessage()
+    message["Subject"] = title
+    message.set_content(message_content)
+    html_content = markdown.markdown(message_content)
+    message.add_alternative(html_content, subtype="html")
+
+    await aiosmtplib.send(
+        message,
+        sender=settings.SMTP_USER,
+        recipients=[user_email],
+        hostname=settings.SMTP_HOST,
+        port=settings.SMTP_PORT,
+        username=settings.SMTP_USER,
+        password=settings.SMTP_PASSWORD,
+    )
+
+
+async def send_invitation_message(
     request: Request,
     project: DbProject,
     invitee_username: str,
     osm_auth: Auth,
     invite_url: str,
+    user_email: str,
+    signin_type: str,
 ):
     """Send an invitation message to a user to join a project."""
-    log.info(f"Sending invitation message to osm user ({invitee_username}).")
-
-    osm_token = get_osm_token(request, osm_auth)
-
     project_url = f"{settings.FMTM_DOMAIN}/project/{project.id}"
     if not project_url.startswith("http"):
         project_url = f"https://{project_url}"
 
+    title = f"You have been invited to join the project {project.name}"
     message_content = dedent(f"""
         You have been invited to join the project **{project.name}**.
 
         To accept the invitation, please click the link below:
         [Accept Invitation]({invite_url})
 
-        You may use this link after accepting the invitation to view the project:
+        You may use this link after accepting the invitation to view the project if you
+        have access:
         [Project]({project_url})
 
         Thank you for being a part of our platform!
     """)
 
-    send_osm_message(
-        osm_token=osm_token,
-        osm_username=invitee_username,
-        title=f"You have been invited to join the project {project.name}",
-        body=message_content,
-    )
-    log.info(f"Invitation message sent to osm user ({invitee_username}).")
+    if signin_type == "osm":
+        osm_token = get_osm_token(request, osm_auth)
+
+        send_osm_message(
+            osm_token=osm_token,
+            osm_username=invitee_username,
+            title=title,
+            body=message_content,
+        )
+        log.info(f"Invitation message sent to osm user ({invitee_username}).")
+
+    elif signin_type == "google":
+        await send_mail(
+            user_email=user_email,
+            title=title,
+            message_content=message_content,
+        )
+        log.info(f"Invitation message sent to email user ({user_email}).")
 
 
 async def get_paginated_users(

src/backend/app/users/user_routes.py

@@ -51,7 +51,7 @@
 from app.users.user_crud import (
     get_paginated_users,
     process_inactive_users,
-    send_invitation_osm_message,
+    send_invitation_message,
 )
 from app.users.user_deps import get_user
 
@@ -145,12 +145,12 @@ async def invite_new_user(
     (e.g. mobile message).
     """
     project = project_user_dict.get("project")
-    osm_user_exists = False
 
     if user_in.osm_username:
-        if osm_user_exists := await check_osm_user(user_in.osm_username):
+        if await check_osm_user(user_in.osm_username):
             username = user_in.osm_username
             signin_type = "osm"
+            domain = settings.FMTM_DOMAIN
         else:
             raise HTTPException(
                 status_code=HTTPStatus.NOT_FOUND,
@@ -159,6 +159,9 @@ async def invite_new_user(
     elif user_in.email:
         username = user_in.email.split("@")[0]
         signin_type = "google"
+        # We use different domain for non OSM users since they can't access the
+        # management interface
+        domain = f"mapper.{settings.FMTM_DOMAIN}"
     else:
         raise HTTPException(
             status_code=HTTPStatus.BAD_REQUEST,
@@ -178,36 +181,26 @@ async def invite_new_user(
             },
         )
 
+    # Generate invite URL
     new_invite = await DbUserInvite.create(db, project.id, user_in)
 
-    # Generate invite URL
-    # TODO create frontend page to handle /invite
-    # TODO save token from URL in localStorage
-    # TODO ask user to login to Field-TM first, present options
-    # TODO once logged in and redirected back to frontend
-    # TODO read the localStorage `invite` key, and call the
-    # TODO /users/invite/{token} endpoint.
     if settings.DEBUG:
         invite_url = (
-            f"http://{settings.FMTM_DOMAIN}:{settings.FMTM_DEV_PORT}"
-            f"/invite?token={new_invite.token}"
+            f"http://{domain}:{settings.FMTM_DEV_PORT}/invite?token={new_invite.token}"
         )
     else:
-        invite_url = f"https://{settings.FMTM_DOMAIN}/invite?token={new_invite.token}"
-
-    # Notify via OSM message
-    if osm_user_exists:
-        background_tasks.add_task(
-            send_invitation_osm_message,
-            request=request,
-            project=project,
-            invitee_username=username,
-            osm_auth=osm_auth,
-            invite_url=invite_url,
-        )
-
-    # TODO Notify via email (consider options)
-
+        invite_url = f"https://{domain}/invite?token={new_invite.token}"
+
+    background_tasks.add_task(
+        send_invitation_message,
+        request=request,
+        project=project,
+        invitee_username=username,
+        osm_auth=osm_auth,
+        invite_url=invite_url,
+        user_email=user_in.email,
+        signin_type=signin_type,
+    )
     return JSONResponse(status_code=HTTPStatus.OK, content={"invite_url": invite_url})