Migrate from Pydantic V1 to V2 (#293)

* Fix: Project do not use the good version of aleph message and sdk

Aleph-message and sdk now uses pydantic v2 but no released yet
Using the github link atm and wait for the release.

* Fix: Replacing deprecated function after pydantic migration

When upgrading the pydantic version, some function are / will become
deprecated. replacing them in those files.

* feat: Adding custom encoding for JSON serialization

Added `default=extended_json_encoder` to `json.dumps()` to handle serialization
of non-native JSON types, such as dates and other complex objects.

Replacing function that are / will be deprecated in pydantic v2 and v3

* Fix: Conflit between pydantic and fastapi version

Pydantic>=2 is not compatible with fastapi<0.100.0

* Fix: Lint test did not pass and conflict between dependencies

Conflit between pydantic and yamlfix
black did not pass

* fix: Wrong aleph-message, aleph-python-sdk version

* fix: Pydantic 2 being more strict about type validation

With Pydantic v2, strict type validation is enforced. The type field is expected
to have the exact value "POST" as a literal.

* change version of typing_extension

* change version of typing_extension

* style: ymlfix

* style: pyproject-fmt

* style: ruff

- Unused varible deleted

* Fix: Functions are deprecated because of the migration to pydantic2

- Replacing parse_obj() by model_validate()
- Replacing copy() by model_copy()
- Replacing dict() by model_dump()

* fix: Warning about dict() being deprecated

* Fix: We can't access size_mib about Ephemeral and Persistant class because of some changes on aleph_message

Before any change, there were classes dedicated to the size of Ephemeral and Persistent class such as
PersistentVolumeSizeMb and EphemeralVolumeSizeMb.
Now, the size is included inside the classes EphemeralVolume and PersistentVolume so have to access the size
by another method.

- Creating a function to get the max and min size, that way it is more rigorous
- Calling this function to get max and min size of the different volumes

* style: mypy raise errors about unexpected arguments

- Refactor the Dict() class inside the mocks test file to allow any key-value pair to be
passed to the model.

* fix: Missing substrate-interface depedency

* Fix: Reverting some changes I made

Arnaud made some changes about getting hashes that were better but I deleted by mistakes

Putting back his code

* Fix: aleph-message version conflicts between aleph-client and aleph-sdk

* Fix: Make changes from Andres review

Reverting changes about accessing data of PersistentVolumeSizeMib and EphemeralVolumeSize
- creating a function to access annotated type because we can no longer do Persistent.le to get the data

* Fix: Solved last issues on some commands and refactored the method to get the annotation rules.

* Fix: Update the SDK dependency to the last 2.0.0 version.

---------

Co-authored-by: Andres D. Molins <[email protected]>

Author: Antonyjin

pyproject.toml

@@ -30,16 +30,17 @@ dynamic = [ "version" ]
 dependencies = [
   "aiodns==3.2",
   "aiohttp==3.11.13",
-  "aleph-message>=0.6.1",
-  "aleph-sdk-python @ git+https://github.com/aleph-im/aleph-sdk-python",
-  "base58==2.1.1",                                                       # Needed now as default with _load_account changement
-  "py-sr25519-bindings==0.2",                                            # Needed for DOT signatures
+  "aleph-message>=1",
+  "aleph-sdk-python>=2,<3",
+  "base58==2.1.1",               # Needed now as default with _load_account changement
+  "py-sr25519-bindings==0.2",    # Needed for DOT signatures
+  "pydantic>=2",
   "pygments==2.19.1",
-  "pynacl==1.5",                                                         # Needed now as default with _load_account changement
+  "pynacl==1.5",                 # Needed now as default with _load_account changement
   "python-magic==0.4.27",
   "rich==13.9.*",
   "setuptools>=65.5",
-  "substrate-interface==1.7.11",                                         # Needed for DOT signatures
+  "substrate-interface==1.7.11", # Needed for DOT signatures
   "textual==0.73",
   "typer==0.15.2",
 ]
@@ -80,7 +81,7 @@ dependencies = [
   "pytest-cov==5.0.0",
   "mypy==1.10.0",
   "base58==2.1.1",
-  "fastapi==0.98.0",
+  "fastapi==0.100.0",
   "httpx==0.27.0",
   "types-requests==2.32.0.20240602",
   "types-setuptools==70.0.0.20240524",
@@ -96,7 +97,7 @@ dependencies = [
   "pytest-cov==5.0.0",
   "mypy==1.10.0",
   "base58==2.1.1",
-  "fastapi==0.98.0",
+  "fastapi==0.100.0",
   "httpx==0.27.0",
 ]
 [tool.hatch.envs.testing.scripts]
@@ -119,7 +120,7 @@ dependencies = [
   "mypy==1.10.0",
   "ruff==0.9.*",
   "isort==5.13.2",
-  "yamlfix==1.16.1",
+  "yamlfix==1.17.0",
   "pyproject-fmt==2.2.1",
 
   "types-requests==2.32.0.20240602",

src/aleph_client/commands/account.py

@@ -267,7 +267,7 @@ def sign_bytes(
     assert message is not None  # to please mypy
     coroutine = account.sign_raw(str(message).encode())
     signature = asyncio.run(coroutine)
-    typer.echo("\nSignature: " + signature.hex())
+    typer.echo("\nSignature: " + f"0x{signature.hex()}")
 
 
 async def get_balance(address: str) -> dict:

src/aleph_client/commands/aggregate.py

@@ -88,7 +88,7 @@ async def forget(
             inline=inline,
             address=address,
         )
-        dumped_content = f"{message.json(indent=4)}"
+        dumped_content = f"{message.model_dump_json(indent=4)}"
 
         if status != MessageStatus.REJECTED:
             if print_message:
@@ -159,7 +159,7 @@ async def post(
             inline=inline,
             address=address,
         )
-        content = f"{message.json(indent=4)}"
+        content = f"{message.model_dump_json(indent=4)}"
 
         if status != MessageStatus.REJECTED:
             if print_message:

src/aleph_client/commands/files.py

@@ -55,7 +55,7 @@ async def pin(
             ref=ref,
         )
         logger.debug("Upload finished")
-        typer.echo(f"{result.json(indent=4)}")
+        typer.echo(f"{result.model_dump_json(indent=4)}")
 
 
 @app.command()
@@ -96,7 +96,7 @@ async def upload(
                 ref=ref,
             )
             logger.debug("Upload finished")
-            typer.echo(f"{result.json(indent=4)}")
+            typer.echo(f"{result.model_dump_json(indent=4)}")
 
 
 @app.command()
@@ -170,7 +170,7 @@ async def forget(
 
     async with AuthenticatedAlephHttpClient(account=account, api_server=settings.API_HOST) as client:
         value = await client.forget(hashes=hashes, reason=reason, channel=channel)
-        typer.echo(f"{value[0].json(indent=4)}")
+        typer.echo(f"{value[0].model_dump_json(indent=4)}")
 
 
 class GetAccountFilesQueryParams(BaseModel):
@@ -264,7 +264,7 @@ async def list_files(
 
         uri = f"{settings.API_HOST}/api/v0/addresses/{address}/files"
         async with aiohttp.ClientSession() as session:
-            response = await session.get(uri, params=query_params.dict())
+            response = await session.get(uri, params=query_params.model_dump())
             if response.status == 200:
                 files_data = await response.json()
                 formatted_files_data = json_lib.dumps(files_data, indent=4)

src/aleph_client/commands/instance/__init__.py

@@ -71,6 +71,7 @@
     filter_only_valid_messages,
     find_sevctl_or_exit,
     found_gpus_by_model,
+    get_annotated_constraint,
     get_or_prompt_volumes,
     setup_logging,
     str_to_datetime,
@@ -91,6 +92,7 @@
 metavar_valid_chains = f"[{'|'.join(hold_chains)}]"
 super_token_chains = get_chains_with_super_token()
 metavar_valid_payg_chains = f"[{'|'.join(super_token_chains)}]"
+max_persistent_volume_size = get_annotated_constraint(PersistentVolumeSizeMib, "le")
 
 
 @app.command()
@@ -119,7 +121,7 @@ async def create(
     vcpus: Annotated[Optional[int], typer.Option(help=help_strings.VCPUS)] = None,
     memory: Annotated[Optional[int], typer.Option(help=help_strings.MEMORY)] = None,
     rootfs_size: Annotated[
-        Optional[int], typer.Option(help=help_strings.ROOTFS_SIZE, max=PersistentVolumeSizeMib.le)
+        Optional[int], typer.Option(help=help_strings.ROOTFS_SIZE, max=max_persistent_volume_size)
     ] = None,
     timeout_seconds: Annotated[float, typer.Option(help=help_strings.TIMEOUT_SECONDS)] = settings.DEFAULT_VM_TIMEOUT,
     ssh_pubkey_file: Annotated[Path, typer.Option(help=help_strings.SSH_PUBKEY_FILE)] = Path(
@@ -341,7 +343,10 @@ async def create(
     disk_size_info = f"Rootfs Size: {round(disk_size/1024, 2)} GiB (defaulted to included storage in tier)"
     if not isinstance(rootfs_size, int):
         rootfs_size = validated_int_prompt(
-            "Custom Rootfs Size (MiB)", min_value=disk_size, max_value=PersistentVolumeSizeMib.le, default=disk_size
+            "Custom Rootfs Size (MiB)",
+            min_value=disk_size,
+            max_value=max_persistent_volume_size,
+            default=disk_size,
         )
     if rootfs_size > disk_size:
         disk_size = rootfs_size
@@ -551,7 +556,7 @@ async def create(
             echo(f"Instance creation failed:\n{e}")
             raise typer.Exit(code=1) from e
         if print_message:
-            echo(f"{message.json(indent=4)}")
+            echo(f"{message.model_dump_json(indent=4)}")
 
         item_hash: ItemHash = message.item_hash
         infos = []
@@ -796,7 +801,7 @@ async def delete(
 
         message, status = await client.forget(hashes=[ItemHash(item_hash)], reason=reason)
         if print_message:
-            echo(f"{message.json(indent=4)}")
+            echo(f"{message.model_dump_json(indent=4)}")
         echo(f"Instance {item_hash} has been deleted.")
 
 
@@ -1018,7 +1023,7 @@ async def list_instances(
             raise typer.Exit(code=1)
         if json:
             for message in messages:
-                echo(message.json(indent=4))
+                echo(message.model_dump_json(indent=4))
         else:
             # Since we filtered on message type, we can safely cast as InstanceMessage.
             messages = cast(builtins.list[InstanceMessage], messages)
@@ -1197,7 +1202,6 @@ async def confidential_init_session(
 
     # Generate sessions certificate files
     if not ((session_dir / "vm_godh.b64").exists() and keep_session):
-
         code, platform_file = await client.get_certificates()
         if code != 200:
             echo(
@@ -1368,7 +1372,7 @@ async def confidential_create(
     vcpus: Annotated[Optional[int], typer.Option(help=help_strings.VCPUS)] = None,
     memory: Annotated[Optional[int], typer.Option(help=help_strings.MEMORY)] = None,
     rootfs_size: Annotated[
-        Optional[int], typer.Option(help=help_strings.ROOTFS_SIZE, max=PersistentVolumeSizeMib.le)
+        Optional[int], typer.Option(help=help_strings.ROOTFS_SIZE, max=max_persistent_volume_size)
     ] = None,
     timeout_seconds: Annotated[float, typer.Option(help=help_strings.TIMEOUT_SECONDS)] = settings.DEFAULT_VM_TIMEOUT,
     ssh_pubkey_file: Annotated[Path, typer.Option(help=help_strings.SSH_PUBKEY_FILE)] = Path(
@@ -1516,7 +1520,7 @@ async def gpu_create(
     vcpus: Annotated[Optional[int], typer.Option(help=help_strings.VCPUS)] = None,
     memory: Annotated[Optional[int], typer.Option(help=help_strings.MEMORY)] = None,
     rootfs_size: Annotated[
-        Optional[int], typer.Option(help=help_strings.ROOTFS_SIZE, max=PersistentVolumeSizeMib.le)
+        Optional[int], typer.Option(help=help_strings.ROOTFS_SIZE, max=max_persistent_volume_size)
     ] = None,
     premium: Annotated[Optional[bool], typer.Option(help=help_strings.GPU_PREMIUM_OPTION)] = None,
     timeout_seconds: Annotated[float, typer.Option(help=help_strings.TIMEOUT_SECONDS)] = settings.DEFAULT_VM_TIMEOUT,

src/aleph_client/commands/message.py

@@ -26,7 +26,6 @@
 from aleph_client.commands import help_strings
 from aleph_client.commands.utils import (
     colorful_json,
-    colorful_message_json,
     colorized_status,
     input_multiline,
     setup_logging,
@@ -55,7 +54,7 @@ async def get(
                 reason = await client.get_message_error(item_hash=ItemHash(item_hash))
                 typer.echo(colorful_json(json.dumps(reason, indent=4)))
             else:
-                typer.echo(colorful_message_json(message))
+                typer.echo(colorful_json(json.dumps(message.model_dump(), indent=4, default=extended_json_encoder)))
 
 
 @app.command()
@@ -109,7 +108,9 @@ async def find(
             ),
             ignore_invalid_messages=ignore_invalid_messages,
         )
-    typer.echo(colorful_json(response.json(sort_keys=True, indent=4)))
+    typer.echo(
+        colorful_json(json.dumps(response.model_dump(), sort_keys=True, indent=4, default=extended_json_encoder))
+    )
 
 
 @app.command()
@@ -165,7 +166,7 @@ async def post(
             storage_engine=storage_engine,
         )
 
-        typer.echo(json.dumps(result.dict(), indent=4, default=extended_json_encoder))
+        typer.echo(json.dumps(result.model_dump(), indent=4, default=extended_json_encoder))
 
 
 @app.command()
@@ -195,7 +196,7 @@ async def amend(
             editor: str = os.getenv("EDITOR", default="nano")
             with tempfile.NamedTemporaryFile(suffix="json") as fd:
                 # Fill in message template
-                fd.write(existing_message.content.json(indent=4).encode())
+                fd.write(existing_message.content.model_dump_json(indent=4).encode())
                 fd.seek(0)
 
                 # Launch editor
@@ -224,7 +225,7 @@ async def amend(
                     message_type=existing_message.type,
                     channel=existing_message.channel,
                 )
-            typer.echo(f"{message.json(indent=4)}")
+            typer.echo(f"{message.model_dump_json(indent=4)}")
 
 
 @app.command()
@@ -273,7 +274,7 @@ async def watch(
             async for message in client.watch_messages(
                 message_filter=MessageFilter(refs=[ref], addresses=[original.content.address])
             ):
-                typer.echo(f"{message.json(indent=indent)}")
+                typer.echo(f"{message.model_dump_json(indent=indent)}")
 
 
 @app.command()

src/aleph_client/commands/program.py

@@ -166,7 +166,7 @@ async def upload(
             )
             logger.debug("Code upload finished")
             if print_messages or print_code_message:
-                typer.echo(f"{user_code.json(indent=4)}")
+                typer.echo(f"{user_code.model_dump_json(indent=4)}")
             program_ref = user_code.item_hash
 
         pricing = await fetch_pricing()
@@ -264,7 +264,7 @@ async def upload(
 
         logger.debug("Program upload finished")
         if print_messages or print_program_message:
-            typer.echo(f"{message.json(indent=4)}")
+            typer.echo(f"{message.model_dump_json(indent=4)}")
 
         item_hash: ItemHash = message.item_hash
         if verbose:
@@ -378,7 +378,7 @@ async def update(
             )
             logger.debug("Code upload finished")
             if print_message:
-                typer.echo(f"{message.json(indent=4)}")
+                typer.echo(f"{message.model_dump_json(indent=4)}")
 
         if verbose:
             hash_base32 = b32encode(b16decode(item_hash.upper())).strip(b"=").lower().decode()
@@ -681,12 +681,12 @@ async def persist(
             return None
 
         # Update content
-        content: ProgramContent = message.content.copy()
+        content: ProgramContent = message.content.model_copy()
         content.on.persistent = True
         content.replaces = message.item_hash
 
         message, _status, _ = await client.submit(
-            content=content.dict(exclude_none=True),
+            content=content.model_dump(exclude_none=True),
             message_type=message.type,
             channel=message.channel,
         )
@@ -778,12 +778,12 @@ async def unpersist(
             return None
 
         # Update content
-        content: ProgramContent = message.content.copy()
+        content: ProgramContent = message.content.model_copy()
         content.on.persistent = False
         content.replaces = message.item_hash
 
         message, _status, _ = await client.submit(
-            content=content.dict(exclude_none=True),
+            content=content.model_dump(exclude_none=True),
             message_type=message.type,
             channel=message.channel,
         )