From a6a37fdc9f721db5fb538e2013118b70494dca74 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Wed, 15 Jan 2025 22:47:12 +0100 Subject: [PATCH 01/14] Ad .vscode folders to .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index ac95efe..12f0eec 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,7 @@ build* # Dummy folder with all gym recordings and metadata /dummy + +#.vscode folders +/cpp/.vscode/* +/.vscode/* From 31b0c04bc5f8abc1c2a4467abf4cae3b6916c79d Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Sun, 19 Jan 2025 15:37:39 +0100 Subject: [PATCH 02/14] Import gymnasium instead of gym This change is needed since gym is no longer actively mantained by open ai and it has been moved to Farama-Foundation "gymnasium" --- python/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/server.py b/python/server.py index 58f5fc3..75f5376 100644 --- a/python/server.py +++ b/python/server.py @@ -6,8 +6,8 @@ from _thread import * import glob -import gym -from gym.wrappers import RecordEpisodeStatistics +import gymnasium as gym +from gymnasium.wrappers import RecordEpisodeStatistics try: import zlib From 02464a66e0affbb1a8e00955de97fc1012540858 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Sun, 19 Jan 2025 15:43:44 +0100 Subject: [PATCH 03/14] Unpack obs, info to adapt to reset API change The gymnasium env.reset method now returns two values: the observation and some additional informations --- python/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/server.py b/python/server.py index 75f5376..d4a396b 100644 --- a/python/server.py +++ b/python/server.py @@ -90,7 +90,7 @@ def create(self, env_id): def reset(self, instance_id): env = self._lookup_env(instance_id) - obs = env.reset() + obs, _ = env.reset() return env.observation_space.to_jsonable(obs) def step(self, instance_id, action, render): From 4b4db81151fae8ea616870ded2746b496490c7f1 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Sun, 19 Jan 2025 15:56:31 +0100 Subject: [PATCH 04/14] Fix typo --- python/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/server.py b/python/server.py index d4a396b..c1e32e4 100644 --- a/python/server.py +++ b/python/server.py @@ -30,7 +30,7 @@ except socket.error as e: print(str(e)) -print('Waitiing for a Connection..') +print('Waiting for a Connection..') ServerSocket.listen(5) From 147bd8337e14e38f59059921dc728e57edc8e086 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Sun, 19 Jan 2025 16:52:32 +0100 Subject: [PATCH 05/14] Pass action as a list The new action_space.from_jsonable api takes a list as parameter --- python/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/server.py b/python/server.py index c1e32e4..4f3ccc6 100644 --- a/python/server.py +++ b/python/server.py @@ -95,7 +95,7 @@ def reset(self, instance_id): def step(self, instance_id, action, render): env = self._lookup_env(instance_id) - action_from_json = env.action_space.from_jsonable(action) + action_from_json = env.action_space.from_jsonable([action]) if (not isinstance(action_from_json, (list))): action_from_json = int(action_from_json) From 4dd546cfef25de3d1a24af8ac23c7e4c43a6d58d Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Sun, 19 Jan 2025 20:40:09 +0100 Subject: [PATCH 06/14] Add render mode to cpp client and environment --- cpp/environment.hpp | 5 +++-- cpp/environment_impl.hpp | 9 +++++---- cpp/example.cpp | 3 ++- cpp/messages.hpp | 4 ++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cpp/environment.hpp b/cpp/environment.hpp index d97657b..9144509 100644 --- a/cpp/environment.hpp +++ b/cpp/environment.hpp @@ -71,7 +71,8 @@ class Environment */ Environment(const std::string& host, const std::string& port, - const std::string& environment); + const std::string& environment, + const std::string& render_mode); /* * Instantiate the environment object using the specified environment name. @@ -79,7 +80,7 @@ class Environment * @param environment Name of the environments used to train/evaluate * the model. */ - void make(const std::string& environment); + void make(const std::string& environment, const std::string& render_mode); /* * Renders the environment. diff --git a/cpp/environment_impl.hpp b/cpp/environment_impl.hpp index 83f3938..b7053b4 100644 --- a/cpp/environment_impl.hpp +++ b/cpp/environment_impl.hpp @@ -27,16 +27,17 @@ inline Environment::Environment(const std::string& host, const std::string& port inline Environment::Environment( const std::string& host, const std::string& port, - const std::string& environment) : + const std::string& environment, + const std::string& render_mode="") : renderValue(false) { client.connect(host, port); - make(environment); + make(environment, render_mode); } -inline void Environment::make(const std::string& environment) +inline void Environment::make(const std::string& environment, const std::string& render_mode=nullptr) { - client.send(messages::EnvironmentName(environment)); + client.send(messages::EnvironmentName(environment, render_mode)); std::string json; client.receive(json); diff --git a/cpp/example.cpp b/cpp/example.cpp index ecc9666..3d1adf4 100644 --- a/cpp/example.cpp +++ b/cpp/example.cpp @@ -16,11 +16,12 @@ int main(int argc, char* argv[]) const std::string environment = "CartPole-v1"; const std::string host = "127.0.0.1"; const std::string port = "4040"; + const std::string render_mode = "rgb_array"; double totalReward = 0; size_t totalSteps = 0; - Environment env(host, port, environment); + Environment env(host, port, environment, render_mode); env.compression(9); env.record_episode_stats.start(); diff --git a/cpp/messages.hpp b/cpp/messages.hpp index 15b4305..dc52993 100644 --- a/cpp/messages.hpp +++ b/cpp/messages.hpp @@ -16,9 +16,9 @@ namespace gym { namespace messages { //! Create message to set the enviroment name. -static inline std::string EnvironmentName(const std::string& name) +static inline std::string EnvironmentName(const std::string& name, const std::string& render_mode) { - return "{\"env\":{\"name\": \"" + name + "\"}}"; + return "{\"env\":{\"name\": \"" + name + "\", \"render_mode\": \"" + render_mode + "\"}}"; } //! Create message to reset the enviroment. From 4ca8e7d98a75eb9f1eac9ebc4568831517588ba5 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Sun, 19 Jan 2025 20:40:49 +0100 Subject: [PATCH 07/14] Receive and use render mode in the server --- python/server.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/server.py b/python/server.py index 4f3ccc6..e631d27 100644 --- a/python/server.py +++ b/python/server.py @@ -77,9 +77,9 @@ def _remove_env(self, instance_id): except KeyError: raise InvalidUsage('Instance_id {} unknown'.format(instance_id)) - def create(self, env_id): + def create(self, env_id, render_mode=None): try: - env = gym.make(env_id) + env = gym.make(env_id, render_mode=render_mode) except gym.error.Error: raise InvalidUsage( "Attempted to look up malformed environment ID '{}'".format(env_id)) @@ -264,12 +264,15 @@ def process_response(response, connection, envs, enviroment, instance_id, close, print(jsonMessage) enviroment = get_optional_params(jsonMessage, "env", "name") + render_mode = get_optional_params(jsonMessage, "env", "render_mode") + render_mode = render_mode if render_mode != "" else None + if isinstance(enviroment, basestring): compressionLevel = 0 if instance_id != None: envs.env_close(instance_id) - instance_id = envs.create(enviroment) + instance_id = envs.create(enviroment, render_mode) data = json.dumps({"instance" : instance_id}, cls = NDArrayEncoder) connection.send(process_data(data, compressionLevel)) return enviroment, instance_id, close, compressionLevel From a48211cdea691bf3de0b145541e1f574f0acdc93 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Wed, 22 Jan 2025 23:55:47 +0100 Subject: [PATCH 08/14] Use human render mode --- cpp/example.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/example.cpp b/cpp/example.cpp index 3d1adf4..1ddc509 100644 --- a/cpp/example.cpp +++ b/cpp/example.cpp @@ -16,7 +16,7 @@ int main(int argc, char* argv[]) const std::string environment = "CartPole-v1"; const std::string host = "127.0.0.1"; const std::string port = "4040"; - const std::string render_mode = "rgb_array"; + const std::string render_mode = "human"; double totalReward = 0; size_t totalSteps = 0; From 0dfb7ca0b4d09a2f7463e5c488272efe891d4885 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Thu, 23 Jan 2025 00:00:30 +0100 Subject: [PATCH 09/14] Adapt to the gymnasium API --- python/server.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/server.py b/python/server.py index e631d27..fa00816 100644 --- a/python/server.py +++ b/python/server.py @@ -100,7 +100,7 @@ def step(self, instance_id, action, render): action_from_json = int(action_from_json) if render: env.render() - [observation, reward, done, info] = env.step(action_from_json) + [observation, reward, done, _, info] = env.step(action_from_json[0]) obs_jsonable = env.observation_space.to_jsonable(observation) return [obs_jsonable, reward, done, info] @@ -324,9 +324,7 @@ def process_response(response, connection, envs, enviroment, instance_id, close, render = True if (render is not None and render == 1) else False - [obs, reward, done, info] = envs.step( - instance_id, action, render) - + [obs, reward, done, info] = envs.step(instance_id, action, render) data = json.dumps({"observation" : obs, "reward" : reward, "done" : done, @@ -376,4 +374,3 @@ def process_response(response, connection, envs, enviroment, instance_id, close, start_new_thread(threaded_client, (Client, )) ThreadCount += 1 print('Thread Number: ' + str(ThreadCount)) -ServerSocket.close() \ No newline at end of file From d4e92603fdb6c632125c22fdd5a650fe655a6c38 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Thu, 23 Jan 2025 00:01:59 +0100 Subject: [PATCH 10/14] Run main loop without threading This is needed for the server to run on MacOS. Otherwise rendering doesn't happen on the main-thread, which is not allowed on MacOs. --- python/server.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/server.py b/python/server.py index fa00816..0dff8d7 100644 --- a/python/server.py +++ b/python/server.py @@ -368,9 +368,19 @@ def process_response(response, connection, envs, enviroment, instance_id, close, return connection.close() +# while True: +# Client, address = ServerSocket.accept() +# print('Connected to: ' + address[0] + ':' + str(address[1])) +# start_new_thread(threaded_client, (Client, )) +# ThreadCount += 1 +# print('Thread Number: ' + str(ThreadCount)) +# ServerSocket.close() + + while True: Client, address = ServerSocket.accept() print('Connected to: ' + address[0] + ':' + str(address[1])) - start_new_thread(threaded_client, (Client, )) ThreadCount += 1 print('Thread Number: ' + str(ThreadCount)) + threaded_client(Client) # Directly call the function without threading +ServerSocket.close() \ No newline at end of file From aeef06b31732ef9439ba2885117ac85af9365e24 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Thu, 23 Jan 2025 23:47:37 +0100 Subject: [PATCH 11/14] Update readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3c789fe..ecc899b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # gym-tcp-api -This project provides a distributed infrastructure (TCP API) to the OpenAI Gym toolkit, allowing development in languages other than python. +This project provides a distributed infrastructure (TCP API) to the Farama-Foundation Gymnasium toolkit, allowing development in languages other than python. The server is written in elixir, enabling a distributed infrastructure. Where each node makes use of a limitted set of processes that can be used to perform time consuming tasks (4 python instances per default). @@ -20,7 +20,7 @@ The server has the following dependencies: Python3 Elixir >= 1.0 - OpenAI Gym. + Farama-Foundation Gymnasium. The c++ example agent has the following dependencies: From abbd03476b7794553b168c299fca9f2af3c6d350 Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Thu, 23 Jan 2025 23:49:25 +0100 Subject: [PATCH 12/14] Add comment --- python/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/server.py b/python/server.py index 0dff8d7..8ef6b9c 100644 --- a/python/server.py +++ b/python/server.py @@ -377,6 +377,8 @@ def process_response(response, connection, envs, enviroment, instance_id, close, # ServerSocket.close() +# This is needed to make it work on MacOS. In fact, MacOS doesn't allow to rendere on threads +# other than the main thread while True: Client, address = ServerSocket.accept() print('Connected to: ' + address[0] + ':' + str(address[1])) From 275513a3f3248c48509370ca032abdf25ddaf78d Mon Sep 17 00:00:00 2001 From: Andrea Novellini Date: Sat, 1 Feb 2025 16:29:39 +0100 Subject: [PATCH 13/14] Update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ecc899b..395d32a 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ We use JSON as the format to cimmunicate with the server. Create the specified environment: - {"env" {"name": "CartPole-v0"}} + {"env" {"name": "CartPole-v0", "render_mode": "human"}} Close the environment: From bab7dbe3f715b0cd04ad6f002b8046e71ed25fe7 Mon Sep 17 00:00:00 2001 From: geonove Date: Sat, 1 Mar 2025 11:51:21 +0100 Subject: [PATCH 14/14] Handle game loop based on OS --- python/server.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/python/server.py b/python/server.py index 8ef6b9c..d5e9afa 100644 --- a/python/server.py +++ b/python/server.py @@ -3,6 +3,7 @@ import numpy as np import socket import os +import platform from _thread import * import glob @@ -368,21 +369,23 @@ def process_response(response, connection, envs, enviroment, instance_id, close, return connection.close() -# while True: -# Client, address = ServerSocket.accept() -# print('Connected to: ' + address[0] + ':' + str(address[1])) -# start_new_thread(threaded_client, (Client, )) -# ThreadCount += 1 -# print('Thread Number: ' + str(ThreadCount)) -# ServerSocket.close() - - -# This is needed to make it work on MacOS. In fact, MacOS doesn't allow to rendere on threads -# other than the main thread -while True: - Client, address = ServerSocket.accept() - print('Connected to: ' + address[0] + ':' + str(address[1])) - ThreadCount += 1 - print('Thread Number: ' + str(ThreadCount)) - threaded_client(Client) # Directly call the function without threading -ServerSocket.close() \ No newline at end of file + +if __name__ == "__main__": + if platform.system() == 'Darwin': + # This is needed to make it work on MacOS. In fact, MacOS doesn't allow to rendere on threads + # other than the main thread + while True: + Client, address = ServerSocket.accept() + print('Connected to: ' + address[0] + ':' + str(address[1])) + ThreadCount += 1 + print('Thread Number: ' + str(ThreadCount)) + threaded_client(Client) # Directly call the function without threading + ServerSocket.close() + else: + while True: + Client, address = ServerSocket.accept() + print('Connected to: ' + address[0] + ':' + str(address[1])) + start_new_thread(threaded_client, (Client, )) + ThreadCount += 1 + print('Thread Number: ' + str(ThreadCount)) + ServerSocket.close() \ No newline at end of file