Edit network config

Author: defaultdino

Pipfile

@@ -15,4 +15,4 @@ transformers = {extras = ["sentencepiece"], version = "*"}
 
 [requires]
 python_version = "3.11"
-python_full_version = "3.11.5"
+python_full_version = "3.11.6"

Pipfile.lock

@@ -18,7 +18,7 @@ services:
     env_file:
       - ./hadoop.env
     networks:
-      - spark-hadoop-network
+      - default
 
   datanode:
     build:
@@ -34,7 +34,7 @@ services:
     env_file:
       - ./hadoop.env
     networks:
-      - spark-hadoop-network
+      - default
 
   resourcemanager:
     build:
@@ -48,7 +48,7 @@ services:
     env_file:
       - ./hadoop.env
     networks:
-      - spark-hadoop-network
+      - default
 
   nodemanager:
     build:
@@ -62,7 +62,7 @@ services:
     env_file:
       - ./hadoop.env
     networks:
-      - spark-hadoop-network
+      - default
 
   historyserver:
     build:
@@ -78,7 +78,7 @@ services:
     env_file:
       - ./hadoop.env
     networks:
-      - spark-hadoop-network
+      - default
 
   spark-master:
     build:
@@ -101,7 +101,7 @@ services:
       - namenode
       - datanode
     networks:
-      - spark-hadoop-network
+      - default
     deploy:
       resources:
         limits:
@@ -126,7 +126,7 @@ services:
     depends_on:
       - spark-master
     networks:
-      - spark-hadoop-network
+      - default
     deploy:
       resources:
         limits:
@@ -139,5 +139,6 @@ volumes:
   hadoop_historyserver:
 
 networks:
-  spark-hadoop-network:
-    driver: bridge
+  default:
+    name: shared-spark-hadoop-network
+    external: true

hadoop-spark-cluster/docker-compose.yml

@@ -40,24 +40,4 @@ MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m
 MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m
 MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
 MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
-MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/
-
-HADOOP_ARCHITECTURE=3.3.6
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/

hadoop-spark-cluster/hadoop.env

@@ -26,7 +26,7 @@ class SparkHadoopSetup:
         Sets up the Spark and Hadoop environment
     """
     def __init__(self):
-        self.network_name = "spark-hadoop-network"
+        self.network_name = "shared-spark-hadoop-network"
         self.hadoop_container_dir = "hadoop-spark-cluster"
 
     def create_spark_hadoop_network(self):