Merge pull request #286 from amdfxlucas/seed-contrib08

support named docker volumes

Author: kevin-w-du

seedemu/compiler/DistributedDocker.py

@@ -1,11 +1,12 @@
 # Distributed Docker Compiler is not maintained
 
 from .Docker import Docker, DockerCompilerFileTemplates
-from seedemu.core import Emulator, ScopedRegistry, Node, Network
+from seedemu.core import Emulator, ScopedRegistry, Node, Network, BaseVolume
 from seedemu.core.enums import NodeRole
 from typing import Dict
 from hashlib import md5
 from os import mkdir, chdir, rmdir
+from yaml import dump
 
 DistributedDockerCompilerFileTemplates: Dict[str, str] = {}
 
@@ -70,6 +71,28 @@ def __compileIxNetWorker(self, net) -> str:
 
     def _doCompile(self, emulator: Emulator):
         registry = emulator.getRegistry()
+
+        self._groupSoftware(emulator)
+
+        toplevelvolumes = '' 
+        if len(topvols := self._getVolumes()) > 0:
+            toplevelvolumes += 'volumes:\n'
+            #topvols = set(map( lambda vol: TopLvlVolume(vol), pool.getVolumes() ))
+
+            for v in  topvols:
+                v.mode = 'toplevel'
+
+            #toplevelvolumes += '\n'.join(map( lambda line: '        ' + line ,dump( topvols ).split('\n') ) ) 
+
+            # sharedFolders/bind mounts do not belong in the top-level volumes section
+            for v in [vv  for  vv in topvols if vv.asDict()['type'] == 'volume' ]: 
+                toplevelvolumes += '  {}:\n'.format(v.asDict()['source']) # why not 'name'
+                lines = dump( v ).rstrip('\n').split('\n')
+                toplevelvolumes += '\n'.join( map( lambda x: '        ' if x[0] != 0 
+                                                else '        ' + x[1] if x[1] != ''
+                                                else '', enumerate(lines ) ) )
+                toplevelvolumes += '\n'
+
         scopes = set()
         for (scope, _, _) in registry.getAll().keys(): scopes.add(scope)
 
@@ -114,6 +137,7 @@ def _doCompile(self, emulator: Emulator):
                 print(DockerCompilerFileTemplates['compose'].format(
                     services = services,
                     networks = networks,
+                    volumes = toplevelvolumes,
                     dummies = self._makeDummies()
                 ), file=open('docker-compose.yml', 'w'))
 

seedemu/compiler/Docker.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from seedemu.core.Emulator import Emulator
-from seedemu.core import Node, Network, Compiler, BaseSystem, BaseOption, Scope, ScopeType, ScopeTier, OptionHandling
+from seedemu.core import Node, Network, Compiler, BaseSystem, BaseOption, Scope, ScopeType, ScopeTier, OptionHandling, BaseVolume
 from seedemu.core.enums import NodeRole, NetworkType
 from .DockerImage import DockerImage
 from .DockerImageConstant import *
@@ -12,7 +12,7 @@
 from ipaddress import IPv4Network, IPv4Address
 from shutil import copyfile
 import json
-
+from yaml import dump
 
 SEEDEMU_INTERNET_MAP_IMAGE='handsonsecurity/seedemu-multiarch-map:buildx-latest'
 SEEDEMU_ETHER_VIEW_IMAGE='handsonsecurity/seedemu-multiarch-etherview:buildx-latest'
@@ -104,6 +104,7 @@
 {services}
 networks:
 {networks}
+{volumes}
 """
 
 DockerCompilerFileTemplates['compose_dummy'] = """\
@@ -159,16 +160,6 @@
 {volumeList}
 """
 
-DockerCompilerFileTemplates['compose_volume'] = """\
-            - type: bind
-              source: {hostPath}
-              target: {nodePath}
-"""
-
-DockerCompilerFileTemplates['compose_storage'] = """\
-            - {nodePath}
-"""
-
 DockerCompilerFileTemplates['compose_service_network'] = """\
             {netId}:
 {address}
@@ -394,6 +385,13 @@ def __init__(
         self._used_images = set()
         self.__image_per_node_list = {}
         self.__config = [] # variables for '.env' file alongside 'docker-compose.yml'
+
+        self.__volumes_dedup = (
+            []
+        )  # unforunately set(()) failed to automatically deduplicate
+        self.__vol_names = []
+        super().__init__()
+
         self.__platform = platform
 
         self.__basesystem_dockerimage_mapping = BASESYSTEM_DOCKERIMAGE_MAPPING_PER_PLATFORM[self.__platform]
@@ -404,6 +402,24 @@ def __init__(
                 priority = 1
             self.addImage(image, priority=priority)
 
+    def _addVolume(self, vol: BaseVolume):
+        """! @brief add a docker volume/bind mount/or tmpfs
+
+        Remember them for later, to generate the top lvl 'volumes:' section of docker-compose.yml
+        """
+        # if vol.type() == 'volume': # then it is a named-volume
+        key = vol.asDict()["source"]
+        if key not in self.__vol_names:
+            self.__volumes_dedup.append(vol)
+            self.__vol_names.append(key)
+        return self
+
+    def _getVolumes(self) -> List[BaseVolume]:
+        """! @brief get all docker volumes/mounts that must appear
+            in docker-compose.yml top-level  'volumes:' section
+        """
+        return self.__volumes_dedup
+
     def optionHandlingCapabilities(self) -> OptionHandling:
         return OptionHandling.DIRECT_DOCKER_COMPOSE | OptionHandling.CREATE_SEPARATE_ENV_FILE
 
@@ -910,28 +926,23 @@ def _getComposeNodeNets(self, node: Node) -> str:
         return node_nets, dummy_addr_map
 
     def _getComposeNodeVolumes(self, node: Node) -> str:
-        _volumes = node.getSharedFolders()
-        storages = node.getPersistentStorages()
+        """ compute the docker-compose 'volumes:' section for this service(emulation node)"""
 
         volumes = ''
+        # svcvols = map( lambda vol: ServiceLvlVolume(vol), node.getCustomVolumes() )
+        svcvols = list (set(node.getDockerVolumes() ))
+        for v in svcvols:
+            v.mode = 'service'
+        yamlvols = '\n'.join(map( lambda line: '        ' + line ,dump( svcvols ).split('\n') ) )
 
-        if len(_volumes) > 0 or len(storages) > 0:
-            lst = ''
+        volumes +='        volumes:\n' + yamlvols if len(node.getDockerVolumes()) > 0 else   ''
 
-            for (nodePath, hostPath) in _volumes.items():
-                lst += DockerCompilerFileTemplates['compose_volume'].format(
-                    hostPath = hostPath,
-                    nodePath = nodePath
-                )
 
-            for path in storages:
-                lst += DockerCompilerFileTemplates['compose_storage'].format(
-                    nodePath = path
-                )
+        # the top-level docker-compose volumes section is rendered at a later stage ..
+        # Remember encountered volumes until then
+        for v in node.getDockerVolumes():
+            self._addVolume(v)
 
-            volumes = DockerCompilerFileTemplates['compose_volumes'].format(
-                volumeList = lst
-            )
         return volumes
 
     def _computeDockerfile(self, node: Node) -> str:
@@ -1260,11 +1271,41 @@ def _doCompile(self, emulator: Emulator):
                 dirName = image.getDirName()
             )
 
+        toplevelvolumes = self._computeComposeTopLvlVolumes()
+
         self._log('creating docker-compose.yml...'.format(scope, name))
         print(DockerCompilerFileTemplates['compose'].format(
             services = self.__services,
             networks = self.__networks,
+            volumes = toplevelvolumes,
             dummies = local_images + self._makeDummies()
         ), file=open('docker-compose.yml', 'w'))
 
         self.generateEnvFile(Scope(ScopeTier.Global),'')
+
+    def _computeComposeTopLvlVolumes(self) -> str:
+        """!@brief render the 'volumes:' section of the docker-compose.yml file
+        It contains named volumes but not bind-mounts.
+        """
+        toplevelvolumes = ''
+        if len(topvols := self._getVolumes()) > 0:
+            hit = False
+            #topvols = set(map( lambda vol: TopLvlVolume(vol), pool.getVolumes() ))
+
+            for v in  topvols:
+                v.mode = 'toplevel'
+
+            #toplevelvolumes += '\n'.join(map( lambda line: '        ' + line ,dump( topvols ).split('\n') ) )
+
+            # sharedFolders/bind mounts do not belong in the top-level volumes section
+            for v in [vv  for  vv in topvols if vv.asDict()['type'] == 'volume' ]:
+                hit = True
+                toplevelvolumes += '  {}:\n'.format(v.asDict()['source']) # why not 'name'
+                lines = dump( v ).rstrip('\n').split('\n')
+                toplevelvolumes += '\n'.join( map( lambda x: '        '
+                                                  if x[0] != 0 else '        ' + x[1]
+                                                  if x[1] != ''else '' , enumerate(lines ) ) )
+                toplevelvolumes += '\n'
+
+            if hit: toplevelvolumes = 'volumes:\n' + toplevelvolumes
+        return toplevelvolumes

seedemu/core/Node.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
+import random
+import string
 from .Printable import Printable
 from .Network import Network
 from .enums import NodeRole
 from .Scope import *
 from .Registry import Registrable
 from .Emulator import Emulator
 from .Customizable import Customizable
+from .Volume import BaseVolume
 from .Configurable import Configurable
 from .enums import NetworkType
 from .Visualization import Vertex
@@ -230,8 +233,7 @@ class Node(Printable, Registrable, Configurable, Vertex, Customizable):
     # Dict of (peername, peerasn) -> (localaddr, netname, netProperties) -- netProperties = (latency, bandwidth, packetDrop, MTU)
     __xcs: Dict[Tuple[str, int], Tuple[IPv4Interface, str, Tuple[int,int,float,int]]]
 
-    __shared_folders: Dict[str, str]
-    __persistent_storages: List[str]
+    __custom_vols: List[BaseVolume]
     __name_servers: List[str]
 
     __geo: Tuple[float,float,str] # (Latitude,Longitude,Address) -- optional parameter that contains the geographical location of the Node
@@ -270,8 +272,7 @@ def __init__(self, name: str, role: NodeRole, asn: int, scope: str = None):
         self.__xcs = {}
         self.__configured = False
 
-        self.__shared_folders = {}
-        self.__persistent_storages = []
+        self.__custom_vols = []
 
         for soft in DEFAULT_SOFTWARE:
             self.__softwares.add(soft)
@@ -384,6 +385,17 @@ def setNameServers(self, servers: List[str]) -> Node:
 
         return self
 
+    def addDockerVolume(self, vol: BaseVolume ):
+        """!@brief adds the docker volume to this node's container
+            It can be a shared folder, named docker volume or tmpfs
+        """
+        self.__custom_vols.append(vol )
+        return self
+
+    def getDockerVolumes(self):
+        """!@brief retrieve any volumes mounted on this node's container"""
+        return self.__custom_vols
+
     # TODO: if a separate .env file is created, or the values are given directly in the docker-compose.yml 'environment' section
     # could be a setting of the Docker compiler
     # def setCustomEnv(self, key: str, actual_value: str, scope: ScopeTier=ScopeTier.Node, use_envsubst: bool=False):
@@ -891,50 +903,43 @@ def getInterfaces(self) -> List[Interface]:
         """
         return self.__interfaces
 
-    def addSharedFolder(self, nodePath: str, hostPath: str) -> Node:
+    def addSharedFolder(self, nodePath: str, hostPath: str, **kwargs) -> Node:
         """!
         @@brief Add a new shared folder between the node and host.
 
         @param nodePath path to the folder inside the container.
         @param hostPath path to the folder on the emulator host node.
+        @param kwargs any other docker volume options i.e. 'readonly'
 
         @returns self, for chaining API calls.
         """
-        self.__shared_folders[nodePath] = hostPath
+        # bind mounts are never named!
+        self.__custom_vols.append(  BaseVolume(source=hostPath, target=nodePath, type='bind', **kwargs) )
 
         return self
 
-    def getSharedFolders(self) -> Dict[str, str]:
-        """!
-        @brief Get shared folders between the node and host.
-
-        @returns dict, where key is the path in container and value is path on
-        host.
-        """
-        return self.__shared_folders
-
-    def addPersistentStorage(self, path: str) -> Node:
+    def addPersistentStorage(self, path: str, name: str=None, **kwargs) -> Node:
         """!
         @brief Add persistent storage to node.
 
         Nodes usually start fresh when you re-start them. This allow setting a
         directory where data will be persistent.
 
         @param path path to put the persistent storage folder in the container.
+        @param name if specified a named-volume is created.
+            By specifying the same name on multiple nodes
+            the volume is effectively shared between those containers.
+        @param kwargs any other docker volume options i.e. 'readonly'
 
         @returns self, for chaining API calls.
         """
-        self.__persistent_storages.append(path)
 
-        return self
+        if name == None: # generate random name for anonymus volume
+            name = ''.join(random.choice(string.ascii_lowercase) for i in range(6))
 
-    def getPersistentStorages(self) -> List[str]:
-        """!
-        @brief Get persistent storage folders on the node.
+        self.__custom_vols.append(  BaseVolume(target=path, type='volume', name=name, source=name, **kwargs) )
 
-        @returns list of persistent storage folder.
-        """
-        return self.__persistent_storages
+        return self
 
     def setGeo(self, Lat: float, Long: float, Address: str="") -> Node:
         """!
@@ -995,7 +1000,7 @@ def copySettings(self, node: Node):
         # if node.getBaseSystem() != None : self.setBaseSystem(node.getClasses())
 
         for (h, n, p) in node.getPorts(): self.addPort(h, n, p)
-        for p in node.getPersistentStorages(): self.addPersistentStorage(p)
+        for v in node.getDockerVolumes(): self.addDockerVolume(v)
         for (c, f) in node.getStartCommands(): self.appendStartCommand(c, f)
         # for (c, f) in node.getUserStartCommands(): self.appendUserStartCommand(c, f)
         for c in node.getBuildCommands(): self.addBuildCommand(c)