Filter optimize

.gitignore

@@ -1,2 +1,12 @@
+# Apple Desktop Services Store
+.DS_Store
+# PyCharm IDE
+.idea
+# Vim swap files
+.*.swp
+# Python
 *.pyc
+#
 *.retry
+# Visual Studio Code
+.vscode

README.md

@@ -1,7 +1,7 @@
 # Ansible Role: aci-model
-A comprehensive Ansible role to model and deploy Cisco ACI fabrics.
+A comprehensive Ansible role to model and deploy Cisco ACI fabrics and a custom Ansible filter for structured data.
 
-This role provides an abstraction layer that is convenient to use. By providing your required configuration (a structured dataset) in your inventory this role will perform the needed actions to ensure that configuration is deployd on your ACI infrastructure.
+This role provides an abstraction layer that is convenient to use. By providing your required configuration (a structured dataset) in your inventory this role will perform the needed actions to ensure that configuration is deployed on your ACI infrastructure.
 
 Using this role you can easily set up demo environment, maintain a lab or use it as the basis for your in-house ACI infrastructure. It can help you understand how ACI works while prototyping and testing. No prior Ansible or ACI knowledge is required to get started.
 
@@ -38,6 +38,14 @@ You need to configure your Ansible to find this Jinja2 filter. There are two way
 
 Because of its general usefulness, we are looking into making this *aci_listify* filter more generic and make it part of the default Ansible filters.
 
+#### The alternative filter plugin
+The alternative filter *aci_listify2* (file: *plugins/filter/aci2.py*) is installed in the same manner as the original filter. It provides the following enhancements:
+
+* Instances of objects can be organized in lists (as in the original filter) or dicts (new).
+* You can append a regex to each key so that only key values that match the regex will appear in the output.
+* This is documented in the file *plugins/filter/aci2.py* itself.
+
+The filter does not depend on this Ansible role. It can be used in any Ansible task to extract a list of items from a structured dict. For this purpose, it suffices to install the filter. You need neither the role nor the playbook or the example inventory.
 
 ## Using the example playbook
 

example-inventory.yaml

@@ -159,18 +159,28 @@ fabric01:
               type: phys
         bd:
         - name: app_bd
-          subnet: 
+          subnet:
           - name: 10.10.10.1
             mask: 24
             scope: private
           vrf: Customer01
         - name: web_bd
-          subnet: 
+          subnet:
           - name: 20.20.20.1
             mask: 24
             scope: public
           vrf: Customer01
-          l3out: 
+          l3out:
+          - name: l3out
+        - name: web_bd1
+          subnet:
+          - name: 20.20.21.1
+            mask: 24
+            scope:
+            - public
+            - shared
+          vrf: Customer01
+          l3out:
           - name: l3out
         vrf:
         - name: Customer01

plugins/filter/aci.py

@@ -1,38 +1,69 @@
-# Copyright: (c) 2017, Ramses Smeyers <[email protected]>
-
+# Copyright: (c) 2020-2023, Tilmann Boess <[email protected]>
+# Based on: (c) 2017, Ramses Smeyers <[email protected]>
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+
 def listify(d, *keys):
-    return listify_worker(d, keys, 0, [], {}, '')
-
-def listify_worker(d, keys, depth, result, cache, prefix):
-    prefix += keys[depth] + '_'
-
-    if keys[depth] in d:
-        for item in d[keys[depth]]:
-            cache_work = cache.copy()
-            if isinstance(item, dict):
-                for k,v in item.items():
-                    if not isinstance(v, dict) and not isinstance(v, list):
-                        cache_key = prefix + k
-                        cache_value = v
-                        cache_work[cache_key] = cache_value
-
-                if len(keys)-1 == depth :
-                    result.append(cache_work)
-                else:
-                    for k,v in item.items():
-                        if k == keys[depth+1]:
-                            if isinstance(v, dict) or isinstance(v, list):
-                                result = listify_worker({k:v}, keys, depth+1, result, cache_work, prefix)
-    return result
+    """Extract key/value data from ACI-model object tree.
+The object tree must have the following structure in order to find matches: dict
+at top-level, list at 2nd level and then alternating dicts and lists.
+The keys must match dict names along a path in this tree down to dict that
+contains at least 1 key/value pair.
+Along this path all key/value pairs for all keys given are fetched.
+Args:
+- d (dict): object tree.
+- *keys: key names to look for in ' d'  in hierarchical order (the keys must form
+    a path in the object tree).
+Returns:
+- list of dicts (key/value-pairs); given keys are concatenated with '_' to form
+    a single key. Example: ('tenant' , 'app' , 'epg') results in 'tenant_app_epg'.
+"""
+
+    def listify_worker(d, keys, depth=0, result=[], cache={}, prefix=''):
+        """Recursive inner function to encapsulate the internal arguments.
+Args:
+- d (dict): subtree of objects for key search (depends on value of ' depth' ).
+- keys (list): list of keys.
+- depth (int): index (corresponding to depth in object tree) of key in key list.
+- result (list): current result list of key/value-pairs.
+- cache (dict): collects key/value pairs common for all items in result list.
+- prefix (str): current prefix for key list in result.
+"""
+        prefix = ''.join((prefix, keys[depth], '_'))
+        if keys[depth] in d:
+            # At level of dict.
+            for item in d[keys[depth]]:
+                # One level below: Loop thru the list in this dict. If 'd[keys[depth]]' is a dict, the next test will fail and the recursion ends.
+                if isinstance(item, dict):
+                    # 'cache_work' holds all key/value pairs along the path.
+                    cache_work = cache.copy()
+                    for k, v in item.items():
+                        # Two levels below: Loop thru the dict.
+                        if isinstance(v, (str, int, float, bool, bytes)) or isinstance(v, list) and all(isinstance(x, (str, int, float, bool, bytes)) for x in v):
+                            # Key/value found. Accept a scalar or a list of scalars as attribute value.
+                            cache_work[''.join((prefix, k))] = v
+                    if len(keys)-1 == depth:
+                        # Max. depth reached.
+                        result.append(cache_work)
+                    else:
+                        # Lookup next key in the dict below.
+                        nextkey = keys[depth+1]
+                        if nextkey in item and isinstance(item[nextkey], list):
+                            # It's useless to test for dict here because the recursion will end in the next level.
+                            result = listify_worker({nextkey: item[nextkey]}, keys, depth+1, result, cache_work, prefix)
+        return result
+        # End of inner function
+
+    return listify_worker(d, keys)
+
 
 class FilterModule(object):
     ''' Ansible core jinja2 filters '''
 
     def filters(self):
         return {
-            'aci_listify': listify,
+            'aci_listify': listify
         }

plugins/filter/aci2.py

@@ -0,0 +1,205 @@
+# Copyright: (c) 2020-2023, Tilmann Boess <[email protected]>
+# Based on: (c) 2017, Ramses Smeyers <[email protected]>
+
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+
+"""
+This is an alternative filter to the original 'aci_listify' in 'aci.py'.
+The instances (e.g. tenant, vrf, leafid, …) can be organized in a dict as
+well as in a list. If you need to lookup instances directly (e.g. by other
+filters), it might be useful to organize your inventory in dicts instead
+of lists.
+
+*** Examples ***
+1. Simple static specification:
+  loop: "{{ aci_topology|aci_listify2('access_policy', 'interface_policy_profile=.+998', 'interface_selector') }}"
+All paths in the output match interface policy profiles that end in «998».
+E.g. interface selectors below a non-matching interface policy profile
+will be suppressed from the output.
+2. Dynamic specification:
+  loop: "{{ LEAFID_ROOT|aci_listify2(leaf_match, port_match, 'type=switch_port') }}"
+  vars:
+    leaf_match: "leafid={{ outer.leafid_Name }}"
+    port_match: "port={{ outer.leafid_port_Name }}"
+Here the regex's for the leafid and the port are determined at runtime in an
+outer task. The outer task sets the dict 'outer' and this dict is referenced
+here.
+'LEAFID_ROOT' is the dict in which to look for the following hierarchy:
+  leafid:
+  # leafid 101: all instances organized in lists.
+  - Name: 101
+    port:
+    - Name: 1
+      type:
+      - Name: vpc
+    - Name: 2
+      type:
+      - Name: port_channel
+    - Name: 3
+      type:
+      - Name: switch_port
+  - Name: 102
+    # leafid 102: organized in dicts and lists.
+    port:
+      # port instances: dict
+      1:
+        Name: 1
+        type:
+          # type instances: dict
+          vpc:
+            Name: vpc
+      2:
+        Name: 2
+        type:
+          # type instances: dict
+          port_channel:
+            Name: port_channel
+      4:
+        Name: 4
+        type:
+        # type instances: list
+        - Name: switch_port
+( … and so on for all leaf-switches and ports …)
+This matches only if:
+* leafid matches the leafid delivered by the outer task.
+* port matches the port delivered by the outer task.
+* The port shall be configured as a simple switchport (not a channel).
+The outer task could be:
+  - name: "example outer task"
+    include_tasks:
+      file: inner.yaml
+    loop: "{{ portlist }}"
+    loop_control:
+      loop_var: outer
+    vars:
+      portlist:
+      - leafid_Name: '10.'
+        leafid_port_Name: '3'
+      - leafid_Name: '.0.'
+        leafid_port_Name: '(2|4)'
+The dict 'portlist' need not be specified here as task variable.
+You can provide it as extra var on the command line and thus specify
+dynamically which ports shall be configured.
+"""
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import re
+
+
+def def_filter():
+    """Outer function that defines the filter and encapsulates initialization of variables.
+"""
+    # Name of the attribute used as «Name». We use uppercase «Name» to
+    # let it appear 1st if YAML/JSON files are sorted by keys.
+    # Change it to your liking.
+    nameAttr = 'Name'
+    # Regex to separate object and instance names.
+    rValue = re.compile('([^=]+)=(.*)')
+
+    def lister(myDict, *myKeys):
+        """Extract key/value data from ACI-model object tree.
+The keys must match dict names along a path in this tree down to a dict that
+contains at least 1 key/value pair.
+Along this path all key/value pairs for all keys given are fetched.
+Args:
+* myDict (dict): object tree.
+* *myKeys: key names to look for in 'myDict' in hierarchical order (the keys
+  must form a path in the object tree).
+* You can append a regex to each key (separated by «=»). Only keys
+  whose name-attribute matches the regex will be included in the result.
+  If the regex is omitted, all keys will be included (backwards compatible).
+Returns:
+* list of dicts (key/value-pairs); given keys are concatenated with '_' to form
+  a single key. Example: ('tenant' , 'app' , 'epg') results in 'tenant_app_epg'.
+"""
+        # keyList will be a copy of the initial list «myKeys».
+        keyList = []
+        # List of regex to match the name attributes.
+        regexList = []
+        for K in myKeys:
+            match = rValue.fullmatch(K)
+            if match:
+                keyList.append(match.group(1))
+                regexList.append(re.compile(match.group(2)))
+            else:
+                keyList.append(K)
+                regexList.append(None)
+
+        def worker(itemList, depth, result, cache, prefix):
+            """Inner function for instance evaluation.
+Args:
+* itemList (list): current instance list in tree (list of dicts, each item
+  is an ACI object).
+* depth (int): index (corresponding to depth in object tree) of key in key list.
+* result (list): current result list of key/value-pairs.
+* cache (dict): collects key/value pairs common for all items in result list.
+* prefix (str): current prefix for key list in result.
+"""
+            for item in itemList:
+                # Save name attribute for later usage.
+                # If name attribute is missing, set to None.
+                name = str(item.get(nameAttr, None))
+                # cache holds the pathed keys (build from the key list).
+                # Each recursive call gets its own copy.
+                subcache = cache.copy()
+                for subKey, subItem in list(item.items()):
+                    if isinstance(subItem, (str, int, float, bool, bytes)) or isinstance(subItem, list) and all(isinstance(x, (str, int, float, bool, bytes)) for x in subItem):
+                        # Key/value found. Accept a scalar or a list of scalars as attribute value.
+                        subcache['%s%s' % (prefix, subKey)] = subItem
+                        # All key/value pairs are evaluated before dicts and lists.
+                        # Otherwise, some attributes might not be transferred from the
+                        # cache to the result list.
+                if regexList[depth] is not None and (name is None or not regexList[depth].fullmatch(name)):
+                    # If regex was specified and the nameAttr does not match, do
+                    # not follow the path but continue with next item. Also a
+                    # non-existing nameAttr attribute is interpreted as non-match.
+                    continue
+                result = finder(item, depth, result, subcache, prefix)
+            return result
+
+        def finder(objDict, depth=-1, result=None, cache=None, prefix=''):
+            """Inner function for tree traversal.
+* objDict (dict): current subtree, top key is name of an ACI object type.
+* depth (int): index (corresponding to depth in object tree) of key in key list.
+* result (list): current result list of key/value-pairs.
+* cache (dict): collects key/value pairs common for all items in result list.
+* prefix (str): current prefix for key list in result.
+"""
+            if result is None:
+                result = []
+            if cache is None:
+                cache = {}
+            depth += 1
+            if depth == len(keyList):
+                # At end of key list: transfer cache to result list.
+                result.append(cache)
+            else:
+                prefix = ''.join((prefix, keyList[depth], '_'))
+                # Check if object type is in tree at given depth.
+                if keyList[depth] in objDict:
+                    # Prepare item list. ACI objects may be stored as list or dict.
+                    if isinstance(objDict[keyList[depth]], list):
+                        itemList = objDict[keyList[depth]]
+                    elif isinstance(objDict[keyList[depth]], dict):
+                        itemList = list(objDict[keyList[depth]].values())
+                    else:
+                        # Neither dict nor list – return to upper level.
+                        return result
+                    result = worker(itemList, depth, result, cache.copy(), prefix)
+            return result
+
+        # End of function: lister
+        return finder(myDict)
+
+    # End of function: def_filter
+    return lister
+
+
+class FilterModule(object):
+    """Jinja2 filters for Ansible."""
+
+    def filters(self):
+        """Name the filter: aci_listify2"""
+        return {'aci_listify2': def_filter()}