Implement report-from-jinja2-file functionality for the 'report' outp…

…ut module

* Try to render a report from default settings (defaults.output.report) or a
  Jinja2 file in the 'reports' subdirectory of the search path
* Rename 'format' output module to 'report' retaining a stub in case anyone
  every used 'format' output
* Add sample 'node addressing' report
* Document the 'report' output module

Also:

* Refactor 'templates.template' into 'templates.render_template'
* Remove 'get_moddir' from common imports and fix any code using it to
  point to 'utils.files'
* Cleanup all calls to 'templates.template' and add error reporting
* Replace incidental calls to 'common.x' with calls to actual utils function

docs/outputs/index.md

@@ -2,9 +2,9 @@
 
 **netlab create** command can call one or more output modules to create files that can be used with Vagrant, Ansible, containerlab, graphviz or *graphite* topology. The output modules are specified with one or more `-o` parameters. When no `-o` parameter is specific, **netlab create** calls *provider* and *ansible* output modules.
 
-Each `-o` parameter specifies an output module, formatting modifiers, and output filename in the **format:modifiers=file(s)** format:
+Each `-o` parameter specifies an output module, formatting modifiers, and output filename in the **module:modifiers=file(s)** format:
 
-* **format** is the desired output module. It can be one of *provider*, *ansible*, *graph*, *yaml*, *json* or *graphite*.
+* **module** is the desired output module. It can be one of *provider*, *ansible*, *graph*, *yaml*, *json* or *format*.
 * Some output modules use optional formatting modifiers -- you can specify Ansible inventory format, graph type, or parts of the transformed data structure that you want to see in YAML or JSON format
 * All output formats support optional destination file name. Default file name is either hard-coded in the module or specified in **defaults.outputs** part of lab topology.
 
@@ -17,6 +17,7 @@ The following output modules are included in **netlab** distribution; you can cr
    provider.md
    ansible.md
    graph.md
+   report.md
    d2.md
    yaml-or-json.md
    devices.md

docs/outputs/report.md

@@ -0,0 +1,9 @@
+# Creating Reports with Custom Output Formats
+
+The *report* output module uses its parameter as the name of a Jinja2 formatting template that is used to create a custom report. For example, `netlab create -o report:addressing` creates an IP addressing report.
+
+The *report* output module tries to use the **defaults.outputs.report.*rname*** topology setting (*rname* is the report name). If that fails, it tries to read the Jinja2 template from **_rname_.j2** file in **reports** subdirectory of current directory, user _netlab_ directory (`~/.netlab`), system _netlab_ directory (`/etc/netlab`) and _netlab_ package directory.
+
+_netlab_ ships with the following built-in reports:
+
+* **addressing** -- Node/interface addressing report

netsim/augment/plugin.py

@@ -10,7 +10,7 @@
 
 from box import Box
 from .. import common
-from ..utils.templates import get_moddir
+from ..utils.files import get_moddir
 from .. import data
 
 def load_plugin_from_path(path: str, plugin: str) -> typing.Optional[object]:

netsim/cli/libvirt.py

@@ -17,9 +17,10 @@
 
 from .. import common
 from .. import read_topology
-from ..utils import strings,status
+from ..utils import strings,status,templates,log
 from . import external_commands
 from ..providers.libvirt import create_vagrant_network,LIBVIRT_MANAGEMENT_NETWORK_NAME
+from ..utils import files as _files
 
 def package_parse(args: typing.List[str], settings: Box) -> argparse.Namespace:
   devs = [ k for k in settings.devices.keys() if settings.devices[k].libvirt.create or settings.devices[k].libvirt.create_template ]
@@ -100,7 +101,7 @@ def lp_preinstall_hook(args: argparse.Namespace,settings: Box) -> None:
   if not 'pre_install' in devdata.libvirt:
     return
   print("Running pre-install hooks")
-  pre_inst_script = common.get_moddir() / "install/libvirt" / devdata.libvirt.pre_install / "run.sh"
+  pre_inst_script = _files.get_moddir() / "install/libvirt" / devdata.libvirt.pre_install / "run.sh"
 
   if not os.access(pre_inst_script, os.X_OK):
     print(" - run file not executable - skipping.")
@@ -115,7 +116,7 @@ def lp_create_bootstrap_iso(args: argparse.Namespace,settings: Box) -> None:
     return
   print("Creating bootstrap ISO image")
 
-  isodir = common.get_moddir() / "install/libvirt" / devdata.libvirt.create_iso
+  isodir = _files.get_moddir() / "install/libvirt" / devdata.libvirt.create_iso
   shutil.rmtree('iso',ignore_errors=True)
   shutil.copytree(isodir,'iso')
   if os.path.exists('bootstrap.iso'):
@@ -141,7 +142,17 @@ def lp_create_vm(args: argparse.Namespace,settings: Box) -> None:
     abort_on_failure(cmd)
   elif devdata.libvirt.create_template:
     data = get_template_data(devdata)
-    template = common.template(devdata.libvirt.create_template,data,"install/libvirt",)
+    tname = devdata.libvirt.create_template
+    try:
+      template = templates.render_template(
+                  j2_file=tname,
+                  data=data,
+                  path="install/libvirt")
+    except Exception as ex:
+      log.fatal(
+        text=f"Error rendering {tname}\n{strings.extra_data_printout(str(ex))}",
+        module='libvirt')
+
     pathlib.Path("template.xml").write_text(template)
     abort_on_failure("virsh define template.xml")
     abort_on_failure("virsh start --console vm_box")
@@ -270,7 +281,7 @@ def libvirt_package(cli_args: typing.List[str], topology: Box) -> None:
     lp_create_box(args,settings)
 
 def config_parse(args: typing.List[str], settings: Box) -> argparse.Namespace:
-  moddir = common.get_moddir()
+  moddir = _files.get_moddir()
   devs = map(
     lambda x: pathlib.Path(x).stem,
     glob.glob(str(moddir / "install/libvirt/*txt")))
@@ -286,7 +297,7 @@ def config_parse(args: typing.List[str], settings: Box) -> argparse.Namespace:
 
 def libvirt_config(cli_args: typing.List[str], settings: Box) -> None:
   args = config_parse(cli_args,settings)
-  helpfile = common.get_moddir() / "install/libvirt" / (args.device+".txt")
+  helpfile = _files.get_moddir() / "install/libvirt" / (args.device+".txt")
   print(helpfile.read_text())
 
 def libvirt_usage() -> None:

netsim/common.py

@@ -15,7 +15,6 @@
 from .utils.log import MissingValue, IncorrectAttr, IncorrectValue, IncorrectType, FatalError, ErrorAbort
 from .utils.log import fatal, error, exit_on_error, set_logging_flags, set_flag, print_verbose, debug_active
 from .utils.strings import extra_data_printout,format_structured_dict,print_structured_dict
-from .utils.templates import template,write_template,find_file,get_moddir
 
 AF_LIST = ['ipv4','ipv6']
 BGP_SESSIONS = ['ibgp','ebgp']

netsim/defaults/hints.yml

@@ -6,3 +6,8 @@ evpn:
     participating nodes and list VLANs in the 'vlans' attribute of that group
   node_bundle: |
     evpn.bundle attribute can be used only in global VRF definition
+report:
+  source: |
+    A report can be specified in a file with .j2 suffix within 'reports' subdirectory in
+    package-, system-, user- or current directory. You can also specify a report in a
+    defaults.outputs.report setting.

netsim/outputs/ansible.py

@@ -11,6 +11,8 @@
 from . import _TopologyOutput,check_writeable
 from ..augment import nodes
 from ..augment import devices
+from ..utils import templates,strings,log
+from ..utils import files as _files
 
 forwarded_port_name = { 'ssh': 'ansible_port', }
 
@@ -194,7 +196,18 @@ def ansible_config(config_file: typing.Union[str,None] = 'ansible.cfg', inventor
     inventory_file = 'hosts.yml'
 
   with open(config_file,"w") as output:
-    output.write(common.template('ansible.cfg.j2',{ 'inventory': inventory_file or 'hosts.yml' },'templates','ansible'))
+    try:
+      cfg_text = templates.render_template(
+                  j2_file='ansible.cfg.j2',
+                  data={'inventory': inventory_file or 'hosts.yml'},
+                  path='templates',
+                  extra_path=_files.get_search_path('ansible'))
+    except Exception as ex:
+      log.fatal(
+        text=f"Error rendering ansible.cfg\n{strings.extra_data_printout(str(ex))}",
+        module='ansible')
+
+    output.write(cfg_text)
     output.close()
     if not common.QUIET:
       print("Created Ansible configuration file: %s" % config_file)

netsim/outputs/format.py

@@ -1,41 +1,9 @@
 #
-# Create YAML or JSON output
+# This is just a dummy output module that was replaced by the new 'report'
+# output module. We kept the old module (pointing to the new one) in case
+# someone already used it in their workflow.
 #
-import typing
+from .report import REPORT
 
-import yaml
-import os
-from box import Box,BoxList
-from jinja2 import Environment, BaseLoader, FileSystemLoader, StrictUndefined, make_logging_undefined
-
-from .. import common
-from .. import data
-from ..augment import topology
-
-from . import _TopologyOutput
-
-class FORMAT(_TopologyOutput):
-
-  def write(self, topo: Box) -> None:
-    outfile = self.settings.filename or '-'
-    modname = type(self).__name__
-
-    if hasattr(self,'filenames'):
-      outfile = self.filenames[0]
-      if len(self.filenames) > 1:
-        common.error('Extra output filename(s) ignored: %s' % str(self.filenames[1:]),common.IncorrectValue,modname)
-
-    cleantopo: typing.Any = topology.cleanup_topology(topo)
-    output = common.open_output_file(outfile)
-
-    for fmt in self.format:
-      if not fmt in self.settings:
-        common.error(f'Unknown template format {fmt}',common.IncorrectValue,modname)
-        print(topo.defaults.outputs.format)
-        continue
-
-      template = Environment(loader=BaseLoader(), \
-              trim_blocks=True,lstrip_blocks=True, \
-              undefined=make_logging_undefined(base=StrictUndefined)).from_string(self.settings[fmt])
-      output.write(template.render(**topo))
-      output.write("\n")
+class FORMAT(REPORT):
+  pass

netsim/outputs/report.py

@@ -0,0 +1,80 @@
+#
+# Create YAML or JSON output
+#
+import typing
+import pathlib
+
+from box import Box
+import jinja2
+
+from .. import common
+from .. import data
+from ..augment import topology
+from ..utils import files as _files
+from ..utils import log,templates,strings
+
+from . import _TopologyOutput
+
+SEARCH_PATH: list = []
+
+def get_format_text(settings: Box, fname: str) -> typing.Optional[str]:
+  global SEARCH_PATH
+  if fname in settings:
+    return settings[fname]
+
+  if not SEARCH_PATH:  
+    SEARCH_PATH = [ pc + "/reports" for pc in _files.get_search_path() ]
+
+  tfile = _files.find_file(fname+".j2",SEARCH_PATH)
+  if tfile is None:
+    return None
+
+  return pathlib.Path(tfile).read_text()
+
+class REPORT(_TopologyOutput):
+
+  def write(self, topo: Box) -> None:
+    outfile = self.settings.filename or '-'
+    modname = type(self).__name__
+
+    if hasattr(self,'filenames'):
+      outfile = self.filenames[0]
+      if len(self.filenames) > 1:
+        common.error('Extra output filename(s) ignored: %s' % str(self.filenames[1:]),common.IncorrectValue,modname)
+
+    output = common.open_output_file(outfile)
+
+    extra_path = _files.get_search_path("reports")
+    for fmt in self.format:
+      if fmt in self.settings:
+        try:
+          output.write(
+            templates.render_template(
+              data=topo.to_dict(),
+              j2_text = self.settings[fmt],
+              path="reports",
+              extra_path=extra_path)+"\n")
+          output.write("\n")
+        except Exception as ex:
+          log.error(
+            text=f"Error rendering topology format {fmt}\n{strings.extra_data_printout(str(ex))}",
+            category=log.IncorrectValue)
+        continue
+
+      try:
+        output.write(
+          templates.render_template(
+            data=topo.to_dict(),
+            j2_file=fmt+".j2",
+            path="reports",
+            extra_path=extra_path)+"\n")
+      except jinja2.exceptions.TemplateNotFound:
+        log.error(
+          text=f'Cannot find "{fmt}" in any of the report directories',
+          category=common.IncorrectValue,
+          module='report',
+          hint='source')
+      except Exception as ex:
+        log.error(
+          text=f"Error rendering {fmt}\n{strings.extra_data_printout(str(ex))}",
+          category=log.IncorrectValue)

netsim/outputs/tools.py

@@ -12,7 +12,8 @@
 from . import _TopologyOutput,check_writeable
 from ..tools import _ToolOutput
 from .common import adjust_inventory_host
-from ..utils import templates
+from ..utils import templates,strings,log
+from ..utils import files as _files
 
 def render_tool_config(tool: str, fmt: str, topology: Box) -> str:
     output_module = _ToolOutput.load(tool)
@@ -41,11 +42,16 @@ def create_tool_config(tool: str, topology: Box) -> None:
       config_src  = f'rendering "{config.render}" format'
     elif 'template' in config:
       template = config.template
-      config_text = templates.template(
-                      j2=config.template,
-                      data=topo_data,
-                      path=f'tools/{tool}',
-                      user_template_path=f'tools/{tool}')
+      try:
+        config_text = templates.render_template(
+                        j2_file=config.template,
+                        data=topo_data,
+                        path=f'tools/{tool}',
+                        extra_path=_files.get_search_path(f'tools/{tool}'))
+      except Exception as ex:
+        log.fatal(
+          text=f"Error rendering {config.template}\n{strings.extra_data_printout(str(ex))}",
+          module='libvirt')
       config_src  = f'from {config.template} template'
     else:
       common.error(f'Unknown tool configuration type\n... tool {tool}\n... config {config}')

netsim/providers/__init__.py

@@ -18,6 +18,8 @@
 from ..callback import Callback
 from ..augment import devices,links
 from ..data import get_box,get_empty_box,filemaps
+from ..utils import files as _files
+from ..utils import templates,log,strings
 
 class _Provider(Callback):
   def __init__(self, provider: str, data: Box) -> None:
@@ -38,10 +40,10 @@ def get_template_path(self) -> str:
     return 'templates/provider/' + self.provider
 
   def get_full_template_path(self) -> str:
-    return str(common.get_moddir()) + '/' + self.get_template_path()
+    return str(_files.get_moddir()) + '/' + self.get_template_path()
 
   def find_extra_template(self, node: Box, fname: str) -> typing.Optional[str]:
-    return common.find_file(fname+'.j2',[ f'./{node.device}','.',f'{ self.get_full_template_path() }/{node.device}'])
+    return _files.find_file(fname+'.j2',[ f'./{node.device}','.',f'{ self.get_full_template_path() }/{node.device}'])
 
   def get_output_name(self, fname: typing.Optional[str], topology: Box) -> str:
     if fname:
@@ -123,7 +125,7 @@ def create_extra_files(
     if not binds:
       return
 
-    sys_folder = str(common.get_moddir())+"/"
+    sys_folder = str(_files.get_moddir())+"/"
     out_folder = f"{self.provider}_files/{node.name}"
 
     bind_dict = filemaps.mapping_to_dict(binds)
@@ -136,20 +138,38 @@ def create_extra_files(
         node_data = node + { 'hostvars': topology.nodes }
         if '/' in file_name:                      # Create subdirectory in out_folder if needed
           pathlib.Path(f"{out_folder}/{os.path.dirname(file_name)}").mkdir(parents=True,exist_ok=True)
-        common.write_template(
-          in_folder=os.path.dirname(template_name),
-          j2=os.path.basename(template_name),
-          data=node_data.to_dict(),
-          out_folder=out_folder, filename=file_name)
+        try:
+          templates.write_template(
+            in_folder=os.path.dirname(template_name),
+            j2=os.path.basename(template_name),
+            data=node_data.to_dict(),
+            out_folder=out_folder, filename=file_name)
+        except Exception as ex:
+          log.fatal(
+            text=f"Error rendering {template_name} into {file_name}\n{strings.extra_data_printout(str(ex))}",
+            module=self.provider)
+
         print( f"Created {out_folder}/{file_name} from {template_name.replace(sys_folder,'')}, mapped to {node.name}:{mapping}" )
       else:
         common.error(f"Cannot find template for {file_name} on node {node.name}",common.MissingValue,'provider')
 
   def create(self, topology: Box, fname: typing.Optional[str]) -> None:
     self.transform(topology)
     fname = self.get_output_name(fname,topology)
+    tname = self.get_root_template()
+    try:
+      r_text = templates.render_template(
+        data=topology.to_dict(),
+        j2_file=tname,
+        path=self.get_template_path(),
+        extra_path=_files.get_search_path(self.provider))
+    except Exception as ex:
+      log.fatal(
+        text=f"Error rendering {fname} from {tname}\n{strings.extra_data_printout(str(ex))}",
+        module=self.provider)
+
     output = common.open_output_file(fname)
-    output.write(common.template(self.get_root_template(),topology.to_dict(),self.get_template_path(),self.provider))
+    output.write(r_text)
     if fname != '-':
       common.close_output_file(output)
       print("Created provider configuration file: %s" % fname)