Source code for nunavut

#
# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# Copyright (C) 2018-2021  UAVCAN Development Team  <uavcan.org>
# This software is distributed under the terms of the MIT License.
#
"""Code generator built on top of pydsdl.

Nunavut uses pydsdl to generate text files using templates. While these
text files are often source code this module could also be used to generate
documentation or data interchange formats like JSON or XML.

The input to the nunavut library is a list of templates and a list of
``pydsdl.pydsdl.CompositeType`` objects. The latter is typically obtained
by calling pydsdl::

    from pydsdl import read_namespace

    compound_types = read_namespace(root_namespace, include_paths)

Next a :class:`nunavut.LanguageContext` is needed which is used to
configure all Nunavut objects for a specific target language ::

    from nunavut.lang import LanguageContext

    # Here we are going to generate C headers.
    language_context = LanguageContext('c')

:class:`nunavut.generators.AbstractGenerator` objects require
a :class:`nunavut.Namespace` tree which can be built from the
pydsdl type map using :meth:`nunavut.build_namespace_tree`::

    from nunavut import build_namespace_tree

    root_namespace = build_namespace_tree(compound_types,
                                          root_ns_folder,
                                          out_dir,
                                          language_context)

Putting this all together, the typical use of this library looks something like this::

    from pydsdl import read_namespace
    from nunavut import build_namespace_tree
    from nunavut.lang import LanguageContext
    from nunavut.jinja import DSDLCodeGenerator

    # parse the dsdl
    compound_types = read_namespace(root_namespace, include_paths)

    # select a target language
    language_context = LanguageContext('c')

    # build the namespace tree
    root_namespace = build_namespace_tree(compound_types,
                                          root_ns_folder,
                                          out_dir,
                                          language_context)

    # give the root namespace to the generator and...
    generator = DSDLCodeGenerator(root_namespace)

    # generate all the code!
    generator.generate_all()

"""

import collections
import pathlib
import sys
import typing

import pydsdl

from .lang import LanguageContext
from .lang._common import IncludeGenerator

# library users can access the utility types directly from the nunvut namespace. Internally
# we us the _utilities package to break circular imports.
from ._utilities import YesNoDefault  # noqa # pylint: disable=unused-import

if sys.version_info[:2] < (3, 5):  # pragma: no cover
    print("A newer version of Python is required", file=sys.stderr)
    sys.exit(1)


__all__ = ["Namespace", "YesNoDefault", "build_namespace_tree", "generate_types"]

# +---------------------------------------------------------------------------+


[docs]class Namespace(pydsdl.Any): """ K-ary tree (where K is the largest set of data types in a single dsdl namespace) where the nodes represent dsdl namespaces and the children are the datatypes and other nested namespaces (with datatypes always being leaf nodes). This structure extends :code:`pydsdl.Any` and is a :code:`pydsdl.pydsdl.CompositeType` via duck typing. :param str full_namespace: The full, dot-separated name of the namepace. This is expected to be a unique identifier. :param pathlib.Path root_namespace_dir: The directory representing the dsdl namespace and containing the namespaces's datatypes and nested namespaces. :param pathlib.PurePath base_output_path: The base path under which all namespaces and datatypes should be generated. :param LanguageContext language_context: The generated software language context the namespace is within. """ DefaultOutputStem = "_" def __init__( self, full_namespace: str, root_namespace_dir: pathlib.Path, base_output_path: pathlib.PurePath, language_context: LanguageContext, ): self._parent = None # type: typing.Optional[Namespace] self._namespace_components = [] # type: typing.List[str] self._namespace_components_stropped = [] # type: typing.List[str] for component in full_namespace.split("."): self._namespace_components_stropped.append(language_context.filter_id_for_target(component, "path")) self._namespace_components.append(component) self._full_namespace = ".".join(self._namespace_components_stropped) self._output_folder = pathlib.Path(base_output_path / pathlib.PurePath(*self._namespace_components_stropped)) output_stem = language_context.get_default_namespace_output_stem() if output_stem is None: output_stem = self.DefaultOutputStem output_path = self._output_folder / pathlib.PurePath(output_stem) self._base_output_path = base_output_path self._output_path = output_path.with_suffix(language_context.get_output_extension()) self._source_folder = pathlib.Path( root_namespace_dir / pathlib.PurePath(*self._namespace_components[1:]) ).resolve() if not self._source_folder.exists(): # to make Python > 3.5 behave the same as Python 3.5 raise FileNotFoundError(self._source_folder) self._short_name = self._namespace_components_stropped[-1] self._data_type_to_outputs = dict() # type: typing.Dict[pydsdl.CompositeType, pathlib.Path] self._nested_namespaces = set() # type: typing.Set[Namespace] self._language_context = language_context @property def output_folder(self) -> pathlib.Path: """ The folder where this namespace's output file and datatypes are generated. """ return self._output_folder
[docs] def get_support_output_folder(self) -> pathlib.PurePath: """ The folder under which support artifacts are generated. """ return self._base_output_path
[docs] def get_language_context(self) -> LanguageContext: """ The generated software language context the namespace is within. """ return self._language_context
[docs] def get_root_namespace(self) -> "Namespace": """ Traverses the namespace tree up to the root and returns the root node. :return: The root namespace object. """ namespace = self # type: Namespace while namespace._parent is not None: namespace = namespace._parent return namespace
[docs] def get_nested_namespaces(self) -> typing.Iterator["Namespace"]: """ Get an iterator over all the nested namespaces within this namespace. This is a shallow iterator that only provides directly nested namespaces. """ return iter(self._nested_namespaces)
[docs] def get_nested_types(self) -> typing.ItemsView[pydsdl.CompositeType, pathlib.Path]: """ Get a view of a tuple relating datatypes in this namespace to the path for the type's generated output. This is a shallow view including only the types directly within this namespace. """ return self._data_type_to_outputs.items()
[docs] def get_all_datatypes(self) -> typing.Generator[typing.Tuple[pydsdl.CompositeType, pathlib.Path], None, None]: """ Generates tuples relating datatypes at and below this namespace to the path for each type's generated output. """ yield from self._recursive_data_type_generator(self)
[docs] def get_all_namespaces(self) -> typing.Generator[typing.Tuple["Namespace", pathlib.Path], None, None]: """ Generates tuples relating nested namespaces at and below this namespace to the path for each namespace's generated output. """ yield from self._recursive_namespace_generator(self)
[docs] def get_all_types(self) -> typing.Generator[typing.Tuple[pydsdl.Any, pathlib.Path], None, None]: """ Generates tuples relating datatypes and nested namespaces at and below this namespace to the path for each type's generated output. """ yield from self._recursive_data_type_and_namespace_generator(self)
[docs] def find_output_path_for_type(self, any_type: pydsdl.Any) -> pathlib.Path: """ Searches the entire namespace tree to find a mapping of the type to an output file path. :param Any any_type: Either a Namespace or pydsdl.CompositeType to find the output path for. :return: The path where a file will be generated for a given type. :raises KeyError: If the type was not found in this namespace tree. """ if isinstance(any_type, Namespace): return any_type._output_path else: try: return self._data_type_to_outputs[any_type] except KeyError: pass # We could get fancier but this should do return self.get_root_namespace()._bfs_search_for_output_path(any_type, set([self]))
# +-----------------------------------------------------------------------+ # | DUCK TYPING: pydsdl.CompositeType # +-----------------------------------------------------------------------+ @property def full_name(self) -> str: return self._full_namespace @property def full_namespace(self) -> str: return self._full_namespace @property def source_file_path(self) -> pathlib.Path: return self._source_folder @property def data_types(self) -> typing.KeysView[pydsdl.CompositeType]: return self._data_type_to_outputs.keys() @property def attributes(self) -> typing.List[pydsdl.CompositeType]: return [] # +-----------------------------------------------------------------------+ # | PYTHON DATA MODEL # +-----------------------------------------------------------------------+ def __eq__(self, other: object) -> bool: if isinstance(other, Namespace): return self._full_namespace == other._full_namespace else: return False def __str__(self) -> str: return self.full_name def __hash__(self) -> int: return hash(self._full_namespace) # +-----------------------------------------------------------------------+ # | PRIVATE # +-----------------------------------------------------------------------+ def _add_data_type(self, dsdl_type: pydsdl.CompositeType, extension: str) -> None: self._data_type_to_outputs[dsdl_type] = pathlib.Path(self._base_output_path) / IncludeGenerator.make_path( dsdl_type, self._language_context.get_target_language(), extension ) def _add_nested_namespace(self, nested: "Namespace") -> None: self._nested_namespaces.add(nested) nested._parent = self def _bfs_search_for_output_path( self, data_type: pydsdl.CompositeType, skip_namespace: typing.Set["Namespace"] ) -> pathlib.Path: search_queue = collections.deque() # type: typing.Deque[Namespace] search_queue.appendleft(self) while len(search_queue) > 0: namespace = search_queue.pop() if namespace not in skip_namespace: try: return namespace._data_type_to_outputs[data_type] except KeyError: pass for nested_namespace in namespace._nested_namespaces: search_queue.appendleft(nested_namespace) raise KeyError(data_type) @classmethod def _recursive_data_type_generator( cls, namespace: "Namespace" ) -> typing.Generator[typing.Tuple[pydsdl.CompositeType, pathlib.Path], None, None]: for data_type, output_path in namespace.get_nested_types(): yield (data_type, output_path) for nested_namespace in namespace.get_nested_namespaces(): yield from cls._recursive_data_type_generator(nested_namespace) @classmethod def _recursive_namespace_generator( cls, namespace: "Namespace" ) -> typing.Generator[typing.Tuple["Namespace", pathlib.Path], None, None]: yield (namespace, namespace._output_path) for nested_namespace in namespace.get_nested_namespaces(): yield from cls._recursive_namespace_generator(nested_namespace) @classmethod def _recursive_data_type_and_namespace_generator( cls, namespace: "Namespace" ) -> typing.Generator[typing.Tuple[pydsdl.Any, pathlib.Path], None, None]: yield (namespace, namespace._output_path) for data_type, output_path in namespace.get_nested_types(): yield (data_type, output_path) for nested_namespace in namespace.get_nested_namespaces(): yield from cls._recursive_data_type_and_namespace_generator(nested_namespace)
# +---------------------------------------------------------------------------+ class _NamespaceFactory: """ Read-through cache and factory for :class:`Namespace` objects. """ def __init__(self, lctx: LanguageContext, base_path: pathlib.PurePath, root_namespace_dir: pathlib.Path): self._lctx = lctx self._base_path = base_path self._namespaces = dict() # type: typing.Dict[str, Namespace] self._root_namespace_dir = root_namespace_dir def get_root_namesapce(self) -> Namespace: try: return next(iter(self._namespaces.values())).get_root_namespace() except StopIteration: pass return self.get_empty_namespace() def get_empty_namespace(self) -> Namespace: return self.get_or_make_namespace("")[0] def get_or_make_namespace(self, full_namespace: str) -> typing.Tuple[Namespace, bool]: try: namespace = self._namespaces[str(full_namespace)] return (namespace, True) except KeyError: pass namespace = Namespace(full_namespace, self._root_namespace_dir, self._base_path, self._lctx) self._namespaces[str(full_namespace)] = namespace return (namespace, False)
[docs]def build_namespace_tree( types: typing.List[pydsdl.CompositeType], root_namespace_dir: str, output_dir: str, language_context: LanguageContext, ) -> Namespace: """Generates a :class:`nunavut.Namespace` tree. Given a list of pydsdl types, this method returns a root :class:`nunavut.Namespace`. The root :class:`nunavut.Namespace` is the top of a tree where each node contains references to nested :class:`nunavut.Namespace` and to any :code:`pydsdl.CompositeType` instances contained within the namespace. :param list types: A list of pydsdl types. :param str root_namespace_dir: A path to the folder which is the root namespace. :param str output_dir: The base directory under which all generated files will be created. :param nunavut.lang.LanguageContext language_context: The language context to use when building :class:`nunavut.Namespace` objects. :return: The root :class:`nunavut.Namespace`. """ namespace_index = set() # type: typing.Set[str] nsf = _NamespaceFactory(language_context, pathlib.PurePath(output_dir), pathlib.Path(root_namespace_dir)) for dsdl_type in types: # For each type we form a path with the output_dir as the base; the intermediate # folders named for the type's namespaces; and a file name that includes the type's # short name, major version, minor version, and the extension argument as a suffix. # Python's pathlib adapts the provided folder and file names to the platform # this script is running on. # We also, lazily, generate Namespace nodes as we encounter new namespaces for the # first time. namespace, did_exist = nsf.get_or_make_namespace(dsdl_type.full_namespace) if not did_exist: # add all namespaces up to root to index so we trigger # empty namespace generation in the final tree building # loop below. for i in range(len(dsdl_type.name_components) - 1, 0, -1): ancestor_ns = ".".join(dsdl_type.name_components[0:i]) if ancestor_ns in namespace_index: break namespace_index.add(ancestor_ns) namespace._add_data_type(dsdl_type, language_context.get_output_extension()) # We now have an index of all namespace names and we have Namespace # objects for non-empty namespaces. This final loop will build any # missing (i.e. empty) namespaces and all the links to form the # namespace tree. for full_namespace in namespace_index: namespace, _ = nsf.get_or_make_namespace(full_namespace) parent_namespace_components = namespace._namespace_components[0:-1] if len(parent_namespace_components) > 0: parent_name = ".".join(parent_namespace_components) parent, _ = nsf.get_or_make_namespace(parent_name) parent._add_nested_namespace(namespace) return nsf.get_root_namesapce()
# +---------------------------------------------------------------------------+ # +---------------------------------------------------------------------------+ # | GENERATION HELPERS # +---------------------------------------------------------------------------+
[docs]def generate_types( language_key: str, root_namespace_dir: pathlib.Path, out_dir: pathlib.Path, omit_serialization_support: bool = True, is_dryrun: bool = False, allow_overwrite: bool = True, lookup_directories: typing.Optional[typing.Iterable[str]] = None, allow_unregulated_fixed_port_id: bool = False, language_options: typing.Optional[typing.Mapping[str, typing.Any]] = None, ) -> None: """ Helper method that uses default settings and built-in templates to generate types for a given language. This method is the most direct way to generate code using Nunavut. :param str language_key: The name of the language to generate source for. See the :doc:`../../docs/templates` for details on available language support. :param pathlib.Path root_namespace_dir: The path to the root of the DSDL types to generate code for. :param pathlib.Path out_dir: The path to generate code at and under. :param bool omit_serialization_support: If True then logic used to serialize and deserialize data is omitted. :param bool is_dryrun: If True then nothing is generated but all other activity is performed and any errors that would have occurred are reported. :param bool allow_overwrite: If True then generated files are allowed to overwrite existing files under the `out_dir` path. :param typing.Optional[typing.Iterable[str]] lookup_directories: Additional directories to search for dependent types referenced by the types provided under the `root_namespace_dir`. Types will not be generated for these unless they are used by a type in the root namespace. :param bool allow_unregulated_fixed_port_id: If True then errors will become warning when using fixed port identifiers for unregulated datatypes. :param typing.Optional[typing.Mapping[str, typing.Any]] language_options: Opaque arguments passed through to the language objects. The supported arguments and valid values are different depending on the language specified by the `language_key` parameter. """ from nunavut.generators import create_generators language_context = LanguageContext( language_key, omit_serialization_support_for_target=omit_serialization_support, language_options=language_options, ) if lookup_directories is None: lookup_directories = [] type_map = pydsdl.read_namespace( str(root_namespace_dir), lookup_directories, allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id ) namespace = build_namespace_tree(type_map, str(root_namespace_dir), str(out_dir), language_context) generator, support_generator = create_generators(namespace) support_generator.generate_all(is_dryrun, allow_overwrite) generator.generate_all(is_dryrun, allow_overwrite)