#
# Copyright (C) OpenCyphal Development Team <opencyphal.org>
# Copyright Amazon.com Inc. or its affiliates.
# SPDX-License-Identifier: MIT
#
"""
Filters for generating python. All filters in this
module will be available in the template's global namespace as ``py``.
"""
from __future__ import annotations
import base64
import builtins
import functools
import gzip
import itertools
import keyword
import pickle
from typing import Any, Dict, Iterable
import pydsdl
from nunavut._dependencies import Dependencies
from nunavut._templates import (
SupportsTemplateContext,
template_context_filter,
template_language_filter,
template_language_int_filter,
template_language_list_filter,
)
from nunavut._utilities import cached_property
from nunavut.lang import Language as BaseLanguage
from nunavut.lang._common import TokenEncoder, UniqueNameGenerator
class Language(BaseLanguage):
"""
Concrete, Python-specific :class:`nunavut.lang.Language` object.
"""
PYTHON_RESERVED_IDENTIFIERS: list[str] = sorted(list(map(str, list(keyword.kwlist) + dir(builtins))))
def _validate_language_options(self, defaults: Dict[str, Any], options: Dict[str, Any]) -> Dict[str, Any]:
# pylint: disable=unused-argument
options["enable_serialization_asserts"] = True # always enable serialization asserts for python
return options
@cached_property
def _token_encoder(self) -> TokenEncoder:
"""
Caching getter to ensure we don't have to recompile TokenEncoders for each filter invocation.
"""
return TokenEncoder(self, additional_reserved_identifiers=self.PYTHON_RESERVED_IDENTIFIERS)
def get_includes(self, dep_types: Dependencies) -> list[str]:
# imports aren't includes
return []
def filter_id(self, instance: Any, id_type: str = "any") -> str:
raw_name = self.default_filter_id_for_target(instance)
return self._token_encoder.strop(raw_name, id_type)
[docs]
@template_context_filter
def filter_to_template_unique_name(context: SupportsTemplateContext, base_token: str) -> str:
# pylint: disable=unused-argument
"""
Filter that takes a base token and forms a name that is very
likely to be unique within the template the filter is invoked. This
name is also very likely to be a valid Python identifier.
.. IMPORTANT::
The exact tokens generated may change between major or minor versions
of this library. The only guarantee provided is that the tokens
will be stable for the same version of this library given the same
input.
Also note that name uniqueness is only likely within a given template.
Between templates there is no guarantee of uniqueness and,
since this library does not lex generated source, there is no guarantee
that the generated name does not conflict with a name generated by
another means.
.. invisible-code-block: python
from nunavut.lang.py import filter_to_template_unique_name
from nunavut.lang._common import UniqueNameGenerator
.. code-block:: python
# Given
template = '{{ "f" | to_template_unique_name }},{{ "f" | to_template_unique_name }},'
template += '{{ "f" | to_template_unique_name }},{{ "bar" | to_template_unique_name }}'
# then
rendered = '_f0_,_f1_,_f2_,_bar0_'
.. invisible-code-block: python
UniqueNameGenerator.reset()
jinja_filter_tester(filter_to_template_unique_name, template, rendered, 'py')
.. code-block:: python
# Given
template = '{{ "i like coffee" | to_template_unique_name }}'
# then
rendered = '_i like coffee0_'
.. invisible-code-block: python
jinja_filter_tester(filter_to_template_unique_name, template, rendered, 'py')
:param str base_token: A token to include in the base name.
:return: A name that is likely to be valid python identifier and is likely to
be unique within the file generated by the current template.
"""
return UniqueNameGenerator.get_instance()("py", base_token, "_", "_")
[docs]
@template_language_filter(__name__)
def filter_id(language: Language, instance: Any, id_type: str = "any") -> str:
"""
Filter that produces a valid Python identifier for a given object. The encoding may not
be reversible.
.. invisible-code-block: python
from nunavut.lang.py import filter_id
.. code-block:: python
# Given
I = 'I like python'
# and
template = '{{ I | id }}'
# then
rendered = 'I_like_python'
.. invisible-code-block: python
jinja_filter_tester(filter_id, template, rendered, 'py', I=I)
.. code-block:: python
# Given
I = '&because'
# and
template = '{{ I | id }}'
# then
rendered = 'zX0026because'
.. invisible-code-block: python
jinja_filter_tester(filter_id, template, rendered, 'py', I=I)
.. code-block:: python
# Given
I = 'if'
# and
template = '{{ I | id }}'
# then
rendered = 'if_'
.. invisible-code-block: python
jinja_filter_tester(filter_id, template, rendered, 'py', I=I)
:param any instance: Any object or data that either has a name property or can be converted
to a string.
:return: A token that is a valid Python identifier, is not a reserved keyword, and is transformed
in a deterministic manner based on the provided instance.
"""
return language.filter_id(instance, id_type)
[docs]
@template_language_filter(__name__)
def filter_full_reference_name(language: Language, t: pydsdl.CompositeType) -> str:
"""
Provides a string that is the full namespace, typename, major, and minor version for a given composite type.
.. invisible-code-block: python
from nunavut.lang.py import filter_full_reference_name
dummy = lambda: None
dummy_version = lambda: None
setattr(dummy, 'version', dummy_version)
.. code-block:: python
# Given
full_name = 'any.str.2Foo'
major = 1
minor = 2
# and
template = '{{ my_obj | full_reference_name }}'
# then
rendered = 'any_.str_.zX0032Foo_1_2'
.. invisible-code-block: python
setattr(dummy_version, 'major', major)
setattr(dummy_version, 'minor', minor)
setattr(dummy, 'full_name', full_name)
setattr(dummy, 'short_name', full_name.split('.')[-1])
jinja_filter_tester(filter_full_reference_name, template, rendered, 'py', my_obj=dummy)
:param pydsdl.CompositeType t: The DSDL type to get the fully-resolved reference name for.
"""
ns_parts = t.full_name.split(".")
if len(ns_parts) > 1:
if language.enable_stropping:
ns = list(map(functools.partial(filter_id, language), ns_parts[:-1]))
else:
ns = ns_parts[:-1]
return ".".join(ns + [language.filter_short_reference_name(t)])
[docs]
@template_language_filter(__name__)
def filter_short_reference_name(language: Language, t: pydsdl.CompositeType) -> str:
"""
Provides a string that is a shorted version of the full reference name. This type is unique only within its
namespace.
.. invisible-code-block: python
from nunavut.lang.py import filter_short_reference_name
dummy = lambda: None
dummy_version = lambda: None
setattr(dummy, 'version', dummy_version)
.. code-block:: python
# Given
short_name = '2Foo'
major = 1
minor = 2
# and
template = '{{ my_obj | short_reference_name }}'
# then
rendered = 'zX0032Foo_1_2'
.. invisible-code-block: python
setattr(dummy_version, 'major', major)
setattr(dummy_version, 'minor', minor)
setattr(dummy, 'short_name', short_name)
jinja_filter_tester(filter_short_reference_name, template, rendered, 'py', my_obj=dummy)
:param pydsdl.CompositeType t: The DSDL type to get the reference name for.
"""
return language.filter_short_reference_name(t)
[docs]
@template_language_list_filter(__name__)
def filter_imports(language: Language, t: pydsdl.CompositeType, sort: bool = True) -> list[str]:
"""
Returns a list of all modules that must be imported to use a given type.
:param pydsdl.CompositeType t: The type to scan for dependencies.
:param bool sort: If true the returned list will be sorted.
:return: a list of python module names the provided type depends on.
"""
# Make a list of all attributes defined by this type
if isinstance(t, pydsdl.ServiceType):
atr = t.request_type.attributes + t.response_type.attributes
else:
atr = t.attributes
def array_w_composite_type(data_type: pydsdl.Any) -> bool:
return isinstance(data_type, pydsdl.ArrayType) and isinstance(data_type.element_type, pydsdl.CompositeType)
# Extract data types of said attributes; for type constructors such as arrays extract the element type
dep_types = [x.data_type for x in atr if isinstance(x.data_type, pydsdl.CompositeType)]
dep_types += [x.data_type.element_type for x in atr if array_w_composite_type(x.data_type)]
# Make a list of unique full namespaces of referenced composites. Keep the original ordering.
namespace_list = []
for dt in dep_types:
ns = dt.full_namespace
if ns not in namespace_list:
namespace_list.append(ns)
if language.enable_stropping:
namespace_list = [".".join([language.filter_id(y) for y in x.split(".")]) for x in namespace_list]
if sort:
return list(sorted(namespace_list))
else:
return namespace_list
[docs]
@template_language_int_filter(__name__)
def filter_longest_id_length(language: Language, attributes: list[pydsdl.Attribute]) -> int:
"""
Return the length of the longest identifier in a list of :class:`pydsdl.Attribute` objects.
.. invisible-code-block: python
from nunavut.lang.py import filter_longest_id_length
.. code-block:: python
# Given
I = ['one.str.int.any', 'three.str.int.any']
# and
template = '{{ I | longest_id_length }}'
# then
rendered = '32'
.. invisible-code-block: python
jinja_filter_tester(filter_longest_id_length, template, rendered, 'py', I=I)
"""
if language.enable_stropping:
return max(map(len, map(functools.partial(filter_id, language), attributes)))
else:
return max(map(len, attributes))
[docs]
def filter_pickle(x: Any) -> str:
"""
Serializes the given object using pickle and then compresses it using gzip and then encodes it using base85.
"""
pck = base64.b85encode(gzip.compress(pickle.dumps(x, protocol=4))).decode().strip() # type: str
segment_gen = map("".join, itertools.zip_longest(*([iter(pck)] * 100), fillvalue=""))
return "\n".join(repr(x) for x in segment_gen)
[docs]
def filter_numpy_scalar_type(t: pydsdl.Any) -> str:
"""
Returns the numpy scalar type that is the closest match to the given DSDL type.
"""
def pick_width(w: int) -> int:
for o in [8, 16, 32, 64]:
if w <= o:
return o
raise ValueError(f"Invalid bit width: {w}") # pragma: no cover
if isinstance(t, pydsdl.BooleanType):
return "_np_.bool_"
if isinstance(t, pydsdl.SignedIntegerType):
return f"_np_.int{pick_width(t.bit_length)}"
if isinstance(t, pydsdl.UnsignedIntegerType):
return f"_np_.uint{pick_width(t.bit_length)}"
if isinstance(t, pydsdl.FloatType):
return f"_np_.float{pick_width(t.bit_length)}"
assert not isinstance(t, pydsdl.PrimitiveType), "Forgot to handle some primitive types"
return "_np_.object_"
[docs]
def filter_newest_minor_version_aliases(tys: Iterable[pydsdl.CompositeType]) -> list[tuple[str, pydsdl.CompositeType]]:
"""
Implementation of https://github.com/OpenCyphal/nunavut/issues/193
"""
tys = list(tys)
return [
(
f"{name}_{major}",
max(
(t for t in tys if t.short_name == name and t.version.major == major),
key=lambda x: int(x.version.minor),
),
)
for name, major in sorted({(x.short_name, x.version.major) for x in tys})
]