500 lines
19 KiB
Python
500 lines
19 KiB
Python
from __future__ import absolute_import, division, print_function, unicode_literals
|
|
|
|
from abc import ABCMeta, abstractmethod
|
|
from contextlib import contextmanager
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import six
|
|
import textwrap
|
|
|
|
from stone.frontend.ir_generator import doc_ref_re
|
|
from stone.ir import (
|
|
is_alias,
|
|
resolve_aliases,
|
|
strip_alias
|
|
)
|
|
|
|
_MYPY = False
|
|
if _MYPY:
|
|
from stone.ir import Api
|
|
import typing # pylint: disable=import-error,useless-suppression
|
|
|
|
# Generic Dict key-val types
|
|
DelimTuple = typing.Tuple[typing.Text, typing.Text]
|
|
K = typing.TypeVar('K')
|
|
V = typing.TypeVar('V')
|
|
|
|
|
|
def remove_aliases_from_api(api):
|
|
# Resolve nested aliases from each namespace first. This way, when we replace an alias with
|
|
# its source later on, it too is alias free.
|
|
for namespace in api.namespaces.values():
|
|
for alias in namespace.aliases:
|
|
# This loops through each alias type chain, resolving each (nested) alias
|
|
# to its underlying type at the end of the chain (see resolve_aliases fn).
|
|
#
|
|
# It will continue until it no longer encounters a type
|
|
# with a data_type attribute - this ensures it resolves aliases
|
|
# that are subtypes of composites e.g. Lists
|
|
curr_type = alias
|
|
while hasattr(curr_type, 'data_type'):
|
|
curr_type.data_type = resolve_aliases(curr_type.data_type)
|
|
curr_type = curr_type.data_type
|
|
# Remove alias layers from each data type
|
|
for namespace in api.namespaces.values():
|
|
for data_type in namespace.data_types:
|
|
for field in data_type.fields:
|
|
strip_alias(field)
|
|
for route in namespace.routes:
|
|
# Strip inner aliases
|
|
strip_alias(route.arg_data_type)
|
|
strip_alias(route.result_data_type)
|
|
strip_alias(route.error_data_type)
|
|
|
|
# Strip top-level aliases
|
|
if is_alias(route.arg_data_type):
|
|
route.arg_data_type = route.arg_data_type.data_type
|
|
if is_alias(route.result_data_type):
|
|
route.result_data_type = route.result_data_type.data_type
|
|
if is_alias(route.error_data_type):
|
|
route.error_data_type = route.error_data_type.data_type
|
|
|
|
# Clear aliases
|
|
namespace.aliases = []
|
|
namespace.alias_by_name = {}
|
|
|
|
return api
|
|
|
|
|
|
@six.add_metaclass(ABCMeta)
|
|
class Backend(object):
|
|
"""
|
|
The parent class for all backends. All backends should extend this
|
|
class to be recognized as such.
|
|
|
|
You will want to implement the generate() function to do the generation
|
|
that you need.
|
|
|
|
Here's roughly what you need to do in generate().
|
|
1. Use the context manager output_to_relative_path() to specify an output file.
|
|
|
|
with output_to_relative_path('generated_code.py'):
|
|
...
|
|
|
|
2. Use the family of emit*() functions to write to the output file.
|
|
|
|
The target_folder_path attribute is the path to the folder where all
|
|
generated files should be created.
|
|
"""
|
|
|
|
# Can be overridden by a subclass
|
|
tabs_for_indents = False
|
|
|
|
# Can be overridden with an argparse.ArgumentParser object.
|
|
cmdline_parser = None # type: argparse.ArgumentParser
|
|
|
|
# Can be overridden by a subclass. If true, stone.data_type.Alias
|
|
# objects will be present in the API object. If false, aliases are masked
|
|
# by replacing them with duplicate type definitions as the source type.
|
|
# For backwards compatibility with existing backends defaults to false.
|
|
preserve_aliases = False
|
|
|
|
def __init__(self, target_folder_path, args):
|
|
# type: (str, typing.Optional[typing.Sequence[str]]) -> None
|
|
"""
|
|
Args:
|
|
target_folder_path (str): Path to the folder where all generated
|
|
files should be created.
|
|
"""
|
|
self.logger = logging.getLogger('Backend<%s>' %
|
|
self.__class__.__name__)
|
|
self.target_folder_path = target_folder_path
|
|
# Output is a list of strings that should be concatenated together for
|
|
# the final output.
|
|
self.output = [] # type: typing.List[typing.Text]
|
|
self.lineno = 1
|
|
self.cur_indent = 0
|
|
self.positional_placeholders = [] # type: typing.List[typing.Text]
|
|
self.named_placeholders = {} # type: typing.Dict[typing.Text, typing.Text]
|
|
|
|
self.args = None # type: typing.Optional[argparse.Namespace]
|
|
|
|
if self.cmdline_parser:
|
|
assert isinstance(self.cmdline_parser, argparse.ArgumentParser), (
|
|
'expected cmdline_parser to be ArgumentParser, got %r' %
|
|
self.cmdline_parser)
|
|
try:
|
|
self.args = self.cmdline_parser.parse_args(args)
|
|
except SystemExit:
|
|
print('Note: This is for backend-specific arguments which '
|
|
'follow arguments to Stone after a "--" delimiter.')
|
|
raise
|
|
|
|
@abstractmethod
|
|
def generate(self, api):
|
|
# type: (Api) -> None
|
|
"""
|
|
Subclasses should override this method. It's the entry point that is
|
|
invoked by the rest of the toolchain.
|
|
|
|
Args:
|
|
api (stone.api.Api): The API specification.
|
|
"""
|
|
raise NotImplementedError
|
|
|
|
@contextmanager
|
|
def output_to_relative_path(self, relative_path, mode='wb'):
|
|
# type: (typing.Text, typing.Text) -> typing.Iterator[None]
|
|
"""
|
|
Sets up backend so that all emits are directed towards the new file
|
|
created at :param:`relative_path`.
|
|
|
|
Clears the output buffer on enter and exit.
|
|
"""
|
|
full_path = os.path.join(self.target_folder_path, relative_path)
|
|
directory = os.path.dirname(full_path)
|
|
if not os.path.exists(directory):
|
|
self.logger.info('Creating %s', directory)
|
|
os.makedirs(directory)
|
|
|
|
self.logger.info('Generating %s', full_path)
|
|
self.clear_output_buffer()
|
|
yield
|
|
with open(full_path, mode) as f:
|
|
f.write(self.output_buffer_to_string().encode('utf-8'))
|
|
self.clear_output_buffer()
|
|
|
|
def output_buffer_to_string(self):
|
|
# type: () -> typing.Text
|
|
"""Returns the contents of the output buffer as a string."""
|
|
return ''.join(self.output).format(
|
|
*self.positional_placeholders,
|
|
**self.named_placeholders)
|
|
|
|
def clear_output_buffer(self):
|
|
self.output = []
|
|
self.positional_placeholders = []
|
|
self.named_placeholders = {}
|
|
|
|
def indent_step(self):
|
|
# type: () -> int
|
|
"""
|
|
Returns the size of a single indentation step.
|
|
"""
|
|
return 1 if self.tabs_for_indents else 4
|
|
|
|
@contextmanager
|
|
def indent(self, dent=None):
|
|
# type: (typing.Optional[int]) -> typing.Iterator[None]
|
|
"""
|
|
For the duration of the context manager, indentation will be increased
|
|
by dent. Dent is in units of spaces or tabs depending on the value of
|
|
the class variable tabs_for_indents. If dent is None, indentation will
|
|
increase by either four spaces or one tab.
|
|
"""
|
|
assert dent is None or dent >= 0, 'dent must be >= 0.'
|
|
if dent is None:
|
|
dent = self.indent_step()
|
|
self.cur_indent += dent
|
|
yield
|
|
self.cur_indent -= dent
|
|
|
|
def make_indent(self):
|
|
# type: () -> typing.Text
|
|
"""
|
|
Returns a string representing the current indentation. Indents can be
|
|
either spaces or tabs, depending on the value of the class variable
|
|
tabs_for_indents.
|
|
"""
|
|
if self.tabs_for_indents:
|
|
return '\t' * self.cur_indent
|
|
else:
|
|
return ' ' * self.cur_indent
|
|
|
|
@contextmanager
|
|
def capture_emitted_output(self, output_buffer):
|
|
# type: (six.StringIO) -> typing.Iterator[None]
|
|
original_output = self.output
|
|
self.output = []
|
|
yield
|
|
output_buffer.write(''.join(self.output))
|
|
self.output = original_output
|
|
|
|
def emit_raw(self, s):
|
|
# type: (typing.Text) -> None
|
|
"""
|
|
Adds the input string to the output buffer. The string must end in a
|
|
newline. It may contain any number of newline characters. No
|
|
indentation is generated.
|
|
"""
|
|
self.lineno += s.count('\n')
|
|
self._append_output(s.replace('{', '{{').replace('}', '}}'))
|
|
if len(s) > 0 and s[-1] != '\n':
|
|
raise AssertionError(
|
|
'Input string to emit_raw must end with a newline.')
|
|
|
|
def _append_output(self, s):
|
|
# type: (typing.Text) -> None
|
|
self.output.append(s)
|
|
|
|
def emit(self, s=''):
|
|
# type: (typing.Text) -> None
|
|
"""
|
|
Adds indentation, then the input string, and lastly a newline to the
|
|
output buffer. If s is an empty string (default) then an empty line is
|
|
created with no indentation.
|
|
"""
|
|
assert isinstance(s, six.text_type), 's must be a unicode string'
|
|
assert '\n' not in s, \
|
|
'String to emit cannot contain newline strings.'
|
|
if s:
|
|
self.emit_raw('%s%s\n' % (self.make_indent(), s))
|
|
else:
|
|
self.emit_raw('\n')
|
|
|
|
def emit_wrapped_text(
|
|
self,
|
|
s, # type: typing.Text
|
|
prefix='', # type: typing.Text
|
|
initial_prefix='', # type: typing.Text
|
|
subsequent_prefix='', # type: typing.Text
|
|
width=80, # type: int
|
|
break_long_words=False, # type: bool
|
|
break_on_hyphens=False # type: bool
|
|
):
|
|
# type: (...) -> None
|
|
"""
|
|
Adds the input string to the output buffer with indentation and
|
|
wrapping. The wrapping is performed by the :func:`textwrap.fill` Python
|
|
library function.
|
|
|
|
Args:
|
|
s (str): The input string to wrap.
|
|
prefix (str): The string to prepend to *every* line.
|
|
initial_prefix (str): The string to prepend to the first line of
|
|
the wrapped string. Note that the current indentation is
|
|
already added to each line.
|
|
subsequent_prefix (str): The string to prepend to every line after
|
|
the first. Note that the current indentation is already added
|
|
to each line.
|
|
width (int): The target width of each line including indentation
|
|
and text.
|
|
break_long_words (bool): Break words longer than width. If false,
|
|
those words will not be broken, and some lines might be longer
|
|
than width.
|
|
break_on_hyphens (bool): Allow breaking hyphenated words. If true,
|
|
wrapping will occur preferably on whitespaces and right after
|
|
hyphens part of compound words.
|
|
"""
|
|
indent = self.make_indent()
|
|
prefix = indent + prefix
|
|
|
|
self.emit_raw(textwrap.fill(s,
|
|
initial_indent=prefix + initial_prefix,
|
|
subsequent_indent=prefix + subsequent_prefix,
|
|
width=width,
|
|
break_long_words=break_long_words,
|
|
break_on_hyphens=break_on_hyphens,
|
|
) + '\n')
|
|
|
|
def emit_placeholder(self, s=''):
|
|
# type: (typing.Text) -> None
|
|
"""
|
|
Emits replacements fields that can be used to format the output string later.
|
|
"""
|
|
self._append_output('{%s}' % s)
|
|
|
|
def add_positional_placeholder(self, s):
|
|
# type: (typing.Text) -> None
|
|
"""
|
|
Format replacement fields corresponding to empty calls to emit_placeholder.
|
|
"""
|
|
self.positional_placeholders.append(s)
|
|
|
|
def add_named_placeholder(self, name, s):
|
|
# type: (typing.Text, typing.Text) -> None
|
|
"""
|
|
Format replacement fields corresponding to non-empty calls to emit_placeholder.
|
|
"""
|
|
self.named_placeholders[name] = s
|
|
|
|
@classmethod
|
|
def process_doc(cls, doc, handler):
|
|
# type: (str, typing.Callable[[str, str], str]) -> typing.Text
|
|
"""
|
|
Helper for parsing documentation references in Stone docstrings and
|
|
replacing them with more suitable annotations for the generated output.
|
|
|
|
Args:
|
|
doc (str): A Stone docstring.
|
|
handler: A function with the following signature:
|
|
`(tag: str, value: str) -> str`. It will be called for every
|
|
reference found in the docstring with the tag and value parsed
|
|
for you. The returned string will be substituted in the
|
|
docstring in place of the reference.
|
|
"""
|
|
assert isinstance(doc, six.text_type), \
|
|
'Expected string (unicode in PY2), got %r.' % type(doc)
|
|
cur_index = 0
|
|
parts = []
|
|
for match in doc_ref_re.finditer(doc):
|
|
# Append the part of the doc that is not part of any reference.
|
|
start, end = match.span()
|
|
parts.append(doc[cur_index:start])
|
|
cur_index = end
|
|
|
|
# Call the handler with the next tag and value.
|
|
tag = match.group('tag')
|
|
val = match.group('val')
|
|
sub = handler(tag, val)
|
|
parts.append(sub)
|
|
parts.append(doc[cur_index:])
|
|
return ''.join(parts)
|
|
|
|
|
|
class CodeBackend(Backend):
|
|
"""
|
|
Extend this instead of :class:`Backend` when generating source code.
|
|
Contains helper functions specific to code generation.
|
|
"""
|
|
# pylint: disable=abstract-method
|
|
|
|
def filter_out_none_valued_keys(self, d):
|
|
# type: (typing.Dict[K, V]) -> typing.Dict[K, V]
|
|
"""Given a dict, returns a new dict with all the same key/values except
|
|
for keys that had values of None."""
|
|
new_d = {}
|
|
for k, v in d.items():
|
|
if v is not None:
|
|
new_d[k] = v
|
|
return new_d
|
|
|
|
def generate_multiline_list(
|
|
self,
|
|
items, # type: typing.List[typing.Text]
|
|
before='', # type: typing.Text
|
|
after='', # type: typing.Text
|
|
delim=('(', ')'), # type: DelimTuple
|
|
compact=True, # type: bool
|
|
sep=',', # type: typing.Text
|
|
skip_last_sep=False # type: bool
|
|
):
|
|
# type: (...) -> None
|
|
"""
|
|
Given a list of items, emits one item per line.
|
|
|
|
This is convenient for function prototypes and invocations, as well as
|
|
for instantiating arrays, sets, and maps in some languages.
|
|
|
|
TODO(kelkabany): A backend that uses tabs cannot be used with this
|
|
if compact is false.
|
|
|
|
Args:
|
|
items (list[str]): Should contain the items to generate a list of.
|
|
before (str): The string to come before the list of items.
|
|
after (str): The string to follow the list of items.
|
|
delim (str, str): The first element is added immediately following
|
|
`before`. The second element is added prior to `after`.
|
|
compact (bool): In compact mode, the enclosing parentheses are on
|
|
the same lines as the first and last list item.
|
|
sep (str): The string that follows each list item when compact is
|
|
true. If compact is false, the separator is omitted for the
|
|
last item.
|
|
skip_last_sep (bool): When compact is false, whether the last line
|
|
should have a trailing separator. Ignored when compact is true.
|
|
"""
|
|
assert len(delim) == 2 and isinstance(delim[0], six.text_type) and \
|
|
isinstance(delim[1], six.text_type), 'delim must be a tuple of two unicode strings.'
|
|
|
|
if len(items) == 0:
|
|
self.emit(before + delim[0] + delim[1] + after)
|
|
return
|
|
if len(items) == 1:
|
|
self.emit(before + delim[0] + items[0] + delim[1] + after)
|
|
return
|
|
|
|
if compact:
|
|
self.emit(before + delim[0] + items[0] + sep)
|
|
def emit_list(items):
|
|
items = items[1:]
|
|
for (i, item) in enumerate(items):
|
|
if i == len(items) - 1:
|
|
self.emit(item + delim[1] + after)
|
|
else:
|
|
self.emit(item + sep)
|
|
if before or delim[0]:
|
|
with self.indent(len(before) + len(delim[0])):
|
|
emit_list(items)
|
|
else:
|
|
emit_list(items)
|
|
else:
|
|
if before or delim[0]:
|
|
self.emit(before + delim[0])
|
|
with self.indent():
|
|
for (i, item) in enumerate(items):
|
|
if i == len(items) - 1 and skip_last_sep:
|
|
self.emit(item)
|
|
else:
|
|
self.emit(item + sep)
|
|
if delim[1] or after:
|
|
self.emit(delim[1] + after)
|
|
elif delim[1]:
|
|
self.emit(delim[1])
|
|
|
|
@contextmanager
|
|
def block(
|
|
self,
|
|
before='', # type: typing.Text
|
|
after='', # type: typing.Text
|
|
delim=('{', '}'), # type: DelimTuple
|
|
dent=None, # type: typing.Optional[int]
|
|
allman=False # type: bool
|
|
):
|
|
# type: (...) -> typing.Iterator[None]
|
|
"""
|
|
A context manager that emits configurable lines before and after an
|
|
indented block of text.
|
|
|
|
This is convenient for class and function definitions in some
|
|
languages.
|
|
|
|
Args:
|
|
before (str): The string to be output in the first line which is
|
|
not indented..
|
|
after (str): The string to be output in the last line which is
|
|
not indented.
|
|
delim (str, str): The first element is added immediately following
|
|
`before` and a space. The second element is added prior to a
|
|
space and then `after`.
|
|
dent (int): The amount to indent the block. If none, the default
|
|
indentation increment is used (four spaces or one tab).
|
|
allman (bool): Indicates whether to use `Allman` style indentation,
|
|
or the default `K&R` style. If there is no `before` string this
|
|
is ignored. For more details about indent styles see
|
|
http://en.wikipedia.org/wiki/Indent_style
|
|
"""
|
|
assert len(delim) == 2, 'delim must be a tuple of length 2'
|
|
assert (isinstance(delim[0], (six.text_type, type(None))) and
|
|
isinstance(delim[1], (six.text_type, type(None)))), (
|
|
'delim must be a tuple of two optional strings.')
|
|
|
|
if before and not allman:
|
|
if delim[0] is not None:
|
|
self.emit('{} {}'.format(before, delim[0]))
|
|
else:
|
|
self.emit(before)
|
|
else:
|
|
if before:
|
|
self.emit(before)
|
|
if delim[0] is not None:
|
|
self.emit(delim[0])
|
|
|
|
with self.indent(dent):
|
|
yield
|
|
|
|
if delim[1] is not None:
|
|
self.emit(delim[1] + after)
|
|
else:
|
|
self.emit(after)
|