Maximum recursion depth exceeded problem

[SlavomirMazurPantheon]

Hello,

after a few hundreds of use of pyang parser call, which uses the IETF plugin, we get the following maximum recursion depth error:

Traceback (most recent call last):
  File "try_pyang.py", line 69, in <module>
    main()
  File "try_pyang.py", line 53, in main
    ctx.validate()
  File "/usr/local/lib/python3.6/dist-packages/pyang/context.py", line 331, in validate
    statements.validate_module(self, m)
  File "/usr/local/lib/python3.6/dist-packages/pyang/statements.py", line 413, in validate_module
    iterate(module, phase)
  File "/usr/local/lib/python3.6/dist-packages/pyang/statements.py", line 370, in iterate
    res = f(ctx, stmt)
  File "/usr/local/lib/python3.6/dist-packages/pyang/statements.py", line 35, in <lambda>
    return lambda *args, **kargs: (one(*args, **kargs), two(*args, **kargs))[1]
  File "/usr/local/lib/python3.6/dist-packages/pyang/statements.py", line 35, in <lambda>
    return lambda *args, **kargs: (one(*args, **kargs), two(*args, **kargs))[1]
  File "/usr/local/lib/python3.6/dist-packages/pyang/statements.py", line 35, in <lambda>
    return lambda *args, **kargs: (one(*args, **kargs), two(*args, **kargs))[1]
  [Previous line repeated 37 more times]
RecursionError: maximum recursion depth exceeded

Workaround for now was to reset the state of the variables in statements.py after each use of the pyang parser.

Minimal, reproducible example:

import io
import time
import sys
from pyang.context import Context
from pyang.error import error_codes
from pyang.repository import FileRepository
from pyang import plugin
from pyang.plugins.depend import emit_depend

DEFAULT_OPTIONS = {
    'path': [],
    'deviations': [],
    'features': [],
    'format': 'yang',
    'keep_comments': True,
    'no_path_recurse': False,
    'trim_yin': False,
    'yang_canonical': False,
    'yang_remove_unused_imports': False,
    # -- errors
    'ignore_error_tags': [],
    'ignore_errors': [],
    'list_errors': True,
    'print_error_code': False,
    'errors': [],
    'warnings': [code for code, desc in error_codes.items() if desc[0] > 4],
    'verbose': True,
}
"""Default options for pyang command line"""

_COPY_OPTIONS = [
    'canonical',
    'max_line_len',
    'max_identifier_len',
    'trim_yin',
    'lax_xpath_checks',
    'strict',
]
"""copy options to pyang context options"""

class objectify(object):
    """Utility for providing object access syntax (.attr) to dicts"""

    def __init__(self, *args, **kwargs):
        for entry in args:
            self.__dict__.update(entry)

        self.__dict__.update(kwargs)

    def __getattr__(self, _):
        return None

    def __setattr__(self, attr, value):
        self.__dict__[attr] = value

def create_context(path='.', *options, **kwargs):
    opts = objectify(DEFAULT_OPTIONS, *options, **kwargs)
    repo = FileRepository(path, no_path_recurse=opts.no_path_recurse)

    ctx = Context(repo)
    ctx.opts = opts

    for attr in _COPY_OPTIONS:
        setattr(ctx, attr, getattr(opts, attr))

    # make a map of features to support, per module (taken from pyang bin)
    for feature_name in opts.features:
        (module_name, features) = _parse_features_string(feature_name)
        ctx.features[module_name] = features

    # apply deviations (taken from pyang bin)
    for file_name in opts.deviations:
        with io.open(file_name, "r", encoding="utf-8") as fd:
            module = ctx.add_module(file_name, fd.read())
            if module is not None:
                ctx.deviation_modules.append(module)
    return ctx

def main():
    # Init plugins
    # NOTE: Path to yang file (replace with your own)
    dir_path = '/var/yang/all_modules/'
    # NOTE: Name of yang file (replace with your own)
    module_name = '[email protected]'
    plugin.plugins = []
    plugin.init([])

    # Create context
    ctx = create_context(dir_path)
    ctx.opts.lint_namespace_prefixes = []
    ctx.opts.lint_modulename_prefixes = []
    ctx.opts.ietf = True
    ctx.opts.depend_recurse = True
    ctx.opts.depend_ignore = []

    # Setup plugins
    for p in plugin.plugins:
        p.setup_ctx(ctx)

    m = []
    with open(dir_path + module_name, 'r', encoding="utf-8") as yang_file:
        module = yang_file.read()
        if module is None:
            print('no module provided')
        m = ctx.add_module('dir_path + module_name', module)
        if m is None:
            m = []
        else:
            m = [m]

    ctx.validate()

    f = io.StringIO()
    emit_depend(ctx, m, f)

if __name__ == "__main__":
    sys.setrecursionlimit(200)
    for i in range(1000):
        start_time = time.time()
        main()
        print("%d --- %s seconds ---" %(i,time.time() - start_time))

NOTE: I manually decreased recursion limit for test purposes, because it would take longer for the error to occur (but it would occur anyway)

[fredgan]

Hi @SlavomirMazurPantheon , I wanted to reproduce the errors with the example you attached, but _parse_features_string(feature_name) on line 68 was not defined. Can you tell me where is the implementation of _parse_features_string(), or can you provide it?

[SlavomirMazurPantheon]

Hello @fredgan,

I'm sorry, I forgot to include it in the minimal, reproducible example. Here is the implementation of the method:

def _parse_features_string(feature_str):
    if feature_str.find(':') == -1:
        return (feature_str, [])

    [module_name, rest] = feature_str.split(':', 1)
    if rest == '':
        return (module_name, [])

    features = rest.split(',')
    return (module_name, features)