pylint
pylint copied to clipboard
Maximum recursion depth crash with pyreverse ``-S`` option
Steps to reproduce
Given this filen named test.py:
import numpy as np
class ExempleClass :
typ = np.uint32
def __init__(self,D):
self.X = np.zeros( D, dtype=self.typ)
I run the command:
Pyreverse -o gv -A -my -S test.py
Current behavior
An error is raised. The error message is very very long. I paste here only the end of the message.
File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\diadefslib.py", line 121, in extract_classes
self.extract_classes(node, anc_level, association_level - 1)
[Previous line repeated 920 more times]
File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\diadefslib.py", line 115, in extract_classes
self.add_class(klass_node)
File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\diadefslib.py", line 85, in add_class
self.linker.visit(node)
File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\utils.py", line 217, in visit
methods[0](node)
File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\inspector.py", line 170, in visit_classdef
self.handle_assignattr_type(assignattr, node)
File "C:\ProgramData\Anaconda3\lib\site-packages\pylint\pyreverse\inspector.py", line 235, in handle_assignattr_type
values = set(node.infer())
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 132, in raise_if_nothing_inferred
yield next(generator)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 96, in wrapped
res = next(generator)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\bases.py", line 136, in _infer_stmts
for inferred in stmt.infer(context=context):
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\util.py", line 160, in limit_inference
yield from islice(iterator, size)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\context.py", line 113, in cache_generator
for result in generator:
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 132, in raise_if_nothing_inferred
yield next(generator)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 96, in wrapped
res = next(generator)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\inference.py", line 227, in infer_call
for callee in self.func.infer(context):
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\node_classes.py", line 357, in infer
return self._explicit_inference(self, context, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\__init__.py", line 95, in _inference_tip_cached
result = func(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\brain\brain_numpy_ndarray.py", line 141, in infer_numpy_ndarray
node = astroid.extract_node(ndarray)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 421, in extract_node
tree = parse(code, module_name=module_name)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 278, in parse
return builder.string_build(code, modname=module_name, path=path)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 144, in string_build
return self._post_build(module, "utf-8")
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 158, in _post_build
self.delayed_assattr(delayed)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\builder.py", line 225, in delayed_assattr
for inferred in node.expr.infer():
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 132, in raise_if_nothing_inferred
yield next(generator)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 96, in wrapped
res = next(generator)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\bases.py", line 136, in _infer_stmts
for inferred in stmt.infer(context=context):
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\util.py", line 160, in limit_inference
yield from islice(iterator, size)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\context.py", line 113, in cache_generator
for result in generator:
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 132, in raise_if_nothing_inferred
yield next(generator)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 93, in wrapped
generator = _func(node, context, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\inference.py", line 850, in infer_assign
stmts = list(self.assigned_stmts(context=context))
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\protocols.py", line 323, in _arguments_infer_argname
functype = self.parent.type
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 72, in __get__
val = self.wrapped(inst)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\scoped_nodes.py", line 1461, in type
for decorator in self.extra_decorators:
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 72, in __get__
val = self.wrapped(inst)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\scoped_nodes.py", line 1424, in extra_decorators
for assign in frame._get_assign_nodes():
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 34, in cached
cache[func] = result = func(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\scoped_nodes.py", line 2927, in _get_assign_nodes
return list(itertools.chain.from_iterable(children_assign_nodes))
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\scoped_nodes.py", line 2925, in <genexpr>
child_node._get_assign_nodes() for child_node in self.body
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 34, in cached
cache[func] = result = func(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\mixins.py", line 153, in _get_assign_nodes
return list(itertools.chain.from_iterable(children_assign_nodes))
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\mixins.py", line 151, in <genexpr>
for child_node in block
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 34, in cached
cache[func] = result = func(*args, **kwargs)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\node_classes.py", line 1953, in _get_assign_nodes
return [self] + list(self.value._get_assign_nodes())
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\decorators.py", line 28, in cached
cache = getattr(instance, "__cache", None)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\node_classes.py", line 2575, in __getattr__
return super().__getattr__(name)
File "C:\ProgramData\Anaconda3\lib\site-packages\astroid\bases.py", line 113, in __getattr__
return getattr(self._proxied, name)
RecursionError: maximum recursion depth exceeded while calling a Python object
Expected behavior
A generation of a file classes.gv
pylint --version output
pylint 2.5.0 astroid 2.4.0 Python 3.7.3 (default, Apr 24 2019, 15:29:51) [MSC v.1915 64 bit (AMD64)]
Look like it's the first issue opened for this recursion bug and the following one are: #3986, #3985, #3982, #3977, #3952, #3913, #3836
A possible temporary solution to the problem was proposed here: https://github.com/PyCQA/pylint/issues/3836#issuecomment-742512541
turns out this is a genuinely large recursion problem, and can be gotten around by putting the following in your pylintrc file:
init-hook='import sys; sys.setrecursionlimit(3 * sys.getrecursionlimit())'
I just checked with pylint 2.7 astroid 2.5, it's not crashing but it's taking really fucking long (more than 10mn and counting) for parsing 5 line of code.
Edit: OOM killed 15mn in. [1] 3619 killed pyreverse -o gv -A -my -S a.py
I have created a .pylintrc
file and added init-hook='import sys; sys.setrecursionlimit(3 * sys.getrecursionlimit())'
, however my pyreverse command still fails at the same point.
In fact, when I set it to init-hook='assert False'
, my pyreverse command still failed the same way. I suspect that the .pylintrc
file isn't being used for pyreverse commands. Perhaps that could be a fix here, allowing pylintrc files to be to be used for pyreverse commands (perhaps by having adding a --rcfile
argument to the pyreverse command)?
Could you please share the versions of pylint
and astroid
you are using?
Ideally with a minimum working example to reproduce the issue. Thank you!
On the latest main (2.14.0-b1) with the following a.py
file:
import numpy as np
class ExempleClass :
typ = np.uint32
def __init__(self,D):
self.X = np.zeros( D, dtype=self.typ)
running pyreverse -o gv -A -my -S a.py
ends up in OOM kill.
The offending option in this case is -S
("show recursively all associated off all associated classes").
The problem is with self.X = np.zeros(D, dtype=self.typ)
.
It is possible to limit the levels of association by using "-s2" (lower case -s
, "show <association_level> levels of associated classes not in
classDiagram
class ExempleClass {
X
typ
}
class ndarray {
T
base : NoneType
ctypes : NoneType
data : NoneType
dtype : NoneType
flags : NoneType
flat
imag
itemsize : NoneType
nbytes : NoneType
ndim : NoneType
real
shape
size : NoneType
strides : NoneType
all(axis, out, keepdims)
any(axis, out, keepdims)
argmax(axis, out)
argmin(axis, out)
argpartition(kth, axis, kind, order)
argsort(axis, kind, order)
astype(dtype, order, casting, subok, copy)
byteswap(inplace)
choose(choices, out, mode)
clip(min, max, out)
compress(condition, axis, out)
conj()
conjugate()
copy(order)
cumprod(axis, dtype, out)
cumsum(axis, dtype, out)
diagonal(offset, axis1, axis2)
dot(b, out)
dump(file)
dumps()
fill(value)
flatten(order)
getfield(dtype, offset)
item()
itemset()
max(axis, out)
mean(axis, dtype, out, keepdims)
min(axis, out, keepdims)
newbyteorder(new_order)
nonzero()
partition(kth, axis, kind, order)
prod(axis, dtype, out, keepdims)
ptp(axis, out)
put(indices, values, mode)
ravel(order)
repeat(repeats, axis)
reshape(shape, order)
resize(new_shape, refcheck)
round(decimals, out)
searchsorted(v, side, sorter)
setfield(val, dtype, offset)
setflags(write, align, uic)
sort(axis, kind, order)
squeeze(axis)
std(axis, dtype, out, ddof, keepdims)
sum(axis, dtype, out, keepdims)
swapaxes(axis1, axis2)
take(indices, axis, out, mode)
tobytes(order)
tofile(fid, sep, format)
tolist()
tostring(order)
trace(offset, axis1, axis2, dtype, out)
transpose()
var(axis, dtype, out, ddof, keepdims)
view(dtype, type)
}
class ndarray {
T : ndarray
base : NoneType
ctypes : NoneType
data : NoneType
dtype : NoneType
flags : NoneType
flat : ndarray
imag : ndarray
itemsize : NoneType
nbytes : NoneType
ndim : NoneType
real : ndarray
shape : ndarray
size : NoneType
strides : NoneType
all(axis, out, keepdims)
any(axis, out, keepdims)
argmax(axis, out)
argmin(axis, out)
argpartition(kth, axis, kind, order)
argsort(axis, kind, order)
astype(dtype, order, casting, subok, copy)
byteswap(inplace)
choose(choices, out, mode)
clip(min, max, out)
compress(condition, axis, out)
conj()
conjugate()
copy(order)
cumprod(axis, dtype, out)
cumsum(axis, dtype, out)
diagonal(offset, axis1, axis2)
dot(b, out)
dump(file)
dumps()
fill(value)
flatten(order)
getfield(dtype, offset)
item()
itemset()
max(axis, out)
mean(axis, dtype, out, keepdims)
min(axis, out, keepdims)
newbyteorder(new_order)
nonzero()
partition(kth, axis, kind, order)
prod(axis, dtype, out, keepdims)
ptp(axis, out)
put(indices, values, mode)
ravel(order)
repeat(repeats, axis)
reshape(shape, order)
resize(new_shape, refcheck)
round(decimals, out)
searchsorted(v, side, sorter)
setfield(val, dtype, offset)
setflags(write, align, uic)
sort(axis, kind, order)
squeeze(axis)
std(axis, dtype, out, ddof, keepdims)
sum(axis, dtype, out, keepdims)
swapaxes(axis1, axis2)
take(indices, axis, out, mode)
tobytes(order)
tofile(fid, sep, format)
tolist()
tostring(order)
trace(offset, axis1, axis2, dtype, out)
transpose()
var(axis, dtype, out, ddof, keepdims)
view(dtype, type)
}
class ndarray {
T : ndarray
base : NoneType
ctypes : NoneType
data : NoneType
dtype : NoneType
flags : NoneType
flat : ndarray
imag : ndarray
itemsize : NoneType
nbytes : NoneType
ndim : NoneType
real : ndarray
shape : ndarray
size : NoneType
strides : NoneType
all(axis, out, keepdims)
any(axis, out, keepdims)
argmax(axis, out)
argmin(axis, out)
argpartition(kth, axis, kind, order)
argsort(axis, kind, order)
astype(dtype, order, casting, subok, copy)
byteswap(inplace)
choose(choices, out, mode)
clip(min, max, out)
compress(condition, axis, out)
conj()
conjugate()
copy(order)
cumprod(axis, dtype, out)
cumsum(axis, dtype, out)
diagonal(offset, axis1, axis2)
dot(b, out)
dump(file)
dumps()
fill(value)
flatten(order)
getfield(dtype, offset)
item()
itemset()
max(axis, out)
mean(axis, dtype, out, keepdims)
min(axis, out, keepdims)
newbyteorder(new_order)
nonzero()
partition(kth, axis, kind, order)
prod(axis, dtype, out, keepdims)
ptp(axis, out)
put(indices, values, mode)
ravel(order)
repeat(repeats, axis)
reshape(shape, order)
resize(new_shape, refcheck)
round(decimals, out)
searchsorted(v, side, sorter)
setfield(val, dtype, offset)
setflags(write, align, uic)
sort(axis, kind, order)
squeeze(axis)
std(axis, dtype, out, ddof, keepdims)
sum(axis, dtype, out, keepdims)
swapaxes(axis1, axis2)
take(indices, axis, out, mode)
tobytes(order)
tofile(fid, sep, format)
tolist()
tostring(order)
trace(offset, axis1, axis2, dtype, out)
transpose()
var(axis, dtype, out, ddof, keepdims)
view(dtype, type)
}
class ndarray {
T : ndarray
base : NoneType
ctypes : NoneType
data : NoneType
dtype : NoneType
flags : NoneType
flat : ndarray
imag : ndarray
itemsize : NoneType
nbytes : NoneType
ndim : NoneType
real : ndarray
shape : ndarray
size : NoneType
strides : NoneType
all(axis, out, keepdims)
any(axis, out, keepdims)
argmax(axis, out)
argmin(axis, out)
argpartition(kth, axis, kind, order)
argsort(axis, kind, order)
astype(dtype, order, casting, subok, copy)
byteswap(inplace)
choose(choices, out, mode)
clip(min, max, out)
compress(condition, axis, out)
conj()
conjugate()
copy(order)
cumprod(axis, dtype, out)
cumsum(axis, dtype, out)
diagonal(offset, axis1, axis2)
dot(b, out)
dump(file)
dumps()
fill(value)
flatten(order)
getfield(dtype, offset)
item()
itemset()
max(axis, out)
mean(axis, dtype, out, keepdims)
min(axis, out, keepdims)
newbyteorder(new_order)
nonzero()
partition(kth, axis, kind, order)
prod(axis, dtype, out, keepdims)
ptp(axis, out)
put(indices, values, mode)
ravel(order)
repeat(repeats, axis)
reshape(shape, order)
resize(new_shape, refcheck)
round(decimals, out)
searchsorted(v, side, sorter)
setfield(val, dtype, offset)
setflags(write, align, uic)
sort(axis, kind, order)
squeeze(axis)
std(axis, dtype, out, ddof, keepdims)
sum(axis, dtype, out, keepdims)
swapaxes(axis1, axis2)
take(indices, axis, out, mode)
tobytes(order)
tofile(fid, sep, format)
tolist()
tostring(order)
trace(offset, axis1, axis2, dtype, out)
transpose()
var(axis, dtype, out, ddof, keepdims)
view(dtype, type)
}
class ndarray {
T : ndarray
base : NoneType
ctypes : NoneType
data : NoneType
dtype : NoneType
flags : NoneType
flat : ndarray
imag : ndarray
itemsize : NoneType
nbytes : NoneType
ndim : NoneType
real : ndarray
shape : ndarray
size : NoneType
strides : NoneType
all(axis, out, keepdims)
any(axis, out, keepdims)
argmax(axis, out)
argmin(axis, out)
argpartition(kth, axis, kind, order)
argsort(axis, kind, order)
astype(dtype, order, casting, subok, copy)
byteswap(inplace)
choose(choices, out, mode)
clip(min, max, out)
compress(condition, axis, out)
conj()
conjugate()
copy(order)
cumprod(axis, dtype, out)
cumsum(axis, dtype, out)
diagonal(offset, axis1, axis2)
dot(b, out)
dump(file)
dumps()
fill(value)
flatten(order)
getfield(dtype, offset)
item()
itemset()
max(axis, out)
mean(axis, dtype, out, keepdims)
min(axis, out, keepdims)
newbyteorder(new_order)
nonzero()
partition(kth, axis, kind, order)
prod(axis, dtype, out, keepdims)
ptp(axis, out)
put(indices, values, mode)
ravel(order)
repeat(repeats, axis)
reshape(shape, order)
resize(new_shape, refcheck)
round(decimals, out)
searchsorted(v, side, sorter)
setfield(val, dtype, offset)
setflags(write, align, uic)
sort(axis, kind, order)
squeeze(axis)
std(axis, dtype, out, ddof, keepdims)
sum(axis, dtype, out, keepdims)
swapaxes(axis1, axis2)
take(indices, axis, out, mode)
tobytes(order)
tofile(fid, sep, format)
tolist()
tostring(order)
trace(offset, axis1, axis2, dtype, out)
transpose()
var(axis, dtype, out, ddof, keepdims)
view(dtype, type)
}
class ndarray {
T : ndarray
base : NoneType
ctypes : NoneType
data : NoneType
dtype : NoneType
flags : NoneType
flat : ndarray
imag : ndarray
itemsize : NoneType
nbytes : NoneType
ndim : NoneType
real : ndarray
shape : ndarray
size : NoneType
strides : NoneType
all(axis, out, keepdims)
any(axis, out, keepdims)
argmax(axis, out)
argmin(axis, out)
argpartition(kth, axis, kind, order)
argsort(axis, kind, order)
astype(dtype, order, casting, subok, copy)
byteswap(inplace)
choose(choices, out, mode)
clip(min, max, out)
compress(condition, axis, out)
conj()
conjugate()
copy(order)
cumprod(axis, dtype, out)
cumsum(axis, dtype, out)
diagonal(offset, axis1, axis2)
dot(b, out)
dump(file)
dumps()
fill(value)
flatten(order)
getfield(dtype, offset)
item()
itemset()
max(axis, out)
mean(axis, dtype, out, keepdims)
min(axis, out, keepdims)
newbyteorder(new_order)
nonzero()
partition(kth, axis, kind, order)
prod(axis, dtype, out, keepdims)
ptp(axis, out)
put(indices, values, mode)
ravel(order)
repeat(repeats, axis)
reshape(shape, order)
resize(new_shape, refcheck)
round(decimals, out)
searchsorted(v, side, sorter)
setfield(val, dtype, offset)
setflags(write, align, uic)
sort(axis, kind, order)
squeeze(axis)
std(axis, dtype, out, ddof, keepdims)
sum(axis, dtype, out, keepdims)
swapaxes(axis1, axis2)
take(indices, axis, out, mode)
tobytes(order)
tofile(fid, sep, format)
tolist()
tostring(order)
trace(offset, axis1, axis2, dtype, out)
transpose()
var(axis, dtype, out, ddof, keepdims)
view(dtype, type)
}
class uint32 {
}
ndarray --* ndarray : T
ndarray --* ndarray : flat
ndarray --* ndarray : imag
ndarray --* ndarray : real
ndarray --* ndarray : shape
ndarray --* ExempleClass : X
uint32 --* ExempleClass : typ
Sounds like the issue could be closed if we put a lower default value when nothing is set ? It's easy to shoot oneself in the foot right now.
Well, -S
explicitly says recursively all associated classes.
It is not turned on by default (*), so I don't see a need to change this behaviour (the recursion error itself should be fixed of course).
(*) this is not entirely true, as using the --class
option to generate a diagram of a specific class sets -ASmy
as default.
But for normal invocations, i.e. pyreverse a.py
, no associated classes outside the own project will be shown.