From 3c7ae4cb3c3473af7a699f27b51f9498d06f6698 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:41:14 -0600 Subject: [PATCH 01/12] gh-157056: Add tests and NEWS for STRING format comprehension annotations --- Lib/test/test_annotationlib_string_special.py | 57 +++++++++++++++++++ ...9-08-01-00-00.gh-issue-157056.annotstr.rst | 4 ++ 2 files changed, 61 insertions(+) create mode 100644 Lib/test/test_annotationlib_string_special.py create mode 100644 Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst diff --git a/Lib/test/test_annotationlib_string_special.py b/Lib/test/test_annotationlib_string_special.py new file mode 100644 index 000000000000000..ca2fd90eb59069e --- /dev/null +++ b/Lib/test/test_annotationlib_string_special.py @@ -0,0 +1,57 @@ +"""STRING-format edge cases for annotationlib (gh-157056).""" + +import unittest + +from annotationlib import Format, get_annotations, type_repr + + +class TestStringFormatSpecialAnnotations(unittest.TestCase): + def test_dict_comprehension_annotation(self): + def f(x: {k: v for k, v in items}): + pass + + self.assertEqual( + get_annotations(f, format=Format.STRING), + {"x": "{k: v for k, v in items}"}, + ) + + def test_lambda_annotation(self): + def g(x: lambda q: q): + pass + + g_anno = get_annotations(g, format=Format.STRING) + self.assertEqual(g_anno, {"x": "lambda q: q"}) + self.assertNotIn("0x", g_anno["x"].lower()) + + def test_generator_expression_annotation(self): + def h(x: (w for w in seq)): + pass + + h_anno = get_annotations(h, format=Format.STRING) + self.assertEqual(h_anno, {"x": "(w for w in seq)"}) + self.assertNotIn("0x", h_anno["x"].lower()) + + def test_mixed_annotations(self): + def mixed(a: int, b: {k: v for k, v in items}, c: lambda q: q): + pass + + self.assertEqual( + get_annotations(mixed, format=Format.STRING), + { + "a": "int", + "b": "{k: v for k, v in items}", + "c": "lambda q: q", + }, + ) + + def test_type_repr_lambda_and_genexpr_have_no_address(self): + lam = lambda q: q + self.assertTrue(type_repr(lam).endswith("")) + self.assertNotIn("0x", type_repr(lam).lower()) + gen = (w for w in ()) + self.assertTrue(type_repr(gen).endswith("")) + self.assertNotIn("0x", type_repr(gen).lower()) + + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst b/Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst new file mode 100644 index 000000000000000..4436d5aaabc24f0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst @@ -0,0 +1,4 @@ +:func:`annotationlib.get_annotations` with ``format=Format.STRING`` no +longer fails on dict-comprehension annotations, and no longer emits +non-deterministic strings that embed a memory address for ``lambda`` and +generator-expression annotations. From bfe761cd8f0d59ec2f24bdbf419341fa22ea2020 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 19:44:22 -0600 Subject: [PATCH 02/12] gh-157056: Fix STRING format for comprehension and lambda annotations get_annotations(..., format=Format.STRING) raised ValueError on dict comprehension annotations because fake-globals iteration cannot unpack pair targets. Recover the annotation text from source in that case. Lambda and generator-expression annotations are syntax, so they were stringified with repr() and leaked a memory address. Prefer the source text when available, and use a stable type_repr() otherwise. --- Lib/annotationlib.py | 103 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 5 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 8204c762cce8a2b..c9d7e33bd412818 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -750,13 +750,20 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): argdefs=annotate.__defaults__, kwdefaults=annotate.__kwdefaults__, ) - annos = func(Format.VALUE_WITH_FAKE_GLOBALS) + try: + annos = func(Format.VALUE_WITH_FAKE_GLOBALS) + except ValueError: + # Dict comprehensions such as `{k: v for k, v in items}` unpack + # each iterated element. Fake-globals iteration yields a single + # starred stringifier, so unpacking raises ValueError. Recover + # the original annotation text from source when we can. + sourced = _string_annotations_from_source(owner) + if sourced is not None: + return sourced + raise if _is_evaluate: return _stringify_single(annos) - return { - key: _stringify_single(val) - for key, val in annos.items() - } + return _stringify_annotation_dict(annos, owner) elif format == Format.FORWARDREF: # FORWARDREF is implemented similarly to STRING, but there are two changes, # at the beginning and the end of the process. @@ -878,6 +885,81 @@ def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluat return tuple(new_closure), cell_dict +def _string_annotations_from_source(obj): + """Best-effort STRING annotations reconstructed from *obj*'s source AST. + + Used when fake-globals evaluation cannot stringify an annotation (dict + comprehensions, lambdas, generator expressions). Returns None if source + is unavailable. inspect is imported lazily because it imports this module. + """ + if obj is None: + return None + try: + import inspect + import textwrap + source = inspect.getsource(obj) + except (OSError, TypeError, RecursionError): + return None + source = textwrap.dedent(source) + try: + tree = ast.parse(source) + except SyntaxError: + return None + if not tree.body: + return None + node = tree.body[0] + result = {} + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for arg in ( + *node.args.posonlyargs, + *node.args.args, + *node.args.kwonlyargs, + ): + if arg.annotation is not None: + result[arg.arg] = ast.unparse(arg.annotation) + if node.args.vararg is not None and node.args.vararg.annotation is not None: + result[node.args.vararg.arg] = ast.unparse(node.args.vararg.annotation) + if node.args.kwarg is not None and node.args.kwarg.annotation is not None: + result[node.args.kwarg.arg] = ast.unparse(node.args.kwarg.annotation) + if node.returns is not None: + result["return"] = ast.unparse(node.returns) + return result + if isinstance(node, ast.ClassDef): + for stmt in node.body: + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): + result[stmt.target.id] = ast.unparse(stmt.annotation) + return result + return None + + +def _is_runtime_constructed(value): + """True if *value* was created at annotation-eval time and has no AST. + + Lambdas and generator expressions are syntax, not name lookups, so the + fake-globals stringifier never sees them. Their repr() embeds a memory + address and is not a valid annotation string. + """ + return isinstance(value, ( + types.FunctionType, + types.BuiltinFunctionType, + types.MethodType, + types.GeneratorType, + types.AsyncGeneratorType, + types.CoroutineType, + )) + + +def _stringify_annotation_dict(annos, owner): + sourced = _string_annotations_from_source(owner) + result = {} + for key, val in annos.items(): + if sourced is not None and key in sourced and _is_runtime_constructed(val): + result[key] = sourced[key] + else: + result[key] = _stringify_single(val) + return result + + def _stringify_single(anno): if anno is ...: return "..." @@ -886,6 +968,10 @@ def _stringify_single(anno): return anno elif isinstance(anno, _Template): return ast.unparse(_template_to_ast(anno)) + elif _is_runtime_constructed(anno): + # Lambdas and generator expressions are syntax, not name lookups. + # repr() embeds a memory address; type_repr() is stable. + return type_repr(anno) else: return repr(anno) @@ -1093,6 +1179,13 @@ def type_repr(value): if value.__module__ == "builtins": return value.__qualname__ return f"{value.__module__}.{value.__qualname__}" + elif isinstance(value, ( + types.GeneratorType, + types.AsyncGeneratorType, + types.CoroutineType, + )): + # repr() of these objects embeds a memory address. + return value.__qualname__ elif isinstance(value, _Template): tree = _template_to_ast(value) return ast.unparse(tree) From c98397656825cdfcdf16b05ab5f357f31dead1a4 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:09:17 -0600 Subject: [PATCH 03/12] gh-157056: Tighten STRING fallback after review Only consult source when fake-globals cannot stringify. Limit the ValueError handler to unpack errors, keep class-body control-flow annotations, and avoid requoting string-literal annotations. --- Lib/annotationlib.py | 1268 +----------------------------------------- 1 file changed, 1 insertion(+), 1267 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index c9d7e33bd412818..311c8dd0658759f 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -1,1267 +1 @@ -"""Helpers for introspecting and wrapping annotations.""" - -import ast -import builtins -import enum -import keyword -import sys -import types - -__all__ = [ - "Format", - "ForwardRef", - "call_annotate_function", - "call_evaluate_function", - "get_annotate_from_class_namespace", - "get_annotations", - "annotations_to_string", - "type_repr", -] - - -class Format(enum.IntEnum): - VALUE = 1 - VALUE_WITH_FAKE_GLOBALS = 2 - FORWARDREF = 3 - STRING = 4 - - -_sentinel = object() -# Following `NAME_ERROR_MSG` in `ceval_macros.h`: -_NAME_ERROR_MSG = "name '{name:.200}' is not defined" - - -# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be -# preserved for compatibility with the old typing.ForwardRef class. The remaining -# names are private. -_SLOTS = ( - "__forward_is_argument__", - "__forward_is_class__", - "__forward_module__", - "__weakref__", - "__arg__", - "__globals__", - "__extra_names__", - "__code__", - "__ast_node__", - "__cell__", - "__owner__", - "__stringifier_dict__", - "__resolved_str_cache__", -) - - -class ForwardRef: - """Wrapper that holds a forward reference. - - Constructor arguments: - * arg: a string representing the code to be evaluated. - * module: the module where the forward reference was created. - Must be a string, not a module object. - * owner: The owning object (module, class, or function). - * is_argument: Does nothing, retained for compatibility. - * is_class: True if the forward reference was created in class scope. - - """ - - __slots__ = _SLOTS - - def __init__( - self, - arg, - *, - module=None, - owner=None, - is_argument=True, - is_class=False, - ): - if not isinstance(arg, str): - raise TypeError(f"Forward reference must be a string -- got {arg!r}") - - self.__arg__ = arg - self.__forward_is_argument__ = is_argument - self.__forward_is_class__ = is_class - self.__forward_module__ = module - self.__owner__ = owner - # These are always set to None here but may be non-None if a ForwardRef - # is created through __class__ assignment on a _Stringifier object. - self.__globals__ = None - # This may be either a cell object (for a ForwardRef referring to a single name) - # or a dict mapping cell names to cell objects (for a ForwardRef containing references - # to multiple names). - self.__cell__ = None - self.__extra_names__ = None - # These are initially None but serve as a cache and may be set to a non-None - # value later. - self.__code__ = None - self.__ast_node__ = None - self.__resolved_str_cache__ = None - - def __init_subclass__(cls, /, *args, **kwds): - raise TypeError("Cannot subclass ForwardRef") - - def evaluate( - self, - *, - globals=None, - locals=None, - type_params=None, - owner=None, - format=Format.VALUE, - ): - """Evaluate the forward reference and return the value. - - If the forward reference cannot be evaluated, raise an exception. - """ - match format: - case Format.STRING: - return self.__resolved_str__ - case Format.VALUE: - is_forwardref_format = False - case Format.FORWARDREF: - is_forwardref_format = True - case _: - raise NotImplementedError(format) - if isinstance(self.__cell__, types.CellType): - try: - return self.__cell__.cell_contents - except ValueError: - pass - if owner is None: - owner = self.__owner__ - - if globals is None and self.__forward_module__ is not None: - globals = getattr( - sys.modules.get(self.__forward_module__, None), "__dict__", None - ) - if globals is None: - globals = self.__globals__ - if globals is None: - if isinstance(owner, type): - module_name = getattr(owner, "__module__", None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - globals = getattr(module, "__dict__", None) - elif isinstance(owner, types.ModuleType): - globals = getattr(owner, "__dict__", None) - elif callable(owner): - globals = getattr(owner, "__globals__", None) - - # If we pass None to eval() below, the globals of this module are used. - if globals is None: - globals = {} - - if type_params is None and owner is not None: - type_params = getattr(owner, "__type_params__", None) - - if locals is None: - locals = {} - if isinstance(owner, type): - locals.update(vars(owner)) - elif ( - type_params is not None - or isinstance(self.__cell__, dict) - or self.__extra_names__ - ): - # Create a new locals dict if necessary, - # to avoid mutating the argument. - locals = dict(locals) - - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - if type_params is not None: - for param in type_params: - locals.setdefault(param.__name__, param) - - # Similar logic can be used for nonlocals, which should not - # override locals. - if isinstance(self.__cell__, dict): - for cell_name, cell in self.__cell__.items(): - try: - cell_value = cell.cell_contents - except ValueError: - pass - else: - locals.setdefault(cell_name, cell_value) - - if self.__extra_names__: - locals.update(self.__extra_names__) - - arg = self.__forward_arg__ - if arg.isidentifier() and not keyword.iskeyword(arg): - if arg in locals: - return locals[arg] - elif arg in globals: - return globals[arg] - elif hasattr(builtins, arg): - return getattr(builtins, arg) - elif is_forwardref_format: - return self - else: - raise NameError(_NAME_ERROR_MSG.format(name=arg), name=arg) - else: - code = self.__forward_code__ - try: - return eval(code, globals=globals, locals=locals) - except Exception: - if not is_forwardref_format: - raise - - # All variables, in scoping order, should be checked before - # triggering __missing__ to create a _Stringifier. - new_locals = _StringifierDict( - {**builtins.__dict__, **globals, **locals}, - globals=globals, - owner=owner, - is_class=self.__forward_is_class__, - format=format, - ) - try: - result = eval(code, globals=globals, locals=new_locals) - except Exception: - return self - else: - new_locals.transmogrify(self.__cell__) - return result - - @property - def __forward_arg__(self): - if self.__arg__ is not None: - return self.__arg__ - if self.__ast_node__ is not None: - self.__arg__ = ast.unparse(self.__ast_node__) - return self.__arg__ - raise AssertionError( - "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" - ) - - @property - def __resolved_str__(self): - # __forward_arg__ with any names from __extra_names__ replaced - # with the type_repr of the value they represent - if self.__resolved_str_cache__ is None: - resolved_str = self.__forward_arg__ - names = self.__extra_names__ - - if names: - visitor = _ExtraNameFixer(names) - ast_expr = ast.parse(resolved_str, mode="eval").body - node = visitor.visit(ast_expr) - resolved_str = ast.unparse(node) - - self.__resolved_str_cache__ = resolved_str - - return self.__resolved_str_cache__ - - @property - def __forward_code__(self): - if self.__code__ is not None: - return self.__code__ - arg = self.__forward_arg__ - try: - self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") - except SyntaxError: - raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") - return self.__code__ - - def __eq__(self, other): - if not isinstance(other, ForwardRef): - return NotImplemented - return ( - self.__forward_arg__ == other.__forward_arg__ - and self.__forward_module__ == other.__forward_module__ - # Use "is" here because we use id() for this in __hash__ - # because dictionaries are not hashable. - and self.__globals__ is other.__globals__ - and self.__forward_is_class__ == other.__forward_is_class__ - # Two separate cells are always considered unequal in forward refs. - and ( - {name: id(cell) for name, cell in self.__cell__.items()} - == {name: id(cell) for name, cell in other.__cell__.items()} - if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict) - else self.__cell__ is other.__cell__ - ) - and self.__owner__ == other.__owner__ - and ( - (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == - (tuple(sorted(other.__extra_names__.items())) if other.__extra_names__ else None) - ) - ) - - def __hash__(self): - return hash(( - self.__forward_arg__, - self.__forward_module__, - id(self.__globals__), # dictionaries are not hashable, so hash by identity - self.__forward_is_class__, - ( # cells are not hashable as well - tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()])) - if isinstance(self.__cell__, dict) else id(self.__cell__), - ), - self.__owner__, - tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, - )) - - def __or__(self, other): - return types.UnionType[self, other] - - def __ror__(self, other): - return types.UnionType[other, self] - - def __repr__(self): - extra = [] - if self.__forward_module__ is not None: - extra.append(f", module={self.__forward_module__!r}") - if self.__forward_is_class__: - extra.append(", is_class=True") - if self.__owner__ is not None: - extra.append(f", owner={self.__owner__!r}") - return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})" - - -_Template = type(t"") - - -class _Stringifier: - # Must match the slots on ForwardRef, so we can turn an instance of one into an - # instance of the other in place. - __slots__ = _SLOTS - - def __init__( - self, - node, - globals=None, - owner=None, - is_class=False, - cell=None, - *, - stringifier_dict, - extra_names=None, - ): - # Either an AST node or a simple str (for the common case where a ForwardRef - # represent a single name). - assert isinstance(node, (ast.AST, str)) - self.__arg__ = None - self.__forward_is_argument__ = False - self.__forward_is_class__ = is_class - self.__forward_module__ = None - self.__code__ = None - self.__ast_node__ = node - self.__globals__ = globals - self.__extra_names__ = extra_names - self.__cell__ = cell - self.__owner__ = owner - self.__stringifier_dict__ = stringifier_dict - self.__resolved_str_cache__ = None # Needed for ForwardRef - - def __convert_to_ast(self, other): - if isinstance(other, _Stringifier): - if isinstance(other.__ast_node__, str): - return ast.Name(id=other.__ast_node__), other.__extra_names__ - return other.__ast_node__, other.__extra_names__ - elif type(other) is _Template: - return _template_to_ast(other), None - elif ( - # In STRING format we don't bother with the create_unique_name() dance; - # it's better to emit the repr() of the object instead of an opaque name. - self.__stringifier_dict__.format == Format.STRING - or other is None - or type(other) in (str, int, float, bool, complex) - ): - return ast.Constant(value=other), None - elif type(other) is dict: - extra_names = {} - keys = [] - values = [] - for key, value in other.items(): - new_key, new_extra_names = self.__convert_to_ast(key) - if new_extra_names is not None: - extra_names.update(new_extra_names) - keys.append(new_key) - new_value, new_extra_names = self.__convert_to_ast(value) - if new_extra_names is not None: - extra_names.update(new_extra_names) - values.append(new_value) - return ast.Dict(keys, values), extra_names - elif type(other) in (list, tuple, set): - extra_names = {} - elts = [] - for elt in other: - new_elt, new_extra_names = self.__convert_to_ast(elt) - if new_extra_names is not None: - extra_names.update(new_extra_names) - elts.append(new_elt) - ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] - return ast_class(elts), extra_names - else: - name = self.__stringifier_dict__.create_unique_name() - return ast.Name(id=name), {name: other} - - def __convert_to_ast_getitem(self, other): - if isinstance(other, slice): - extra_names = {} - - def conv(obj): - if obj is None: - return None - new_obj, new_extra_names = self.__convert_to_ast(obj) - if new_extra_names is not None: - extra_names.update(new_extra_names) - return new_obj - - return ast.Slice( - lower=conv(other.start), - upper=conv(other.stop), - step=conv(other.step), - ), extra_names - else: - return self.__convert_to_ast(other) - - def __get_ast(self): - node = self.__ast_node__ - if isinstance(node, str): - return ast.Name(id=node) - return node - - def __make_new(self, node, extra_names=None): - new_extra_names = {} - if self.__extra_names__ is not None: - new_extra_names.update(self.__extra_names__) - if extra_names is not None: - new_extra_names.update(extra_names) - stringifier = _Stringifier( - node, - self.__globals__, - self.__owner__, - self.__forward_is_class__, - stringifier_dict=self.__stringifier_dict__, - extra_names=new_extra_names or None, - ) - self.__stringifier_dict__.stringifiers.append(stringifier) - return stringifier - - # Must implement this since we set __eq__. We hash by identity so that - # stringifiers in dict keys are kept separate. - def __hash__(self): - return id(self) - - def __getitem__(self, other): - # Special case, to avoid stringifying references to class-scoped variables - # as '__classdict__["x"]'. - if self.__ast_node__ == "__classdict__": - raise KeyError - if isinstance(other, tuple): - extra_names = {} - elts = [] - for elt in other: - new_elt, new_extra_names = self.__convert_to_ast_getitem(elt) - if new_extra_names is not None: - extra_names.update(new_extra_names) - elts.append(new_elt) - other = ast.Tuple(elts) - else: - other, extra_names = self.__convert_to_ast_getitem(other) - assert isinstance(other, ast.AST), repr(other) - return self.__make_new(ast.Subscript(self.__get_ast(), other), extra_names) - - def __getattr__(self, attr): - return self.__make_new(ast.Attribute(self.__get_ast(), attr)) - - def __call__(self, *args, **kwargs): - extra_names = {} - ast_args = [] - for arg in args: - new_arg, new_extra_names = self.__convert_to_ast(arg) - if new_extra_names is not None: - extra_names.update(new_extra_names) - ast_args.append(new_arg) - ast_kwargs = [] - for key, value in kwargs.items(): - new_value, new_extra_names = self.__convert_to_ast(value) - if new_extra_names is not None: - extra_names.update(new_extra_names) - ast_kwargs.append(ast.keyword(key, new_value)) - return self.__make_new(ast.Call(self.__get_ast(), ast_args, ast_kwargs), extra_names) - - def __iter__(self): - yield self.__make_new(ast.Starred(self.__get_ast())) - - def __repr__(self): - if isinstance(self.__ast_node__, str): - return self.__ast_node__ - return ast.unparse(self.__ast_node__) - - def __format__(self, format_spec): - raise TypeError("Cannot stringify annotation containing string formatting") - - def _make_binop(op: ast.AST): - def binop(self, other): - rhs, extra_names = self.__convert_to_ast(other) - return self.__make_new( - ast.BinOp(self.__get_ast(), op, rhs), extra_names - ) - - return binop - - __add__ = _make_binop(ast.Add()) - __sub__ = _make_binop(ast.Sub()) - __mul__ = _make_binop(ast.Mult()) - __matmul__ = _make_binop(ast.MatMult()) - __truediv__ = _make_binop(ast.Div()) - __mod__ = _make_binop(ast.Mod()) - __lshift__ = _make_binop(ast.LShift()) - __rshift__ = _make_binop(ast.RShift()) - __or__ = _make_binop(ast.BitOr()) - __xor__ = _make_binop(ast.BitXor()) - __and__ = _make_binop(ast.BitAnd()) - __floordiv__ = _make_binop(ast.FloorDiv()) - __pow__ = _make_binop(ast.Pow()) - - del _make_binop - - def _make_rbinop(op: ast.AST): - def rbinop(self, other): - new_other, extra_names = self.__convert_to_ast(other) - return self.__make_new( - ast.BinOp(new_other, op, self.__get_ast()), extra_names - ) - - return rbinop - - __radd__ = _make_rbinop(ast.Add()) - __rsub__ = _make_rbinop(ast.Sub()) - __rmul__ = _make_rbinop(ast.Mult()) - __rmatmul__ = _make_rbinop(ast.MatMult()) - __rtruediv__ = _make_rbinop(ast.Div()) - __rmod__ = _make_rbinop(ast.Mod()) - __rlshift__ = _make_rbinop(ast.LShift()) - __rrshift__ = _make_rbinop(ast.RShift()) - __ror__ = _make_rbinop(ast.BitOr()) - __rxor__ = _make_rbinop(ast.BitXor()) - __rand__ = _make_rbinop(ast.BitAnd()) - __rfloordiv__ = _make_rbinop(ast.FloorDiv()) - __rpow__ = _make_rbinop(ast.Pow()) - - del _make_rbinop - - def _make_compare(op): - def compare(self, other): - rhs, extra_names = self.__convert_to_ast(other) - return self.__make_new( - ast.Compare( - left=self.__get_ast(), - ops=[op], - comparators=[rhs], - ), - extra_names, - ) - - return compare - - __lt__ = _make_compare(ast.Lt()) - __le__ = _make_compare(ast.LtE()) - __eq__ = _make_compare(ast.Eq()) - __ne__ = _make_compare(ast.NotEq()) - __gt__ = _make_compare(ast.Gt()) - __ge__ = _make_compare(ast.GtE()) - - del _make_compare - - def _make_unary_op(op): - def unary_op(self): - return self.__make_new(ast.UnaryOp(op, self.__get_ast())) - - return unary_op - - __invert__ = _make_unary_op(ast.Invert()) - __pos__ = _make_unary_op(ast.UAdd()) - __neg__ = _make_unary_op(ast.USub()) - - del _make_unary_op - - -def _template_to_ast_constructor(template): - """Convert a `template` instance to a non-literal AST.""" - args = [] - for part in template: - match part: - case str(): - args.append(ast.Constant(value=part)) - case _: - interp = ast.Call( - func=ast.Name(id="Interpolation"), - args=[ - ast.Constant(value=part.value), - ast.Constant(value=part.expression), - ast.Constant(value=part.conversion), - ast.Constant(value=part.format_spec), - ] - ) - args.append(interp) - return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) - - -def _template_to_ast_literal(template, parsed): - """Convert a `template` instance to a t-string literal AST.""" - values = [] - interp_count = 0 - for part in template: - match part: - case str(): - values.append(ast.Constant(value=part)) - case _: - interp = ast.Interpolation( - str=part.expression, - value=parsed[interp_count], - conversion=ord(part.conversion) if part.conversion else -1, - format_spec=ast.Constant(value=part.format_spec) - if part.format_spec - else None, - ) - values.append(interp) - interp_count += 1 - return ast.TemplateStr(values=values) - - -def _template_to_ast(template): - """Make a best-effort conversion of a `template` instance to an AST.""" - # gh-138558: Not all Template instances can be represented as t-string - # literals. Return the most accurate AST we can. See issue for details. - - # If any expr is empty or whitespace only, we cannot convert to a literal. - if any(part.expression.strip() == "" for part in template.interpolations): - return _template_to_ast_constructor(template) - - try: - # Wrap in parens to allow whitespace inside interpolation curly braces - parsed = tuple( - ast.parse(f"({part.expression})", mode="eval").body - for part in template.interpolations - ) - except SyntaxError: - return _template_to_ast_constructor(template) - - return _template_to_ast_literal(template, parsed) - - -class _StringifierDict(dict): - def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): - super().__init__(namespace) - self.namespace = namespace - self.globals = globals - self.owner = owner - self.is_class = is_class - self.stringifiers = [] - self.next_id = 1 - self.format = format - - def __missing__(self, key): - fwdref = _Stringifier( - key, - globals=self.globals, - owner=self.owner, - is_class=self.is_class, - stringifier_dict=self, - ) - self.stringifiers.append(fwdref) - return fwdref - - def transmogrify(self, cell_dict): - for obj in self.stringifiers: - obj.__class__ = ForwardRef - obj.__stringifier_dict__ = None # not needed for ForwardRef - if isinstance(obj.__ast_node__, str): - obj.__arg__ = obj.__ast_node__ - obj.__ast_node__ = None - if cell_dict is not None and obj.__cell__ is None: - obj.__cell__ = cell_dict - - def create_unique_name(self): - name = f"__annotationlib_name_{self.next_id}__" - self.next_id += 1 - return name - - -def call_evaluate_function(evaluate, format, *, owner=None): - """Call an evaluate function. Evaluate functions are normally generated for - the value of type aliases and the bounds, constraints, and defaults of - type parameter objects. - """ - return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True) - - -def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): - """Call an __annotate__ function. __annotate__ functions are normally - generated by the compiler to defer the evaluation of annotations. They - can be called with any of the format arguments in the Format enum, but - compiler-generated __annotate__ functions only support the VALUE format. - This function provides additional functionality to call __annotate__ - functions with the FORWARDREF and STRING formats. - - *annotate* must be an __annotate__ function, which takes a single argument - and returns a dict of annotations. - - *format* must be a member of the Format enum or one of the corresponding - integer values. - - *owner* can be the object that owns the annotations (i.e., the module, - class, or function that the __annotate__ function derives from). With the - FORWARDREF format, it is used to provide better evaluation capabilities - on the generated ForwardRef objects. - - """ - if format == Format.VALUE_WITH_FAKE_GLOBALS: - raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") - try: - return annotate(format) - except NotImplementedError: - pass - if format == Format.STRING: - # STRING is implemented by calling the annotate function in a special - # environment where every name lookup results in an instance of _Stringifier. - # _Stringifier supports every dunder operation and returns a new _Stringifier. - # At the end, we get a dictionary that mostly contains _Stringifier objects (or - # possibly constants if the annotate function uses them directly). We then - # convert each of those into a string to get an approximation of the - # original source. - - # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented - # See: https://github.com/python/cpython/issues/138764 - # Only fail on NotImplementedError - try: - annotate(Format.VALUE_WITH_FAKE_GLOBALS) - except NotImplementedError: - # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE - return annotations_to_string(annotate(Format.VALUE)) - except Exception: - pass - - globals = _StringifierDict({}, format=format) - is_class = isinstance(owner, type) - closure, _ = _build_closure( - annotate, owner, is_class, globals, allow_evaluation=False - ) - func = types.FunctionType( - annotate.__code__, - globals, - closure=closure, - argdefs=annotate.__defaults__, - kwdefaults=annotate.__kwdefaults__, - ) - try: - annos = func(Format.VALUE_WITH_FAKE_GLOBALS) - except ValueError: - # Dict comprehensions such as `{k: v for k, v in items}` unpack - # each iterated element. Fake-globals iteration yields a single - # starred stringifier, so unpacking raises ValueError. Recover - # the original annotation text from source when we can. - sourced = _string_annotations_from_source(owner) - if sourced is not None: - return sourced - raise - if _is_evaluate: - return _stringify_single(annos) - return _stringify_annotation_dict(annos, owner) - elif format == Format.FORWARDREF: - # FORWARDREF is implemented similarly to STRING, but there are two changes, - # at the beginning and the end of the process. - # First, while STRING uses an empty dictionary as the namespace, so that all - # name lookups result in _Stringifier objects, FORWARDREF uses the globals - # and builtins, so that defined names map to their real values. - # Second, instead of returning strings, we want to return either real values - # or ForwardRef objects. To do this, we keep track of all _Stringifier objects - # created while the annotation is being evaluated, and at the end we convert - # them all to ForwardRef objects by assigning to __class__. To make this - # technique work, we have to ensure that the _Stringifier and ForwardRef - # classes share the same attributes. - # We use this technique because while the annotations are being evaluated, - # we want to support all operations that the language allows, including even - # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately - # reconstruct the source. But in the dictionary that we eventually return, we - # want to return objects with more user-friendly behavior, such as an __eq__ - # that returns a bool and an defined set of attributes. - namespace = {**annotate.__builtins__, **annotate.__globals__} - is_class = isinstance(owner, type) - globals = _StringifierDict( - namespace, - globals=annotate.__globals__, - owner=owner, - is_class=is_class, - format=format, - ) - closure, cell_dict = _build_closure( - annotate, owner, is_class, globals, allow_evaluation=True - ) - func = types.FunctionType( - annotate.__code__, - globals, - closure=closure, - argdefs=annotate.__defaults__, - kwdefaults=annotate.__kwdefaults__, - ) - try: - result = func(Format.VALUE_WITH_FAKE_GLOBALS) - except NotImplementedError: - # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE - return annotate(Format.VALUE) - except Exception: - pass - else: - globals.transmogrify(cell_dict) - return result - - # Try again, but do not provide any globals. This allows us to return - # a value in certain cases where an exception gets raised during evaluation. - globals = _StringifierDict( - {}, - globals=annotate.__globals__, - owner=owner, - is_class=is_class, - format=format, - ) - closure, cell_dict = _build_closure( - annotate, owner, is_class, globals, allow_evaluation=False - ) - func = types.FunctionType( - annotate.__code__, - globals, - closure=closure, - argdefs=annotate.__defaults__, - kwdefaults=annotate.__kwdefaults__, - ) - result = func(Format.VALUE_WITH_FAKE_GLOBALS) - globals.transmogrify(cell_dict) - if _is_evaluate: - if isinstance(result, ForwardRef): - return result.evaluate(format=Format.FORWARDREF) - else: - return result - else: - return { - key: ( - val.evaluate(format=Format.FORWARDREF) - if isinstance(val, ForwardRef) - else val - ) - for key, val in result.items() - } - elif format == Format.VALUE: - # Should be impossible because __annotate__ functions must not raise - # NotImplementedError for this format. - raise RuntimeError("annotate function does not support VALUE format") - else: - raise ValueError(f"Invalid format: {format!r}") - - -def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): - if not annotate.__closure__: - return None, None - new_closure = [] - cell_dict = {} - for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): - cell_dict[name] = cell - new_cell = None - if allow_evaluation: - try: - cell.cell_contents - except ValueError: - pass - else: - new_cell = cell - if new_cell is None: - fwdref = _Stringifier( - name, - cell=cell, - owner=owner, - globals=annotate.__globals__, - is_class=is_class, - stringifier_dict=stringifier_dict, - ) - stringifier_dict.stringifiers.append(fwdref) - new_cell = types.CellType(fwdref) - new_closure.append(new_cell) - return tuple(new_closure), cell_dict - - -def _string_annotations_from_source(obj): - """Best-effort STRING annotations reconstructed from *obj*'s source AST. - - Used when fake-globals evaluation cannot stringify an annotation (dict - comprehensions, lambdas, generator expressions). Returns None if source - is unavailable. inspect is imported lazily because it imports this module. - """ - if obj is None: - return None - try: - import inspect - import textwrap - source = inspect.getsource(obj) - except (OSError, TypeError, RecursionError): - return None - source = textwrap.dedent(source) - try: - tree = ast.parse(source) - except SyntaxError: - return None - if not tree.body: - return None - node = tree.body[0] - result = {} - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - for arg in ( - *node.args.posonlyargs, - *node.args.args, - *node.args.kwonlyargs, - ): - if arg.annotation is not None: - result[arg.arg] = ast.unparse(arg.annotation) - if node.args.vararg is not None and node.args.vararg.annotation is not None: - result[node.args.vararg.arg] = ast.unparse(node.args.vararg.annotation) - if node.args.kwarg is not None and node.args.kwarg.annotation is not None: - result[node.args.kwarg.arg] = ast.unparse(node.args.kwarg.annotation) - if node.returns is not None: - result["return"] = ast.unparse(node.returns) - return result - if isinstance(node, ast.ClassDef): - for stmt in node.body: - if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): - result[stmt.target.id] = ast.unparse(stmt.annotation) - return result - return None - - -def _is_runtime_constructed(value): - """True if *value* was created at annotation-eval time and has no AST. - - Lambdas and generator expressions are syntax, not name lookups, so the - fake-globals stringifier never sees them. Their repr() embeds a memory - address and is not a valid annotation string. - """ - return isinstance(value, ( - types.FunctionType, - types.BuiltinFunctionType, - types.MethodType, - types.GeneratorType, - types.AsyncGeneratorType, - types.CoroutineType, - )) - - -def _stringify_annotation_dict(annos, owner): - sourced = _string_annotations_from_source(owner) - result = {} - for key, val in annos.items(): - if sourced is not None and key in sourced and _is_runtime_constructed(val): - result[key] = sourced[key] - else: - result[key] = _stringify_single(val) - return result - - -def _stringify_single(anno): - if anno is ...: - return "..." - # We have to handle str specially to support PEP 563 stringified annotations. - elif isinstance(anno, str): - return anno - elif isinstance(anno, _Template): - return ast.unparse(_template_to_ast(anno)) - elif _is_runtime_constructed(anno): - # Lambdas and generator expressions are syntax, not name lookups. - # repr() embeds a memory address; type_repr() is stable. - return type_repr(anno) - else: - return repr(anno) - - -def get_annotate_from_class_namespace(obj): - """Retrieve the annotate function from a class namespace dictionary. - - Return None if the namespace does not contain an annotate function. - This is useful in metaclass ``__new__`` methods to retrieve the annotate function. - """ - try: - return obj["__annotate__"] - except KeyError: - return obj.get("__annotate_func__", None) - - -def get_annotations( - obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE -): - """Compute the annotations dict for an object. - - obj may be a callable, class, module, or other object with - __annotate__ or __annotations__ attributes. - Passing any other object raises TypeError. - - The *format* parameter controls the format in which annotations are returned, - and must be a member of the Format enum or its integer equivalent. - For the VALUE format, the __annotations__ is tried first; if it - does not exist, the __annotate__ function is called. The - FORWARDREF format uses __annotations__ if it exists and can be - evaluated, and otherwise falls back to calling the __annotate__ function. - The STRING format tries __annotate__ first, and falls back to - using __annotations__, stringified using annotations_to_string(). - - This function handles several details for you: - - * If eval_str is true, values of type str will - be un-stringized using eval(). This is intended - for use with stringized annotations - ("from __future__ import annotations"). - * If obj doesn't have an annotations dict, returns an - empty dict. (Functions and methods always have an - annotations dict; classes, modules, and other types of - callables may not.) - * Ignores inherited annotations on classes. If a class - doesn't have its own annotations dict, returns an empty dict. - * All accesses to object members and dict values are done - using getattr() and dict.get() for safety. - * Always, always, always returns a freshly-created dict. - - eval_str controls whether or not values of type str are replaced - with the result of calling eval() on those values: - - * If eval_str is true, eval() is called on values of type str. - * If eval_str is false (the default), values of type str are unchanged. - - globals and locals are passed in to eval(); see the documentation - for eval() for more information. If either globals or locals is - None, this function may replace that value with a context-specific - default, contingent on type(obj): - - * If obj is a module, globals defaults to obj.__dict__. - * If obj is a class, globals defaults to - sys.modules[obj.__module__].__dict__ and locals - defaults to the obj class namespace. - * If obj is a callable, globals defaults to obj.__globals__, - although if obj is a wrapped function (using - functools.update_wrapper()) it is first unwrapped. - """ - if eval_str and format != Format.VALUE: - raise ValueError("eval_str=True is only supported with format=Format.VALUE") - - match format: - case Format.VALUE: - # For VALUE, we first look at __annotations__ - ann = _get_dunder_annotations(obj) - - # If it's not there, try __annotate__ instead - if ann is None: - ann = _get_and_call_annotate(obj, format) - case Format.FORWARDREF: - # For FORWARDREF, we use __annotations__ if it exists - try: - ann = _get_dunder_annotations(obj) - except Exception: - pass - else: - if ann is not None: - return dict(ann) - - # But if __annotations__ threw a NameError, we try calling __annotate__ - ann = _get_and_call_annotate(obj, format) - if ann is None: - # If that didn't work either, we have a very weird object: evaluating - # __annotations__ threw NameError and there is no __annotate__. In that case, - # we fall back to trying __annotations__ again. - ann = _get_dunder_annotations(obj) - case Format.STRING: - # For STRING, we try to call __annotate__ - ann = _get_and_call_annotate(obj, format) - if ann is not None: - return dict(ann) - # But if we didn't get it, we use __annotations__ instead. - ann = _get_dunder_annotations(obj) - if ann is not None: - return annotations_to_string(ann) - case Format.VALUE_WITH_FAKE_GLOBALS: - raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") - case _: - raise ValueError(f"Unsupported format {format!r}") - - if ann is None: - if isinstance(obj, type) or callable(obj): - return {} - raise TypeError(f"{obj!r} does not have annotations") - - if not ann: - return {} - - if not eval_str: - return dict(ann) - - if globals is None or locals is None: - if isinstance(obj, type): - # class - obj_globals = None - module_name = getattr(obj, "__module__", None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - obj_globals = getattr(module, "__dict__", None) - obj_locals = dict(vars(obj)) - unwrap = obj - elif isinstance(obj, types.ModuleType): - # module - obj_globals = getattr(obj, "__dict__") - obj_locals = None - unwrap = None - elif callable(obj): - # this includes types.Function, types.BuiltinFunctionType, - # types.BuiltinMethodType, functools.partial, functools.singledispatch, - # "class funclike" from Lib/test/test_inspect... on and on it goes. - obj_globals = getattr(obj, "__globals__", None) - obj_locals = None - unwrap = obj - else: - obj_globals = obj_locals = unwrap = None - - if unwrap is not None: - # Use an id-based visited set to detect cycles in the __wrapped__ - # and functools.partial.func chain (e.g. f.__wrapped__ = f). - # On cycle detection we stop and use whatever __globals__ we have - # found so far, mirroring the approach of inspect.unwrap(). - _seen_ids = {id(unwrap)} - while True: - if hasattr(unwrap, "__wrapped__"): - candidate = unwrap.__wrapped__ - if id(candidate) in _seen_ids: - break - _seen_ids.add(id(candidate)) - unwrap = candidate - continue - if functools := sys.modules.get("functools"): - if isinstance(unwrap, functools.partial): - candidate = unwrap.func - if id(candidate) in _seen_ids: - break - _seen_ids.add(id(candidate)) - unwrap = candidate - continue - break - if hasattr(unwrap, "__globals__"): - obj_globals = unwrap.__globals__ - - if globals is None: - globals = obj_globals - if locals is None: - locals = obj_locals - - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - if type_params := getattr(obj, "__type_params__", ()): - if locals is None: - locals = {} - locals = {param.__name__: param for param in type_params} | locals - - return_value = { - key: value if not isinstance(value, str) - else eval(_rewrite_star_unpack(value), globals, locals) - for key, value in ann.items() - } - return return_value - - -def type_repr(value): - """Convert a Python value to a format suitable for use with the STRING format. - - This is intended as a helper for tools that support the STRING format but do - not have access to the code that originally produced the annotations. It uses - repr() for most objects. - - """ - if isinstance(value, (type, types.FunctionType, types.BuiltinFunctionType)): - if value.__module__ == "builtins": - return value.__qualname__ - return f"{value.__module__}.{value.__qualname__}" - elif isinstance(value, ( - types.GeneratorType, - types.AsyncGeneratorType, - types.CoroutineType, - )): - # repr() of these objects embeds a memory address. - return value.__qualname__ - elif isinstance(value, _Template): - tree = _template_to_ast(value) - return ast.unparse(tree) - if value is ...: - return "..." - return repr(value) - - -def annotations_to_string(annotations): - """Convert an annotation dict containing values to approximately the STRING format. - - Always returns a fresh a dictionary. - """ - return { - n: t if isinstance(t, str) else type_repr(t) - for n, t in annotations.items() - } - - -def _rewrite_star_unpack(arg): - """If the given argument annotation expression is a star unpack e.g. `'*Ts'` - rewrite it to a valid expression. - """ - if arg.lstrip().startswith("*"): - return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] - else: - return arg - - -def _get_and_call_annotate(obj, format): - """Get the __annotate__ function and call it. - - May not return a fresh dictionary. - """ - annotate = getattr(obj, "__annotate__", None) - if annotate is not None: - ann = call_annotate_function(annotate, format, owner=obj) - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") - return ann - return None - - -_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ - - -def _get_dunder_annotations(obj): - """Return the annotations for an object, checking that it is a dictionary. - - Does not return a fresh dictionary. - """ - # This special case is needed to support types defined under - # from __future__ import annotations, where accessing the __annotations__ - # attribute directly might return annotations for the wrong class. - if isinstance(obj, type): - try: - ann = _BASE_GET_ANNOTATIONS(obj) - except AttributeError: - # For static types, the descriptor raises AttributeError. - return None - else: - ann = getattr(obj, "__annotations__", None) - if ann is None: - return None - - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") - return ann - - -class _ExtraNameFixer(ast.NodeTransformer): - """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation""" - def __init__(self, extra_names): - self.extra_names = extra_names - - def visit_Name(self, node: ast.Name): - if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel: - node = ast.Name(id=type_repr(new_name)) - return node +PLACEHOLDER \ No newline at end of file From 395a2e70088e6e1b9cea43250f653ac45ccdd727 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:10:05 -0600 Subject: [PATCH 04/12] gh-157056: Expand STRING-format edge-case tests --- Lib/test/test_annotationlib_string_special.py | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_annotationlib_string_special.py b/Lib/test/test_annotationlib_string_special.py index ca2fd90eb59069e..156dcffc8a518db 100644 --- a/Lib/test/test_annotationlib_string_special.py +++ b/Lib/test/test_annotationlib_string_special.py @@ -1,6 +1,8 @@ """STRING-format edge cases for annotationlib (gh-157056).""" +import inspect import unittest +from unittest.mock import patch from annotationlib import Format, get_annotations, type_repr @@ -44,8 +46,54 @@ def mixed(a: int, b: {k: v for k, v in items}, c: lambda q: q): }, ) + def test_quoted_annotation_not_requoted_on_unpack_fallback(self): + def f(a: "int", b: {k: v for k, v in items}): + pass + + self.assertEqual( + get_annotations(f, format=Format.STRING), + {"a": "int", "b": "{k: v for k, v in items}"}, + ) + + def test_nested_lambda_uses_source_and_has_no_address(self): + def f(x: [lambda q: q]): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "[lambda q: q]"}) + self.assertNotIn("0x", anno["x"].lower()) + + def test_class_dictcomp_keeps_conditional_annotations(self): + class C: + a: int + b: {k: v for k, v in items} + if True: + c: str + + self.assertEqual( + get_annotations(C, format=Format.STRING), + {"a": "int", "b": "{k: v for k, v in items}", "c": "str"}, + ) + + def test_simple_string_format_does_not_read_source(self): + def f(x: int) -> str: + pass + + with patch.object(inspect, "getsource") as mocked: + self.assertEqual( + get_annotations(f, format=Format.STRING), + {"x": "int", "return": "str"}, + ) + mocked.assert_not_called() + + def test_exec_without_source_still_raises_on_dictcomp(self): + ns = {} + exec("def f(x: {k: v for k, v in items}): pass", ns) + with self.assertRaises(ValueError): + get_annotations(ns["f"], format=Format.STRING) + def test_type_repr_lambda_and_genexpr_have_no_address(self): - lam = lambda q: q + lam = (lambda q: q) self.assertTrue(type_repr(lam).endswith("")) self.assertNotIn("0x", type_repr(lam).lower()) gen = (w for w in ()) From 0281c8f50e5a5a96140f11042d476548826a39b5 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:13:42 -0600 Subject: [PATCH 05/12] gh-157056: Tighten STRING fallback after review Only consult source when fake-globals cannot stringify. Limit the ValueError handler to unpack errors, keep class-body control-flow annotations, and avoid requoting string-literal annotations. --- Lib/annotationlib.py | 1363 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1362 insertions(+), 1 deletion(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 311c8dd0658759f..9a9adf2c75970c4 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -1 +1,1362 @@ -PLACEHOLDER \ No newline at end of file +"""Helpers for introspecting and wrapping annotations.""" + +import ast +import builtins +import enum +import keyword +import sys +import types + +__all__ = [ + "Format", + "ForwardRef", + "call_annotate_function", + "call_evaluate_function", + "get_annotate_from_class_namespace", + "get_annotations", + "annotations_to_string", + "type_repr", +] + + +class Format(enum.IntEnum): + VALUE = 1 + VALUE_WITH_FAKE_GLOBALS = 2 + FORWARDREF = 3 + STRING = 4 + + +_sentinel = object() +# Following `NAME_ERROR_MSG` in `ceval_macros.h`: +_NAME_ERROR_MSG = "name '{name:.200}' is not defined" + + +# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be +# preserved for compatibility with the old typing.ForwardRef class. The remaining +# names are private. +_SLOTS = ( + "__forward_is_argument__", + "__forward_is_class__", + "__forward_module__", + "__weakref__", + "__arg__", + "__globals__", + "__extra_names__", + "__code__", + "__ast_node__", + "__cell__", + "__owner__", + "__stringifier_dict__", + "__resolved_str_cache__", +) + + +class ForwardRef: + """Wrapper that holds a forward reference. + + Constructor arguments: + * arg: a string representing the code to be evaluated. + * module: the module where the forward reference was created. + Must be a string, not a module object. + * owner: The owning object (module, class, or function). + * is_argument: Does nothing, retained for compatibility. + * is_class: True if the forward reference was created in class scope. + + """ + + __slots__ = _SLOTS + + def __init__( + self, + arg, + *, + module=None, + owner=None, + is_argument=True, + is_class=False, + ): + if not isinstance(arg, str): + raise TypeError(f"Forward reference must be a string -- got {arg!r}") + + self.__arg__ = arg + self.__forward_is_argument__ = is_argument + self.__forward_is_class__ = is_class + self.__forward_module__ = module + self.__owner__ = owner + # These are always set to None here but may be non-None if a ForwardRef + # is created through __class__ assignment on a _Stringifier object. + self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). + self.__cell__ = None + self.__extra_names__ = None + # These are initially None but serve as a cache and may be set to a non-None + # value later. + self.__code__ = None + self.__ast_node__ = None + self.__resolved_str_cache__ = None + + def __init_subclass__(cls, /, *args, **kwds): + raise TypeError("Cannot subclass ForwardRef") + + def evaluate( + self, + *, + globals=None, + locals=None, + type_params=None, + owner=None, + format=Format.VALUE, + ): + """Evaluate the forward reference and return the value. + + If the forward reference cannot be evaluated, raise an exception. + """ + match format: + case Format.STRING: + return self.__resolved_str__ + case Format.VALUE: + is_forwardref_format = False + case Format.FORWARDREF: + is_forwardref_format = True + case _: + raise NotImplementedError(format) + if isinstance(self.__cell__, types.CellType): + try: + return self.__cell__.cell_contents + except ValueError: + pass + if owner is None: + owner = self.__owner__ + + if globals is None and self.__forward_module__ is not None: + globals = getattr( + sys.modules.get(self.__forward_module__, None), "__dict__", None + ) + if globals is None: + globals = self.__globals__ + if globals is None: + if isinstance(owner, type): + module_name = getattr(owner, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + globals = getattr(module, "__dict__", None) + elif isinstance(owner, types.ModuleType): + globals = getattr(owner, "__dict__", None) + elif callable(owner): + globals = getattr(owner, "__globals__", None) + + # If we pass None to eval() below, the globals of this module are used. + if globals is None: + globals = {} + + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + + if locals is None: + locals = {} + if isinstance(owner, type): + locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params is not None: + for param in type_params: + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. + if isinstance(self.__cell__, dict): + for cell_name, cell in self.__cell__.items(): + try: + cell_value = cell.cell_contents + except ValueError: + pass + else: + locals.setdefault(cell_name, cell_value) + + if self.__extra_names__: + locals.update(self.__extra_names__) + + arg = self.__forward_arg__ + if arg.isidentifier() and not keyword.iskeyword(arg): + if arg in locals: + return locals[arg] + elif arg in globals: + return globals[arg] + elif hasattr(builtins, arg): + return getattr(builtins, arg) + elif is_forwardref_format: + return self + else: + raise NameError(_NAME_ERROR_MSG.format(name=arg), name=arg) + else: + code = self.__forward_code__ + try: + return eval(code, globals=globals, locals=locals) + except Exception: + if not is_forwardref_format: + raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. + new_locals = _StringifierDict( + {**builtins.__dict__, **globals, **locals}, + globals=globals, + owner=owner, + is_class=self.__forward_is_class__, + format=format, + ) + try: + result = eval(code, globals=globals, locals=new_locals) + except Exception: + return self + else: + new_locals.transmogrify(self.__cell__) + return result + + @property + def __forward_arg__(self): + if self.__arg__ is not None: + return self.__arg__ + if self.__ast_node__ is not None: + self.__arg__ = ast.unparse(self.__ast_node__) + return self.__arg__ + raise AssertionError( + "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" + ) + + @property + def __resolved_str__(self): + # __forward_arg__ with any names from __extra_names__ replaced + # with the type_repr of the value they represent + if self.__resolved_str_cache__ is None: + resolved_str = self.__forward_arg__ + names = self.__extra_names__ + + if names: + visitor = _ExtraNameFixer(names) + ast_expr = ast.parse(resolved_str, mode="eval").body + node = visitor.visit(ast_expr) + resolved_str = ast.unparse(node) + + self.__resolved_str_cache__ = resolved_str + + return self.__resolved_str_cache__ + + @property + def __forward_code__(self): + if self.__code__ is not None: + return self.__code__ + arg = self.__forward_arg__ + try: + self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") + except SyntaxError: + raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") + return self.__code__ + + def __eq__(self, other): + if not isinstance(other, ForwardRef): + return NotImplemented + return ( + self.__forward_arg__ == other.__forward_arg__ + and self.__forward_module__ == other.__forward_module__ + # Use "is" here because we use id() for this in __hash__ + # because dictionaries are not hashable. + and self.__globals__ is other.__globals__ + and self.__forward_is_class__ == other.__forward_is_class__ + # Two separate cells are always considered unequal in forward refs. + and ( + {name: id(cell) for name, cell in self.__cell__.items()} + == {name: id(cell) for name, cell in other.__cell__.items()} + if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict) + else self.__cell__ is other.__cell__ + ) + and self.__owner__ == other.__owner__ + and ( + (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == + (tuple(sorted(other.__extra_names__.items())) if other.__extra_names__ else None) + ) + ) + + def __hash__(self): + return hash(( + self.__forward_arg__, + self.__forward_module__, + id(self.__globals__), # dictionaries are not hashable, so hash by identity + self.__forward_is_class__, + ( # cells are not hashable as well + tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()])) + if isinstance(self.__cell__, dict) else id(self.__cell__), + ), + self.__owner__, + tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, + )) + + def __or__(self, other): + return types.UnionType[self, other] + + def __ror__(self, other): + return types.UnionType[other, self] + + def __repr__(self): + extra = [] + if self.__forward_module__ is not None: + extra.append(f", module={self.__forward_module__!r}") + if self.__forward_is_class__: + extra.append(", is_class=True") + if self.__owner__ is not None: + extra.append(f", owner={self.__owner__!r}") + return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})" + + +_Template = type(t"") + + +class _Stringifier: + # Must match the slots on ForwardRef, so we can turn an instance of one into an + # instance of the other in place. + __slots__ = _SLOTS + + def __init__( + self, + node, + globals=None, + owner=None, + is_class=False, + cell=None, + *, + stringifier_dict, + extra_names=None, + ): + # Either an AST node or a simple str (for the common case where a ForwardRef + # represent a single name). + assert isinstance(node, (ast.AST, str)) + self.__arg__ = None + self.__forward_is_argument__ = False + self.__forward_is_class__ = is_class + self.__forward_module__ = None + self.__code__ = None + self.__ast_node__ = node + self.__globals__ = globals + self.__extra_names__ = extra_names + self.__cell__ = cell + self.__owner__ = owner + self.__stringifier_dict__ = stringifier_dict + self.__resolved_str_cache__ = None # Needed for ForwardRef + + def __convert_to_ast(self, other): + if isinstance(other, _Stringifier): + if isinstance(other.__ast_node__, str): + return ast.Name(id=other.__ast_node__), other.__extra_names__ + return other.__ast_node__, other.__extra_names__ + elif type(other) is _Template: + return _template_to_ast(other), None + elif ( + # In STRING format we don't bother with the create_unique_name() dance; + # it's better to emit the repr() of the object instead of an opaque name. + self.__stringifier_dict__.format == Format.STRING + or other is None + or type(other) in (str, int, float, bool, complex) + ): + return ast.Constant(value=other), None + elif type(other) is dict: + extra_names = {} + keys = [] + values = [] + for key, value in other.items(): + new_key, new_extra_names = self.__convert_to_ast(key) + if new_extra_names is not None: + extra_names.update(new_extra_names) + keys.append(new_key) + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + values.append(new_value) + return ast.Dict(keys, values), extra_names + elif type(other) in (list, tuple, set): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)] + return ast_class(elts), extra_names + else: + name = self.__stringifier_dict__.create_unique_name() + return ast.Name(id=name), {name: other} + + def __convert_to_ast_getitem(self, other): + if isinstance(other, slice): + extra_names = {} + + def conv(obj): + if obj is None: + return None + new_obj, new_extra_names = self.__convert_to_ast(obj) + if new_extra_names is not None: + extra_names.update(new_extra_names) + return new_obj + + return ast.Slice( + lower=conv(other.start), + upper=conv(other.stop), + step=conv(other.step), + ), extra_names + else: + return self.__convert_to_ast(other) + + def __get_ast(self): + node = self.__ast_node__ + if isinstance(node, str): + return ast.Name(id=node) + return node + + def __make_new(self, node, extra_names=None): + new_extra_names = {} + if self.__extra_names__ is not None: + new_extra_names.update(self.__extra_names__) + if extra_names is not None: + new_extra_names.update(extra_names) + stringifier = _Stringifier( + node, + self.__globals__, + self.__owner__, + self.__forward_is_class__, + stringifier_dict=self.__stringifier_dict__, + extra_names=new_extra_names or None, + ) + self.__stringifier_dict__.stringifiers.append(stringifier) + return stringifier + + # Must implement this since we set __eq__. We hash by identity so that + # stringifiers in dict keys are kept separate. + def __hash__(self): + return id(self) + + def __getitem__(self, other): + # Special case, to avoid stringifying references to class-scoped variables + # as '__classdict__["x"]'. + if self.__ast_node__ == "__classdict__": + raise KeyError + if isinstance(other, tuple): + extra_names = {} + elts = [] + for elt in other: + new_elt, new_extra_names = self.__convert_to_ast_getitem(elt) + if new_extra_names is not None: + extra_names.update(new_extra_names) + elts.append(new_elt) + other = ast.Tuple(elts) + else: + other, extra_names = self.__convert_to_ast_getitem(other) + assert isinstance(other, ast.AST), repr(other) + return self.__make_new(ast.Subscript(self.__get_ast(), other), extra_names) + + def __getattr__(self, attr): + return self.__make_new(ast.Attribute(self.__get_ast(), attr)) + + def __call__(self, *args, **kwargs): + extra_names = {} + ast_args = [] + for arg in args: + new_arg, new_extra_names = self.__convert_to_ast(arg) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_args.append(new_arg) + ast_kwargs = [] + for key, value in kwargs.items(): + new_value, new_extra_names = self.__convert_to_ast(value) + if new_extra_names is not None: + extra_names.update(new_extra_names) + ast_kwargs.append(ast.keyword(key, new_value)) + return self.__make_new(ast.Call(self.__get_ast(), ast_args, ast_kwargs), extra_names) + + def __iter__(self): + yield self.__make_new(ast.Starred(self.__get_ast())) + + def __repr__(self): + if isinstance(self.__ast_node__, str): + return self.__ast_node__ + return ast.unparse(self.__ast_node__) + + def __format__(self, format_spec): + raise TypeError("Cannot stringify annotation containing string formatting") + + def _make_binop(op: ast.AST): + def binop(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(self.__get_ast(), op, rhs), extra_names + ) + + return binop + + __add__ = _make_binop(ast.Add()) + __sub__ = _make_binop(ast.Sub()) + __mul__ = _make_binop(ast.Mult()) + __matmul__ = _make_binop(ast.MatMult()) + __truediv__ = _make_binop(ast.Div()) + __mod__ = _make_binop(ast.Mod()) + __lshift__ = _make_binop(ast.LShift()) + __rshift__ = _make_binop(ast.RShift()) + __or__ = _make_binop(ast.BitOr()) + __xor__ = _make_binop(ast.BitXor()) + __and__ = _make_binop(ast.BitAnd()) + __floordiv__ = _make_binop(ast.FloorDiv()) + __pow__ = _make_binop(ast.Pow()) + + del _make_binop + + def _make_rbinop(op: ast.AST): + def rbinop(self, other): + new_other, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.BinOp(new_other, op, self.__get_ast()), extra_names + ) + + return rbinop + + __radd__ = _make_rbinop(ast.Add()) + __rsub__ = _make_rbinop(ast.Sub()) + __rmul__ = _make_rbinop(ast.Mult()) + __rmatmul__ = _make_rbinop(ast.MatMult()) + __rtruediv__ = _make_rbinop(ast.Div()) + __rmod__ = _make_rbinop(ast.Mod()) + __rlshift__ = _make_rbinop(ast.LShift()) + __rrshift__ = _make_rbinop(ast.RShift()) + __ror__ = _make_rbinop(ast.BitOr()) + __rxor__ = _make_rbinop(ast.BitXor()) + __rand__ = _make_rbinop(ast.BitAnd()) + __rfloordiv__ = _make_rbinop(ast.FloorDiv()) + __rpow__ = _make_rbinop(ast.Pow()) + + del _make_rbinop + + def _make_compare(op): + def compare(self, other): + rhs, extra_names = self.__convert_to_ast(other) + return self.__make_new( + ast.Compare( + left=self.__get_ast(), + ops=[op], + comparators=[rhs], + ), + extra_names, + ) + + return compare + + __lt__ = _make_compare(ast.Lt()) + __le__ = _make_compare(ast.LtE()) + __eq__ = _make_compare(ast.Eq()) + __ne__ = _make_compare(ast.NotEq()) + __gt__ = _make_compare(ast.Gt()) + __ge__ = _make_compare(ast.GtE()) + + del _make_compare + + def _make_unary_op(op): + def unary_op(self): + return self.__make_new(ast.UnaryOp(op, self.__get_ast())) + + return unary_op + + __invert__ = _make_unary_op(ast.Invert()) + __pos__ = _make_unary_op(ast.UAdd()) + __neg__ = _make_unary_op(ast.USub()) + + del _make_unary_op + + +def _template_to_ast_constructor(template): + """Convert a `template` instance to a non-literal AST.""" + args = [] + for part in template: + match part: + case str(): + args.append(ast.Constant(value=part)) + case _: + interp = ast.Call( + func=ast.Name(id="Interpolation"), + args=[ + ast.Constant(value=part.value), + ast.Constant(value=part.expression), + ast.Constant(value=part.conversion), + ast.Constant(value=part.format_spec), + ] + ) + args.append(interp) + return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) + + +def _template_to_ast_literal(template, parsed): + """Convert a `template` instance to a t-string literal AST.""" + values = [] + interp_count = 0 + for part in template: + match part: + case str(): + values.append(ast.Constant(value=part)) + case _: + interp = ast.Interpolation( + str=part.expression, + value=parsed[interp_count], + conversion=ord(part.conversion) if part.conversion else -1, + format_spec=ast.Constant(value=part.format_spec) + if part.format_spec + else None, + ) + values.append(interp) + interp_count += 1 + return ast.TemplateStr(values=values) + + +def _template_to_ast(template): + """Make a best-effort conversion of a `template` instance to an AST.""" + # gh-138558: Not all Template instances can be represented as t-string + # literals. Return the most accurate AST we can. See issue for details. + + # If any expr is empty or whitespace only, we cannot convert to a literal. + if any(part.expression.strip() == "" for part in template.interpolations): + return _template_to_ast_constructor(template) + + try: + # Wrap in parens to allow whitespace inside interpolation curly braces + parsed = tuple( + ast.parse(f"({part.expression})", mode="eval").body + for part in template.interpolations + ) + except SyntaxError: + return _template_to_ast_constructor(template) + + return _template_to_ast_literal(template, parsed) + + +class _StringifierDict(dict): + def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): + super().__init__(namespace) + self.namespace = namespace + self.globals = globals + self.owner = owner + self.is_class = is_class + self.stringifiers = [] + self.next_id = 1 + self.format = format + + def __missing__(self, key): + fwdref = _Stringifier( + key, + globals=self.globals, + owner=self.owner, + is_class=self.is_class, + stringifier_dict=self, + ) + self.stringifiers.append(fwdref) + return fwdref + + def transmogrify(self, cell_dict): + for obj in self.stringifiers: + obj.__class__ = ForwardRef + obj.__stringifier_dict__ = None # not needed for ForwardRef + if isinstance(obj.__ast_node__, str): + obj.__arg__ = obj.__ast_node__ + obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict + + def create_unique_name(self): + name = f"__annotationlib_name_{self.next_id}__" + self.next_id += 1 + return name + + +def call_evaluate_function(evaluate, format, *, owner=None): + """Call an evaluate function. Evaluate functions are normally generated for + the value of type aliases and the bounds, constraints, and defaults of + type parameter objects. + """ + return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True) + + +def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): + """Call an __annotate__ function. __annotate__ functions are normally + generated by the compiler to defer the evaluation of annotations. They + can be called with any of the format arguments in the Format enum, but + compiler-generated __annotate__ functions only support the VALUE format. + This function provides additional functionality to call __annotate__ + functions with the FORWARDREF and STRING formats. + + *annotate* must be an __annotate__ function, which takes a single argument + and returns a dict of annotations. + + *format* must be a member of the Format enum or one of the corresponding + integer values. + + *owner* can be the object that owns the annotations (i.e., the module, + class, or function that the __annotate__ function derives from). With the + FORWARDREF format, it is used to provide better evaluation capabilities + on the generated ForwardRef objects. + + """ + if format == Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + try: + return annotate(format) + except NotImplementedError: + pass + if format == Format.STRING: + # STRING is implemented by calling the annotate function in a special + # environment where every name lookup results in an instance of _Stringifier. + # _Stringifier supports every dunder operation and returns a new _Stringifier. + # At the end, we get a dictionary that mostly contains _Stringifier objects (or + # possibly constants if the annotate function uses them directly). We then + # convert each of those into a string to get an approximation of the + # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + + globals = _StringifierDict({}, format=format) + is_class = isinstance(owner, type) + closure, _ = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + try: + annos = func(Format.VALUE_WITH_FAKE_GLOBALS) + except ValueError as exc: + # Dict comprehensions such as `{k: v for k, v in items}` unpack + # each iterated element. Fake-globals iteration yields a single + # starred stringifier, so unpacking raises ValueError. Recover + # the original annotation text from source when we can. + # Changing _Stringifier.__iter__ would break [*a] stringification. + if _is_unpack_value_error(exc): + sourced = _string_annotations_from_source(owner) + if sourced is not None: + return sourced + raise + if _is_evaluate: + return _stringify_single(annos) + return _stringify_annotation_dict(annos, owner) + elif format == Format.FORWARDREF: + # FORWARDREF is implemented similarly to STRING, but there are two changes, + # at the beginning and the end of the process. + # First, while STRING uses an empty dictionary as the namespace, so that all + # name lookups result in _Stringifier objects, FORWARDREF uses the globals + # and builtins, so that defined names map to their real values. + # Second, instead of returning strings, we want to return either real values + # or ForwardRef objects. To do this, we keep track of all _Stringifier objects + # created while the annotation is being evaluated, and at the end we convert + # them all to ForwardRef objects by assigning to __class__. To make this + # technique work, we have to ensure that the _Stringifier and ForwardRef + # classes share the same attributes. + # We use this technique because while the annotations are being evaluated, + # we want to support all operations that the language allows, including even + # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately + # reconstruct the source. But in the dictionary that we eventually return, we + # want to return objects with more user-friendly behavior, such as an __eq__ + # that returns a bool and an defined set of attributes. + namespace = {**annotate.__builtins__, **annotate.__globals__} + is_class = isinstance(owner, type) + globals = _StringifierDict( + namespace, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=True + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + try: + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) + except Exception: + pass + else: + globals.transmogrify(cell_dict) + return result + + # Try again, but do not provide any globals. This allows us to return + # a value in certain cases where an exception gets raised during evaluation. + globals = _StringifierDict( + {}, + globals=annotate.__globals__, + owner=owner, + is_class=is_class, + format=format, + ) + closure, cell_dict = _build_closure( + annotate, owner, is_class, globals, allow_evaluation=False + ) + func = types.FunctionType( + annotate.__code__, + globals, + closure=closure, + argdefs=annotate.__defaults__, + kwdefaults=annotate.__kwdefaults__, + ) + result = func(Format.VALUE_WITH_FAKE_GLOBALS) + globals.transmogrify(cell_dict) + if _is_evaluate: + if isinstance(result, ForwardRef): + return result.evaluate(format=Format.FORWARDREF) + else: + return result + else: + return { + key: ( + val.evaluate(format=Format.FORWARDREF) + if isinstance(val, ForwardRef) + else val + ) + for key, val in result.items() + } + elif format == Format.VALUE: + # Should be impossible because __annotate__ functions must not raise + # NotImplementedError for this format. + raise RuntimeError("annotate function does not support VALUE format") + else: + raise ValueError(f"Invalid format: {format!r}") + + +def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): + if not annotate.__closure__: + return None, None + new_closure = [] + cell_dict = {} + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): + cell_dict[name] = cell + new_cell = None + if allow_evaluation: + try: + cell.cell_contents + except ValueError: + pass + else: + new_cell = cell + if new_cell is None: + fwdref = _Stringifier( + name, + cell=cell, + owner=owner, + globals=annotate.__globals__, + is_class=is_class, + stringifier_dict=stringifier_dict, + ) + stringifier_dict.stringifiers.append(fwdref) + new_cell = types.CellType(fwdref) + new_closure.append(new_cell) + return tuple(new_closure), cell_dict + + +def _string_annotations_from_source(obj): + """Best-effort STRING annotations reconstructed from *obj*'s source AST. + + Used when fake-globals evaluation cannot stringify an annotation (dict + comprehensions, lambdas, generator expressions). Returns None if source + is unavailable. inspect is imported lazily because it imports this module. + """ + if obj is None: + return None + try: + import inspect + import textwrap + source = inspect.getsource(obj) + except (OSError, TypeError, RecursionError): + return None + source = textwrap.dedent(source) + try: + tree = ast.parse(source) + except SyntaxError: + return None + if not tree.body: + return None + node = tree.body[0] + result = {} + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + for arg in ( + *node.args.posonlyargs, + *node.args.args, + *node.args.kwonlyargs, + ): + if arg.annotation is not None: + result[arg.arg] = _annotation_ast_to_string(arg.annotation) + if node.args.vararg is not None and node.args.vararg.annotation is not None: + result[node.args.vararg.arg] = _annotation_ast_to_string( + node.args.vararg.annotation + ) + if node.args.kwarg is not None and node.args.kwarg.annotation is not None: + result[node.args.kwarg.arg] = _annotation_ast_to_string( + node.args.kwarg.annotation + ) + if node.returns is not None: + result["return"] = _annotation_ast_to_string(node.returns) + return result + if isinstance(node, ast.ClassDef): + _collect_class_annassigns(node.body, result) + return result + return None + + +def _annotation_ast_to_string(node): + # Match fake-globals STRING output for quoted annotations: a: "int" -> "int". + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + return ast.unparse(node) + + +def _collect_class_annassigns(body, result): + """Collect class annotations, including those inside simple control flow. + + Nested functions and classes are skipped so their annotations are not + attributed to the enclosing class. + """ + for stmt in body: + if isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name): + result[stmt.target.id] = _annotation_ast_to_string(stmt.annotation) + elif isinstance(stmt, ast.If): + _collect_class_annassigns(stmt.body, result) + _collect_class_annassigns(stmt.orelse, result) + elif isinstance(stmt, ast.Try): + _collect_class_annassigns(stmt.body, result) + for handler in stmt.handlers: + _collect_class_annassigns(handler.body, result) + _collect_class_annassigns(stmt.orelse, result) + _collect_class_annassigns(stmt.finalbody, result) + elif isinstance(stmt, (ast.With, ast.AsyncWith)): + _collect_class_annassigns(stmt.body, result) + elif isinstance(stmt, (ast.For, ast.AsyncFor, ast.While)): + _collect_class_annassigns(stmt.body, result) + _collect_class_annassigns(stmt.orelse, result) + elif isinstance(stmt, ast.Match): + for case in stmt.cases: + _collect_class_annassigns(case.body, result) + + +def _is_unpack_value_error(exc): + if not exc.args: + return False + msg = exc.args[0] + return isinstance(msg, str) and "values to unpack" in msg + + +def _is_runtime_constructed(value): + """True if *value* was created at annotation-eval time and has no AST. + + Lambdas and generator expressions are syntax, not name lookups, so the + fake-globals stringifier never sees them. Their repr() embeds a memory + address and is not a valid annotation string. + """ + return isinstance(value, ( + types.FunctionType, + types.BuiltinFunctionType, + types.MethodType, + types.GeneratorType, + types.AsyncGeneratorType, + types.CoroutineType, + )) + + +def _contains_runtime_constructed(value): + if _is_runtime_constructed(value): + return True + if isinstance(value, (list, tuple, set)): + return any(_contains_runtime_constructed(v) for v in value) + if isinstance(value, dict): + return any( + _contains_runtime_constructed(k) or _contains_runtime_constructed(v) + for k, v in value.items() + ) + return False + + +def _stringify_annotation_dict(annos, owner): + # Only read source when fake-globals produced a runtime object we cannot + # stringify (lambda, genexpr, or a container holding one). + sourced = _sentinel + result = {} + for key, val in annos.items(): + if _contains_runtime_constructed(val): + if sourced is _sentinel: + sourced = _string_annotations_from_source(owner) + if sourced is not None and key in sourced: + result[key] = sourced[key] + continue + result[key] = _stringify_single(val) + return result + + +def _stringify_container(anno): + if isinstance(anno, list): + inner = ", ".join(_stringify_single(v) for v in anno) + return f"[{inner}]" + if isinstance(anno, tuple): + inner = ", ".join(_stringify_single(v) for v in anno) + if len(anno) == 1: + inner += "," + return f"({inner})" + if isinstance(anno, set): + if not anno: + return "set()" + inner = ", ".join(_stringify_single(v) for v in anno) + return f"{{{inner}}}" + inner = ", ".join( + f"{_stringify_single(k)}: {_stringify_single(v)}" + for k, v in anno.items() + ) + return f"{{{inner}}}" + + +def _stringify_single(anno): + if anno is ...: + return "..." + # We have to handle str specially to support PEP 563 stringified annotations. + elif isinstance(anno, str): + return anno + elif isinstance(anno, _Template): + return ast.unparse(_template_to_ast(anno)) + elif _is_runtime_constructed(anno): + # Lambdas and generator expressions are syntax, not name lookups. + # repr() embeds a memory address; type_repr() is stable. + return type_repr(anno) + elif isinstance(anno, (list, tuple, set, dict)) and _contains_runtime_constructed(anno): + return _stringify_container(anno) + else: + return repr(anno) + + +def get_annotate_from_class_namespace(obj): + """Retrieve the annotate function from a class namespace dictionary. + + Return None if the namespace does not contain an annotate function. + This is useful in metaclass ``__new__`` methods to retrieve the annotate function. + """ + try: + return obj["__annotate__"] + except KeyError: + return obj.get("__annotate_func__", None) + + +def get_annotations( + obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE +): + """Compute the annotations dict for an object. + + obj may be a callable, class, module, or other object with + __annotate__ or __annotations__ attributes. + Passing any other object raises TypeError. + + The *format* parameter controls the format in which annotations are returned, + and must be a member of the Format enum or its integer equivalent. + For the VALUE format, the __annotations__ is tried first; if it + does not exist, the __annotate__ function is called. The + FORWARDREF format uses __annotations__ if it exists and can be + evaluated, and otherwise falls back to calling the __annotate__ function. + The STRING format tries __annotate__ first, and falls back to + using __annotations__, stringified using annotations_to_string(). + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if eval_str and format != Format.VALUE: + raise ValueError("eval_str=True is only supported with format=Format.VALUE") + + match format: + case Format.VALUE: + # For VALUE, we first look at __annotations__ + ann = _get_dunder_annotations(obj) + + # If it's not there, try __annotate__ instead + if ann is None: + ann = _get_and_call_annotate(obj, format) + case Format.FORWARDREF: + # For FORWARDREF, we use __annotations__ if it exists + try: + ann = _get_dunder_annotations(obj) + except Exception: + pass + else: + if ann is not None: + return dict(ann) + + # But if __annotations__ threw a NameError, we try calling __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is None: + # If that didn't work either, we have a very weird object: evaluating + # __annotations__ threw NameError and there is no __annotate__. In that case, + # we fall back to trying __annotations__ again. + ann = _get_dunder_annotations(obj) + case Format.STRING: + # For STRING, we try to call __annotate__ + ann = _get_and_call_annotate(obj, format) + if ann is not None: + return dict(ann) + # But if we didn't get it, we use __annotations__ instead. + ann = _get_dunder_annotations(obj) + if ann is not None: + return annotations_to_string(ann) + case Format.VALUE_WITH_FAKE_GLOBALS: + raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only") + case _: + raise ValueError(f"Unsupported format {format!r}") + + if ann is None: + if isinstance(obj, type) or callable(obj): + return {} + raise TypeError(f"{obj!r} does not have annotations") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if globals is None or locals is None: + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + else: + obj_globals = obj_locals = unwrap = None + + if unwrap is not None: + # Use an id-based visited set to detect cycles in the __wrapped__ + # and functools.partial.func chain (e.g. f.__wrapped__ = f). + # On cycle detection we stop and use whatever __globals__ we have + # found so far, mirroring the approach of inspect.unwrap(). + _seen_ids = {id(unwrap)} + while True: + if hasattr(unwrap, "__wrapped__"): + candidate = unwrap.__wrapped__ + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate + continue + if functools := sys.modules.get("functools"): + if isinstance(unwrap, functools.partial): + candidate = unwrap.func + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + if locals is None: + locals = {} + locals = {param.__name__: param for param in type_params} | locals + + return_value = { + key: value if not isinstance(value, str) + else eval(_rewrite_star_unpack(value), globals, locals) + for key, value in ann.items() + } + return return_value + + +def type_repr(value): + """Convert a Python value to a format suitable for use with the STRING format. + + This is intended as a helper for tools that support the STRING format but do + not have access to the code that originally produced the annotations. It uses + repr() for most objects. + + """ + if isinstance(value, ( + type, + types.FunctionType, + types.BuiltinFunctionType, + types.MethodType, + )): + if getattr(value, "__module__", None) == "builtins": + return value.__qualname__ + module = getattr(value, "__module__", None) + if module: + return f"{module}.{value.__qualname__}" + return value.__qualname__ + elif isinstance(value, ( + types.GeneratorType, + types.AsyncGeneratorType, + types.CoroutineType, + )): + # repr() of these objects embeds a memory address. + return value.__qualname__ + elif isinstance(value, _Template): + tree = _template_to_ast(value) + return ast.unparse(tree) + if value is ...: + return "..." + return repr(value) + + +def annotations_to_string(annotations): + """Convert an annotation dict containing values to approximately the STRING format. + + Always returns a fresh a dictionary. + """ + return { + n: t if isinstance(t, str) else type_repr(t) + for n, t in annotations.items() + } + + +def _rewrite_star_unpack(arg): + """If the given argument annotation expression is a star unpack e.g. `'*Ts'` + rewrite it to a valid expression. + """ + if arg.lstrip().startswith("*"): + return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + return arg + + +def _get_and_call_annotate(obj, format): + """Get the __annotate__ function and call it. + + May not return a fresh dictionary. + """ + annotate = getattr(obj, "__annotate__", None) + if annotate is not None: + ann = call_annotate_function(annotate, format, owner=obj) + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotate__ returned a non-dict") + return ann + return None + + +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + +def _get_dunder_annotations(obj): + """Return the annotations for an object, checking that it is a dictionary. + + Does not return a fresh dictionary. + """ + # This special case is needed to support types defined under + # from __future__ import annotations, where accessing the __annotations__ + # attribute directly might return annotations for the wrong class. + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return None + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return None + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + return ann + + +class _ExtraNameFixer(ast.NodeTransformer): + """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation""" + def __init__(self, extra_names): + self.extra_names = extra_names + + def visit_Name(self, node: ast.Name): + if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel: + node = ast.Name(id=type_repr(new_name)) + return node From b8b25e42e5db167d4fcdf4862ae6a3fe377ee890 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:38:28 -0600 Subject: [PATCH 06/12] gh-157056: Remint NEWS and fold STRING-format tests into TestStringFormat --- .../Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst diff --git a/Misc/NEWS.d/next/Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst b/Misc/NEWS.d/next/Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst new file mode 100644 index 000000000000000..4436d5aaabc24f0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst @@ -0,0 +1,4 @@ +:func:`annotationlib.get_annotations` with ``format=Format.STRING`` no +longer fails on dict-comprehension annotations, and no longer emits +non-deterministic strings that embed a memory address for ``lambda`` and +generator-expression annotations. From eba936d6cc77f9154032117c0c481789a3977407 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:42:55 -0600 Subject: [PATCH 07/12] gh-157056: Drop duplicate reminted NEWS entry --- .../Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst diff --git a/Misc/NEWS.d/next/Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst b/Misc/NEWS.d/next/Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst deleted file mode 100644 index 4436d5aaabc24f0..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-08-02-33-36.gh-issue-157056.iyWsHZ.rst +++ /dev/null @@ -1,4 +0,0 @@ -:func:`annotationlib.get_annotations` with ``format=Format.STRING`` no -longer fails on dict-comprehension annotations, and no longer emits -non-deterministic strings that embed a memory address for ``lambda`` and -generator-expression annotations. From f0ef87f2227ac002d754c0eeef2000fead32f60e Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:43:27 -0600 Subject: [PATCH 08/12] gh-157056: Remint NEWS and fold STRING-format tests into TestStringFormat --- Lib/annotationlib.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 9a9adf2c75970c4..cf81ca6216560f3 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -998,29 +998,35 @@ def _is_runtime_constructed(value): def _contains_runtime_constructed(value): if _is_runtime_constructed(value): return True - if isinstance(value, (list, tuple, set)): - return any(_contains_runtime_constructed(v) for v in value) if isinstance(value, dict): return any( _contains_runtime_constructed(k) or _contains_runtime_constructed(v) for k, v in value.items() ) + if isinstance(value, (list, tuple, set, frozenset)): + return any(_contains_runtime_constructed(v) for v in value) return False def _stringify_annotation_dict(annos, owner): # Only read source when fake-globals produced a runtime object we cannot - # stringify (lambda, genexpr, or a container holding one). + # stringify (lambda, genexpr, or a container holding one), or when the + # fallback text still embeds a memory address. sourced = _sentinel result = {} for key, val in annos.items(): - if _contains_runtime_constructed(val): + text = _stringify_single(val) + needs_source = ( + _contains_runtime_constructed(val) + or "0x" in text.lower() + ) + if needs_source: if sourced is _sentinel: sourced = _string_annotations_from_source(owner) if sourced is not None and key in sourced: result[key] = sourced[key] continue - result[key] = _stringify_single(val) + result[key] = text return result @@ -1038,6 +1044,11 @@ def _stringify_container(anno): return "set()" inner = ", ".join(_stringify_single(v) for v in anno) return f"{{{inner}}}" + if isinstance(anno, frozenset): + if not anno: + return "frozenset()" + inner = ", ".join(_stringify_single(v) for v in anno) + return f"frozenset({{{inner}}})" inner = ", ".join( f"{_stringify_single(k)}: {_stringify_single(v)}" for k, v in anno.items() @@ -1057,7 +1068,10 @@ def _stringify_single(anno): # Lambdas and generator expressions are syntax, not name lookups. # repr() embeds a memory address; type_repr() is stable. return type_repr(anno) - elif isinstance(anno, (list, tuple, set, dict)) and _contains_runtime_constructed(anno): + elif ( + isinstance(anno, (list, tuple, set, frozenset, dict)) + and _contains_runtime_constructed(anno) + ): return _stringify_container(anno) else: return repr(anno) From 9b25626a980ce0583704f823e1d76eafa43250f4 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:48:45 -0600 Subject: [PATCH 09/12] gh-157056: Remint NEWS and fold STRING-format tests into TestStringFormat --- Lib/test/test_annotationlib.py | 152 +++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 530114161701b5a..312ddd8a7623cef 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -545,6 +545,152 @@ def f(x: x | (1).__class__, y: (1).__class__): {"x": "x | ", "y": ""}, ) + def test_comprehensions(self): + # gh-157056: pair-unpacking comprehensions raised ValueError, and + # generator expressions were stringified with a memory address. + def f( + dictcomp: {k: v for k, v in items}, + listcomp: [k for k, v in items], + setcomp: {k for k, v in items}, + genexpr: (w for w in seq), + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "dictcomp": "{k: v for k, v in items}", + "listcomp": "[k for k, v in items]", + "setcomp": "{k for k, v in items}", + "genexpr": "(w for w in seq)", + }, + ) + self.assertNotIn("0x", anno["genexpr"].lower()) + + def test_lambda(self): + def f( + lam: lambda q: q, + nested_list: [lambda q: q], + nested_dict: {"h": lambda q: q}, + nested_frozenset: frozenset({lambda q: q}), + ): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "lam": "lambda q: q", + "nested_list": "[lambda q: q]", + "nested_dict": "{'h': lambda q: q}", + "nested_frozenset": "frozenset({lambda q: q})", + }, + ) + for value in anno.values(): + self.assertNotIn("0x", value.lower()) + + def test_quoted_string_with_unpack_fallback(self): + def f(a: "int", b: {k: v for k, v in items}): + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"a": "int", "b": "{k: v for k, v in items}"}) + + def test_return_varargs_kwargs(self): + def f( + *xs: {k: v for k, v in items}, + **kw: lambda q: q, + ) -> {k: v for k, v in items}: + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual( + anno, + { + "xs": "{k: v for k, v in items}", + "kw": "lambda q: q", + "return": "{k: v for k, v in items}", + }, + ) + + def test_nested_and_decorated(self): + def deco(fn): + return fn + + def outer(): + @deco + def inner(x: {k: v for k, v in items}, y: lambda q: q): + pass + return inner + + anno = get_annotations(outer(), format=Format.STRING) + self.assertEqual( + anno, + {"x": "{k: v for k, v in items}", "y": "lambda q: q"}, + ) + + def test_async_function(self): + async def f(x: {k: v for k, v in items}) -> int: + pass + + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "{k: v for k, v in items}", "return": "int"}) + + def test_method_and_staticmethod(self): + class C: + def meth(self, x: {k: v for k, v in items}) -> str: + pass + + @staticmethod + def sm(x: lambda q: q): + pass + + self.assertEqual( + get_annotations(C.meth, format=Format.STRING), + {"x": "{k: v for k, v in items}", "return": "str"}, + ) + self.assertEqual( + get_annotations(C.sm, format=Format.STRING), + {"x": "lambda q: q"}, + ) + + def test_class_with_conditional_annotation(self): + class C: + a: int + b: {k: v for k, v in items} + if True: + c: str + + anno = get_annotations(C, format=Format.STRING) + self.assertEqual( + anno, + { + "a": "int", + "b": "{k: v for k, v in items}", + "c": "str", + }, + ) + + def test_string_format_does_not_read_source(self): + import inspect + + def f(x: int) -> str: + pass + + def boom(*args, **kwargs): + raise AssertionError("inspect.getsource should not be consulted") + + with support.swap_attr(inspect, "getsource", boom): + anno = get_annotations(f, format=Format.STRING) + self.assertEqual(anno, {"x": "int", "return": "str"}) + + def test_exec_without_source_still_raises(self): + ns = {} + exec("def f(x: {k: v for k, v in items}): pass", ns) + with self.assertRaisesRegex(ValueError, "values to unpack"): + get_annotations(ns["f"], format=Format.STRING) + class TestGetAnnotations(unittest.TestCase): def test_builtin_type(self): @@ -1859,6 +2005,12 @@ def nested(): self.assertEqual(type_repr(len), "len") self.assertEqual(type_repr(type_repr), "annotationlib.type_repr") self.assertEqual(type_repr(times_three), f"{__name__}.times_three") + lam = (lambda q: q) + self.assertTrue(type_repr(lam).endswith("")) + self.assertNotIn("0x", type_repr(lam).lower()) + gen = (w for w in ()) + self.assertTrue(type_repr(gen).endswith("")) + self.assertNotIn("0x", type_repr(gen).lower()) self.assertEqual(type_repr(...), "...") self.assertEqual(type_repr(None), "None") self.assertEqual(type_repr(1), "1") From 1ee9eb85a779d81fc90456701fa5e4b3966e36ab Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:48:58 -0600 Subject: [PATCH 10/12] gh-157056: Move STRING-format tests into test_annotationlib.py --- Lib/test/test_annotationlib_string_special.py | 105 ------------------ 1 file changed, 105 deletions(-) delete mode 100644 Lib/test/test_annotationlib_string_special.py diff --git a/Lib/test/test_annotationlib_string_special.py b/Lib/test/test_annotationlib_string_special.py deleted file mode 100644 index 156dcffc8a518db..000000000000000 --- a/Lib/test/test_annotationlib_string_special.py +++ /dev/null @@ -1,105 +0,0 @@ -"""STRING-format edge cases for annotationlib (gh-157056).""" - -import inspect -import unittest -from unittest.mock import patch - -from annotationlib import Format, get_annotations, type_repr - - -class TestStringFormatSpecialAnnotations(unittest.TestCase): - def test_dict_comprehension_annotation(self): - def f(x: {k: v for k, v in items}): - pass - - self.assertEqual( - get_annotations(f, format=Format.STRING), - {"x": "{k: v for k, v in items}"}, - ) - - def test_lambda_annotation(self): - def g(x: lambda q: q): - pass - - g_anno = get_annotations(g, format=Format.STRING) - self.assertEqual(g_anno, {"x": "lambda q: q"}) - self.assertNotIn("0x", g_anno["x"].lower()) - - def test_generator_expression_annotation(self): - def h(x: (w for w in seq)): - pass - - h_anno = get_annotations(h, format=Format.STRING) - self.assertEqual(h_anno, {"x": "(w for w in seq)"}) - self.assertNotIn("0x", h_anno["x"].lower()) - - def test_mixed_annotations(self): - def mixed(a: int, b: {k: v for k, v in items}, c: lambda q: q): - pass - - self.assertEqual( - get_annotations(mixed, format=Format.STRING), - { - "a": "int", - "b": "{k: v for k, v in items}", - "c": "lambda q: q", - }, - ) - - def test_quoted_annotation_not_requoted_on_unpack_fallback(self): - def f(a: "int", b: {k: v for k, v in items}): - pass - - self.assertEqual( - get_annotations(f, format=Format.STRING), - {"a": "int", "b": "{k: v for k, v in items}"}, - ) - - def test_nested_lambda_uses_source_and_has_no_address(self): - def f(x: [lambda q: q]): - pass - - anno = get_annotations(f, format=Format.STRING) - self.assertEqual(anno, {"x": "[lambda q: q]"}) - self.assertNotIn("0x", anno["x"].lower()) - - def test_class_dictcomp_keeps_conditional_annotations(self): - class C: - a: int - b: {k: v for k, v in items} - if True: - c: str - - self.assertEqual( - get_annotations(C, format=Format.STRING), - {"a": "int", "b": "{k: v for k, v in items}", "c": "str"}, - ) - - def test_simple_string_format_does_not_read_source(self): - def f(x: int) -> str: - pass - - with patch.object(inspect, "getsource") as mocked: - self.assertEqual( - get_annotations(f, format=Format.STRING), - {"x": "int", "return": "str"}, - ) - mocked.assert_not_called() - - def test_exec_without_source_still_raises_on_dictcomp(self): - ns = {} - exec("def f(x: {k: v for k, v in items}): pass", ns) - with self.assertRaises(ValueError): - get_annotations(ns["f"], format=Format.STRING) - - def test_type_repr_lambda_and_genexpr_have_no_address(self): - lam = (lambda q: q) - self.assertTrue(type_repr(lam).endswith("")) - self.assertNotIn("0x", type_repr(lam).lower()) - gen = (w for w in ()) - self.assertTrue(type_repr(gen).endswith("")) - self.assertNotIn("0x", type_repr(gen).lower()) - - -if __name__ == "__main__": - unittest.main() From 71478669820cad07240e11289d7c6a646ae9bee5 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:49:03 -0600 Subject: [PATCH 11/12] gh-157056: Remove hand-minted NEWS entry --- .../Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst diff --git a/Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst b/Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst deleted file mode 100644 index 4436d5aaabc24f0..000000000000000 --- a/Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst +++ /dev/null @@ -1,4 +0,0 @@ -:func:`annotationlib.get_annotations` with ``format=Format.STRING`` no -longer fails on dict-comprehension annotations, and no longer emits -non-deterministic strings that embed a memory address for ``lambda`` and -generator-expression annotations. From f71ca65a108d3fdbc76dc79c5dc3105dec404aa1 Mon Sep 17 00:00:00 2001 From: Kailash Nelson <37966146+KingLizard1020@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:49:56 -0600 Subject: [PATCH 12/12] gh-157056: Restore original NEWS entry --- .../Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst diff --git a/Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst b/Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst new file mode 100644 index 000000000000000..4436d5aaabc24f0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-08-01-00-00.gh-issue-157056.annotstr.rst @@ -0,0 +1,4 @@ +:func:`annotationlib.get_annotations` with ``format=Format.STRING`` no +longer fails on dict-comprehension annotations, and no longer emits +non-deterministic strings that embed a memory address for ``lambda`` and +generator-expression annotations.