From 0ce97cc14a210232914f0cc7c1e127f4162e0a8e Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Mon, 1 Apr 2024 15:48:24 +0100 Subject: [PATCH 1/2] Add a failing test --- .../test/fixtures/pyflakes/F821_26.pyi | 18 +++++++++++++++++- crates/ruff_python_semantic/src/model.rs | 3 +++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/ruff_linter/resources/test/fixtures/pyflakes/F821_26.pyi b/crates/ruff_linter/resources/test/fixtures/pyflakes/F821_26.pyi index f87819ef8004b..73b2b01a0b19f 100644 --- a/crates/ruff_linter/resources/test/fixtures/pyflakes/F821_26.pyi +++ b/crates/ruff_linter/resources/test/fixtures/pyflakes/F821_26.pyi @@ -1,6 +1,6 @@ """Tests for constructs allowed in `.pyi` stub files but not at runtime""" -from typing import Optional, TypeAlias, Union +from typing import Generic, NewType, Optional, TypeAlias, TypeVar, Union __version__: str __author__: str @@ -33,6 +33,19 @@ class Leaf: ... class Tree(list[Tree | Leaf]): ... # valid in a `.pyi` stub file, not in a `.py` runtime file class Tree2(list["Tree | Leaf"]): ... # always okay +# Generic bases can have forward references in stubs +class Foo(Generic[T]): ... +T = TypeVar("T") +class Bar(Foo[Baz]): ... +class Baz: ... + +# bases in general can be forward references in stubs +class Eggs(Spam): ... +class Spam: ... + +# NewType can have forward references +MyNew = NewType("MyNew", MyClass) + # Annotations are treated as assignments in .pyi files, but not in .py files class MyClass: foo: int @@ -42,3 +55,6 @@ class MyClass: baz: MyClass eggs = baz # valid in a `.pyi` stub file, not in a `.py` runtime file eggs = "baz" # always okay + +class Blah: + class Blah2(Blah): ... diff --git a/crates/ruff_python_semantic/src/model.rs b/crates/ruff_python_semantic/src/model.rs index ac735d97acd29..7656661802b93 100644 --- a/crates/ruff_python_semantic/src/model.rs +++ b/crates/ruff_python_semantic/src/model.rs @@ -1794,6 +1794,9 @@ bitflags! { /// /// `__future__`-style type annotations are only enabled if the `annotations` feature /// is enabled via `from __future__ import annotations`. + /// + /// Note that this flag is only set when we are actually *visiting* the deferred definition, + /// not when we "pass by" it when initially traversing the source tree. const FUTURE_TYPE_DEFINITION = 1 << 6; /// The model is in an exception handler. From 0819d861ad3ff33b6c90f9d3340260caa772b658 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 4 Apr 2024 21:39:51 +0100 Subject: [PATCH 2/2] Fix the failing test --- .../ruff_linter/src/checkers/ast/deferred.rs | 3 ++ crates/ruff_linter/src/checkers/ast/mod.rs | 29 +++++++++++++++++ crates/ruff_python_semantic/src/model.rs | 31 ++++++++++++++++++- 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/crates/ruff_linter/src/checkers/ast/deferred.rs b/crates/ruff_linter/src/checkers/ast/deferred.rs index e1ca4a8656375..7f390e7afd577 100644 --- a/crates/ruff_linter/src/checkers/ast/deferred.rs +++ b/crates/ruff_linter/src/checkers/ast/deferred.rs @@ -12,6 +12,8 @@ pub(crate) struct Visit<'a> { pub(crate) type_param_definitions: Vec<(&'a Expr, Snapshot)>, pub(crate) functions: Vec, pub(crate) lambdas: Vec, + /// N.B. This field should always be empty unless it's a stub file + pub(crate) class_bases: Vec<(&'a Expr, Snapshot)>, } impl Visit<'_> { @@ -22,6 +24,7 @@ impl Visit<'_> { && self.type_param_definitions.is_empty() && self.functions.is_empty() && self.lambdas.is_empty() + && self.class_bases.is_empty() } } diff --git a/crates/ruff_linter/src/checkers/ast/mod.rs b/crates/ruff_linter/src/checkers/ast/mod.rs index 6af274cb8f331..a05f531e15db5 100644 --- a/crates/ruff_linter/src/checkers/ast/mod.rs +++ b/crates/ruff_linter/src/checkers/ast/mod.rs @@ -712,7 +712,9 @@ impl<'a> Visitor<'a> for Checker<'a> { } if let Some(arguments) = arguments { + self.semantic.flags |= SemanticModelFlags::CLASS_BASE; self.visit_arguments(arguments); + self.semantic.flags -= SemanticModelFlags::CLASS_BASE; } let definition = docstrings::extraction::extract_definition( @@ -935,6 +937,16 @@ impl<'a> Visitor<'a> for Checker<'a> { fn visit_expr(&mut self, expr: &'a Expr) { // Step 0: Pre-processing + if self.source_type.is_stub() + && self.semantic.in_class_base() + && !self.semantic.in_deferred_class_base() + { + self.visit + .class_bases + .push((expr, self.semantic.snapshot())); + return; + } + if !self.semantic.in_typing_literal() && !self.semantic.in_deferred_type_definition() && self.semantic.in_type_definition() @@ -1965,6 +1977,22 @@ impl<'a> Checker<'a> { scope.add(id, binding_id); } + fn visit_deferred_class_bases(&mut self) { + let snapshot = self.semantic.snapshot(); + let deferred_bases = std::mem::take(&mut self.visit.class_bases); + debug_assert!( + self.source_type.is_stub() || deferred_bases.is_empty(), + "Class bases should never be deferred outside of stub files" + ); + for (expr, snapshot) in deferred_bases { + self.semantic.restore(snapshot); + // Set this flag to avoid infinite recursion, or we'll just defer it again: + self.semantic.flags |= SemanticModelFlags::DEFERRED_CLASS_BASE; + self.visit_expr(expr); + } + self.semantic.restore(snapshot); + } + fn visit_deferred_future_type_definitions(&mut self) { let snapshot = self.semantic.snapshot(); while !self.visit.future_type_definitions.is_empty() { @@ -2097,6 +2125,7 @@ impl<'a> Checker<'a> { /// annotations. fn visit_deferred(&mut self, allocator: &'a typed_arena::Arena) { while !self.visit.is_empty() { + self.visit_deferred_class_bases(); self.visit_deferred_functions(); self.visit_deferred_type_param_definitions(); self.visit_deferred_lambdas(); diff --git a/crates/ruff_python_semantic/src/model.rs b/crates/ruff_python_semantic/src/model.rs index 7656661802b93..c181179aa2f91 100644 --- a/crates/ruff_python_semantic/src/model.rs +++ b/crates/ruff_python_semantic/src/model.rs @@ -1489,7 +1489,8 @@ impl<'a> SemanticModel<'a> { .intersects(SemanticModelFlags::FUTURE_TYPE_DEFINITION) } - /// Return `true` if the model is in any kind of deferred type definition. + /// Return `true` if the model is visiting any kind of type definition + /// that was previously deferred when initially traversing the AST pub const fn in_deferred_type_definition(&self) -> bool { self.flags .intersects(SemanticModelFlags::DEFERRED_TYPE_DEFINITION) @@ -1603,6 +1604,20 @@ impl<'a> SemanticModel<'a> { .intersects(SemanticModelFlags::DUNDER_ALL_DEFINITION) } + /// Return `true` if the model is visiting an item in a class's bases tuple + /// (e.g. `Foo` in `class Bar(Foo): ...`) + pub const fn in_class_base(&self) -> bool { + self.flags.intersects(SemanticModelFlags::CLASS_BASE) + } + + /// Return `true` if the model is visiting an item in a class's bases tuple + /// that was initially deferred while traversing the AST. + /// (This only happens in stub files.) + pub const fn in_deferred_class_base(&self) -> bool { + self.flags + .intersects(SemanticModelFlags::DEFERRED_CLASS_BASE) + } + /// Return an iterator over all bindings shadowed by the given [`BindingId`], within the /// containing scope, and across scopes. pub fn shadowed_bindings( @@ -1978,6 +1993,20 @@ bitflags! { /// ``` const F_STRING_REPLACEMENT_FIELD = 1 << 23; + /// The model is visiting the bases tuple of a class. + /// + /// For example, the model could be visiting `Foo` or `Bar` in: + /// + /// ```python + /// class Baz(Foo, Bar): + /// pass + /// ``` + const CLASS_BASE = 1 << 24; + + /// The model is visiting a class base that was initially deferred + /// while traversing the AST. (This only happens in stub files.) + const DEFERRED_CLASS_BASE = 1 << 25; + /// The context is in any type annotation. const ANNOTATION = Self::TYPING_ONLY_ANNOTATION.bits() | Self::RUNTIME_EVALUATED_ANNOTATION.bits() | Self::RUNTIME_REQUIRED_ANNOTATION.bits();