Skip to content

Commit

Permalink
Improve internal documentation for the semantic model (#10788)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlexWaygood
AlexWaygood committed Apr 6, 2024
1 parent 7fb5f47 commit 1dc9310
Show file tree
Hide file tree
Showing 5 changed files with 142 additions and 39 deletions.
21 changes: 8 additions & 13 deletions crates/ruff_linter/src/checkers/ast/analyze/expression.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
if let Some(operator) = typing::to_pep604_operator(value, slice, &checker.semantic)
{
if checker.enabled(Rule::FutureRewritableTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py310
&& checker.settings.target_version >= PythonVersion::Py37
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing
{
Expand All @@ -48,7 +47,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
if checker.source_type.is_stub()
|| checker.settings.target_version >= PythonVersion::Py310
|| (checker.settings.target_version >= PythonVersion::Py37
&& checker.semantic.future_annotations()
&& checker.semantic.future_annotations_or_stub()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing)
{
Expand All @@ -60,9 +59,8 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {

// Ex) list[...]
if checker.enabled(Rule::FutureRequiredTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py39
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
&& typing::is_pep585_generic(value, &checker.semantic)
{
Expand Down Expand Up @@ -186,10 +184,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
typing::to_pep585_generic(expr, &checker.semantic)
{
if checker.enabled(Rule::FutureRewritableTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py39
&& checker.settings.target_version >= PythonVersion::Py37
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing
{
Expand All @@ -200,7 +197,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
if checker.source_type.is_stub()
|| checker.settings.target_version >= PythonVersion::Py39
|| (checker.settings.target_version >= PythonVersion::Py37
&& checker.semantic.future_annotations()
&& checker.semantic.future_annotations_or_stub()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing)
{
Expand Down Expand Up @@ -270,10 +267,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
]) {
if let Some(replacement) = typing::to_pep585_generic(expr, &checker.semantic) {
if checker.enabled(Rule::FutureRewritableTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py39
&& checker.settings.target_version >= PythonVersion::Py37
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing
{
Expand All @@ -286,7 +282,7 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
if checker.source_type.is_stub()
|| checker.settings.target_version >= PythonVersion::Py39
|| (checker.settings.target_version >= PythonVersion::Py37
&& checker.semantic.future_annotations()
&& checker.semantic.future_annotations_or_stub()
&& checker.semantic.in_annotation()
&& !checker.settings.pyupgrade.keep_runtime_typing)
{
Expand Down Expand Up @@ -1176,9 +1172,8 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
}) => {
// Ex) `str | None`
if checker.enabled(Rule::FutureRequiredTypeAnnotation) {
if !checker.source_type.is_stub()
if !checker.semantic.future_annotations_or_stub()
&& checker.settings.target_version < PythonVersion::Py310
&& !checker.semantic.future_annotations()
&& checker.semantic.in_annotation()
{
flake8_future_annotations::rules::future_required_type_annotation(
Expand Down
9 changes: 5 additions & 4 deletions crates/ruff_linter/src/checkers/ast/annotation.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,10 @@ impl AnnotationContext {
_ => {}
}

// If `__future__` annotations are enabled, then annotations are never evaluated
// at runtime, so we can treat them as typing-only.
if semantic.future_annotations() {
// If `__future__` annotations are enabled or it's a stub file,
// then annotations are never evaluated at runtime,
// so we can treat them as typing-only.
if semantic.future_annotations_or_stub() {
return Self::TypingOnly;
}

Expand Down Expand Up @@ -87,7 +88,7 @@ impl AnnotationContext {
semantic,
) {
Self::RuntimeRequired
} else if semantic.future_annotations() {
} else if semantic.future_annotations_or_stub() {
Self::TypingOnly
} else {
Self::RuntimeEvaluated
Expand Down
73 changes: 67 additions & 6 deletions crates/ruff_linter/src/checkers/ast/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -935,9 +935,12 @@ impl<'a> Visitor<'a> for Checker<'a> {
fn visit_expr(&mut self, expr: &'a Expr) {
// Step 0: Pre-processing
if !self.semantic.in_typing_literal()
// `in_deferred_type_definition()` will only be `true` if we're now visiting the deferred nodes
// after having already traversed the source tree once. If we're now visiting the deferred nodes,
// we can't defer again, or we'll infinitely recurse!
&& !self.semantic.in_deferred_type_definition()
&& self.semantic.in_type_definition()
&& self.semantic.future_annotations()
&& self.semantic.future_annotations_or_stub()
&& (self.semantic.in_annotation() || self.source_type.is_stub())
{
if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = expr {
Expand Down Expand Up @@ -1964,13 +1967,40 @@ impl<'a> Checker<'a> {
scope.add(id, binding_id);
}

/// After initial traversal of the AST, visit all "future type definitions".
///
/// A "future type definition" is a type definition where [PEP 563] semantics
/// apply (i.e., an annotation in a module that has `from __future__ import annotations`
/// at the top of the file, or an annotation in a stub file). These type definitions
/// support forward references, so they are deferred on initial traversal
/// of the source tree.
///
/// For example:
/// ```python
/// from __future__ import annotations
///
/// def foo() -> Bar: # <-- return annotation is a "future type definition"
/// return Bar()
///
/// class Bar: pass
/// ```
///
/// [PEP 563]: https://peps.python.org/pep-0563/
fn visit_deferred_future_type_definitions(&mut self) {
let snapshot = self.semantic.snapshot();
while !self.visit.future_type_definitions.is_empty() {
let type_definitions = std::mem::take(&mut self.visit.future_type_definitions);
for (expr, snapshot) in type_definitions {
self.semantic.restore(snapshot);

// Type definitions should only be considered "`__future__` type definitions"
// if they are annotations in a module where `from __future__ import
// annotations` is active, or they are type definitions in a stub file.
debug_assert!(
self.semantic.future_annotations_or_stub()
&& (self.source_type.is_stub() || self.semantic.in_annotation())
);

self.semantic.flags |= SemanticModelFlags::TYPE_DEFINITION
| SemanticModelFlags::FUTURE_TYPE_DEFINITION;
self.visit_expr(expr);
Expand All @@ -1979,6 +2009,19 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}

/// After initial traversal of the AST, visit all [type parameter definitions].
///
/// Type parameters natively support forward references,
/// so are always deferred during initial traversal of the source tree.
///
/// For example:
/// ```python
/// class Foo[T: Bar]: pass # <-- Forward reference used in definition of type parameter `T`
/// type X[T: Bar] = Foo[T] # <-- Ditto
/// class Bar: pass
/// ```
///
/// [type parameter definitions]: https://docs.python.org/3/reference/executionmodel.html#annotation-scopes
fn visit_deferred_type_param_definitions(&mut self) {
let snapshot = self.semantic.snapshot();
while !self.visit.type_param_definitions.is_empty() {
Expand All @@ -1994,6 +2037,17 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}

/// After initial traversal of the AST, visit all "string type definitions",
/// i.e., type definitions that are enclosed within quotes so as to allow
/// the type definition to use forward references.
///
/// For example:
/// ```python
/// def foo() -> "Bar": # <-- return annotation is a "string type definition"
/// return Bar()
///
/// class Bar: pass
/// ```
fn visit_deferred_string_type_definitions(&mut self, allocator: &'a typed_arena::Arena<Expr>) {
let snapshot = self.semantic.snapshot();
while !self.visit.string_type_definitions.is_empty() {
Expand All @@ -2006,7 +2060,7 @@ impl<'a> Checker<'a> {

self.semantic.restore(snapshot);

if self.semantic.in_annotation() && self.semantic.future_annotations() {
if self.semantic.in_annotation() && self.semantic.future_annotations_or_stub() {
if self.enabled(Rule::QuotedAnnotation) {
pyupgrade::rules::quoted_annotation(self, value, range);
}
Expand Down Expand Up @@ -2042,6 +2096,11 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}

/// After initial traversal of the AST, visit all function bodies.
///
/// Function bodies are always deferred on initial traversal of the source tree,
/// as the body of a function may validly contain references to global-scope symbols
/// that were not yet defined at the point when the function was defined.
fn visit_deferred_functions(&mut self) {
let snapshot = self.semantic.snapshot();
while !self.visit.functions.is_empty() {
Expand All @@ -2065,8 +2124,9 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}

/// Visit all deferred lambdas. Returns a list of snapshots, such that the caller can restore
/// the semantic model to the state it was in before visiting the deferred lambdas.
/// After initial traversal of the source tree has been completed,
/// visit all lambdas. Lambdas are deferred during the initial traversal
/// for the same reason as function bodies.
fn visit_deferred_lambdas(&mut self) {
let snapshot = self.semantic.snapshot();
while !self.visit.lambdas.is_empty() {
Expand All @@ -2092,8 +2152,9 @@ impl<'a> Checker<'a> {
self.semantic.restore(snapshot);
}

/// Recursively visit all deferred AST nodes, including lambdas, functions, and type
/// annotations.
/// After initial traversal of the source tree has been completed,
/// recursively visit all AST nodes that were deferred on the first pass.
/// This includes lambdas, functions, type parameters, and type annotations.
fn visit_deferred(&mut self, allocator: &'a typed_arena::Arena<Expr>) {
while !self.visit.is_empty() {
self.visit_deferred_functions();
Expand Down

0 comments on commit 1dc9310

Please sign in to comment.