Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Performance: Token Type Utility Analyzer: Avoid allocations #6785

Merged
merged 14 commits into from Mar 1, 2023
Merged

Performance: Token Type Utility Analyzer: Avoid allocations #6785

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
9b77a0e
Refactor TokenTypeAnalyzerBase
martin-strecker-sonarsource Feb 24, 2023
3a52260
Simplify classification methods.
martin-strecker-sonarsource Feb 24, 2023
e9b2f70
Formatting
martin-strecker-sonarsource Feb 24, 2023
6b37c7f
Rename method
martin-strecker-sonarsource Feb 24, 2023
7daf5e6
EnableConcurrentExecution
martin-strecker-sonarsource Feb 24, 2023
925d4e8
Fix "Take" for IdentifierTokenCountThreshold
martin-strecker-sonarsource Feb 24, 2023
b82a6f0
Formatting
martin-strecker-sonarsource Feb 24, 2023
dadb051
Disable EnableConcurrentExecution
martin-strecker-sonarsource Feb 24, 2023
959d51d
Apply suggestions from code review
martin-strecker-sonarsource Feb 27, 2023
c5e8b1d
Make IterateTrivia static local function
martin-strecker-sonarsource Feb 27, 2023
618cb72
Fix syntax error
martin-strecker-sonarsource Feb 27, 2023
a6ee74f
Indentation
martin-strecker-sonarsource Feb 27, 2023
97232cb
using static SonarAnalyzer.Protobuf.TokenTypeInfo.Types;
martin-strecker-sonarsource Feb 27, 2023
24ac1a6
Move TokenInfo() and make it static
martin-strecker-sonarsource Mar 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
18 changes: 12 additions & 6 deletions analyzers/src/SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs
View file
Open in desktop
Expand Up @@ -25,12 +25,15 @@ public class TokenTypeAnalyzer : TokenTypeAnalyzerBase<SyntaxKind>
{
protected override ILanguageFacade<SyntaxKind> Language { get; } = CSharpFacade.Instance;

protected override TokenClassifierBase GetTokenClassifier(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifierTokens) =>
new TokenClassifier(token, semanticModel, skipIdentifierTokens);
protected override TokenClassifierBase GetTokenClassifier(SemanticModel semanticModel, bool skipIdentifierTokens) =>
new TokenClassifier(semanticModel, skipIdentifierTokens);

protected override TriviaClassifierBase GetTriviaClassifier() =>
new TriviaClassifier();

private sealed class TokenClassifier : TokenClassifierBase
{
public TokenClassifier(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifiers) : base(token, semanticModel, skipIdentifiers) { }
public TokenClassifier(SemanticModel semanticModel, bool skipIdentifiers) : base(semanticModel, skipIdentifiers) { }

protected override SyntaxNode GetBindableParent(SyntaxToken token) =>
token.GetBindableParent();
Expand All @@ -41,9 +44,6 @@ private sealed class TokenClassifier : TokenClassifierBase
protected override bool IsKeyword(SyntaxToken token) =>
SyntaxFacts.IsKeywordKind(token.Kind());

protected override bool IsRegularComment(SyntaxTrivia trivia) =>
trivia.IsAnyKind(SyntaxKind.SingleLineCommentTrivia, SyntaxKind.MultiLineCommentTrivia);

protected override bool IsNumericLiteral(SyntaxToken token) =>
token.IsKind(SyntaxKind.NumericLiteralToken);

Expand All @@ -63,6 +63,12 @@ private sealed class TokenClassifier : TokenClassifierBase
SyntaxKind.InterpolatedStringTextToken,
SyntaxKind.InterpolatedStringEndToken,
SyntaxKindEx.InterpolatedRawStringEndToken);
}

private sealed class TriviaClassifier : TriviaClassifierBase
{
protected override bool IsRegularComment(SyntaxTrivia trivia) =>
trivia.IsAnyKind(SyntaxKind.SingleLineCommentTrivia, SyntaxKind.MultiLineCommentTrivia);

protected override bool IsDocComment(SyntaxTrivia trivia) =>
trivia.IsAnyKind(SyntaxKind.SingleLineDocumentationCommentTrivia, SyntaxKind.MultiLineDocumentationCommentTrivia);
Expand Down
188 changes: 86 additions & 102 deletions analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs
View file
Open in desktop
Expand Up @@ -34,36 +34,62 @@ public abstract class TokenTypeAnalyzerBase<TSyntaxKind> : UtilityAnalyzerBase<T

protected TokenTypeAnalyzerBase() : base(DiagnosticId, Title) { }

protected abstract TokenClassifierBase GetTokenClassifier(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifierTokens);
protected abstract TokenClassifierBase GetTokenClassifier(SemanticModel semanticModel, bool skipIdentifierTokens);
protected abstract TriviaClassifierBase GetTriviaClassifier();
Tim-Pohlmann marked this conversation as resolved.
Show resolved Hide resolved

protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, SemanticModel semanticModel)
{
var tokens = syntaxTree.GetRoot().DescendantTokens();
var identifierTokenKind = Language.SyntaxKind.IdentifierToken; // Performance optimization
var skipIdentifierTokens = tokens.Count(token => Language.Syntax.IsKind(token, identifierTokenKind)) > IdentifierTokenCountThreshold;
var skipIdentifierTokens = tokens
.Where(token => Language.Syntax.IsKind(token, identifierTokenKind))
.Take(IdentifierTokenCountThreshold + 1)
.Count() > IdentifierTokenCountThreshold;
martin-strecker-sonarsource marked this conversation as resolved.
Show resolved Hide resolved

var tokenClassifier = GetTokenClassifier(semanticModel, skipIdentifierTokens);
var triviaClassifier = GetTriviaClassifier();
var spans = new List<TokenTypeInfo.Types.TokenInfo>();
Tim-Pohlmann marked this conversation as resolved.
Show resolved Hide resolved
// The second iteration of the tokens is intended since there is no processing done and we want to avoid copying all the tokens to a second collection.
foreach (var token in tokens)
{
spans.AddRange(GetTokenClassifier(token, semanticModel, skipIdentifierTokens).Spans);
if (token.HasLeadingTrivia)
{
IterateTrivia(token.LeadingTrivia);
}
if (tokenClassifier.ClassifyToken(token) is { } tokenClassification)
{
spans.Add(tokenClassification);
}
if (token.HasTrailingTrivia)
{
IterateTrivia(token.TrailingTrivia);
}
Tim-Pohlmann marked this conversation as resolved.
Show resolved Hide resolved
}

var tokenTypeInfo = new TokenTypeInfo
{
FilePath = syntaxTree.FilePath
};

tokenTypeInfo.TokenInfo.AddRange(spans.OrderBy(s => s.TextRange.StartLine).ThenBy(s => s.TextRange.StartOffset));
tokenTypeInfo.TokenInfo.AddRange(spans);
return tokenTypeInfo;

void IterateTrivia(SyntaxTriviaList triviaList)
{
foreach (var trivia in triviaList)
{
if (triviaClassifier.ClassifyTrivia(trivia) is { } triviaClassification)
{
spans.Add(triviaClassification);
}
}
}
}

protected abstract class TokenClassifierBase
{
private readonly SyntaxToken token;
private readonly SemanticModel semanticModel;
private readonly bool skipIdentifiers;
private readonly List<TokenTypeInfo.Types.TokenInfo> spans = new();
private static readonly ISet<MethodKind> ConstructorKinds = new HashSet<MethodKind>
{
MethodKind.Constructor,
Expand All @@ -80,130 +106,88 @@ protected abstract class TokenClassifierBase
};

protected abstract SyntaxNode GetBindableParent(SyntaxToken token);
protected abstract bool IsDocComment(SyntaxTrivia trivia);
protected abstract bool IsRegularComment(SyntaxTrivia trivia);
protected abstract bool IsKeyword(SyntaxToken token);
protected abstract bool IsIdentifier(SyntaxToken token);
protected abstract bool IsNumericLiteral(SyntaxToken token);
protected abstract bool IsStringLiteral(SyntaxToken token);

protected TokenClassifierBase(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifiers)
protected TokenClassifierBase(SemanticModel semanticModel, bool skipIdentifiers)
{
this.token = token;
this.semanticModel = semanticModel;
this.skipIdentifiers = skipIdentifiers;
}

public IEnumerable<TokenTypeInfo.Types.TokenInfo> Spans
{
get
{
spans.Clear();
ClassifyToken();

foreach (var trivia in token.LeadingTrivia)
{
ClassifyTrivia(trivia);
}

foreach (var trivia in token.TrailingTrivia)
{
ClassifyTrivia(trivia);
}

return spans;
}
}

private void CollectClassified(TokenType tokenType, TextSpan span)
{
if (string.IsNullOrWhiteSpace(token.SyntaxTree.GetText().GetSubText(span).ToString()))
{
return;
}

spans.Add(new TokenTypeInfo.Types.TokenInfo
private TokenTypeInfo.Types.TokenInfo TokenInfo(SyntaxToken token, TokenType tokenType) =>
string.IsNullOrWhiteSpace(token.ValueText)
? null
: new()
Tim-Pohlmann marked this conversation as resolved.
Show resolved Hide resolved
{
TokenType = tokenType,
TextRange = GetTextRange(Location.Create(token.SyntaxTree, span).GetLineSpan())
});
}
TextRange = GetTextRange(token.GetLocation().GetLineSpan()),
};

private void ClassifyToken()
{
if (IsKeyword(token))
public TokenTypeInfo.Types.TokenInfo ClassifyToken(SyntaxToken token) =>
token switch
{
CollectClassified(TokenType.Keyword, token.Span);
}
else if (IsStringLiteral(token))
{
CollectClassified(TokenType.StringLiteral, token.Span);
}
else if (IsNumericLiteral(token))
{
CollectClassified(TokenType.NumericLiteral, token.Span);
}
else if (IsIdentifier(token) && !skipIdentifiers)
{
ClassifyIdentifier();
}
}
_ when IsKeyword(token) => TokenInfo(token, TokenType.Keyword),
_ when IsStringLiteral(token) => TokenInfo(token, TokenType.StringLiteral),
_ when IsNumericLiteral(token) => TokenInfo(token, TokenType.NumericLiteral),
_ when IsIdentifier(token) && !skipIdentifiers => ClassifyIdentifier(token),
_ => null,
};

private void ClassifyIdentifier()
private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token)
Tim-Pohlmann marked this conversation as resolved.
Show resolved Hide resolved
{
if (semanticModel.GetDeclaredSymbol(token.Parent) is { } declaration)
{
ClassifyIdentifier(declaration);
}
else if (GetBindableParent(token) is { } parent && semanticModel.GetSymbolInfo(parent).Symbol is { } symbol)
{
ClassifyIdentifier(symbol);
}
}

private void ClassifyIdentifier(ISymbol symbol)
{
if (symbol.Kind == SymbolKind.Alias)
{
ClassifyIdentifier(((IAliasSymbol)symbol).Target);
}
else if (symbol is IMethodSymbol ctorSymbol && ConstructorKinds.Contains(ctorSymbol.MethodKind))
{
CollectClassified(TokenType.TypeName, token.Span);
return ClassifyIdentifier(token, declaration);
}
else if (token.ToString() == "var" && VarSymbolKinds.Contains(symbol.Kind))
else if (GetBindableParent(token) is { } parent && semanticModel.GetSymbolInfo(parent).Symbol is { } symbol)
{
CollectClassified(TokenType.Keyword, token.Span);
return ClassifyIdentifier(token, symbol);
}
else if (token.ToString() == "value" && symbol.Kind == SymbolKind.Parameter && symbol.IsImplicitlyDeclared)
else
{
CollectClassified(TokenType.Keyword, token.Span);
}
else if (symbol.Kind == SymbolKind.NamedType || symbol.Kind == SymbolKind.TypeParameter)
{
CollectClassified(TokenType.TypeName, token.Span);
}
else if (symbol.Kind == SymbolKind.DynamicType)
{
CollectClassified(TokenType.Keyword, token.Span);
return null;
}
}

private void ClassifyTrivia(SyntaxTrivia trivia)
{
if (IsRegularComment(trivia))
private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token, ISymbol symbol) =>
symbol switch
{
IAliasSymbol alias => ClassifyIdentifier(token, alias.Target),
IMethodSymbol ctorSymbol when ConstructorKinds.Contains(ctorSymbol.MethodKind) => TokenInfo(token, TokenType.TypeName),
_ when token.ValueText == "var" && VarSymbolKinds.Contains(symbol.Kind) => TokenInfo(token, TokenType.Keyword),
{ Kind: SymbolKind.Parameter, IsImplicitlyDeclared: true } when token.ValueText == "value" => TokenInfo(token, TokenType.Keyword),
{ Kind: SymbolKind.NamedType or SymbolKind.TypeParameter } => TokenInfo(token, TokenType.TypeName),
{ Kind: SymbolKind.DynamicType } => TokenInfo(token, TokenType.Keyword),
_ => null,
};
}

protected abstract class TriviaClassifierBase
{
protected abstract bool IsDocComment(SyntaxTrivia trivia);
protected abstract bool IsRegularComment(SyntaxTrivia trivia);

public TokenTypeInfo.Types.TokenInfo ClassifyTrivia(SyntaxTrivia trivia) =>
trivia switch
{
CollectClassified(TokenType.Comment, trivia.Span);
}
else if (IsDocComment(trivia))
_ when IsRegularComment(trivia) => TokenInfo(trivia.SyntaxTree, TokenType.Comment, trivia.Span),
_ when IsDocComment(trivia) => ClassifyDocComment(trivia),
// Handle preprocessor directives here
_ => null,
};

private TokenTypeInfo.Types.TokenInfo TokenInfo(SyntaxTree tree, TokenType tokenType, TextSpan span) =>
new()
{
ClassifyDocComment(trivia);
}
// Handle preprocessor directives here
}
TokenType = tokenType,
TextRange = GetTextRange(Location.Create(tree, span).GetLineSpan())
};

private void ClassifyDocComment(SyntaxTrivia trivia) =>
CollectClassified(TokenType.Comment, trivia.FullSpan);
private TokenTypeInfo.Types.TokenInfo ClassifyDocComment(SyntaxTrivia trivia) =>
TokenInfo(trivia.SyntaxTree, TokenType.Comment, trivia.FullSpan);
}
}
}
18 changes: 12 additions & 6 deletions analyzers/src/SonarAnalyzer.VisualBasic/Rules/Utilities/TokenTypeAnalyzer.cs
View file
Open in desktop
Expand Up @@ -25,12 +25,15 @@ public class TokenTypeAnalyzer : TokenTypeAnalyzerBase<SyntaxKind>
{
protected override ILanguageFacade<SyntaxKind> Language { get; } = VisualBasicFacade.Instance;

protected override TokenClassifierBase GetTokenClassifier(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifierTokens) =>
new TokenClassifier(token, semanticModel, skipIdentifierTokens);
protected override TokenClassifierBase GetTokenClassifier(SemanticModel semanticModel, bool skipIdentifierTokens) =>
new TokenClassifier(semanticModel, skipIdentifierTokens);

protected override TriviaClassifierBase GetTriviaClassifier() =>
new TriviaClassifier();

private sealed class TokenClassifier : TokenClassifierBase
{
public TokenClassifier(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifiers) : base(token, semanticModel, skipIdentifiers) { }
public TokenClassifier(SemanticModel semanticModel, bool skipIdentifiers) : base(semanticModel, skipIdentifiers) { }

protected override SyntaxNode GetBindableParent(SyntaxToken token) =>
token.GetBindableParent();
Expand All @@ -41,9 +44,6 @@ private sealed class TokenClassifier : TokenClassifierBase
protected override bool IsKeyword(SyntaxToken token) =>
SyntaxFacts.IsKeywordKind(token.Kind());

protected override bool IsRegularComment(SyntaxTrivia trivia) =>
trivia.IsKind(SyntaxKind.CommentTrivia);

protected override bool IsNumericLiteral(SyntaxToken token) =>
token.IsAnyKind(SyntaxKind.DecimalLiteralToken, SyntaxKind.FloatingLiteralToken, SyntaxKind.IntegerLiteralToken);

Expand All @@ -53,6 +53,12 @@ private sealed class TokenClassifier : TokenClassifierBase
SyntaxKind.CharacterLiteralToken,
SyntaxKind.InterpolatedStringTextToken,
SyntaxKind.EndOfInterpolatedStringToken);
}

private sealed class TriviaClassifier : TriviaClassifierBase
{
protected override bool IsRegularComment(SyntaxTrivia trivia) =>
trivia.IsKind(SyntaxKind.CommentTrivia);

protected override bool IsDocComment(SyntaxTrivia trivia) =>
trivia.IsKind(SyntaxKind.DocumentationCommentTrivia);
Expand Down