From 9b77a0e467893e9009cd5b956a94bd13543caede Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Fri, 24 Feb 2023 10:58:56 +0100 Subject: [PATCH 01/14] Refactor TokenTypeAnalyzerBase --- .../Rules/Utilities/TokenTypeAnalyzer.cs | 17 +- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 147 ++++++++++-------- .../Rules/Utilities/TokenTypeAnalyzer.cs | 17 +- 3 files changed, 107 insertions(+), 74 deletions(-) diff --git a/analyzers/src/SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs b/analyzers/src/SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs index 83011352439..d8ce07bcea4 100644 --- a/analyzers/src/SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs +++ b/analyzers/src/SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs @@ -25,12 +25,14 @@ public class TokenTypeAnalyzer : TokenTypeAnalyzerBase { protected override ILanguageFacade Language { get; } = CSharpFacade.Instance; - protected override TokenClassifierBase GetTokenClassifier(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifierTokens) => - new TokenClassifier(token, semanticModel, skipIdentifierTokens); + protected override TokenClassifierBase GetTokenClassifier(SemanticModel semanticModel, bool skipIdentifierTokens) => + new TokenClassifier(semanticModel, skipIdentifierTokens); + protected override TriviaClassifierBase GetTriviaClassifier() => + new TriviaClassifier(); private sealed class TokenClassifier : TokenClassifierBase { - public TokenClassifier(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifiers) : base(token, semanticModel, skipIdentifiers) { } + public TokenClassifier(SemanticModel semanticModel, bool skipIdentifiers) : base(semanticModel, skipIdentifiers) { } protected override SyntaxNode GetBindableParent(SyntaxToken token) => token.GetBindableParent(); @@ -41,9 +43,6 @@ private sealed class TokenClassifier : TokenClassifierBase protected override bool IsKeyword(SyntaxToken token) => SyntaxFacts.IsKeywordKind(token.Kind()); - protected override bool IsRegularComment(SyntaxTrivia trivia) => - trivia.IsAnyKind(SyntaxKind.SingleLineCommentTrivia, SyntaxKind.MultiLineCommentTrivia); - protected override bool IsNumericLiteral(SyntaxToken token) => token.IsKind(SyntaxKind.NumericLiteralToken); @@ -63,6 +62,12 @@ private sealed class TokenClassifier : TokenClassifierBase SyntaxKind.InterpolatedStringTextToken, SyntaxKind.InterpolatedStringEndToken, SyntaxKindEx.InterpolatedRawStringEndToken); + } + + private sealed class TriviaClassifier: TriviaClassifierBase + { + protected override bool IsRegularComment(SyntaxTrivia trivia) => + trivia.IsAnyKind(SyntaxKind.SingleLineCommentTrivia, SyntaxKind.MultiLineCommentTrivia); protected override bool IsDocComment(SyntaxTrivia trivia) => trivia.IsAnyKind(SyntaxKind.SingleLineDocumentationCommentTrivia, SyntaxKind.MultiLineDocumentationCommentTrivia); diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index a4fe75b2e66..6feb29311e1 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -34,7 +34,8 @@ public abstract class TokenTypeAnalyzerBase : UtilityAnalyzerBase Language.Syntax.IsKind(token, identifierTokenKind)) > IdentifierTokenCountThreshold; + var tokenClassifier = GetTokenClassifier(semanticModel, skipIdentifierTokens); + var triviaClassifier = GetTriviaClassifier(); var spans = new List(); // The second iteration of the tokens is intended since there is no processing done and we want to avoid copying all the tokens to a second collection. foreach (var token in tokens) { - spans.AddRange(GetTokenClassifier(token, semanticModel, skipIdentifierTokens).Spans); + if (token.HasLeadingTrivia) + { + IterateTrivia(token.LeadingTrivia); + } + if (tokenClassifier.ClassifyToken(token) is { } tokenClassification) + { + spans.Add(tokenClassification); + } + if (token.HasTrailingTrivia) + { + IterateTrivia(token.TrailingTrivia); + } } var tokenTypeInfo = new TokenTypeInfo @@ -56,14 +70,23 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem tokenTypeInfo.TokenInfo.AddRange(spans.OrderBy(s => s.TextRange.StartLine).ThenBy(s => s.TextRange.StartOffset)); return tokenTypeInfo; + + void IterateTrivia(SyntaxTriviaList triviaList) + { + foreach (var trivia in triviaList) + { + if (triviaClassifier.ClassifyTrivia(trivia) is { } triviaClassification) + { + spans.Add(triviaClassification); + } + } + } } protected abstract class TokenClassifierBase { - private readonly SyntaxToken token; private readonly SemanticModel semanticModel; private readonly bool skipIdentifiers; - private readonly List spans = new(); private static readonly ISet ConstructorKinds = new HashSet { MethodKind.Constructor, @@ -80,130 +103,130 @@ protected abstract class TokenClassifierBase }; protected abstract SyntaxNode GetBindableParent(SyntaxToken token); - protected abstract bool IsDocComment(SyntaxTrivia trivia); - protected abstract bool IsRegularComment(SyntaxTrivia trivia); protected abstract bool IsKeyword(SyntaxToken token); protected abstract bool IsIdentifier(SyntaxToken token); protected abstract bool IsNumericLiteral(SyntaxToken token); protected abstract bool IsStringLiteral(SyntaxToken token); - protected TokenClassifierBase(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifiers) + protected TokenClassifierBase(SemanticModel semanticModel, bool skipIdentifiers) { - this.token = token; this.semanticModel = semanticModel; this.skipIdentifiers = skipIdentifiers; } - public IEnumerable Spans - { - get - { - spans.Clear(); - ClassifyToken(); - - foreach (var trivia in token.LeadingTrivia) - { - ClassifyTrivia(trivia); - } - - foreach (var trivia in token.TrailingTrivia) + private TokenTypeInfo.Types.TokenInfo TokenInfo(SyntaxToken token, TokenType tokenType) => + string.IsNullOrWhiteSpace(token.ValueText) + ? null + : new() { - ClassifyTrivia(trivia); - } - - return spans; - } - } + TokenType = tokenType, + TextRange = GetTextRange(token.GetLocation().GetLineSpan()), + }; - private void CollectClassified(TokenType tokenType, TextSpan span) - { - if (string.IsNullOrWhiteSpace(token.SyntaxTree.GetText().GetSubText(span).ToString())) - { - return; - } - - spans.Add(new TokenTypeInfo.Types.TokenInfo - { - TokenType = tokenType, - TextRange = GetTextRange(Location.Create(token.SyntaxTree, span).GetLineSpan()) - }); - } - - private void ClassifyToken() + public TokenTypeInfo.Types.TokenInfo ClassifyToken(SyntaxToken token) { if (IsKeyword(token)) { - CollectClassified(TokenType.Keyword, token.Span); + return TokenInfo(token, TokenType.Keyword); } else if (IsStringLiteral(token)) { - CollectClassified(TokenType.StringLiteral, token.Span); + return TokenInfo(token, TokenType.StringLiteral); } else if (IsNumericLiteral(token)) { - CollectClassified(TokenType.NumericLiteral, token.Span); + return TokenInfo(token, TokenType.NumericLiteral); } else if (IsIdentifier(token) && !skipIdentifiers) { - ClassifyIdentifier(); + return ClassifyIdentifier(token); + } + else + { + return null; } } - private void ClassifyIdentifier() + private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token) { if (semanticModel.GetDeclaredSymbol(token.Parent) is { } declaration) { - ClassifyIdentifier(declaration); + return ClassifyIdentifier(token, declaration); + } + else if (GetBindableParent(token) is { } parent && semanticModel.GetSymbolInfo(parent).Symbol is { } symbol) + { + return ClassifyIdentifier(token, symbol); } - else if (GetBindableParent(token) is { } parent && semanticModel.GetSymbolInfo(parent).Symbol is { } symbol) + else { - ClassifyIdentifier(symbol); + return null; } } - private void ClassifyIdentifier(ISymbol symbol) + private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token, ISymbol symbol) { if (symbol.Kind == SymbolKind.Alias) { - ClassifyIdentifier(((IAliasSymbol)symbol).Target); + return ClassifyIdentifier(token, ((IAliasSymbol)symbol).Target); } else if (symbol is IMethodSymbol ctorSymbol && ConstructorKinds.Contains(ctorSymbol.MethodKind)) { - CollectClassified(TokenType.TypeName, token.Span); + return TokenInfo(token, TokenType.TypeName); } else if (token.ToString() == "var" && VarSymbolKinds.Contains(symbol.Kind)) { - CollectClassified(TokenType.Keyword, token.Span); + return TokenInfo(token, TokenType.Keyword); } else if (token.ToString() == "value" && symbol.Kind == SymbolKind.Parameter && symbol.IsImplicitlyDeclared) { - CollectClassified(TokenType.Keyword, token.Span); + return TokenInfo(token, TokenType.Keyword); } else if (symbol.Kind == SymbolKind.NamedType || symbol.Kind == SymbolKind.TypeParameter) { - CollectClassified(TokenType.TypeName, token.Span); + return TokenInfo(token, TokenType.TypeName); } else if (symbol.Kind == SymbolKind.DynamicType) { - CollectClassified(TokenType.Keyword, token.Span); + return TokenInfo(token, TokenType.Keyword); + } + else + { + return null; } } + } + + protected abstract class TriviaClassifierBase + { + protected abstract bool IsDocComment(SyntaxTrivia trivia); + protected abstract bool IsRegularComment(SyntaxTrivia trivia); - private void ClassifyTrivia(SyntaxTrivia trivia) + public TokenTypeInfo.Types.TokenInfo ClassifyTrivia(SyntaxTrivia trivia) { if (IsRegularComment(trivia)) { - CollectClassified(TokenType.Comment, trivia.Span); + return CollectClassified(trivia.SyntaxTree, TokenType.Comment, trivia.Span); } else if (IsDocComment(trivia)) { - ClassifyDocComment(trivia); + return ClassifyDocComment(trivia); + } + else + { + return null; } // Handle preprocessor directives here } - private void ClassifyDocComment(SyntaxTrivia trivia) => - CollectClassified(TokenType.Comment, trivia.FullSpan); + private TokenTypeInfo.Types.TokenInfo ClassifyDocComment(SyntaxTrivia trivia) => + CollectClassified(trivia.SyntaxTree, TokenType.Comment, trivia.FullSpan); + + private TokenTypeInfo.Types.TokenInfo CollectClassified(SyntaxTree tree, TokenType tokenType, TextSpan span) => + new() + { + TokenType = tokenType, + TextRange = GetTextRange(Location.Create(tree, span).GetLineSpan()) + }; } } } diff --git a/analyzers/src/SonarAnalyzer.VisualBasic/Rules/Utilities/TokenTypeAnalyzer.cs b/analyzers/src/SonarAnalyzer.VisualBasic/Rules/Utilities/TokenTypeAnalyzer.cs index dd91aed69d4..da551d8f679 100644 --- a/analyzers/src/SonarAnalyzer.VisualBasic/Rules/Utilities/TokenTypeAnalyzer.cs +++ b/analyzers/src/SonarAnalyzer.VisualBasic/Rules/Utilities/TokenTypeAnalyzer.cs @@ -25,12 +25,14 @@ public class TokenTypeAnalyzer : TokenTypeAnalyzerBase { protected override ILanguageFacade Language { get; } = VisualBasicFacade.Instance; - protected override TokenClassifierBase GetTokenClassifier(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifierTokens) => - new TokenClassifier(token, semanticModel, skipIdentifierTokens); + protected override TokenClassifierBase GetTokenClassifier(SemanticModel semanticModel, bool skipIdentifierTokens) => + new TokenClassifier(semanticModel, skipIdentifierTokens); + protected override TriviaClassifierBase GetTriviaClassifier() => + new TriviaClassifier(); private sealed class TokenClassifier : TokenClassifierBase { - public TokenClassifier(SyntaxToken token, SemanticModel semanticModel, bool skipIdentifiers) : base(token, semanticModel, skipIdentifiers) { } + public TokenClassifier(SemanticModel semanticModel, bool skipIdentifiers) : base(semanticModel, skipIdentifiers) { } protected override SyntaxNode GetBindableParent(SyntaxToken token) => token.GetBindableParent(); @@ -41,9 +43,6 @@ private sealed class TokenClassifier : TokenClassifierBase protected override bool IsKeyword(SyntaxToken token) => SyntaxFacts.IsKeywordKind(token.Kind()); - protected override bool IsRegularComment(SyntaxTrivia trivia) => - trivia.IsKind(SyntaxKind.CommentTrivia); - protected override bool IsNumericLiteral(SyntaxToken token) => token.IsAnyKind(SyntaxKind.DecimalLiteralToken, SyntaxKind.FloatingLiteralToken, SyntaxKind.IntegerLiteralToken); @@ -53,6 +52,12 @@ private sealed class TokenClassifier : TokenClassifierBase SyntaxKind.CharacterLiteralToken, SyntaxKind.InterpolatedStringTextToken, SyntaxKind.EndOfInterpolatedStringToken); + } + + private sealed class TriviaClassifier : TriviaClassifierBase + { + protected override bool IsRegularComment(SyntaxTrivia trivia) => + trivia.IsKind(SyntaxKind.CommentTrivia); protected override bool IsDocComment(SyntaxTrivia trivia) => trivia.IsKind(SyntaxKind.DocumentationCommentTrivia); From 3a52260f3ba1764efe9f843adccf0cf2e47757a0 Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Fri, 24 Feb 2023 11:42:46 +0100 Subject: [PATCH 02/14] Simplify classification methods. --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 108 ++++++------------ 1 file changed, 33 insertions(+), 75 deletions(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index 6feb29311e1..69c7a76170e 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -68,7 +68,7 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem FilePath = syntaxTree.FilePath }; - tokenTypeInfo.TokenInfo.AddRange(spans.OrderBy(s => s.TextRange.StartLine).ThenBy(s => s.TextRange.StartOffset)); + tokenTypeInfo.TokenInfo.AddRange(spans); return tokenTypeInfo; void IterateTrivia(SyntaxTriviaList triviaList) @@ -118,34 +118,20 @@ protected TokenClassifierBase(SemanticModel semanticModel, bool skipIdentifiers) string.IsNullOrWhiteSpace(token.ValueText) ? null : new() - { - TokenType = tokenType, - TextRange = GetTextRange(token.GetLocation().GetLineSpan()), - }; - - public TokenTypeInfo.Types.TokenInfo ClassifyToken(SyntaxToken token) - { - if (IsKeyword(token)) - { - return TokenInfo(token, TokenType.Keyword); - } - else if (IsStringLiteral(token)) - { - return TokenInfo(token, TokenType.StringLiteral); - } - else if (IsNumericLiteral(token)) { - return TokenInfo(token, TokenType.NumericLiteral); - } - else if (IsIdentifier(token) && !skipIdentifiers) - { - return ClassifyIdentifier(token); - } - else + TokenType = tokenType, + TextRange = GetTextRange(token.GetLocation().GetLineSpan()), + }; + + public TokenTypeInfo.Types.TokenInfo ClassifyToken(SyntaxToken token) => + token switch { - return null; - } - } + _ when IsKeyword(token) => TokenInfo(token, TokenType.Keyword), + _ when IsStringLiteral(token) => TokenInfo(token, TokenType.StringLiteral), + _ when IsNumericLiteral(token) => TokenInfo(token, TokenType.NumericLiteral), + _ when IsIdentifier(token) && !skipIdentifiers => ClassifyIdentifier(token), + _ => null, + }; private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token) { @@ -163,37 +149,17 @@ private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token) } } - private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token, ISymbol symbol) - { - if (symbol.Kind == SymbolKind.Alias) - { - return ClassifyIdentifier(token, ((IAliasSymbol)symbol).Target); - } - else if (symbol is IMethodSymbol ctorSymbol && ConstructorKinds.Contains(ctorSymbol.MethodKind)) - { - return TokenInfo(token, TokenType.TypeName); - } - else if (token.ToString() == "var" && VarSymbolKinds.Contains(symbol.Kind)) - { - return TokenInfo(token, TokenType.Keyword); - } - else if (token.ToString() == "value" && symbol.Kind == SymbolKind.Parameter && symbol.IsImplicitlyDeclared) - { - return TokenInfo(token, TokenType.Keyword); - } - else if (symbol.Kind == SymbolKind.NamedType || symbol.Kind == SymbolKind.TypeParameter) - { - return TokenInfo(token, TokenType.TypeName); - } - else if (symbol.Kind == SymbolKind.DynamicType) + private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token, ISymbol symbol) => + symbol switch { - return TokenInfo(token, TokenType.Keyword); - } - else - { - return null; - } - } + IAliasSymbol alias => ClassifyIdentifier(token, alias.Target), + IMethodSymbol ctorSymbol when ConstructorKinds.Contains(ctorSymbol.MethodKind) => TokenInfo(token, TokenType.TypeName), + _ when token.ValueText == "var" && VarSymbolKinds.Contains(symbol.Kind) => TokenInfo(token, TokenType.Keyword), + { Kind: SymbolKind.Parameter, IsImplicitlyDeclared: true } when token.ValueText == "value" => TokenInfo(token, TokenType.Keyword), + { Kind: SymbolKind.NamedType or SymbolKind.TypeParameter } => TokenInfo(token, TokenType.TypeName), + { Kind: SymbolKind.DynamicType } => TokenInfo(token, TokenType.Keyword), + _ => null, + }; } protected abstract class TriviaClassifierBase @@ -201,25 +167,14 @@ protected abstract class TriviaClassifierBase protected abstract bool IsDocComment(SyntaxTrivia trivia); protected abstract bool IsRegularComment(SyntaxTrivia trivia); - public TokenTypeInfo.Types.TokenInfo ClassifyTrivia(SyntaxTrivia trivia) - { - if (IsRegularComment(trivia)) - { - return CollectClassified(trivia.SyntaxTree, TokenType.Comment, trivia.Span); - } - else if (IsDocComment(trivia)) + public TokenTypeInfo.Types.TokenInfo ClassifyTrivia(SyntaxTrivia trivia) => + trivia switch { - return ClassifyDocComment(trivia); - } - else - { - return null; - } - // Handle preprocessor directives here - } - - private TokenTypeInfo.Types.TokenInfo ClassifyDocComment(SyntaxTrivia trivia) => - CollectClassified(trivia.SyntaxTree, TokenType.Comment, trivia.FullSpan); + _ when IsRegularComment(trivia) => CollectClassified(trivia.SyntaxTree, TokenType.Comment, trivia.Span), + _ when IsDocComment(trivia) => ClassifyDocComment(trivia), + // Handle preprocessor directives here + _ => null, + }; private TokenTypeInfo.Types.TokenInfo CollectClassified(SyntaxTree tree, TokenType tokenType, TextSpan span) => new() @@ -227,6 +182,9 @@ public TokenTypeInfo.Types.TokenInfo ClassifyTrivia(SyntaxTrivia trivia) TokenType = tokenType, TextRange = GetTextRange(Location.Create(tree, span).GetLineSpan()) }; + + private TokenTypeInfo.Types.TokenInfo ClassifyDocComment(SyntaxTrivia trivia) => + CollectClassified(trivia.SyntaxTree, TokenType.Comment, trivia.FullSpan); } } } From e9b2f70a9c062897176ac145ef8bd9f37e8573a7 Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Fri, 24 Feb 2023 13:06:44 +0100 Subject: [PATCH 03/14] Formatting --- .../SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs | 3 ++- .../Rules/Utilities/TokenTypeAnalyzer.cs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/analyzers/src/SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs b/analyzers/src/SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs index d8ce07bcea4..299cc07b3ad 100644 --- a/analyzers/src/SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs +++ b/analyzers/src/SonarAnalyzer.CSharp/Rules/Utilities/TokenTypeAnalyzer.cs @@ -27,6 +27,7 @@ public class TokenTypeAnalyzer : TokenTypeAnalyzerBase protected override TokenClassifierBase GetTokenClassifier(SemanticModel semanticModel, bool skipIdentifierTokens) => new TokenClassifier(semanticModel, skipIdentifierTokens); + protected override TriviaClassifierBase GetTriviaClassifier() => new TriviaClassifier(); @@ -64,7 +65,7 @@ private sealed class TokenClassifier : TokenClassifierBase SyntaxKindEx.InterpolatedRawStringEndToken); } - private sealed class TriviaClassifier: TriviaClassifierBase + private sealed class TriviaClassifier : TriviaClassifierBase { protected override bool IsRegularComment(SyntaxTrivia trivia) => trivia.IsAnyKind(SyntaxKind.SingleLineCommentTrivia, SyntaxKind.MultiLineCommentTrivia); diff --git a/analyzers/src/SonarAnalyzer.VisualBasic/Rules/Utilities/TokenTypeAnalyzer.cs b/analyzers/src/SonarAnalyzer.VisualBasic/Rules/Utilities/TokenTypeAnalyzer.cs index da551d8f679..03bd726b0f2 100644 --- a/analyzers/src/SonarAnalyzer.VisualBasic/Rules/Utilities/TokenTypeAnalyzer.cs +++ b/analyzers/src/SonarAnalyzer.VisualBasic/Rules/Utilities/TokenTypeAnalyzer.cs @@ -27,6 +27,7 @@ public class TokenTypeAnalyzer : TokenTypeAnalyzerBase protected override TokenClassifierBase GetTokenClassifier(SemanticModel semanticModel, bool skipIdentifierTokens) => new TokenClassifier(semanticModel, skipIdentifierTokens); + protected override TriviaClassifierBase GetTriviaClassifier() => new TriviaClassifier(); From 6b37c7f6e616184fca5a19138f62cf996f8aca12 Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Fri, 24 Feb 2023 13:23:11 +0100 Subject: [PATCH 04/14] Rename method --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index 69c7a76170e..fba23581e9c 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -41,7 +41,7 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem { var tokens = syntaxTree.GetRoot().DescendantTokens(); var identifierTokenKind = Language.SyntaxKind.IdentifierToken; // Performance optimization - var skipIdentifierTokens = tokens.Count(token => Language.Syntax.IsKind(token, identifierTokenKind)) > IdentifierTokenCountThreshold; + var skipIdentifierTokens = tokens.Take(IdentifierTokenCountThreshold + 1).Count(token => Language.Syntax.IsKind(token, identifierTokenKind)) > IdentifierTokenCountThreshold; var tokenClassifier = GetTokenClassifier(semanticModel, skipIdentifierTokens); var triviaClassifier = GetTriviaClassifier(); @@ -170,13 +170,13 @@ protected abstract class TriviaClassifierBase public TokenTypeInfo.Types.TokenInfo ClassifyTrivia(SyntaxTrivia trivia) => trivia switch { - _ when IsRegularComment(trivia) => CollectClassified(trivia.SyntaxTree, TokenType.Comment, trivia.Span), + _ when IsRegularComment(trivia) => TokenInfo(trivia.SyntaxTree, TokenType.Comment, trivia.Span), _ when IsDocComment(trivia) => ClassifyDocComment(trivia), // Handle preprocessor directives here _ => null, }; - private TokenTypeInfo.Types.TokenInfo CollectClassified(SyntaxTree tree, TokenType tokenType, TextSpan span) => + private TokenTypeInfo.Types.TokenInfo TokenInfo(SyntaxTree tree, TokenType tokenType, TextSpan span) => new() { TokenType = tokenType, @@ -184,7 +184,7 @@ trivia switch }; private TokenTypeInfo.Types.TokenInfo ClassifyDocComment(SyntaxTrivia trivia) => - CollectClassified(trivia.SyntaxTree, TokenType.Comment, trivia.FullSpan); + TokenInfo(trivia.SyntaxTree, TokenType.Comment, trivia.FullSpan); } } } From 7daf5e6ade221bb9ba9ea1b3db0dfdffea7de045 Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Fri, 24 Feb 2023 13:26:06 +0100 Subject: [PATCH 05/14] EnableConcurrentExecution --- .../SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs index 9ccc48ce48f..d996d542ed9 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs @@ -36,7 +36,7 @@ public abstract class UtilityAnalyzerBase : SonarDiagnosticAnalyzer protected virtual bool AnalyzeTestProjects => true; protected string OutPath { get; set; } protected bool IsTestProject { get; set; } - protected override bool EnableConcurrentExecution => false; + protected override bool EnableConcurrentExecution => true; protected UtilityAnalyzerBase(string diagnosticId, string title) => rule = DiagnosticDescriptorFactory.CreateUtility(diagnosticId, title); From 925d4e83db8f9923b9c0a7d9a7090655aaa4d717 Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Fri, 24 Feb 2023 13:48:44 +0100 Subject: [PATCH 06/14] Fix "Take" for IdentifierTokenCountThreshold --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index fba23581e9c..dd83236be60 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -41,7 +41,7 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem { var tokens = syntaxTree.GetRoot().DescendantTokens(); var identifierTokenKind = Language.SyntaxKind.IdentifierToken; // Performance optimization - var skipIdentifierTokens = tokens.Take(IdentifierTokenCountThreshold + 1).Count(token => Language.Syntax.IsKind(token, identifierTokenKind)) > IdentifierTokenCountThreshold; + var skipIdentifierTokens = tokens.Where(token => Language.Syntax.IsKind(token, identifierTokenKind)).Take(IdentifierTokenCountThreshold + 1).Count() > IdentifierTokenCountThreshold; var tokenClassifier = GetTokenClassifier(semanticModel, skipIdentifierTokens); var triviaClassifier = GetTriviaClassifier(); From b82a6f0acc5acc80d3b5ff400ba4b3692590b33b Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Fri, 24 Feb 2023 13:49:29 +0100 Subject: [PATCH 07/14] Formatting --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index dd83236be60..ba14f9e6758 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -41,7 +41,10 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem { var tokens = syntaxTree.GetRoot().DescendantTokens(); var identifierTokenKind = Language.SyntaxKind.IdentifierToken; // Performance optimization - var skipIdentifierTokens = tokens.Where(token => Language.Syntax.IsKind(token, identifierTokenKind)).Take(IdentifierTokenCountThreshold + 1).Count() > IdentifierTokenCountThreshold; + var skipIdentifierTokens = tokens + .Where(token => Language.Syntax.IsKind(token, identifierTokenKind)) + .Take(IdentifierTokenCountThreshold + 1) + .Count() > IdentifierTokenCountThreshold; var tokenClassifier = GetTokenClassifier(semanticModel, skipIdentifierTokens); var triviaClassifier = GetTriviaClassifier(); From dadb05134b3a8dc482db8fe2a4622fa915b6a14d Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Fri, 24 Feb 2023 14:24:21 +0100 Subject: [PATCH 08/14] Disable EnableConcurrentExecution --- .../SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs index d996d542ed9..9ccc48ce48f 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/UtilityAnalyzerBase.cs @@ -36,7 +36,7 @@ public abstract class UtilityAnalyzerBase : SonarDiagnosticAnalyzer protected virtual bool AnalyzeTestProjects => true; protected string OutPath { get; set; } protected bool IsTestProject { get; set; } - protected override bool EnableConcurrentExecution => true; + protected override bool EnableConcurrentExecution => false; protected UtilityAnalyzerBase(string diagnosticId, string title) => rule = DiagnosticDescriptorFactory.CreateUtility(diagnosticId, title); From 959d51d5b7d75bbcd3a04bb8bd2db2125bff8f55 Mon Sep 17 00:00:00 2001 From: Martin Strecker <103252490+martin-strecker-sonarsource@users.noreply.github.com> Date: Mon, 27 Feb 2023 16:53:57 +0100 Subject: [PATCH 09/14] Apply suggestions from code review Co-authored-by: Tim Pohlmann --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index ba14f9e6758..d990ee75d70 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -43,8 +43,8 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem var identifierTokenKind = Language.SyntaxKind.IdentifierToken; // Performance optimization var skipIdentifierTokens = tokens .Where(token => Language.Syntax.IsKind(token, identifierTokenKind)) - .Take(IdentifierTokenCountThreshold + 1) - .Count() > IdentifierTokenCountThreshold; + .Skip(IdentifierTokenCountThreshold) + .Any() var tokenClassifier = GetTokenClassifier(semanticModel, skipIdentifierTokens); var triviaClassifier = GetTriviaClassifier(); From c5e8b1da55700af5ad01f2a13fea0847516e7461 Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Mon, 27 Feb 2023 17:04:38 +0100 Subject: [PATCH 10/14] Make IterateTrivia static local function --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index d990ee75d70..76f1351a7be 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -54,7 +54,7 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem { if (token.HasLeadingTrivia) { - IterateTrivia(token.LeadingTrivia); + IterateTrivia(triviaClassifier, spans, token.LeadingTrivia); } if (tokenClassifier.ClassifyToken(token) is { } tokenClassification) { @@ -62,7 +62,7 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem } if (token.HasTrailingTrivia) { - IterateTrivia(token.TrailingTrivia); + IterateTrivia(triviaClassifier, spans, token.TrailingTrivia); } } @@ -74,7 +74,7 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem tokenTypeInfo.TokenInfo.AddRange(spans); return tokenTypeInfo; - void IterateTrivia(SyntaxTriviaList triviaList) + static void IterateTrivia(TriviaClassifierBase triviaClassifier, List spans, SyntaxTriviaList triviaList) { foreach (var trivia in triviaList) { From 618cb72606b7f2c02f993d7bca84466d9ac19788 Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Mon, 27 Feb 2023 17:05:08 +0100 Subject: [PATCH 11/14] Fix syntax error --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index 76f1351a7be..1e2b5ba2ec0 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -44,7 +44,7 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem var skipIdentifierTokens = tokens .Where(token => Language.Syntax.IsKind(token, identifierTokenKind)) .Skip(IdentifierTokenCountThreshold) - .Any() + .Any(); var tokenClassifier = GetTokenClassifier(semanticModel, skipIdentifierTokens); var triviaClassifier = GetTriviaClassifier(); From a6ee74f8f60dcaa7bec2979d611d1eca40e6b6ea Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Mon, 27 Feb 2023 17:07:29 +0100 Subject: [PATCH 12/14] Indentation --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index 1e2b5ba2ec0..85f54613a4d 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -119,12 +119,12 @@ protected TokenClassifierBase(SemanticModel semanticModel, bool skipIdentifiers) private TokenTypeInfo.Types.TokenInfo TokenInfo(SyntaxToken token, TokenType tokenType) => string.IsNullOrWhiteSpace(token.ValueText) - ? null - : new() - { - TokenType = tokenType, - TextRange = GetTextRange(token.GetLocation().GetLineSpan()), - }; + ? null + : new() + { + TokenType = tokenType, + TextRange = GetTextRange(token.GetLocation().GetLineSpan()), + }; public TokenTypeInfo.Types.TokenInfo ClassifyToken(SyntaxToken token) => token switch From 97232cb6c87792acdc18f71c0cfa247cd9f8965e Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Mon, 27 Feb 2023 17:13:55 +0100 Subject: [PATCH 13/14] using static SonarAnalyzer.Protobuf.TokenTypeInfo.Types; --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index 85f54613a4d..d8d1333bb77 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -20,6 +20,7 @@ using Microsoft.CodeAnalysis.Text; using SonarAnalyzer.Protobuf; +using static SonarAnalyzer.Protobuf.TokenTypeInfo.Types; namespace SonarAnalyzer.Rules { @@ -48,7 +49,7 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem var tokenClassifier = GetTokenClassifier(semanticModel, skipIdentifierTokens); var triviaClassifier = GetTriviaClassifier(); - var spans = new List(); + var spans = new List(); // The second iteration of the tokens is intended since there is no processing done and we want to avoid copying all the tokens to a second collection. foreach (var token in tokens) { @@ -74,7 +75,7 @@ protected sealed override TokenTypeInfo CreateMessage(SyntaxTree syntaxTree, Sem tokenTypeInfo.TokenInfo.AddRange(spans); return tokenTypeInfo; - static void IterateTrivia(TriviaClassifierBase triviaClassifier, List spans, SyntaxTriviaList triviaList) + static void IterateTrivia(TriviaClassifierBase triviaClassifier, List spans, SyntaxTriviaList triviaList) { foreach (var trivia in triviaList) { @@ -117,7 +118,7 @@ protected TokenClassifierBase(SemanticModel semanticModel, bool skipIdentifiers) this.skipIdentifiers = skipIdentifiers; } - private TokenTypeInfo.Types.TokenInfo TokenInfo(SyntaxToken token, TokenType tokenType) => + private TokenInfo TokenInfo(SyntaxToken token, TokenType tokenType) => string.IsNullOrWhiteSpace(token.ValueText) ? null : new() @@ -126,7 +127,7 @@ protected TokenClassifierBase(SemanticModel semanticModel, bool skipIdentifiers) TextRange = GetTextRange(token.GetLocation().GetLineSpan()), }; - public TokenTypeInfo.Types.TokenInfo ClassifyToken(SyntaxToken token) => + public TokenInfo ClassifyToken(SyntaxToken token) => token switch { _ when IsKeyword(token) => TokenInfo(token, TokenType.Keyword), @@ -136,7 +137,7 @@ token switch _ => null, }; - private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token) + private TokenInfo ClassifyIdentifier(SyntaxToken token) { if (semanticModel.GetDeclaredSymbol(token.Parent) is { } declaration) { @@ -152,7 +153,7 @@ private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token) } } - private TokenTypeInfo.Types.TokenInfo ClassifyIdentifier(SyntaxToken token, ISymbol symbol) => + private TokenInfo ClassifyIdentifier(SyntaxToken token, ISymbol symbol) => symbol switch { IAliasSymbol alias => ClassifyIdentifier(token, alias.Target), @@ -170,7 +171,7 @@ protected abstract class TriviaClassifierBase protected abstract bool IsDocComment(SyntaxTrivia trivia); protected abstract bool IsRegularComment(SyntaxTrivia trivia); - public TokenTypeInfo.Types.TokenInfo ClassifyTrivia(SyntaxTrivia trivia) => + public TokenInfo ClassifyTrivia(SyntaxTrivia trivia) => trivia switch { _ when IsRegularComment(trivia) => TokenInfo(trivia.SyntaxTree, TokenType.Comment, trivia.Span), @@ -179,14 +180,14 @@ trivia switch _ => null, }; - private TokenTypeInfo.Types.TokenInfo TokenInfo(SyntaxTree tree, TokenType tokenType, TextSpan span) => + private TokenInfo TokenInfo(SyntaxTree tree, TokenType tokenType, TextSpan span) => new() { TokenType = tokenType, TextRange = GetTextRange(Location.Create(tree, span).GetLineSpan()) }; - private TokenTypeInfo.Types.TokenInfo ClassifyDocComment(SyntaxTrivia trivia) => + private TokenInfo ClassifyDocComment(SyntaxTrivia trivia) => TokenInfo(trivia.SyntaxTree, TokenType.Comment, trivia.FullSpan); } } From 24ac1a6ab353d28e5732b58e69201ccd74b5b25f Mon Sep 17 00:00:00 2001 From: Martin Strecker Date: Wed, 1 Mar 2023 08:36:06 +0100 Subject: [PATCH 14/14] Move TokenInfo() and make it static --- .../Rules/Utilities/TokenTypeAnalyzerBase.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs index d8d1333bb77..a4b2116e209 100644 --- a/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs +++ b/analyzers/src/SonarAnalyzer.Common/Rules/Utilities/TokenTypeAnalyzerBase.cs @@ -118,15 +118,6 @@ protected TokenClassifierBase(SemanticModel semanticModel, bool skipIdentifiers) this.skipIdentifiers = skipIdentifiers; } - private TokenInfo TokenInfo(SyntaxToken token, TokenType tokenType) => - string.IsNullOrWhiteSpace(token.ValueText) - ? null - : new() - { - TokenType = tokenType, - TextRange = GetTextRange(token.GetLocation().GetLineSpan()), - }; - public TokenInfo ClassifyToken(SyntaxToken token) => token switch { @@ -137,6 +128,15 @@ token switch _ => null, }; + private static TokenInfo TokenInfo(SyntaxToken token, TokenType tokenType) => + string.IsNullOrWhiteSpace(token.ValueText) + ? null + : new() + { + TokenType = tokenType, + TextRange = GetTextRange(token.GetLocation().GetLineSpan()), + }; + private TokenInfo ClassifyIdentifier(SyntaxToken token) { if (semanticModel.GetDeclaredSymbol(token.Parent) is { } declaration)