PHP 8.3 | Tokenizer/PHP: add support for typed OO constants

src/Tokenizers/PHP.php

@@ -526,8 +526,9 @@ protected function tokenize($string)
         $numTokens         = count($tokens);
         $lastNotEmptyToken = 0;
 
-        $insideInlineIf = [];
-        $insideUseGroup = false;
+        $insideInlineIf         = [];
+        $insideUseGroup         = false;
+        $insideConstDeclaration = false;
 
         $commentTokenizer = new Comment();
 
@@ -608,7 +609,8 @@ protected function tokenize($string)
             if ($tokenIsArray === true
                 && isset(Util\Tokens::$contextSensitiveKeywords[$token[0]]) === true
                 && (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
-                || $finalTokens[$lastNotEmptyToken]['content'] === '&')
+                || $finalTokens[$lastNotEmptyToken]['content'] === '&'
+                || $insideConstDeclaration === true)
             ) {
                 if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
                     $preserveKeyword = false;
@@ -665,6 +667,30 @@ protected function tokenize($string)
                     }
                 }//end if
 
+                // Types in typed constants should not be touched, but the constant name should be.
+                if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                    && $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
+                    || $insideConstDeclaration === true
+                ) {
+                    $preserveKeyword = true;
+
+                    // Find the next non-empty token.
+                    for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
+                        if (is_array($tokens[$i]) === true
+                            && isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
+                        ) {
+                            continue;
+                        }
+
+                        break;
+                    }
+
+                    if ($tokens[$i] === '=' || $tokens[$i] === ';') {
+                        $preserveKeyword        = false;
+                        $insideConstDeclaration = false;
+                    }
+                }//end if
+
                 if ($finalTokens[$lastNotEmptyToken]['content'] === '&') {
                     $preserveKeyword = true;
 
@@ -698,6 +724,26 @@ protected function tokenize($string)
                 }
             }//end if
 
+            /*
+                Mark the start of a constant declaration to allow for handling keyword to T_STRING
+                convertion for constant names using reserved keywords.
+            */
+
+            if ($tokenIsArray === true && $token[0] === T_CONST) {
+                $insideConstDeclaration = true;
+            }
+
+            /*
+                Close an open "inside constant declaration" marker when no keyword convertion was needed.
+            */
+
+            if ($insideConstDeclaration === true
+                && $tokenIsArray === false
+                && ($token[0] === '=' || $token[0] === ';')
+            ) {
+                $insideConstDeclaration = false;
+            }
+
             /*
                 Special case for `static` used as a function name, i.e. `static()`.
             */
@@ -1869,6 +1915,20 @@ protected function tokenize($string)
                 $newToken            = [];
                 $newToken['content'] = '?';
 
+                // For typed constants, we only need to check the token before the ? to be sure.
+                if ($finalTokens[$lastNotEmptyToken]['code'] === T_CONST) {
+                    $newToken['code'] = T_NULLABLE;
+                    $newToken['type'] = 'T_NULLABLE';
+
+                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                        echo "\t\t* token $stackPtr changed from ? to T_NULLABLE".PHP_EOL;
+                    }
+
+                    $finalTokens[$newStackPtr] = $newToken;
+                    $newStackPtr++;
+                    continue;
+                }
+
                 /*
                  * Check if the next non-empty token is one of the tokens which can be used
                  * in type declarations. If not, it's definitely a ternary.
@@ -2236,7 +2296,30 @@ function return types. We want to keep the parenthesis map clean,
                 if ($tokenIsArray === true && $token[0] === T_STRING) {
                     $preserveTstring = false;
 
-                    if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
+                    // True/false/parent/self/static in typed constants should be fixed to their own token,
+                    // but the constant name should not be.
+                    if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                        && $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
+                        || $insideConstDeclaration === true
+                    ) {
+                        // Find the next non-empty token.
+                        for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
+                            if (is_array($tokens[$i]) === true
+                                && isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
+                            ) {
+                                continue;
+                            }
+
+                            break;
+                        }
+
+                        if ($tokens[$i] === '=') {
+                            $preserveTstring        = true;
+                            $insideConstDeclaration = false;
+                        }
+                    } else if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                        && $finalTokens[$lastNotEmptyToken]['code'] !== T_CONST
+                    ) {
                         $preserveTstring = true;
 
                         // Special case for syntax like: return new self/new parent
@@ -3008,6 +3091,12 @@ protected function processAdditional()
                         $suspectedType = 'return';
                     }
 
+                    if ($this->tokens[$x]['code'] === T_EQUAL) {
+                        // Possible constant declaration, the `T_STRING` name will have been skipped over already.
+                        $suspectedType = 'constant';
+                        break;
+                    }
+
                     break;
                 }//end for
 
@@ -3049,6 +3138,11 @@ protected function processAdditional()
                         break;
                     }
 
+                    if ($suspectedType === 'constant' && $this->tokens[$x]['code'] === T_CONST) {
+                        $confirmed = true;
+                        break;
+                    }
+
                     if ($suspectedType === 'property or parameter'
                         && (isset(Util\Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true
                         || $this->tokens[$x]['code'] === T_VAR

tests/Core/Tokenizer/ArrayKeywordTest.inc

@@ -21,15 +21,21 @@ $var = array(
 );
 
 /* testFunctionDeclarationParamType */
-function foo(array $a) {}
+function typedParam(array $a) {}
 
 /* testFunctionDeclarationReturnType */
-function foo($a) : int|array|null {}
+function returnType($a) : int|array|null {}
 
 class Bar {
     /* testClassConst */
     const ARRAY = [];
 
     /* testClassMethod */
     public function array() {}
+
+    /* testOOConstType */
+    const array /* testTypedOOConstName */ ARRAY = /* testOOConstDefault */ array();
+
+    /* testOOPropertyType */
+    protected array $property;
 }

tests/Core/Tokenizer/ArrayKeywordTest.php

@@ -68,6 +68,9 @@ public static function dataArrayKeyword()
             'nested: inner array'                   => [
                 'testMarker' => '/* testNestedArray */',
             ],
+            'OO constant default value'             => [
+                'testMarker' => '/* testOOConstDefault */',
+            ],
         ];
 
     }//end dataArrayKeyword()
@@ -122,6 +125,12 @@ public static function dataArrayType()
             'function union return type' => [
                 'testMarker' => '/* testFunctionDeclarationReturnType */',
             ],
+            'OO constant type'           => [
+                'testMarker' => '/* testOOConstType */',
+            ],
+            'OO property type'           => [
+                'testMarker' => '/* testOOPropertyType */',
+            ],
         ];
 
     }//end dataArrayType()
@@ -167,13 +176,17 @@ public function testNotArrayKeyword($testMarker, $testContent='array')
     public static function dataNotArrayKeyword()
     {
         return [
-            'class-constant-name' => [
+            'class-constant-name'            => [
                 'testMarker'  => '/* testClassConst */',
                 'testContent' => 'ARRAY',
             ],
-            'class-method-name'   => [
+            'class-method-name'              => [
                 'testMarker' => '/* testClassMethod */',
             ],
+            'class-constant-name-after-type' => [
+                'testMarker'  => '/* testTypedOOConstName */',
+                'testContent' => 'ARRAY',
+            ],
         ];
 
     }//end dataNotArrayKeyword()

tests/Core/Tokenizer/BitwiseOrTest.inc

@@ -9,6 +9,30 @@ $result = $value | $test /* testBitwiseOr2 */ | $another;
 
 class TypeUnion
 {
+    /* testTypeUnionOOConstSimple */
+    public const Foo|Bar SIMPLE = new Foo;
+
+    /* testTypeUnionOOConstReverseModifierOrder */
+    protected final const int|float MODIFIERS_REVERSED /* testBitwiseOrOOConstDefaultValue */ = E_WARNING | E_NOTICE;
+
+    const
+        /* testTypeUnionOOConstMulti1 */
+        array |
+        /* testTypeUnionOOConstMulti2 */
+        Traversable | // phpcs:ignore Stnd.Cat.Sniff
+        false
+        /* testTypeUnionOOConstMulti3 */
+        | null MULTI_UNION = false;
+
+    /* testTypeUnionOOConstNamespaceRelative */
+    final protected const namespace\Sub\NameA|namespace\Sub\NameB NAMESPACE_RELATIVE = new namespace\Sub\NameB;
+
+    /* testTypeUnionOOConstPartiallyQualified */
+    const Partially\Qualified\NameA|Partially\Qualified\NameB PARTIALLY_QUALIFIED = new Partially\Qualified\NameA;
+
+    /* testTypeUnionOOConstFullyQualified */
+    const \Fully\Qualified\NameA|\Fully\Qualified\NameB FULLY_QUALIFIED = new \Fully\Qualified\NameB();
+
     /* testTypeUnionPropertySimple */
     public static Foo|Bar $obj;
 

tests/Core/Tokenizer/BitwiseOrTest.php

@@ -47,6 +47,7 @@ public static function dataBitwiseOr()
         return [
             'in simple assignment 1'                    => ['/* testBitwiseOr1 */'],
             'in simple assignment 2'                    => ['/* testBitwiseOr2 */'],
+            'in OO constant default value'              => ['/* testBitwiseOrOOConstDefaultValue */'],
             'in property default value'                 => ['/* testBitwiseOrPropertyDefaultValue */'],
             'in method parameter default value'         => ['/* testBitwiseOrParamDefaultValue */'],
             'in return statement'                       => ['/* testBitwiseOr3 */'],
@@ -97,6 +98,14 @@ public function testTypeUnion($testMarker)
     public static function dataTypeUnion()
     {
         return [
+            'type for OO constant'                                     => ['/* testTypeUnionOOConstSimple */'],
+            'type for OO constant, reversed modifier order'            => ['/* testTypeUnionOOConstReverseModifierOrder */'],
+            'type for OO constant, first of multi-union'               => ['/* testTypeUnionOOConstMulti1 */'],
+            'type for OO constant, middle of multi-union + comments'   => ['/* testTypeUnionOOConstMulti2 */'],
+            'type for OO constant, last of multi-union'                => ['/* testTypeUnionOOConstMulti3 */'],
+            'type for OO constant, using namespace relative names'     => ['/* testTypeUnionOOConstNamespaceRelative */'],
+            'type for OO constant, using partially qualified names'    => ['/* testTypeUnionOOConstPartiallyQualified */'],
+            'type for OO constant, using fully qualified names'        => ['/* testTypeUnionOOConstFullyQualified */'],
             'type for static property'                                 => ['/* testTypeUnionPropertySimple */'],
             'type for static property, reversed modifier order'        => ['/* testTypeUnionPropertyReverseModifierOrder */'],
             'type for property, first of multi-union'                  => ['/* testTypeUnionPropertyMulti1 */'],

tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc

@@ -76,6 +76,12 @@ class ContextSensitiveKeywords
     const /* testAnd */ AND = 'LOGICAL_AND';
     const /* testOr */ OR = 'LOGICAL_OR';
     const /* testXor */ XOR = 'LOGICAL_XOR';
+
+    const /* testArrayIsTstringInConstType */ array /* testArrayNameForTypedConstant */ ARRAY = /* testArrayIsKeywordInConstDefault */ array();
+    const /* testStaticIsKeywordAsConstType */ static /* testStaticIsNameForTypedConstant */ STATIC = new /* testStaticIsKeywordAsConstDefault */ static;
+
+    const int|bool /* testPrivateNameForUnionTypedConstant */ PRIVATE = 'PRIVATE';
+    const Foo&Bar /* testFinalNameForIntersectionTypedConstant */ FINAL = 'FINAL';
 }
 
 namespace /* testKeywordAfterNamespaceShouldBeString */ class;

tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php

@@ -118,6 +118,12 @@ public static function dataStrings()
             'constant declaration: or'                        => ['/* testOr */'],
             'constant declaration: xor'                       => ['/* testXor */'],
 
+            'constant declaration: array in type'             => ['/* testArrayIsTstringInConstType */'],
+            'constant declaration: array, name after type'    => ['/* testArrayNameForTypedConstant */'],
+            'constant declaration: static, name after type'   => ['/* testStaticIsNameForTypedConstant */'],
+            'constant declaration: private, name after type'  => ['/* testPrivateNameForUnionTypedConstant */'],
+            'constant declaration: final, name after type'    => ['/* testFinalNameForIntersectionTypedConstant */'],
+
             'namespace declaration: class'                    => ['/* testKeywordAfterNamespaceShouldBeString */'],
             'namespace declaration (partial): my'             => ['/* testNamespaceNameIsString1 */'],
             'namespace declaration (partial): class'          => ['/* testNamespaceNameIsString2 */'],
@@ -179,6 +185,19 @@ public static function dataKeywords()
                 'testMarker'        => '/* testNamespaceIsKeyword */',
                 'expectedTokenType' => 'T_NAMESPACE',
             ],
+            'array: default value in const decl'     => [
+                'testMarker'        => '/* testArrayIsKeywordInConstDefault */',
+                'expectedTokenType' => 'T_ARRAY',
+            ],
+            'static: type in constant declaration'   => [
+                'testMarker'        => '/* testStaticIsKeywordAsConstType */',
+                'expectedTokenType' => 'T_STATIC',
+            ],
+            'static: value in constant declaration'  => [
+                'testMarker'        => '/* testStaticIsKeywordAsConstDefault */',
+                'expectedTokenType' => 'T_STATIC',
+            ],
+
             'abstract: class declaration'            => [
                 'testMarker'        => '/* testAbstractIsKeyword */',
                 'expectedTokenType' => 'T_ABSTRACT',

tests/Core/Tokenizer/OtherContextSensitiveKeywordsTest.inc

@@ -51,3 +51,17 @@ function standAloneFalseTrueNullTypesAndMore(
     || $a === /* testNullIsKeywordInComparison */ null
     ) {}
 }
+
+class TypedConstProp {
+    const /* testFalseIsKeywordAsConstType */ false /* testFalseIsNameForTypedConstant */ FALSE = /* testFalseIsKeywordAsConstDefault */ false;
+    const /* testTrueIsKeywordAsConstType */ true /* testTrueIsNameForTypedConstant */ TRUE = /* testTrueIsKeywordAsConstDefault */ true;
+    const /* testNullIsKeywordAsConstType */ null /* testNullIsNameForTypedConstant */ NULL = /* testNullIsKeywordAsConstDefault */ null;
+    const /* testSelfIsKeywordAsConstType */ self /* testSelfIsNameForTypedConstant */ SELF = new /* testSelfIsKeywordAsConstDefault */ self;
+    const /* testParentIsKeywordAsConstType */ parent /* testParentIsNameForTypedConstant */ PARENT = new /* testParentIsKeywordAsConstDefault */ parent;
+
+    public /* testFalseIsKeywordAsPropertyType */ false $false = /* testFalseIsKeywordAsPropertyDefault */ false;
+    protected readonly /* testTrueIsKeywordAsPropertyType */ true $true = /* testTrueIsKeywordAsPropertyDefault */ true;
+    static private /* testNullIsKeywordAsPropertyType */ null $null = /* testNullIsKeywordAsPropertyDefault */ null;
+    var /* testSelfIsKeywordAsPropertyType */ self $self = new /* testSelfIsKeywordAsPropertyDefault */ self;
+    protected /* testParentIsKeywordAsPropertyType */ parent $parent = new /* testParentIsKeywordAsPropertyDefault */ parent;
+}