PHP 8.3 | Tokenizer/PHP: add support for typed OO constants

PHP 8.3 introduced typed OO constants, where the type is between the `const` keyword and the constant name.

All type variations are supported, including nullable types, union types, intersection types, with the exception of `callable`, `void` and `never`.
`self` and `static` types are only allowed in Enum constants.

This PR adds support for typed OO constants in the Tokenizer layer of PHPCS.

The following issues had to be fixed to support typed constants:
1. Consistently tokenizing the constant _name_ as `T_STRING`, even if the name mirrors a reserved keyword, like `foreach` or a special keyword, like `self` or `true`.
2. Tokenizing a `?` at the start of a constant type declaration as `T_NULLABLE`.
3. Tokenizing a `|` and `&` operators within a constant type declaration as `T_TYPE_UNION` and `T_TYPE_INTERSECTION` respectively.

Each and every part of the above has been covered by extensive tests.
Includes additional tests safeguarding that the `array` keyword when used in a type declaration for a constant is tokenized as `T_STRING`.

Ref: https://wiki.php.net/rfc/typed_class_constants

src/Tokenizers/PHP.php

@@ -526,8 +526,9 @@ protected function tokenize($string)
         $numTokens         = count($tokens);
         $lastNotEmptyToken = 0;
 
-        $insideInlineIf = [];
-        $insideUseGroup = false;
+        $insideInlineIf         = [];
+        $insideUseGroup         = false;
+        $insideConstDeclaration = false;
 
         $commentTokenizer = new Comment();
 
@@ -608,7 +609,8 @@ protected function tokenize($string)
             if ($tokenIsArray === true
                 && isset(Util\Tokens::$contextSensitiveKeywords[$token[0]]) === true
                 && (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
-                || $finalTokens[$lastNotEmptyToken]['content'] === '&')
+                || $finalTokens[$lastNotEmptyToken]['content'] === '&'
+                || $insideConstDeclaration === true)
             ) {
                 if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
                     $preserveKeyword = false;
@@ -665,6 +667,30 @@ protected function tokenize($string)
                     }
                 }//end if
 
+                // Types in typed constants should not be touched, but the constant name should be.
+                if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                    && $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
+                    || $insideConstDeclaration === true
+                ) {
+                    $preserveKeyword = true;
+
+                    // Find the next non-empty token.
+                    for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
+                        if (is_array($tokens[$i]) === true
+                            && isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
+                        ) {
+                            continue;
+                        }
+
+                        break;
+                    }
+
+                    if ($tokens[$i] === '=' || $tokens[$i] === ';') {
+                        $preserveKeyword        = false;
+                        $insideConstDeclaration = false;
+                    }
+                }//end if
+
                 if ($finalTokens[$lastNotEmptyToken]['content'] === '&') {
                     $preserveKeyword = true;
 
@@ -698,6 +724,26 @@ protected function tokenize($string)
                 }
             }//end if
 
+            /*
+                Mark the start of a constant declaration to allow for handling keyword to T_STRING
+                convertion for constant names using reserved keywords.
+            */
+
+            if ($tokenIsArray === true && $token[0] === T_CONST) {
+                $insideConstDeclaration = true;
+            }
+
+            /*
+                Close an open "inside constant declaration" marker when no keyword convertion was needed.
+            */
+
+            if ($insideConstDeclaration === true
+                && $tokenIsArray === false
+                && ($token[0] === '=' || $token[0] === ';')
+            ) {
+                $insideConstDeclaration = false;
+            }
+
             /*
                 Special case for `static` used as a function name, i.e. `static()`.
             */
@@ -1869,6 +1915,20 @@ protected function tokenize($string)
                 $newToken            = [];
                 $newToken['content'] = '?';
 
+                // For typed constants, we only need to check the token before the ? to be sure.
+                if ($finalTokens[$lastNotEmptyToken]['code'] === T_CONST) {
+                    $newToken['code'] = T_NULLABLE;
+                    $newToken['type'] = 'T_NULLABLE';
+
+                    if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                        echo "\t\t* token $stackPtr changed from ? to T_NULLABLE".PHP_EOL;
+                    }
+
+                    $finalTokens[$newStackPtr] = $newToken;
+                    $newStackPtr++;
+                    continue;
+                }
+
                 /*
                  * Check if the next non-empty token is one of the tokens which can be used
                  * in type declarations. If not, it's definitely a ternary.
@@ -2236,7 +2296,30 @@ function return types. We want to keep the parenthesis map clean,
                 if ($tokenIsArray === true && $token[0] === T_STRING) {
                     $preserveTstring = false;
 
-                    if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
+                    // True/false/parent/self/static in typed constants should be fixed to their own token,
+                    // but the constant name should not be.
+                    if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                        && $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
+                        || $insideConstDeclaration === true
+                    ) {
+                        // Find the next non-empty token.
+                        for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
+                            if (is_array($tokens[$i]) === true
+                                && isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
+                            ) {
+                                continue;
+                            }
+
+                            break;
+                        }
+
+                        if ($tokens[$i] === '=') {
+                            $preserveTstring        = true;
+                            $insideConstDeclaration = false;
+                        }
+                    } else if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
+                        && $finalTokens[$lastNotEmptyToken]['code'] !== T_CONST
+                    ) {
                         $preserveTstring = true;
 
                         // Special case for syntax like: return new self/new parent
@@ -3008,6 +3091,12 @@ protected function processAdditional()
                         $suspectedType = 'return';
                     }
 
+                    if ($this->tokens[$x]['code'] === T_EQUAL) {
+                        // Possible constant declaration, the `T_STRING` name will have been skipped over already.
+                        $suspectedType = 'constant';
+                        break;
+                    }
+
                     break;
                 }//end for
 
@@ -3049,6 +3138,11 @@ protected function processAdditional()
                         break;
                     }
 
+                    if ($suspectedType === 'constant' && $this->tokens[$x]['code'] === T_CONST) {
+                        $confirmed = true;
+                        break;
+                    }
+
                     if ($suspectedType === 'property or parameter'
                         && (isset(Util\Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true
                         || $this->tokens[$x]['code'] === T_VAR

tests/Core/Tokenizer/ArrayKeywordTest.inc

@@ -21,15 +21,21 @@ $var = array(
 );
 
 /* testFunctionDeclarationParamType */
-function foo(array $a) {}
+function typedParam(array $a) {}
 
 /* testFunctionDeclarationReturnType */
-function foo($a) : int|array|null {}
+function returnType($a) : int|array|null {}
 
 class Bar {
     /* testClassConst */
     const ARRAY = [];
 
     /* testClassMethod */
     public function array() {}
+
+    /* testOOConstType */
+    const array /* testTypedOOConstName */ ARRAY = /* testOOConstDefault */ array();
+
+    /* testOOPropertyType */
+    protected array $property;
 }

tests/Core/Tokenizer/ArrayKeywordTest.php

@@ -68,6 +68,9 @@ public static function dataArrayKeyword()
             'nested: inner array'                   => [
                 'testMarker' => '/* testNestedArray */',
             ],
+            'OO constant default value'             => [
+                'testMarker' => '/* testOOConstDefault */',
+            ],
         ];
 
     }//end dataArrayKeyword()
@@ -122,6 +125,12 @@ public static function dataArrayType()
             'function union return type' => [
                 'testMarker' => '/* testFunctionDeclarationReturnType */',
             ],
+            'OO constant type'           => [
+                'testMarker' => '/* testOOConstType */',
+            ],
+            'OO property type'           => [
+                'testMarker' => '/* testOOPropertyType */',
+            ],
         ];
 
     }//end dataArrayType()
@@ -167,13 +176,17 @@ public function testNotArrayKeyword($testMarker, $testContent='array')
     public static function dataNotArrayKeyword()
     {
         return [
-            'class-constant-name' => [
+            'class-constant-name'            => [
                 'testMarker'  => '/* testClassConst */',
                 'testContent' => 'ARRAY',
             ],
-            'class-method-name'   => [
+            'class-method-name'              => [
                 'testMarker' => '/* testClassMethod */',
             ],
+            'class-constant-name-after-type' => [
+                'testMarker'  => '/* testTypedOOConstName */',
+                'testContent' => 'ARRAY',
+            ],
         ];
 
     }//end dataNotArrayKeyword()

tests/Core/Tokenizer/BitwiseOrTest.inc

@@ -9,6 +9,30 @@ $result = $value | $test /* testBitwiseOr2 */ | $another;
 
 class TypeUnion
 {
+    /* testTypeUnionOOConstSimple */
+    public const Foo|Bar SIMPLE = new Foo;
+
+    /* testTypeUnionOOConstReverseModifierOrder */
+    protected final const int|float MODIFIERS_REVERSED /* testBitwiseOrOOConstDefaultValue */ = E_WARNING | E_NOTICE;
+
+    const
+        /* testTypeUnionOOConstMulti1 */
+        array |
+        /* testTypeUnionOOConstMulti2 */
+        Traversable | // phpcs:ignore Stnd.Cat.Sniff
+        false
+        /* testTypeUnionOOConstMulti3 */
+        | null MULTI_UNION = false;
+
+    /* testTypeUnionOOConstNamespaceRelative */
+    final protected const namespace\Sub\NameA|namespace\Sub\NameB NAMESPACE_RELATIVE = new namespace\Sub\NameB;
+
+    /* testTypeUnionOOConstPartiallyQualified */
+    const Partially\Qualified\NameA|Partially\Qualified\NameB PARTIALLY_QUALIFIED = new Partially\Qualified\NameA;
+
+    /* testTypeUnionOOConstFullyQualified */
+    const \Fully\Qualified\NameA|\Fully\Qualified\NameB FULLY_QUALIFIED = new \Fully\Qualified\NameB();
+
     /* testTypeUnionPropertySimple */
     public static Foo|Bar $obj;
 

tests/Core/Tokenizer/BitwiseOrTest.php

@@ -47,6 +47,7 @@ public static function dataBitwiseOr()
         return [
             'in simple assignment 1'                    => ['/* testBitwiseOr1 */'],
             'in simple assignment 2'                    => ['/* testBitwiseOr2 */'],
+            'in OO constant default value'              => ['/* testBitwiseOrOOConstDefaultValue */'],
             'in property default value'                 => ['/* testBitwiseOrPropertyDefaultValue */'],
             'in method parameter default value'         => ['/* testBitwiseOrParamDefaultValue */'],
             'in return statement'                       => ['/* testBitwiseOr3 */'],
@@ -97,6 +98,14 @@ public function testTypeUnion($testMarker)
     public static function dataTypeUnion()
     {
         return [
+            'type for OO constant'                                     => ['/* testTypeUnionOOConstSimple */'],
+            'type for OO constant, reversed modifier order'            => ['/* testTypeUnionOOConstReverseModifierOrder */'],
+            'type for OO constant, first of multi-union'               => ['/* testTypeUnionOOConstMulti1 */'],
+            'type for OO constant, middle of multi-union + comments'   => ['/* testTypeUnionOOConstMulti2 */'],
+            'type for OO constant, last of multi-union'                => ['/* testTypeUnionOOConstMulti3 */'],
+            'type for OO constant, using namespace relative names'     => ['/* testTypeUnionOOConstNamespaceRelative */'],
+            'type for OO constant, using partially qualified names'    => ['/* testTypeUnionOOConstPartiallyQualified */'],
+            'type for OO constant, using fully qualified names'        => ['/* testTypeUnionOOConstFullyQualified */'],
             'type for static property'                                 => ['/* testTypeUnionPropertySimple */'],
             'type for static property, reversed modifier order'        => ['/* testTypeUnionPropertyReverseModifierOrder */'],
             'type for property, first of multi-union'                  => ['/* testTypeUnionPropertyMulti1 */'],

tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc

@@ -76,6 +76,12 @@ class ContextSensitiveKeywords
     const /* testAnd */ AND = 'LOGICAL_AND';
     const /* testOr */ OR = 'LOGICAL_OR';
     const /* testXor */ XOR = 'LOGICAL_XOR';
+
+    const /* testArrayIsTstringInConstType */ array /* testArrayNameForTypedConstant */ ARRAY = /* testArrayIsKeywordInConstDefault */ array();
+    const /* testStaticIsKeywordAsConstType */ static /* testStaticIsNameForTypedConstant */ STATIC = new /* testStaticIsKeywordAsConstDefault */ static;
+
+    const int|bool /* testPrivateNameForUnionTypedConstant */ PRIVATE = 'PRIVATE';
+    const Foo&Bar /* testFinalNameForIntersectionTypedConstant */ FINAL = 'FINAL';
 }
 
 namespace /* testKeywordAfterNamespaceShouldBeString */ class;

tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php

@@ -118,6 +118,12 @@ public static function dataStrings()
             'constant declaration: or'                        => ['/* testOr */'],
             'constant declaration: xor'                       => ['/* testXor */'],
 
+            'constant declaration: array in type'             => ['/* testArrayIsTstringInConstType */'],
+            'constant declaration: array, name after type'    => ['/* testArrayNameForTypedConstant */'],
+            'constant declaration: static, name after type'   => ['/* testStaticIsNameForTypedConstant */'],
+            'constant declaration: private, name after type'  => ['/* testPrivateNameForUnionTypedConstant */'],
+            'constant declaration: final, name after type'    => ['/* testFinalNameForIntersectionTypedConstant */'],
+
             'namespace declaration: class'                    => ['/* testKeywordAfterNamespaceShouldBeString */'],
             'namespace declaration (partial): my'             => ['/* testNamespaceNameIsString1 */'],
             'namespace declaration (partial): class'          => ['/* testNamespaceNameIsString2 */'],
@@ -179,6 +185,19 @@ public static function dataKeywords()
                 'testMarker'        => '/* testNamespaceIsKeyword */',
                 'expectedTokenType' => 'T_NAMESPACE',
             ],
+            'array: default value in const decl'     => [
+                'testMarker'        => '/* testArrayIsKeywordInConstDefault */',
+                'expectedTokenType' => 'T_ARRAY',
+            ],
+            'static: type in constant declaration'   => [
+                'testMarker'        => '/* testStaticIsKeywordAsConstType */',
+                'expectedTokenType' => 'T_STATIC',
+            ],
+            'static: value in constant declaration'  => [
+                'testMarker'        => '/* testStaticIsKeywordAsConstDefault */',
+                'expectedTokenType' => 'T_STATIC',
+            ],
+
             'abstract: class declaration'            => [
                 'testMarker'        => '/* testAbstractIsKeyword */',
                 'expectedTokenType' => 'T_ABSTRACT',

tests/Core/Tokenizer/OtherContextSensitiveKeywordsTest.inc

@@ -51,3 +51,17 @@ function standAloneFalseTrueNullTypesAndMore(
     || $a === /* testNullIsKeywordInComparison */ null
     ) {}
 }
+
+class TypedConstProp {
+    const /* testFalseIsKeywordAsConstType */ false /* testFalseIsNameForTypedConstant */ FALSE = /* testFalseIsKeywordAsConstDefault */ false;
+    const /* testTrueIsKeywordAsConstType */ true /* testTrueIsNameForTypedConstant */ TRUE = /* testTrueIsKeywordAsConstDefault */ true;
+    const /* testNullIsKeywordAsConstType */ null /* testNullIsNameForTypedConstant */ NULL = /* testNullIsKeywordAsConstDefault */ null;
+    const /* testSelfIsKeywordAsConstType */ self /* testSelfIsNameForTypedConstant */ SELF = new /* testSelfIsKeywordAsConstDefault */ self;
+    const /* testParentIsKeywordAsConstType */ parent /* testParentIsNameForTypedConstant */ PARENT = new /* testParentIsKeywordAsConstDefault */ parent;
+
+    public /* testFalseIsKeywordAsPropertyType */ false $false = /* testFalseIsKeywordAsPropertyDefault */ false;
+    protected readonly /* testTrueIsKeywordAsPropertyType */ true $true = /* testTrueIsKeywordAsPropertyDefault */ true;
+    static private /* testNullIsKeywordAsPropertyType */ null $null = /* testNullIsKeywordAsPropertyDefault */ null;
+    var /* testSelfIsKeywordAsPropertyType */ self $self = new /* testSelfIsKeywordAsPropertyDefault */ self;
+    protected /* testParentIsKeywordAsPropertyType */ parent $parent = new /* testParentIsKeywordAsPropertyDefault */ parent;
+}