Skip to content

Commit

Permalink
Merge pull request #321 from PHPCSStandards/php-8.3/tokenizer-php-all…
Browse files Browse the repository at this point in the history 
…ow-for-typed-constants

PHP 8.3 | Tokenizer/PHP: add support for typed OO constants
  • Loading branch information
@jrfnl
jrfnl committed Feb 8, 2024
2 parents 10249a2 + 53dd20c commit de3b745
Show file tree
Hide file tree
Showing 13 changed files with 963 additions and 8 deletions.
102 changes: 98 additions & 4 deletions src/Tokenizers/PHP.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -526,8 +526,9 @@ protected function tokenize($string)
$numTokens = count($tokens);
$lastNotEmptyToken = 0;

$insideInlineIf = [];
$insideUseGroup = false;
$insideInlineIf = [];
$insideUseGroup = false;
$insideConstDeclaration = false;

$commentTokenizer = new Comment();

Expand Down Expand Up @@ -608,7 +609,8 @@ protected function tokenize($string)
if ($tokenIsArray === true
&& isset(Util\Tokens::$contextSensitiveKeywords[$token[0]]) === true
&& (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
|| $finalTokens[$lastNotEmptyToken]['content'] === '&')
|| $finalTokens[$lastNotEmptyToken]['content'] === '&'
|| $insideConstDeclaration === true)
) {
if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
$preserveKeyword = false;
Expand Down Expand Up @@ -665,6 +667,30 @@ protected function tokenize($string)
}
}//end if

// Types in typed constants should not be touched, but the constant name should be.
if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
&& $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
|| $insideConstDeclaration === true
) {
$preserveKeyword = true;

// Find the next non-empty token.
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
if (is_array($tokens[$i]) === true
&& isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
) {
continue;
}

break;
}

if ($tokens[$i] === '=' || $tokens[$i] === ';') {
$preserveKeyword = false;
$insideConstDeclaration = false;
}
}//end if

if ($finalTokens[$lastNotEmptyToken]['content'] === '&') {
$preserveKeyword = true;

Expand Down Expand Up @@ -698,6 +724,26 @@ protected function tokenize($string)
}
}//end if

/*
Mark the start of a constant declaration to allow for handling keyword to T_STRING
convertion for constant names using reserved keywords.
*/

if ($tokenIsArray === true && $token[0] === T_CONST) {
$insideConstDeclaration = true;
}

/*
Close an open "inside constant declaration" marker when no keyword convertion was needed.
*/

if ($insideConstDeclaration === true
&& $tokenIsArray === false
&& ($token[0] === '=' || $token[0] === ';')
) {
$insideConstDeclaration = false;
}

/*
Special case for `static` used as a function name, i.e. `static()`.
*/
Expand Down Expand Up @@ -1869,6 +1915,20 @@ protected function tokenize($string)
$newToken = [];
$newToken['content'] = '?';

// For typed constants, we only need to check the token before the ? to be sure.
if ($finalTokens[$lastNotEmptyToken]['code'] === T_CONST) {
$newToken['code'] = T_NULLABLE;
$newToken['type'] = 'T_NULLABLE';

if (PHP_CODESNIFFER_VERBOSITY > 1) {
echo "\t\t* token $stackPtr changed from ? to T_NULLABLE".PHP_EOL;
}

$finalTokens[$newStackPtr] = $newToken;
$newStackPtr++;
continue;
}

/*
* Check if the next non-empty token is one of the tokens which can be used
* in type declarations. If not, it's definitely a ternary.
Expand Down Expand Up @@ -2236,7 +2296,30 @@ function return types. We want to keep the parenthesis map clean,
if ($tokenIsArray === true && $token[0] === T_STRING) {
$preserveTstring = false;

if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true) {
// True/false/parent/self/static in typed constants should be fixed to their own token,
// but the constant name should not be.
if ((isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
&& $finalTokens[$lastNotEmptyToken]['code'] === T_CONST)
|| $insideConstDeclaration === true
) {
// Find the next non-empty token.
for ($i = ($stackPtr + 1); $i < $numTokens; $i++) {
if (is_array($tokens[$i]) === true
&& isset(Util\Tokens::$emptyTokens[$tokens[$i][0]]) === true
) {
continue;
}

break;
}

if ($tokens[$i] === '=') {
$preserveTstring = true;
$insideConstDeclaration = false;
}
} else if (isset($this->tstringContexts[$finalTokens[$lastNotEmptyToken]['code']]) === true
&& $finalTokens[$lastNotEmptyToken]['code'] !== T_CONST
) {
$preserveTstring = true;

// Special case for syntax like: return new self/new parent
Expand Down Expand Up @@ -3008,6 +3091,12 @@ protected function processAdditional()
$suspectedType = 'return';
}

if ($this->tokens[$x]['code'] === T_EQUAL) {
// Possible constant declaration, the `T_STRING` name will have been skipped over already.
$suspectedType = 'constant';
break;
}

break;
}//end for

Expand Down Expand Up @@ -3049,6 +3138,11 @@ protected function processAdditional()
break;
}

if ($suspectedType === 'constant' && $this->tokens[$x]['code'] === T_CONST) {
$confirmed = true;
break;
}

if ($suspectedType === 'property or parameter'
&& (isset(Util\Tokens::$scopeModifiers[$this->tokens[$x]['code']]) === true
|| $this->tokens[$x]['code'] === T_VAR
Expand Down
10 changes: 8 additions & 2 deletions tests/Core/Tokenizer/ArrayKeywordTest.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,21 @@ $var = array(
);

/* testFunctionDeclarationParamType */
function foo(array $a) {}
function typedParam(array $a) {}

/* testFunctionDeclarationReturnType */
function foo($a) : int|array|null {}
function returnType($a) : int|array|null {}

class Bar {
/* testClassConst */
const ARRAY = [];

/* testClassMethod */
public function array() {}

/* testOOConstType */
const array /* testTypedOOConstName */ ARRAY = /* testOOConstDefault */ array();

/* testOOPropertyType */
protected array $property;
}
17 changes: 15 additions & 2 deletions tests/Core/Tokenizer/ArrayKeywordTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ public static function dataArrayKeyword()
'nested: inner array' => [
'testMarker' => '/* testNestedArray */',
],
'OO constant default value' => [
'testMarker' => '/* testOOConstDefault */',
],
];

}//end dataArrayKeyword()
Expand Down Expand Up @@ -122,6 +125,12 @@ public static function dataArrayType()
'function union return type' => [
'testMarker' => '/* testFunctionDeclarationReturnType */',
],
'OO constant type' => [
'testMarker' => '/* testOOConstType */',
],
'OO property type' => [
'testMarker' => '/* testOOPropertyType */',
],
];

}//end dataArrayType()
Expand Down Expand Up @@ -167,13 +176,17 @@ public function testNotArrayKeyword($testMarker, $testContent='array')
public static function dataNotArrayKeyword()
{
return [
'class-constant-name' => [
'class-constant-name' => [
'testMarker' => '/* testClassConst */',
'testContent' => 'ARRAY',
],
'class-method-name' => [
'class-method-name' => [
'testMarker' => '/* testClassMethod */',
],
'class-constant-name-after-type' => [
'testMarker' => '/* testTypedOOConstName */',
'testContent' => 'ARRAY',
],
];

}//end dataNotArrayKeyword()
Expand Down
24 changes: 24 additions & 0 deletions tests/Core/Tokenizer/BitwiseOrTest.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,30 @@ $result = $value | $test /* testBitwiseOr2 */ | $another;

class TypeUnion
{
/* testTypeUnionOOConstSimple */
public const Foo|Bar SIMPLE = new Foo;

/* testTypeUnionOOConstReverseModifierOrder */
protected final const int|float MODIFIERS_REVERSED /* testBitwiseOrOOConstDefaultValue */ = E_WARNING | E_NOTICE;

const
/* testTypeUnionOOConstMulti1 */
array |
/* testTypeUnionOOConstMulti2 */
Traversable | // phpcs:ignore Stnd.Cat.Sniff
false
/* testTypeUnionOOConstMulti3 */
| null MULTI_UNION = false;

/* testTypeUnionOOConstNamespaceRelative */
final protected const namespace\Sub\NameA|namespace\Sub\NameB NAMESPACE_RELATIVE = new namespace\Sub\NameB;
/* testTypeUnionOOConstPartiallyQualified */
const Partially\Qualified\NameA|Partially\Qualified\NameB PARTIALLY_QUALIFIED = new Partially\Qualified\NameA;
/* testTypeUnionOOConstFullyQualified */
const \Fully\Qualified\NameA|\Fully\Qualified\NameB FULLY_QUALIFIED = new \Fully\Qualified\NameB();

/* testTypeUnionPropertySimple */
public static Foo|Bar $obj;

Expand Down
9 changes: 9 additions & 0 deletions tests/Core/Tokenizer/BitwiseOrTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public static function dataBitwiseOr()
return [
'in simple assignment 1' => ['/* testBitwiseOr1 */'],
'in simple assignment 2' => ['/* testBitwiseOr2 */'],
'in OO constant default value' => ['/* testBitwiseOrOOConstDefaultValue */'],
'in property default value' => ['/* testBitwiseOrPropertyDefaultValue */'],
'in method parameter default value' => ['/* testBitwiseOrParamDefaultValue */'],
'in return statement' => ['/* testBitwiseOr3 */'],
Expand Down Expand Up @@ -97,6 +98,14 @@ public function testTypeUnion($testMarker)
public static function dataTypeUnion()
{
return [
'type for OO constant' => ['/* testTypeUnionOOConstSimple */'],
'type for OO constant, reversed modifier order' => ['/* testTypeUnionOOConstReverseModifierOrder */'],
'type for OO constant, first of multi-union' => ['/* testTypeUnionOOConstMulti1 */'],
'type for OO constant, middle of multi-union + comments' => ['/* testTypeUnionOOConstMulti2 */'],
'type for OO constant, last of multi-union' => ['/* testTypeUnionOOConstMulti3 */'],
'type for OO constant, using namespace relative names' => ['/* testTypeUnionOOConstNamespaceRelative */'],
'type for OO constant, using partially qualified names' => ['/* testTypeUnionOOConstPartiallyQualified */'],
'type for OO constant, using fully qualified names' => ['/* testTypeUnionOOConstFullyQualified */'],
'type for static property' => ['/* testTypeUnionPropertySimple */'],
'type for static property, reversed modifier order' => ['/* testTypeUnionPropertyReverseModifierOrder */'],
'type for property, first of multi-union' => ['/* testTypeUnionPropertyMulti1 */'],
Expand Down
6 changes: 6 additions & 0 deletions tests/Core/Tokenizer/ContextSensitiveKeywordsTest.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ class ContextSensitiveKeywords
const /* testAnd */ AND = 'LOGICAL_AND';
const /* testOr */ OR = 'LOGICAL_OR';
const /* testXor */ XOR = 'LOGICAL_XOR';

const /* testArrayIsTstringInConstType */ array /* testArrayNameForTypedConstant */ ARRAY = /* testArrayIsKeywordInConstDefault */ array();
const /* testStaticIsKeywordAsConstType */ static /* testStaticIsNameForTypedConstant */ STATIC = new /* testStaticIsKeywordAsConstDefault */ static;

const int|bool /* testPrivateNameForUnionTypedConstant */ PRIVATE = 'PRIVATE';
const Foo&Bar /* testFinalNameForIntersectionTypedConstant */ FINAL = 'FINAL';
}

namespace /* testKeywordAfterNamespaceShouldBeString */ class;
Expand Down
19 changes: 19 additions & 0 deletions tests/Core/Tokenizer/ContextSensitiveKeywordsTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,12 @@ public static function dataStrings()
'constant declaration: or' => ['/* testOr */'],
'constant declaration: xor' => ['/* testXor */'],

'constant declaration: array in type' => ['/* testArrayIsTstringInConstType */'],
'constant declaration: array, name after type' => ['/* testArrayNameForTypedConstant */'],
'constant declaration: static, name after type' => ['/* testStaticIsNameForTypedConstant */'],
'constant declaration: private, name after type' => ['/* testPrivateNameForUnionTypedConstant */'],
'constant declaration: final, name after type' => ['/* testFinalNameForIntersectionTypedConstant */'],

'namespace declaration: class' => ['/* testKeywordAfterNamespaceShouldBeString */'],
'namespace declaration (partial): my' => ['/* testNamespaceNameIsString1 */'],
'namespace declaration (partial): class' => ['/* testNamespaceNameIsString2 */'],
Expand Down Expand Up @@ -179,6 +185,19 @@ public static function dataKeywords()
'testMarker' => '/* testNamespaceIsKeyword */',
'expectedTokenType' => 'T_NAMESPACE',
],
'array: default value in const decl' => [
'testMarker' => '/* testArrayIsKeywordInConstDefault */',
'expectedTokenType' => 'T_ARRAY',
],
'static: type in constant declaration' => [
'testMarker' => '/* testStaticIsKeywordAsConstType */',
'expectedTokenType' => 'T_STATIC',
],
'static: value in constant declaration' => [
'testMarker' => '/* testStaticIsKeywordAsConstDefault */',
'expectedTokenType' => 'T_STATIC',
],

'abstract: class declaration' => [
'testMarker' => '/* testAbstractIsKeyword */',
'expectedTokenType' => 'T_ABSTRACT',
Expand Down
14 changes: 14 additions & 0 deletions tests/Core/Tokenizer/OtherContextSensitiveKeywordsTest.inc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,17 @@ function standAloneFalseTrueNullTypesAndMore(
|| $a === /* testNullIsKeywordInComparison */ null
) {}
}

class TypedConstProp {
const /* testFalseIsKeywordAsConstType */ false /* testFalseIsNameForTypedConstant */ FALSE = /* testFalseIsKeywordAsConstDefault */ false;
const /* testTrueIsKeywordAsConstType */ true /* testTrueIsNameForTypedConstant */ TRUE = /* testTrueIsKeywordAsConstDefault */ true;
const /* testNullIsKeywordAsConstType */ null /* testNullIsNameForTypedConstant */ NULL = /* testNullIsKeywordAsConstDefault */ null;
const /* testSelfIsKeywordAsConstType */ self /* testSelfIsNameForTypedConstant */ SELF = new /* testSelfIsKeywordAsConstDefault */ self;
const /* testParentIsKeywordAsConstType */ parent /* testParentIsNameForTypedConstant */ PARENT = new /* testParentIsKeywordAsConstDefault */ parent;

public /* testFalseIsKeywordAsPropertyType */ false $false = /* testFalseIsKeywordAsPropertyDefault */ false;
protected readonly /* testTrueIsKeywordAsPropertyType */ true $true = /* testTrueIsKeywordAsPropertyDefault */ true;
static private /* testNullIsKeywordAsPropertyType */ null $null = /* testNullIsKeywordAsPropertyDefault */ null;
var /* testSelfIsKeywordAsPropertyType */ self $self = new /* testSelfIsKeywordAsPropertyDefault */ self;
protected /* testParentIsKeywordAsPropertyType */ parent $parent = new /* testParentIsKeywordAsPropertyDefault */ parent;
}

0 comments on commit de3b745

Please sign in to comment.