Added Laravel project

This commit is contained in:
2017-09-17 00:35:10 +02:00
parent a3c19304d5
commit ecf605b8f5
6246 changed files with 682270 additions and 2 deletions

View File

@@ -0,0 +1,7 @@
/.idea
/.php_cs.cache
/composer.lock
/src/autoload.php
/tools
/vendor
/build

View File

@@ -0,0 +1,67 @@
<?php
$finder = Symfony\CS\Finder\DefaultFinder::create()
->files()
->in('src')
->in('tests')
->name('*.php');
return Symfony\CS\Config\Config::create()
->setUsingCache(true)
->level(\Symfony\CS\FixerInterface::NONE_LEVEL)
->fixers(
array(
'align_double_arrow',
'align_equals',
'concat_with_spaces',
'duplicate_semicolon',
'elseif',
'empty_return',
'encoding',
'eof_ending',
'extra_empty_lines',
'function_call_space',
'function_declaration',
'indentation',
'join_function',
'line_after_namespace',
'linefeed',
'list_commas',
'lowercase_constants',
'lowercase_keywords',
'method_argument_space',
'multiple_use',
'namespace_no_leading_whitespace',
'no_blank_lines_after_class_opening',
'no_empty_lines_after_phpdocs',
'parenthesis',
'php_closing_tag',
'phpdoc_indent',
'phpdoc_no_access',
'phpdoc_no_empty_return',
'phpdoc_no_package',
'phpdoc_params',
'phpdoc_scalar',
'phpdoc_separation',
'phpdoc_to_comment',
'phpdoc_trim',
'phpdoc_types',
'phpdoc_var_without_name',
'remove_lines_between_uses',
'return',
'self_accessor',
'short_array_syntax',
'short_tag',
'single_line_after_imports',
'single_quote',
'spaces_before_semicolon',
'spaces_cast',
'ternary_spaces',
'trailing_spaces',
'trim_array_spaces',
'unused_use',
'visibility',
'whitespacy_lines'
)
)
->finder($finder);

View File

@@ -0,0 +1,33 @@
os:
- linux
language: php
before_install:
- wget https://phar.io/releases/phive.phar
- wget https://phar.io/releases/phive.phar.asc
- gpg --keyserver hkps.pool.sks-keyservers.net --recv-keys 0x9B2D5D79
- gpg --verify phive.phar.asc phive.phar
- chmod +x phive.phar
- sudo mv phive.phar /usr/bin/phive
install:
- ant setup
script: ./tools/phpunit
php:
- 5.6
- 7.0
- 7.1
- 7.0snapshot
- 7.1snapshot
- master
matrix:
allow_failures:
- php: master
fast_finish: true
notifications:
email: false

View File

@@ -0,0 +1,30 @@
Tokenizer
Copyright (c) 2017 Arne Blankerts <arne@blankerts.de> and contributors
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of Arne Blankerts nor the names of contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT * NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

View File

@@ -0,0 +1,49 @@
# Tokenizer
A small library for converting tokenized PHP source code into XML.
[![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/theseer/tokenizer/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/theseer/tokenizer/?branch=master)
[![Code Coverage](https://scrutinizer-ci.com/g/theseer/tokenizer/badges/coverage.png?b=master)](https://scrutinizer-ci.com/g/theseer/tokenizer/?branch=master)
[![Build Status](https://scrutinizer-ci.com/g/theseer/tokenizer/badges/build.png?b=master)](https://scrutinizer-ci.com/g/theseer/tokenizer/build-status/master)
## Installation
You can add this library as a local, per-project dependency to your project using [Composer](https://getcomposer.org/):
composer require theseer/tokenizer
If you only need this library during development, for instance to run your project's test suite, then you should add it as a development-time dependency:
composer require --dev theseer/tokenizer
## Usage examples
```php
$tokenizer = new TheSeer\Tokenizer\Tokenizer();
$tokens = $tokenizer->parse(file_get_contents(__DIR__ . '/src/XMLSerializer.php'));
$serializer = new TheSeer\Tokenizer\XMLSerializer();
$xml = $serializer->toXML($tokens);
echo $xml;
```
The generated XML structure looks something like this:
```xml
<?xml version="1.0"?>
<source xmlns="https://github.com/theseer/tokenizer">
<line no="1">
<token name="T_OPEN_TAG">&lt;?php </token>
<token name="T_DECLARE">declare</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_STRING">strict_types</token>
<token name="T_WHITESPACE"> </token>
<token name="T_EQUAL">=</token>
<token name="T_WHITESPACE"> </token>
<token name="T_LNUMBER">1</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_SEMICOLON">;</token>
</line>
</source>
```

View File

@@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<project name="Tokenizer" default="setup">
<target name="setup" depends="clean,install-tools,generate-autoloader"/>
<target name="clean" unless="clean.done" description="Cleanup build artifacts">
<delete dir="${basedir}/tools"/>
<delete dir="${basedir}/vendor"/>
<delete file="${basedir}/src/autoload.php"/>
<property name="clean.done" value="true"/>
</target>
<target name="prepare" unless="prepare.done" depends="clean" description="Prepare for build">
<property name="prepare.done" value="true"/>
</target>
<target name="-tools-installed">
<available file="${basedir}/tools" property="tools-installed" type="dir"/>
</target>
<target name="install-tools" unless="tools-installed" depends="-tools-installed" description="Install tools with Phive">
<exec executable="phive" taskname="phive">
<arg value="install"/>
<arg value="--trust-gpg-keys" />
<arg value="4AA394086372C20A,2A8299CE842DD38C" />
</exec>
</target>
<target name="generate-autoloader" depends="install-tools" description="Generate autoloader using PHPAB">
<exec executable="${basedir}/tools/phpab" taskname="phpab">
<arg value="--output"/>
<arg path="${basedir}/src/autoload.php"/>
<arg path="${basedir}/src"/>
</exec>
</target>
<target name="test" depends="generate-autoloader" description="Run tests">
<exec executable="${basedir}/tools/phpunit" taskname="phpunit"/>
</target>
</project>

View File

@@ -0,0 +1,27 @@
{
"name": "theseer/tokenizer",
"description": "A small library for converting tokenized PHP source code into XML and potentially other formats",
"license": "BSD-3-Clause",
"authors": [
{
"name": "Arne Blankerts",
"email": "arne@blankerts.de",
"role": "Developer"
}
],
"support": {
"issues": "https://github.com/theseer/tokenizer/issues"
},
"require": {
"php": "^7.0",
"ext-xmlwriter": "*",
"ext-dom": "*",
"ext-tokenizer": "*"
},
"autoload": {
"classmap": [
"src/"
]
}
}

View File

@@ -0,0 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<phive xmlns="https://phar.io/phive">
<phar name="phpunit" version="^6.0" installed="6.0.10" location="./tools/phpunit"/>
<phar name="phpab" version="^1.23.0" installed="1.23.0" location="./tools/phpab"/>
</phive>

View File

@@ -0,0 +1,25 @@
<?xml version="1.0" encoding="UTF-8"?>
<phpunit xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:noNamespaceSchemaLocation="https://schema.phpunit.de/5.7/phpunit.xsd"
bootstrap="src/autoload.php"
backupGlobals="false"
beStrictAboutOutputDuringTests="true"
beStrictAboutTestsThatDoNotTestAnything="true"
beStrictAboutTodoAnnotatedTests="true"
verbose="true">
<testsuite>
<directory suffix="Test.php">tests</directory>
</testsuite>
<filter>
<whitelist processUncoveredFilesFromWhitelist="true">
<directory suffix=".php">src</directory>
</whitelist>
</filter>
<logging>
<log type="coverage-html" target="build/coverage/html" title="phpDox"
charset="UTF-8" highlight="true" lowUpperBound="60" highLowerBound="90"/>
</logging>
</phpunit>

View File

@@ -0,0 +1,6 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
class Exception extends \Exception {
}

View File

@@ -0,0 +1,28 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
class NamespaceUri {
/** @var string */
private $value;
/**
* @param string $value
*/
public function __construct(string $value) {
$this->ensureValidUri($value);
$this->value = $value;
}
public function asString(): string {
return $this->value;
}
private function ensureValidUri($value) {
if (strpos($value, ':') === false) {
throw new NamespaceUriException(
sprintf("Namespace URI '%s' must contain at least one colon", $value)
);
}
}
}

View File

@@ -0,0 +1,6 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
class NamespaceUriException extends Exception {
}

View File

@@ -0,0 +1,55 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
class Token {
/**
* @var int
*/
private $line;
/**
* @var string
*/
private $name;
/**
* @var string
*/
private $value;
/**
* Token constructor.
*
* @param int $line
* @param string $name
* @param string $value
*/
public function __construct(int $line, string $name, string $value) {
$this->line = $line;
$this->name = $name;
$this->value = $value;
}
/**
* @return int
*/
public function getLine(): int {
return $this->line;
}
/**
* @return string
*/
public function getName(): string {
return $this->name;
}
/**
* @return string
*/
public function getValue(): string {
return $this->value;
}
}

View File

@@ -0,0 +1,128 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
class TokenCollection implements \ArrayAccess, \Iterator, \Countable {
/**
* @var Token[]
*/
private $tokens = [];
/**
* @var int
*/
private $pos;
/**
* @param Token $token
*/
public function addToken(Token $token) {
$this->tokens[] = $token;
}
/**
* @return Token
*/
public function current(): Token {
return current($this->tokens);
}
/**
* @return int
*/
public function key(): int {
return key($this->tokens);
}
/**
* @return void
*/
public function next() {
next($this->tokens);
$this->pos++;
}
/**
* @return bool
*/
public function valid(): bool {
return $this->count() > $this->pos;
}
/**
* @return void
*/
public function rewind() {
reset($this->tokens);
$this->pos = 0;
}
/**
* @return int
*/
public function count(): int {
return count($this->tokens);
}
/**
* @param mixed $offset
*
* @return bool
*/
public function offsetExists($offset): bool {
return isset($this->tokens[$offset]);
}
/**
* @param mixed $offset
*
* @return Token
* @throws TokenCollectionException
*/
public function offsetGet($offset): Token {
if (!$this->offsetExists($offset)) {
throw new TokenCollectionException(
sprintf('No Token at offest %s', $offset)
);
}
return $this->tokens[$offset];
}
/**
* @param mixed $offset
* @param Token $value
*
* @throws TokenCollectionException
*/
public function offsetSet($offset, $value) {
if (!is_int($offset)) {
$type = gettype($offset);
throw new TokenCollectionException(
sprintf(
'Offset must be of type integer, %s given',
$type === 'object' ? get_class($value) : $type
)
);
}
if (!$value instanceof Token) {
$type = gettype($value);
throw new TokenCollectionException(
sprintf(
'Value must be of type %s, %s given',
Token::class,
$type === 'object' ? get_class($value) : $type
)
);
}
$this->tokens[$offset] = $value;
}
/**
* @param mixed $offset
*/
public function offsetUnset($offset) {
unset($this->tokens[$offset]);
}
}

View File

@@ -0,0 +1,6 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
class TokenCollectionException extends Exception {
}

View File

@@ -0,0 +1,82 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
class Tokenizer {
/**
* Token Map for "non-tokens"
*
* @var array
*/
private $map = [
'(' => 'T_OPEN_BRACKET',
')' => 'T_CLOSE_BRACKET',
'[' => 'T_OPEN_SQUARE',
']' => 'T_CLOSE_SQUARE',
'{' => 'T_OPEN_CURLY',
'}' => 'T_CLOSE_CURLY',
';' => 'T_SEMICOLON',
'.' => 'T_DOT',
',' => 'T_COMMA',
'=' => 'T_EQUAL',
'<' => 'T_LT',
'>' => 'T_GT',
'+' => 'T_PLUS',
'-' => 'T_MINUS',
'*' => 'T_MULT',
'/' => 'T_DIV',
'?' => 'T_QUESTION_MARK',
'!' => 'T_EXCLAMATION_MARK',
':' => 'T_COLON',
'"' => 'T_DOUBLE_QUOTES',
'@' => 'T_AT',
'&' => 'T_AMPERSAND',
'%' => 'T_PERCENT',
'|' => 'T_PIPE',
'$' => 'T_DOLLAR',
'^' => 'T_CARET',
'~' => 'T_TILDE',
'`' => 'T_BACKTICK'
];
public function parse(string $source): TokenCollection {
$result = new TokenCollection();
$tokens = token_get_all($source);
$lastToken = new Token(
$tokens[0][2],
'Placeholder',
''
);
foreach ($tokens as $pos => $tok) {
if (is_string($tok)) {
$token = new Token(
$lastToken->getLine(),
$this->map[$tok],
$tok
);
$result->addToken($token);
$lastToken = $token;
continue;
}
$line = $tok[2];
$values = preg_split('/\R+/Uu', $tok[1]);
foreach ($values as $v) {
$token = new Token(
$line,
token_name($tok[0]),
$v
);
$result->addToken($token);
$line++;
$lastToken = $token;
}
}
return $result;
}
}

View File

@@ -0,0 +1,94 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
use DOMDocument;
class XMLSerializer {
/**
* @var \XMLWriter
*/
private $writer;
/**
* @var Token
*/
private $previousToken;
/**
* @var NamespaceUri
*/
private $xmlns;
/**
* XMLSerializer constructor.
*
* @param NamespaceUri $xmlns
*/
public function __construct(NamespaceUri $xmlns = null) {
if ($xmlns === null) {
$xmlns = new NamespaceUri('https://github.com/theseer/tokenizer');
}
$this->xmlns = $xmlns;
}
/**
* @param TokenCollection $tokens
*
* @return DOMDocument
*/
public function toDom(TokenCollection $tokens): DOMDocument {
$dom = new DOMDocument();
$dom->preserveWhiteSpace = false;
$dom->loadXML($this->toXML($tokens));
return $dom;
}
/**
* @param TokenCollection $tokens
*
* @return string
*/
public function toXML(TokenCollection $tokens): string {
$this->writer = new \XMLWriter();
$this->writer->openMemory();
$this->writer->setIndent(true);
$this->writer->startDocument();
$this->writer->startElement('source');
$this->writer->writeAttribute('xmlns', $this->xmlns->asString());
$this->writer->startElement('line');
$this->writer->writeAttribute('no', '1');
$this->previousToken = $tokens[0];
foreach ($tokens as $token) {
$this->addToken($token);
}
$this->writer->endElement();
$this->writer->endElement();
$this->writer->endDocument();
return $this->writer->outputMemory();
}
/**
* @param Token $token
*/
private function addToken(Token $token) {
if ($this->previousToken->getLine() < $token->getLine()) {
$this->writer->endElement();
$this->writer->startElement('line');
$this->writer->writeAttribute('no', (string)$token->getLine());
$this->previousToken = $token;
}
if ($token->getValue() !== '') {
$this->writer->startElement('token');
$this->writer->writeAttribute('name', $token->getName());
$this->writer->writeRaw(htmlspecialchars($token->getValue(), ENT_NOQUOTES | ENT_DISALLOWED | ENT_XML1));
$this->writer->endElement();
}
}
}

View File

@@ -0,0 +1,29 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
use PHPUnit\Framework\TestCase;
/**
* @covers \TheSeer\Tokenizer\NamespaceUri
*/
class NamespaceUriTest extends TestCase {
public function testCanBeConstructedWithValidNamespace() {
$this->assertInstanceOf(
NamespaceUri::class,
new NamespaceUri('a:b')
);
}
public function testInvalidNamespaceThrowsException() {
$this->expectException(NamespaceUriException::class);
new NamespaceUri('invalid-no-colon');
}
public function testStringRepresentationCanBeRetrieved() {
$this->assertEquals(
'a:b',
(new NamespaceUri('a:b'))->asString()
);
}
}

View File

@@ -0,0 +1,72 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
use PHPUnit\Framework\TestCase;
/**
* @covers \TheSeer\Tokenizer\TokenCollection
*/
class TokenCollectionTest extends TestCase {
/** @var TokenCollection */
private $collection;
protected function setUp() {
$this->collection = new TokenCollection();
}
public function testCollectionIsInitiallyEmpty() {
$this->assertCount(0, $this->collection);
}
public function testTokenCanBeAddedToCollection() {
$token = $this->createMock(Token::class);
$this->collection->addToken($token);
$this->assertCount(1, $this->collection);
$this->assertSame($token, $this->collection[0]);
}
public function testCanIterateOverTokens() {
$token = $this->createMock(Token::class);
$this->collection->addToken($token);
$this->collection->addToken($token);
foreach($this->collection as $position => $current) {
$this->assertInternalType('integer', $position);
$this->assertSame($token, $current);
}
}
public function testOffsetCanBeUnset() {
$token = $this->createMock(Token::class);
$this->collection->addToken($token);
$this->assertCount(1, $this->collection);
unset($this->collection[0]);
$this->assertCount(0, $this->collection);
}
public function testTokenCanBeSetViaOffsetPosition() {
$token = $this->createMock(Token::class);
$this->collection[0] = $token;
$this->assertCount(1, $this->collection);
$this->assertSame($token, $this->collection[0]);
}
public function testTryingToUseNonIntegerOffsetThrowsException() {
$this->expectException(TokenCollectionException::class);
$this->collection['foo'] = $this->createMock(Token::class);
}
public function testTryingToSetNonTokenAtOffsetThrowsException() {
$this->expectException(TokenCollectionException::class);
$this->collection[0] = 'abc';
}
public function testTryingToGetTokenAtNonExistingOffsetThrowsException() {
$this->expectException(TokenCollectionException::class);
$x = $this->collection[3];
}
}

View File

@@ -0,0 +1,31 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
use PHPUnit\Framework\TestCase;
class TokenTest extends TestCase {
/** @var Token */
private $token;
protected function setUp() {
$this->token = new Token(1,'test-dummy', 'blank');
}
public function testTokenCanBeCreated() {
$this->assertInstanceOf(Token::class, $this->token);
}
public function testTokenLineCanBeRetrieved() {
$this->assertEquals(1, $this->token->getLine());
}
public function testTokenNameCanBeRetrieved() {
$this->assertEquals('test-dummy', $this->token->getName());
}
public function testTokenValueCanBeRetrieved() {
$this->assertEquals('blank', $this->token->getValue());
}
}

View File

@@ -0,0 +1,21 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
use PHPUnit\Framework\TestCase;
/**
* @covers \TheSeer\Tokenizer\Tokenizer
*/
class TokenizerTest extends TestCase {
public function testValidSourceGetsParsed() {
$tokenizer = new Tokenizer();
$result = $tokenizer->parse(file_get_contents(__DIR__ . '/_files/test.php'));
$expected = unserialize(
file_get_contents(__DIR__ . '/_files/test.php.tokens'),
[TokenCollection::class]
);
$this->assertEquals($expected, $result);
}
}

View File

@@ -0,0 +1,43 @@
<?php declare(strict_types = 1);
namespace TheSeer\Tokenizer;
use PHPUnit\Framework\TestCase;
/**
* @covers \TheSeer\Tokenizer\XMLSerializer
*/
class XMLSerializerTest extends TestCase {
/** @var TokenCollection $tokens */
private $tokens;
protected function setUp() {
$this->tokens = unserialize(
file_get_contents(__DIR__ . '/_files/test.php.tokens'),
[TokenCollection::class]
);
}
public function testCanBeSerializedToXml() {
$expected = file_get_contents(__DIR__ . '/_files/test.php.xml');
$serializer = new XMLSerializer();
$this->assertEquals($expected, $serializer->toXML($this->tokens));
}
public function testCanBeSerializedToDomDocument() {
$serializer = new XMLSerializer();
$result = $serializer->toDom($this->tokens);
$this->assertInstanceOf(\DOMDocument::class, $result);
$this->assertEquals('source', $result->documentElement->localName);
}
public function testCanBeSerializedToXmlWithCustomNamespace() {
$expected = file_get_contents(__DIR__ . '/_files/customns.xml');
$serializer = new XMLSerializer(new NamespaceUri('custom:xml:namespace'));
$this->assertEquals($expected, $serializer->toXML($this->tokens));
}
}

View File

@@ -0,0 +1,177 @@
<?xml version="1.0"?>
<source xmlns="custom:xml:namespace">
<line no="1">
<token name="T_OPEN_TAG">&lt;?php </token>
<token name="T_DECLARE">declare</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_STRING">strict_types</token>
<token name="T_WHITESPACE"> </token>
<token name="T_EQUAL">=</token>
<token name="T_WHITESPACE"> </token>
<token name="T_LNUMBER">1</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="2">
<token name="T_NAMESPACE">namespace</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">foo</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="3"/>
<line no="4">
<token name="T_CLASS">class</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">bar</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="5">
<token name="T_WHITESPACE"> </token>
<token name="T_CONST">const</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">x</token>
<token name="T_WHITESPACE"> </token>
<token name="T_EQUAL">=</token>
<token name="T_WHITESPACE"> </token>
<token name="T_CONSTANT_ENCAPSED_STRING">'abc'</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="6"/>
<line no="7">
<token name="T_WHITESPACE"> </token>
<token name="T_DOC_COMMENT">/** @var int */</token>
</line>
<line no="8">
<token name="T_WHITESPACE"> </token>
<token name="T_PRIVATE">private</token>
<token name="T_WHITESPACE"> </token>
<token name="T_VARIABLE">$y</token>
<token name="T_WHITESPACE"> </token>
<token name="T_EQUAL">=</token>
<token name="T_WHITESPACE"> </token>
<token name="T_LNUMBER">1</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="9"/>
<line no="10">
<token name="T_WHITESPACE"> </token>
<token name="T_PUBLIC">public</token>
<token name="T_WHITESPACE"> </token>
<token name="T_FUNCTION">function</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">__construct</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="11">
<token name="T_WHITESPACE"> </token>
<token name="T_COMMENT">// do something</token>
</line>
<line no="12">
<token name="T_WHITESPACE"> </token>
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="13"/>
<line no="14">
<token name="T_WHITESPACE"> </token>
<token name="T_PUBLIC">public</token>
<token name="T_WHITESPACE"> </token>
<token name="T_FUNCTION">function</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">getY</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_COLON">:</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">int</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="15">
<token name="T_WHITESPACE"> </token>
<token name="T_RETURN">return</token>
<token name="T_WHITESPACE"> </token>
<token name="T_VARIABLE">$this</token>
<token name="T_OBJECT_OPERATOR">-&gt;</token>
<token name="T_STRING">y</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="16">
<token name="T_WHITESPACE"> </token>
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="17"/>
<line no="18">
<token name="T_WHITESPACE"> </token>
<token name="T_PUBLIC">public</token>
<token name="T_WHITESPACE"> </token>
<token name="T_FUNCTION">function</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">getSomeX</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_COLON">:</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">string</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="19">
<token name="T_WHITESPACE"> </token>
<token name="T_RETURN">return</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">self</token>
<token name="T_DOUBLE_COLON">::</token>
<token name="T_STRING">x</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="20">
<token name="T_WHITESPACE"> </token>
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="21"/>
<line no="22">
<token name="T_WHITESPACE"> </token>
<token name="T_PUBLIC">public</token>
<token name="T_WHITESPACE"> </token>
<token name="T_FUNCTION">function</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">some</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_STRING">bar</token>
<token name="T_WHITESPACE"> </token>
<token name="T_VARIABLE">$b</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_COLON">:</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">string</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="23">
<token name="T_WHITESPACE"> </token>
<token name="T_RETURN">return</token>
<token name="T_WHITESPACE"> </token>
<token name="T_VARIABLE">$b</token>
<token name="T_OBJECT_OPERATOR">-&gt;</token>
<token name="T_STRING">getSomeX</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_WHITESPACE"> </token>
<token name="T_DOT">.</token>
<token name="T_WHITESPACE"> </token>
<token name="T_CONSTANT_ENCAPSED_STRING">'-def'</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="24">
<token name="T_WHITESPACE"> </token>
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="25">
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="26"/>
</source>

View File

@@ -0,0 +1,25 @@
<?php declare(strict_types = 1);
namespace foo;
class bar {
const x = 'abc';
/** @var int */
private $y = 1;
public function __construct() {
// do something
}
public function getY(): int {
return $this->y;
}
public function getSomeX(): string {
return self::x;
}
public function some(bar $b): string {
return $b->getSomeX() . '-def';
}
}

Binary file not shown.

View File

@@ -0,0 +1,177 @@
<?xml version="1.0"?>
<source xmlns="https://github.com/theseer/tokenizer">
<line no="1">
<token name="T_OPEN_TAG">&lt;?php </token>
<token name="T_DECLARE">declare</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_STRING">strict_types</token>
<token name="T_WHITESPACE"> </token>
<token name="T_EQUAL">=</token>
<token name="T_WHITESPACE"> </token>
<token name="T_LNUMBER">1</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="2">
<token name="T_NAMESPACE">namespace</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">foo</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="3"/>
<line no="4">
<token name="T_CLASS">class</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">bar</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="5">
<token name="T_WHITESPACE"> </token>
<token name="T_CONST">const</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">x</token>
<token name="T_WHITESPACE"> </token>
<token name="T_EQUAL">=</token>
<token name="T_WHITESPACE"> </token>
<token name="T_CONSTANT_ENCAPSED_STRING">'abc'</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="6"/>
<line no="7">
<token name="T_WHITESPACE"> </token>
<token name="T_DOC_COMMENT">/** @var int */</token>
</line>
<line no="8">
<token name="T_WHITESPACE"> </token>
<token name="T_PRIVATE">private</token>
<token name="T_WHITESPACE"> </token>
<token name="T_VARIABLE">$y</token>
<token name="T_WHITESPACE"> </token>
<token name="T_EQUAL">=</token>
<token name="T_WHITESPACE"> </token>
<token name="T_LNUMBER">1</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="9"/>
<line no="10">
<token name="T_WHITESPACE"> </token>
<token name="T_PUBLIC">public</token>
<token name="T_WHITESPACE"> </token>
<token name="T_FUNCTION">function</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">__construct</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="11">
<token name="T_WHITESPACE"> </token>
<token name="T_COMMENT">// do something</token>
</line>
<line no="12">
<token name="T_WHITESPACE"> </token>
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="13"/>
<line no="14">
<token name="T_WHITESPACE"> </token>
<token name="T_PUBLIC">public</token>
<token name="T_WHITESPACE"> </token>
<token name="T_FUNCTION">function</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">getY</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_COLON">:</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">int</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="15">
<token name="T_WHITESPACE"> </token>
<token name="T_RETURN">return</token>
<token name="T_WHITESPACE"> </token>
<token name="T_VARIABLE">$this</token>
<token name="T_OBJECT_OPERATOR">-&gt;</token>
<token name="T_STRING">y</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="16">
<token name="T_WHITESPACE"> </token>
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="17"/>
<line no="18">
<token name="T_WHITESPACE"> </token>
<token name="T_PUBLIC">public</token>
<token name="T_WHITESPACE"> </token>
<token name="T_FUNCTION">function</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">getSomeX</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_COLON">:</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">string</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="19">
<token name="T_WHITESPACE"> </token>
<token name="T_RETURN">return</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">self</token>
<token name="T_DOUBLE_COLON">::</token>
<token name="T_STRING">x</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="20">
<token name="T_WHITESPACE"> </token>
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="21"/>
<line no="22">
<token name="T_WHITESPACE"> </token>
<token name="T_PUBLIC">public</token>
<token name="T_WHITESPACE"> </token>
<token name="T_FUNCTION">function</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">some</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_STRING">bar</token>
<token name="T_WHITESPACE"> </token>
<token name="T_VARIABLE">$b</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_COLON">:</token>
<token name="T_WHITESPACE"> </token>
<token name="T_STRING">string</token>
<token name="T_WHITESPACE"> </token>
<token name="T_OPEN_CURLY">{</token>
</line>
<line no="23">
<token name="T_WHITESPACE"> </token>
<token name="T_RETURN">return</token>
<token name="T_WHITESPACE"> </token>
<token name="T_VARIABLE">$b</token>
<token name="T_OBJECT_OPERATOR">-&gt;</token>
<token name="T_STRING">getSomeX</token>
<token name="T_OPEN_BRACKET">(</token>
<token name="T_CLOSE_BRACKET">)</token>
<token name="T_WHITESPACE"> </token>
<token name="T_DOT">.</token>
<token name="T_WHITESPACE"> </token>
<token name="T_CONSTANT_ENCAPSED_STRING">'-def'</token>
<token name="T_SEMICOLON">;</token>
</line>
<line no="24">
<token name="T_WHITESPACE"> </token>
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="25">
<token name="T_CLOSE_CURLY">}</token>
</line>
<line no="26"/>
</source>