abstract class Lexer

DocLexer.php in drupal/core/vendor/doctrine/common/lib/Doctrine/Common/Annotations/DocLexer.php

abstract class Lexer {

  /**
   * @var array Array of scanned tokens
   */
  private $tokens = array();

  /**
   * @var integer Current lexer position in input string
   */
  private $position = 0;

  /**
   * @var integer Current peek of current lexer position
   */
  private $peek = 0;

  /**
   * @var array The next token in the input.
   */
  public $lookahead;

  /**
   * @var array The last matched/seen token.
   */
  public $token;

  /**
   * Sets the input data to be tokenized.
   *
   * The Lexer is immediately reset and the new input tokenized.
   * Any unprocessed tokens from any previous input are lost.
   *
   * @param string $input The input to be tokenized.
   */
  public function setInput($input) {
    $this->tokens = array();
    $this
      ->reset();
    $this
      ->scan($input);
  }

  /**
   * Resets the lexer.
   */
  public function reset() {
    $this->lookahead = null;
    $this->token = null;
    $this->peek = 0;
    $this->position = 0;
  }

  /**
   * Resets the peek pointer to 0.
   */
  public function resetPeek() {
    $this->peek = 0;
  }

  /**
   * Resets the lexer position on the input to the given position.
   *
   * @param integer $position Position to place the lexical scanner
   */
  public function resetPosition($position = 0) {
    $this->position = $position;
  }

  /**
   * Checks whether a given token matches the current lookahead.
   *
   * @param integer|string $token
   * @return boolean
   */
  public function isNextToken($token) {
    return null !== $this->lookahead && $this->lookahead['type'] === $token;
  }

  /**
   * Checks whether any of the given tokens matches the current lookahead
   *
   * @param array $tokens
   * @return boolean
   */
  public function isNextTokenAny(array $tokens) {
    return null !== $this->lookahead && in_array($this->lookahead['type'], $tokens, true);
  }

  /**
   * Moves to the next token in the input string.
   *
   * A token is an associative array containing three items:
   *  - 'value'    : the string value of the token in the input string
   *  - 'type'     : the type of the token (identifier, numeric, string, input
   *                 parameter, none)
   *  - 'position' : the position of the token in the input string
   *
   * @return array|null the next token; null if there is no more tokens left
   */
  public function moveNext() {
    $this->peek = 0;
    $this->token = $this->lookahead;
    $this->lookahead = isset($this->tokens[$this->position]) ? $this->tokens[$this->position++] : null;
    return $this->lookahead !== null;
  }

  /**
   * Tells the lexer to skip input tokens until it sees a token with the given value.
   *
   * @param string $type The token type to skip until.
   */
  public function skipUntil($type) {
    while ($this->lookahead !== null && $this->lookahead['type'] !== $type) {
      $this
        ->moveNext();
    }
  }

  /**
   * Checks if given value is identical to the given token
   *
   * @param mixed $value
   * @param integer $token
   * @return boolean
   */
  public function isA($value, $token) {
    return $this
      ->getType($value) === $token;
  }

  /**
   * Moves the lookahead token forward.
   *
   * @return array | null The next token or NULL if there are no more tokens ahead.
   */
  public function peek() {
    if (isset($this->tokens[$this->position + $this->peek])) {
      return $this->tokens[$this->position + $this->peek++];
    }
    else {
      return null;
    }
  }

  /**
   * Peeks at the next token, returns it and immediately resets the peek.
   *
   * @return array|null The next token or NULL if there are no more tokens ahead.
   */
  public function glimpse() {
    $peek = $this
      ->peek();
    $this->peek = 0;
    return $peek;
  }

  /**
   * Scans the input string for tokens.
   *
   * @param string $input a query string
   */
  protected function scan($input) {
    static $regex;
    if (!isset($regex)) {
      $regex = '/(' . implode(')|(', $this
        ->getCatchablePatterns()) . ')|' . implode('|', $this
        ->getNonCatchablePatterns()) . '/i';
    }
    $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_OFFSET_CAPTURE;
    $matches = preg_split($regex, $input, -1, $flags);
    foreach ($matches as $match) {

      // Must remain before 'value' assignment since it can change content
      $type = $this
        ->getType($match[0]);
      $this->tokens[] = array(
        'value' => $match[0],
        'type' => $type,
        'position' => $match[1],
      );
    }
  }

  /**
   * Gets the literal for a given token.
   *
   * @param integer $token
   * @return string
   */
  public function getLiteral($token) {
    $className = get_class($this);
    $reflClass = new \ReflectionClass($className);
    $constants = $reflClass
      ->getConstants();
    foreach ($constants as $name => $value) {
      if ($value === $token) {
        return $className . '::' . $name;
      }
    }
    return $token;
  }

  /**
   * Lexical catchable patterns.
   *
   * @return array
   */
  protected abstract function getCatchablePatterns();

  /**
   * Lexical non-catchable patterns.
   *
   * @return array
   */
  protected abstract function getNonCatchablePatterns();

  /**
   * Retrieve token type. Also processes the token value if necessary.
   *
   * @param string $value
   * @return integer
   */
  protected abstract function getType(&$value);

}