search.extender.inc

<?php

/**
 * @file
 * Search query extender and helper functions.
 */

/**
 * Do a query on the full-text search index for a word or words.
 *
 * This function is normally only called by each module that supports the
 * indexed search (and thus, implements hook_update_index()).
 *
 * Results are retrieved in two logical passes. However, the two passes are
 * joined together into a single query. And in the case of most simple
 * queries the second pass is not even used.
 *
 * The first pass selects a set of all possible matches, which has the benefit
 * of also providing the exact result set for simple "AND" or "OR" searches.
 *
 * The second portion of the query further refines this set by verifying
 * advanced text conditions (such as negative or phrase matches).
 *
 * The used query object has the tag 'search_$module' and can be further
 * extended with hook_query_alter().
 */
class SearchQuery extends SelectQueryExtender {

  /**
   * The search query that is used for searching.
   *
   * @var string
   */
  protected $searchExpression;

  /**
   * Type of search (search module).
   *
   * This maps to the value of the type column in search_index, and is equal
   * to the machine-readable name of the module that implements
   * hook_search_info().
   *
   * @var string
   */
  protected $type;

  /**
   * Positive and negative search keys.
   *
   * @var array
   */
  protected $keys = array(
    'positive' => array(),
    'negative' => array(),
  );

  /**
   * Indicates whether the first pass query requires complex conditions (LIKE).
   *
   * @var boolean.
   */
  protected $simple = TRUE;

  /**
   * Conditions that are used for exact searches.
   *
   * This is always used for the second pass query but not for the first pass,
   * unless $this->simple is FALSE.
   *
   * @var DatabaseCondition
   */
  protected $conditions;

  /**
   * Indicates how many matches for a search query are necessary.
   *
   * @var int
   */
  protected $matches = 0;

  /**
   * Array of search words.
   *
   * These words have to match against {search_index}.word.
   *
   * @var array
   */
  protected $words = array();

  /**
   * Multiplier for the normalized search score.
   *
   * This value is calculated by the first pass query and multiplied with the
   * actual score of a specific word to make sure that the resulting calculated
   * score is between 0 and 1.
   *
   * @var float
   */
  protected $normalize;

  /**
   * Indicates whether the first pass query has been executed.
   *
   * @var boolean
   */
  protected $executedFirstPass = FALSE;

  /**
   * Stores score expressions.
   *
   * @var array
   *
   * @see addScore()
   */
  protected $scores = array();

  /**
   * Stores arguments for score expressions.
   *
   * @var array
   */
  protected $scoresArguments = array();

  /**
   * Stores multipliers for score expressions.
   *
   * @var array
   */
  protected $multiply = array();

  /**
   * Whether or not search expressions were ignored.
   *
   * The maximum number of AND/OR combinations exceeded can be configured to
   * avoid Denial-of-Service attacks. Expressions beyond the limit are ignored.
   *
   * @var boolean
   */
  protected $expressionsIgnored = FALSE;

  /**
   * Sets up the search query expression.
   *
   * @param $query
   *   A search query string, which can contain options.
   * @param $module
   *   The search module. This maps to {search_index}.type in the database.
   *
   * @return
   *   The SearchQuery object.
   */
  public function searchExpression($expression, $module) {
    $this->searchExpression = $expression;
    $this->type = $module;

    // Add a search_* tag. This needs to be added before any preExecute methods
    // for decorated queries are called, as $this->prepared will be set to TRUE
    // and tags added in the execute method will never get used. For example,
    // if $query is extended by 'SearchQuery' then 'PagerDefault', the
    // search-specific tag will be added too late (when preExecute() has
    // already been called from the PagerDefault extender), and as a
    // consequence will not be available to hook_query_alter() implementations,
    // nor will the correct hook_query_TAG_alter() implementations get invoked.
    // See node_search_execute().
    $this
      ->addTag('search_' . $module);
    return $this;
  }

  /**
   * Applies a search option and removes it from the search query string.
   *
   * These options are in the form option:value,value2,value3.
   *
   * @param $option
   *   Name of the option.
   * @param $column
   *   Name of the database column to which the value should be applied.
   *
   * @return
   *   TRUE if a value for that option was found, FALSE if not.
   */
  public function setOption($option, $column) {
    if ($values = search_expression_extract($this->searchExpression, $option)) {
      $or = db_or();
      foreach (explode(',', $values) as $value) {
        $or
          ->condition($column, $value);
      }
      $this
        ->condition($or);
      $this->searchExpression = search_expression_insert($this->searchExpression, $option);
      return TRUE;
    }
    return FALSE;
  }

  /**
   * Parses the search query into SQL conditions.
   *
   * We build two queries that match the dataset bodies.
   */
  protected function parseSearchExpression() {

    // Matchs words optionally prefixed by a dash. A word in this case is
    // something between two spaces, optionally quoted.
    preg_match_all('/ (-?)("[^"]+"|[^" ]+)/i', ' ' . $this->searchExpression, $keywords, PREG_SET_ORDER);
    if (count($keywords) == 0) {
      return;
    }

    // Classify tokens.
    $or = FALSE;
    $warning = '';
    $limit_combinations = variable_get('search_and_or_limit', 7);

    // The first search expression does not count as AND.
    $and_count = -1;
    $or_count = 0;
    foreach ($keywords as $match) {
      if ($or_count && $and_count + $or_count >= $limit_combinations) {

        // Ignore all further search expressions to prevent Denial-of-Service
        // attacks using a high number of AND/OR combinations.
        $this->expressionsIgnored = TRUE;
        break;
      }
      $phrase = FALSE;

      // Strip off phrase quotes.
      if ($match[2][0] == '"') {
        $match[2] = substr($match[2], 1, -1);
        $phrase = TRUE;
        $this->simple = FALSE;
      }

      // Simplify keyword according to indexing rules and external
      // preprocessors. Use same process as during search indexing, so it
      // will match search index.
      $words = search_simplify($match[2]);

      // Re-explode in case simplification added more words, except when
      // matching a phrase.
      $words = $phrase ? array(
        $words,
      ) : preg_split('/ /', $words, -1, PREG_SPLIT_NO_EMPTY);

      // Negative matches.
      if ($match[1] == '-') {
        $this->keys['negative'] = array_merge($this->keys['negative'], $words);
      }
      elseif ($match[2] == 'OR' && count($this->keys['positive'])) {
        $last = array_pop($this->keys['positive']);

        // Starting a new OR?
        if (!is_array($last)) {
          $last = array(
            $last,
          );
        }
        $this->keys['positive'][] = $last;
        $or = TRUE;
        $or_count++;
        continue;
      }
      elseif ($match[2] == 'AND' || $match[2] == 'and') {
        $warning = $match[2];
        continue;
      }
      else {
        if ($match[2] == 'or') {
          $warning = $match[2];
        }
        if ($or) {

          // Add to last element (which is an array).
          $this->keys['positive'][count($this->keys['positive']) - 1] = array_merge($this->keys['positive'][count($this->keys['positive']) - 1], $words);
        }
        else {
          $this->keys['positive'] = array_merge($this->keys['positive'], $words);
          $and_count++;
        }
      }
      $or = FALSE;
    }

    // Convert keywords into SQL statements.
    $this->conditions = db_and();
    $simple_and = FALSE;
    $simple_or = FALSE;

    // Positive matches.
    foreach ($this->keys['positive'] as $key) {

      // Group of ORed terms.
      if (is_array($key) && count($key)) {
        $simple_or = TRUE;
        $any = FALSE;
        $queryor = db_or();
        foreach ($key as $or) {
          list($num_new_scores) = $this
            ->parseWord($or);
          $any |= $num_new_scores;
          $queryor
            ->condition('d.data', "% {$or} %", 'LIKE');
        }
        if (count($queryor)) {
          $this->conditions
            ->condition($queryor);

          // A group of OR keywords only needs to match once.
          $this->matches += $any > 0;
        }
      }
      else {
        $simple_and = TRUE;
        list($num_new_scores, $num_valid_words) = $this
          ->parseWord($key);
        $this->conditions
          ->condition('d.data', "% {$key} %", 'LIKE');
        if (!$num_valid_words) {
          $this->simple = FALSE;
        }

        // Each AND keyword needs to match at least once.
        $this->matches += $num_new_scores;
      }
    }
    if ($simple_and && $simple_or) {
      $this->simple = FALSE;
    }

    // Negative matches.
    foreach ($this->keys['negative'] as $key) {
      $this->conditions
        ->condition('d.data', "% {$key} %", 'NOT LIKE');
      $this->simple = FALSE;
    }
    if ($warning == 'or') {
      drupal_set_message(t('Search for either of the two terms with uppercase <strong>OR</strong>. For example, <strong>cats OR dogs</strong>.'));
    }
  }

  /**
   * Helper function for parseQuery().
   */
  protected function parseWord($word) {
    $num_new_scores = 0;
    $num_valid_words = 0;

    // Determine the scorewords of this word/phrase.
    $split = explode(' ', $word);
    foreach ($split as $s) {
      $num = is_numeric($s);
      if ($num || drupal_strlen($s) >= variable_get('minimum_word_size', 3)) {
        if (!isset($this->words[$s])) {
          $this->words[$s] = $s;
          $num_new_scores++;
        }
        $num_valid_words++;
      }
    }

    // Return matching snippet and number of added words.
    return array(
      $num_new_scores,
      $num_valid_words,
    );
  }

  /**
   * Executes the first pass query.
   *
   * This can either be done explicitly, so that additional scores and
   * conditions can be applied to the second pass query, or implicitly by
   * addScore() or execute().
   *
   * @return
   *   TRUE if search items exist, FALSE if not.
   */
  public function executeFirstPass() {
    $this
      ->parseSearchExpression();
    if (count($this->words) == 0) {
      form_set_error('keys', format_plural(variable_get('minimum_word_size', 3), 'You must include at least one positive keyword with 1 character or more.', 'You must include at least one positive keyword with @count characters or more.'));
      return FALSE;
    }
    if ($this->expressionsIgnored) {
      drupal_set_message(t('Your search used too many AND/OR expressions. Only the first @count terms were included in this search.', array(
        '@count' => variable_get('search_and_or_limit', 7),
      )), 'warning');
    }
    $this->executedFirstPass = TRUE;
    if (!empty($this->words)) {
      $or = db_or();
      foreach ($this->words as $word) {
        $or
          ->condition('i.word', $word);
      }
      $this
        ->condition($or);
    }

    // Build query for keyword normalization.
    $this
      ->join('search_total', 't', 'i.word = t.word');
    $this
      ->condition('i.type', $this->type)
      ->groupBy('i.type')
      ->groupBy('i.sid')
      ->having('COUNT(*) >= :matches', array(
      ':matches' => $this->matches,
    ));

    // Clone the query object to do the firstPass query;
    $first = clone $this->query;

    // For complex search queries, add the LIKE conditions to the first pass query.
    if (!$this->simple) {
      $first
        ->join('search_dataset', 'd', 'i.sid = d.sid AND i.type = d.type');
      $first
        ->condition($this->conditions);
    }

    // Calculate maximum keyword relevance, to normalize it.
    $first
      ->addExpression('SUM(i.score * t.count)', 'calculated_score');
    $this->normalize = $first
      ->range(0, 1)
      ->orderBy('calculated_score', 'DESC')
      ->execute()
      ->fetchField();
    if ($this->normalize) {
      return TRUE;
    }
    return FALSE;
  }

  /**
   * Adds a custom score expression to the search query.
   *
   * Score expressions are used to order search results. If no calls to
   * addScore() have taken place, a default keyword relevance score will be
   * used. However, if at least one call to addScore() has taken place, the
   * keyword relevance score is not automatically added.
   *
   * Note that you must use this method to add ordering to your searches, and
   * not call orderBy() directly, when using the SearchQuery extender. This is
   * because of the two-pass system the SearchQuery class uses to normalize
   * scores.
   *
   * @param $score
   *   The score expression, which should evaluate to a number between 0 and 1.
   *   The string 'i.relevance' in a score expression will be replaced by a
   *   measure of keyword relevance between 0 and 1.
   * @param $arguments
   *   Query arguments needed to provide values to the score expression.
   * @param $multiply
   *   If set, the score is multiplied with this value. However, all scores
   *   with multipliers are then divided by the total of all multipliers, so
   *   that overall, the normalization is maintained.
   *
   * @return object
   *   The updated query object.
   */
  public function addScore($score, $arguments = array(), $multiply = FALSE) {
    if ($multiply) {
      $i = count($this->multiply);

      // Modify the score expression so it is multiplied by the multiplier,
      // with a divisor to renormalize.
      $score = "CAST(:multiply_{$i} AS DECIMAL) * COALESCE(( " . $score . "), 0) / CAST(:total_{$i} AS DECIMAL)";

      // Add an argument for the multiplier. The :total_$i argument is taken
      // care of in the execute() method, which is when the total divisor is
      // calculated.
      $arguments[':multiply_' . $i] = $multiply;
      $this->multiply[] = $multiply;
    }
    $this->scores[] = $score;
    $this->scoresArguments += $arguments;
    return $this;
  }

  /**
   * Executes the search.
   *
   * If not already done, this executes the first pass query. Then the complex
   * conditions are applied to the query including score expressions and
   * ordering.
   *
   * @return
   *   FALSE if the first pass query returned no results, and a database result
   *   set if there were results.
   */
  public function execute() {
    if (!$this->executedFirstPass) {
      $this
        ->executeFirstPass();
    }
    if (!$this->normalize) {
      return new DatabaseStatementEmpty();
    }

    // Add conditions to query.
    $this
      ->join('search_dataset', 'd', 'i.sid = d.sid AND i.type = d.type');
    $this
      ->condition($this->conditions);
    if (empty($this->scores)) {

      // Add default score.
      $this
        ->addScore('i.relevance');
    }
    if (count($this->multiply)) {

      // Re-normalize scores with multipliers by dividing by the total of all
      // multipliers. The expressions were altered in addScore(), so here just
      // add the arguments for the total.
      $i = 0;
      $sum = array_sum($this->multiply);
      foreach ($this->multiply as $total) {
        $this->scoresArguments[':total_' . $i] = $sum;
        $i++;
      }
    }

    // Replace the pseudo-expression 'i.relevance' with a measure of keyword
    // relevance in all score expressions, using string replacement. Careful
    // though! If you just print out a float, some locales use ',' as the
    // decimal separator in PHP, while SQL always uses '.'. So, make sure to
    // set the number format correctly.
    $relevance = number_format(1.0 / $this->normalize, 10, '.', '');
    $this->scores = str_replace('i.relevance', '(' . $relevance . ' * i.score * t.count)', $this->scores);

    // Add all scores together to form a query field.
    $this
      ->addExpression('SUM(' . implode(' + ', $this->scores) . ')', 'calculated_score', $this->scoresArguments);

    // If an order has not yet been set for this query, add a default order
    // that sorts by the calculated sum of scores.
    if (count($this
      ->getOrderBy()) == 0) {
      $this
        ->orderBy('calculated_score', 'DESC');
    }

    // Add useful metadata.
    $this
      ->addMetaData('normalize', $this->normalize)
      ->fields('i', array(
      'type',
      'sid',
    ));
    return $this->query
      ->execute();
  }

  /**
   * Builds the default count query for SearchQuery.
   *
   * Since SearchQuery always uses GROUP BY, we can default to a subquery. We
   * also add the same conditions as execute() because countQuery() is called
   * first.
   */
  public function countQuery() {

    // Clone the inner query.
    $inner = clone $this->query;

    // Add conditions to query.
    $inner
      ->join('search_dataset', 'd', 'i.sid = d.sid AND i.type = d.type');
    $inner
      ->condition($this->conditions);

    // Remove existing fields and expressions, they are not needed for a count
    // query.
    $fields =& $inner
      ->getFields();
    $fields = array();
    $expressions =& $inner
      ->getExpressions();
    $expressions = array();

    // Add the sid as the only field and count them as a subquery.
    $count = db_select($inner
      ->fields('i', array(
      'sid',
    )), NULL, array(
      'target' => 'slave',
    ));

    // Add the COUNT() expression.
    $count
      ->addExpression('COUNT(*)');
    return $count;
  }

}