Rewrite i18n extraction to use PHP-Parser rather than Xhpast

Summary:
Also:
- Teach it that `count` returns a number. Maybe things should use `phutil_count` instead, but in practice they sometimes don't.
- Let it trace through certain variable assignments.
- Tell it that certain methods of certain classes take arguments that are pre-typed, and use this for edge types.

Part of T16378, T16289, and T15862 (but doesn't close any of them)

Test Plan:
- Run `./bin/i18n extract --clean` on the old version before the patch.
- Save the `src/.cache/i18n_strings.json` file somewhere.
- Run `./bin/i18n extract --clean` on the new version with this patch.
- Save the `src/.cache/i18n_strings.json` file somewhere.
- Compare the two files. The expected differences are a few more params show up as numbers, but nothing else.

Reviewers: O1 Blessed Committers, mainframe98, avivey

Reviewed By: O1 Blessed Committers, avivey

Subscribers: avivey, aklapper, tobiaswiese, valerio.bozzolan, Matthew, Cigaryno

Differential Revision: https://we.phorge.it/D26554

src/infrastructure/edges/type/PhabricatorEdgeType.php

···
       $actor);
   }
 
+  /**
+    * Returns a string representing how adding this instances of this
+    * edge should be rendered in history of an object.
+    *
+    * Note that the parameters to this function (and overriding functions in
+    * subclasses) are special-cased by the i18n string extractor;
+    * the `add_count` parameter is treated as a number if passed to `pht`
+    * (even though the extractor can't generally parse numbers
+    *  passed as method arguments)
+    */
   public function getTransactionAddString(
     $actor,
     $add_count,
···
       $add_count,
       $add_edges);
   }
-
+  /**
+    * Returns a string representing how adding removing instances of this
+    * edge should be rendered in history of an object.
+    *
+    * Note that the parameters to this function (and overriding functions in
+    * subclasses) are special-cased by the i18n string extractor;
+    * the `rem_count` parameter is treated as a number if passed to `pht`
+    * (even though the extractor can't generally parse numbers
+    *  passed as method arguments)
+    */
   public function getTransactionRemoveString(
     $actor,
     $rem_count,
···
       $rem_edges);
   }
 
+  /**
+    * Returns a string representing how adding and removing instances of this
+    * edge should be rendered in history of an object.
+    *
+    * Note that the parameters to this function (and overriding functions in
+    * subclasses) are special-cased by the i18n string extractor;
+    * the `*_count` parameters are treated as a numbers if passed to `pht`
+    * (even though the extractor can't generally parse numbers
+    *  passed as method arguments)
+    */
   public function getTransactionEditString(
     $actor,
     $total_count,
···
       $rem_edges);
   }
 
+  /**
+    * Returns a string representing how adding instances of this
+    * edge should be rendered in the feed.
+    *
+    * Note that the parameters to this function (and overriding functions in
+    * subclasses) are special-cased by the i18n string extractor;
+    * the `add_count` parameter is treated as a number if passed to `pht`
+    * (even though the extractor can't generally parse numbers
+    *  passed as method arguments)
+    */
   public function getFeedAddString(
     $actor,
     $object,
···
       $add_edges);
   }
 
+  /**
+    * Returns a string representing how removing instances of this
+    * edge should be rendered in the feed.
+    *
+    * Note that the parameters to this function (and overriding functions in
+    * subclasses) are special-cased by the i18n string extractor;
+    * the `rem_count` parameter is treated as a number if passed to `pht`
+    * (even though the extractor can't generally parse numbers
+     * passed as method arguments)
+    */
   public function getFeedRemoveString(
     $actor,
     $object,
···
       $rem_edges);
   }
 
+  /**
+    * Returns a string representing how adding and removing instances of this
+    * edge should be rendered in the feed.
+    *
+    * Note that the parameters to this function (and overriding functions in
+    * subclasses) are special-cased by the i18n string extractor;
+    * the `*_count` parameters are treated as numbers if passed to `pht`
+    * (even though the extractor can't generally parse numbers
+    *  passed as method arguments)
+    */
   public function getFeedEditString(
     $actor,
     $object,

src/infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php

···
 <?php
 
+/**
+ * @phutil-external-symbol class PhpParser\NodeTraverser
+ * @phutil-external-symbol class PhorgePHPParserExtractor
+ */
 final class PhabricatorInternationalizationManagementExtractWorkflow
   extends PhabricatorInternationalizationManagementWorkflow {
 
···
   private function extractFiles($root_path, array $files) {
     $hashes = array();
 
-    $futures = array();
-    foreach ($files as $file => $hash) {
-      $full_path = $root_path.DIRECTORY_SEPARATOR.$file;
-      $data = Filesystem::readFile($full_path);
-      $futures[$full_path] = PhutilXHPASTBinary::getParserFuture($data);
-
-      $hashes[$full_path] = $hash;
-    }
-
     $bar = id(new PhutilConsoleProgressBar())
-      ->setTotal(count($futures));
+      ->setTotal(count($files));
 
     $messages = array();
     $results = array();
 
-    $futures = id(new FutureIterator($futures))
-      ->limit(8);
-    foreach ($futures as $full_path => $future) {
+    $parser = PhutilPHPParserLibrary::getParser();
+    // Load this class now (once PHP-Parser is built so its base class exists)
+    // this is not in the autoloader to avoid errors from tests that try
+    // to load every class without installing PHP-Parser
+    $root = dirname(phutil_get_library_root('phorge'));
+    require_once $root.'/support/php-parser/PhorgePHPParserExtractor.php';
+    foreach ($files as $file => $hash) {
       $bar->update(1);
-
-      $hash = $hashes[$full_path];
-
+      $full_path = $root_path.DIRECTORY_SEPARATOR.$file;
+      $hashes[$full_path] = $hash;
+      $file = Filesystem::readablePath($full_path, $root_path);
       try {
-        $tree = XHPASTTree::newFromDataAndResolvedExecFuture(
-          Filesystem::readFile($full_path),
-          $future->resolve());
+        $data = Filesystem::readFile($full_path);
+        $tree = $parser->parse($data);
+        $visitor = new PhorgePHPParserExtractor($file);
+        id(new PhpParser\NodeTraverser($visitor))->traverse($tree);
       } catch (Exception $ex) {
         $messages[] = pht(
-          'WARNING: Failed to extract strings from file "%s": %s',
+          'Failed to parse file "%s": %s',
           $full_path,
           $ex->getMessage());
         continue;
       }
-
-      $root = $tree->getRootNode();
-      $calls = $root->selectDescendantsOfType('n_FUNCTION_CALL');
-      foreach ($calls as $call) {
-        $name = $call->getChildByIndex(0)->getConcreteString();
-        if ($name != 'pht') {
-          continue;
-        }
-
-        $params = $call->getChildByIndex(1, 'n_CALL_PARAMETER_LIST');
-        $string_node = $params->getChildByIndex(0);
-        $string_line = $string_node->getLineNumber();
-        try {
-          $string_value = $string_node->evalStatic();
-
-          $args = $params->getChildren();
-          $args = array_slice($args, 1);
-
-          $types = array();
-          foreach ($args as $child) {
-            $type = null;
-
-            switch ($child->getTypeName()) {
-              case 'n_FUNCTION_CALL':
-                $call = $child->getChildByIndex(0);
-                if ($call->getTypeName() == 'n_SYMBOL_NAME') {
-                  switch ($call->getConcreteString()) {
-                    case 'phutil_count':
-                      $type = 'number';
-                      break;
-                    case 'phutil_person':
-                      $type = 'person';
-                      break;
-                  }
-                }
-                break;
-              case 'n_NEW':
-                $class = $child->getChildByIndex(0);
-                if ($class->getTypeName() == 'n_CLASS_NAME') {
-                  switch ($class->getConcreteString()) {
-                    case 'PhutilNumber':
-                      $type = 'number';
-                      break;
-                  }
-                }
-                break;
-              default:
-                break;
-            }
-
-            $types[] = $type;
-          }
+      $results[$hash] = $visitor->getResults();
+      $messages += $visitor->getWarnings();
+    }
 
-          $results[$hash][] = array(
-            'string' => $string_value,
-            'file' => Filesystem::readablePath($full_path, $root_path),
-            'line' => $string_line,
-            'types' => $types,
-          );
-        } catch (Exception $ex) {
-          $messages[] = pht(
-            'WARNING: Failed to evaluate pht() call on line %d in "%s": %s',
-            $call->getLineNumber(),
-            $full_path,
-            $ex->getMessage());
-        }
-      }
-
-      $tree->dispose();
-    }
     $bar->done();
 
     foreach ($messages as $message) {

support/php-parser/PhorgePHPParserExtractor.php

···
+<?php
+
+final class PhorgePHPParserExtractor extends PhpParser\NodeVisitorAbstract {
+  private static $knownTypes = array(
+    'PhabricatorEdgeType' => array(
+      'getTransactionAddString' => array(null, 'number', null),
+      'getTransactionRemoveString' => array(null, 'number', null),
+      'getTransactionEditString' => array(
+         null,
+         'number',
+         'number',
+         null,
+         'number',
+         null,
+      ),
+      'getFeedAddString' => array(null, null, 'number', null),
+      'getFeedRemoveString' => array(null, null, 'number', null),
+      'getFeedEditString' => array(
+        null,
+        null,
+        'number',
+        'number',
+        null,
+        'number',
+        null,
+      ),
+    ),
+  );
+  private $classDecl = null;
+  private $classExtends = null;
+  private $methodDecl = null;
+  private $paramVars = [];
+  private $filename = null;
+  private $results = [];
+  private $warnings = [];
+  private $variableAssignments = [];
+
+  public function getWarnings() {
+    return $this->warnings;
+  }
+
+  public function getResults() {
+    return $this->results;
+  }
+
+  public function __construct($filename) {
+    $this->filename = $filename;
+  }
+
+  private function getName($node) {
+    $name = $node->name;
+    if (is_string($name)) {
+      return $name;
+    }
+    if ($name instanceof PhpParser\Node\Name ||
+      $name instanceof PhpParser\Node\Identifier
+    ) {
+      return $name->name;
+    }
+    return null;
+  }
+
+  private function typeNode(PhpParser\Node $node, array $seen = array()) {
+    if ($node instanceof PhpParser\Node\Scalar\Int_) {
+      return 'number';
+    } else if ($node instanceof PhpParser\Node\Expr\BinaryOp) {
+      // i.e count($foos) + count($bars)
+      $type1 = $this->typeNode($node->left, $seen);
+      $type2 = $this->typeNode($node->right, $seen);
+      if ($type1 === $type2) {
+        return $type1;
+      }
+    } else if ($node instanceof PhpParser\Node\Expr\FuncCall) {
+      switch ($this->getName($node)) {
+        case 'phutil_count':
+        case 'count':
+          return 'number';
+        case 'phutil_person':
+          return 'person';
+      }
+    } else if ($node instanceof PhpParser\Node\Expr\New_ ) {
+      if ($this->getName($node->class) == 'PhutilNumber') {
+        return 'number';
+      }
+    } else if ($node instanceof PhpParser\Node\Expr\Variable) {
+      $name = $this->getName($node);
+      if (isset($seen[$name])) {
+        return null;
+      }
+      if (isset($this->variableAssignments[$name])) {
+        return $this->typeNode(
+          $this->variableAssignments[$name],
+          $seen + [$name => true]);
+      }
+      $known = static::$knownTypes;
+      if (isset($known[$this->classExtends])) {
+        $class_data = $known[$this->classExtends];
+      } else if (isset($known[$this->classDecl])) {
+        $class_data = $known[$this->classDecl];
+      }
+      if (isset($class_data[$this->methodDecl])) {
+        $types = $class_data[$this->methodDecl];
+        if ($name && isset($this->paramVars[$name])) {
+          if (isset($types[$this->paramVars[$name]])) {
+            return $types[$this->paramVars[$name]];
+          }
+        }
+      }
+    }
+    return null;
+  }
+
+  public function enterNode(PhpParser\Node $node) {
+    if ($node instanceof PhpParser\Node\Stmt\Class_) {
+      $this->classDecl = $this->getName($node);
+      if ($node->extends) {
+        $this->classExtends = $this->getName($node->extends);
+      }
+    } else if ($node instanceof PhpParser\Node\Stmt\ClassMethod) {
+      $this->methodDecl = $this->getName($node);
+      $this->paramVars = [];
+      $i = 0;
+      foreach ($node->params as $param) {
+        $name = $this->getName($param->var);
+        if ($name) {
+          $this->paramVars[$name] = $i;
+        }
+        $i++;
+      }
+    } else if ($node instanceof PhpParser\Node\Expr\Assign) {
+      if ($node->var instanceof PhpParser\Node\Expr\Variable) {
+        $name = $this->getName($node->var);
+        // $name could be null if this is a "variable variable"
+        // (like $$foo = bar)
+        if ($name) {
+          $this->variableAssignments[$name] = $node->expr;
+        }
+      }
+    } else if ($node instanceof PhpParser\Node\Expr\FuncCall) {
+      if ($this->getName($node) == 'pht') {
+        $types = [];
+        $args = $node->args;
+        $first = array_shift($args);
+        try {
+          $key = id(new PhpParser\ConstExprEvaluator())
+            ->evaluateSilently($first->value);
+        } catch (PhpParser\ConstExprEvaluationException $ex) {
+          $this->warnings[] = pht(
+            'Failed to evaluate pht() call on line %d in "%s": %s',
+            $first->getStartLine(),
+            $this->filename,
+            $ex->getMessage());
+          return;
+        }
+        foreach ($args as $child) {
+          $types[] = $this->typeNode($child->value);
+        }
+        $this->results[] = array(
+          'string' => $key,
+          'line' => $first->getStartLine(),
+          'types' => $types,
+          'file' => $this->filename,
+        );
+      }
+    }
+  }
+
+  public function leaveNode(PhpParser\Node $node) {
+     if ($node instanceof PhpParser\Node\Stmt\Class_) {
+      $this->classDecl = null;
+      $this->classExtends = null;
+     } else if ($node instanceof PhpParser\Node\Stmt\ClassMethod) {
+      $this->methodDecl = null;
+      $this->paramVars = [];
+      $this->variableAssignments = [];
+     }
+  }
+}