Support PHP-Parser in the PHPAST application

Summary:
The PHPAST application allows viewing the xhpast token stream and AST.
It now also supports PHP-Parser generated ASTs and token streams.

Ref T16289

Test Plan:
* Apply D26522
* Enable prototypes via http://phorge.localhost/config/edit/phabricator.show-prototypes/
* Run `./bin/storage upgrade` to apply the database change
* Go to http://phorge.localhost/xhpast/ and click on the new "Use PHPAST" button in the top right
* Enter some PHP code
* Observe output and compare by doing the same on http://phorge.localhost/xhpast/
* After finishing testing this patch, `DROP TABLE {$NAMESPACE}_xhpast.phpast_parsetree; DELETE FROM phabricator_meta_data.patch_status WHERE patch = "phabricator:20251124.phpast.parsetree.sql";`

Reviewers: O1 Blessed Committers, aklapper

Reviewed By: O1 Blessed Committers, aklapper

Subscribers: aklapper, tobiaswiese, valerio.bozzolan, Matthew, Cigaryno

Maniphest Tasks: T16289

Differential Revision: https://we.phorge.it/D26523

resources/sql/autopatches/20251124.phpast.parsetree.sql

···
+CREATE TABLE {$NAMESPACE}_xhpast.phpast_parsetree (
+  `id` INT UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  `authorPHID` VARBINARY(64) DEFAULT NULL,
+  `input` LONGTEXT NOT NULL,
+  `error` LONGTEXT DEFAULT NULL,
+  `tokenStream` LONGBLOB NOT NULL,
+  `tree` LONGTEXT NOT NULL,
+  `dateCreated` INT UNSIGNED NOT NULL,
+  `dateModified` INT UNSIGNED NOT NULL
+);

src/__phutil_library_map__.php

···
     'PhorgeCodeWarningSetupCheck' => 'applications/config/check/PhorgeCodeWarningSetupCheck.php',
     'PhorgeFlagFlaggedObjectCustomField' => 'applications/flag/customfield/PhorgeFlagFlaggedObjectCustomField.php',
     'PhorgeFlagFlaggedObjectFieldStorage' => 'applications/flag/customfield/PhorgeFlagFlaggedObjectFieldStorage.php',
+    'PhorgePHPASTParseTree' => 'applications/phpast/storage/PhorgePHPASTParseTree.php',
+    'PhorgePHPASTViewFrameController' => 'applications/phpast/controller/PhorgePHPASTViewFrameController.php',
+    'PhorgePHPASTViewFramesetController' => 'applications/phpast/controller/PhorgePHPASTViewFramesetController.php',
+    'PhorgePHPASTViewInputController' => 'applications/phpast/controller/PhorgePHPASTViewInputController.php',
+    'PhorgePHPASTViewPanelController' => 'applications/phpast/controller/PhorgePHPASTViewPanelController.php',
+    'PhorgePHPASTViewRunController' => 'applications/phpast/controller/PhorgePHPASTViewRunController.php',
+    'PhorgePHPASTViewStreamController' => 'applications/phpast/controller/PhorgePHPASTViewStreamController.php',
+    'PhorgePHPASTViewTreeController' => 'applications/phpast/controller/PhorgePHPASTViewTreeController.php',
     'PhorgeSystemDeprecationWarningListener' => 'applications/system/events/PhorgeSystemDeprecationWarningListener.php',
     'PhortuneAccount' => 'applications/phortune/storage/PhortuneAccount.php',
     'PhortuneAccountAddManagerController' => 'applications/phortune/controller/account/PhortuneAccountAddManagerController.php',
···
     'PhorgeCodeWarningSetupCheck' => 'PhabricatorSetupCheck',
     'PhorgeFlagFlaggedObjectCustomField' => 'PhabricatorCustomField',
     'PhorgeFlagFlaggedObjectFieldStorage' => 'Phobject',
+    'PhorgePHPASTParseTree' => 'PhabricatorXHPASTDAO',
+    'PhorgePHPASTViewFrameController' => 'PhabricatorXHPASTViewController',
+    'PhorgePHPASTViewFramesetController' => 'PhabricatorXHPASTViewController',
+    'PhorgePHPASTViewInputController' => 'PhorgePHPASTViewPanelController',
+    'PhorgePHPASTViewPanelController' => 'PhabricatorXHPASTViewController',
+    'PhorgePHPASTViewRunController' => 'PhabricatorXHPASTViewController',
+    'PhorgePHPASTViewStreamController' => 'PhorgePHPASTViewPanelController',
+    'PhorgePHPASTViewTreeController' => 'PhorgePHPASTViewPanelController',
     'PhorgeSystemDeprecationWarningListener' => 'PhabricatorEventListener',
     'PhortuneAccount' => array(
       'PhortuneDAO',

src/applications/phpast/application/PhabricatorPHPASTApplication.php

···
         'stream/(?P<id>[1-9]\d*)/'
           => 'PhabricatorXHPASTViewStreamController',
       ),
+      '/phpast/' => array(
+        '' => PhorgePHPASTViewRunController::class,
+        'view/(?P<id>[1-9]\d*)/'
+          => PhorgePHPASTViewFrameController::class,
+        'frameset/(?P<id>[1-9]\d*)/'
+          => PhorgePHPASTViewFramesetController::class,
+        'input/(?P<id>[1-9]\d*)/'
+          => PhorgePHPASTViewInputController::class,
+        'tree/(?P<id>[1-9]\d*)/'
+          => PhorgePHPASTViewTreeController::class,
+        'stream/(?P<id>[1-9]\d*)/'
+          => PhorgePHPASTViewStreamController::class,
+      ),
     );
   }
 

src/applications/phpast/controller/PhabricatorXHPASTViewRunController.php

···
 final class PhabricatorXHPASTViewRunController
   extends PhabricatorXHPASTViewController {
 
+  protected function buildApplicationCrumbs() {
+    return parent::buildApplicationCrumbs()
+      ->addAction(
+        id(new PHUIListItemView())
+          ->setName(pht('Use PHPAST'))
+          ->setHref('/phpast/')
+          ->setIcon('fa-random'));
+  }
+
   public function handleRequest(AphrontRequest $request) {
     $viewer = $this->getViewer();
 
···
     }
 
     $form = id(new AphrontFormView())
-      ->setUser($viewer)
+      ->setViewer($viewer)
       ->appendChild(
         id(new AphrontFormTextAreaControl())
           ->setLabel(pht('Source'))
···
 
     return $this->newPage()
       ->setTitle($title)
+      ->setCrumbs(
+        id(new PHUICrumbsView())
+          ->addAction(
+            id(new PHUIListItemView())
+              ->setName(pht('Use PHPAST'))
+              ->setHref('/phpast/')
+              ->setIcon('fa-random')))
       ->appendChild($view);
 
   }

src/applications/phpast/controller/PhorgePHPASTViewFrameController.php

···
+<?php
+
+final class PhorgePHPASTViewFrameController
+  extends PhabricatorXHPASTViewController {
+
+  public function shouldAllowPublic() {
+    return true;
+  }
+
+  public function handleRequest(AphrontRequest $request) {
+    $id = $request->getURIData('id');
+
+    return $this->newPage()
+     ->setApplicationName('PHPASTView')
+     ->setBaseURI('/phpast/')
+     ->setTitle(pht('PHPAST View'))
+     ->setGlyph("\xE2\x96\xA0")
+     ->appendChild(phutil_tag(
+        'iframe',
+        array(
+          'src'         => "/phpast/frameset/{$id}/",
+          'frameborder' => '0',
+          'style'       => 'width: 100%; height: 800px;',
+        '',
+      )));
+  }
+}

src/applications/phpast/controller/PhorgePHPASTViewFramesetController.php

···
+<?php
+
+final class PhorgePHPASTViewFramesetController
+  extends PhabricatorXHPASTViewController {
+
+  public function shouldAllowPublic() {
+    return true;
+  }
+
+  public function handleRequest(AphrontRequest $request) {
+    $id = $request->getURIData('id');
+
+    return id(new AphrontWebpageResponse())
+      ->setFrameable(true)
+      ->setContent(phutil_tag(
+        'frameset',
+        array('cols' => '33%, 34%, 33%'),
+          array(
+            phutil_tag('frame', array('src' => "/phpast/input/{$id}/")),
+            phutil_tag('frame', array('src' => "/phpast/tree/{$id}/")),
+            phutil_tag('frame', array('src' => "/phpast/stream/{$id}/")),
+        )));
+  }
+}

src/applications/phpast/controller/PhorgePHPASTViewInputController.php

···
+<?php
+
+final class PhorgePHPASTViewInputController
+  extends PhorgePHPASTViewPanelController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $input = $this->getStorageTree()->getInput();
+    return $this->buildPHPASTViewPanelResponse($input);
+  }
+}

src/applications/phpast/controller/PhorgePHPASTViewPanelController.php

···
+<?php
+
+abstract class PhorgePHPASTViewPanelController
+  extends PhabricatorXHPASTViewController {
+
+  private $id;
+  private $storageTree;
+
+  public function shouldAllowPublic() {
+    return true;
+  }
+
+  public function willProcessRequest(array $data) {
+    // Load the parser now to ensure the autoloader for PHP-Parser is available.
+    PhutilPHPParserLibrary::getParser();
+
+    $this->id = $data['id'];
+    $this->storageTree = id(new PhorgePHPASTParseTree())
+      ->load($this->id);
+
+    if (!$this->storageTree) {
+      throw new Exception(pht('No such AST!'));
+    }
+  }
+
+  protected function getStorageTree() {
+    return $this->storageTree;
+  }
+
+  protected function buildPHPASTViewPanelResponse($content) {
+    $content = hsprintf(
+      '<!DOCTYPE html>'.
+      '<html>'.
+        '<head>'.
+          '<style>
+body {
+  white-space: pre;
+  font: 10px "Monaco";
+  cursor: pointer;
+}
+
+.token {
+  padding: 2px 4px;
+  margin: 2px 2px;
+  border: 1px solid #bbbbbb;
+  line-height: 24px;
+}
+
+ul {
+  margin: 0 0 0 1em;
+  padding: 0;
+  list-style: none;
+  line-height: 1em;
+}
+
+li {
+  margin: 0;
+  padding: 0;
+}
+
+li span {
+  background: #dddddd;
+  padding: 3px 6px;
+}
+
+          </style>'.
+        '</head>'.
+        '<body>%s</body>'.
+      '</html>',
+      $content);
+
+    return id(new AphrontWebpageResponse())
+      ->setFrameable(true)
+      ->setContent($content);
+  }
+
+}

src/applications/phpast/controller/PhorgePHPASTViewRunController.php

···
+<?php
+
+/**
+ * @phutil-external-symbol class PhpParser\Error
+ */
+final class PhorgePHPASTViewRunController
+  extends PhabricatorXHPASTViewController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $viewer = $this->getViewer();
+
+    if ($request->isFormPost()) {
+      $source = $request->getStr('source');
+
+      $storage_tree = id(new PhorgePHPASTParseTree())
+        ->setInput($source)
+        ->setAuthorPHID($viewer->getPHID());
+
+      $parser = PhutilPHPParserLibrary::getParser();
+      $exposes_token_stream = version_compare(
+        PhutilPHPParserLibrary::getVersion(),
+        PhutilPHPParserLibrary::EXPECTED_VERSION,
+        '>=');
+
+      try {
+        $storage_tree->setTree($parser->parse($source));
+        if ($exposes_token_stream) {
+          $storage_tree->setTokenStream($parser->getTokens());
+        }
+      } catch (PhpParser\Error $ex) {
+        $storage_tree->setError($ex->getMessageWithColumnInfo($source));
+      }
+
+      $storage_tree->save();
+
+      return id(new AphrontRedirectResponse())
+        ->setURI('/phpast/view/'.$storage_tree->getID().'/');
+    }
+
+    $form = id(new AphrontFormView())
+      ->setViewer($viewer)
+      ->appendChild(
+        id(new AphrontFormTextAreaControl())
+          ->setLabel(pht('Source'))
+          ->setName('source')
+          ->setValue("<?php\n\n")
+          ->setHeight(AphrontFormTextAreaControl::HEIGHT_VERY_TALL))
+      ->appendChild(
+        id(new AphrontFormSubmitControl())
+          ->setValue(pht('Parse')));
+
+    $form_box = id(new PHUIObjectBoxView())
+      ->setHeaderText(pht('Generate PHP AST'))
+      ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
+      ->setForm($form);
+
+    $title = pht('PHPAST View');
+    $header = id(new PHUIHeaderView())
+      ->setHeader($title)
+      ->setHeaderIcon('fa-ambulance');
+
+    $view = id(new PHUITwoColumnView())
+      ->setHeader($header)
+      ->setFooter(array(
+        $form_box,
+      ));
+
+    return $this->newPage()
+      ->setTitle($title)
+      ->setCrumbs(
+        id(new PHUICrumbsView())
+          ->addAction(
+            id(new PHUIListItemView())
+              ->setName(pht('Use XHPAST'))
+              ->setHref('/xhpast/')
+              ->setIcon('fa-random')))
+      ->appendChild($view);
+
+  }
+
+}

src/applications/phpast/controller/PhorgePHPASTViewStreamController.php

···
+<?php
+
+final class PhorgePHPASTViewStreamController
+  extends PhorgePHPASTViewPanelController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $storage = $this->getStorageTree();
+    $err = $storage->getError();
+    $token_stream = $storage->getTokenStream();
+
+    if ($err) {
+      return $this->buildPHPASTViewPanelResponse($err);
+    }
+
+    $tokens = array();
+    foreach ($token_stream as $id => $token) {
+      $seq = $id;
+      $name = $token->getTokenName();
+      $title = pht('Token %d: %s', $seq, $name);
+
+      $tokens[] = phutil_tag(
+        'span',
+        array(
+          'title' => $title,
+          'class' => 'token',
+        ),
+        $token->text);
+    }
+
+    return $this->buildPHPASTViewPanelResponse(
+      phutil_implode_html('', $tokens));
+  }
+}

src/applications/phpast/controller/PhorgePHPASTViewTreeController.php

···
+<?php
+
+/**
+ * @phutil-external-symbol class PhpParser\Node
+ */
+final class PhorgePHPASTViewTreeController
+  extends PhorgePHPASTViewPanelController {
+
+  public function handleRequest(AphrontRequest $request) {
+    $storage = $this->getStorageTree();
+    $err = $storage->getError();
+    $ast = $storage->getTree();
+
+    if ($err) {
+      return $this->buildPHPASTViewPanelResponse($err);
+    }
+
+    return $this->buildPHPASTViewPanelResponse($this->buildTree($ast));
+  }
+
+  protected function buildTree(array $nodes) {
+    $tree = array();
+
+    foreach ($nodes as $node) {
+      $tree[] = phutil_tag(
+      'li',
+      array(),
+      phutil_tag(
+        'span',
+        array(
+          'title' => $node->getType(),
+        ),
+        $node->getType()));
+
+      foreach ($node->getSubNodeNames() as $sub_node_name) {
+        $sub_node = $node->{$sub_node_name};
+
+        if (is_array($sub_node) && $sub_node) {
+          $tree[] = $this->buildTree($sub_node);
+        } else if ($sub_node instanceof PhpParser\Node) {
+          $tree[] = $this->buildTree(array($sub_node));
+        }
+      }
+    }
+
+    return phutil_tag(
+      'ul',
+      array(),
+      phutil_implode_html("\n", $tree));
+  }
+
+}

src/applications/phpast/storage/PhorgePHPASTParseTree.php

···
+<?php
+
+/**
+ * @phutil-external-symbol class PhpParser\JsonDecoder
+ */
+final class PhorgePHPASTParseTree extends PhabricatorXHPASTDAO {
+
+  protected $authorPHID;
+  protected $input;
+  protected $tree;
+  protected $error;
+  protected $tokenStream;
+
+  protected function getConfiguration() {
+    return array(
+      self::CONFIG_SERIALIZATION => array(
+        'tokenStream' => self::SERIALIZATION_PHP,
+        'tree' => self::SERIALIZATION_JSON,
+      ),
+      self::CONFIG_BINARY => array(
+        'tokenStream' => true,
+      ),
+      self::CONFIG_COLUMN_SCHEMA => array(
+        'authorPHID' => 'phid?',
+        'error' => 'text?',
+        'input' => 'text',
+      ),
+    ) + parent::getConfiguration();
+  }
+
+  protected function applyLiskDataSerialization(array &$data, $deserialize) {
+    // applyLiskDataSerialization overwrites $data, so capture the
+    // JSON before it does so.
+    $tree = $data['tree'];
+
+    parent::applyLiskDataSerialization($data, $deserialize);
+
+    if ($deserialize) {
+      $data['tree'] = id(new PhpParser\JsonDecoder())->decode($tree);
+    }
+  }
+
+  public function getTableName() {
+    return 'phpast_parsetree';
+  }
+}