Fix almost all remaining schemata issues

Summary:
Ref T1191. This fixes nearly every remaining blocker for utf8mb4 -- primarily, overlong keys.

Remaining issue is https://secure.phabricator.com/T1191#77467

Test Plan: I'll annotate inline.

Reviewers: btrahan

Reviewed By: btrahan

Subscribers: epriestley, hach-que

Maniphest Tasks: T6099, T6129, T6133, T6134, T6150, T6148, T6147, T6146, T6105, T1191

Differential Revision: https://secure.phabricator.com/D10601

src/applications/auth/storage/PhabricatorAuthProviderConfig.php

···
       self::CONFIG_COLUMN_SCHEMA => array(
         'isEnabled' => 'bool',
         'providerClass' => 'text128',
-        'providerType' => 'text64',
+        'providerType' => 'text32',
         'providerDomain' => 'text128',
         'shouldAllowLogin' => 'bool',
         'shouldAllowRegistration' => 'bool',

src/applications/conduit/storage/PhabricatorConduitMethodCallLog.php

···
       self::CONFIG_COLUMN_SCHEMA => array(
         'id' => 'id64',
         'connectionID' => 'id64?',
-        'method' => 'text255',
+        'method' => 'text64',
         'error' => 'text255',
         'duration' => 'uint64',
         'callerPHID' => 'phid?',

src/applications/config/controller/PhabricatorConfigDatabaseController.php

···
 abstract class PhabricatorConfigDatabaseController
   extends PhabricatorConfigController {
 
-  const MAX_INNODB_KEY_LENGTH = 767;
-
   protected function buildSchemaQuery() {
     $conf = PhabricatorEnv::newObjectFromConfig(
       'mysql.configuration-provider',

src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php

···
     $nullable_issue = PhabricatorConfigStorageSchema::ISSUE_NULLABLE;
     $unique_issue = PhabricatorConfigStorageSchema::ISSUE_UNIQUE;
     $columns_issue = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS;
+    $longkey_issue = PhabricatorConfigStorageSchema::ISSUE_LONGKEY;
 
     $database = $comp->getDatabase($database_name);
     if (!$database) {
···
       if ($size) {
         $size_formatted = $this->renderAttr(
           $size,
-          ($size > self::MAX_INNODB_KEY_LENGTH));
+          $key->hasIssue($longkey_issue));
       }
 
       $key_rows[] = array(

src/applications/config/schema/PhabricatorConfigColumnSchema.php

···
     $type = $this->getColumnType();
 
     $matches = null;
-    if (preg_match('/^varchar\((\d+)\)$/', $type, $matches)) {
+    if (preg_match('/^(?:var)?char\((\d+)\)$/', $type, $matches)) {
       // For utf8mb4, each character requires 4 bytes.
       $size = (int)$matches[1];
       if ($prefix && $prefix < $size) {
···
     }
 
     $matches = null;
-    if (preg_match('/^char\((\d+)\)$/', $type, $matches)) {
-      // We use char() only for fixed-length binary data, so its size
+    if (preg_match('/^(?:var)?binary\((\d+)\)$/', $type, $matches)) {
+      // binary()/varbinary() store fixed-length binary data, so their size
       // is always the column size.
       $size = (int)$matches[1];
       if ($prefix && $prefix < $size) {

src/applications/config/schema/PhabricatorConfigKeySchema.php

···
 final class PhabricatorConfigKeySchema
   extends PhabricatorConfigStorageSchema {
 
+  const MAX_INNODB_KEY_LENGTH = 767;
+
   private $columnNames;
   private $unique;
+  private $table;
+  private $indexType;
+
+  public function setIndexType($index_type) {
+    $this->indexType = $index_type;
+    return $this;
+  }
+
+  public function getIndexType() {
+    return $this->indexType;
+  }
+
+  public function setProperty($property) {
+    $this->property = $property;
+    return $this;
+  }
+
+  public function getProperty() {
+    return $this->property;
+  }
 
   public function setUnique($unique) {
     $this->unique = $unique;
···
     return $this->unique;
   }
 
+  public function setTable(PhabricatorConfigTableSchema $table) {
+    $this->table = $table;
+    return $this;
+  }
+
+  public function getTable() {
+    return $this->table;
+  }
+
   public function setColumnNames(array $column_names) {
     $this->columnNames = array_values($column_names);
     return $this;
···
     }
   }
 
+  public function getKeyByteLength() {
+    $size = 0;
+    foreach ($this->getColumnNames() as $column_spec) {
+      list($column_name, $prefix) = $this->getKeyColumnAndPrefix($column_spec);
+      $column = $this->getTable()->getColumn($column_name);
+      if (!$column) {
+        $size = 0;
+        break;
+      }
+      $size += $column->getKeyByteLength($prefix);
+    }
+
+    return $size;
+  }
+
   public function compareToSimilarSchema(
     PhabricatorConfigStorageSchema $expect) {
 
···
       $issues[] = self::ISSUE_UNIQUE;
     }
 
+    // A fulltext index can be of any length.
+    if ($this->getIndexType() != 'FULLTEXT') {
+      if ($this->getKeyByteLength() > self::MAX_INNODB_KEY_LENGTH) {
+        $issues[] = self::ISSUE_LONGKEY;
+      }
+    }
+
     return $issues;
   }
 
   public function newEmptyClone() {
     $clone = clone $this;
+    $this->table = null;
     return $clone;
   }
 

src/applications/config/schema/PhabricatorConfigSchemaQuery.php

···
           $key_schema = id(new PhabricatorConfigKeySchema())
             ->setName($key_name)
             ->setColumnNames($column_names)
-            ->setUnique(!$head['Non_unique']);
+            ->setUnique(!$head['Non_unique'])
+            ->setIndexType($head['Index_type']);
 
           $table_schema->addKey($key_schema);
         }

src/applications/config/schema/PhabricatorConfigSchemaSpec.php

···
         ->setColumnNames(idx($key_spec, 'columns', array()));
 
       $key->setUnique((bool)idx($key_spec, 'unique'));
+      $key->setIndexType(idx($key_spec, 'type', 'BTREE'));
 
       $table->addKey($key);
     }

src/applications/config/schema/PhabricatorConfigStorageSchema.php

···
   const ISSUE_NULLABLE = 'nullable';
   const ISSUE_KEYCOLUMNS = 'keycolumns';
   const ISSUE_UNIQUE = 'unique';
+  const ISSUE_LONGKEY = 'longkey';
   const ISSUE_SUBWARN = 'subwarn';
   const ISSUE_SUBFAIL = 'subfail';
 
···
         return pht('Key on Wrong Columns');
       case self::ISSUE_UNIQUE:
         return pht('Key has Wrong Uniqueness');
+      case self::ISSUE_LONGKEY:
+        return pht('Key is Too Long');
       case self::ISSUE_SUBWARN:
         return pht('Subschemata Have Warnings');
       case self::ISSUE_SUBFAIL:
···
       case self::ISSUE_NULLABLE:
         return pht('This schema has the wrong nullable setting.');
       case self::ISSUE_KEYCOLUMNS:
-        return pht('This schema is on the wrong columns.');
+        return pht('This key is on the wrong columns.');
       case self::ISSUE_UNIQUE:
         return pht('This key has the wrong uniqueness setting.');
+      case self::ISSUE_LONGKEY:
+        return pht('This key is too long for utf8mb4.');
       case self::ISSUE_SUBWARN:
         return pht('Subschemata have setup warnings.');
       case self::ISSUE_SUBFAIL:
···
       case self::ISSUE_SURPLUSKEY:
       case self::ISSUE_UNIQUE:
       case self::ISSUE_KEYCOLUMNS:
+      case self::ISSUE_LONGKEY:
         return self::STATUS_WARN;
       default:
         throw new Exception(pht('Unknown schema issue "%s"!', $issue));

src/applications/config/schema/PhabricatorConfigTableSchema.php

···
       throw new Exception(
         pht('Trying to add duplicate key "%s"!', $name));
     }
+    $key->setTable($this);
     $this->keys[$name] = $key;
     return $this;
   }
···
   }
 
   protected function getSubschemata() {
-    return array_merge($this->getColumns(), $this->getKeys());
+    // NOTE: Keys and columns may have the same name, so make sure we return
+    // everything.
+
+    return array_merge(
+      array_values($this->columns),
+      array_values($this->keys));
   }
 
   public function setCollation($collation) {

src/applications/differential/storage/DifferentialDiffProperty.php

···
         'data' => self::SERIALIZATION_JSON,
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
+        'name' => 'text128',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'diffID' => array(

src/applications/diviner/storage/DivinerLiveSymbol.php

···
           ),
         ),
         'name' => array(
-          'columns' => array('name'),
+          'columns' => array('name(64)'),
         ),
         'key_slug' => array(
           'columns' => array('titleSlugHash'),

src/applications/files/storage/PhabricatorTransformedFile.php

···
   public function getConfiguration() {
     return array(
       self::CONFIG_COLUMN_SCHEMA => array(
-        'transform' => 'text255',
+        'transform' => 'text128',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'originalPHID' => array(

src/applications/herald/controller/HeraldRuleController.php

···
     $rule->attachActions($actions);
 
     if (!$errors) {
-      try {
+      $edit_action = $rule->getID() ? 'edit' : 'create';
 
-        $edit_action = $rule->getID() ? 'edit' : 'create';
-
-        $rule->openTransaction();
-          $rule->save();
-          $rule->saveConditions($conditions);
-          $rule->saveActions($actions);
-          $rule->logEdit($request->getUser()->getPHID(), $edit_action);
-        $rule->saveTransaction();
-
-      } catch (AphrontDuplicateKeyQueryException $ex) {
-        $e_name = pht('Not Unique');
-        $errors[] = pht('Rule name is not unique. Choose a unique name.');
-      }
+      $rule->openTransaction();
+        $rule->save();
+        $rule->saveConditions($conditions);
+        $rule->saveActions($actions);
+        $rule->logEdit($request->getUser()->getPHID(), $edit_action);
+      $rule->saveTransaction();
     }
 
     return array($e_name, $errors);

src/applications/herald/storage/HeraldRule.php

···
         'contentType' => 'text255',
         'mustMatchAll' => 'bool',
         'configVersion' => 'uint32',
-        'ruleType' => 'text255',
+        'ruleType' => 'text32',
         'isDisabled' => 'uint32',
         'triggerObjectPHID' => 'phid?',
 
···
         'repetitionPolicy' => 'uint32?',
       ),
       self::CONFIG_KEY_SCHEMA => array(
-        'key_phid' => null,
-        'phid' => array(
-          'columns' => array('phid'),
-          'unique' => true,
-        ),
-        'authorPHID' => array(
-          'columns' => array('authorPHID', 'name'),
-          'unique' => true,
+        'key_author' => array(
+          'columns' => array('authorPHID'),
         ),
-        'IDX_RULE_TYPE' => array(
+        'key_ruletype' => array(
           'columns' => array('ruleType'),
         ),
         'key_trigger' => array(

src/applications/herald/storage/transcript/HeraldTranscript.php

···
         'host' => 'text255',
         'duration' => 'double',
         'dryRun' => 'bool',
+        'garbageCollected' => 'bool',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_phid' => null,

src/applications/macro/storage/PhabricatorFileImageMacro.php

···
     return array(
       self::CONFIG_AUX_PHID  => true,
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
+        'name' => 'text128',
         'authorPHID' => 'phid?',
         'isDisabled' => 'bool',
         'audioPHID' => 'phid?',

src/applications/mailinglists/storage/PhabricatorMetaMTAMailingList.php

···
     return array(
       self::CONFIG_AUX_PHID => true,
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
-        'email' => 'text255',
+        'name' => 'text128',
+        'email' => 'text128',
         'uri' => 'text255?',
       ),
       self::CONFIG_KEY_SCHEMA => array(

src/applications/metamta/storage/PhabricatorMetaMTAMail.php

···
         'parameters'  => self::SERIALIZATION_JSON,
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
-        'status' => 'text255',
+        'status' => 'text32',
         'relatedPHID' => 'phid?',
 
         // T6203/NULLABILITY

src/applications/owners/storage/PhabricatorOwnersPackage.php

···
       self::CONFIG_TIMESTAMPS => false,
       self::CONFIG_AUX_PHID   => true,
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
+        'name' => 'text128',
         'originalName' => 'text255',
         'description' => 'text',
         'primaryOwnerPHID' => 'phid?',

src/applications/people/storage/PhabricatorExternalAccount.php

···
         'accountType' => 'text16',
         'accountDomain' => 'text64',
         'accountSecret' => 'text?',
-        'accountID' => 'text160',
+        'accountID' => 'text64',
         'displayName' => 'text255?',
         'username' => 'text255?',
         'realName' => 'text255?',

src/applications/people/storage/PhabricatorUserSchemaSpec.php

···
       ),
       array(
         'token' => array(
-          'columns' => array('token'),
+          'columns' => array('token(128)'),
         ),
         'userID' => array(
           'columns' => array('userID'),

src/applications/phame/storage/PhameBlog.php

···
       self::CONFIG_COLUMN_SCHEMA => array(
         'name' => 'text64',
         'description' => 'text',
-        'domain' => 'text255?',
+        'domain' => 'text128?',
 
         // T6203/NULLABILITY
         // These policies should always be non-null.

src/applications/phragment/storage/PhragmentFragment.php

···
     return array(
       self::CONFIG_AUX_PHID => true,
       self::CONFIG_COLUMN_SCHEMA => array(
-        'path' => 'text255',
+        'path' => 'text128',
         'depth' => 'uint32',
         'latestVersionPHID' => 'phid?',
       ),

src/applications/phragment/storage/PhragmentSnapshot.php

···
     return array(
       self::CONFIG_AUX_PHID => true,
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
+        'name' => 'text128',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'key_name' => array(

src/applications/phriction/storage/PhrictionContent.php

···
           'columns' => array('authorPHID'),
         ),
         'slug' => array(
-          'columns' => array('slug(255)'),
+          'columns' => array('slug'),
         ),
       ),
     ) + parent::getConfiguration();

src/applications/project/storage/PhabricatorProject.php

···
         'subprojectPHIDs' => self::SERIALIZATION_JSON,
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
+        'name' => 'text128',
         'status' => 'text32',
         'phrictionSlug' => 'text128?',
         'isMembershipLocked' => 'bool',

src/applications/releeph/storage/ReleephProject.php

···
         'details' => self::SERIALIZATION_JSON,
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
+        'name' => 'text128',
         'trunkBranch' => 'text255',
         'isActive' => 'bool',
       ),

src/applications/repository/storage/PhabricatorRepository.php

···
           'unique' => true,
         ),
         'key_name' => array(
-          'columns' => array('name'),
+          'columns' => array('name(128)'),
         ),
         'key_vcs' => array(
           'columns' => array('versionControlSystem'),

src/applications/repository/storage/PhabricatorRepositoryArcanistProject.php

···
         'symbolIndexProjects'  => self::SERIALIZATION_JSON,
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
+        'name' => 'text128',
         'repositoryID' => 'id?',
       ),
       self::CONFIG_KEY_SCHEMA => array(

src/applications/repository/storage/PhabricatorRepositoryBranch.php

···
   public function getConfiguration() {
     return array(
       self::CONFIG_COLUMN_SCHEMA => array(
-        'name' => 'text255',
+        'name' => 'text128',
         'lintCommit' => 'text40?',
       ),
       self::CONFIG_KEY_SCHEMA => array(

src/applications/repository/storage/PhabricatorRepositoryCommitData.php

···
         'commitDetails' => self::SERIALIZATION_JSON,
       ),
       self::CONFIG_COLUMN_SCHEMA => array(
-        'authorName' => 'text255',
+        'authorName' => 'text',
         'commitMessage' => 'text',
       ),
       self::CONFIG_KEY_SCHEMA => array(
         'commitID' => array(
           'columns' => array('commitID'),
           'unique' => true,
-        ),
-        'authorName' => array(
-          'columns' => array('authorName'),
         ),
       ),
     ) + parent::getConfiguration();

src/applications/repository/storage/PhabricatorRepositorySchemaSpec.php

···
       id(new PhabricatorRepository())->getApplicationName(),
       PhabricatorRepository::TABLE_BADCOMMIT,
       array(
-        'fullCommitName' => 'text255',
+        'fullCommitName' => 'text64',
         'description' => 'text',
       ),
       array(

src/applications/search/storage/document/PhabricatorSearchDocumentField.php

···
         'phid' => array(
           'columns' => array('phid'),
         ),
-
-        // NOTE: This is a fulltext index! Be careful!
         'corpus' => array(
           'columns' => array('corpus'),
+          'type' => 'FULLTEXT',
         ),
       ),
     ) + parent::getConfiguration();

src/infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php

···
   public function getConfiguration() {
     return array(
       self::CONFIG_COLUMN_SCHEMA => array(
-        'taskClass' => 'text255',
-        'leaseOwner' => 'text255?',
+        'taskClass' => 'text64',
+        'leaseOwner' => 'text64?',
         'leaseExpires' => 'epoch?',
         'failureCount' => 'uint32',
         'failureTime' => 'epoch?',

src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php

···
   }
 
   public function execute(PhutilArgumentParser $args) {
+    $force = $args->getArg('force');
+
     $this->requireAllPatchesApplied();
-    $this->adjustSchemata();
+    $this->adjustSchemata($force);
     return 0;
   }
 
···
     return array($comp, $expect, $actual);
   }
 
-  private function adjustSchemata() {
+  private function adjustSchemata($force) {
     $console = PhutilConsole::getConsole();
 
     $console->writeOut(
···
 
     $table->draw();
 
-    $console->writeOut(
-      "\n%s\n",
-      pht(
-        "Found %s issues(s) with schemata, detailed above.\n\n".
-        "You can review issues in more detail from the web interface, ".
-        "in Config > Database Status.\n\n".
-        "MySQL needs to copy table data to make some adjustments, so these ".
-        "migrations may take some time.".
+    if (!$force) {
+      $console->writeOut(
+        "\n%s\n",
+        pht(
+          "Found %s issues(s) with schemata, detailed above.\n\n".
+          "You can review issues in more detail from the web interface, ".
+          "in Config > Database Status.\n\n".
+          "MySQL needs to copy table data to make some adjustments, so these ".
+          "migrations may take some time.".
 
-        // TODO: Remove warning once this stabilizes.
-        "\n\n".
-        "WARNING: This workflow is new and unstable. If you continue, you ".
-        "may unrecoverably destory data. Make sure you have a backup before ".
-        "you proceed.",
+          // TODO: Remove warning once this stabilizes.
+          "\n\n".
+          "WARNING: This workflow is new and unstable. If you continue, you ".
+          "may unrecoverably destory data. Make sure you have a backup before ".
+          "you proceed.",
 
-        new PhutilNumber(count($adjustments))));
+          new PhutilNumber(count($adjustments))));
 
-    $prompt = pht('Fix these schema issues?');
-    if (!phutil_console_confirm($prompt, $default_no = true)) {
-      return;
+      $prompt = pht('Fix these schema issues?');
+      if (!phutil_console_confirm($prompt, $default_no = true)) {
+        return;
+      }
     }
 
     $console->writeOut(
···
               break;
             case 'key':
               if (($phase == 0) && $adjust['exists']) {
+                if ($adjust['name'] == 'PRIMARY') {
+                  $key_name = 'PRIMARY KEY';
+                } else {
+                  $key_name = qsprintf($conn, 'KEY %T', $adjust['name']);
+                }
+
                 queryfx(
                   $conn,
-                  'ALTER TABLE %T.%T DROP KEY %T',
+                  'ALTER TABLE %T.%T DROP %Q',
                   $adjust['database'],
                   $adjust['table'],
-                  $adjust['name']);
+                  $key_name);
               }
 
               if (($phase == 2) && $adjust['keep']) {
+                // Different keys need different creation syntax. Notable
+                // special cases are primary keys and fulltext keys.
+                if ($adjust['name'] == 'PRIMARY') {
+                  $key_name = 'PRIMARY KEY';
+                } else if ($adjust['indexType'] == 'FULLTEXT') {
+                  $key_name = qsprintf($conn, 'FULLTEXT %T', $adjust['name']);
+                } else {
+                  if ($adjust['unique']) {
+                    $key_name = qsprintf(
+                      $conn,
+                      'UNIQUE KEY %T',
+                      $adjust['name']);
+                  } else {
+                    $key_name = qsprintf(
+                      $conn,
+                      '/* NONUNIQUE */ KEY %T',
+                      $adjust['name']);
+                  }
+                }
+
                 queryfx(
                   $conn,
-                  'ALTER TABLE %T.%T ADD %Q KEY %T (%Q)',
+                  'ALTER TABLE %T.%T ADD %Q (%Q)',
                   $adjust['database'],
                   $adjust['table'],
-                  $adjust['unique'] ? 'UNIQUE' : '/* NONUNIQUE */',
-                  $adjust['name'],
+                  $key_name,
                   implode(', ', $adjust['columns']));
               }
               break;
···
     foreach ($failed as $failure) {
       list($adjust, $ex) = $failure;
 
-      $pieces = array_select_keys($adjust, array('database', 'table', 'naeme'));
+      $pieces = array_select_keys($adjust, array('database', 'table', 'name'));
       $pieces = array_filter($pieces);
       $target = implode('.', $pieces);
 
···
     $issue_missingkey = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY;
     $issue_columns = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS;
     $issue_unique = PhabricatorConfigStorageSchema::ISSUE_UNIQUE;
-
+    $issue_longkey = PhabricatorConfigStorageSchema::ISSUE_LONGKEY;
 
     $adjustments = array();
     foreach ($comp->getDatabases() as $database_name => $database) {
···
             $issues[] = $issue_unique;
           }
 
+          // NOTE: We can't really fix this, per se, but we may need to remove
+          // the key to change the column type. In the best case, the new
+          // column type won't be overlong and recreating the key really will
+          // fix the issue. In the worst case, we get the right column type and
+          // lose the key, which is still better than retaining the key having
+          // the wrong column type.
+          if ($key->hasIssue($issue_longkey)) {
+            $issues[] = $issue_longkey;
+          }
+
           if ($issues) {
             $adjustment = array(
               'kind' => 'key',
···
               $adjustment += array(
                 'columns' => $expect_key->getColumnNames(),
                 'unique' => $expect_key->getUnique(),
+                'indexType' => $expect_key->getIndexType(),
               );
             }
 

src/infrastructure/storage/schema/PhabricatorStorageSchemaSpec.php

···
       'meta_data',
       'patch_status',
       array(
-        'patch' => 'text255',
+        'patch' => 'text128',
         'applied' => 'uint32',
       ),
       array(