Document configuration of file upload limits

Summary: I have a patch which makes uploads all fancy and adds progress bars, but document the landscape first since it's quite complicated.

Test Plan: Generated, read docs. Configured `storage.upload-size-limit` to various values.

Reviewers: btrahan, vrana

Reviewed By: vrana

CC: aran

Maniphest Tasks: T875

Differential Revision: https://secure.phabricator.com/D2381

conf/default.conf.php

···
   // fits within configured limits.
   'storage.engine-selector' => 'PhabricatorDefaultFileStorageEngineSelector',
 
+  // Set the size of the largest file a user may upload. This is used to render
+  // text like "Maximum file size: 10MB" on interfaces where users can upload
+  // files, and files larger than this size will be rejected.
+  //
+  // Specify this limit in bytes, or using a "K", "M", or "G" suffix.
+  //
+  // NOTE: Setting this to a large size is NOT sufficient to allow users to
+  // upload large files. You must also configure a number of other settings. To
+  // configure file upload limits, consult the article "Configuring File Upload
+  // Limits" in the documentation. Once you've configured some limit across all
+  // levels of the server, you can set this limit to an appropriate value and
+  // the UI will then reflect the actual configured limit.
+  'storage.upload-size-limit'   => null,
+
   // Phabricator puts databases in a namespace, which defualts to "phabricator"
   // -- for instance, the Differential database is named
   // "phabricator_differential" by default. You can change this namespace if you

src/__phutil_library_map__.php

···
     'PhabricatorUIListFilterExample' => 'applications/uiexample/examples/listfilter',
     'PhabricatorUIPagerExample' => 'applications/uiexample/examples/pager',
     'PhabricatorUITooltipExample' => 'applications/uiexample/examples/tooltip',
+    'PhabricatorUnitsTestCase' => 'view/utils/__tests__',
     'PhabricatorUser' => 'applications/people/storage/user',
     'PhabricatorUserAccountSettingsPanelController' => 'applications/people/controller/settings/panels/account',
     'PhabricatorUserConduitSettingsPanelController' => 'applications/people/controller/settings/panels/conduit',
···
     'javelin_render_tag' => 'infrastructure/javelin/markup',
     'phabricator_date' => 'view/utils',
     'phabricator_datetime' => 'view/utils',
+    'phabricator_format_bytes' => 'view/utils',
     'phabricator_format_local_time' => 'view/utils',
     'phabricator_format_relative_time' => 'view/utils',
     'phabricator_format_units_generic' => 'view/utils',
     'phabricator_on_relative_date' => 'view/utils',
+    'phabricator_parse_bytes' => 'view/utils',
     'phabricator_relative_date' => 'view/utils',
     'phabricator_render_form' => 'infrastructure/javelin/markup',
     'phabricator_time' => 'view/utils',
···
     'PhabricatorUIListFilterExample' => 'PhabricatorUIExample',
     'PhabricatorUIPagerExample' => 'PhabricatorUIExample',
     'PhabricatorUITooltipExample' => 'PhabricatorUIExample',
+    'PhabricatorUnitsTestCase' => 'PhabricatorTestCase',
     'PhabricatorUser' => 'PhabricatorUserDAO',
     'PhabricatorUserAccountSettingsPanelController' => 'PhabricatorUserSettingsPanelController',
     'PhabricatorUserConduitSettingsPanelController' => 'PhabricatorUserSettingsPanelController',

src/applications/files/controller/list/PhabricatorFileListController.php

···
     $request = $this->getRequest();
     $user = $request->getUser();
 
+    $limit_text = PhabricatorFileUploadView::renderUploadLimit();
+
     if ($this->useBasicUploader()) {
 
       $upload_panel = new PhabricatorFileUploadView();
···
 
       $upload_panel = new AphrontPanelView();
       $upload_panel->setHeader('Upload Files');
+      $upload_panel->setCaption($limit_text);
       $upload_panel->setCreateButton('Basic Uploader',
         $request->getRequestURI()->setQueryParam('basic_uploader', true)
       );

src/applications/files/view/upload/PhabricatorFileUploadView.php

···
 <?php
 
 /*
- * Copyright 2011 Facebook, Inc.
+ * Copyright 2012 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
···
         id(new AphrontFormFileControl())
           ->setLabel('File')
           ->setName('file')
-          ->setError(true))
+          ->setError(true)
+          ->setCaption(self::renderUploadLimit()))
       ->appendChild(
         id(new AphrontFormTextControl())
           ->setLabel('Name')
···
     $panel->setWidth(AphrontPanelView::WIDTH_FULL);
 
     return $panel->render();
+  }
+
+  public static function renderUploadLimit() {
+    $limit = PhabricatorEnv::getEnvConfig('storage.upload-size-limit');
+    $limit = phabricator_parse_bytes($limit);
+    if ($limit) {
+      $formatted = phabricator_format_bytes($limit);
+      return 'Maximum file size: '.phutil_escape_html($formatted);
+    }
+
+    $doc_href = PhabricatorEnv::getDocLink(
+      'articles/Configuring_File_Upload_Limits.html');
+    $doc_link = phutil_render_tag(
+      'a',
+      array(
+        'href'    => $doc_href,
+        'target'  => '_blank',
+      ),
+      'Configuring File Upload Limits');
+
+    return 'Upload limit is not configured, see '.$doc_link.'.';
   }
 }
 

src/applications/files/view/upload/__init__.php

···
 
 
 
+phutil_require_module('phabricator', 'infrastructure/env');
 phutil_require_module('phabricator', 'view/base');
 phutil_require_module('phabricator', 'view/form/base');
 phutil_require_module('phabricator', 'view/form/control/file');
 phutil_require_module('phabricator', 'view/form/control/submit');
 phutil_require_module('phabricator', 'view/form/control/text');
 phutil_require_module('phabricator', 'view/layout/panel');
+phutil_require_module('phabricator', 'view/utils');
 
+phutil_require_module('phutil', 'markup');
 phutil_require_module('phutil', 'utils');
 
 

src/docs/configuration/configuring_file_storage.diviner

···
 
 Continue by:
 
+  - configuring file size upload limits with
+    @{article:Configuring File Upload Limits}; or
   - returning to the @{article:Configuration Guide}.

src/docs/configuration/configuring_file_upload_limits.diviner

···
+@title Configuring File Upload Limits
+@group config
+
+Explains limits on file upload sizes.
+
+= Overview =
+
+File uploads are limited by a large number of pieces of configuration, at
+multiple layers of the application. Generally, the minimum value of all the
+limits is the effective one. To upload large files, you need to increase all
+the limits above the maximum file size you want to support. The settings which
+limit uploads are:
+
+  - **HTTP Server**: The HTTP server may set a limit on the maximum request
+    size. If you exceed this limit, you'll see a default server page with an
+    HTTP error. These directives limit the total size of the request body,
+    so they must be somewhat larger than the desired maximum filesize.
+    - **Apache**: Apache limits requests with the Apache `LimitRequestBody`
+      directive.
+    - **nginx**: nginx limits requests with the nginx `client_max_body_size`
+      directive. This often defaults to `1M`.
+    - **lighttpd**: lighttpd limits requests with the lighttpd
+      `server.max-request-size` directive.
+  - **PHP**: PHP has several directives which limit uploads. These directives
+    are found in `php.ini`.
+    - **upload_max_filesize**: Maximum file size PHP will accept in a file
+      upload. If you exceed this, Phabricator will give you a useful error. This
+      often defaults to `2M`.
+    - **post_max_size**: Maximum POST request size PHP will accept. If you
+      exceed this, Phabricator will give you a useful error. This often defaults
+      to `8M`.
+    - **memory_limit**: For some uploads, file data will be read into memory
+      before Phabricator can adjust the memory limit. If you exceed this, PHP
+      may give you a useful error, depending on your configuration.
+    - **max_input_vars**: When files are uploaded via HTML5 drag and drop file
+      upload APIs, PHP parses the file body as though it contained normal POST
+      parameters, and may trigger `max_input_vars` if a file has a lot of
+      brackets in it. You may need to set it to some astronomically high value.
+  - **Storage Engines**: Some storage engines can be configured not to accept
+    files over a certain size. To upload a file, you must have at least one
+    configured storage engine which can accept it. Phabricator should give you
+    useful errors if any of these fail.
+    - **MySQL Engine**: Upload size is limited by the Phabricator setting
+      `storage.mysql-engine.max-size`, which is in turn limited by the MySQL
+      setting `max_allowed_packet`. This often defaults to `1M`.
+    - **Amazon S3**: Upload size is limited by Phabricator's implementation to
+      `5G`.
+    - **Local Disk**: Upload size is limited only by free disk space.
+  - **Resource Constraints**: File uploads are limited by resource constraints
+    on the application server. In particular, some uploaded files are written
+    to disk in their entirety before being moved to storage engines, and all
+    uploaded files are read into memory before being moved. These hard limits
+    should be large for most servers, but will fundamentally prevent Phabricator
+    from processing truly enormous files (GB/TB scale). Phabricator is probably
+    not the best application for this in any case.
+  - **Phabricator Master Limit**: The master limit, `storage.upload-size-limit`,
+    is used to show upload limits in the UI.
+
+Phabricator can't read some of these settings, so it can't figure out what the
+current limit is or be much help at all in configuring it. Thus, you need to
+manually configure all of these limits and then tell Phabricator what you set
+them to. Follow these steps:
+
+  - Pick some limit you want to set, like `100M`.
+  - Configure all of the settings mentioned above to be a bit bigger than the
+    limit you want to enforce (**note that there are some security implications
+    to raising these limits**; principally, your server may become easier to
+    attack with a denial-of-service).
+  - Set `storage.upload-size-limit` to the limit you want.
+  - The UI should now show your limit.
+  - Upload a big file to make sure it works.
+
+= Next Steps =
+
+Continue by:
+
+  - configuring file storage with @{article:Configuring File Storage}; or
+  - retuning to the @{article:Configuration Guide}.

src/view/utils/__tests__/PhabricatorUnitsTestCase.php

···
+<?php
+
+/*
+ * Copyright 2012 Facebook, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+final class PhabricatorUnitsTestCase extends PhabricatorTestCase {
+
+  public function testByteFormatting() {
+    $tests = array(
+      1               => '1 B',
+      1000            => '1 KB',
+      1000000         => '1 MB',
+      10000000        => '10 MB',
+      100000000       => '100 MB',
+      1000000000      => '1 GB',
+      1000000000000   => '1 TB',
+      999             => '999 B',
+    );
+
+    foreach ($tests as $input => $expect) {
+      $this->assertEqual(
+        $expect,
+        phabricator_format_bytes($input),
+        'phabricator_format_bytes('.$input.')');
+    }
+  }
+
+  public function testByteParsing() {
+    $tests = array(
+      '1'             => 1,
+      '1k'            => 1000,
+      '1K'            => 1000,
+      '1kB'           => 1000,
+      '1Kb'           => 1000,
+      '1KB'           => 1000,
+      '1MB'           => 1000000,
+      '1GB'           => 1000000000,
+      '1TB'           => 1000000000000,
+      '1.5M'          => 1500000,
+      '1 000'         => 1000,
+      '1,234.56 KB'   => 1234560,
+    );
+
+    foreach ($tests as $input => $expect) {
+      $this->assertEqual(
+        $expect,
+        phabricator_parse_bytes($input),
+        'phabricator_parse_bytes('.$input.')');
+    }
+  }
+
+}

src/view/utils/__tests__/__init__.php

···
 
 
 phutil_require_source('PhabricatorLocalTimeTestCase.php');
+phutil_require_source('PhabricatorUnitsTestCase.php');

src/view/utils/viewutils.php

···
     $precision = 0);
 }
 
+
+/**
+ * Format a byte count for human consumption, e.g. "10MB" instead of
+ * "10000000".
+ *
+ * @param int Number of bytes.
+ * @return string Human-readable description.
+ */
+function phabricator_format_bytes($bytes) {
+  return phabricator_format_units_generic(
+    $bytes,
+    // NOTE: Using the SI version of these units rather than the 1024 version.
+    array(1000, 1000, 1000, 1000, 1000),
+    array('B', 'KB', 'MB', 'GB', 'TB', 'PB'),
+    $precision = 0);
+}
+
+
+/**
+ * Parse a human-readable byte description (like "6MB") into an integer.
+ *
+ * @param string  Human-readable description.
+ * @return int    Number of represented bytes.
+ */
+function phabricator_parse_bytes($input) {
+  $bytes = trim($input);
+  if (!strlen($bytes)) {
+    return null;
+  }
+
+  // NOTE: Assumes US-centric numeral notation.
+  $bytes = preg_replace('/[ ,]/', '', $bytes);
+
+  $matches = null;
+  if (!preg_match('/^(?:\d+(?:[.]\d+)?)([kmgtp]?)b?$/i', $bytes, $matches)) {
+    throw new Exception("Unable to parse byte size '{$input}'!");
+  }
+
+  $scale = array(
+    'k' => 1000,
+    'm' => 1000 * 1000,
+    'g' => 1000 * 1000 * 1000,
+    't' => 1000 * 1000 * 1000 * 1000,
+  );
+
+  $bytes = (float)$bytes;
+  if ($matches[1]) {
+    $bytes *= $scale[strtolower($matches[1])];
+  }
+
+  return (int)$bytes;
+}
+
+
 function phabricator_format_units_generic(
   $n,
   array $scales,
···
 
   $scale = array_shift($scales);
   $label = array_shift($labels);
-  while ($n > $scale && count($labels)) {
+  while ($n >= $scale && count($labels)) {
     $remainder += ($n % $scale) * $accum;
     $n /= $scale;
     $accum *= $scale;

webroot/index.php

···
 
 error_reporting(E_ALL | E_STRICT);
 
+if ($_SERVER['REQUEST_METHOD'] == 'POST' && !$_POST) {
+  $size = ini_get('post_max_size');
+  phabricator_fatal(
+    "Request size exceeds PHP 'post_max_size' ('{$size}').");
+}
+
 $required_version = '5.2.3';
 if (version_compare(PHP_VERSION, $required_version) < 0) {
   phabricator_fatal_config_error(