Add basic support for a "Must Encrypt" mail flag which prevents unsecured content transmission

Summary:
Ref T13053. See PHI291. For particularly sensitive objects (like security issues), installs may reasonably wish to prevent details from being sent in plaintext over email.

This adds a "Must Encrypt" mail behavior, which discards mail content and all identifying details, replacing it with a link to the `/mail/` application. Users can follow the link to view the message over HTTPS.

The flag discards body content, attachments, and headers which imply things about the content of the object. It retains threading headers and headers which may uniquely identify the object as long as they don't disclose anyting about the content.

The `bin/mail list-outbound` command now flags these messages with a `#` mark.

The `bin/mail show-outbound` command now shows sent/suppressed headers and the body content as delivered (if it differs from the original body content).

The `/mail/` web UI now shows a tag for messages marked with this flag.

For now, there is no way to actually set this flag on mail.

Test Plan:
- Forced this flag on, made comments and took actions to send mail.
- Reviewed mail with `bin/mail` and `/mail/` in the web UI, saw all content information omitted.

Reviewers: amckinley

Reviewed By: amckinley

Maniphest Tasks: T13053

Differential Revision: https://secure.phabricator.com/D18983

src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php

···
     $color = PhabricatorMailOutboundStatus::getStatusColor($status);
     $header->setStatus($icon, $color, $name);
 
+    if ($mail->getMustEncrypt()) {
+      Javelin::initBehavior('phabricator-tooltips');
+      $header->addTag(
+        id(new PHUITagView())
+          ->setType(PHUITagView::TYPE_SHADE)
+          ->setColor('blue')
+          ->setName(pht('Must Encrypt'))
+          ->setIcon('fa-shield blue')
+          ->addSigil('has-tooltip')
+          ->setMetadata(
+            array(
+              'tip' => pht(
+                'Message content can only be transmitted over secure '.
+                'channels.'),
+            )));
+    }
+
     $crumbs = $this->buildApplicationCrumbs()
       ->addTextCrumb(pht('Mail %d', $mail->getID()))
       ->setBorder(true);

src/applications/metamta/management/PhabricatorMailManagementListOutboundWorkflow.php

···
     $table = id(new PhutilConsoleTable())
       ->setShowHeader(false)
       ->addColumn('id',      array('title' => pht('ID')))
+      ->addColumn('encrypt', array('title' => pht('#')))
       ->addColumn('status',  array('title' => pht('Status')))
       ->addColumn('subject', array('title' => pht('Subject')));
 
···
 
       $table->addRow(array(
         'id'      => $mail->getID(),
+        'encrypt' => ($mail->getMustEncrypt() ? '#' : ' '),
         'status'  => PhabricatorMailOutboundStatus::getStatusName($status),
         'subject' => $mail->getSubject(),
       ));

src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php

···
 
       $info = array();
 
-      $info[] = pht('PROPERTIES');
+      $info[] = $this->newSectionHeader(pht('PROPERTIES'));
       $info[] = pht('ID: %d', $message->getID());
       $info[] = pht('Status: %s', $message->getStatus());
       $info[] = pht('Related PHID: %s', $message->getRelatedPHID());
···
 
       $ignore = array(
         'body' => true,
+        'body.sent' => true,
         'html-body' => true,
         'headers' => true,
         'attachments' => true,
         'headers.sent' => true,
+        'headers.unfiltered' => true,
         'authors.sent' => true,
       );
 
       $info[] = null;
-      $info[] = pht('PARAMETERS');
+      $info[] = $this->newSectionHeader(pht('PARAMETERS'));
       $parameters = $message->getParameters();
       foreach ($parameters as $key => $value) {
         if (isset($ignore[$key])) {
···
       }
 
       $info[] = null;
-      $info[] = pht('HEADERS');
+      $info[] = $this->newSectionHeader(pht('HEADERS'));
 
       $headers = $message->getDeliveredHeaders();
-      if (!$headers) {
+      $unfiltered = $message->getUnfilteredHeaders();
+      if (!$unfiltered) {
         $headers = $message->generateHeaders();
+        $unfiltered = $headers;
       }
 
+      $header_map = array();
       foreach ($headers as $header) {
         list($name, $value) = $header;
-        $info[] = "{$name}: {$value}";
+        $header_map[$name.':'.$value] = true;
+      }
+
+      foreach ($unfiltered as $header) {
+        list($name, $value) = $header;
+        $was_sent = isset($header_map[$name.':'.$value]);
+
+        if ($was_sent) {
+          $marker = ' ';
+        } else {
+          $marker = '#';
+        }
+
+        $info[] = "{$marker} {$name}: {$value}";
       }
 
       $attachments = idx($parameters, 'attachments');
       if ($attachments) {
         $info[] = null;
-        $info[] = pht('ATTACHMENTS');
+
+        $info[] = $this->newSectionHeader(pht('ATTACHMENTS'));
+
         foreach ($attachments as $attachment) {
           $info[] = idx($attachment, 'filename', pht('Unnamed File'));
         }
···
       $actors = $message->getDeliveredActors();
       if ($actors) {
         $info[] = null;
-        $info[] = pht('RECIPIENTS');
+
+        $info[] = $this->newSectionHeader(pht('RECIPIENTS'));
+
         foreach ($actors as $actor_phid => $actor_info) {
           $actor = idx($all_actors, $actor_phid);
           if ($actor) {
···
       }
 
       $info[] = null;
-      $info[] = pht('TEXT BODY');
+      $info[] = $this->newSectionHeader(pht('TEXT BODY'));
       if (strlen($message->getBody())) {
-        $info[] = $message->getBody();
+        $info[] = tsprintf('%B', $message->getBody());
       } else {
         $info[] = pht('(This message has no text body.)');
       }
 
+      $delivered_body = $message->getDeliveredBody();
+      if ($delivered_body !== null) {
+        $info[] = null;
+        $info[] = $this->newSectionHeader(pht('BODY AS DELIVERED'), true);
+        $info[] = tsprintf('%B', $delivered_body);
+      }
+
       $info[] = null;
-      $info[] = pht('HTML BODY');
+      $info[] = $this->newSectionHeader(pht('HTML BODY'));
       if (strlen($message->getHTMLBody())) {
         $info[] = $message->getHTMLBody();
         $info[] = null;
···
       if ($message_key != $last_key) {
         $console->writeOut("\n%s\n\n", str_repeat('-', 80));
       }
+    }
+  }
+
+  private function newSectionHeader($label, $emphasize = false) {
+    if ($emphasize) {
+      return tsprintf('**<bg:yellow> %s </bg>**', $label);
+    } else {
+      return tsprintf('**<bg:blue> %s </bg>**', $label);
     }
   }
 

src/applications/metamta/storage/PhabricatorMetaMTAMail.php

···
   public function __construct() {
 
     $this->status     = PhabricatorMailOutboundStatus::STATUS_QUEUE;
-    $this->parameters = array('sensitive' => true);
+    $this->parameters = array(
+      'sensitive' => true,
+      'mustEncrypt' => false,
+    );
 
     parent::__construct();
   }
···
     return $this->getParam('sensitive', true);
   }
 
+  public function setMustEncrypt($bool) {
+    $this->setParam('mustEncrypt', $bool);
+    return $this;
+  }
+
+  public function getMustEncrypt() {
+    return $this->getParam('mustEncrypt', false);
+  }
+
   public function setHTMLBody($html) {
     $this->setParam('html-body', $html);
     return $this;
···
       unset($params['is-first-message']);
 
       $is_threaded = (bool)idx($params, 'thread-id');
+      $must_encrypt = $this->getMustEncrypt();
 
       $reply_to_name = idx($params, 'reply-to-name', '');
       unset($params['reply-to-name']);
···
               mpull($cc_actors, 'getEmailAddress'));
             break;
           case 'attachments':
+            // If the mail content must be encrypted, don't add attachments.
+            if ($must_encrypt) {
+              break;
+            }
+
             $value = $this->getAttachments();
             foreach ($value as $attachment) {
               $mailer->addAttachment(
···
 
             $subject[] = trim(idx($params, 'subject-prefix'));
 
-            $vary_prefix = idx($params, 'vary-subject-prefix');
-            if ($vary_prefix != '') {
-              if ($this->shouldVarySubject($preferences)) {
-                $subject[] = $vary_prefix;
+            // If mail content must be encrypted, we replace the subject with
+            // a generic one.
+            if ($must_encrypt) {
+              $subject[] = pht('Object Updated');
+            } else {
+              $vary_prefix = idx($params, 'vary-subject-prefix');
+              if ($vary_prefix != '') {
+                if ($this->shouldVarySubject($preferences)) {
+                  $subject[] = $vary_prefix;
+                }
               }
+
+              $subject[] = $value;
             }
 
-            $subject[] = $value;
-
             $mailer->setSubject(implode(' ', array_filter($subject)));
             break;
           case 'thread-id':
···
         }
       }
 
-      $body = idx($params, 'body', '');
+      $raw_body = idx($params, 'body', '');
+      $body = $raw_body;
+      if ($must_encrypt) {
+        $parts = array();
+        $parts[] = pht(
+          'The content for this message can only be transmitted over a '.
+          'secure channel. To view the message content, follow this '.
+          'link:');
+
+        $parts[] = PhabricatorEnv::getProductionURI($this->getURI());
+
+        $body = implode("\n\n", $parts);
+      } else {
+        $body = $raw_body;
+      }
+
       $max = PhabricatorEnv::getEnvConfig('metamta.email-body-limit');
       if (strlen($body) > $max) {
         $body = id(new PhutilUTF8StringTruncator())
···
       }
       $mailer->setBody($body);
 
-      $html_emails = $this->shouldSendHTML($preferences);
-      if ($html_emails && isset($params['html-body'])) {
+      // If we sent a different message body than we were asked to, record
+      // what we actually sent to make debugging and diagnostics easier.
+      if ($body !== $raw_body) {
+        $this->setParam('body.sent', $body);
+      }
+
+      if ($must_encrypt) {
+        $send_html = false;
+      } else {
+        $send_html = $this->shouldSendHTML($preferences);
+      }
+
+      if ($send_html && isset($params['html-body'])) {
         $mailer->setHTMLBody($params['html-body']);
       }
 
       // Pass the headers to the mailer, then save the state so we can show
-      // them in the web UI.
-      foreach ($headers as $header) {
+      // them in the web UI. If the mail must be encrypted, we remove headers
+      // which are not on a strict whitelist to avoid disclosing information.
+      $filtered_headers = $this->filterHeaders($headers, $must_encrypt);
+      foreach ($filtered_headers as $header) {
         list($header_key, $header_value) = $header;
         $mailer->addHeader($header_key, $header_value);
       }
-      $this->setParam('headers.sent', $headers);
+      $this->setParam('headers.unfiltered', $headers);
+      $this->setParam('headers.sent', $filtered_headers);
 
       // Save the final deliverability outcomes and reasoning so we can
       // explain why things happened the way they did.
···
     // Some clients respect this to suppress OOF and other auto-responses.
     $headers[] = array('X-Auto-Response-Suppress', 'All');
 
-    // If the message has mailtags, filter out any recipients who don't want
-    // to receive this type of mail.
     $mailtags = $this->getParam('mailtags');
     if ($mailtags) {
       $tag_header = array();
···
       $headers[] = array('Precedence', 'bulk');
     }
 
+    if ($this->getMustEncrypt()) {
+      $headers[] = array('X-Phabricator-Must-Encrypt', 'Yes');
+    }
+
     return $headers;
   }
 
···
     return $this->getParam('headers.sent');
   }
 
+  public function getUnfilteredHeaders() {
+    $unfiltered = $this->getParam('headers.unfiltered');
+
+    if ($unfiltered === null) {
+      // Older versions of Phabricator did not filter headers, and thus did
+      // not record unfiltered headers. If we don't have unfiltered header
+      // data just return the delivered headers for compatibility.
+      return $this->getDeliveredHeaders();
+    }
+
+    return $unfiltered;
+  }
+
   public function getDeliveredActors() {
     return $this->getParam('actors.sent');
   }
···
 
   public function getDeliveredRoutingMap() {
     return $this->getParam('routingmap.sent');
+  }
+
+  public function getDeliveredBody() {
+    return $this->getParam('body.sent');
+  }
+
+  private function filterHeaders(array $headers, $must_encrypt) {
+    if (!$must_encrypt) {
+      return $headers;
+    }
+
+    $whitelist = array(
+      'In-Reply-To',
+      'Message-ID',
+      'Precedence',
+      'References',
+      'Thread-Index',
+
+      'X-Mail-Transport-Agent',
+      'X-Auto-Response-Suppress',
+
+      'X-Phabricator-Sent-This-Message',
+      'X-Phabricator-Must-Encrypt',
+    );
+
+    // NOTE: The major header we want to drop is "X-Phabricator-Mail-Tags".
+    // This header contains a significant amount of meaningful information
+    // about the object.
+
+    $whitelist_map = array();
+    foreach ($whitelist as $term) {
+      $whitelist_map[phutil_utf8_strtolower($term)] = true;
+    }
+
+    foreach ($headers as $key => $header) {
+      list($name, $value) = $header;
+      $name = phutil_utf8_strtolower($name);
+
+      if (!isset($whitelist_map[$name])) {
+        unset($headers[$key]);
+      }
+    }
+
+    return $headers;
+  }
+
+  public function getURI() {
+    return '/mail/detail/'.$this->getID().'/';
   }