osu-web / app / Console / Commands / EsIndexWiki.php

<?php

// Copyright (c) ppy Pty Ltd <[e-mail]>. Licensed under the GNU Affero General Public License v3.0.
// See the LICENCE file in the repository root for full licence text.

namespace App\Console\Commands;

use App\Libraries\Elasticsearch\Es;
use App\Libraries\Elasticsearch\Indexing;
use App\Libraries\Elasticsearch\Search;
use App\Libraries\Elasticsearch\Sort;
use App\Libraries\OsuWiki;
use App\Libraries\Search\BasicSearch;
use App\Libraries\Wiki\WikiSitemap;
use App\Models\Wiki\Page;
use Illuminate\Console\Command;

class EsIndexWiki extends Command
{
    /**
     * The name and signature of the console command.
     *
     * @var string
     */
    protected $signature = 'es:index-wiki {--create-only} {--inplace} {--cleanup}';

    /**
     * The console command description.
     *
     * @var string
     */
    protected $description = 'Re-indexes wiki pages';

    private $cleanup;
    private $createOnly;
    private $indexName;
    private $indicesToRemove;
    private $inplace;

    public function handle()
    {
        $this->readOptions();

        $alias = Page::esIndexName();
        $oldIndices = Indexing::getOldIndices($alias);

        if (!$this->inplace || empty($oldIndices)) {
            $this->indexName = Page::esTimestampedIndexName();
        } else {
            $this->indexName = $oldIndices[0];
        }

        $this->indicesToRemove = array_filter($oldIndices, function ($index) {
            // because removing the index we just wrote to would be silly.
            return $this->indexName !== $index;
        });

        $continue = $this->starterMessage();
        if (!$continue) {
            $this->error('User aborted!');
            return;
        }

        if (!Es::getClient()->indices()->exists(['index' => [$this->indexName]])) {
            $this->info("Creating '{$this->indexName}'...");
            Page::esCreateIndex($this->indexName);
        }

        $this->reindex();

        Indexing::updateAlias($alias, $this->indexName);

        $this->updateSitemap();

        $this->finish();
    }

    private function finish()
    {
        if (!$this->cleanup) {
            return;
        }

        foreach ($this->indicesToRemove as $index) {
            $this->warn("Removing '{$index}'...");
            Indexing::deleteIndex($index);
        }
    }

    private function newBaseSearch(): Search
    {
        return (new BasicSearch($this->indexName))
            ->query(['match_all' => new \stdClass()])
            ->sort([new Sort('path.keyword', 'asc'), new Sort('locale.keyword', 'asc')])
            ->source(false);
    }

    private function readOptions()
    {
        $this->createOnly = $this->option('create-only');
        $this->inplace = $this->option('inplace');
        $this->cleanup = $this->option('cleanup');
    }

    private function reindex()
    {
        if ($this->createOnly) {
            return;
        }
        // for storing the paths as keys; the values don't matter in practise.
        $paths = [];

        $this->line('Fetching page list from Github...');
        OsuWiki::getPageList()->each(function ($path) use (&$paths) {
            $path = str_replace_first('wiki/', '', $path);
            $paths[$path] = false;
        });

        $this->line(count($paths).' pages found');

        if ($this->inplace) {
            $this->line('Fetching existing list...');
            $cursor = ['', '']; // number of params for initial cursor must match number of sorts used.
            while ($cursor !== null) {
                $search = $this->newBaseSearch()->searchAfter(array_values($cursor));
                $response = $search->response();

                foreach ($response as $hit) {
                    $paths[$hit['_id']] = true;
                }

                $cursor = $search->getSortCursor();
            }
        }

        $total = count($paths);
        $this->line("Total: {$total} documents");
        $bar = $this->output->createProgressBar($total);

        foreach ($paths as $path => $_inEs) {
            $pagePath = Page::parsePagePath($path);
            $page = new Page($pagePath['path'], $pagePath['locale']);
            $page->sync(true, $this->indexName);

            if (!$page->isVisible()) {
                $this->warn("delete {$pagePath['locale']}: {$pagePath['path']}");
                $page->esDeleteDocument(['index' => $this->indexName]);
            }

            $bar->advance();
        }

        $bar->finish();
        $this->line('');
    }

    private function starterMessage()
    {
        if ($this->cleanup) {
            $this->warn(
                "The following indices will be deleted on completion!\n"
                .implode("\n", $this->indicesToRemove)
            );
        }

        return $this->option('no-interaction') || $this->confirm("This index to {$this->indexName}, begin indexing?", true);
    }

    private function updateSitemap()
    {
        $this->line('Updating wiki sitemap...');
        WikiSitemap::expire();
        WikiSitemap::get();
    }
}