Index tsv file on download

.gitignore

···
 /storage/htmlpurifier/
 /storage/paypal_auth.cache
 
-/database/ip2asn-combined.tsv
+/database/ip2asn.idx
+/database/ip2asn.tsv

Dockerfile.deployment

···
 COPY . .
 RUN mkdir -p bootstrap/cache storage/logs storage/framework/cache storage/framework/views storage/framework/sessions public/uploads public/uploads-avatar public/uploads-replay
 RUN composer dump-autoload
-RUN ./bin/update_ip2asn.sh
 
 ARG APP_URL
 ARG DOCS_URL
 RUN yarn production
 
+RUN php artisan ip2asn:update
 RUN php artisan scribe:generate
 
 RUN rm -rf node_modules

app/Console/Commands/Ip2AsnUpdate.php

···
+<?php
+
+// Copyright (c) ppy Pty Ltd <[e-mail]>. Licensed under the GNU Affero General Public License v3.0.
+// See the LICENCE file in the repository root for full licence text.
+
+namespace App\Console\Commands;
+
+use App\Libraries\Ip2AsnUpdater;
+use Illuminate\Console\Command;
+
+class Ip2AsnUpdate extends Command
+{
+    protected $signature = 'ip2asn:update';
+    protected $description = 'Update or initialise ip2asn database';
+
+    public function handle()
+    {
+        (new Ip2AsnUpdater())->run();
+    }
+}

app/Console/Kernel.php

···
         Commands\EsIndexScoresSetSchema::class,
         Commands\EsIndexWiki::class,
 
+        Commands\Ip2AsnUpdate::class,
+
         // modding stuff
         Commands\ModdingRankCommand::class,
 

app/Libraries/Ip2Asn.php

···
 
 class Ip2Asn
 {
-    private array $db;
-    private $fh;
+    private $dbFh;
+    private string $index;
+    private int $count;
 
     public function __construct()
     {
-        $this->fh = fopen(database_path('ip2asn-combined.tsv'), 'r');
-        if ($this->fh === false) {
+        $this->dbFh = fopen(Ip2AsnUpdater::getDbPath(), 'r');
+        $this->index = file_get_contents(Ip2AsnUpdater::getIndexPath());
+        if ($this->dbFh === false || $this->index === false) {
             throw new Exception('failed opening ip2asn database');
         }
-        $this->db = $this->parseFileForDb();
+
+        // 4 bytes per entry (int32)
+        $this->count = strlen($this->index) / 4;
     }
 
     public function lookup(string $ip): string
     {
         $start = 0;
-        $end = count($this->db) - 1;
+        $end = $this->count - 1;
         $search = inet_pton($this->prefixIPv4($ip));
 
         if ($search === false) {
···
 
         while ($start <= $end) {
             $current = (int) (($start + $end) / 2);
-            $loc = $this->db[$current];
-            fseek($this->fh, $loc);
-            $row = fgets($this->fh);
+            $loc = unpack('l', substr($this->index, $current * 4, 4))[1];
+            fseek($this->dbFh, $loc);
+            $row = fgets($this->dbFh);
             $data = explode("\t", $row, 4);
             $compare = inet_pton($this->prefixIPv4($data[1]));
             $asn = $data[2];
···
         }
 
         return $lastInnerSearchAsn ?? $asn;
-    }
-
-    private function parseFileForDb(): array
-    {
-        fseek($this->fh, 0);
-
-        $ret = [ftell($this->fh)];
-        while (($row = fgets($this->fh)) !== false) {
-            if ($row !== '') {
-                $ret[] = ftell($this->fh);
-            }
-        }
-
-        return $ret;
     }
 
     /**

app/Libraries/Ip2AsnUpdater.php

···
+<?php
+
+// Copyright (c) ppy Pty Ltd <[e-mail]>. Licensed under the GNU Affero General Public License v3.0.
+// See the LICENCE file in the repository root for full licence text.
+
+declare(strict_types=1);
+
+namespace App\Libraries;
+
+class Ip2AsnUpdater
+{
+    public static function getDbPath()
+    {
+        return database_path('ip2asn.tsv');
+    }
+
+    public static function getIndexPath()
+    {
+        return database_path('ip2asn.idx');
+    }
+
+    private string $tsv;
+
+    public function run(): void
+    {
+        $dbPath = static::getDbPath();
+        $indexPath = static::getIndexPath();
+
+        $dbExists = file_exists($dbPath);
+
+        $dbExpireTime = time() - (24 * 3600);
+        $newDb = !$dbExists || filemtime($dbPath) < $dbExpireTime;
+
+        $newIndex = !$dbExists || !file_exists($indexPath) || filemtime($dbPath) > filemtime($indexPath);
+        if (!$newDb && !$newIndex) {
+            return;
+        }
+
+        $tsv = $newDb
+            ? gzdecode(file_get_contents('https://iptoasn.com/data/ip2asn-combined.tsv.gz'))
+            : file_get_contents($dbPath);
+
+        $currentLine = 0;
+        $index = pack('l', $currentLine);
+        while (($currentLine = strpos($tsv, "\n", $currentLine)) !== false) {
+            $currentLine++;
+            if (isset($tsv[$currentLine])) {
+                $index .= pack('l', $currentLine);
+            }
+        }
+        if ($newDb) {
+            file_put_contents($dbPath, $tsv);
+        }
+        file_put_contents($indexPath, $index);
+    }
+}

bin/update_ip2asn.sh

···
-#!/bin/sh
-
-set -e
-set -u
-
-echo 'Updating ip2asn database...'
-
-outfile="$(dirname "${0}")/../database/ip2asn-combined.tsv"
-curl https://iptoasn.com/data/ip2asn-combined.tsv.gz | gzip -d > "${outfile}.new"
-mv "${outfile}.new" "${outfile}"
-
-echo 'Finished updating ip2asn database.'

build.sh

···
   echo "OSU_SKIP_ASSET_BUILD set, skipping javascript asset build."
 fi
 
-./bin/update_ip2asn.sh
+php artisan ip2asn:update

docker/development/prepare.sh

···
     cp .docker/.my.cnf.example .docker/.my.cnf
 fi
 
-./bin/update_ip2asn.sh
+if [ ! -f database/ip2asn.tsv ]; then
+    _run artisan ip2asn:update
+fi
 
 echo "Preparation completed. Adjust .env file if needed and run 'docker compose up' followed by running migration."