rake task for importing & printing progress

Rakefile

···
 
 if ENV['ARLOG'] == '1'
   ActiveRecord::Base.logger = Logger.new(STDOUT)
+else
+  ActiveRecord::Base.logger = nil
 end

app/importer.rb

···
 require_relative 'models/user'
 
 class Importer
-  attr_accessor :post_queue
+  attr_accessor :post_queue, :report
 
   def initialize(user_did)
     @did = DID.new(user_did)
···
 
   def import_likes(time_limit = nil)
     params = { repo: @did, collection: 'app.bsky.feed.like', limit: 100 }
+    count = 0
 
     loop do
       response = @minisky.get_request('com.atproto.repo.listRecords', params)
 
       records = response['records']
       cursor = response['cursor']
+
+      count += records.length
+      @report&.update(importer: { imported_likes: count, oldest_date: Time.parse(records.last['value']['createdAt']) })
 
       process_likes(records)
       params[:cursor] = cursor
···
 
           if @post_queue
             @post_queue.push(like_stub)
+            @report&.update(queue: { length: @post_queue.length })
           end
         end
       rescue StandardError => e

app/post_downloader.rb

···
 require_relative 'models/user'
 
 class PostDownloader
+  attr_accessor :report
+
   def initialize
     @pds_cache = {}
   end
 
   def import_from_queue(queue)
+    count = 0
+
     loop do
       like = queue.pop
+      @report&.update(queue: { length: queue.length })
 
       begin
         post = import_post(like.post_uri)
         like.update!(post: post, post_uri: nil)
+        count += 1
+
+        @report&.update(downloader: { downloaded_posts: count, oldest_date: like.time })
       rescue StandardError => e
         puts "Error in PostDownloader: #{like.post_uri}: #{e}"
       end

lib/tasks/import.rake

···
+require_relative '../../app/importer'
+require_relative '../../app/post_downloader'
+
+class ImportReport
+  def initialize
+    @data = {}
+    @start = Time.now
+  end
+
+  def update(data)
+    data.each do |k, v|
+      @data[k] ||= {}
+      @data[k].update(v)
+    end
+
+    render
+  end
+
+  def render
+    print " " * 80 + "\r"
+    puts "Elapsed time: #{(Time.now - @start).to_i} s"
+    print " " * 80 + "\r"
+    puts "Importer: imported likes = #{@data.dig(:importer, :imported_likes) || 0} (until: #{@data.dig(:importer, :oldest_date)})"
+    print " " * 80 + "\r"
+    puts "Downloader: imported posts = #{@data.dig(:downloader, :downloaded_posts) || 0} (until: #{@data.dig(:downloader, :oldest_date)})"
+    print "\e[3A"
+  end
+end
+
+task :import_user do
+  unless ENV['USER']
+    raise "Required USER parameter missing"
+  end
+
+  queue = Queue.new
+  report = ImportReport.new
+
+  importer = Importer.new(ENV['USER'])
+  importer.post_queue = queue
+  importer.report = report
+
+  downloader = PostDownloader.new
+  downloader.report = report
+
+  Thread.new { downloader.import_from_queue(queue) }
+
+  importer.run_import(ENV['UNTIL'])
+end