Don't forget to lycansubscribe

extracted import code from rake to ImportManager

Changed files
+71 -44
app
lib
tasks
+61
app/import_manager.rb
··· 1 + require_relative 'importers/likes_importer' 2 + require_relative 'importers/posts_importer' 3 + require_relative 'importers/reposts_importer' 4 + require_relative 'item_queue' 5 + require_relative 'post_downloader' 6 + 7 + class ImportManager 8 + attr_accessor :report, :time_limit 9 + 10 + def initialize(user) 11 + @user = user 12 + end 13 + 14 + def start(sets, include_pending) 15 + queued_items = [] 16 + importers = [] 17 + sets = [sets] unless sets.is_a?(Array) 18 + 19 + sets.each do |set| 20 + case set 21 + when 'likes' 22 + queued_items += @user.likes.pending.to_a if include_pending 23 + importers << LikesImporter.new(@user) 24 + when 'reposts' 25 + queued_items += @user.reposts.pending.to_a if include_pending 26 + importers << RepostsImporter.new(@user) 27 + when 'posts' 28 + queued_items += @user.quotes.pending.to_a + @user.pins.pending.to_a if include_pending 29 + importers << PostsImporter.new(@user) 30 + when 'all' 31 + queued_items += @user.all_pending_items if include_pending 32 + importers += [ 33 + LikesImporter.new(@user), 34 + RepostsImporter.new(@user), 35 + PostsImporter.new(@user) 36 + ] 37 + else 38 + raise "Invalid collection: #{set}" 39 + end 40 + end 41 + 42 + queue = ItemQueue.new(queued_items) 43 + 44 + downloader = PostDownloader.new 45 + downloader.report = @report 46 + 47 + download_thread = Thread.new { downloader.import_from_queue(queue) } 48 + 49 + import_threads = importers.map do |import| 50 + import.item_queue = queue 51 + import.report = @report 52 + 53 + Thread.new { import.run_import(@time_limit) } 54 + end 55 + 56 + import_threads.each { |i| i.join } 57 + 58 + downloader.stop_when_empty = true 59 + download_thread.join 60 + end 61 + end
+3 -3
app/importers/base_importer.rb
··· 8 8 class BaseImporter 9 9 attr_accessor :item_queue, :report 10 10 11 - def initialize(user_did) 12 - @did = DID.new(user_did) 13 - @user = User.find_or_create_by!(did: user_did) 11 + def initialize(user) 12 + @did = DID.new(user.did) 13 + @user = user 14 14 @imported_count = 0 15 15 end 16 16
+7 -41
lib/tasks/import.rake
··· 1 - require_relative '../../app/importers/likes_importer' 2 - require_relative '../../app/importers/posts_importer' 3 - require_relative '../../app/importers/reposts_importer' 4 - 5 1 require_relative '../../app/console_report' 2 + require_relative '../../app/import_manager' 6 3 require_relative '../../app/item_queue' 4 + require_relative '../../app/models/user' 7 5 require_relative '../../app/post_downloader' 8 6 9 7 task :import_user do ··· 14 12 user = User.find_or_create_by!(did: ENV['USER']) 15 13 pending = !ENV['SKIP_PENDING'] 16 14 17 - case ENV['COLLECTION'] 18 - when 'likes' 19 - queue = ItemQueue.new(pending ? user.likes.pending.to_a : []) 20 - importers = [LikesImporter.new(ENV['USER'])] 21 - when 'reposts' 22 - queue = ItemQueue.new(pending ? user.reposts.pending.to_a : []) 23 - importers = [RepostsImporter.new(ENV['USER'])] 24 - when 'posts' 25 - queue = ItemQueue.new(pending ? user.quotes.pending.to_a + user.pins.pending.to_a : []) 26 - importers = [PostsImporter.new(ENV['USER'])] 27 - when 'all' 28 - queue = ItemQueue.new(pending ? user.all_pending_items : []) 29 - importers = [ 30 - LikesImporter.new(ENV['USER']), 31 - RepostsImporter.new(ENV['USER']), 32 - PostsImporter.new(ENV['USER']) 33 - ] 34 - when nil 15 + unless ENV['COLLECTION'] 35 16 raise "Required COLLECTION parameter missing" 36 - else 37 - raise "Invalid collection: #{ENV['COLLECTION']}" 38 17 end 39 18 40 - report = ConsoleReport.new 41 - 42 - downloader = PostDownloader.new 43 - downloader.report = report 44 - 45 - download_thread = Thread.new { downloader.import_from_queue(queue) } 19 + import = ImportManager.new(user) 20 + import.report = ConsoleReport.new 21 + import.time_limit = ENV['UNTIL'] 46 22 47 23 trap("SIGINT") { 48 24 puts "\n\n\n\n\n" 49 25 exit 50 26 } 51 27 52 - import_threads = importers.map do |import| 53 - import.item_queue = queue 54 - import.report = report 55 - 56 - Thread.new { import.run_import(ENV['UNTIL']) } 57 - end 58 - 59 - import_threads.each { |i| i.join } 60 - 61 - downloader.stop_when_empty = true 62 - download_thread.join 28 + import.start(ENV['COLLECTION'], pending) 63 29 64 30 puts "\n\n\n\n\n" 65 31 end