Don't forget to lycansubscribe

multi-import of all items in parallel

Changed files
+22 -7
app
models
lib
tasks
+4
app/models/user.rb
··· 17 17 has_many :quotes, foreign_key: 'actor_id' 18 18 has_many :pins, foreign_key: 'actor_id' 19 19 has_many :imports 20 + 21 + def all_pending_items 22 + [:likes, :reposts, :quotes, :pins].map { |x| self.send(x).pending.to_a }.reduce(&:+) 23 + end 20 24 end
+18 -7
lib/tasks/import.rake
··· 17 17 case ENV['COLLECTION'] 18 18 when 'likes' 19 19 queue = ItemQueue.new(pending ? user.likes.pending.to_a : []) 20 - importer = LikesImporter.new(ENV['USER']) 20 + importers = [LikesImporter.new(ENV['USER'])] 21 21 when 'reposts' 22 22 queue = ItemQueue.new(pending ? user.reposts.pending.to_a : []) 23 - importer = RepostsImporter.new(ENV['USER']) 23 + importers = [RepostsImporter.new(ENV['USER'])] 24 24 when 'posts' 25 25 queue = ItemQueue.new(pending ? user.quotes.pending.to_a + user.pins.pending.to_a : []) 26 - importer = PostsImporter.new(ENV['USER']) 26 + importers = [PostsImporter.new(ENV['USER'])] 27 + when 'all' 28 + queue = ItemQueue.new(pending ? user.all_pending_items : []) 29 + importers = [ 30 + LikesImporter.new(ENV['USER']), 31 + RepostsImporter.new(ENV['USER']), 32 + PostsImporter.new(ENV['USER']) 33 + ] 27 34 when nil 28 35 raise "Required COLLECTION parameter missing" 29 36 else ··· 32 39 33 40 report = ImportReport.new 34 41 35 - importer.item_queue = queue 36 - importer.report = report 37 - 38 42 downloader = PostDownloader.new 39 43 downloader.report = report 40 44 ··· 45 49 exit 46 50 } 47 51 48 - importer.run_import(ENV['UNTIL']) 52 + import_threads = importers.map do |import| 53 + import.item_queue = queue 54 + import.report = report 55 + 56 + Thread.new { import.run_import(ENV['UNTIL']) } 57 + end 58 + 59 + import_threads.each { |i| i.join } 49 60 50 61 downloader.stop_when_empty = true 51 62 download_thread.join