+61
app/import_manager.rb
+61
app/import_manager.rb
···
1
+
require_relative 'importers/likes_importer'
2
+
require_relative 'importers/posts_importer'
3
+
require_relative 'importers/reposts_importer'
4
+
require_relative 'item_queue'
5
+
require_relative 'post_downloader'
6
+
7
+
class ImportManager
8
+
attr_accessor :report, :time_limit
9
+
10
+
def initialize(user)
11
+
@user = user
12
+
end
13
+
14
+
def start(sets, include_pending)
15
+
queued_items = []
16
+
importers = []
17
+
sets = [sets] unless sets.is_a?(Array)
18
+
19
+
sets.each do |set|
20
+
case set
21
+
when 'likes'
22
+
queued_items += @user.likes.pending.to_a if include_pending
23
+
importers << LikesImporter.new(@user)
24
+
when 'reposts'
25
+
queued_items += @user.reposts.pending.to_a if include_pending
26
+
importers << RepostsImporter.new(@user)
27
+
when 'posts'
28
+
queued_items += @user.quotes.pending.to_a + @user.pins.pending.to_a if include_pending
29
+
importers << PostsImporter.new(@user)
30
+
when 'all'
31
+
queued_items += @user.all_pending_items if include_pending
32
+
importers += [
33
+
LikesImporter.new(@user),
34
+
RepostsImporter.new(@user),
35
+
PostsImporter.new(@user)
36
+
]
37
+
else
38
+
raise "Invalid collection: #{set}"
39
+
end
40
+
end
41
+
42
+
queue = ItemQueue.new(queued_items)
43
+
44
+
downloader = PostDownloader.new
45
+
downloader.report = @report
46
+
47
+
download_thread = Thread.new { downloader.import_from_queue(queue) }
48
+
49
+
import_threads = importers.map do |import|
50
+
import.item_queue = queue
51
+
import.report = @report
52
+
53
+
Thread.new { import.run_import(@time_limit) }
54
+
end
55
+
56
+
import_threads.each { |i| i.join }
57
+
58
+
downloader.stop_when_empty = true
59
+
download_thread.join
60
+
end
61
+
end
+3
-3
app/importers/base_importer.rb
+3
-3
app/importers/base_importer.rb
+7
-41
lib/tasks/import.rake
+7
-41
lib/tasks/import.rake
···
1
-
require_relative '../../app/importers/likes_importer'
2
-
require_relative '../../app/importers/posts_importer'
3
-
require_relative '../../app/importers/reposts_importer'
4
-
5
1
require_relative '../../app/console_report'
2
+
require_relative '../../app/import_manager'
6
3
require_relative '../../app/item_queue'
4
+
require_relative '../../app/models/user'
7
5
require_relative '../../app/post_downloader'
8
6
9
7
task :import_user do
···
14
12
user = User.find_or_create_by!(did: ENV['USER'])
15
13
pending = !ENV['SKIP_PENDING']
16
14
17
-
case ENV['COLLECTION']
18
-
when 'likes'
19
-
queue = ItemQueue.new(pending ? user.likes.pending.to_a : [])
20
-
importers = [LikesImporter.new(ENV['USER'])]
21
-
when 'reposts'
22
-
queue = ItemQueue.new(pending ? user.reposts.pending.to_a : [])
23
-
importers = [RepostsImporter.new(ENV['USER'])]
24
-
when 'posts'
25
-
queue = ItemQueue.new(pending ? user.quotes.pending.to_a + user.pins.pending.to_a : [])
26
-
importers = [PostsImporter.new(ENV['USER'])]
27
-
when 'all'
28
-
queue = ItemQueue.new(pending ? user.all_pending_items : [])
29
-
importers = [
30
-
LikesImporter.new(ENV['USER']),
31
-
RepostsImporter.new(ENV['USER']),
32
-
PostsImporter.new(ENV['USER'])
33
-
]
34
-
when nil
15
+
unless ENV['COLLECTION']
35
16
raise "Required COLLECTION parameter missing"
36
-
else
37
-
raise "Invalid collection: #{ENV['COLLECTION']}"
38
17
end
39
18
40
-
report = ConsoleReport.new
41
-
42
-
downloader = PostDownloader.new
43
-
downloader.report = report
44
-
45
-
download_thread = Thread.new { downloader.import_from_queue(queue) }
19
+
import = ImportManager.new(user)
20
+
import.report = ConsoleReport.new
21
+
import.time_limit = ENV['UNTIL']
46
22
47
23
trap("SIGINT") {
48
24
puts "\n\n\n\n\n"
49
25
exit
50
26
}
51
27
52
-
import_threads = importers.map do |import|
53
-
import.item_queue = queue
54
-
import.report = report
55
-
56
-
Thread.new { import.run_import(ENV['UNTIL']) }
57
-
end
58
-
59
-
import_threads.each { |i| i.join }
60
-
61
-
downloader.stop_when_empty = true
62
-
download_thread.join
28
+
import.start(ENV['COLLECTION'], pending)
63
29
64
30
puts "\n\n\n\n\n"
65
31
end