separate out blob file migrator
This commit is contained in:
@@ -12,6 +12,10 @@ namespace :blob_file do
|
||||
RubyProf.start if profile
|
||||
|
||||
def migrate_impl(batch_size, start_at)
|
||||
def n2d(n)
|
||||
ActiveSupport::NumberHelper.number_to_delimited(n).rjust(8)
|
||||
end
|
||||
|
||||
num_migrated = 0
|
||||
num_processed = 0
|
||||
start_time = Time.now
|
||||
@@ -26,8 +30,8 @@ namespace :blob_file do
|
||||
num_processed += batch.size
|
||||
rate = batch_migrated.to_f / (Time.now - start_time)
|
||||
puts [
|
||||
"[migrated: #{ActiveSupport::NumberHelper.number_to_delimited(num_migrated).rjust(8)}]",
|
||||
"[processed: #{ActiveSupport::NumberHelper.number_to_delimited(num_processed).rjust(8)}]",
|
||||
"[migrated: #{n2d(num_migrated)}]",
|
||||
"[processed: #{n2d(num_processed)}]",
|
||||
"[rate: #{rate.round(1).to_s.rjust(5)}/second]",
|
||||
"[last: '#{HexUtil.bin2hex(batch.last.sha256)}']",
|
||||
].join(" ")
|
||||
|
||||
@@ -7,7 +7,7 @@ namespace :e621 do
|
||||
desc "download files for posts missing files"
|
||||
task download_missing_files: %i[environment set_logger_stdout] do
|
||||
relation =
|
||||
Domain::E621::Post
|
||||
Domain::Post::E621Post
|
||||
.where(file: nil, state: :ok)
|
||||
.where.not(file_url_str: nil)
|
||||
puts "will download #{relation.count} posts"
|
||||
@@ -19,7 +19,7 @@ namespace :e621 do
|
||||
desc "scan e621 user favs"
|
||||
task scan_user_favs: :environment do
|
||||
while user =
|
||||
Domain::E621::User
|
||||
Domain::User::E621User
|
||||
.where(scanned_favs_at: nil)
|
||||
.where(num_other_favs_cached: ..200)
|
||||
.order("RANDOM()")
|
||||
@@ -32,9 +32,9 @@ namespace :e621 do
|
||||
task scan_user_favs_descending: %i[environment set_logger_stdout] do
|
||||
user_query =
|
||||
lambda do
|
||||
Domain::E621::User
|
||||
Domain::User::E621User
|
||||
.where(scanned_favs_status: nil)
|
||||
.or(Domain::E621::User.where.not(scanned_favs_status: "error"))
|
||||
.or(Domain::User::E621User.where.not(scanned_favs_status: "error"))
|
||||
.where(scanned_favs_at: nil)
|
||||
.where.not(num_other_favs_cached: nil)
|
||||
.order(num_other_favs_cached: :desc)
|
||||
|
||||
Reference in New Issue
Block a user