separate out blob file migrator

This commit is contained in:
Dylan Knutson
2025-02-17 04:57:43 +00:00
parent ca924afd49
commit 25a86d6464
16 changed files with 464 additions and 78 deletions

View File

@@ -12,6 +12,10 @@ namespace :blob_file do
RubyProf.start if profile
def migrate_impl(batch_size, start_at)
def n2d(n)
ActiveSupport::NumberHelper.number_to_delimited(n).rjust(8)
end
num_migrated = 0
num_processed = 0
start_time = Time.now
@@ -26,8 +30,8 @@ namespace :blob_file do
num_processed += batch.size
rate = batch_migrated.to_f / (Time.now - start_time)
puts [
"[migrated: #{ActiveSupport::NumberHelper.number_to_delimited(num_migrated).rjust(8)}]",
"[processed: #{ActiveSupport::NumberHelper.number_to_delimited(num_processed).rjust(8)}]",
"[migrated: #{n2d(num_migrated)}]",
"[processed: #{n2d(num_processed)}]",
"[rate: #{rate.round(1).to_s.rjust(5)}/second]",
"[last: '#{HexUtil.bin2hex(batch.last.sha256)}']",
].join(" ")

View File

@@ -7,7 +7,7 @@ namespace :e621 do
desc "download files for posts missing files"
task download_missing_files: %i[environment set_logger_stdout] do
relation =
Domain::E621::Post
Domain::Post::E621Post
.where(file: nil, state: :ok)
.where.not(file_url_str: nil)
puts "will download #{relation.count} posts"
@@ -19,7 +19,7 @@ namespace :e621 do
desc "scan e621 user favs"
task scan_user_favs: :environment do
while user =
Domain::E621::User
Domain::User::E621User
.where(scanned_favs_at: nil)
.where(num_other_favs_cached: ..200)
.order("RANDOM()")
@@ -32,9 +32,9 @@ namespace :e621 do
task scan_user_favs_descending: %i[environment set_logger_stdout] do
user_query =
lambda do
Domain::E621::User
Domain::User::E621User
.where(scanned_favs_status: nil)
.or(Domain::E621::User.where.not(scanned_favs_status: "error"))
.or(Domain::User::E621User.where.not(scanned_favs_status: "error"))
.where(scanned_favs_at: nil)
.where.not(num_other_favs_cached: nil)
.order(num_other_favs_cached: :desc)