better progress bar, new thumbnails/fingerprints bust cache

This commit is contained in:
Dylan Knutson
2025-06-26 07:02:53 +00:00
parent c20e7a0b7e
commit e3b2463cbe
5 changed files with 51 additions and 22 deletions

View File

@@ -292,28 +292,31 @@ task run_fa_user_avatar_jobs: :environment do
end
end
task create_post_file_fingerprints: :environment do
task create_post_file_fingerprints: %i[environment set_logger_stdout] do
def migrate_posts_for_user(user)
puts "migrating posts for #{user.to_param}"
posts = user.posts.includes(files: %i[blob thumbnails bit_fingerprints])
pb =
ProgressBar.create(
total: user.posts.count,
format: "%t: %c/%C %B %p%% %a %e",
total: posts.count,
progress_mark: " ",
remainder_mark: " ",
format: "%B %c/%C (%r/sec) %J%% %a %E",
)
user
.posts
.includes(:files)
.find_in_batches(batch_size: 64) do |batch|
ReduxApplicationRecord.transaction do
batch.each { |post| migrate_post(post) }
posts.find_in_batches(batch_size: 64) do |batch|
ReduxApplicationRecord.transaction do
batch.each do |post|
migrate_post(post)
pb.progress = [pb.progress + 1, pb.total].min
end
end
end
end
def migrate_post(post)
puts "migrating #{post.id} / #{post.to_param} / '#{post.title_for_view}'"
puts "#{post.creator&.url_name} (#{post.creator&.user_user_follows_to_count}) :: #{post.to_param} / '#{post.title_for_view}'"
ColorLogger.quiet do
post.files.each do |file|
migrate_post_file(file)
@@ -324,9 +327,8 @@ task create_post_file_fingerprints: :environment do
end
def migrate_post_file(post_file)
job = Domain::PostFileThumbnailJob.new
ColorLogger.quiet do
job.perform({ post_file: })
Domain::PostFileThumbnailJob.new.perform({ post_file: })
rescue => e
puts "error: #{e.message}"
end
@@ -334,7 +336,13 @@ task create_post_file_fingerprints: :environment do
if ENV["post_file_descending"].present?
total = 49_783_962 # cache this value
pb = ProgressBar.create(total:, format: "%t: %c/%C %B %p%% %a %e")
pb =
ProgressBar.create(
total:,
progress_mark: " ",
remainder_mark: " ",
format: "%B %c/%C (%r/sec) %J%% %a %E",
)
i = 0
Domain::PostFile
.where(state: "ok")
@@ -354,7 +362,13 @@ task create_post_file_fingerprints: :environment do
elsif ENV["posts_descending"].present?
# total = Domain::Post.count
total = 66_431_808 # cache this value
pb = ProgressBar.create(total:, format: "%t: %c/%C %B %p%% %a %e")
pb =
ProgressBar.create(
total:,
progress_mark: " ",
remainder_mark: " ",
format: "%B %c/%C (%r/sec) %J%% %a %E",
)
Domain::Post.find_each(order: :desc) do |post|
migrate_post(post) unless post.is_a?(Domain::Post::InkbunnyPost)
pb.progress = [pb.progress + 1, pb.total].min
@@ -371,7 +385,7 @@ task create_post_file_fingerprints: :environment do
migrated_users = migrated_file.readlines.map(&:strip)
users =
Domain::User::FaUser.order(
Arel.sql("json_attributes->>'num_watched_by' DESC NULLS LAST"),
Arel.sql("user_user_follows_to_count DESC NULLS LAST"),
).pluck(:id)
users.each do |user_id|