dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

inkbunny missing posts task fixes

Browse Source
This commit is contained in:
Dylan Knutson
2025-08-18 23:58:06 +00:00
parent a68e5b0112
commit 7f521b30e9
4 changed files with 20 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/jobs/domain/static_file_job_helper.rb
View File

@@ -105,7 +105,9 @@ module Domain::StaticFileJobHelper
end end
end end
ensure ensure
post_file.save! if post_file post_file.save!
post = post_file.post
post.touch if post
if should_enqueue_thumbnail_job if should_enqueue_thumbnail_job
defer_job(Domain::PostFileThumbnailJob, { post_file: }) defer_job(Domain::PostFileThumbnailJob, { post_file: })
end end

10
app/lib/enqueue_job_base.rb
View File

@@ -45,8 +45,8 @@ class EnqueueJobBase < Tasks::InterruptableTask
def start_enqueuing def start_enqueuing
end end
sig { params(block: T.proc.void).void } sig { params(always_recheck: T::Boolean, block: T.proc.void).void }
def enqueue(&block) def enqueue(always_recheck: false, &block)
# if we're under the high water mark, we can just enqueue and return # if we're under the high water mark, we can just enqueue and return
# so we get called again as soon as possible # so we get called again as soon as possible
if @inferred_queue_size < high_water_mark if @inferred_queue_size < high_water_mark
@@ -64,7 +64,11 @@ class EnqueueJobBase < Tasks::InterruptableTask
end end
block.call block.call
@inferred_queue_size += 1 if always_recheck
@inferred_queue_size = queue_size
else
@inferred_queue_size += 1
end
@total_performed += 1 @total_performed += 1
end end

17
app/lib/tasks/inkbunny/enqueue_missing_posts_task.rb
View File

@@ -60,12 +60,10 @@ class Tasks::Inkbunny::EnqueueMissingPostsTask < EnqueueJobBase
missing_ib_post_ids = T.cast(missing_ib_post_ids, T::Array[Integer]) missing_ib_post_ids = T.cast(missing_ib_post_ids, T::Array[Integer])
if found_min_id = missing_ib_post_ids.min if found_min_id = missing_ib_post_ids.min
enqueue do enqueue(always_recheck: true) do
ColorLogger.quiet do Domain::Inkbunny::Job::UpdatePostsJob.perform_now(
Domain::Inkbunny::Job::UpdatePostsJob.perform_now( ib_post_ids: missing_ib_post_ids,
ib_post_ids: missing_ib_post_ids, )
)
end
end end
# Move to continue from the lowest ID we just processed # Move to continue from the lowest ID we just processed
max_ib_post_id = found_min_id max_ib_post_id = found_min_id
@@ -82,13 +80,16 @@ class Tasks::Inkbunny::EnqueueMissingPostsTask < EnqueueJobBase
end end
# Stop if we've reached the beginning # Stop if we've reached the beginning
save_progress([max_ib_post_id, 0].max.to_s) max_ib_post_id = [max_ib_post_id, 0].max
save_progress(max_ib_post_id.to_s)
logger.info("saved progress: #{max_ib_post_id}")
break if max_ib_post_id <= 0 break if max_ib_post_id <= 0
break if interrupted?
end end
end end
sig { override.returns(Integer) } sig { override.returns(Integer) }
def queue_size def queue_size
count_failed_in_queue("inkbunny") count_failed_in_queue(%w[inkbunny static_file])
end end
end end

2
rake/ib.rake
View File

@@ -10,7 +10,7 @@ namespace :ib do
desc "Enqueue missing posts" desc "Enqueue missing posts"
task enqueue_missing_posts: :environment do task enqueue_missing_posts: :environment do
Tasks::Inkbunny::EnqueueMissingPostsTask.new.run Tasks::Inkbunny::EnqueueMissingPostsTask.new(start_at: ENV["start_at"]).run
end end
desc "set auth credentials" desc "set auth credentials"