dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

look at wildcard queues

Browse Source
This commit is contained in:
Dylan Knutson
2023-04-14 09:26:08 -07:00
parent 2f7fbf29fe
commit 1be5811e14
2 changed files with 31 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
Rakefile
View File

@@ -48,11 +48,12 @@ task :good_job do
"GOOD_JOB_MAX_THREADS" => "4",
"GOOD_JOB_QUEUES" => [
"manual:4",
"static_file,fa_user_avatar:3",
"+fa_user_avatar,static_file,fa_post:1",
"+fa_user_page,fa_user_gallery,fa_post:3",
"fa_user_follows:2",
proxy == "serverhost-1" ? nil : "twitter_timeline_tweets:1",
# "static_file,fa_user_avatar:3",
# "+fa_user_avatar,static_file,fa_post:1",
# "+fa_user_page,fa_user_gallery,fa_post:3",
# "fa_user_follows:2",
# proxy == "serverhost-1" ? nil : "twitter_timeline_tweets:1",
proxy == "serverhost-1" ? "-twitter_timeline_tweets:6" : "*:6",
].reject(&:nil?).join(";"),
}

49
app/lib/domain/fa/user_enqueuer.rb
View File

@@ -27,35 +27,35 @@ class Domain::Fa::UserEnqueuer
end
end
measure("enqueue jobs") do
bulk_enqueue_jobs do
rows.each do |user|
types = []
if user.state == "ok"
if user.due_for_page_scan?
Domain::Fa::Job::UserPageJob.perform_later({ user: user })
types << "page"
end
if user.due_for_gallery_scan?
Domain::Fa::Job::UserGalleryJob.perform_later({ user: user })
types << "gallery"
end
if user.due_for_follows_scan?
Domain::Fa::Job::UserFollowsJob.perform_later({ user: user })
types << "follows"
end
# bulk_enqueue_jobs do
rows.each do |user|
types = []
if user.state == "ok"
if user.due_for_page_scan?
Domain::Fa::Job::UserPageJob.perform_later({ user: user })
types << "page"
end
avatar = user.avatar_or_create
if avatar.file.nil? && avatar.state == "ok"
Domain::Fa::Job::UserAvatarJob.perform_later({ user: user })
types << "avatar"
if user.due_for_gallery_scan?
Domain::Fa::Job::UserGalleryJob.perform_later({ user: user })
types << "gallery"
end
if user.due_for_follows_scan?
Domain::Fa::Job::UserFollowsJob.perform_later({ user: user })
types << "follows"
end
types = types.map { |t| t.bold }.join("|")
logger.info "#{types} - #{user.url_name.bold} - #{user.id.to_s.bold}"
end
avatar = user.avatar_or_create
if avatar.file.nil? && avatar.state == "ok"
Domain::Fa::Job::UserAvatarJob.perform_later({ user: user })
types << "avatar"
end
types = types.map { |t| t.bold }.join("|")
logger.info "#{types} - #{user.url_name.bold} - #{user.id.to_s.bold}"
end
end
# end
throw StopIteration if rows.empty?
else
logger.info(
@@ -79,6 +79,7 @@ class Domain::Fa::UserEnqueuer
"fa_user_gallery",
"fa_post",
"fa_user_avatar",
"static_file",
].map do |queue_name|
GoodJob::Job.where(
queue_name: queue_name,