dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

skip page / gallery scan in gallery job

Browse Source
This commit is contained in:
dymk
2023-03-01 10:50:39 -08:00
parent d3a864b92f
commit eacbf7f43b
2 changed files with 41 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
app/jobs/domain/fa/job/fa_job_base.rb
View File

@@ -31,6 +31,8 @@ class Domain::Fa::Job::FaJobBase < Scraper::JobBase
page,
caused_by_entry,
enqueue_posts_pri:,
enqueue_page_scan: true,
enqueue_gallery_scan: true,
page_desc: nil
)
fatal_error("not a listings page") unless page.probably_listings_page?
@@ -50,7 +52,12 @@ class Domain::Fa::Job::FaJobBase < Scraper::JobBase
listing_page_stats.total_seen += 1
update_post_from_listings_page(job_type, post, submission, caused_by_entry)
enqueue_user_scan(post.creator, caused_by_entry) if post.creator
enqueue_user_scan(
post.creator,
caused_by_entry,
enqueue_page_scan: enqueue_page_scan,
enqueue_gallery_scan: enqueue_gallery_scan,
) if post.creator
case post.state.to_sym
when :ok
@@ -87,44 +94,45 @@ class Domain::Fa::Job::FaJobBase < Scraper::JobBase
post.save!
end
def enqueue_user_scan(user, caused_by_entry)
def enqueue_user_scan(
user,
caused_by_entry,
enqueue_page_scan: true,
enqueue_gallery_scan: true
)
@users_enqueued_for_page_scan ||= Set.new
@users_enqueued_for_gallery_scan ||= Set.new
args = if user.persisted?
{ user: user }
else
unless user.url_name
logger.warn "user does not have a url name and is not persisted, skipping (#{user.name})"
return
end
{ url_name: user.url_name }
{ user: user }
else
unless user.url_name
logger.warn "user does not have a url name and is not persisted, skipping (#{user.name})"
return
end
if user.due_for_page_scan?
logger.info(
"enqueue user page job for #{user.url_name.bold}, " +
"last scanned #{time_ago_in_words(user.scanned_page_at)}"
)
enqueue_job(Domain::Fa::Job::UserPageJob, args.merge(caused_by_entry: caused_by_entry))
else
# logger.debug(
# "already scanned user page for #{user.url_name.white.bold} #{time_ago_in_words(user.scanned_page_at)}"
# )
end if @users_enqueued_for_page_scan.add?(user.url_name)
{ url_name: user.url_name }
end.merge({ caused_by_entry: caused_by_entry })
if user.due_for_gallery_scan?
logger.info(
"enqueue user gallery job for #{user.url_name.bold}, " +
"last scanned #{time_ago_in_words(user.scanned_gallery_at)}"
)
enqueue_job(Domain::Fa::Job::UserGalleryJob, args.merge(caused_by_entry: caused_by_entry))
else
# logger.debug(
# "already scanned user gallery for #{user.url_name.white.bold} #{time_ago_in_words(user.scanned_gallery_at)}"
# )
end if @users_enqueued_for_gallery_scan.add?(user.url_name)
if enqueue_page_scan && @users_enqueued_for_page_scan.add?(user.url_name)
if user.due_for_page_scan?
logger.info(
"enqueue user page job for #{user.url_name.bold}, " +
"last scanned #{time_ago_in_words(user.scanned_page_at)}"
)
enqueue_job(Domain::Fa::Job::UserPageJob, args)
end
end
if enqueue_gallery_scan && @users_enqueued_for_gallery_scan.add?(user.url_name)
if user.due_for_gallery_scan?
logger.info(
"enqueue user gallery job for #{user.url_name.bold}, " +
"last scanned #{time_ago_in_words(user.scanned_gallery_at)}"
)
enqueue_job(Domain::Fa::Job::UserGalleryJob, args)
end
end
end
def enqueue_post_scan(post, caused_by_entry, enqueue_pri)

1
app/jobs/domain/fa/job/user_gallery_job.rb
View File

@@ -104,6 +104,7 @@ class Domain::Fa::Job::UserGalleryJob < Domain::Fa::Job::FaJobBase
listing_page_stats = update_and_enqueue_posts_from_listings_page(
:gallery_page, page, response.log_entry,
enqueue_posts_pri: :low,
enqueue_gallery_scan: false,
page_desc: "#{folder[:title]}@#{page_number}",
)
total_num_new_posts_seen += listing_page_stats.new_seen