48337c08bc
- Updated job classes in the FA and Inkbunny domains to enforce strict typing with Sorbet, including the addition of type signatures in various job classes. - Refactored initialization methods to improve argument handling and ensure proper type management. - Enhanced error handling and logging mechanisms across job classes for better traceability. - Improved job argument handling, including the introduction of `first_log_entry` and `causing_log_entry` for better context in job processing. - Streamlined the processing of API responses and improved the management of user and post data. These changes aim to enhance type safety, maintainability, and robustness of the job processing logic.
66 lines
1.7 KiB
Ruby
66 lines
1.7 KiB
Ruby
# typed: strict
|
|
class Domain::Fa::Job::BrowsePageJob < Domain::Fa::Job::Base
|
|
queue_as :fa_browse_page
|
|
|
|
sig { params(args: T.untyped).void }
|
|
def initialize(*args)
|
|
super(*T.unsafe(args))
|
|
@page_number = T.let(1, Integer)
|
|
@total_num_new_posts_seen = T.let(0, Integer)
|
|
@total_num_posts_seen = T.let(0, Integer)
|
|
end
|
|
|
|
sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
|
|
def perform(args)
|
|
while true
|
|
break unless scan_browse_page
|
|
break if @page_number > 150
|
|
@page_number += 1
|
|
end
|
|
|
|
logger.info(
|
|
[
|
|
"[finished]",
|
|
"[total new: #{@total_num_new_posts_seen.to_s.bold}]",
|
|
"[total seen: #{@total_num_posts_seen.to_s.bold}]",
|
|
"[pages: #{@page_number.to_s.bold}]",
|
|
].join(" "),
|
|
)
|
|
end
|
|
|
|
private
|
|
|
|
sig { returns(T::Boolean) }
|
|
def scan_browse_page
|
|
if @page_number == 1
|
|
url = "https://www.furaffinity.net/browse/"
|
|
else
|
|
url = "https://www.furaffinity.net/browse/#{@page_number}/"
|
|
end
|
|
|
|
response = http_client.get(url, caused_by_entry: causing_log_entry)
|
|
log_entry = response.log_entry
|
|
self.first_log_entry ||= log_entry
|
|
|
|
if response.status_code != 200
|
|
fatal_error(
|
|
"non 200 response for /browse: #{response.status_code.to_s.underline}",
|
|
)
|
|
end
|
|
|
|
page = Domain::Fa::Parser::Page.new(response.body)
|
|
listing_page_stats =
|
|
update_and_enqueue_posts_from_listings_page(
|
|
:browse_page,
|
|
page,
|
|
enqueue_posts_pri: :high,
|
|
page_desc: "Browse@#{@page_number}",
|
|
fill_id_gaps: true,
|
|
)
|
|
|
|
@total_num_new_posts_seen += listing_page_stats.new_seen
|
|
@total_num_posts_seen += listing_page_stats.total_seen
|
|
listing_page_stats.new_seen > 0
|
|
end
|
|
end
|