incremental impl in user page job

app/helpers/domain/users_helper.rb

@@ -147,7 +147,13 @@ module Domain::UsersHelper
         hover_title: user.gallery_scan.interval.inspect,
       )
       rows << StatRow.new(
-        name: "Follows scanned",
+        name: "Followers scanned",
+        value: user.followed_by_scan,
+        fa_icon_class: icon_for.call(user.followed_by_scan.due?),
+        hover_title: user.followed_by_scan.interval.inspect,
+      )
+      rows << StatRow.new(
+        name: "Followed scanned",
         value: user.follows_scan,
         fa_icon_class: icon_for.call(user.follows_scan.due?),
         hover_title: user.follows_scan.interval.inspect,

app/jobs/domain/fa/job/base.rb

@@ -1,7 +1,6 @@
 # typed: strict
 class Domain::Fa::Job::Base < Scraper::JobBase
   abstract!
-
   discard_on ActiveJob::DeserializationError
   include HasBulkEnqueueJobs
 
@@ -12,6 +11,20 @@ class Domain::Fa::Job::Base < Scraper::JobBase
 
   protected
 
+  BUGGY_USER_URL_NAMES = T.let(["click here", ".."], T::Array[String])
+
+  sig { params(user: Domain::User::FaUser).returns(T::Boolean) }
+  def buggy_user?(user)
+    if BUGGY_USER_URL_NAMES.include?(user.url_name)
+      logger.error(
+        format_tags("buggy user", make_tag("url_name", user.url_name)),
+      )
+      return true
+    end
+
+    false
+  end
+
   sig { returns(T::Boolean) }
   def skip_enqueue_found_links?
     !!arguments[0][:skip_enqueue_found_links]
@@ -246,6 +259,9 @@ class Domain::Fa::Job::Base < Scraper::JobBase
         )
       end
 
+      # don't enqueue any other jobs if the user page hasn't been scanned yet
+      return unless user.scanned_page_at?
+
       if user.gallery_scan.due? &&
            defer_job(Domain::Fa::Job::UserGalleryJob, args)
         logger.info(
@@ -332,11 +348,18 @@ class Domain::Fa::Job::Base < Scraper::JobBase
   sig do
     params(
       user: Domain::User::FaUser,
-      page: Domain::Fa::Parser::Page,
       response: Scraper::HttpClient::Response,
-    ).void
+    ).returns(T.nilable(Domain::Fa::Parser::Page))
   end
-  def update_user_fields_from_page(user, page, response)
+  def update_user_from_user_page(user, response)
+    disabled_or_not_found = user_disabled_or_not_found?(user, response)
+    user.scanned_page_at = Time.current
+    user.last_user_page_log_entry = response.log_entry
+    return nil if disabled_or_not_found
+
+    page = Domain::Fa::Parser::Page.new(response.body)
+    return nil unless page.probably_user_page?
+
     user_page = page.user_page
     user.name = user_page.name
     user.registered_at = user_page.registered_since
@@ -348,13 +371,11 @@ class Domain::Fa::Job::Base < Scraper::JobBase
     user.num_favorites = user_page.num_favorites
     user.profile_html =
       user_page.profile_html.encode("UTF-8", invalid: :replace, undef: :replace)
-    user.last_user_page_id = response.log_entry.id
-    user.scanned_page_at = Time.current
-    user.save!
-
     if url = user_page.profile_thumb_url
       enqueue_user_avatar(user, url)
     end
+
+    page
   end
 
   sig { params(user: Domain::User::FaUser, avatar_url_str: String).void }
@@ -488,4 +509,61 @@ class Domain::Fa::Job::Base < Scraper::JobBase
       ),
     )
   end
+
+  DISABLED_PAGE_PATTERNS =
+    T.let(
+      [
+        /User ".+" has voluntarily disabled access/,
+        /The page you are trying to reach is currently pending deletion/,
+      ],
+      T::Array[Regexp],
+    )
+
+  NOT_FOUND_PAGE_PATTERNS =
+    T.let(
+      [
+        /User ".+" was not found in our database\./,
+        /The username ".+" could not be found\./,
+      ],
+      T::Array[Regexp],
+    )
+
+  module DisabledOrNotFoundResult
+    class Stop < T::Struct
+      include T::Struct::ActsAsComparable
+
+      const :message, String
+    end
+
+    class Ok < T::Struct
+      include T::Struct::ActsAsComparable
+
+      const :page, Domain::Fa::Parser::Page
+    end
+  end
+
+  sig do
+    params(
+      user: Domain::User::FaUser,
+      response: Scraper::HttpClient::Response,
+    ).returns(T::Boolean)
+  end
+  def user_disabled_or_not_found?(user, response)
+    if response.status_code != 200
+      fatal_error(
+        "http #{response.status_code}, log entry #{response.log_entry.id}",
+      )
+    end
+
+    if DISABLED_PAGE_PATTERNS.any? { |pattern| response.body =~ pattern }
+      user.state_account_disabled!
+      user.is_disabled = true
+      true
+    elsif NOT_FOUND_PAGE_PATTERNS.any? { |pattern| response.body =~ pattern }
+      user.state_error!
+      true
+    else
+      false
+    end
+  end
 end

app/jobs/domain/fa/job/favs_job.rb

@@ -22,6 +22,7 @@ class Domain::Fa::Job::FavsJob < Domain::Fa::Job::Base
     full_scan = !!args[:full_scan]
     user = user_from_args!(create_if_missing: true)
     logger.push_tags(make_arg_tag(user))
+    return if buggy_user?(user)
 
     unless user_due_for_favs_scan?(user)
       logger.warn(format_tags("user not due for favs scan, skipping"))
@@ -137,16 +138,14 @@ class Domain::Fa::Job::FavsJob < Domain::Fa::Job::Base
       )
     end
 
-    if Domain::Fa::Job::ScanUserUtils.user_disabled_or_not_found?(
-         user,
-         response,
-       )
-      logger.error(format_tags("account disabled / not found", "aborting"))
-      user.scanned_favs_at = Time.zone.now
-      return ScanPageResult::Stop.new
-    end
+    disabled_or_not_found = user_disabled_or_not_found?(user, response)
+    user.scanned_favs_at = Time.current
+
+    return ScanPageResult::Stop.new if disabled_or_not_found
 
     page_parser = Domain::Fa::Parser::Page.new(response.body)
+    return ScanPageResult::Stop.new unless page_parser.probably_listings_page?
+
     listing_page_stats =
       update_and_enqueue_posts_from_listings_page(
         ListingPageType::FavsPage.new(page_number: @page_id, user:),
@@ -159,49 +158,5 @@ class Domain::Fa::Job::FavsJob < Domain::Fa::Job::Base
       posts_created_ids: listing_page_stats.new_posts.map(&:id).compact.to_set,
       keep_scanning: @page_id.present?,
     )
-
-    # unless page_parser.probably_listings_page?
-    #   fatal_error("not a favs listing page")
-    # end
-    # submissions = page_parser.submissions_parsed
-
-    # existing_fa_id_to_post_id =
-    #   Domain::Post::FaPost
-    #     .where(fa_id: submissions.map(&:id))
-    #     .pluck(:fa_id, :id)
-    #     .to_h
-
-    # created_posts = T.let([], T::Array[Domain::Post::FaPost])
-    # posts_to_save = T.let([], T::Array[Domain::Post::FaPost])
-    # submissions.each do |submission_parser_helper|
-    #   post =
-    #     Domain::Post::FaPost.find_or_initialize_by_submission_parser(
-    #       submission_parser_helper,
-    #       first_seen_log_entry: response.log_entry,
-    #     )
-    #   created_posts << post if post.new_record?
-
-    #   if post.new_record? || !post.state_ok?
-    #     posts_to_save << post
-    #     post.state_ok!
-    #     post.enqueue_job_after_save(
-    #       Domain::Fa::Job::ScanPostJob,
-    #       { post:, caused_by_entry: causing_log_entry },
-    #     )
-    #   end
-    # end
-
-    # bulk_enqueue_jobs { posts_to_save.each(&:save!) }
-
-    # last_page_post_ids = T.let(Set.new, T::Set[Integer])
-    # created_posts.each { |post| last_page_post_ids.add(T.must(post.id)) }
-    # existing_fa_id_to_post_id.values.each { |id| last_page_post_ids.add(id) }
-
-    # ScanPageResult::Ok.new(
-    #   faved_post_ids_on_page: last_page_post_ids,
-    #   posts_created_ids: created_posts.map(&:id).compact.to_set,
-    #   keep_scanning: @page_id.present?,
-    #   page_parser: page,
-    # )
   end
 end

app/jobs/domain/fa/job/scan_user_utils.rb

@@ -1,110 +0,0 @@
-# typed: strict
-class Domain::Fa::Job::ScanUserUtils
-  extend T::Sig
-  DISABLED_PAGE_PATTERNS =
-    T.let(
-      [
-        /User ".+" has voluntarily disabled access/,
-        /User ".+" was not found in our database\./,
-        /The page you are trying to reach is currently pending deletion/,
-        /The username ".+" could not be found\./,
-      ],
-      T::Array[Regexp],
-    )
-
-  sig do
-    params(
-      user: Domain::User::FaUser,
-      response: Scraper::HttpClient::Response,
-    ).returns(T::Boolean)
-  end
-  def self.user_disabled_or_not_found?(user, response)
-    if DISABLED_PAGE_PATTERNS.any? { |pattern| response.body =~ pattern }
-      user.state_account_disabled!
-      user.page_scan_error =
-        "account disabled or not found, see last_user_page_id"
-      user.last_user_page_id = response.log_entry.id
-      user.save!
-      true
-    else
-      false
-    end
-  end
-
-  module DisabledOrNotFoundResult
-    class Fatal < T::Struct
-      include T::Struct::ActsAsComparable
-
-      const :message, String
-    end
-
-    class Stop < T::Struct
-      include T::Struct::ActsAsComparable
-
-      const :message, String
-    end
-
-    class Ok < T::Struct
-      include T::Struct::ActsAsComparable
-
-      const :page, Domain::Fa::Parser::Page
-    end
-  end
-
-  sig do
-    params(
-      user: Domain::User::FaUser,
-      response: Scraper::HttpClient::Response,
-    ).returns(
-      T.any(
-        DisabledOrNotFoundResult::Fatal,
-        DisabledOrNotFoundResult::Stop,
-        DisabledOrNotFoundResult::Ok,
-      ),
-    )
-  end
-  def self.check_disabled_or_not_found(user, response)
-    if response.status_code != 200
-      return(
-        DisabledOrNotFoundResult::Fatal.new(
-          message:
-            "http #{response.status_code}, log entry #{response.log_entry.id}",
-        )
-      )
-    end
-
-    page = Domain::Fa::Parser::Page.new(response.body, require_logged_in: false)
-    if page.probably_user_page?
-      return DisabledOrNotFoundResult::Ok.new(page: page)
-    end
-
-    if response.body =~ /has voluntarily disabled access/
-      user.state = "ok"
-      user.is_disabled = true
-      user.last_user_page_id = response.log_entry.id
-      user.save!
-      try_name = /User "(.+)" has voluntarily disabled/.match(response.body)
-      user.name ||= try_name && try_name[1] || user.url_name
-      user.save!
-      return DisabledOrNotFoundResult::Stop.new(message: "account disabled")
-    end
-
-    if response.body =~ /This user cannot be found./ ||
-         response.body =~
-           /The page you are trying to reach is currently pending deletion/
-      user.state = "error"
-      user.page_scan_error = "account not found, see last_user_page_id"
-      user.last_user_page_id = response.log_entry.id
-      user.save!
-      user.name ||= user.url_name
-      user.save!
-      return DisabledOrNotFoundResult::Stop.new(message: "account not found")
-    end
-
-    return(
-      DisabledOrNotFoundResult::Fatal.new(
-        message: "not a user page - log entry #{response.log_entry.id}",
-      )
-    )
-  end
-end

app/jobs/domain/fa/job/user_avatar_job.rb

@@ -7,6 +7,7 @@ class Domain::Fa::Job::UserAvatarJob < Domain::Fa::Job::Base
   def perform(args)
     avatar = avatar_from_args!
     user = T.cast(avatar.user, Domain::User::FaUser)
+    return if buggy_user?(user)
 
     logger.push_tags(make_arg_tag(avatar), make_arg_tag(user))
 

app/jobs/domain/fa/job/user_follows_job.rb

@@ -26,6 +26,7 @@ class Domain::Fa::Job::UserFollowsJob < Domain::Fa::Job::Base
   sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
     user = user_from_args!
+    return if buggy_user?(user)
 
     if !user.follows_scan.due? && !force_scan?
       logger.warn("scanned #{user.follows_scan.ago_in_words}, skipping")

app/jobs/domain/fa/job/user_gallery_job.rb

@@ -28,15 +28,13 @@ class Domain::Fa::Job::UserGalleryJob < Domain::Fa::Job::Base
   sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
     user = user_from_args!
+    return if buggy_user?(user)
 
     if user.state != "ok" && user.scanned_gallery_at
       logger.warn("state == #{user.state} and already scanned, skipping")
       return
     end
 
-    # buggy (sentinal) user
-    return if user.id == 117_552 && user.url_name == "click here"
-
     @go_until_end = user.gallery_scan.at.nil?
 
     if !user.gallery_scan.due? && !force_scan?
@@ -95,16 +93,12 @@ class Domain::Fa::Job::UserGalleryJob < Domain::Fa::Job::Base
       response = http_client.get(page_url)
       fatal_error("failed to scan folder page") if response.status_code != 200
 
-      if Domain::Fa::Job::ScanUserUtils.user_disabled_or_not_found?(
-           user,
-           response,
-         )
-        logger.error("account disabled / not found, abort")
-        user.state = "account_disabled"
-        return :break
-      end
+      disabled_or_not_found = user_disabled_or_not_found?(user, response)
+      user.scanned_gallery_at = Time.current
+      return :break if disabled_or_not_found
 
       page = Domain::Fa::Parser::Page.new(response.body)
+      fatal_error("not a listings page") unless page.probably_listings_page?
 
       # newly instantiated users don't have a name yet, but can derive it from the gallery page
       user.name ||= page.user_page.name || user.url_name

app/jobs/domain/fa/job/user_incremental_job.rb

@@ -7,9 +7,7 @@ module Domain::Fa::Job
     def perform(args)
       user = user_from_args!
       logger.push_tags(make_arg_tag(user))
-
-      # buggy user
-      return if user.id == 117_552 && user.url_name == "click here"
+      return if buggy_user?(user)
 
       # this is similar to a user page job, and will update the user page
       # however, it will incrementally update user favs & follows / following:
@@ -31,33 +29,16 @@ module Domain::Fa::Job
 
       response =
         http_client.get("https://www.furaffinity.net/user/#{user.url_name}/")
-      logger.push_tags(make_arg_tag(response.log_entry))
-
-      ret =
-        Domain::Fa::Job::ScanUserUtils.check_disabled_or_not_found(
-          user,
-          response,
-        )
-
-      case ret
-      when ScanUserUtils::DisabledOrNotFoundResult::Ok
-        page = ret.page
-        logger.info(format_tags("user page is ok"))
-      when ScanUserUtils::DisabledOrNotFoundResult::Stop
-        logger.error(format_tags(ret.message))
-        return
-      when ScanUserUtils::DisabledOrNotFoundResult::Fatal
-        fatal_error(format_tags(ret.message))
+      logger.tagged(make_arg_tag(response.log_entry)) do
+        page = update_user_from_user_page(user, response)
+        if page
+          check_favs(user, page.user_page.recent_fav_fa_ids)
+          check_watchers(user, page.user_page.recent_watchers)
+          check_watching(user, page.user_page.recent_watching)
+        end
+        user.scanned_incremental_at = Time.current
+        logger.info(format_tags("completed page scan"))
       end
-
-      update_user_fields_from_page(user, page, response)
-
-      check_favs(user, page.user_page.recent_fav_fa_ids)
-      check_watchers(user, page.user_page.recent_watchers)
-      check_watching(user, page.user_page.recent_watching)
-
-      user.scanned_incremental_at = Time.current
-      logger.info(format_tags("completed page scan"))
     ensure
       user.save! if user
 

app/jobs/domain/fa/job/user_page_job.rb

@@ -1,6 +1,5 @@
 # typed: strict
 class Domain::Fa::Job::UserPageJob < Domain::Fa::Job::Base
-  ScanUserUtils = Domain::Fa::Job::ScanUserUtils
   queue_as :fa_user_page
   queue_with_priority do
     T.bind(self, Domain::Fa::Job::UserPageJob)
@@ -10,9 +9,7 @@ class Domain::Fa::Job::UserPageJob < Domain::Fa::Job::Base
   sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
     user = user_from_args!
-
-    # buggy (sentinal) user
-    return if user.id == 117_552 && user.url_name == "click here"
+    return if buggy_user?(user)
 
     if !user.page_scan.due? && !force_scan?
       logger.warn("scanned #{user.page_scan.ago_in_words}, skipping")
@@ -22,27 +19,132 @@ class Domain::Fa::Job::UserPageJob < Domain::Fa::Job::Base
     response =
       http_client.get("https://www.furaffinity.net/user/#{user.url_name}/")
 
-    ret =
-      Domain::Fa::Job::ScanUserUtils.check_disabled_or_not_found(user, response)
-    case ret
-    when ScanUserUtils::DisabledOrNotFoundResult::Ok
-      page = ret.page
-    when ScanUserUtils::DisabledOrNotFoundResult::Stop
-      logger.error(ret.message)
-      return
-    when ScanUserUtils::DisabledOrNotFoundResult::Fatal
-      fatal_error(ret.message)
+    page = update_user_from_user_page(user, response)
+    user_page = page&.user_page
+
+    if user.state_ok? && user_page
+      check_skip_gallery_scan(user)
+      check_skip_favs_scan(user, user_page)
+      check_skip_followed_users_scan(user, user_page)
+      check_skip_followed_by_users_scan(user, user_page)
     end
 
-    update_user_fields_from_page(user, page, response)
-    user.save!
+    enqueue_user_scan(user)
     logger.info "completed page scan"
   ensure
-    if response && response.status_code == 200
+    user.save! if user
+    if user && response && (response.status_code == 200)
       enqueue_jobs_from_found_links(
         response.log_entry,
         suppress_jobs: [{ job: self.class, url_name: user.url_name }],
       )
     end
   end
+
+  private
+
+  sig { params(user: Domain::User::FaUser).void }
+  def check_skip_gallery_scan(user)
+    # if the user has no submissions, we don't need to scan their gallery
+    if user.num_submissions == 0
+      logger.info(format_tags("skipping gallery scan, 0 submissions"))
+      user.scanned_gallery_at = Time.current
+    end
+  end
+
+  sig do
+    params(
+      user: Domain::User::FaUser,
+      user_page: Domain::Fa::Parser::UserPageHelper,
+    ).void
+  end
+  def check_skip_favs_scan(user, user_page)
+    recent_faved_fa_ids = user_page.recent_fav_fa_ids
+    if recent_faved_fa_ids.empty?
+      logger.info(format_tags("skipping favs scan, 0 favorites"))
+      user.scanned_favs_at = Time.current
+    elsif recent_faved_fa_ids.count < 8
+      logger.info(
+        format_tags(
+          "skipping favs scan, #{recent_faved_fa_ids.count} favorites < threshold",
+        ),
+      )
+      faved_posts =
+        recent_faved_fa_ids.map do |fa_id|
+          Domain::Post::FaPost.find_or_create_by(fa_id:)
+        end
+      user.user_post_favs.upsert_all(
+        faved_posts.map(&:id).compact.map { |post_id| { post_id: } },
+        unique_by: %i[user_id post_id],
+      )
+      user.scanned_favs_at = Time.current
+    end
+  end
+
+  sig do
+    params(
+      user: Domain::User::FaUser,
+      user_page: Domain::Fa::Parser::UserPageHelper,
+    ).void
+  end
+  def check_skip_followed_users_scan(user, user_page)
+    recent_watching = user_page.recent_watching
+    if recent_watching.empty?
+      logger.info(format_tags("skipping followed users scan, 0 watching"))
+      user.scanned_follows_at = Time.current
+    elsif recent_watching.count < 12
+      logger.info(
+        format_tags(
+          "skipping followed users scan, #{recent_watching.count} watching < threshold",
+        ),
+      )
+      watched_users =
+        recent_watching.map do |recent_user|
+          Domain::User::FaUser.find_or_create_by(
+            url_name: recent_user.url_name,
+          ) { |user| user.name = recent_user.name }
+        end
+
+      user.user_user_follows_from.upsert_all(
+        watched_users.map(&:id).compact.map { |user_id| { to_id: user_id } },
+        unique_by: %i[from_id to_id],
+      )
+      user.scanned_follows_at = Time.current
+    end
+  end
+
+  sig do
+    params(
+      user: Domain::User::FaUser,
+      user_page: Domain::Fa::Parser::UserPageHelper,
+    ).void
+  end
+  def check_skip_followed_by_users_scan(user, user_page)
+    recent_watchers = user_page.recent_watchers
+    if recent_watchers.empty?
+      logger.info(format_tags("skipping followed by scan, 0 watched"))
+      user.scanned_followed_by_at = Time.current
+    elsif recent_watchers.count < 12
+      logger.info(
+        format_tags(
+          "skipping followed by scan, #{recent_watchers.count} watchers < threshold",
+        ),
+      )
+      watched_by_users =
+        recent_watchers.map do |recent_user|
+          Domain::User::FaUser.find_or_create_by(
+            url_name: recent_user.url_name,
+          ) { |user| user.name = recent_user.name }
+        end
+
+      user.user_user_follows_to.upsert_all(
+        watched_by_users
+          .map(&:id)
+          .compact
+          .map { |user_id| { from_id: user_id } },
+        unique_by: %i[from_id to_id],
+      )
+      user.scanned_followed_by_at = Time.current
+    end
+  end
 end

app/lib/domain/fa/parser/user_page_helper.rb

@@ -230,6 +230,14 @@ class Domain::Fa::Parser::UserPageHelper < Domain::Fa::Parser::Base
       end
   end
 
+  class JSONSubmissionData < T::ImmutableStruct
+    include T::Struct::ActsAsComparable
+
+    const :fa_id, Integer
+    const :title, String
+    const :creator, Domain::User::FaUser
+  end
+
   class RecentUser < T::Struct
     include T::Struct::ActsAsComparable
     extend T::Sig

app/models/domain/user/fa_user.rb

@@ -17,6 +17,7 @@ class Domain::User::FaUser < Domain::User
   attr_json_due_timestamp :scanned_gallery_at, 3.years
   attr_json_due_timestamp :scanned_page_at, 3.months
   attr_json_due_timestamp :scanned_follows_at, 3.months
+  attr_json_due_timestamp :scanned_followed_by_at, 3.months
   attr_json_due_timestamp :scanned_favs_at, 1.month
   attr_json_due_timestamp :scanned_incremental_at, 1.month
   attr_json :registered_at, :datetime

sorbet/rbi/dsl/domain/user/fa_user.rbi

@@ -15,6 +15,9 @@ class Domain::User::FaUser
   sig { returns(HasTimestampsWithDueAt::TimestampScanInfo) }
   def favs_scan; end
 
+  sig { returns(HasTimestampsWithDueAt::TimestampScanInfo) }
+  def followed_by_scan; end
+
   sig { returns(HasTimestampsWithDueAt::TimestampScanInfo) }
   def follows_scan; end
 
@@ -2030,6 +2033,9 @@ class Domain::User::FaUser
     sig { void }
     def restore_scanned_favs_at!; end
 
+    sig { void }
+    def restore_scanned_followed_by_at!; end
+
     sig { void }
     def restore_scanned_follows_at!; end
 
@@ -2198,6 +2204,12 @@ class Domain::User::FaUser
     sig { returns(T::Boolean) }
     def saved_change_to_scanned_favs_at?; end
 
+    sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
+    def saved_change_to_scanned_followed_by_at; end
+
+    sig { returns(T::Boolean) }
+    def saved_change_to_scanned_followed_by_at?; end
+
     sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
     def saved_change_to_scanned_follows_at; end
 
@@ -2301,6 +2313,61 @@ class Domain::User::FaUser
     sig { void }
     def scanned_favs_at_will_change!; end
 
+    sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
+    def scanned_followed_by_at; end
+
+    sig { params(value: T.nilable(::ActiveSupport::TimeWithZone)).returns(T.nilable(::ActiveSupport::TimeWithZone)) }
+    def scanned_followed_by_at=(value); end
+
+    sig { returns(T::Boolean) }
+    def scanned_followed_by_at?; end
+
+    sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
+    def scanned_followed_by_at_before_last_save; end
+
+    sig { returns(T.untyped) }
+    def scanned_followed_by_at_before_type_cast; end
+
+    sig { returns(T::Boolean) }
+    def scanned_followed_by_at_came_from_user?; end
+
+    sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
+    def scanned_followed_by_at_change; end
+
+    sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
+    def scanned_followed_by_at_change_to_be_saved; end
+
+    sig do
+      params(
+        from: T.nilable(::ActiveSupport::TimeWithZone),
+        to: T.nilable(::ActiveSupport::TimeWithZone)
+      ).returns(T::Boolean)
+    end
+    def scanned_followed_by_at_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
+
+    sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
+    def scanned_followed_by_at_in_database; end
+
+    sig { returns(T.nilable([T.nilable(::ActiveSupport::TimeWithZone), T.nilable(::ActiveSupport::TimeWithZone)])) }
+    def scanned_followed_by_at_previous_change; end
+
+    sig do
+      params(
+        from: T.nilable(::ActiveSupport::TimeWithZone),
+        to: T.nilable(::ActiveSupport::TimeWithZone)
+      ).returns(T::Boolean)
+    end
+    def scanned_followed_by_at_previously_changed?(from: T.unsafe(nil), to: T.unsafe(nil)); end
+
+    sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
+    def scanned_followed_by_at_previously_was; end
+
+    sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
+    def scanned_followed_by_at_was; end
+
+    sig { void }
+    def scanned_followed_by_at_will_change!; end
+
     sig { returns(T.nilable(::ActiveSupport::TimeWithZone)) }
     def scanned_follows_at; end
 
@@ -2793,6 +2860,9 @@ class Domain::User::FaUser
     sig { returns(T::Boolean) }
     def will_save_change_to_scanned_favs_at?; end
 
+    sig { returns(T::Boolean) }
+    def will_save_change_to_scanned_followed_by_at?; end
+
     sig { returns(T::Boolean) }
     def will_save_change_to_scanned_follows_at?; end
 

spec/jobs/domain/fa/job/browse_page_job_spec.rb

@@ -91,31 +91,6 @@ describe Domain::Fa::Job::BrowsePageJob do
     end
   end
 
-  shared_examples "enqueue user gallery scan" do |expect_to_enqueue|
-    if expect_to_enqueue
-      it "enqueues user gallery job" do
-        expect(
-          SpecUtil.enqueued_job_args(Domain::Fa::Job::UserGalleryJob),
-        ).to match(
-          [
-            hash_including(
-              user: find_creator.call,
-              caused_by_entry: log_entries[0],
-            ),
-          ],
-        )
-      end
-    end
-
-    unless expect_to_enqueue
-      it "does not enqueue user gallery job" do
-        expect(SpecUtil.enqueued_jobs(Domain::Fa::Job::UserGalleryJob)).to eq(
-          [],
-        )
-      end
-    end
-  end
-
   it "enqueues one" do
     expect do
       ret = described_class.perform_later({})
@@ -329,7 +304,6 @@ describe Domain::Fa::Job::BrowsePageJob do
       include_examples "enqueue post scan", true
       include_examples "enqueue file scan", false
       include_examples "enqueue user page scan", true
-      include_examples "enqueue user gallery scan", true
     end
 
     context "and post page scanned" do
@@ -344,7 +318,6 @@ describe Domain::Fa::Job::BrowsePageJob do
       include_examples "enqueue post scan", false
       include_examples "enqueue file scan", true
       include_examples "enqueue user page scan", true
-      include_examples "enqueue user gallery scan", true
     end
 
     context "and post file scanned" do
@@ -359,7 +332,6 @@ describe Domain::Fa::Job::BrowsePageJob do
       include_examples "enqueue post scan", false
       include_examples "enqueue file scan", false
       include_examples "enqueue user page scan", true
-      include_examples "enqueue user gallery scan", true
     end
 
     context "and post is marked as removed" do
@@ -377,7 +349,6 @@ describe Domain::Fa::Job::BrowsePageJob do
       include_examples "enqueue post scan", true
       include_examples "enqueue file scan", false
       include_examples "enqueue user page scan", true
-      include_examples "enqueue user gallery scan", true
     end
 
     context "and post scanned but file is a terminal error state" do
@@ -392,7 +363,6 @@ describe Domain::Fa::Job::BrowsePageJob do
       include_examples "enqueue post scan", true
       include_examples "enqueue file scan", false
       include_examples "enqueue user page scan", true
-      include_examples "enqueue user gallery scan", true
     end
 
     context "and user gallery already scanned" do
@@ -406,7 +376,6 @@ describe Domain::Fa::Job::BrowsePageJob do
       include_examples "enqueue post scan", true
       include_examples "enqueue file scan", false
       include_examples "enqueue user page scan", true
-      include_examples "enqueue user gallery scan", false
     end
 
     context "and user page already scanned" do
@@ -420,7 +389,6 @@ describe Domain::Fa::Job::BrowsePageJob do
       include_examples "enqueue post scan", true
       include_examples "enqueue file scan", false
       include_examples "enqueue user page scan", false
-      include_examples "enqueue user gallery scan", true
     end
   end
 

spec/jobs/domain/fa/job/favs_job_spec.rb

@@ -35,7 +35,7 @@ describe Domain::Fa::Job::FavsJob do
     end
   end
 
-  shared_context "user is disabled" do
+  shared_context "user is not found" do
     let(:user_url_name) { "caffeinatedarson" }
     let(:client_mock_config) do
       [
@@ -52,6 +52,23 @@ describe Domain::Fa::Job::FavsJob do
     end
   end
 
+  shared_context "user is disabled" do
+    let(:user_url_name) { "ground-lion" }
+    let(:client_mock_config) do
+      [
+        {
+          uri: "https://www.furaffinity.net/favorites/ground-lion/",
+          status_code: 200,
+          content_type: "text/html",
+          contents:
+            SpecUtil.read_fixture_file(
+              "domain/fa/favorites/favs_account_disabled_user_ground_lion.html",
+            ),
+        },
+      ]
+    end
+  end
+
   context "the user does not yet exist" do
     include_context "user has no favs"
 
@@ -96,7 +113,27 @@ describe Domain::Fa::Job::FavsJob do
     end
   end
 
-  context "the user has been disabled" do
+  context "the user is not found" do
+    include_context "user exists"
+    include_context "user is not found"
+
+    it "marks the user as scanned" do
+      expect do
+        perform_now(args)
+        user.reload
+      end.to change(user, :scanned_favs_at).from(nil).to(
+        be_within(1.second).of(Time.now),
+      )
+    end
+    it "changes user account state to error" do
+      expect do
+        perform_now(args)
+        user.reload
+      end.to change(user, :state).from("ok").to("error")
+    end
+  end
+
+  context "the user is disabled" do
     include_context "user exists"
     include_context "user is disabled"
 

spec/jobs/domain/fa/job/scan_post_job_spec.rb

@@ -86,25 +86,6 @@ describe Domain::Fa::Job::ScanPostJob do
         SpecUtil.enqueued_job_args(Domain::Fa::Job::UserPageJob),
       ).to include(hash_including({ user: creator }))
     end
-
-    it "enqueues a user gallery job" do
-      creator = Domain::User::FaUser.find_by(url_name: "-creeps")
-      expect(creator).to be_nil
-      perform_now({ fa_id: 59_714_213 })
-      creator = Domain::User::FaUser.find_by(url_name: "-creeps")
-      expect(creator).not_to be_nil
-      expect(
-        SpecUtil.enqueued_job_args(Domain::Fa::Job::UserGalleryJob),
-      ).to include(hash_including({ user: creator }))
-    end
-
-    it "enqueues a user favs job" do
-      creator = Domain::User::FaUser.find_by(url_name: "-creeps")
-      expect(creator).to be_nil
-      perform_now({ fa_id: 59_714_213 })
-      creator = Domain::User::FaUser.find_by(url_name: "-creeps")
-      expect(creator).not_to be_nil
-    end
   end
 
   context "when scanning a post" do

spec/jobs/domain/fa/job/user_gallery_job_spec.rb

@@ -153,9 +153,6 @@ describe Domain::Fa::Job::UserGalleryJob do
           change do
             SpecUtil.enqueued_job_args(Domain::Fa::Job::ScanPostJob).size
           end.by(52),
-          change do
-            SpecUtil.enqueued_job_args(Domain::Fa::Job::FavsJob).size
-          end.by(1),
         ].reduce(:and)
 
         expect { perform_now(args) }.to(expectations)
@@ -247,11 +244,11 @@ describe Domain::Fa::Job::UserGalleryJob do
     include_context "user model exists"
     include_context "user is not found on fa"
 
-    it "updates user state" do
+    it "updates user state to error" do
       expect do
         perform_now(args)
         user.reload
-      end.to(change(user, :state).from("ok").to("account_disabled"))
+      end.to(change(user, :state).from("ok").to("error"))
     end
   end
 end

spec/jobs/domain/fa/job/user_page_job_spec.rb

@@ -25,19 +25,303 @@ describe Domain::Fa::Job::UserPageJob do
       ]
     end
 
-    it "succeeds" do
+    let(:user) { Domain::User::FaUser.find_by(url_name: "meesh") }
+
+    it "records the right stats" do
       perform_now({ url_name: "meesh" })
-      user = Domain::User::FaUser.find_by(url_name: "meesh")
       expect(user).to_not be_nil
-      avatar = user.avatar
-      expect(avatar).to_not be_nil
-      expect(avatar.url_str).to eq(
-        "https://a.furaffinity.net/1635789297/meesh.gif",
+      expect(user.num_pageviews).to eq(3_061_083)
+      expect(user.num_submissions).to eq(1590)
+      expect(user.num_favorites).to eq(1_422_886)
+      expect(user.num_comments_recieved).to eq(47_931)
+      expect(user.num_comments_given).to eq(17_741)
+      expect(user.num_journals).to eq(5)
+    end
+
+    it "enqueues a favs job scan" do
+      perform_now({ url_name: "meesh" })
+      expect(SpecUtil.enqueued_job_args(Domain::Fa::Job::FavsJob)).to match(
+        [hash_including(user:, caused_by_entry: @log_entries[0])],
       )
-      expect(avatar.state).to eq("pending")
+    end
+
+    context "the user does not yet exist" do
+      it "the user is created" do
+        expect do perform_now({ url_name: "meesh" }) end.to change {
+          Domain::User::FaUser.find_by(url_name: "meesh")
+        }.from(nil).to(be_present)
+      end
+
+      it "enqueues a user avatar job" do
+        perform_now({ url_name: "meesh" })
+        expect(user).to_not be_nil
+        avatar = user.avatar
+        expect(avatar).to_not be_nil
+        expect(avatar.url_str).to eq(
+          "https://a.furaffinity.net/1635789297/meesh.gif",
+        )
+        expect(avatar.state).to eq("pending")
+        expect(
+          SpecUtil.enqueued_job_args(Domain::Fa::Job::UserAvatarJob),
+        ).to match([hash_including(avatar:, caused_by_entry: @log_entries[0])])
+      end
+
+      it "enqueues a gallery job" do
+        perform_now({ url_name: "meesh" })
+        expect(user).to_not be_nil
+        expect(
+          SpecUtil.enqueued_job_args(Domain::Fa::Job::UserGalleryJob),
+        ).to match([hash_including(user:, caused_by_entry: @log_entries[0])])
+      end
+    end
+
+    context "the user exists" do
+      let!(:user) { create(:domain_user_fa_user, url_name: "meesh") }
+      context "gallery scan was recently performed" do
+        before do
+          user.scanned_gallery_at = 1.day.ago
+          user.save!
+        end
+        it "does not enqueue a gallery job" do
+          perform_now({ url_name: "meesh" })
+          expect(
+            SpecUtil.enqueued_job_args(Domain::Fa::Job::UserGalleryJob),
+          ).to be_empty
+        end
+      end
+
+      context "the gallery scan was not recently performed" do
+        before do
+          user.scanned_gallery_at = 10.years.ago
+          user.save!
+        end
+        it "enqueues a gallery job" do
+          perform_now({ url_name: "meesh" })
+          expect(
+            SpecUtil.enqueued_job_args(Domain::Fa::Job::UserGalleryJob),
+          ).to match([hash_including(user:, caused_by_entry: @log_entries[0])])
+        end
+      end
+    end
+  end
+
+  context "all watched users fit in the recently watched section" do
+    let(:client_mock_config) do
+      [
+        {
+          uri: "https://www.furaffinity.net/user/llllvi/",
+          status_code: 200,
+          content_type: "text/html",
+          contents:
+            SpecUtil.read_fixture_file(
+              "domain/fa/user_page/user_page_llllvi_few_watched_users.html",
+            ),
+        },
+      ]
+    end
+
+    let(:user) { Domain::User::FaUser.find_by(url_name: "llllvi") }
+
+    it "does not enqueue a follows job" do
+      perform_now({ url_name: "llllvi" })
       expect(
-        SpecUtil.enqueued_job_args(Domain::Fa::Job::UserAvatarJob),
-      ).to match([hash_including(avatar:, caused_by_entry: @log_entries[0])])
+        SpecUtil.enqueued_job_args(Domain::Fa::Job::UserFollowsJob),
+      ).to be_empty
+    end
+
+    it "adds watched users to the user's followed_users" do
+      perform_now({ url_name: "llllvi" })
+      expect(user.followed_users.count).to eq(6)
+      expect(user.followed_users.map(&:url_name)).to match_array(
+        %w[
+          koul
+          artii
+          aquadragon35
+          incredibleediblecalico
+          nummynumz
+          fidchellvore
+        ],
+      )
+    end
+
+    it "marks scanned_follows_at as recent" do
+      perform_now({ url_name: "llllvi" })
+      expect(user.scanned_follows_at).to be_within(3.seconds).of(Time.current)
+    end
+
+    it "does not add any users to followed_by_users" do
+      perform_now({ url_name: "llllvi" })
+      expect(user.followed_by_users.count).to eq(0)
+    end
+
+    it "works when the user already has some followed users" do
+      user = create(:domain_user_fa_user, url_name: "llllvi")
+      followed_user = create(:domain_user_fa_user, url_name: "koul")
+      user.followed_users << followed_user
+      perform_now({ url_name: "llllvi" })
+      expect(user.followed_users.count).to eq(6)
+      expect(user.followed_by_users.count).to eq(0)
+    end
+  end
+
+  context "the user has no submissions" do
+    let(:client_mock_config) do
+      [
+        {
+          uri: "https://www.furaffinity.net/user/sealingthedeal/",
+          status_code: 200,
+          content_type: "text/html",
+          contents:
+            SpecUtil.read_fixture_file(
+              "domain/fa/user_page/user_page_sealingthedeal_no_submissions.html",
+            ),
+        },
+      ]
+    end
+
+    it "records the right number of submissions" do
+      perform_now({ url_name: "sealingthedeal" })
+      user = Domain::User::FaUser.find_by(url_name: "sealingthedeal")
+      expect(user).to_not be_nil
+      expect(user.num_submissions).to eq(0)
+    end
+
+    it "does not enqueue a gallery job" do
+      perform_now({ url_name: "sealingthedeal" })
+      expect(
+        SpecUtil.enqueued_job_args(Domain::Fa::Job::UserGalleryJob),
+      ).to be_empty
+    end
+  end
+
+  context "the user has a single scrap submission and few watchers" do
+    let(:client_mock_config) do
+      [
+        {
+          uri: "https://www.furaffinity.net/user/zzreg/",
+          status_code: 200,
+          content_type: "text/html",
+          contents:
+            SpecUtil.read_fixture_file(
+              "domain/fa/user_page/user_page_zzreg_one_scrap_submission.html",
+            ),
+        },
+      ]
+    end
+
+    let(:user) { Domain::User::FaUser.find_by(url_name: "zzreg") }
+
+    it "records the right number of submissions" do
+      perform_now({ url_name: "zzreg" })
+      expect(user.num_submissions).to eq(1)
+    end
+
+    it "enqueues a gallery job" do
+      perform_now({ url_name: "zzreg" })
+      expect(
+        SpecUtil.enqueued_job_args(Domain::Fa::Job::UserGalleryJob),
+      ).to match([hash_including(user:, caused_by_entry: @log_entries[0])])
+    end
+
+    it "adds watchers to followed_by_users" do
+      perform_now({ url_name: "zzreg" })
+      expect(user.followed_by_users.count).to eq(5)
+      expect(user.followed_by_users.map(&:url_name)).to match_array(
+        %w[noneedtothankme karenpls azureparagon zarmir iginger],
+      )
+    end
+
+    it "works when the user already has some followed_by_users" do
+      user = create(:domain_user_fa_user, url_name: "zzreg")
+      followed_by_user =
+        create(:domain_user_fa_user, url_name: "noneedtothankme")
+      user.followed_by_users << followed_by_user
+      perform_now({ url_name: "zzreg" })
+      expect(user.followed_by_users.count).to eq(5)
+      expect(user.followed_users.count).to eq(0)
+    end
+
+    it "marks scanned_followed_by_at as recent" do
+      perform_now({ url_name: "zzreg" })
+      expect(user.scanned_followed_by_at).to be_within(3.seconds).of(
+        Time.current,
+      )
+    end
+
+    it "does not mark scanned_follows_at as recent" do
+      perform_now({ url_name: "zzreg" })
+      expect(user.scanned_follows_at).to be_nil
+    end
+  end
+
+  context "the user has no recent favories" do
+    let(:client_mock_config) do
+      [
+        {
+          uri: "https://www.furaffinity.net/user/angu/",
+          status_code: 200,
+          content_type: "text/html",
+          contents:
+            SpecUtil.read_fixture_file(
+              "domain/fa/user_page/user_page_angu_no_recent_favorites.html",
+            ),
+        },
+      ]
+    end
+
+    let(:user) { Domain::User::FaUser.find_by(url_name: "angu") }
+
+    it "does not enqueue a favs job" do
+      perform_now({ url_name: "angu" })
+      expect(SpecUtil.enqueued_job_args(Domain::Fa::Job::FavsJob)).to be_empty
+    end
+
+    it "marks scanned_favs_at as recent" do
+      perform_now({ url_name: "angu" })
+      expect(user.scanned_favs_at).to be_within(3.seconds).of(Time.current)
+    end
+  end
+
+  context "all favorites fit in the recently faved section" do
+    let(:client_mock_config) do
+      [
+        {
+          uri: "https://www.furaffinity.net/user/lleaued/",
+          status_code: 200,
+          content_type: "text/html",
+          contents:
+            SpecUtil.read_fixture_file(
+              "domain/fa/user_page/user_page_lleaued_few_recent_favorites.html",
+            ),
+        },
+      ]
+    end
+
+    let(:user) { Domain::User::FaUser.find_by(url_name: "lleaued") }
+
+    it "does not enqueue a favs job" do
+      perform_now({ url_name: "lleaued" })
+      expect(SpecUtil.enqueued_job_args(Domain::Fa::Job::FavsJob)).to be_empty
+    end
+
+    it "marks scanned_favs_at as recent" do
+      perform_now({ url_name: "lleaued" })
+      expect(user.scanned_favs_at).to be_within(3.seconds).of(Time.current)
+    end
+
+    it "adds posts to the user's favorites" do
+      perform_now({ url_name: "lleaued" })
+      expect(user.faved_posts.count).to eq(1)
+      expect(user.faved_posts.map(&:fa_id)).to eq([51_355_154])
+    end
+
+    it "works when the user already has some favorites" do
+      user = create(:domain_user_fa_user, url_name: "lleaued")
+      post = create(:domain_post_fa_post, fa_id: 51_355_154)
+      user.user_post_favs.create!(post_id: post.id)
+      perform_now({ url_name: "lleaued" })
+      expect(user.faved_posts.count).to eq(1)
+      expect(user.faved_posts.map(&:fa_id)).to eq([51_355_154])
     end
   end