fix for routes

This commit introduces comprehensive support for post groups across different domains:
- Created PostGroupsController to handle viewing post groups
- Added new views for displaying post groups and their associated posts
- Implemented policies for post groups
- Enhanced models to support post group functionality
- Updated routes to support post group navigation
- Added helper methods for post group interactions
- Improved GoodJob argument rendering for post groups

The changes provide a unified way to view and interact with post collections across different domains like Inkbunny and E621.

.devcontainer/Dockerfile.devcontainer

@@ -49,12 +49,16 @@ RUN \
     libvips42 \
     libyaml-dev \
     patch \
-    rustc \
     uuid-dev \
     zlib1g-dev \
-    watchman
+    watchman \
+    ffmpeg \
+    ffmpegthumbnailer \
+    abiword \
+    pdftohtml \
+    libreoffice
 
-# Install postgres 17 client
+# Install postgres 15 client
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
     sudo install -d /usr/share/postgresql-common/pgdg && \
@@ -62,7 +66,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
     apt update && \
     apt-get install --no-install-recommends --no-install-suggests -y \
-    postgresql-client-17
+    postgresql-client-15
 
 # Install & configure delta diff tool
 RUN wget -O- https://github.com/dandavison/delta/releases/download/0.18.2/git-delta_0.18.2_amd64.deb > /tmp/git-delta.deb && \

.devcontainer/Dockerfile.postgres

@@ -1,7 +1,7 @@
-FROM postgres:17
+FROM postgres:15
 
 RUN apt-get update && apt-get install -y \
-    postgresql-17-pgvector \
+    postgresql-15-pgvector \
     && rm -rf /var/lib/apt/lists/*
 
 COPY create-tablespaces.bash /docker-entrypoint-initdb.d/00-create-tablespaces.bash

.devcontainer/docker-compose.yml

@@ -32,7 +32,7 @@ services:
       POSTGRES_PASSWORD: postgres
 
   pgadmin:
-    image: dpage/pgadmin4:8.14.0
+    image: dpage/pgadmin4:9
     restart: unless-stopped
     environment:
       PGADMIN_DEFAULT_EMAIL: admin@example.com

.devcontainer/fish-shell-conf-d/rustup.fish

@@ -0,0 +1 @@
+source "$HOME/.cargo/env.fish"

.devcontainer/install-extensions.sh

@@ -18,3 +18,5 @@ install_extension KoichiSasada.vscode-rdbg
 install_extension qwtel.sqlite-viewer
 install_extension esbenp.prettier-vscode
 install_extension ms-azuretools.vscode-docker
+install_extension 1YiB.rust-bundle
+install_extension rust-lang.rust-analyzer

.dockerignore

@@ -12,3 +12,4 @@ launch.json
 settings.json
 *.export
 .devcontainer
+user_scripts/dist

.gitignore

@@ -11,6 +11,8 @@ core
 lib/xdiff
 ext/xdiff/Makefile
 ext/xdiff/xdiff
+user_scripts/dist
+migrated_files.txt
 
 # use yarn to manage node_modules
 package-lock.json

.vscode/settings.json

@@ -12,7 +12,7 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "[erb]": {
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
+    "editor.defaultFormatter": "aliariff.vscode-erb-beautify"
   },
   "[typescript]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"

Dockerfile

@@ -35,6 +35,7 @@ RUN \
 
 # preinstall gems that take a long time to install
 RUN MAKE="make -j12" gem install bundler           -v '2.5.6' --verbose
+RUN MAKE="make -j12" gem install rice              -v '4.3.3' --verbose
 RUN MAKE="make -j12" gem install faiss             -v '0.3.2' --verbose
 RUN MAKE="make -j12" gem install rails_live_reload -v '0.3.6' --verbose
 RUN bundle config --global frozen 1
@@ -51,7 +52,17 @@ RUN \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
     apt-get update && \
     apt-get install --no-install-recommends --no-install-suggests -y \
-    libvips42 ca-certificates curl gnupg nodejs libpq-dev
+    libvips42 \
+    ca-certificates \
+    curl \
+    gnupg \
+    nodejs \
+    libpq-dev \
+    ffmpeg \
+    ffmpegthumbnailer \
+    abiword \
+    pdftohtml \
+    libreoffice
 
 COPY --from=native-gems /usr/src/app/gems/xdiff-rb /gems/xdiff-rb
 COPY --from=native-gems /usr/src/app/gems/rb-bsdiff /gems/rb-bsdiff
@@ -72,6 +83,9 @@ COPY . .
 RUN RAILS_ENV=production bin/rails assets:precompile
 RUN mkdir -p tmp/pids
 
+# build user scripts
+RUN yarn build:user-scripts
+
 # create user with id=1000 gid=1000
 RUN groupadd -g 1000 app && \
     useradd -m -d /home/app -s /bin/bash -u 1000 -g 1000 app

Gemfile

@@ -19,6 +19,7 @@ gem "pry-stack_explorer"
 
 # Use the Puma web server [https://github.com/puma/puma]
 gem "puma", "~> 5.0"
+gem "thruster"
 
 # # Use JavaScript with ESM import maps [https://github.com/rails/importmap-rails]
 # gem "importmap-rails"
@@ -124,8 +125,16 @@ gem "ripcord"
 gem "ruby-prof"
 gem "ruby-prof-speedscope"
 gem "ruby-vips"
+gem "dhash-vips"
+gem "ffmpeg", git: "https://github.com/instructure/ruby-ffmpeg", tag: "v6.1.2"
 gem "table_print"
 gem "zstd-ruby"
+gem "rouge"
+gem "docx"
+gem "ruby-bbcode"
+gem "dtext_rb",
+    git: "https://github.com/e621ng/dtext_rb",
+    ref: "5ef8fd7a5205c832f4c18197911717e7d491494e"
 
 # gem "pghero", git: "https://github.com/dymk/pghero", ref: "e314f99"
 gem "pghero", "~> 3.6"
@@ -143,6 +152,7 @@ end
 
 group :production do
   gem "sd_notify"
+  gem "cloudflare-rails"
 end
 
 gem "rack", "~> 2.2"
@@ -150,6 +160,7 @@ gem "rack-cors"
 gem "react_on_rails"
 gem "sanitize", "~> 6.1"
 gem "shakapacker", "~> 6.6"
+gem "timeout"
 
 group :development do
   gem "prettier_print"
@@ -169,5 +180,6 @@ gem "pundit", "~> 2.4"
 gem "prometheus_exporter", "~> 2.2"
 
 gem "sorbet-static-and-runtime"
-gem "tapioca", require: false, group: %i[development test]
+gem "tapioca", require: false
 gem "rspec-sorbet", group: [:test]
+gem "sorbet-struct-comparable"

Gemfile.lock

@@ -1,3 +1,18 @@
+GIT
+  remote: https://github.com/e621ng/dtext_rb
+  revision: 5ef8fd7a5205c832f4c18197911717e7d491494e
+  ref: 5ef8fd7a5205c832f4c18197911717e7d491494e
+  specs:
+    dtext_rb (1.11.0)
+
+GIT
+  remote: https://github.com/instructure/ruby-ffmpeg
+  revision: a3404b8fa275e2eb9549f074906461b0266a70ea
+  tag: v6.1.2
+  specs:
+    ffmpeg (6.1.2)
+      multi_json (~> 1.8)
+
 GIT
   remote: https://github.com/railsjazz/rails_live_reload
   revision: dcd3b73904594e2c5134c2f6e05954f3937a8d29
@@ -116,6 +131,11 @@ GEM
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
+    cloudflare-rails (6.2.0)
+      actionpack (>= 7.1.0, < 8.1.0)
+      activesupport (>= 7.1.0, < 8.1.0)
+      railties (>= 7.1.0, < 8.1.0)
+      zeitwerk (>= 2.5.0)
     coderay (1.1.3)
     colorize (1.1.0)
     concurrent-ruby (1.3.4)
@@ -143,6 +163,8 @@ GEM
       railties (>= 4.1.0)
       responders
       warden (~> 1.2.3)
+    dhash-vips (0.2.3.0)
+      ruby-vips (~> 2.0, != 2.1.1, != 2.1.0)
     diff-lcs (1.5.1)
     diffy (3.4.3)
     discard (1.4.0)
@@ -150,6 +172,9 @@ GEM
     disco (0.5.1)
       libmf (>= 0.4)
       numo-narray (>= 0.9.2)
+    docx (0.8.0)
+      nokogiri (~> 1.13, >= 1.13.0)
+      rubyzip (~> 2.0)
     domain_name (0.6.20240107)
     drb (2.2.1)
     erubi (1.13.1)
@@ -252,6 +277,7 @@ GEM
     mini_mime (1.1.5)
     minitest (5.25.4)
     msgpack (1.7.5)
+    multi_json (1.15.0)
     neighbor (0.5.1)
       activerecord (>= 7)
     net-imap (0.5.4)
@@ -392,6 +418,7 @@ GEM
     rexml (3.4.0)
     rice (4.3.3)
     ripcord (2.0.0)
+    rouge (4.5.1)
     rspec (3.13.0)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -415,6 +442,8 @@ GEM
     rspec-sorbet (1.9.2)
       sorbet-runtime
     rspec-support (3.13.2)
+    ruby-bbcode (2.1.1)
+      activesupport (>= 4.2.2)
     ruby-prof (1.7.1)
     ruby-prof-speedscope (0.3.0)
       ruby-prof (~> 1.0)
@@ -451,6 +480,8 @@ GEM
     sorbet-static-and-runtime (0.5.11711)
       sorbet (= 0.5.11711)
       sorbet-runtime (= 0.5.11711)
+    sorbet-struct-comparable (1.3.0)
+      sorbet-runtime (>= 0.5)
     spoom (1.5.0)
       erubi (>= 1.10.0)
       prism (>= 0.28.0)
@@ -491,6 +522,10 @@ GEM
       thor (>= 1.2.0)
       yard-sorbet
     thor (1.3.2)
+    thruster (0.1.11-aarch64-linux)
+    thruster (0.1.11-arm64-darwin)
+    thruster (0.1.11-x86_64-darwin)
+    thruster (0.1.11-x86_64-linux)
     timeout (0.4.3)
     turbo-rails (2.0.11)
       actionpack (>= 6.0.0)
@@ -538,6 +573,7 @@ DEPENDENCIES
   attr_json
   bootsnap
   capybara
+  cloudflare-rails
   colorize
   concurrent-ruby-edge
   concurrent-ruby-ext
@@ -547,11 +583,15 @@ DEPENDENCIES
   db-query-matchers (~> 0.14)
   debug (~> 1.10)
   devise (~> 4.9)
+  dhash-vips
   diffy
   discard
   disco
+  docx
+  dtext_rb!
   factory_bot_rails
   faiss
+  ffmpeg!
   good_job (~> 4.6)
   htmlbeautifier
   http (~> 5.2)
@@ -584,8 +624,10 @@ DEPENDENCIES
   rb-bsdiff!
   react_on_rails
   ripcord
+  rouge
   rspec-rails (~> 7.0)
   rspec-sorbet
+  ruby-bbcode
   ruby-prof
   ruby-prof-speedscope
   ruby-vips
@@ -596,6 +638,7 @@ DEPENDENCIES
   shakapacker (~> 6.6)
   shoulda-matchers
   sorbet-static-and-runtime
+  sorbet-struct-comparable
   sprockets-rails
   sqlite3 (~> 1.4)
   stackprof
@@ -604,6 +647,8 @@ DEPENDENCIES
   table_print
   tailwindcss-rails (~> 3.0)
   tapioca
+  thruster
+  timeout
   turbo-rails
   tzinfo-data
   web-console

Procfile.dev

@@ -1,5 +1,5 @@
-rails: RAILS_ENV=development bundle exec rails s -p 3000
+rails: RAILS_ENV=development HTTP_PORT=3000 TARGET_PORT=3003 rdbg --command --nonstop --open -- thrust ./bin/rails server -p 3003
 wp-client: RAILS_ENV=development HMR=true ./bin/webpacker-dev-server
 wp-server: RAILS_ENV=development HMR=true SERVER_BUNDLE_ONLY=yes ./bin/webpacker --watch
-css: RAILS_ENV=development yarn "build:css[debug]" --watch
+css: tailwindcss -c ./config/tailwind.config.js -i ./app/assets/stylesheets/application.tailwind.css -o ./app/assets/builds/tailwind.css --watch
 prometheus_exporter: RAILS_ENV=development bundle exec prometheus_exporter --bind 0.0.0.0 --prefix redux_ --label '{"environment": "development"}'

Procfile.production

@@ -1,3 +1,3 @@
-rails: RAILS_ENV=production bundle exec rails s -b 0.0.0.0 -p 3000
+rails: RAILS_ENV=production HTTP_PORT=3000 TARGET_PORT=3003 thrust ./bin/rails server -p 3003
 tail: tail -f log/production.log
 prometheus_exporter: RAILS_ENV=production bundle exec prometheus_exporter --bind 0.0.0.0 --prefix redux_ --label '{"environment": "production"}'

Procfile.staging

@@ -1,4 +1,4 @@
-rails: RAILS_ENV=staging ./bin/rails s -p 3001
+rails: RAILS_ENV=staging HTTP_PORT=3001 TARGET_PORT=3002 bundle exec thrust ./bin/rails server -p 3002
 wp-client: RAILS_ENV=development HMR=true ./bin/webpacker-dev-server
 wp-server: RAILS_ENV=development HMR=true SERVER_BUNDLE_ONLY=yes ./bin/webpacker --watch
 css: RAILS_ENV=development yarn "build:css[debug]" --watch

Rakefile

@@ -92,3 +92,528 @@ task :reverse_csv do
   in_csv.reverse_each { |row| out_csv << row.map(&:second) }
   out_csv.close
 end
+
+task migrate_to_domain: :environment do
+  only_user = ENV["only_user"]
+  allowed_domains = %w[e621 fa ib]
+  only_domains = (ENV["only_domains"] || "").split(",")
+  only_domains = allowed_domains if only_domains.empty?
+  if (only_domains - allowed_domains).any?
+    raise "only_domains must be a subset of #{allowed_domains.join(", ")}"
+  end
+
+  migrator = Domain::MigrateToDomain.new
+
+  if only_domains.include?("e621")
+    # migrator.migrate_e621_users(only_user: only_user)
+    # migrator.migrate_e621_posts(only_user: only_user)
+    migrator.migrate_e621_users_favs(only_user: only_user)
+  end
+
+  if only_domains.include?("fa")
+    # migrator.migrate_fa_users(only_user: only_user)
+    # migrator.migrate_fa_posts(only_user: only_user)
+    # migrator.migrate_fa_users_favs(only_user: only_user)
+    migrator.migrate_fa_users_followed_users(only_user: only_user)
+  end
+
+  if only_domains.include?("ib")
+    migrator.migrate_inkbunny_users(only_user: only_user)
+    migrator.migrate_inkbunny_posts(only_user: only_user)
+    migrator.migrate_inkbunny_pools(only_user: nil) if only_user.nil?
+  end
+end
+
+task infer_last_submission_log_entries: :environment do
+  only_fa_id = ENV["only_fa_id"]
+  start = ENV["start_at"]&.to_i || nil
+
+  if only_fa_id
+    relation = Domain::Fa::Post.where(fa_id: only_fa_id)
+  else
+    relation =
+      Domain::Fa::Post
+        .where(state: :ok)
+        .where(last_submission_page_id: nil)
+        .or(Domain::Fa::Post.where(state: :ok).where(posted_at: nil))
+  end
+
+  relation.find_each(batch_size: 10, start:) do |post|
+    parts = ["[id: #{post.id}]", "[fa_id: #{post.fa_id}]"]
+
+    log_entry = post.guess_last_submission_page
+    unless log_entry
+      parts << "[no log entry]"
+      next
+    end
+
+    contents = log_entry.response&.contents
+    unless contents
+      parts << "[no contents]"
+      next
+    end
+
+    parser = Domain::Fa::Parser::Page.new(contents)
+    if parser.submission_not_found?
+      parts << "[removed]"
+      post.state = :removed
+    else
+      posted_at = parser.submission.posted_date
+      post.posted_at ||= posted_at
+      parts << "[posted at: #{posted_at}]"
+    end
+
+    if post.last_submission_page_id.present? &&
+         log_entry.id != post.last_submission_page_id
+      parts << "[overwrite]"
+    end
+    post.last_submission_page_id = log_entry.id
+
+    parts << "[log entry: #{log_entry.id}]"
+    parts << "[uri: #{log_entry.uri.to_s}]"
+    post.save!
+  rescue => e
+    parts << "[error: #{e.message}]"
+  ensure
+    puts parts.join(" ")
+  end
+end
+
+task fix_fa_post_files: :environment do
+  file_ids = ENV["file_ids"]&.split(",") || raise("need 'file_ids'")
+  Domain::Fa::Post
+    .where(file_id: file_ids)
+    .find_each { |post| post.fix_file_by_uri! }
+end
+
+task fix_fa_post_files_by_csv: :environment do
+  require "csv"
+
+  csv_file = ENV["csv_file"] || raise("need 'csv_file'")
+  CSV
+    .open(csv_file, headers: true)
+    .each do |row|
+      id = row["id"].to_i
+      post = Domain::Fa::Post.find(id)
+      post.fix_file_by_uri!
+    end
+end
+
+task fix_buggy_fa_posts: :environment do
+  post_fa_ids = %w[7704069 7704068 6432347 6432346].map(&:to_i)
+
+  require "uri"
+
+  post_fa_ids.each do |fa_id|
+    post = Domain::Fa::Post.find_by(fa_id: fa_id)
+    next unless post&.file
+    post_file_url_str = Addressable::URI.parse(post.file_url_str).to_s
+    file_url_str = Addressable::URI.parse(CGI.unescape(post.file.uri.to_s)).to_s
+    hle = post.guess_last_submission_page
+
+    parser = Domain::Fa::Parser::Page.new(hle.response.contents)
+    if parser.submission_not_found?
+      post.file = nil
+      post.save!
+      puts "submission not found"
+    else
+      submission = parser.submission
+      full_res_img = Addressable::URI.parse(submission.full_res_img)
+      full_res_img.scheme = "https" if full_res_img.scheme.blank?
+      matches = full_res_img.to_s == post.file_url_str
+    end
+  end
+end
+
+task enqueue_fa_posts_missing_files: %i[environment set_logger_stdout] do
+  Domain::Post::FaPost
+    .where(state: "ok")
+    .where
+    .missing(:file)
+    .find_each(order: :desc) do |post|
+      Domain::Fa::Job::ScanPostJob.perform_now(post:)
+    end
+end
+
+task fix_e621_post_files: :environment do
+  query = Domain::Post::E621Post.where(state: "ok").where.missing(:files)
+  limit = ENV["limit"]&.to_i
+  puts "query: #{query.to_sql}"
+
+  query.find_each(batch_size: 10) do |post|
+    Domain::E621::Task::FixE621PostMissingFiles.new.run(post)
+    if limit
+      limit -= 1
+      if limit.zero?
+        puts "limit reached"
+        break
+      end
+    end
+  end
+end
+
+task fix_ok_e621_posts_missing_files: :environment do
+  query = Domain::Post::E621Post.where(state: "ok").where.missing(:file)
+  progress_bar =
+    ProgressBar.create(total: query.count, format: "%t: %c/%C %B %p%% %a %e")
+  query.find_each(batch_size: 10) do |post|
+    Domain::E621::Job::ScanPostJob.perform_now(post:)
+    progress_bar.progress = [progress_bar.progress + 1, progress_bar.total].min
+  end
+end
+
+task perform_good_jobs: :environment do
+  job_class = ENV["job_class"]
+  job_id = ENV["job_id"]
+  limit = ENV["limit"]&.to_i
+
+  if !job_id.present? && !job_class.present?
+    raise "need 'job_id' or 'job_class'"
+  end
+
+  relation =
+    if job_id
+      job =
+        GoodJob::Job.find_by(id: job_id) ||
+          GoodJob::Execution.find_by(id: job_id)&.job
+      if job.nil?
+        puts "no job found with id #{job_id}"
+        exit 1
+      end
+      puts "found job with id #{job.id}" if job.id != job_id
+      GoodJob::Job.where(id: job.id)
+    else
+      GoodJob::Job.queued.where(job_class: job_class).order(created_at: :asc)
+    end
+
+  relation.find_each(batch_size: 1) do |job|
+    job = T.cast(job, GoodJob::Job)
+
+    # Get the actual job instance and deserialize arguments
+    serialized_args = job.serialized_params["arguments"]
+    if serialized_args.nil?
+      puts "No arguments found for job #{job.id}"
+      next
+    end
+
+    deserialized_args = ActiveJob::Arguments.deserialize(serialized_args)
+    job_instance = job.job_class.constantize.new
+    job_instance.deserialize(job.serialized_params)
+
+    puts "Running job #{job.id} (#{job.job_class})"
+
+    # Create execution record
+    execution =
+      GoodJob::Execution.create!(
+        active_job_id: job.active_job_id,
+        job_class: job.job_class,
+        queue_name: job.queue_name,
+        serialized_params: job.serialized_params,
+        scheduled_at: job.scheduled_at,
+        created_at: Time.current,
+        updated_at: Time.current,
+        process_id: SecureRandom.uuid,
+      )
+
+    start_time = Time.current
+
+    # Temporarily disable concurrency limits
+    job_class = job.job_class.constantize
+    old_config = job_class.good_job_concurrency_config
+    job_class.good_job_concurrency_config = { total_limit: nil }
+
+    begin
+      # Perform the job with deserialized arguments
+      GoodJob::CurrentThread.job = job
+      job.update!(performed_at: Time.current)
+      job_instance.arguments = deserialized_args
+      job_instance.perform_now
+
+      # Update execution and job records
+      execution.update!(
+        finished_at: Time.current,
+        error: nil,
+        error_event: nil,
+        duration: Time.current - start_time,
+      )
+      job.update!(finished_at: Time.current)
+      puts "Job completed successfully"
+    rescue => e
+      puts "Job failed: #{e.message}"
+      # Update execution and job records with error
+      execution.update!(
+        finished_at: Time.current,
+        error: e.message,
+        error_event: "execution_failed",
+        error_backtrace: e.backtrace,
+        duration: Time.current - start_time,
+      )
+      job.update!(
+        error: "#{e.class}: #{e.message}",
+        error_event: "execution_failed",
+      )
+      raise e
+    ensure
+      job.update!(
+        executions_count: GoodJob::Execution.where(active_job_id: job.id).count,
+      )
+      # Restore original concurrency config
+      job_class.good_job_concurrency_config = old_config
+      GoodJob::CurrentThread.job = nil
+    end
+
+    if limit
+      limit -= 1
+      if limit.zero?
+        puts "limit reached"
+        break
+      end
+    end
+  end
+end
+
+task fix_removed_fa_posts: :environment do
+  colorize_state = ->(state) do
+    case state
+    when "ok"
+      "ok".green
+    when "removed"
+      "removed".red
+    else
+      state.to_s
+    end.bold
+  end
+
+  last_fa_id = ENV["start_at"]&.to_i
+  while true
+    query =
+      Domain::Post::FaPost
+        .where(state: "removed")
+        .where.not(title: nil)
+        .order(fa_id: :desc)
+    query = query.where(fa_id: ...last_fa_id) if last_fa_id
+    post = query.first
+    break unless post
+    last_fa_id = post.fa_id
+
+    puts "[before] [post.state: #{colorize_state.call(post.state)}] [post.file.id: #{post.file&.id}] [post.id: #{post.id}] [post.fa_id: #{post.fa_id}] [post.title: #{post.title}]"
+    Domain::Fa::Job::ScanPostJob.perform_now(post: post, force_scan: true)
+    post.reload
+    puts "[after] [post.state: #{colorize_state.call(post.state)}] [post.file.id: #{post.file&.id}] [post.id: #{post.id}] [post.fa_id: #{post.fa_id}] [post.title: #{post.title}]"
+    sleep 2
+  end
+rescue => e
+  puts "error: #{e.message}"
+  binding.pry
+end
+
+task fix_fa_user_avatars: :environment do
+  new_users_missing_avatar =
+    Domain::User::FaUser.where.missing(:avatar).select(:url_name)
+  old_users_with_avatar =
+    Domain::Fa::User
+      .where(url_name: new_users_missing_avatar)
+      .includes(:avatar)
+      .filter(&:avatar)
+
+  old_users_with_avatar.each do |old_user|
+    old_avatar = old_user.avatar
+    new_user = Domain::User::FaUser.find_by(url_name: old_user.url_name)
+
+    if old_avatar.log_entry.nil?
+      puts "enqueue fresh download for #{old_user.url_name}"
+      new_avatar = Domain::UserAvatar.new
+      new_user.avatar = new_avatar
+      new_user.save!
+      Domain::Fa::Job::UserAvatarJob.perform_now(avatar: new_avatar)
+      new_avatar.reload
+
+      binding.pry
+      next
+    end
+
+    new_avatar = Domain::UserAvatar.new
+    new_avatar.log_entry_id = old_avatar.log_entry_id
+    new_avatar.last_log_entry_id = old_avatar.log_entry_id
+    new_avatar.url_str = old_avatar.file_url_str
+    new_avatar.downloaded_at = old_avatar.log_entry&.created_at
+    new_avatar.state =
+      case old_avatar.state
+      when "ok"
+        old_avatar.log_entry_id.present? ? "ok" : "pending"
+      when "file_not_found"
+        new_avatar.error_message = old_avatar.state
+        "file_404"
+      else
+        new_avatar.error_message = old_avatar.state
+        "http_error"
+      end
+    new_user.avatar = new_avatar
+    new_user.save!
+    puts "migrated #{old_user.url_name}"
+  rescue => e
+    puts "error: #{e.message}"
+    binding.pry
+  end
+end
+
+task run_fa_user_avatar_jobs: :environment do
+  avatars =
+    Domain::UserAvatar
+      .where(state: "pending")
+      .joins(:user)
+      .where(user: { type: Domain::User::FaUser.name })
+
+  puts "count: #{avatars.count}"
+
+  avatars.each do |avatar|
+    Domain::Fa::Job::UserAvatarJob.perform_now(avatar:)
+    avatar.reload
+    puts "perform avatar job for #{avatar.user.url_name} - #{avatar.state.bold}"
+  end
+end
+
+task sample_migrated_favs: :environment do
+  new_user = Domain::User::FaUser.where.not(migrated_user_favs_at: nil).last
+  old_user = Domain::Fa::User.find_by(url_name: new_user.url_name)
+
+  puts "user: #{new_user.url_name}"
+  puts "old fav count: #{old_user.fav_posts.count}"
+  puts "new fav count: #{new_user.faved_posts.count}"
+end
+
+task create_post_file_fingerprints: :environment do
+  def migrate_posts_for_user(user)
+    puts "migrating posts for #{user.to_param}"
+    pb =
+      ProgressBar.create(
+        total: user.posts.count,
+        format: "%t: %c/%C %B %p%% %a %e",
+      )
+
+    user
+      .posts
+      .includes(:files)
+      .find_in_batches(batch_size: 64) do |batch|
+        ReduxApplicationRecord.transaction do
+          batch.each { |post| migrate_post(post) }
+          pb.progress = [pb.progress + 1, pb.total].min
+        end
+      end
+  end
+
+  def migrate_post(post)
+    puts "migrating #{post.id} / #{post.to_param} / '#{post.title_for_view}'"
+    ColorLogger.quiet do
+      post.files.each do |file|
+        migrate_post_file(file)
+      rescue StandardError => e
+        puts "error: #{e.message}"
+      end
+    end
+  end
+
+  def migrate_post_file(post_file)
+    job = Domain::PostFileThumbnailJob.new
+    ColorLogger.quiet do
+      job.perform({ post_file: })
+    rescue => e
+      puts "error: #{e.message}"
+    end
+  end
+
+  if ENV["post_file_descending"].present?
+    total = 49_783_962 # cache this value
+    pb = ProgressBar.create(total:, format: "%t: %c/%C %B %p%% %a %e")
+    i = 0
+    Domain::PostFile
+      .where(state: "ok")
+      .includes(:blob)
+      .find_each(
+        order: :desc,
+        batch_size: 32,
+        start: ENV["start_at"],
+      ) do |post_file|
+        i += 1
+        if i % 100 == 0
+          puts "migrating #{post_file.id} / #{post_file.post.title_for_view}"
+        end
+        migrate_post_file(post_file)
+        pb.progress = [pb.progress + 1, pb.total].min
+      end
+  elsif ENV["posts_descending"].present?
+    # total = Domain::Post.count
+    total = 66_431_808 # cache this value
+    pb = ProgressBar.create(total:, format: "%t: %c/%C %B %p%% %a %e")
+    Domain::Post.find_each(order: :desc) do |post|
+      migrate_post(post) unless post.is_a?(Domain::Post::InkbunnyPost)
+      pb.progress = [pb.progress + 1, pb.total].min
+    end
+  elsif ENV["user"].present?
+    for_user = ENV["user"] || raise("need 'user'")
+    user = DomainController.find_model_from_param(Domain::User, for_user)
+    raise "user '#{for_user}' not found" unless user
+    migrate_posts_for_user(user)
+  elsif ENV["users_descending"].present?
+    # all users with posts, ordered by post count descending
+    migrated_file = File.open("migrated_files.txt", "a+")
+    migrated_file.seek(0)
+    migrated_users = migrated_file.readlines.map(&:strip)
+    users =
+      Domain::User::FaUser.order(
+        Arel.sql("json_attributes->>'num_watched_by' DESC NULLS LAST"),
+      ).pluck(:id)
+
+    users.each do |user_id|
+      user = Domain::User::FaUser.find(user_id)
+      next if migrated_users.include?(user.to_param)
+      puts "migrating posts for #{user.to_param} (#{user.num_watched_by} watched by)"
+      migrate_posts_for_user(user)
+      migrated_file.write("#{user.to_param}\n")
+      migrated_file.flush
+    end
+    migrated_file.close
+  else
+    raise "need 'user' or 'users_descending'"
+  end
+end
+
+task enqueue_pending_post_files: :environment do
+  query = Domain::PostFile.where(state: "pending")
+  puts "enqueueing #{query.count} pending post files"
+  query.find_in_batches(batch_size: 100, start: ENV["start_at"]) do |batch|
+    while (
+            queue_size =
+              GoodJob::Job.where(
+                job_class: "Job::PostFileJob",
+                performed_at: nil,
+                scheduled_at: nil,
+                error: nil,
+              ).count
+          ) > 100
+      puts "queue size: #{queue_size}"
+      sleep 10
+    end
+    batch.each do |post_file|
+      Job::PostFileJob.set(priority: 10).perform_later(post_file:)
+    end
+  end
+end
+
+task find_post_files_with_empty_response: :environment do
+  query =
+    Domain::PostFile
+      .where(state: "ok", retry_count: 0)
+      .joins(:log_entry)
+      .where(http_log_entries: { response_sha256: BlobFile::EMPTY_FILE_SHA256 })
+
+  pb = ProgressBar.create(total: query.count, format: "%t: %c/%C %B %p%% %a %e")
+
+  query.find_each(batch_size: 10) do |post_file|
+    # puts "post_file: #{post_file.id} / '#{post_file.post.to_param}'"
+    post_file.state_pending!
+    post_file.save!
+    Job::PostFileJob.perform_now(post_file:)
+    pb.progress = [pb.progress + 1, pb.total].min
+  end
+end

TODO.md

@@ -6,4 +6,36 @@
 - [x] Attach logs to jobs, page to view jobs and their logs
 - [ ] Standardize all the embeddings tables to use the same schema (item_id, embedding)
 - [ ] Bluesky scraper
-- [ ] Download favs / votes for E621 users
+- [x] Download favs / votes for E621 users
+- [ ] Automatically enqueue jobs for FA users to do incremental scans of profiles
+- [ ] Fix FA posts that start with "Font size adjustment: smallerlarger"
+- [ ] Convert logger .prefix=... into .tagged(...)
+- [x] `make_tag` should be smart about the objects it takes
+- [ ] Convert all `state: string` attributes to enums in ActiveRecord models
+- [ ] Create `belongs_to_log_entry` macro for ActiveRecord models
+- [x] Use StaticFileJobHelper for Domain::Fa::Job::ScanFileJob
+- [ ] Unify HTTP client configs for all domains, so the same job type can be used for different domains
+- [ ] put abstract `external_url_for_view` in a module
+- [ ] backfill descriptions on inkbunny posts
+- [ ] store deep update json on inkbunny posts
+- [x] limit number of users, or paginate for "users who favorited this post" page
+- [ ] manual good job runner does not indicate if the job threw an exception - check return value of #perform, maybe?
+- [ ] FA user favs job should stop when in incremental mode when all posts on the page are already known favs (e.g. pages with only 47 posts are not a false positive)
+- [x] Factor out FA listings page enqueue logic into common location; use in Gallery and Favs jobs
+- [ ] Add followers / following to FA user show page
+- [x] Parse E621 source url for inkbunny posts & users
+- [x] Parse E621 source url for fa users
+- [ ] Parse BBCode in post descriptions
+  - example post with bbcode: https://refurrer.com/posts/ib/3452498
+- [ ] Show tags on fa posts, ib posts
+- [ ] Sofurry implmentation
+- [ ] Make unified Static file job
+- [ ] Make unified Avatar file job
+- [ ] ko-fi domain icon
+- [ ] tumblr domain icon
+- [ ] Do PCA on user factors table to display a 2D plot of users
+- [ ] Use links found in descriptions to indicate re-scanning a post? (e.g. for comic next/prev links)
+- [ ] fix for IDs that have a dot in them - e.g. https://refurrer.com/users/fa@jakke.
+- [ ] Rich inline links to e621 e.g. https://refurrer.com/posts/fa@60070060
+- [ ] Find FaPost that have favs recorded but no scan / file, enqueue scan
+- [ ] Bunch of posts with empty responses: posts = Domain::Post.joins(files: :log_entry).where(files: { http_log_entries: { response_sha256: BlobFile::EMPTY_FILE_SHA256 }}).limit(10)

app/assets/images/domain-icons/sorbet/rbi/dsl/.gitattributes

@@ -0,0 +1 @@
+**/*.rbi linguist-generated=true

app/assets/images/domain-icons/sorbet/rbi/dsl/active_support/callbacks.rbi

@@ -0,0 +1,23 @@
+# typed: true
+
+# DO NOT EDIT MANUALLY
+# This is an autogenerated file for dynamic methods in `ActiveSupport::Callbacks`.
+# Please instead update this file by running `bin/tapioca dsl ActiveSupport::Callbacks`.
+
+
+module ActiveSupport::Callbacks
+  include GeneratedInstanceMethods
+
+  mixes_in_class_methods GeneratedClassMethods
+
+  module GeneratedClassMethods
+    def __callbacks; end
+    def __callbacks=(value); end
+    def __callbacks?; end
+  end
+
+  module GeneratedInstanceMethods
+    def __callbacks; end
+    def __callbacks?; end
+  end
+end

app/assets/images/generic-domain.svg

@@ -0,0 +1,19 @@
+<svg
+  width="16"
+  height="16"
+  viewBox="0 0 16 16"
+  fill="none"
+  xmlns="http://www.w3.org/2000/svg"
+>
+  <!-- Background circle -->
+  <circle cx="8" cy="8" r="7" fill="#E0E0E0" />
+
+  <!-- Stylized "www" text -->
+  <path
+    d="M4 8.5C4 6.5 5 5.5 6 5.5C7 5.5 8 6.5 8 8.5C8 6.5 9 5.5 10 5.5C11 5.5 12 6.5 12 8.5"
+    stroke="#666666"
+    stroke-width="1.5"
+    stroke-linecap="round"
+    fill="none"
+  />
+</svg>

app/assets/stylesheets/application.tailwind.css

@@ -7,17 +7,25 @@
 }
 
 .sky-section {
-  @apply divide-y divide-slate-300 overflow-hidden border border-slate-300 bg-slate-100 md:rounded-lg;
+  @apply divide-y divide-slate-300 overflow-hidden border border-slate-300 bg-slate-100 sm:rounded-lg;
 }
 
 .section-header {
   @apply px-4 py-3 font-medium text-slate-900;
 }
 
+.sky-section-header {
+  @apply px-4 py-3 font-medium text-slate-900;
+}
+
 .sky-link {
   @apply text-sky-600 underline decoration-dotted transition-colors hover:text-sky-800;
 }
 
+.blue-link {
+  @apply text-blue-600 transition-colors hover:text-blue-800 hover:underline;
+}
+
 .scroll-shadows {
   background:
     /* Shadow Cover TOP */
@@ -39,3 +47,15 @@
     100% 10px;
   background-attachment: local, local, scroll, scroll;
 }
+
+.log-entry-table-header-cell {
+  @apply border-b border-slate-200 bg-slate-50 px-2 py-1 text-xs font-medium uppercase tracking-wider text-slate-500;
+}
+
+.log-entry-table-row-cell {
+  @apply flex items-center border-b border-slate-200 px-2 py-1 text-sm group-hover:bg-slate-50;
+}
+
+.rich-text-content blockquote {
+  @apply my-4 border-s-4 border-gray-300 bg-slate-200 p-4 italic leading-relaxed;
+}

app/assets/stylesheets/good_job_custom.css

@@ -3,25 +3,25 @@
   font-weight: bold;
 }
 .ansi-black {
-  color: #000000;
+  color: #333333;
 }
 .ansi-red {
-  color: #cd0000;
+  color: #cd3333;
 }
 .ansi-green {
-  color: #00cd00;
+  color: #33cd33;
 }
 .ansi-yellow {
-  color: #cdcd00;
+  color: #cdcd33;
 }
 .ansi-blue {
-  color: #0000ee;
+  color: #3333ee;
 }
 .ansi-magenta {
-  color: #cd00cd;
+  color: #cd33cd;
 }
 .ansi-cyan {
-  color: #00cdcd;
+  color: #33cdcd;
 }
 .ansi-white {
   color: #e5e5e5;
@@ -32,51 +32,63 @@
   color: #7f7f7f;
 }
 .ansi-bright-red {
-  color: #ff0000;
+  color: #990000;
 }
 .ansi-bright-green {
-  color: #00ff00;
+  color: #009900;
 }
 .ansi-bright-yellow {
-  color: #ffff00;
+  color: #999900;
 }
 .ansi-bright-blue {
-  color: #5c5cff;
+  color: #5c5c99;
 }
 .ansi-bright-magenta {
-  color: #ff00ff;
+  color: #990099;
 }
 .ansi-bright-cyan {
-  color: #00ffff;
+  color: #009999;
 }
 .ansi-bright-white {
-  color: #ffffff;
+  color: #999999;
 }
 
 .log-uuid {
+  min-width: 20px;
   max-width: 100px;
   overflow: hidden;
-  /* white-space: nowrap; */
   text-overflow: ellipsis;
 }
 
-/* Log line container */
-.log-line {
+/* All log lines container */
+.good-job-log-lines {
+  overflow-x: auto;
+}
+
+/* Single log line container */
+.good-job-log-line {
   font-family: monospace;
   font-size: 0.8rem;
   line-height: 1;
   margin: 2px 0;
   padding: 2px 4px;
   display: flex;
+  white-space: nowrap;
+  width: max-content; /* Make width match the content width */
 }
 
-.log-line > span {
+.good-job-log-line:hover {
+  background-color: #ccc;
+}
+
+.good-job-log-line > span {
   display: inline-block;
   white-space: pre;
 }
 
 .good-job-execution-log {
-  background: #3d3d3d;
+  color: #333;
+  background: #f0f0f0;
 }
 
 .text-truncate-link {
@@ -85,3 +97,35 @@
   overflow: hidden;
   text-overflow: ellipsis;
 }
+
+.good-job-arg-name {
+  white-space: nowrap;
+}
+
+.good-job-arg-grid {
+  display: grid;
+  grid-template-columns: auto 1fr;
+}
+
+.good-job-arg-value,
+.good-job-arg-name {
+  padding: 0.35em 0.4em;
+}
+
+.good-job-arg-name,
+.good-job-arg-value {
+  border-bottom: 1px solid #e0e0e0;
+}
+
+.good-job-arg-row {
+  display: contents;
+}
+
+.good-job-arg-row:hover > * {
+  background-color: #ccc;
+}
+
+/* This ensures the last row doesn't have a bottom border */
+.good-job-arg-grid .good-job-arg-row:last-child * {
+  border-bottom: none;
+}

app/controllers/application_controller.rb

@@ -5,6 +5,17 @@ class ApplicationController < ActionController::Base
   include Pundit::Authorization
   include Devise::Controllers::Helpers::ClassMethods
 
+  sig { returns(T.nilable(IpAddressRole)) }
+  def current_ip_address_role
+    @current_ip_address_role ||= IpAddressRole.for_ip(request.remote_ip)
+  end
+  helper_method :current_ip_address_role
+
+  sig { returns(T.nilable(T.any(User, IpAddressRole))) }
+  def pundit_user
+    current_user || current_ip_address_role
+  end
+
   before_action do
     if Rails.env.development? || Rails.env.staging?
       Rack::MiniProfiler.authorize_request
@@ -18,13 +29,6 @@ class ApplicationController < ActionController::Base
 
   protected
 
-  def set_ivfflat_probes!
-    ReduxApplicationRecord.connection.execute("SET ivfflat.max_probes = 10")
-    ReduxApplicationRecord.connection.execute("SET ivfflat.probes = 10")
-  end
-
-  protected
-
   def prometheus_client
     PrometheusExporter::Client.default
   end

app/controllers/blob_entries_controller.rb

@@ -1,69 +1,196 @@
-# typed: false
+# typed: strict
 class BlobEntriesController < ApplicationController
   skip_before_action :authenticate_user!, only: [:show]
 
+  sig { void }
   def show
     thumb = params[:thumb]
-    raise("invalid thumb #{thumb}") if thumb.present? && !thumb_params(thumb)
+    if thumb.present? && !thumb_params(thumb)
+      raise ActionController::BadRequest.new("invalid thumbnail #{thumb}")
+    end
 
-    expires_dur = 1.year
+    if thumb.present?
+      expires_dur = 1.week
+    else
+      expires_dur = 1.year
+    end
     response.headers["Expires"] = expires_dur.from_now.httpdate
     expires_in expires_dur, public: true
 
-    sha256 = params[:sha256]
-    etag = sha256
-    etag += "-#{thumb}" if thumb
-    return unless stale?(last_modified: Time.at(0), strong_etag: etag)
+    unless stale?(
+             last_modified: Time.at(0),
+             strong_etag: strong_etag_for_request,
+           )
+      return
+    end
 
-    # images, videos, etc
-    blob_entry = BlobEntry.find(HexUtil.hex2bin(sha256))
-    if helpers.is_send_data_content_type?(blob_entry.content_type)
-      if !thumb.blank? &&
-           helpers.is_thumbable_content_type?(blob_entry.content_type)
-        filename = "thumb-#{thumb}-#{sha256}"
-        filename = filename[..File.extname(filename).length]
-        filename += ".jpeg"
+    sha256 = T.let(params[:sha256], String)
+    raise ActionController::BadRequest.new("no file specified") if sha256.blank?
 
-        width, height = thumb_params(thumb)
-        image =
-          Vips::Image.thumbnail_buffer(
-            blob_entry.contents,
-            width,
-            height: height,
-          )
-        resized_image_contents = image.jpegsave_buffer
-
-        send_data(
-          resized_image_contents,
-          type: "image/jpg",
-          disposition: "inline",
-          filename: filename,
-        )
-      else
-        ext = helpers.ext_for_content_type(blob_entry.content_type)
-        ext = ".#{ext}" if ext
-        send_data(
-          blob_entry.contents,
-          type: blob_entry.content_type,
-          disposition: "inline",
-          filename: "data#{ext}",
-        )
-      end
-    elsif blob_entry.content_type =~ %r{text/plain}
-      render plain: blob_entry.contents
-    elsif blob_entry.content_type.starts_with? "text/html"
-      render html: blob_entry.contents.html_safe
-    elsif blob_entry.content_type.starts_with? "application/json"
-      pretty_json = JSON.pretty_generate(JSON.parse blob_entry.contents)
-      render html:
-               "<html><body><pre>#{pretty_json}</pre></body></html>".html_safe
+    if show_blob_file(sha256, thumb)
+      return
+    elsif BlobFile.migrate_sha256!(sha256) && show_blob_file(sha256, thumb)
+      return
     else
-      render plain: "no renderer for #{blob_entry.content_type}"
+      raise ActiveRecord::RecordNotFound
     end
   end
 
   private
 
+  sig { params(sha256: String, thumb: T.nilable(String)).returns(T::Boolean) }
+  def show_blob_file(sha256, thumb)
+    if thumb
+      thumb_params = thumb_params(thumb)
+      if thumb_params.nil?
+        raise ActionController::BadRequest.new("invalid thumbnail: #{thumb}")
+      end
+
+      # if the requested format is gif, and the thumbnail type is content-container, we want to
+      # thumbnail the gif into another gif. Else, always thumbnail into a jpeg.
+      file_ext = "jpeg"
+      if params[:format] == "gif" && thumb == "content-container"
+        file_ext = "gif"
+      end
+
+      width, height = thumb_params
+      filename = "thumb-#{sha256}-#{thumb}.#{file_ext}"
+      cache_key = "vips:#{filename}"
+      thumb_data =
+        Rack::MiniProfiler.step("vips: load from cache") do
+          Rails
+            .cache
+            .fetch(cache_key, expires_in: 1.day) do
+              blob_file = BlobFile.find_by(sha256: HexUtil.hex2bin(sha256))
+              if blob_file
+                content_type =
+                  blob_file.content_type || "application/octet-stream"
+                if helpers.is_renderable_video_type?(content_type)
+                  thumbnail_video_file(blob_file, width, height, file_ext)
+                elsif helpers.is_renderable_image_type?(content_type)
+                  thumbnail_image_file(blob_file, width, height, file_ext)
+                end
+              end
+            end
+        end
+
+      if !thumb_data
+        Rails.cache.delete(cache_key)
+        return false
+      end
+
+      send_data(
+        thumb_data[0],
+        type: thumb_data[1],
+        disposition: "inline",
+        filename: filename,
+      )
+    else
+      blob_file = BlobFile.find_by(sha256: HexUtil.hex2bin(sha256))
+      return false if !blob_file
+
+      content_type = blob_file.content_type || "application/octet-stream"
+      send_file(
+        blob_file.absolute_file_path,
+        type: content_type,
+        disposition: "inline",
+      )
+    end
+
+    return true
+  end
+
+  sig do
+    params(
+      blob_file: BlobFile,
+      width: Integer,
+      height: Integer,
+      thumb: String,
+    ).returns(T.nilable([String, String]))
+  end
+  def thumbnail_video_file(blob_file, width, height, thumb)
+    video_file = blob_file.absolute_file_path
+    temp_thumb_file = Tempfile.new(%w[video-thumb .png])
+    process_result =
+      system(
+        "ffmpegthumbnailer",
+        "-f", # overlay video strip indicator
+        "-i",
+        video_file,
+        "-o",
+        T.must(temp_thumb_file.path),
+        "-s",
+        "#{width}",
+        "-c",
+        "jpeg",
+      )
+    if !process_result
+      temp_thumb_file.unlink
+      return nil
+    end
+
+    thumb_data_tmp = File.read(T.must(temp_thumb_file.path), mode: "rb")
+    temp_thumb_file.unlink
+    [thumb_data_tmp, "image/jpeg"]
+  end
+
+  # Returns a tuple of the thumbnail data and the content type
+  sig do
+    params(
+      blob_file: BlobFile,
+      width: Integer,
+      height: Integer,
+      file_ext: String,
+    ).returns(T.nilable([String, String]))
+  end
+  def thumbnail_image_file(blob_file, width, height, file_ext)
+    if file_ext == "gif"
+      Rack::MiniProfiler.step("vips: load gif") do
+        # Use libvips' gifload with n=-1 to load all frames
+        image = Vips::Image.gifload(blob_file.absolute_file_path, n: -1)
+        num_frames = image.get("n-pages")
+        image_width, image_height = image.width, (image.height / num_frames)
+
+        if width >= image_width && height >= image_height
+          logger.info("gif is already smaller than requested thumbnail size")
+          return [
+            File.read(blob_file.absolute_file_path, mode: "rb"),
+            "image/gif"
+          ]
+        end
+
+        Rack::MiniProfiler.step("vips: thumbnail gif") do
+          image = image.thumbnail_image(width, height: height)
+          image_buffer =
+            image.gifsave_buffer(
+              dither: 1,
+              effort: 1,
+              interframe_maxerror: 16,
+              interpalette_maxerror: 10,
+              interlace: true,
+            )
+          [image_buffer, "image/gif"]
+        end
+      end
+    else
+      # Original static image thumbnailing logic
+      image_buffer =
+        Rack::MiniProfiler.step("vips: load image") do
+          T.unsafe(Vips::Image).thumbnail(
+            blob_file.absolute_file_path,
+            width,
+            height: height,
+          )
+        end
+
+      Rack::MiniProfiler.step("vips: thumbnail image") do
+        logger.info("rendering thumbnail as jpeg")
+        [image_buffer.jpegsave_buffer(interlace: true, Q: 95), "image/jpeg"]
+      end
+    end
+  end
+
+  sig { params(thumb: String).returns(T.nilable([Integer, Integer])) }
   def thumb_params(thumb)
     case thumb
     when "32-avatar"
@@ -76,6 +203,13 @@ class BlobEntriesController < ApplicationController
       [400, 300]
     when "medium"
       [800, 600]
+    when "content-container"
+      [768, 2048]
     end
   end
+
+  sig { returns(String) }
+  def strong_etag_for_request
+    [params[:sha256], params[:thumb], params[:format]].compact.join("-")
+  end
 end

app/controllers/domain/e621/posts_controller.rb

@@ -1,6 +0,0 @@
-# typed: true
-class Domain::E621::PostsController < ApplicationController
-  def show
-    @post = Domain::E621::Post.find_by!(e621_id: params[:e621_id])
-  end
-end

app/controllers/domain/fa/api_controller.rb

@@ -4,44 +4,26 @@ class Domain::Fa::ApiController < ApplicationController
   before_action :validate_api_token!
 
   skip_before_action :verify_authenticity_token,
-                     only: %i[enqueue_objects object_statuses]
+                     only: %i[enqueue_objects object_statuses similar_users]
 
-  skip_before_action :validate_api_token!, only: %i[search_user_names]
-
-  def search_user_names
-    name = params[:name]
-    limit = (params[:limit] || 5).to_i.clamp(0, 15)
-    users = users_for_name(name, limit: limit)
-    if !Rails.env.production? && name == "error"
-      render status: 500, json: { error: "an error!" }
-    else
-      render json: { users: users }
-    end
-  end
+  skip_before_action :validate_api_token!,
+                     only: %i[object_statuses similar_users]
 
   def object_statuses
-    fa_ids = (params[:fa_ids] || []).map(&:to_i)
-    url_names = (params[:url_names] || [])
+    fa_ids = (params[:fa_ids] || []).reject(&:blank?).map(&:to_i)
+    url_names = (params[:url_names] || []).reject(&:blank?)
 
-    jobs_async =
-      GoodJob::Job
-        .select(:id, :queue_name, :serialized_params)
-        .where(queue_name: "manual", finished_at: nil)
-        .where(
-          [
-            "(serialized_params->'exception_executions' = '{}')",
-            "(serialized_params->'exception_executions' is null)",
-          ].join(" OR "),
-        )
-        .load_async
-
-    users_async = Domain::Fa::User.where(url_name: url_names).load_async
+    url_name_to_user =
+      Domain::User::FaUser
+        .where(url_name: url_names)
+        .map { |user| [T.must(user.url_name), user] }
+        .to_h
 
     fa_id_to_post =
-      Domain::Fa::Post
+      Domain::Post::FaPost
         .includes(:file)
         .where(fa_id: fa_ids)
-        .map { |post| [post.fa_id, post] }
+        .map { |post| [T.must(post.fa_id), post] }
         .to_h
 
     posts_response = {}
@@ -50,96 +32,64 @@ class Domain::Fa::ApiController < ApplicationController
     fa_ids.each do |fa_id|
       post = fa_id_to_post[fa_id]
 
-      post_response =
-        T.let(
-          {
-            terminal_state: false,
-            seen_at: time_ago_or_never(post&.created_at),
-            scanned_at: "never",
-            downloaded_at: "never",
-          },
-          T::Hash[Symbol, T.untyped],
-        )
-
       if post
-        post_response[:info_url] = domain_fa_post_url(fa_id: post.fa_id)
-        post_response[:scanned_at] = time_ago_or_never(post.scanned_at)
+        post_state =
+          if post.file.present?
+            "have_file"
+          elsif post.scanned_at?
+            "scanned_post"
+          else
+            post.state
+          end
 
-        if post.file.present?
-          post_response[:downloaded_at] = time_ago_or_never(
-            post.file&.created_at,
-          )
-          post_response[:state] = "have_file"
-          post_response[:terminal_state] = true
-        elsif post.scanned?
-          post_response[:state] = "scanned_post"
-        else
-          post_response[:state] = post.state
-        end
+        post_response = {
+          state: post_state,
+          seen_at: time_ago_or_never(post.created_at),
+          object_url: request.base_url + helpers.domain_post_path(post),
+          post_scan: {
+            last_at: time_ago_or_never(post.scanned_at),
+            due_for_scan: !post.scanned_at?,
+          },
+          file_scan: {
+            last_at: time_ago_or_never(post.file&.created_at),
+            due_for_scan: !post.file&.created_at?,
+          },
+        }
       else
-        post_response[:state] = "not_seen"
+        post_response = { state: "not_seen" }
       end
 
       posts_response[fa_id] = post_response
     end
 
-    url_name_to_user = users_async.map { |user| [user.url_name, user] }.to_h
-
     url_names.each do |url_name|
       user = url_name_to_user[url_name]
 
       if user
         user_response = {
           created_at: time_ago_or_never(user.created_at),
-          scanned_gallery_at: time_ago_or_never(user.scanned_gallery_at),
-          scanned_page_at: time_ago_or_never(user.scanned_page_at),
+          state: user.state,
+          object_url: request.base_url + helpers.domain_user_path(user),
+          page_scan: {
+            last_at: time_ago_or_never(user.scanned_page_at),
+            due_for_scan: user.page_scan.due?,
+          },
+          gallery_scan: {
+            last_at: time_ago_or_never(user.gallery_scan.at),
+            due_for_scan: user.gallery_scan.due?,
+          },
+          favs_scan: {
+            last_at: time_ago_or_never(user.favs_scan.at),
+            due_for_scan: user.favs_scan.due?,
+          },
         }
-        states = []
-        states << "page" unless user.due_for_page_scan?
-        states << "gallery" unless user.due_for_gallery_scan?
-        states << "seen" if states.empty?
-
-        user_response[:state] = states.join(",")
-
-        if user.scanned_gallery_at && user.scanned_page_at
-          user_response[:terminal_state] = true
-        end
       else
-        user_response = { state: "not_seen", terminal_state: false }
+        user_response = { state: "not_seen" }
       end
       users_response[url_name] = user_response
     end
 
-    queue_depths = Hash.new { |hash, key| hash[key] = 0 }
-
-    jobs_async.each do |job|
-      queue_depths[job.serialized_params["job_class"]] += 1
-    end
-
-    queue_depths =
-      queue_depths
-        .map do |key, value|
-          [
-            key
-              .delete_prefix("Domain::Fa::Job::")
-              .split("::")
-              .last
-              .underscore
-              .delete_suffix("_job")
-              .gsub("_", " "),
-            value,
-          ]
-        end
-        .to_h
-
-    render json: {
-             posts: posts_response,
-             users: users_response,
-             queues: {
-               total_depth: queue_depths.values.sum,
-               depths: queue_depths,
-             },
-           }
+    render json: { posts: posts_response, users: users_response }
   end
 
   def enqueue_objects
@@ -182,94 +132,8 @@ class Domain::Fa::ApiController < ApplicationController
            }
   end
 
-  def similar_users
-    url_name = params[:url_name]
-    exclude_url_name = params[:exclude_url_name]
-
-    user = Domain::Fa::User.find_by(url_name: url_name)
-    if user.nil?
-      render status: 404,
-             json: {
-               error: "user '#{url_name}' not found",
-               error_type: "user_not_found",
-             }
-      return
-    end
-
-    all_similar_users = helpers.similar_users_by_followed(user, limit: 10)
-
-    if all_similar_users.nil?
-      render status: 500,
-             json: {
-               error:
-                 "user '#{url_name}' has not had recommendations computed yet",
-               error_type: "recs_not_computed",
-             }
-      return
-    end
-
-    all_similar_users = users_list_to_similar_list(all_similar_users)
-
-    not_followed_similar_users = nil
-    if exclude_url_name
-      exclude_folowed_by_user =
-        Domain::Fa::User.find_by(url_name: exclude_url_name)
-      not_followed_similar_users =
-        if exclude_folowed_by_user.nil?
-          # TODO - enqueue a manual UserFollowsJob for this user and have client
-          # re-try the request later
-          {
-            error: "user '#{exclude_url_name}' not found",
-            error_type: "exclude_user_not_found",
-          }
-        elsif exclude_folowed_by_user.scanned_follows_at.nil?
-          {
-            error:
-              "user '#{exclude_url_name}' followers list hasn't been scanned",
-            error_type: "exclude_user_not_scanned",
-          }
-        else
-          users_list_to_similar_list(
-            helpers.similar_users_by_followed(
-              user,
-              limit: 10,
-              exclude_followed_by: exclude_folowed_by_user,
-            ),
-          )
-        end
-    end
-
-    render json: {
-             all: all_similar_users,
-             not_followed: not_followed_similar_users,
-           }
-  end
-
   private
 
-  def get_best_user_page_http_log_entry_for(user)
-    for_path =
-      proc do |uri_path|
-        HttpLogEntry
-          .where(
-            uri_scheme: "https",
-            uri_host: "www.furaffinity.net",
-            uri_path: uri_path,
-          )
-          .order(created_at: :desc)
-          .first
-          &.response
-      end
-
-    for_hle_id =
-      proc { |hle_id| hle_id && HttpLogEntry.find_by(id: hle_id)&.response }
-
-    # older versions don't end in a trailing slash
-    hle_id = user.log_entry_detail && user.log_entry_detail["last_user_page_id"]
-    for_hle_id.call(hle_id) || for_path.call("/user/#{user.url_name}/") ||
-      for_path.call("/user/#{user.url_name}")
-  end
-
   def defer_post_scan(post, fa_id)
     if !post || !post.scanned?
       defer_manual(Domain::Fa::Job::ScanPostJob, { fa_id: fa_id }, -17)
@@ -335,75 +199,6 @@ class Domain::Fa::ApiController < ApplicationController
     end
   end
 
-  def users_for_name(name, limit: 10)
-    users =
-      Domain::Fa::User
-        .where(
-          [
-            "(name ilike :name) OR (url_name ilike :name)",
-            { name: "#{ReduxApplicationRecord.sanitize_sql_like(name)}%" },
-          ],
-        )
-        .includes(:avatar)
-        .select(:id, :state, :state_detail, :log_entry_detail, :name, :url_name)
-        .select(
-          "(SELECT COUNT(*) FROM domain_fa_posts WHERE creator_id = domain_fa_users.id) as num_posts",
-        )
-        .order(name: :asc)
-        .limit(limit)
-
-    users.map do |user|
-      {
-        id: user.id,
-        name: user.name,
-        url_name: user.url_name,
-        thumb: helpers.fa_user_avatar_path(user, thumb: "64-avatar"),
-        show_path: domain_fa_user_path(user.url_name),
-        # `num_posts` is a manually added column, so we need to use T.unsafe to
-        # access it
-        num_posts: T.unsafe(user).num_posts,
-      }
-    end
-  end
-
-  def users_list_to_similar_list(users_list)
-    users_list.map do |user|
-      profile_thumb_url = user.avatar&.file_uri&.to_s
-      profile_thumb_url ||
-        begin
-          profile_page_response = get_best_user_page_http_log_entry_for(user)
-          if profile_page_response
-            parser =
-              Domain::Fa::Parser::Page.new(
-                profile_page_response.contents,
-                require_logged_in: false,
-              )
-            profile_thumb_url = parser.user_page.profile_thumb_url
-          else
-            if user.due_for_follows_scan?
-              Domain::Fa::Job::UserFollowsJob.set(
-                { priority: -20 },
-              ).perform_later({ user: user })
-            end
-            if user.due_for_page_scan?
-              Domain::Fa::Job::UserPageJob.set({ priority: -20 }).perform_later(
-                { user: user },
-              )
-            end
-          end
-        rescue StandardError
-          logger.error("error getting profile_thumb_url: #{$!.message}")
-        end
-
-      {
-        name: user.name,
-        url_name: user.url_name,
-        profile_thumb_url: profile_thumb_url,
-        url: "https://www.furaffinity.net/user/#{user.url_name}/",
-      }
-    end
-  end
-
   API_TOKENS = {
     "a4eb03ac-b33c-439c-9b51-a834d1c5cf48" => "dymk",
     "56cc81fe-8c00-4436-8981-4580eab00e66" => "taargus",

app/controllers/domain/fa/posts_controller.rb

@@ -1,139 +0,0 @@
-# typed: true
-
-class Domain::Fa::PostsController < ApplicationController
-  before_action :set_ivfflat_probes!, only: %i[show]
-  before_action :set_domain_fa_post, only: %i[show scan_post]
-
-  skip_before_action :verify_authenticity_token,
-                     only: %i[try_scan_post try_scan_posts]
-
-  skip_before_action :authenticate_user!, only: %i[show index]
-
-  # This action is always scoped to a user, so the :user_url_name parameter is required.
-  # GET /domain/fa/users/:user_url_name/posts
-  def index
-    @user = Domain::Fa::User.find_by!(url_name: params[:user_url_name])
-    relation = policy_scope(@user.posts)
-    @posts =
-      relation
-        .includes(:creator, :file)
-        .order(fa_id: :desc)
-        .page(params[:page])
-        .per(50)
-        .without_count
-  end
-
-  # GET /domain/fa/posts/:fa_id
-  def show
-  end
-
-  # GET /domain/fa/posts/:fa_id/favorites
-  def favorites
-    @post = Domain::Fa::Post.find_by!(fa_id: params[:fa_id])
-  end
-
-  def scan_post
-    if try_enqueue_post_scan(@post, @post.fa_id)
-      redirect_to domain_fa_post_path(@post.fa_id), notice: "Enqueued for scan"
-    else
-      redirect_to domain_fa_post_path(@post.fa_id), notice: "Already scanned"
-    end
-  end
-
-  def try_scan_post
-    fa_id = params[:fa_id]&.to_i || raise("need fa_id parameter")
-    post = Domain::Fa::Post.find_by(fa_id: fa_id)
-    enqueued = try_enqueue_post_scan(post, fa_id)
-
-    if post && (file = post.file).present?
-      state_string =
-        "downloaded #{helpers.time_ago_in_words(file.created_at, include_seconds: true)} ago"
-    elsif post && post.scanned?
-      state_string =
-        "scanned #{helpers.time_ago_in_words(post.scanned_at, include_seconds: true)} ago"
-    else
-      state_string = []
-      !post ? state_string << "not seen" : state_string << "#{post.state}"
-
-      state_string << "enqueued" if enqueued
-
-      state_string = state_string.join(", ")
-    end
-
-    render json: {
-             enqueued: enqueued,
-             title: post&.title,
-             state: state_string,
-             is_terminal_state: post&.scanned? && post.file&.present? || false,
-           }
-  end
-
-  def try_scan_posts
-    Rails.logger.info "params: #{params.inspect}"
-    fa_ids = params[:fa_ids].map(&:to_i)
-    fa_id_to_post =
-      Domain::Fa::Post
-        .where(fa_id: fa_ids)
-        .map { |post| [post.fa_id, post] }
-        .to_h
-
-    response = {}
-
-    fa_ids.each do |fa_id|
-      post = fa_id_to_post[fa_id]
-      if post.nil?
-        state = "not_seen"
-      elsif post.file.present?
-        state = "have_file"
-      elsif post.scanned?
-        state = "scanned"
-      else
-        state = "state_#{post.state}"
-      end
-
-      response[fa_id] = {
-        state: state,
-        enqueued: try_enqueue_post_scan(post, fa_id),
-      }
-    end
-    render json: response
-  end
-
-  private
-
-  def try_enqueue_post_scan(post, fa_id)
-    @@already_enqueued_fa_ids ||= Set.new
-    unless @@already_enqueued_fa_ids.add?(fa_id)
-      Rails.logger.info "Already enqueued #{fa_id}, skipping"
-      return false
-    end
-
-    if !post || !post.scanned?
-      Rails.logger.info "Enqueue scan #{fa_id}"
-      Domain::Fa::Job::ScanPostJob.set(
-        priority: -15,
-        queue: "manual",
-      ).perform_later({ fa_id: fa_id })
-      return true
-    end
-
-    if post && post.file_uri && !post.file.present?
-      Rails.logger.info "Enqueue file #{fa_id}"
-      Domain::Fa::Job::ScanFileJob.set(
-        priority: -15,
-        queue: "manual",
-      ).perform_later({ post: post })
-      return true
-    end
-
-    false
-  end
-
-  # Use callbacks to share common setup or constraints between actions.
-  def set_domain_fa_post
-    @post =
-      Domain::Fa::Post.includes(:creator, file: :response).find_by!(
-        fa_id: params[:fa_id],
-      )
-  end
-end

app/controllers/domain/fa/users_controller.rb

@@ -1,28 +0,0 @@
-# typed: true
-
-class Domain::Fa::UsersController < ApplicationController
-  before_action :set_ivfflat_probes!, only: %i[show]
-  before_action :set_user, only: %i[show]
-  skip_before_action :authenticate_user!, only: %i[show]
-
-  # GET /domain/fa/users or /domain/fa/users.json
-  def index
-    authorize Domain::Fa::User
-    @users =
-      policy_scope(Domain::Fa::User).includes({ avatar: [:file] }).page(
-        params[:page],
-      )
-  end
-
-  # GET /domain/fa/users/1 or /domain/fa/users/1.json
-  def show
-    authorize @user
-  end
-
-  private
-
-  # Use callbacks to share common setup or constraints between actions.
-  def set_user
-    @user = Domain::Fa::User.find_by(url_name: params[:url_name])
-  end
-end

app/controllers/domain/inkbunny/posts_controller.rb

@@ -1,19 +0,0 @@
-# typed: false
-class Domain::Inkbunny::PostsController < ApplicationController
-  skip_before_action :authenticate_user!, only: %i[show index]
-
-  def index
-    relation = Domain::Inkbunny::Post.includes(:creator, :files)
-
-    if params[:user_id].present?
-      @user = Domain::Inkbunny::User.find(params[:user_id])
-      relation = relation.where(creator: @user)
-    end
-
-    @posts = relation.order(ib_post_id: :desc).page(params[:page]).per(50)
-  end
-
-  def show
-    @post = Domain::Inkbunny::Post.find_by!(ib_post_id: params[:ib_post_id])
-  end
-end

app/controllers/domain/inkbunny/users_controller.rb

@@ -1,6 +0,0 @@
-# typed: true
-class Domain::Inkbunny::UsersController < ApplicationController
-  def show
-    @user = Domain::Inkbunny::User.find_by(name: params[:name])
-  end
-end

app/controllers/domain/post_groups_controller.rb

@@ -0,0 +1,26 @@
+# typed: true
+
+class Domain::PostGroupsController < DomainController
+  extend T::Sig
+  extend T::Helpers
+
+  skip_before_action :authenticate_user!, only: %i[show]
+  before_action :set_post_group!, only: %i[show]
+
+  # GET /pools/:id
+  sig(:final) { void }
+  def show
+    authorize @post_group
+  end
+
+  private
+
+  sig { override.returns(DomainController::DomainParamConfig) }
+  def self.param_config
+    DomainController::DomainParamConfig.new(
+      post_group_id_param: :id,
+      post_id_param: :domain_post_id,
+      user_id_param: :domain_user_id,
+    )
+  end
+end

app/controllers/domain/posts_controller.rb

@@ -0,0 +1,296 @@
+# typed: true
+
+require "open-uri"
+require "tempfile"
+require "base64"
+
+class Domain::PostsController < DomainController
+  extend T::Sig
+  extend T::Helpers
+
+  skip_before_action :authenticate_user!,
+                     only: %i[
+                       show
+                       index
+                       user_favorite_posts
+                       user_created_posts
+                       visual_search
+                       visual_results
+                     ]
+  before_action :set_post!, only: %i[show]
+  before_action :set_user!, only: %i[user_favorite_posts user_created_posts]
+  before_action :set_post_group!, only: %i[posts_in_group]
+
+  class PostsIndexViewConfig < T::ImmutableStruct
+    include T::Struct::ActsAsComparable
+
+    const :show_domain_filters, T::Boolean
+    const :show_creator_links, T::Boolean
+    const :index_type_header, String
+  end
+
+  sig { void }
+  def initialize
+    super
+    @posts_index_view_config =
+      PostsIndexViewConfig.new(
+        show_domain_filters: false,
+        show_creator_links: false,
+        index_type_header: "all_posts",
+      )
+  end
+
+  # GET /posts
+  sig(:final) { void }
+  def index
+    @posts_index_view_config =
+      PostsIndexViewConfig.new(
+        show_domain_filters: true,
+        show_creator_links: true,
+        index_type_header: "all_posts",
+      )
+    authorize Domain::Post
+    @posts = posts_relation(Domain::Post.all).without_count
+    active_sources = (params[:sources] || DomainSourceHelper.all_source_names)
+    unless DomainSourceHelper.has_all_sources?(active_sources)
+      postable_types =
+        DomainSourceHelper.source_names_to_class_names(active_sources)
+      @posts = @posts.where(type: postable_types) if postable_types.any?
+    end
+  end
+
+  # GET /posts/:id
+  sig(:final) { void }
+  def show
+    authorize @post
+  end
+
+  sig(:final) { void }
+  def user_favorite_posts
+    @posts_index_view_config =
+      PostsIndexViewConfig.new(
+        show_domain_filters: false,
+        show_creator_links: true,
+        index_type_header: "user_favorites",
+      )
+
+    @user = T.must(@user)
+    authorize @user
+    @posts = posts_relation(@user.faved_posts)
+    authorize @posts
+    render :index
+  end
+
+  sig(:final) { void }
+  def user_created_posts
+    @posts_index_view_config =
+      PostsIndexViewConfig.new(
+        show_domain_filters: false,
+        show_creator_links: true,
+        index_type_header: "user_created",
+      )
+
+    @user = T.must(@user)
+    authorize @user
+    @posts = posts_relation(@user.posts)
+    authorize @posts
+    render :index
+  end
+
+  sig(:final) { void }
+  def posts_in_group
+    @posts_index_view_config =
+      PostsIndexViewConfig.new(
+        show_domain_filters: false,
+        show_creator_links: true,
+        index_type_header: "posts_in_group",
+      )
+
+    authorize @post_group
+    @posts = posts_relation(T.must(@post_group).posts)
+    render :index
+  end
+
+  # GET /posts/visual_search
+  sig(:final) { void }
+  def visual_search
+    authorize Domain::Post
+  end
+
+  sig { params(content_type: T.nilable(String)).returns(T::Boolean) }
+  def check_content_type!(content_type)
+    return false unless content_type
+
+    ret =
+      Domain::PostFile::Thumbnail::THUMBABLE_CONTENT_TYPES.any? do |type|
+        content_type.match?(type)
+      end
+
+    unless ret
+      flash.now[:error] = "The uploaded file is not a valid image format."
+      render :visual_search
+    end
+
+    ret
+  end
+
+  # POST /posts/visual_search
+  sig(:final) { void }
+  def visual_results
+    authorize Domain::Post
+
+    # Process the uploaded image or URL
+    image_result = process_image_input
+    return unless image_result
+
+    image_path, content_type = image_result
+
+    # Create thumbnail for the view if possible
+    @uploaded_image_data_uri = create_thumbnail(image_path, content_type)
+    @uploaded_hash_value = generate_fingerprint(image_path)
+    @uploaded_detail_hash_value = generate_detail_fingerprint(image_path)
+    @post_file_fingerprints =
+      find_similar_fingerprints(@uploaded_hash_value).to_a
+    @post_file_fingerprints.sort! do |a, b|
+      helpers.calculate_similarity_percentage(
+        b.fingerprint_detail_value,
+        @uploaded_detail_hash_value,
+      ) <=>
+        helpers.calculate_similarity_percentage(
+          a.fingerprint_detail_value,
+          @uploaded_detail_hash_value,
+        )
+    end
+    @post_file_fingerprints = @post_file_fingerprints.take(10)
+    @posts = @post_file_fingerprints.map(&:post_file).compact.map(&:post)
+  ensure
+    # Clean up any temporary files
+    if @temp_file
+      @temp_file.unlink
+      @temp_file = nil
+    end
+  end
+
+  private
+
+  # Process the uploaded file or URL and return [image_path, content_type] or nil on failure
+  sig { returns(T.nilable([String, String])) }
+  def process_image_input
+    if params[:image_file].present?
+      process_uploaded_file
+    elsif params[:image_url].present?
+      process_image_url
+    else
+      flash.now[:error] = "Please upload an image or provide an image URL."
+      render :visual_search
+      nil
+    end
+  end
+
+  # Process an uploaded file and return [image_path, content_type] or nil on failure
+  sig { returns(T.nilable([String, String])) }
+  def process_uploaded_file
+    image_file = params[:image_file]
+    content_type = T.must(image_file.content_type)
+
+    return nil unless check_content_type!(content_type)
+
+    image_path = T.must(image_file.tempfile.path)
+    [image_path, content_type]
+  end
+
+  # Process an image URL and return [image_path, content_type] or nil on failure
+  sig { returns(T.nilable([String, String])) }
+  def process_image_url
+    # Download the image to a temporary file
+    image_url = params[:image_url]
+    image_io = URI.open(image_url)
+
+    if image_io.nil?
+      flash.now[:error] = "The URL does not point to a valid image format."
+      render :visual_search
+      return nil
+    end
+
+    content_type = T.must(T.unsafe(image_io).content_type)
+    return nil unless check_content_type!(content_type)
+
+    # Save to temp file
+    extension = helpers.extension_for_content_type(content_type) || "jpg"
+    @temp_file = Tempfile.new(["image", ".#{extension}"])
+    @temp_file.binmode
+    image_data = image_io.read
+    @temp_file.write(image_data)
+    @temp_file.close
+
+    image_path = T.must(@temp_file.path)
+    [image_path, content_type]
+  rescue StandardError => e
+    Rails.logger.error("Error processing image URL: #{e.message}")
+    flash.now[:error] = "Error downloading search image"
+    render :visual_search
+    nil
+  end
+
+  # Create a thumbnail from the image and return the data URI
+  sig do
+    params(image_path: String, content_type: String).returns(T.nilable(String))
+  end
+  def create_thumbnail(image_path, content_type)
+    helpers.create_image_thumbnail_data_uri(image_path, content_type)
+  end
+
+  # Generate a fingerprint from the image path
+  sig { params(image_path: String).returns(String) }
+  def generate_fingerprint(image_path)
+    # Use the new from_file_path method to create a fingerprint
+    Domain::PostFile::BitFingerprint.from_file_path(image_path)
+  end
+
+  # Generate a detail fingerprint from the image path
+  sig { params(image_path: String).returns(String) }
+  def generate_detail_fingerprint(image_path)
+    Domain::PostFile::BitFingerprint.detail_from_file_path(image_path)
+  end
+
+  # Find similar images based on the fingerprint
+  sig { params(fingerprint_value: String).returns(ActiveRecord::Relation) }
+  def find_similar_fingerprints(fingerprint_value)
+    # Use the model's similar_to_fingerprint method directly
+
+    subquery = <<~SQL
+      (
+        select distinct on (post_file_id) *, (fingerprint_value <~> '#{ActiveRecord::Base.connection.quote_string(fingerprint_value)}') as distance
+        from #{Domain::PostFile::BitFingerprint.table_name}
+        order by post_file_id, distance asc
+      ) subquery
+    SQL
+
+    Domain::PostFile::BitFingerprint
+      .select("*")
+      .from(subquery)
+      .order("distance ASC")
+      .limit(32)
+  end
+
+  sig { override.returns(DomainController::DomainParamConfig) }
+  def self.param_config
+    DomainController::DomainParamConfig.new(
+      post_id_param: :id,
+      user_id_param: :domain_user_id,
+      post_group_id_param: :domain_post_group_id,
+    )
+  end
+
+  sig(:final) do
+    params(starting_relation: ActiveRecord::Relation).returns(
+      T.all(ActiveRecord::Relation, Kaminari::ActiveRecordRelationMethods),
+    )
+  end
+  def posts_relation(starting_relation)
+    relation = starting_relation
+    relation = T.unsafe(policy_scope(relation)).page(params[:page]).per(50)
+    relation = relation.order(relation.klass.post_order_attribute => :desc)
+    relation
+  end
+end

app/controllers/domain/users_controller.rb

@@ -0,0 +1,253 @@
+# typed: true
+class Domain::UsersController < DomainController
+  extend T::Sig
+  extend T::Helpers
+
+  before_action :set_user!, only: %i[show followed_by following]
+  before_action :set_post!, only: %i[users_faving_post]
+  skip_before_action :authenticate_user!,
+                     only: %i[
+                       show
+                       search_by_name
+                       users_faving_post
+                       similar_users
+                     ]
+
+  # GET /users
+  sig(:final) { void }
+  def index
+    authorize Domain::User
+    @users = policy_scope(Domain::User).order(created_at: :desc)
+  end
+
+  sig(:final) { void }
+  def followed_by
+    @user = T.must(@user)
+    authorize @user
+    @users =
+      @user
+        .followed_by_users
+        .includes(avatar: :log_entry)
+        .page(params[:page])
+        .per(50)
+    @index_type = :followed_by
+    render :index
+  end
+
+  sig(:final) { void }
+  def following
+    @user = T.must(@user)
+    authorize @user
+    @users =
+      @user
+        .followed_users
+        .includes(avatar: :log_entry)
+        .page(params[:page])
+        .per(50)
+    @index_type = :following
+    render :index
+  end
+
+  sig(:final) { void }
+  def users_faving_post
+    @post = T.must(@post)
+    authorize @post
+    @users =
+      T
+        .unsafe(@post)
+        .faving_users
+        .includes(avatar: :log_entry)
+        .page(params[:page])
+        .per(50)
+
+    @index_type = :users_faving_post
+    render :index
+  end
+
+  # GET /users/:id
+  sig(:final) { void }
+  def show
+    authorize @user
+  end
+
+  sig(:final) { void }
+  def search_by_name
+    authorize Domain::User
+    name = params[:name]&.downcase
+    name = ReduxApplicationRecord.sanitize_sql_like(name)
+    @user_search_names =
+      Domain::UserSearchName
+        .select("domain_user_search_names.*, domain_users.*")
+        .select("levenshtein(name, '#{name}') as distance")
+        .select(
+          "(SELECT COUNT(*) FROM domain_user_post_creations dupc WHERE dupc.user_id = domain_users.id) as num_posts",
+        )
+        .joins(:user)
+        .where(
+          "(name ilike ?) OR (similarity(dmetaphone(name), dmetaphone(?)) > 0.8)",
+          "%#{name}%",
+          name,
+        )
+        .where(
+          "NOT EXISTS (
+            SELECT 1
+            FROM domain_user_search_names dns2
+            WHERE dns2.user_id = domain_user_search_names.user_id
+            AND levenshtein(dns2.name, ?) < levenshtein(domain_user_search_names.name, ?)
+          )",
+          name,
+          name,
+        )
+        .order("distance ASC")
+        .limit(10)
+  end
+
+  sig { void }
+  def similar_users
+    url_name = params[:url_name]
+    exclude_url_name = params[:exclude_url_name]
+
+    user = Domain::User::FaUser.find_by(url_name: url_name)
+    if user.nil?
+      render status: 404,
+             json: {
+               error: "user '#{url_name}' not found",
+               error_type: "user_not_found",
+             }
+      return
+    end
+
+    all_similar_users =
+      users_similar_to_by_followers(user, limit: 10).map do |u|
+        user_to_similarity_entry(u)
+      end
+
+    if all_similar_users.nil?
+      render status: 500,
+             json: {
+               error:
+                 "user '#{url_name}' has not had recommendations computed yet",
+               error_type: "recs_not_computed",
+             }
+      return
+    end
+
+    not_followed_similar_users = nil
+    if exclude_url_name
+      exclude_followed_by =
+        Domain::User::FaUser.find_by(url_name: exclude_url_name)
+      if exclude_followed_by.nil?
+        render status: 500,
+               json: {
+                 error: "user '#{exclude_url_name}' not found",
+                 error_type: "exclude_user_not_found",
+               }
+        return
+      elsif exclude_followed_by.scanned_follows_at.nil?
+        render status: 500,
+               json: {
+                 error:
+                   "user '#{exclude_url_name}' followers list hasn't been scanned",
+                 error_type: "exclude_user_not_scanned",
+               }
+        return
+      else
+        not_followed_similar_users =
+          users_similar_to_by_followers(
+            user,
+            limit: 10,
+            exclude_followed_by: exclude_followed_by,
+          ).map { |u| user_to_similarity_entry(u) }
+      end
+    end
+
+    render json: {
+             all: all_similar_users,
+             not_followed: not_followed_similar_users,
+           }
+  end
+
+  private
+
+  sig { override.returns(DomainController::DomainParamConfig) }
+  def self.param_config
+    DomainController::DomainParamConfig.new(
+      user_id_param: :id,
+      post_id_param: :domain_post_id,
+      post_group_id_param: :domain_post_group_id,
+    )
+  end
+
+  sig { params(user: Domain::User::FaUser).returns(T::Hash[Symbol, T.untyped]) }
+  def user_to_similarity_entry(user)
+    profile_thumb_url = user.avatar&.log_entry&.uri_str
+    profile_thumb_url ||=
+      begin
+        pp_log_entry = get_best_user_page_http_log_entry_for(user)
+        if pp_log_entry && (response_bytes = pp_log_entry.response_bytes)
+          parser =
+            Domain::Fa::Parser::Page.new(
+              response_bytes,
+              require_logged_in: false,
+            )
+          parser.user_page.profile_thumb_url
+        end
+      rescue StandardError
+        logger.error("error getting profile_thumb_url: #{$!.message}")
+      end || "https://a.furaffinity.net/0/#{user.url_name}.gif"
+
+    {
+      name: user.name,
+      url_name: user.url_name,
+      profile_thumb_url: profile_thumb_url,
+      external_url: "https://www.furaffinity.net/user/#{user.url_name}/",
+      refurrer_url: request.base_url + helpers.domain_user_path(user),
+    }
+  end
+
+  sig { params(user: Domain::User::FaUser).returns(T.nilable(HttpLogEntry)) }
+  def get_best_user_page_http_log_entry_for(user)
+    for_path =
+      proc do |uri_path|
+        HttpLogEntry
+          .where(
+            uri_scheme: "https",
+            uri_host: "www.furaffinity.net",
+            uri_path: uri_path,
+          )
+          .order(created_at: :desc)
+          .first
+      end
+
+    # older versions don't end in a trailing slash
+    user.last_user_page_log_entry || for_path.call("/user/#{user.url_name}/") ||
+      for_path.call("/user/#{user.url_name}")
+  end
+
+  sig do
+    params(
+      user: Domain::User::FaUser,
+      limit: Integer,
+      exclude_followed_by: T.nilable(Domain::User::FaUser),
+    ).returns(T::Array[Domain::User::FaUser])
+  end
+  def users_similar_to_by_followers(user, limit: 10, exclude_followed_by: nil)
+    factors = Domain::Factors::UserUserFollowToFactors.find_by(user: user)
+    return [] if factors.nil?
+
+    relation =
+      Domain::NeighborFinder
+        .find_neighbors(factors)
+        .limit(limit)
+        .includes(:user)
+
+    if exclude_followed_by
+      relation =
+        relation.where.not(
+          user_id: exclude_followed_by.followed_users.select(:to_id),
+        )
+    end
+
+    relation.map(&:user)
+  end
+end

app/controllers/domain_controller.rb

@@ -0,0 +1,73 @@
+# typed: strict
+class DomainController < ApplicationController
+  extend T::Sig
+  extend T::Helpers
+  abstract!
+
+  class DomainParamConfig < T::ImmutableStruct
+    include T::Struct::ActsAsComparable
+
+    const :post_id_param, Symbol
+    const :user_id_param, Symbol
+    const :post_group_id_param, Symbol
+  end
+
+  sig { void }
+  def initialize
+    super
+    @post = T.let(nil, T.nilable(Domain::Post))
+    @user = T.let(nil, T.nilable(Domain::User))
+    @post_group = T.let(nil, T.nilable(Domain::PostGroup))
+  end
+
+  protected
+
+  sig { abstract.returns(DomainParamConfig) }
+  def self.param_config
+  end
+
+  sig(:final) { void }
+  def set_post!
+    @post =
+      self.class.find_model_from_param(
+        Domain::Post,
+        params[self.class.param_config.post_id_param],
+      ) || raise(ActiveRecord::RecordNotFound)
+  end
+
+  sig(:final) { void }
+  def set_user!
+    @user =
+      self.class.find_model_from_param(
+        Domain::User,
+        params[self.class.param_config.user_id_param],
+      ) || raise(ActiveRecord::RecordNotFound)
+  end
+
+  sig(:final) { void }
+  def set_post_group!
+    @post_group =
+      self.class.find_model_from_param(
+        Domain::PostGroup,
+        params[self.class.param_config.post_group_id_param],
+      ) || raise(ActiveRecord::RecordNotFound)
+  end
+
+  public
+
+  sig(:final) do
+    type_parameters(:Klass)
+      .params(
+        klass:
+          T.all(
+            T.class_of(ReduxApplicationRecord),
+            HasCompositeToParam::ClassMethods[T.type_parameter(:Klass)],
+          ),
+        param: T.nilable(String),
+      )
+      .returns(T.nilable(T.type_parameter(:Klass)))
+  end
+  def self.find_model_from_param(klass, param)
+    klass.find_by_param(param)
+  end
+end

app/controllers/indexed_posts_controller.rb

@@ -1,13 +0,0 @@
-# typed: false
-class IndexedPostsController < ApplicationController
-  def index
-    @posts = IndexedPost.all
-    active_sources = (params[:sources] || SourceHelper.all_source_names)
-    unless SourceHelper.has_all_sources?(active_sources)
-      postable_types = SourceHelper.source_names_to_class_names(active_sources)
-      @posts =
-        @posts.where(postable_type: postable_types) if postable_types.any?
-    end
-    @posts = @posts.order(created_at: :desc).page(params[:page]).per(50)
-  end
-end

app/controllers/log_entries_controller.rb

@@ -56,7 +56,7 @@ class LogEntriesController < ApplicationController
     HttpLogEntry
       .joins(:response)
       .includes(:response)
-      .select("http_log_entries.*, blob_entries_p.size")
+      .select("http_log_entries.*, blob_files.size_bytes")
       .find_each(batch_size: 100, order: :desc) do |log_entry|
         break if log_entry.created_at < @time_window.ago
         @last_window_count += 1
@@ -76,7 +76,7 @@ class LogEntriesController < ApplicationController
       HttpLogEntry.includes(
         :caused_by_entry,
         :triggered_entries,
-        response: :base,
+        :response,
       ).find(params[:id])
   end
 end

app/controllers/pages_controller.rb

@@ -1,8 +1,12 @@
 # typed: true
 class PagesController < ApplicationController
-  skip_before_action :authenticate_user!, only: [:root]
+  skip_before_action :authenticate_user!, only: %i[root furecs_user_script]
 
   def root
     render :root
   end
+
+  def furecs_user_script
+    render :furecs_user_script
+  end
 end

app/controllers/state/ip_address_roles_controller.rb

@@ -0,0 +1,85 @@
+# typed: true
+class State::IpAddressRolesController < ApplicationController
+  before_action :set_ip_address_role, only: %i[edit update destroy toggle]
+  before_action :authorize_ip_address_roles
+
+  # GET /state/ip_address_roles
+  def index
+    @ip_address_roles = IpAddressRole.all.order(created_at: :desc)
+  end
+
+  # GET /state/ip_address_roles/new
+  def new
+    @ip_address_role = IpAddressRole.new
+  end
+
+  # GET /state/ip_address_roles/1/edit
+  def edit
+  end
+
+  # POST /state/ip_address_roles
+  def create
+    @ip_address_role = IpAddressRole.new(ip_address_role_params)
+
+    if @ip_address_role.save
+      redirect_to state_ip_address_roles_path,
+                  notice: "IP address role was successfully created."
+    else
+      render :new
+    end
+  end
+
+  # PATCH/PUT /state/ip_address_roles/1
+  def update
+    if @ip_address_role.update(ip_address_role_params)
+      redirect_to state_ip_address_roles_path,
+                  notice: "IP address role was successfully updated."
+    else
+      render :edit
+    end
+  end
+
+  # DELETE /state/ip_address_roles/1
+  def destroy
+    @ip_address_role.destroy
+    redirect_to state_ip_address_roles_path,
+                notice: "IP address role was successfully deleted."
+  end
+
+  def toggle
+    @ip_address_role.update!(active: !@ip_address_role.active)
+    redirect_to state_ip_address_roles_path
+  rescue ActiveRecord::RecordInvalid => e
+    redirect_to state_ip_address_roles_path,
+                alert: "Failed to update status: #{e.message}"
+  end
+
+  private
+
+  # Use callbacks to share common setup or constraints between actions
+  def set_ip_address_role
+    @ip_address_role = IpAddressRole.find(params[:id])
+  end
+
+  # Only allow a list of trusted parameters through
+  def ip_address_role_params
+    params.require(:ip_address_role).permit(
+      :ip_address,
+      :role,
+      :description,
+      :active,
+    )
+  end
+
+  # Authorize all actions based on the current action
+  def authorize_ip_address_roles
+    case action_name.to_sym
+    when :index, :new, :edit
+      authorize IpAddressRole, policy_class: State::IpAddressRolePolicy
+    when :create
+      authorize IpAddressRole, policy_class: State::IpAddressRolePolicy
+    when :update, :destroy, :toggle
+      authorize @ip_address_role, policy_class: State::IpAddressRolePolicy
+    end
+  end
+end

app/controllers/user_scripts_controller.rb

@@ -1,21 +1,24 @@
 # typed: true
 class UserScriptsController < ApplicationController
   skip_before_action :authenticate_user!, only: [:get]
+  skip_before_action :verify_authenticity_token, only: [:get]
+
+  ALLOWED_SCRIPTS = %w[object_statuses.user.js furecs.user.js].freeze
 
   def get
     expires_in 1.hour, public: true
-    response.cache_control[:public] = false
-    response.cache_control[:private] = true
+    response.cache_control[:public] = true
+    response.cache_control[:private] = false
 
     script = params[:script]
-    case script
-    when "furecs.user.js"
-      send_file(
-        Rails.root.join("user_scripts/furecs.user.js"),
-        type: "application/json",
-      )
-    else
+    unless ALLOWED_SCRIPTS.include?(script)
       render status: 404, text: "not found"
+      return
     end
+
+    send_file(
+      Rails.root.join("user_scripts/dist/#{script}"),
+      type: "application/javascript",
+    )
   end
 end

app/helpers/domain/descriptions_helper.rb

@@ -0,0 +1,348 @@
+# typed: strict
+# frozen_string_literal: true
+require "dtext"
+
+module Domain::DescriptionsHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  include Domain::PostsHelper
+  include Domain::DomainsHelper
+  include Domain::UsersHelper
+  requires_ancestor { Object }
+
+  abstract!
+
+  sig do
+    params(assumed_host: String, url_string: String).returns(
+      T.nilable(Addressable::URI),
+    )
+  end
+  def try_parse_uri(assumed_host, url_string)
+    extracted = URI.extract(url_string).first || url_string
+
+    # if the url string starts with a slash, add the assumed host to it
+    extracted = assumed_host + extracted if extracted.starts_with?("/")
+
+    # if the url string doesn't have a protocol, add https:// to it
+    unless extracted.starts_with?("http") && extracted.include?("://")
+      extracted = "https://" + extracted
+    end
+
+    uri = Addressable::URI.parse(extracted)
+    uri.host ||= assumed_host
+    uri.scheme ||= "https"
+    uri
+  rescue Addressable::URI::InvalidURIError
+    nil
+  end
+
+  sig { params(text: String, url: String).returns(T::Boolean) }
+  def text_same_as_url?(text, url)
+    text = text.strip.downcase
+    url = url.strip.downcase
+    ["", "http://", "https://"].any? { |prefix| "#{prefix}#{text}" == url }
+  end
+
+  sig { params(model: HasDescriptionHtmlForView).returns(T.nilable(String)) }
+  def description_section_class_for_model(model)
+    case model
+    when Domain::Post::FaPost, Domain::User::FaUser
+      "bg-slate-700 p-4 text-slate-200 text-sm"
+    when Domain::Post::E621Post, Domain::User::E621User
+      "bg-slate-700 p-4 text-slate-200 text-sm"
+    else
+      nil
+    end
+  end
+
+  WEAK_URL_MATCHER_REGEX =
+    %r{(http(s)?:\/\/.)?(www\.)?[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)}
+
+  ALLOWED_INFERRED_URL_DOMAINS =
+    T.let(
+      %w[furaffinity.net inkbunny.net e621.net]
+        .flat_map { |domain| [domain, "www.#{domain}"] }
+        .freeze,
+      T::Array[String],
+    )
+
+  sig { params(model: HasDescriptionHtmlForView).returns(T.nilable(String)) }
+  def sanitize_description_html(model)
+    html = model.description_html_for_view
+    return nil if html.blank?
+
+    case model
+    when Domain::Post::E621Post
+      dtext_result = DText.parse(html)
+      return nil if dtext_result.blank?
+      html = dtext_result[0]
+    else
+      # profiles often contain bbcode, so first re-parse that
+      # for some reason, lots of duplicate <br> tags appear as well
+      html = html.gsub("<br>", "").strip
+      html = try_convert_bbcode_to_html(html)
+    end
+
+    replacements = {}
+
+    # Transform bare text that is not contained within an anchor tag into an anchor tag
+    text_link_transformer =
+      lambda do |env|
+        node = T.cast(env[:node], Nokogiri::XML::Node)
+        return if env[:is_allowlisted]
+
+        next unless node.text?
+        next unless node.ancestors("a").empty?
+        next unless (node_text = T.cast(node.text, T.nilable(String)))
+        next unless (match = node_text.match(WEAK_URL_MATCHER_REGEX))
+        next unless (url_text = match[0])
+        unless (
+                 uri =
+                   try_parse_uri(model.description_html_base_domain, url_text)
+               )
+          next
+        end
+        unless ALLOWED_PLAIN_TEXT_URL_DOMAINS.any? { |domain|
+                 url_matches_domain?(domain, uri.host)
+               }
+          next
+        end
+
+        before, after = node.text.split(url_text, 2)
+        new_node = "#{before}<a href=\"#{url_text}\">#{url_text}</a>#{after}"
+        node.replace(new_node)
+      end
+
+    tag_class_and_style_transformer =
+      lambda do |env|
+        node = T.cast(env[:node], Nokogiri::XML::Node)
+        node_name = T.cast(env[:node_name], String)
+        return if env[:is_allowlisted] || !node.element?
+
+        # Convert bbcode_center class to text-align: center style
+        # and remove all other styling
+        add_node_styles = []
+        if node["class"]&.include?("bbcode_center")
+          add_node_styles << "text-align: center"
+        end
+
+        node.name = "div" if node_name == "code"
+        node.remove_attribute("class")
+        # add to original styles
+        node["style"] = (node["style"] || "")
+          .split(";")
+          .map(&:strip)
+          .concat(add_node_styles)
+          .map { |s| s + ";" }
+          .join(" ")
+      end
+
+    link_to_model_link_transformer =
+      lambda do |env|
+        node = T.cast(env[:node], Nokogiri::XML::Node)
+        node_name = T.cast(env[:node_name], String)
+        next if env[:is_allowlisted] || !node.element?
+
+        # Only allow and transform FA links
+        if node_name == "a"
+          href_str = node["href"]&.downcase || ""
+          url = try_parse_uri(model.description_html_base_domain, href_str)
+          next { node_whitelist: [] } if url.nil?
+
+          found_link = link_for_source(url.to_s)
+          if found_link.present? && (found_model = found_link.model)
+            partial, locals =
+              case found_model
+              when Domain::Post
+                [
+                  "domain/has_description_html/inline_link_domain_post",
+                  {
+                    post: found_model,
+                    link_text: node.text,
+                    visual_style: "description-section-link",
+                  },
+                ]
+              when Domain::User
+                [
+                  "domain/has_description_html/inline_link_domain_user",
+                  {
+                    user: found_model,
+                    link_text: node.text,
+                    visual_style: "description-section-link",
+                  },
+                ]
+              else
+                raise "Unknown model type: #{found_link.model.class}"
+              end
+
+            replacements[node] = Nokogiri::HTML5.fragment(
+              render(partial:, locals:),
+            )
+            next { node_whitelist: [node] }
+          else
+            if ALLOWED_EXTERNAL_LINK_DOMAINS.any? { |domain|
+                 url_matches_domain?(domain, url.host)
+               }
+              if node.text.blank? || text_same_as_url?(node.text, url.to_s)
+                title = title_for_url(url.to_s)
+              else
+                title = node.text
+              end
+
+              replacements[node] = Nokogiri::HTML5.fragment(
+                render(
+                  partial: "domain/has_description_html/inline_link_external",
+                  locals: {
+                    url: url.to_s,
+                    title:,
+                    icon_path: icon_path_for_domain(url.host),
+                  },
+                ),
+              )
+              next { node_whitelist: [node] }
+            end
+          end
+        end
+      end
+
+    disallowed_link_transformer =
+      lambda do |env|
+        node = T.cast(env[:node], Nokogiri::XML::Node)
+        node_name = T.cast(env[:node_name], String)
+        return if env[:is_allowlisted] || !node.element?
+
+        if node_name == "a"
+          # by the time we're here, we know this is not a valid link node,
+          # and it should be replaced with its text
+          node.replace(node.inner_html)
+        end
+      end
+
+    sanitizer =
+      Sanitize.new(
+        elements: %w[a code div br img b i span strong hr p],
+        attributes: {
+          "a" => %w[href class],
+          :all => %w[class style],
+        },
+        css: {
+          properties: %w[font-size color text-align class],
+        },
+        transformers: [
+          text_link_transformer,
+          tag_class_and_style_transformer,
+          link_to_model_link_transformer,
+          disallowed_link_transformer,
+        ],
+      )
+
+    fragment = Nokogiri::HTML5.fragment(sanitizer.send(:preprocess, html))
+    sanitizer.node!(fragment)
+    replacements.each { |node, replacement| node.replace(replacement) }
+    raw fragment.to_html(preserve_newline: true)
+  rescue StandardError
+    raise if Rails.env == "staging" || Rails.env.test? || Rails.env.development?
+    # if anything goes wrong in production, bail out and don't display anything
+    "(error generating description)"
+  end
+
+  sig { params(visual_style: String).returns(String) }
+  def link_classes_for_visual_style(visual_style)
+    case visual_style
+    when "sky-link"
+      "blue-link truncate"
+    when "description-section-link"
+      [
+        "text-sky-200 border-slate-200",
+        "border border-transparent hover:border-slate-300 hover:text-sky-800 hover:bg-slate-100",
+        "rounded-md px-1 transition-all",
+        "inline-flex items-center align-bottom",
+      ].join(" ")
+    else
+      "blue-link"
+    end
+  end
+
+  sig do
+    params(user: Domain::User, visual_style: String, icon_size: String).returns(
+      T::Hash[Symbol, T.untyped],
+    )
+  end
+  def props_for_user_hover_preview(user, visual_style, icon_size)
+    cache_key = [
+      user,
+      policy(user),
+      "popover_inline_link_domain_user",
+      icon_size,
+    ]
+    Rails
+      .cache
+      .fetch(cache_key) do
+        num_posts =
+          user.has_created_posts? ? user.user_post_creations.count : nil
+        registered_at = domain_user_registered_at_string_for_view(user)
+        num_followed_by =
+          user.has_followed_by_users? ? user.user_user_follows_to.count : nil
+        num_followed =
+          user.has_followed_users? ? user.user_user_follows_from.count : nil
+        avatar_thumb_size = icon_size == "large" ? "64-avatar" : "32-avatar"
+
+        {
+          iconSize: icon_size,
+          linkText: user.name_for_view,
+          userId: user.to_param,
+          userName: user.name_for_view,
+          userPath: domain_user_path(user),
+          userSmallAvatarPath:
+            domain_user_avatar_img_src_path(
+              user.avatar,
+              thumb: avatar_thumb_size,
+            ),
+          userAvatarPath: domain_user_avatar_img_src_path(user.avatar),
+          userAvatarAlt: "View #{user.name_for_view}'s profile",
+          userDomainIcon: domain_model_icon_path(user),
+          userNumPosts: num_posts,
+          userRegisteredAt: registered_at,
+          userNumFollowedBy: num_followed_by,
+          userNumFollowed: num_followed,
+        }
+      end
+      .then do |props|
+        props[:visualStyle] = visual_style
+        props
+      end
+  end
+
+  sig do
+    params(post: Domain::Post, link_text: String, visual_style: String).returns(
+      T::Hash[Symbol, T.untyped],
+    )
+  end
+  def props_for_post_hover_preview(post, link_text, visual_style)
+    cache_key = [post, policy(post), "popover_inline_link_domain_post"]
+    Rails
+      .cache
+      .fetch(cache_key) do
+        {
+          linkText: link_text,
+          postId: post.to_param,
+          postTitle: post.title,
+          postPath: Rails.application.routes.url_helpers.domain_post_path(post),
+          postThumbnailPath: thumbnail_for_post_path(post),
+          postThumbnailAlt: "View on #{domain_name_for_model(post)}",
+          postDomainIcon: domain_model_icon_path(post),
+        }.then do |props|
+          if creator = post.primary_creator_for_view
+            props[:creatorName] = creator.name_for_view
+            props[:creatorAvatarPath] = user_avatar_path_for_view(creator)
+          end
+          props
+        end
+      end
+      .then do |props|
+        props[:visualStyle] = visual_style
+        props
+      end
+  end
+end

app/helpers/domain/domain_model_helper.rb

@@ -0,0 +1,46 @@
+# typed: strict
+module Domain::DomainModelHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  abstract!
+
+  HasDomainTypeType =
+    T.type_alias { T.any(HasDomainType, HasDomainType::ClassMethods) }
+
+  sig { params(model: HasDomainTypeType).returns(String) }
+  def domain_name_for_model(model)
+    case model.domain_type
+    when Domain::DomainType::Fa
+      "FurAffinity"
+    when Domain::DomainType::E621
+      "E621"
+    when Domain::DomainType::Inkbunny
+      "Inkbunny"
+    end
+  end
+
+  sig { params(model: HasDomainTypeType).returns(String) }
+  def domain_abbreviation_for_model(model)
+    case model.domain_type
+    when Domain::DomainType::Fa
+      "FA"
+    when Domain::DomainType::E621
+      "E621"
+    when Domain::DomainType::Inkbunny
+      "IB"
+    end
+  end
+
+  sig { params(model: Domain::Post).returns(String) }
+  def title_for_post_model(model)
+    case model
+    when Domain::Post::FaPost
+      model.title
+    when Domain::Post::E621Post
+      model.title
+    when Domain::Post::InkbunnyPost
+      model.title
+    end || "(unknown)"
+  end
+end

app/helpers/domain/domain_type.rb

@@ -0,0 +1,9 @@
+# typed: strict
+# Enum represents the domain of a post or user, e.g. "FurAffinity", "E621", "Inkbunny"
+class Domain::DomainType < T::Enum
+  enums do
+    Fa = new
+    E621 = new
+    Inkbunny = new
+  end
+end

app/helpers/domain/domains_helper.rb

@@ -0,0 +1,133 @@
+# typed: strict
+module Domain::DomainsHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  abstract!
+
+  # If a URL is detected in plain text and is one of these domains,
+  # it will be converted to an anchor tag.
+  ALLOWED_PLAIN_TEXT_URL_DOMAINS = %w[
+    e621.net
+    furaffinity.net
+    inkbunny.net
+  ].freeze
+
+  # If a link is detected in an anchor tag and is one of these domains,
+  # it will be converted to a link.
+  ALLOWED_EXTERNAL_LINK_DOMAINS =
+    T.let(
+      (
+        %w[
+          archiveofourown.org
+          behance.net
+          bigcartel.com
+          boosty.to
+          bsky.app
+          carrd.co
+          deviantart.com
+          discord.gg
+          dribbble.com
+          e621.net
+          facebook.com
+          furaffinity.net
+          gumroad.com
+          hipolink.me
+          inkbunny.net
+          itch.io
+          instagram.com
+          ko-fi.com
+          livejournal.com
+          mstdn.social
+          patreon.com
+          pinterest.com
+          pixiv.net
+          redbubble.com
+          spreadshirt.com
+          spreadshirt.de
+          t.me
+          tumblr.com
+          twitch.tv
+          twitter.com
+          vimeo.com
+          weasyl.com
+          x.com
+          youtube.com
+        ] + ALLOWED_PLAIN_TEXT_URL_DOMAINS
+      ).freeze,
+      T::Array[String],
+    )
+
+  DOMAIN_TO_ICON_PATH =
+    T.let(
+      {
+        "bigcartel.com" => "bigcartel.png",
+        "boosty.to" => "boosty.png",
+        "bsky.app" => "bsky.png",
+        "carrd.co" => "carrd.png",
+        "deviantart.com" => "deviantart.png",
+        "e621.net" => "e621.png",
+        "furaffinity.net" => "fa.png",
+        "ib.metapix.net" => "inkbunny.png",
+        "inkbunny.net" => "inkbunny.png",
+        "itaku.ee" => "itaku.png",
+        "ko-fi.com" => "ko-fi.png",
+        "newgrounds.com" => "newgrounds.png",
+        "patreon.com" => "patreon.png",
+        "pixiv.net" => "pixiv.png",
+        "redbubble.com" => "redbubble.png",
+        "spreadshirt.com" => "spreadshirt.png",
+        "spreadshirt.de" => "spreadshirt.png",
+        "subscribestar.com" => "subscribestar.png",
+        "subscribestar.adult" => "subscribestar.png",
+        "gumroad.com" => "gumroad.png",
+        "itch.io" => "itch-io.png",
+        "t.me" => "telegram.png",
+        "tumblr.com" => "tumblr.png",
+        "twitter.com" => "x-twitter.png",
+        "weasyl.com" => "weasyl.png",
+        "wixmp.com" => "deviantart.png",
+        "x.com" => "x-twitter.png",
+      }.freeze,
+      T::Hash[String, String],
+    )
+
+  DOMAIN_TITLE_MAPPERS =
+    T.let(
+      [
+        [%r{://t.me/([^/]+)}, ->(match) { match[1] }],
+        [%r{://bsky.app/profile/([^/]+)}, ->(match) { match[1] }],
+        [%r{://(.*\.)?x.com/([^/]+)}, ->(match) { match[2] }],
+        [%r{://(.*\.)?twitter.com/([^/]+)}, ->(match) { match[2] }],
+        [%r{://(.*\.)?patreon.com/([^/]+)}, ->(match) { match[2] }],
+        [%r{://(.*\.)?furaffinity.net/user/([^/]+)}, ->(match) { match[2] }],
+      ],
+      T::Array[[Regexp, T.proc.params(match: MatchData).returns(String)]],
+    )
+
+  sig { params(domain: String, host: String).returns(T::Boolean) }
+  def url_matches_domain?(domain, host)
+    host == domain || host.end_with?(".#{domain}")
+  end
+
+  sig { params(domain: String).returns(T.nilable(String)) }
+  def icon_path_for_domain(domain)
+    for test_domain, icon in DOMAIN_TO_ICON_PATH
+      if url_matches_domain?(test_domain, domain)
+        return asset_path("domain-icons/#{icon}")
+      end
+    end
+    nil
+  end
+
+  sig { params(url: String).returns(String) }
+  def title_for_url(url)
+    url = url.to_s
+    for mapper in DOMAIN_TITLE_MAPPERS
+      if (match = mapper[0].match(url)) && (group = mapper[1].call(match))
+        return group
+      end
+    end
+    url
+  end
+end

app/helpers/domain/fa/posts_helper.rb

@@ -69,8 +69,18 @@ module Domain::Fa::PostsHelper
               # by default, assume the host is www.furaffinity.net
               href = node["href"]&.downcase || ""
               href = "//" + href if href.match?(/^(www\.)?furaffinity\.net/)
-              uri = URI.parse(href)
+              uri =
+                begin
+                  URI.parse(href)
+                rescue URI::InvalidURIError
+                  nil
+                end
+
+              valid_type = !uri.is_a?(URI::MailTo)
+              next { node_whitelist: [node] } if uri.nil? || !valid_type
+
               uri.host ||= "www.furaffinity.net"
+              uri.scheme ||= "https"
               path = uri.path
 
               fa_host_matcher = /^(www\.)?furaffinity\.net$/

app/helpers/domain/fa/users_helper.rb

@@ -1,5 +1,7 @@
 # typed: false
 module Domain::Fa::UsersHelper
+  extend T::Sig
+
   def avatar_url(sha256, thumb: "32-avatar")
     blob_path(HexUtil.bin2hex(sha256), format: "jpg", thumb: thumb)
   end
@@ -86,15 +88,31 @@ module Domain::Fa::UsersHelper
         )
   end
 
+  # TODO - remove this once we've migrated similarity scores to new user model
+  sig do
+    params(
+      user: Domain::User::FaUser,
+      limit: Integer,
+      exclude_followed_by: T.nilable(Domain::User::FaUser),
+    ).returns(T::Array[Domain::User::FaUser])
+  end
   def similar_users_by_followed(user, limit: 10, exclude_followed_by: nil)
-    if user.disco.nil?
-      nil
-    else
-      ReduxApplicationRecord.connection.execute("SET ivfflat.probes = 32")
-      user.similar_users_by_followed(
-        exclude_followed_by: exclude_followed_by,
-      ).limit(limit)
+    factors = Domain::Factors::UserUserFollowToFactors.find_by(user: user)
+    return [] if factors.nil?
+
+    relation =
+      Domain::NeighborFinder
+        .find_neighbors(factors)
+        .limit(limit)
+        .includes(:user)
+    if exclude_followed_by
+      relation =
+        relation.where.not(
+          user_id: exclude_followed_by.followed_users.select(:to_id),
+        )
     end
+
+    relation.map { |factor| factor.user }
   end
 
   def fa_user_account_status(user)

app/helpers/domain/model_helper.rb

@@ -0,0 +1,39 @@
+# typed: strict
+module Domain::ModelHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  include Pundit::Authorization
+  abstract!
+
+  sig do
+    params(
+      model: HasViewPrefix,
+      partial: String,
+      as: Symbol,
+      expires_in: ActiveSupport::Duration,
+      cache_key: T.untyped,
+    ).returns(T.nilable(String))
+  end
+  def render_for_model(model, partial, as:, expires_in: 1.hour, cache_key: nil)
+    cache_key ||= [model, policy(model), partial]
+    Rails
+      .cache
+      .fetch(cache_key, expires_in:) do
+        prefixes = lookup_context.prefixes
+        partial_path =
+          prefixes
+            .map { |prefix| "#{prefix}/#{model.class.view_prefix}/_#{partial}" }
+            .find { |view| lookup_context.exists?(view) } ||
+            prefixes
+              .map { |prefix| "#{prefix}/default/_#{partial}" }
+              .find { |view| lookup_context.exists?(view) } ||
+            Kernel.raise("no partial found for #{partial} in #{prefixes}")
+
+        partial_path = partial_path.split("/")
+        T.must(partial_path.last).delete_prefix!("_")
+        partial_path = partial_path.join("/")
+        render partial: partial_path, locals: { as => model }
+      end
+  end
+end

app/helpers/domain/pagination_helper.rb

@@ -0,0 +1,71 @@
+# typed: strict
+module Domain::PaginationHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  abstract!
+
+  sig do
+    params(
+      collection: T.untyped,
+      link_text: String,
+      link_opts: T.untyped,
+    ).returns(String)
+  end
+  def collection_previous_page_link(collection, link_text, link_opts)
+    request_url = Addressable::URI.parse(request.url)
+    page = collection.current_page - 1
+    set_uri_page!(request_url, page)
+    path = get_uri_path_and_query(request_url)
+    link_to link_text, path, link_opts
+  end
+
+  sig do
+    params(
+      collection: T.untyped,
+      link_text: String,
+      link_opts: T.untyped,
+    ).returns(String)
+  end
+  def collection_next_page_link(collection, link_text, link_opts)
+    request_url = Addressable::URI.parse(request.url)
+    page = collection.current_page + 1
+    set_uri_page!(request_url, page)
+    path = get_uri_path_and_query(request_url)
+    link_to link_text, path, link_opts
+  end
+
+  sig do
+    params(
+      collection: T.untyped,
+      link_text: String,
+      link_opts: T.untyped,
+    ).returns(String)
+  end
+  def collection_previous_page_link(collection, link_text, link_opts)
+    request_url = Addressable::URI.parse(request.url)
+    page = collection.current_page - 1
+    set_uri_page!(request_url, page)
+    path = get_uri_path_and_query(request_url)
+    link_to link_text, path, link_opts
+  end
+
+  sig { params(uri: Addressable::URI, page: T.nilable(Integer)).void }
+  def set_uri_page!(uri, page)
+    query_values = uri.query_values || {}
+    if page.nil? || page <= 1
+      query_values.delete("page")
+    else
+      query_values["page"] = page
+    end
+
+    uri.query_values = query_values.empty? ? nil : query_values
+  end
+
+  sig { params(uri: Addressable::URI).returns(String) }
+  def get_uri_path_and_query(uri)
+    path = uri.path
+    path += "?#{uri.query}" if uri.query.present?
+    path
+  end
+end

app/helpers/domain/post_groups_helper.rb

@@ -0,0 +1,17 @@
+# typed: strict
+module Domain::PostGroupsHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  abstract!
+
+  sig { params(post_group: Domain::PostGroup).returns(String) }
+  def domain_post_group_path(post_group)
+    "#{domain_post_groups_path}/#{post_group.to_param}"
+  end
+
+  sig { returns(String) }
+  def domain_post_groups_path
+    "/pools"
+  end
+end

app/helpers/domain/posts_helper.rb

@@ -0,0 +1,492 @@
+# typed: strict
+module Domain::PostsHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  include LogEntriesHelper
+  include Domain::UsersHelper
+  include PathsHelper
+  include Domain::DomainsHelper
+  include Domain::DomainModelHelper
+  include Pundit::Authorization
+  require "base64"
+  abstract!
+
+  class DomainData < T::Struct
+    include T::Struct::ActsAsComparable
+
+    const :domain_icon_path, String
+    const :domain_icon_title, String
+  end
+
+  DEFAULT_DOMAIN_DATA =
+    DomainData.new(
+      domain_icon_path: "generic-domain.svg",
+      domain_icon_title: "Unknown",
+    )
+
+  DOMAIN_DATA =
+    T.let(
+      {
+        Domain::DomainType::Fa =>
+          DomainData.new(
+            domain_icon_path: "domain-icons/fa.png",
+            domain_icon_title: "Furaffinity",
+          ),
+        Domain::DomainType::E621 =>
+          DomainData.new(
+            domain_icon_path: "domain-icons/e621.png",
+            domain_icon_title: "E621",
+          ),
+        Domain::DomainType::Inkbunny =>
+          DomainData.new(
+            domain_icon_path: "domain-icons/inkbunny.png",
+            domain_icon_title: "Inkbunny",
+          ),
+      },
+      T::Hash[Domain::DomainType, DomainData],
+    )
+
+  sig { params(model: HasDomainType).returns(String) }
+  def domain_model_icon_path(model)
+    path =
+      if (domain_data = DOMAIN_DATA[model.domain_type])
+        domain_data.domain_icon_path
+      else
+        DEFAULT_DOMAIN_DATA.domain_icon_path
+      end
+    asset_path(path)
+  end
+
+  sig { params(post: Domain::Post).returns(T.nilable(Domain::PostFile)) }
+  def gallery_file_for_post(post)
+    file = post.primary_file_for_view
+    return nil unless file.present?
+    return nil unless file.state_ok?
+    return nil unless file.log_entry_id.present?
+    content_type = file.log_entry&.content_type
+    return nil unless content_type.present?
+    return nil unless is_thumbable_content_type?(content_type)
+    file
+  end
+
+  sig { params(post: Domain::Post).returns(T.nilable(String)) }
+  def gallery_file_info_for_post(post)
+    file = post.primary_file_for_view
+    return nil unless file.present?
+    return nil unless file.state_ok?
+    return nil unless file.log_entry_id.present?
+    content_type = file.log_entry&.content_type || ""
+    pretty_content_type(content_type)
+  end
+
+  sig { params(post: Domain::Post).returns(T.nilable(String)) }
+  def thumbnail_for_post_path(post)
+    return nil unless policy(post).view_file?
+    file = gallery_file_for_post(post)
+    return nil unless file.present?
+    return nil unless file.state_ok?
+    return nil unless file.log_entry_id.present?
+    if (log_entry = file.log_entry) &&
+         (response_sha256 = log_entry.response_sha256)
+      blob_path(HexUtil.bin2hex(response_sha256), format: "jpg", thumb: "small")
+    end
+  end
+
+  # Create a data URI thumbnail from an image file
+  sig do
+    params(file_path: String, content_type: String, max_size: Integer).returns(
+      T.nilable(String),
+    )
+  end
+  def create_image_thumbnail_data_uri(file_path, content_type, max_size = 180)
+    # Load the Vips library properly instead of using require directly
+    begin
+      # Load the image
+      image = ::Vips::Image.new_from_file(file_path)
+
+      # Calculate the scaling factor to keep within max_size
+      scale = [max_size.to_f / image.width, max_size.to_f / image.height].min
+
+      # Only scale down, not up
+      scale = 1.0 if scale > 1.0
+
+      # Resize the image (use nearest neighbor for speed as this is just a thumbnail)
+      thumbnail = image.resize(scale)
+
+      # Get the image data in the original format
+      # For JPEG use quality 85 for a good balance of quality vs size
+      output_format = content_type.split("/").last
+
+      case output_format
+      when "jpeg", "jpg"
+        image_data = thumbnail.write_to_buffer(".jpg", Q: 85)
+      when "png"
+        image_data = thumbnail.write_to_buffer(".png", compression: 6)
+      when "gif"
+        image_data = thumbnail.write_to_buffer(".gif")
+      else
+        # Default to JPEG if format not recognized
+        image_data = thumbnail.write_to_buffer(".jpg", Q: 85)
+        content_type = "image/jpeg"
+      end
+
+      # Create the data URI
+      "data:#{content_type};base64,#{Base64.strict_encode64(image_data)}"
+    rescue => e
+      Rails.logger.error("Error creating thumbnail: #{e.message}")
+      nil
+    end
+  end
+
+  sig { params(content_type: String).returns(String) }
+  def pretty_content_type(content_type)
+    case content_type
+    when %r{text/plain}
+      "Plain Text Document"
+    when %r{application/pdf}
+      "PDF Document"
+    when %r{application/msword}
+      "Microsoft Word Document"
+    when %r{application/vnd\.openxmlformats-officedocument\.wordprocessingml\.document}
+      "Microsoft Word Document (OpenXML)"
+    when %r{application/vnd\.oasis\.opendocument\.text}
+      "OpenDocument Text"
+    when %r{application/rtf}
+      "Rich Text Document"
+    when %r{image/jpeg}
+      "JPEG Image"
+    when %r{image/png}
+      "PNG Image"
+    when %r{image/gif}
+      "GIF Image"
+    when %r{video/webm}
+      "Webm Video"
+    when %r{audio/mpeg}
+      "MP3 Audio"
+    when %r{audio/mp3}
+      "MP3 Audio"
+    when %r{audio/wav}
+      "WAV Audio"
+    else
+      content_type.split(";").first&.split("/")&.last&.titleize || "Unknown"
+    end
+  end
+
+  sig { params(post: Domain::Post).returns(T.nilable(String)) }
+  def gallery_file_size_for_post(post)
+    file = post.primary_file_for_view
+    return nil unless file.present?
+    return nil unless file.state_ok?
+    return nil unless file.log_entry_id.present?
+    file.log_entry&.response_size&.then { |size| number_to_human_size(size) }
+  end
+
+  sig { params(url: String).returns(T.nilable(String)) }
+  def icon_asset_for_url(url)
+    domain = extract_domain(url)
+    return nil unless domain
+    icon_path_for_domain(domain)
+  end
+
+  sig { params(category: Symbol).returns(String) }
+  def tailwind_tag_category_class(category)
+    case category
+    when :general
+      "bg-blue-300" # Light blue
+    when :artist
+      "bg-indigo-300" # Light indigo
+    when :copyright
+      "bg-purple-300" # Light purple
+    when :character
+      "bg-green-300" # Light green
+    when :species
+      "bg-teal-300" # Light teal
+    when :invalid
+      "bg-slate-300" # Medium gray
+    when :meta
+      "bg-amber-300" # Light amber
+    when :lore
+      "bg-cyan-300" # Light cyan
+    else
+      "bg-white" # White (default)
+    end
+  end
+
+  sig { returns(T::Array[Symbol]) }
+  def tag_category_order
+    TAG_CATEGORY_ORDER
+  end
+
+  sig { params(category: Symbol).returns(T.nilable(String)) }
+  def font_awesome_category_icon(category)
+    case category
+    when :artist
+      "fa-brush"
+    when :species
+      "fa-paw"
+    when :character
+      "fa-user"
+    when :copyright
+      "fa-copyright"
+    when :general
+      "fa-tag"
+    when :lore
+      "fa-book"
+    when :meta
+      "fa-info"
+    when :invalid
+      "fa-ban"
+    end
+  end
+
+  class LinkForSource < T::ImmutableStruct
+    include T::Struct::ActsAsComparable
+
+    const :model, ReduxApplicationRecord
+    const :title, String
+    const :model_path, String
+    const :icon_path, T.nilable(String)
+  end
+
+  class SourceResult < T::ImmutableStruct
+    include T::Struct::ActsAsComparable
+
+    const :model, ReduxApplicationRecord
+    const :title, String
+  end
+
+  class SourceMatcher < T::ImmutableStruct
+    extend T::Generic
+    include T::Struct::ActsAsComparable
+
+    const :hosts, T::Array[String]
+    const :patterns, T::Array[Regexp]
+    const :find_proc,
+          T
+            .proc
+            .params(helper: Domain::PostsHelper, match: MatchData, url: String)
+            .returns(T.nilable(SourceResult))
+  end
+
+  FA_HOSTS = %w[*.furaffinity.net furaffinity.net]
+  FA_CDN_HOSTS = %w[d.furaffinity.net *.facdn.net facdn.net]
+  IB_HOSTS = %w[*.inkbunny.net inkbunny.net]
+  IB_CDN_HOSTS = %w[*.ib.metapix.net ib.metapix.net]
+  E621_HOSTS = %w[www.e621.net e621.net]
+
+  URL_SUFFIX_QUERY = T.let(<<-SQL.strip.chomp.freeze, String)
+    lower(json_attributes->>'url_str') = lower(?)
+  SQL
+
+  MATCHERS =
+    T.let(
+      [
+        # Furaffinity posts
+        SourceMatcher.new(
+          hosts: FA_HOSTS,
+          patterns: [
+            %r{/view/(\d+)/?},
+            %r{/full/(\d+)/?},
+            %r{/controls/submissions/changeinfo/(\d+)/?},
+          ],
+          find_proc: ->(helper, match, _) do
+            if post = Domain::Post::FaPost.find_by(fa_id: match[1])
+              SourceResult.new(
+                model: post,
+                title: helper.title_for_post_model(post),
+              )
+            end
+          end,
+        ),
+        # Furaffinity posts via direct file URL
+        SourceMatcher.new(
+          hosts: FA_CDN_HOSTS,
+          patterns: [/.+/],
+          find_proc: ->(helper, _, url) do
+            url = Addressable::URI.parse(url)
+
+            post_file =
+              Domain::PostFile.where(
+                "lower(json_attributes->>'url_str') IN (?, ?, ?, ?, ?, ?)",
+                "d.furaffinity.net#{url.host}/#{url.path}",
+                "//d.furaffinity.net#{url.host}/#{url.path}",
+                "https://d.furaffinity.net#{url.host}/#{url.path}",
+                "d.facdn.net#{url.host}/#{url.path}",
+                "//d.facdn.net#{url.host}/#{url.path}",
+                "https://d.facdn.net#{url.host}/#{url.path}",
+              ).first
+
+            if post_file && (post = post_file.post)
+              title =
+                T.bind(self, Domain::PostsHelper).title_for_post_model(post)
+              SourceResult.new(model: post, title:)
+            end
+          end,
+        ),
+        # Furaffinity users
+        SourceMatcher.new(
+          hosts: FA_HOSTS,
+          patterns: [%r{/user/([^/]+)/?}],
+          find_proc: ->(helper, match, _) do
+            if user = Domain::User::FaUser.find_by(url_name: match[1])
+              SourceResult.new(
+                model: user,
+                title: user.name_for_view || "unknown",
+              )
+            end
+          end,
+        ),
+        # Inkbunny posts
+        SourceMatcher.new(
+          hosts: IB_HOSTS,
+          patterns: [%r{/s/(\d+)/?}, %r{/submissionview\.php\?id=(\d+)/?}],
+          find_proc: ->(helper, match, _) do
+            if post = Domain::Post::InkbunnyPost.find_by(ib_id: match[1])
+              title = helper.title_for_post_model(post)
+              SourceResult.new(model: post, title:)
+            end
+          end,
+        ),
+        # Inkbunny posts via direct file URL
+        SourceMatcher.new(
+          hosts: IB_CDN_HOSTS,
+          patterns: [//],
+          find_proc: ->(helper, _, url) do
+            url = Addressable::URI.parse(url)
+            if post_file =
+                 Domain::PostFile.where(
+                   "#{URL_SUFFIX_QUERY}",
+                   "ib.metapix.net#{url.path}",
+                 ).first
+              if post = post_file.post
+                title = helper.title_for_post_model(post)
+                SourceResult.new(model: post, title:)
+              end
+            end
+          end,
+        ),
+        # Inkbunny users
+        SourceMatcher.new(
+          hosts: IB_HOSTS,
+          patterns: [%r{/(\w+)/?$}],
+          find_proc: ->(_, match, _) do
+            if user =
+                 Domain::User::InkbunnyUser.where(
+                   "lower(json_attributes->>'name') = lower(?)",
+                   match[1],
+                 ).first
+              SourceResult.new(
+                model: user,
+                title: user.name_for_view || "unknown",
+              )
+            end
+          end,
+        ),
+        # E621 posts
+        SourceMatcher.new(
+          hosts: E621_HOSTS,
+          patterns: [%r{/posts/(\d+)/?}],
+          find_proc: ->(helper, match, _) do
+            if post = Domain::Post::E621Post.find_by(e621_id: match[1])
+              SourceResult.new(
+                model: post,
+                title: helper.title_for_post_model(post),
+              )
+            end
+          end,
+        ),
+        # E621 users
+        SourceMatcher.new(
+          hosts: E621_HOSTS,
+          patterns: [%r{/users/(\d+)/?}],
+          find_proc: ->(helper, match, _) do
+            if user = Domain::User::E621User.find_by(e621_id: match[1])
+              SourceResult.new(
+                model: user,
+                title: user.name_for_view || "unknown",
+              )
+            end
+          end,
+        ),
+      ],
+      T::Array[SourceMatcher],
+    )
+
+  sig { params(source: String).returns(T.nilable(LinkForSource)) }
+  def link_for_source(source)
+    return nil if source.blank?
+
+    # normalize the source to a lowercase string with a protocol
+    source.downcase!
+    source = "https://" + source unless source.include?("://")
+    begin
+      uri = URI.parse(source)
+    rescue StandardError
+      return nil
+    end
+    uri_host = uri.host
+    return nil if uri_host.blank?
+
+    for matcher in MATCHERS
+      if matcher.hosts.any? { |host|
+           File.fnmatch?(host, uri_host, File::FNM_PATHNAME)
+         }
+        for pattern in matcher.patterns
+          if (match = pattern.match(uri.to_s))
+            object = matcher.find_proc.call(self, match, uri.to_s)
+            return nil unless object
+            model = object.model
+
+            if model.is_a?(Domain::Post)
+              model_path =
+                Rails.application.routes.url_helpers.domain_post_path(model)
+            elsif model.is_a?(Domain::User)
+              model_path =
+                Rails.application.routes.url_helpers.domain_user_path(model)
+              icon_path =
+                domain_user_avatar_img_src_path(
+                  model.avatar,
+                  thumb: "64-avatar",
+                )
+            else
+              model_path = "#"
+            end
+
+            return(
+              LinkForSource.new(
+                model:,
+                title: object.title,
+                model_path:,
+                icon_path:,
+              )
+            )
+          end
+        end
+      end
+    end
+    nil
+  end
+
+  sig { params(post: Domain::Post::FaPost).returns(T::Array[String]) }
+  def keywords_for_fa_post(post)
+    post.keywords.map(&:strip).reject(&:blank?).compact
+  end
+
+  private
+
+  sig { params(url: String).returns(T.nilable(String)) }
+  def extract_domain(url)
+    URI.parse(url).host
+  rescue URI::InvalidURIError
+    nil
+  end
+
+  private
+
+  TAG_CATEGORY_ORDER =
+    T.let(
+      %i[artist copyright character species general meta lore invalid].freeze,
+      T::Array[Symbol],
+    )
+end

app/helpers/domain/users_helper.rb

@@ -0,0 +1,256 @@
+# typed: strict
+module Domain::UsersHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  include Pundit::Authorization
+  abstract!
+
+  sig { params(user: Domain::User).returns(T.nilable(String)) }
+  def user_avatar_path_for_view(user)
+    if avatar = user.avatar
+      domain_user_avatar_img_src_path(avatar, thumb: "32-avatar")
+    end
+  end
+
+  sig { params(user: Domain::User).returns(T.nilable(String)) }
+  def user_name_for_view(user)
+    if avatar = user.avatar
+      domain_user_avatar_img_src_path(avatar, thumb: "32-avatar")
+    end
+  end
+
+  sig { params(user: Domain::User).returns(T.nilable(String)) }
+  def domain_user_registered_at_string_for_view(user)
+    ts = domain_user_registered_at_ts_for_view(user)
+    ts ? time_ago_in_words(ts) : nil
+  end
+
+  sig do
+    params(user: Domain::User).returns(T.nilable(ActiveSupport::TimeWithZone))
+  end
+  def domain_user_registered_at_ts_for_view(user)
+    case user
+    when Domain::User::FaUser, Domain::User::E621User
+      user.registered_at
+    else
+      nil
+    end
+  end
+
+  sig do
+    params(
+      avatar: T.nilable(Domain::UserAvatar),
+      thumb: T.nilable(String),
+    ).returns(String)
+  end
+  def domain_user_avatar_img_src_path(avatar, thumb: nil)
+    cache_key = ["domain_user_avatar_img_src_path", avatar&.id, thumb]
+    Rails
+      .cache
+      .fetch(cache_key, expires_in: 1.day) do
+        if (sha256 = avatar&.log_entry&.response_sha256)
+          Rails.application.routes.url_helpers.blob_path(
+            HexUtil.bin2hex(sha256),
+            format: "jpg",
+            thumb: thumb,
+          )
+        elsif avatar && avatar.state_file_404? &&
+              (sha256 = avatar.last_log_entry&.response_sha256)
+          Rails.application.routes.url_helpers.blob_path(
+            HexUtil.bin2hex(sha256),
+            format: "jpg",
+            thumb: thumb,
+          )
+        else
+          # default / 'not found' avatar image
+          # "/blobs/9080fd4e7e23920eb2dccfe2d86903fc3e748eebb2e5aa8c657bbf6f3d941cdc/contents.jpg"
+          asset_path("user-circle.svg")
+        end
+      end
+  end
+  sig do
+    params(
+      avatar: T.nilable(Domain::UserAvatar),
+      thumb: T.nilable(String),
+    ).returns(String)
+  end
+  def domain_user_avatar_img_tag(avatar, thumb: nil)
+    if (sha256 = avatar&.log_entry&.response_sha256)
+      image_tag(
+        blob_path(HexUtil.bin2hex(sha256), format: "jpg", thumb: thumb),
+        class: "inline-block h-4 w-4 flex-shrink-0 rounded-sm object-cover",
+        alt: avatar&.user&.name_for_view || "user avatar",
+      )
+    else
+      raw <<~SVG
+      <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="w-4 h-4">
+        <path stroke-linecap="round" stroke-linejoin="round" d="M15.75 6a3.75 3.75 0 11-7.5 0 3.75 3.75 0 017.5 0zM4.501 20.118a7.5 7.5 0 0114.998 0A17.933 17.933 0 0112 21.75c-2.676 0-5.216-.584-7.499-1.632z" />
+      </svg>
+      SVG
+    end
+  end
+
+  sig { params(user: Domain::User).returns(String) }
+  def site_name_for_user(user)
+    case user
+    when Domain::User::E621User
+      "E621"
+    when Domain::User::FaUser
+      "Furaffinity"
+    when Domain::User::InkbunnyUser
+      "Inkbunny"
+    else
+      Kernel.raise "Unknown user type: #{user.class}"
+    end
+  end
+
+  sig { params(user: Domain::User).returns(String) }
+  def site_icon_path_for_user(user)
+    case user
+    when Domain::User::E621User
+      asset_path("domain-icons/e621.png")
+    when Domain::User::FaUser
+      asset_path("domain-icons/fa.png")
+    when Domain::User::InkbunnyUser
+      asset_path("domain-icons/inkbunny.png")
+    else
+      Kernel.raise "Unknown user type: #{user.class}"
+    end
+  end
+
+  sig { params(user: Domain::User).returns(String) }
+  def domain_user_path(user)
+    "#{domain_users_path}/#{user.to_param}"
+  end
+
+  sig { returns(String) }
+  def domain_users_path
+    "/users"
+  end
+
+  sig { params(user: Domain::User).returns(String) }
+  def domain_user_posts_path(user)
+    "#{domain_user_path(user)}/posts"
+  end
+
+  sig { params(user: Domain::User).returns(String) }
+  def domain_user_favorites_path(user)
+    "#{domain_user_path(user)}/favorites"
+  end
+
+  sig { params(user: Domain::User).returns(String) }
+  def domain_user_followed_by_path(user)
+    "#{domain_user_path(user)}/followed_by"
+  end
+
+  sig { params(user: Domain::User).returns(String) }
+  def domain_user_following_path(user)
+    "#{domain_user_path(user)}/following"
+  end
+
+  class StatRow < T::ImmutableStruct
+    include T::Struct::ActsAsComparable
+
+    const :name, String
+    const :value,
+          T.nilable(
+            T.any(String, Integer, HasTimestampsWithDueAt::TimestampScanInfo),
+          )
+    const :link_to, T.nilable(String)
+    const :fa_icon_class, T.nilable(String)
+    const :hover_title, T.nilable(String)
+  end
+
+  sig { params(user: Domain::User).returns(T::Array[StatRow]) }
+  def stat_rows_for_user(user)
+    rows = T.let([], T::Array[StatRow])
+    if user.has_faved_posts?
+      rows << StatRow.new(name: "Favorites", value: user.user_post_favs.count)
+    end
+    if user.has_followed_by_users?
+      can_view_link = policy(user).followed_by?
+      rows << StatRow.new(
+        name: "Followed by",
+        value: user.user_user_follows_to.count,
+        link_to: can_view_link ? domain_user_followed_by_path(user) : nil,
+      )
+    end
+    if user.has_followed_users?
+      can_view_link = policy(user).following?
+      rows << StatRow.new(
+        name: "Following",
+        value: user.user_user_follows_from.count,
+        link_to: can_view_link ? domain_user_following_path(user) : nil,
+      )
+    end
+
+    can_view_timestamps = policy(user).view_page_scanned_at_timestamps?
+    can_view_log_entries = policy(user).view_log_entries?
+    icon_for =
+      Kernel.proc do |due_for_scan|
+        due_for_scan ? "fa-hourglass-half" : "fa-check"
+      end
+
+    if user.is_a?(Domain::User::FaUser) && can_view_timestamps
+      if can_view_log_entries && hle = user.guess_last_user_page_log_entry
+        rows << StatRow.new(
+          name: "Page scanned",
+          value: user.page_scan,
+          link_to: log_entry_path(hle),
+          fa_icon_class: icon_for.call(user.page_scan.due?),
+          hover_title: user.page_scan.interval.inspect,
+        )
+      else
+        rows << StatRow.new(
+          name: "Page",
+          value: user.page_scan,
+          fa_icon_class: icon_for.call(user.page_scan.due?),
+          hover_title: user.page_scan.interval.inspect,
+        )
+      end
+
+      rows << StatRow.new(
+        name: "Favs",
+        value: user.favs_scan,
+        fa_icon_class: icon_for.call(user.favs_scan.due?),
+        hover_title: user.favs_scan.interval.inspect,
+      )
+      rows << StatRow.new(
+        name: "Followed by",
+        value: user.followed_by_scan,
+        fa_icon_class: icon_for.call(user.followed_by_scan.due?),
+        hover_title: user.followed_by_scan.interval.inspect,
+      )
+      rows << StatRow.new(
+        name: "Following",
+        value: user.follows_scan,
+        fa_icon_class: icon_for.call(user.follows_scan.due?),
+        hover_title: user.follows_scan.interval.inspect,
+      )
+      rows << StatRow.new(
+        name: "Gallery",
+        value: user.gallery_scan,
+        fa_icon_class: icon_for.call(user.gallery_scan.due?),
+        hover_title: user.gallery_scan.interval.inspect,
+      )
+    elsif user.is_a?(Domain::User::E621User) && can_view_timestamps
+      if user.favs_are_hidden
+        rows << StatRow.new(name: "Favorites hidden", value: "yes")
+      else
+        rows << StatRow.new(
+          name: "Server favorites",
+          value: user.num_other_favs_cached,
+        )
+      end
+      rows << StatRow.new(
+        name: "Favorites",
+        value: user.favs_scan,
+        fa_icon_class: icon_for.call(user.favs_scan.due?),
+        hover_title: user.favs_scan.interval.inspect,
+      )
+    end
+
+    rows
+  end
+end

app/helpers/domain/visual_search_helper.rb

@@ -0,0 +1,63 @@
+# typed: true
+module Domain
+  module VisualSearchHelper
+    # Calculate the similarity percentage between two fingerprint hash values
+    # @param hash_value [String] The hash value to compare
+    # @param reference_hash_value [String] The reference hash value to compare against
+    # @return [Float] The similarity percentage between 0 and 100
+    def calculate_similarity_percentage(hash_value, reference_hash_value)
+      # Calculate hamming distance between the two hash values
+      distance =
+        DHashVips::IDHash.distance(
+          hash_value.to_i(2),
+          reference_hash_value.to_i(2),
+        )
+
+      max_distance = 256
+      # # Calculate similarity percentage based on distance
+      ((max_distance - distance) / max_distance.to_f * 100).round(1).clamp(
+        0,
+        100,
+      )
+    end
+
+    # Determine the background color class based on similarity percentage
+    # @param similarity_percentage [Float] The similarity percentage between 0 and 100
+    # @return [String] The Tailwind CSS background color class
+    def match_badge_bg_color(similarity_percentage)
+      case similarity_percentage
+      when 90..100
+        "bg-green-600"
+      when 70..89
+        "bg-blue-600"
+      when 50..69
+        "bg-amber-500"
+      else
+        "bg-slate-700"
+      end
+    end
+
+    # Determine the text color class based on similarity percentage
+    # @param similarity_percentage [Float] The similarity percentage between 0 and 100
+    # @return [String] The Tailwind CSS text color class
+    def match_text_color(similarity_percentage)
+      case similarity_percentage
+      when 90..100
+        "text-green-700"
+      when 70..89
+        "text-blue-700"
+      when 50..69
+        "text-amber-700"
+      else
+        "text-slate-700"
+      end
+    end
+
+    # Get the CSS classes for the match percentage badge
+    # @param similarity_percentage [Float] The similarity percentage between 0 and 100
+    # @return [String] The complete CSS classes for the match percentage badge
+    def match_badge_classes(similarity_percentage)
+      "#{match_badge_bg_color(similarity_percentage)} text-white font-semibold text-xs rounded-full px-3 py-1 shadow-md"
+    end
+  end
+end

app/helpers/domain_source_helper.rb

@@ -0,0 +1,28 @@
+# typed: strict
+module DomainSourceHelper
+  extend T::Sig
+
+  sig { returns(T::Hash[String, String]) }
+  def self.source_name_to_class_name
+    {
+      "furaffinity" => "Domain::Post::FaPost",
+      "e621" => "Domain::Post::E621Post",
+      "inkbunny" => "Domain::Post::InkbunnyPost",
+    }
+  end
+
+  sig { returns(T::Array[String]) }
+  def self.all_source_names
+    source_name_to_class_name.keys
+  end
+
+  sig { params(list: T::Array[String]).returns(T::Array[String]) }
+  def self.source_names_to_class_names(list)
+    list.map { |source| source_name_to_class_name[source] }.compact
+  end
+
+  sig { params(list: T::Array[String]).returns(T::Boolean) }
+  def self.has_all_sources?(list)
+    list.sort == all_source_names.sort
+  end
+end

app/helpers/good_job_helper.rb

@@ -4,11 +4,14 @@
 module GoodJobHelper
   extend T::Sig
   extend T::Helpers
-  extend self
+  abstract!
 
   class AnsiSegment < T::Struct
+    include T::Struct::ActsAsComparable
+
     const :text, String
     const :class_names, T::Array[String]
+    const :url, T.nilable(String)
   end
 
   # ANSI escape code pattern
@@ -59,6 +62,7 @@ module GoodJobHelper
           AnsiSegment.new(
             text: segment.text[idx...idx + 36],
             class_names: ["log-uuid"],
+            url: "/jobs/jobs/#{segment.text[idx...idx + 36]}",
           ),
           AnsiSegment.new(
             text: segment.text[idx + 36..],
@@ -71,15 +75,87 @@ module GoodJobHelper
     end
   end
 
-  sig { params(job: GoodJob::Job).returns(T::Hash[String, T.untyped]) }
+  class JobArg < T::Struct
+    const :key, Symbol
+    const :value, T.untyped
+    const :inferred, T::Boolean
+  end
+
+  sig { params(job: GoodJob::Job).returns(T::Array[JobArg]) }
   def arguments_for_job(job)
     deserialized =
       T.cast(
         ActiveJob::Arguments.deserialize(job.serialized_params).to_h,
         T::Hash[String, T.untyped],
       )
-    args = deserialized["arguments"].first
-    args.sort_by { |key, _| key.to_s }.to_h
+    args_hash =
+      T.cast(deserialized["arguments"].first, T::Hash[Symbol, T.untyped])
+    args =
+      args_hash.map { |key, value| JobArg.new(key:, value:, inferred: false) }
+
+    if args_hash[:fa_id] && !args_hash[:post] &&
+         args << JobArg.new(
+           key: :post,
+           value:
+             Domain::Post::FaPost.find_by(fa_id: args_hash[:fa_id]) ||
+               "post not found",
+           inferred: true,
+         )
+    end
+
+    if args_hash[:url_name] && !args_hash[:user]
+      args << JobArg.new(
+        key: :user,
+        value:
+          Domain::User::FaUser.find_by(url_name: args_hash[:url_name]) ||
+            "user not found",
+        inferred: true,
+      )
+    end
+
+    if job_id = args_hash[:caused_by_job_id]
+      job = GoodJob::Job.find_by(id: job_id)
+      if job
+        args << JobArg.new(key: :caused_by_job, value: job, inferred: false)
+        args.delete_if { |arg| arg.key == :caused_by_job_id }
+      end
+    end
+
+    args.sort_by(&:key)
+  end
+
+  sig { params(state: String).returns(String) }
+  def post_file_state_badge_class(state)
+    case state
+    when "ok"
+      "bg-success"
+    when "terminal_error"
+      "bg-danger"
+    when "retryable_error"
+      "bg-warning text-dark"
+    when "pending"
+      "bg-info"
+    else
+      "bg-secondary"
+    end
+  end
+
+  sig { params(state: String).returns(String) }
+  def post_file_state_icon_class(state)
+    base =
+      case state
+      when "ok"
+        "fa-check"
+      when "terminal_error"
+        "fa-circle-xmark"
+      when "retryable_error"
+        "fa-triangle-exclamation"
+      when "pending"
+        "fa-clock"
+      else
+        "fa-question"
+      end
+    "fa-solid #{base}"
   end
 
   private

app/helpers/indexable_posts_helper.rb

@@ -1,22 +0,0 @@
-# typed: false
-module IndexablePostsHelper
-  def show_path(indexed_post)
-    case indexed_post.postable_type
-    when "Domain::Fa::Post"
-      # need to use the helper here because the postable is not loaded
-      Rails.application.routes.url_helpers.domain_fa_post_path(
-        indexed_post.postable,
-      )
-    when "Domain::E621::Post"
-      Rails.application.routes.url_helpers.domain_e621_post_path(
-        indexed_post.postable,
-      )
-    when "Domain::Inkbunny::Post"
-      Rails.application.routes.url_helpers.domain_inkbunny_post_path(
-        indexed_post.postable,
-      )
-    else
-      raise("Unsupported postable type: #{indexed_post.postable_type}")
-    end
-  end
-end

app/helpers/ip_address_helper.rb

@@ -0,0 +1,35 @@
+# typed: strict
+module IpAddressHelper
+  extend T::Sig
+
+  # Formats an IPAddr object to display properly with CIDR notation if it's a subnet
+  # @param ip_addr [IPAddr, nil] The IP address object to format or nil
+  # @return [String] A formatted string representation of the IP address
+  sig { params(ip_addr: T.nilable(IPAddr)).returns(String) }
+  def format_ip_address(ip_addr)
+    if ip_addr.nil?
+      ""
+    else
+      # For IPv4, check if the prefix is not 32 (full mask)
+      # For IPv6, check if the prefix is not 128 (full mask)
+      if (ip_addr.ipv4? && ip_addr.prefix < 32) ||
+           (ip_addr.ipv6? && ip_addr.prefix < 128)
+        # This is a CIDR range
+        "#{ip_addr.to_s}/#{ip_addr.prefix}"
+      else
+        # Single IP address
+        ip_addr.to_s
+      end
+    end
+  end
+
+  # Determines if the provided IP address is a CIDR range
+  # @param ip_addr [IPAddr, nil] The IP address to check or nil
+  # @return [Boolean] true if the address is a CIDR range, false otherwise
+  sig { params(ip_addr: T.nilable(IPAddr)).returns(T::Boolean) }
+  def cidr_range?(ip_addr)
+    return false if ip_addr.nil?
+
+    format_ip_address(ip_addr).include?("/")
+  end
+end

app/helpers/log_entries_helper.rb

@@ -1,54 +1,359 @@
-# typed: true
+# typed: strict
 module LogEntriesHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  abstract!
+
+  sig { params(content_type: String).returns(T::Boolean) }
   def is_send_data_content_type?(content_type)
     is_renderable_image_type?(content_type) ||
       is_renderable_video_type?(content_type) ||
+      is_renderable_audio_type?(content_type) ||
       is_flash_content_type?(content_type)
   end
 
+  sig { params(uri_path: String).returns(T::Array[[String, String]]) }
   def path_iterative_parts(uri_path)
     path_parts = uri_path.split("/")
+
     (1...path_parts.length).map do |i|
       [
-        path_parts[i],
-        path_parts[0..i].join("/") + (i == path_parts.length - 1 ? "" : "/"),
+        T.must(path_parts[i]),
+        T.must(path_parts[0..i]).join("/") +
+          (i == path_parts.length - 1 ? "" : "/"),
       ]
     end
   end
 
-  def ext_for_content_type(content_type)
-    case content_type
-    when "image/jpeg"
-      "jpeg"
-    when "image/jpg"
-      "jpg"
-    when "image/png"
-      "png"
-    when "image/gif"
+  sig { params(content_type: String).returns(T.nilable(String)) }
+  def thumbnail_extension_for_content_type(content_type)
+    return nil unless is_thumbable_content_type?(content_type)
+    extension = extension_for_content_type(content_type)
+    if extension == "gif"
       "gif"
-    when "video/webm"
+    else
+      "jpeg"
+    end
+  end
+
+  sig { params(content_type: String).returns(T.nilable(String)) }
+  def extension_for_content_type(content_type)
+    content_type = content_type.split(";")[0]
+    return nil unless content_type
+
+    extension = Rack::Mime::MIME_TYPES.invert[content_type]
+    return extension[1..] if extension
+
+    case content_type
+    when %r{image/jpeg}
+      "jpeg"
+    when %r{image/jpg}
+      "jpg"
+    when %r{image/png}
+      "png"
+    when %r{image/gif}
+      "gif"
+    when %r{video/webm}
       "webm"
+    when %r{audio/mpeg}
+      "mp3"
+    when %r{audio/mp3}
+      "mp3"
+    when %r{audio/wav}
+      "wav"
+    when %r{application/pdf}
+      "pdf"
+    when %r{application/rtf}
+      "rtf"
+    when %r{application/msword}
+      "doc"
+    when %r{application/vnd\.openxmlformats-officedocument\.wordprocessingml\.document}
+      "docx"
+    when %r{application/vnd\.oasis\.opendocument\.text}
+      "odt"
     else
       nil
     end
   end
 
+  sig { params(content_type: String).returns(T::Boolean) }
   def is_renderable_image_type?(content_type)
     %w[image/jpeg image/jpg image/png image/gif].any? do |ct|
       content_type.starts_with?(ct)
     end
   end
 
-  def is_thumbable_content_type?(content_type)
-    %w[video/webm].any? { |ct| content_type.starts_with?(ct) } ||
-      is_renderable_image_type?(content_type)
+  sig { params(content_type: String).returns(T::Boolean) }
+  def is_json_content_type?(content_type)
+    content_type.starts_with?("application/json")
   end
 
+  sig { params(content_type: String).returns(T::Boolean) }
+  def is_rich_text_content_type?(content_type)
+    %w[
+      application/pdf
+      application/rtf
+      application/msword
+      text/plain
+      application/vnd.oasis.opendocument.text
+      application/vnd.openxmlformats-officedocument.wordprocessingml.document
+    ].any? { |ct| content_type.starts_with?(ct) }
+  end
+
+  sig { params(content_type: String).returns(T::Boolean) }
   def is_renderable_video_type?(content_type)
     %w[video/mp4 video/webm].any? { |ct| content_type.starts_with?(ct) }
   end
 
+  sig { params(content_type: String).returns(T::Boolean) }
+  def is_renderable_audio_type?(content_type)
+    %w[audio/mpeg audio/mp3 audio/wav audio/ogg].any? do |ct|
+      content_type.starts_with?(ct)
+    end
+  end
+
+  sig { params(content_type: String).returns(T::Boolean) }
   def is_flash_content_type?(content_type)
-    content_type =~ %r{application/x-shockwave-flash}
+    content_type.match? %r{application/x-shockwave-flash}
+  end
+
+  sig { params(content_type: String).returns(T::Boolean) }
+  def is_thumbable_content_type?(content_type)
+    is_renderable_video_type?(content_type) ||
+      is_renderable_image_type?(content_type)
+  end
+
+  sig { params(rich_text_body: String).returns(T.nilable(String)) }
+  def convert_with_pdftohtml(rich_text_body)
+    stdin, stdout, stderr, wait_thr =
+      Open3.popen3(
+        "pdftohtml",
+        "-i", # ignore images
+        "-s", # generate single HTML page
+        "-nodrm", # ignore drm
+        "-enc",
+        "UTF-8",
+        "-stdout",
+        "-", # read from stdin
+        "-", # write to stdout (???)
+      )
+    stdin.binmode
+    stdin.write(rich_text_body)
+    stdin.close
+
+    stdout_str = stdout.read
+    exit_status = T.cast(wait_thr.value, Process::Status)
+    return nil unless exit_status.success?
+
+    # For PDFs, handle both HTML entities and Unicode NBSPs
+    # First replace the actual unicode NBSP character (U+00A0)
+    # stdout_str.gsub!(/[[:space:]]+/, " ")
+    stdout_str.gsub!(/\u00A0/, " ")
+    stdout_str.gsub!(/ /i, " ")
+    stdout_str.gsub!(/ /, " ")
+    stdout_str.gsub!(/ /i, " ")
+
+    stdout_str
+  ensure
+    stdin&.close
+    stdout&.close
+    stderr&.close
+  end
+
+  sig { params(rich_text_body: String).returns(T.nilable(String)) }
+  def convert_with_abiword(rich_text_body)
+    stdin, stdout, stderr, wait_thr =
+      Open3.popen3(
+        "abiword",
+        "--display=0",
+        "--to=html",
+        "--to-name=fd://1",
+        "fd://0",
+      )
+    stdin.binmode
+    stdin.write(rich_text_body)
+    stdin.close
+
+    stdout_str = stdout.read
+    exit_status = T.cast(wait_thr.value, Process::Status)
+    return nil unless exit_status.success?
+
+    stdout_str.gsub!(/Abiword HTML Document/, "")
+    stdout_str = try_convert_bbcode_to_html(stdout_str)
+    stdout_str.gsub!(%r{<br\s*/>}, "")
+
+    stdout_str
+  ensure
+    stdin&.close
+    stdout&.close
+    stderr&.close
+  end
+
+  sig { params(rich_text_body: String).returns(T.nilable(String)) }
+  def convert_with_libreoffice(rich_text_body)
+    tempfile = Tempfile.new(%w[test .doc], binmode: true)
+    tempfile.write(rich_text_body)
+    tempfile.flush
+
+    stdin, stdout, stderr, wait_thr =
+      Open3.popen3(
+        "libreoffice",
+        "--display",
+        "0",
+        "--headless",
+        "--convert-to",
+        "html",
+        T.must(tempfile.path),
+        "--cat",
+      )
+    stdin.binmode
+    stdin.write(rich_text_body)
+    stdin.close
+
+    stdout_str = stdout.read
+    exit_status = T.cast(wait_thr.value, Process::Status)
+    return nil unless exit_status.success?
+
+    stdout_str
+  ensure
+    stdin&.close
+    stdout&.close
+    stderr&.close
+    tempfile&.close
+  end
+
+  sig { params(rich_text_body: String).returns(String) }
+  def try_convert_bbcode_to_html(rich_text_body)
+    rich_text_body.bbcode_to_html(false)
+  rescue StandardError
+    rich_text_body
+  end
+
+  sig { params(log_entry: HttpLogEntry).returns(T.nilable(String)) }
+  def render_rich_text_content(log_entry)
+    content_type = log_entry.content_type
+    rich_text_body = log_entry.response_bytes
+    return nil if rich_text_body.blank? || content_type.blank?
+
+    is_plain_text = content_type.starts_with?("text/plain")
+
+    if is_plain_text
+      # rich_text_body.gsub!(/(\r\n|\n|\r)+/, "<br />")
+      rich_text_body = rich_text_body.force_encoding("UTF-8")
+      document_html = try_convert_bbcode_to_html(rich_text_body)
+    elsif content_type.starts_with?("application/pdf")
+      document_html = convert_with_pdftohtml(rich_text_body)
+    else
+      document_html =
+        convert_with_abiword(rich_text_body) ||
+          convert_with_libreoffice(rich_text_body)
+    end
+
+    return nil if document_html.blank?
+    sanitize_rich_text_document_html(document_html, is_plain_text)
+  end
+
+  sig do
+    params(document_html: String, is_plain_text: T::Boolean).returns(String)
+  end
+  def sanitize_rich_text_document_html(document_html, is_plain_text)
+    quote_transformer =
+      Kernel.lambda do |env|
+        node = env[:node]
+        if node["class"]
+          classes = node["class"].split(" ").map(&:strip).compact
+          node.remove_attribute("class")
+          if classes.include?("quote")
+            # write div to be a blockquote
+            node.name = "blockquote"
+          end
+        end
+        node
+      end
+
+    clean_plain_text_node =
+      Kernel.lambda do |node|
+        node = T.cast(node, Nokogiri::XML::Node)
+        if node.text?
+          node_text = node.text.strip
+          if node_text.empty?
+            node.unlink
+          else
+            # collect all the subsequent nodes that are not a block element
+            # and replace the current node with a <p> containing the text
+            # and the collected nodes
+            current_node = node
+            inline_elements = []
+            while (next_sibling = current_node.next_sibling) &&
+                    (next_sibling.name != "br") && (next_sibling.name != "p")
+              inline_elements << next_sibling
+              current_node = next_sibling
+            end
+
+            node_html = [node_text]
+            inline_elements.each do |inline_element|
+              inline_element.unlink
+              node_html << inline_element.to_html
+            end
+            node.replace("<p>#{node_html.join(" ")}</p>")
+          end
+        end
+      end
+
+    plain_text_transformer =
+      Kernel.lambda do |env|
+        # within a div, wrap bare text nodes in a <p>
+        node = T.cast(env[:node], Nokogiri::XML::Node)
+        node_name = T.cast(env[:node_name], String)
+        if node_name == "div"
+          current_child = T.unsafe(node.children.first)
+          while current_child.present?
+            clean_plain_text_node.call(current_child)
+            current_child = current_child.next_sibling
+          end
+        elsif node.text? && node.parent&.name == "#document-fragment"
+          clean_plain_text_node.call(node)
+        end
+        { node_allowlist: [node] }
+      end
+
+    # remove_empty_newline_transformer =
+    #   Kernel.lambda do |env|
+    #     node = env[:node]
+    #     node.unlink if node.text? && node.text.strip.chomp.blank?
+    #   end
+
+    # remove_multiple_br_transformer =
+    #   Kernel.lambda do |env|
+    #     node = env[:node]
+    #     if node.name == "br"
+    #       node.unlink if node.previous_sibling&.name == "br"
+    #     end
+    #   end
+
+    sanitizer =
+      Sanitize.new(
+        elements: %w[span div p i b strong em blockquote br],
+        attributes: {
+          all: %w[style class],
+        },
+        css: {
+          properties: %w[color text-align margin-bottom],
+        },
+        transformers: [
+          quote_transformer,
+          # is_plain_text ? remove_empty_newline_transformer : nil,
+          is_plain_text ? plain_text_transformer : nil,
+          # is_plain_text ? remove_multiple_br_transformer : nil,
+        ].compact,
+      )
+
+    fragment = sanitizer.fragment(document_html).strip
+    if is_plain_text
+      fragment.gsub!("<br>", "")
+      fragment.gsub!("<br />", "")
+    end
+    raw fragment
   end
 end

app/helpers/paths_helper.rb

@@ -0,0 +1,66 @@
+# typed: strict
+# frozen_string_literal: true
+module PathsHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+  abstract!
+
+  # sig do
+  #   params(post: Domain::Post, params: T::Hash[Symbol, T.untyped]).returns(
+  #     String,
+  #   )
+  # end
+  # def domain_post_path(post, params = {})
+  #   to_path("#{domain_posts_path}/#{post.to_param}", params)
+  # end
+
+  # sig do
+  #   params(post: Domain::Post, params: T::Hash[Symbol, T.untyped]).returns(
+  #     String,
+  #   )
+  # end
+  # def domain_post_faved_by_path(post, params = {})
+  #   to_path("#{domain_post_path(post)}/faved_by", params)
+  # end
+
+  # sig { params(params: T::Hash[Symbol, T.untyped]).returns(String) }
+  # def domain_posts_path(params = {})
+  #   to_path("/posts", params)
+  # end
+
+  # sig do
+  #   params(
+  #     post_group: Domain::PostGroup,
+  #     params: T::Hash[Symbol, T.untyped],
+  #   ).returns(String)
+  # end
+  # def domain_post_group_posts_path(post_group, params = {})
+  #   to_path("#{domain_post_group_path(post_group)}/posts", params)
+  # end
+
+  # sig do
+  #   params(
+  #     post_group: Domain::PostGroup,
+  #     params: T::Hash[Symbol, T.untyped],
+  #   ).returns(String)
+  # end
+  # def domain_post_group_path(post_group, params = {})
+  #   to_path("#{domain_post_groups_path}/#{post_group.to_param}", params)
+  # end
+
+  # sig { params(params: T::Hash[Symbol, T.untyped]).returns(String) }
+  # def domain_post_groups_path(params = {})
+  #   to_path("/pools", params)
+  # end
+
+  private
+
+  sig do
+    params(path: String, params: T::Hash[Symbol, T.untyped]).returns(String)
+  end
+  def to_path(path, params = {})
+    params_string = params.reject { |k, v| v.blank? }.to_query
+    "#{path}#{"?#{params_string}" if params_string.present?}"
+  end
+end

app/helpers/timestamp_helper.rb

@@ -0,0 +1,13 @@
+# typed: strict
+module TimestampHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+
+  sig do
+    params(timestamp: T.nilable(ActiveSupport::TimeWithZone)).returns(String)
+  end
+  def time_ago_or_never(timestamp)
+    timestamp ? time_ago_in_words(timestamp) + " ago" : "never"
+  end
+end

app/helpers/ui_helper.rb

@@ -0,0 +1,41 @@
+# typed: strict
+# frozen_string_literal: true
+module UiHelper
+  extend T::Sig
+  extend T::Helpers
+  include HelpersInterface
+
+  sig do
+    params(
+      title: String,
+      collapsible: T::Boolean,
+      initially_collapsed: T::Boolean,
+      font_size_adjustable: T::Boolean,
+      kwargs: T.untyped,
+      block: T.proc.void,
+    ).returns(String)
+  end
+  def sky_section_tag(
+    title,
+    collapsible: false,
+    initially_collapsed: false,
+    font_size_adjustable: false,
+    **kwargs,
+    &block
+  )
+    content = capture(&block)
+    kwargs[:class] ||= "bg-slate-100 p-4"
+
+    render(
+      partial: "shared/section_controls/sky_section",
+      locals: {
+        title: title,
+        content: content,
+        collapsible: collapsible,
+        initially_collapsed: initially_collapsed,
+        font_size_adjustable: font_size_adjustable,
+        container_class: kwargs[:class],
+      },
+    )
+  end
+end

app/javascript/bundles/Main/components/HoverPreview.tsx

@@ -0,0 +1,62 @@
+import * as React from 'react';
+import { ReactNode } from 'react';
+import { createPortal } from 'react-dom';
+import {
+  useHoverPreview,
+  getPreviewContainerClassName,
+} from '../utils/hoverPreviewUtils';
+
+export interface HoverPreviewProps {
+  children: ReactNode;
+  previewContent: ReactNode;
+  previewClassName?: string;
+  maxWidth?: string;
+  maxHeight?: string;
+}
+
+export const HoverPreview: React.FC<HoverPreviewProps> = ({
+  children,
+  previewContent,
+  previewClassName = '',
+  maxWidth = '300px',
+  maxHeight = '500px',
+}) => {
+  // Use shared hover preview hook
+  const {
+    containerRef,
+    previewRef,
+    portalContainer,
+    showPreview,
+    previewStyle,
+    handleMouseEnter,
+    handleMouseLeave,
+  } = useHoverPreview();
+
+  return (
+    <span
+      ref={containerRef}
+      onMouseEnter={handleMouseEnter}
+      onMouseLeave={handleMouseLeave}
+    >
+      {children}
+
+      {showPreview &&
+        createPortal(
+          <div
+            ref={previewRef}
+            className={`${getPreviewContainerClassName(maxWidth, maxHeight)} ${previewClassName}`}
+            style={{
+              ...previewStyle,
+            }}
+            onMouseEnter={handleMouseEnter}
+            onMouseLeave={handleMouseLeave}
+          >
+            {previewContent}
+          </div>,
+          portalContainer,
+        )}
+    </span>
+  );
+};
+
+export default HoverPreview;

app/javascript/bundles/Main/components/ListItem.tsx

@@ -16,6 +16,7 @@ interface PropTypes {
   selected: boolean;
   href?: string;
   style: 'item' | 'info' | 'error';
+  domainIcon?: string;
 }
 
 export default function ListItem({
@@ -26,6 +27,7 @@ export default function ListItem({
   style,
   href,
   subtext,
+  domainIcon,
 }: PropTypes) {
   const iconClassName = ['ml-2'];
   const textClassName = [
@@ -55,7 +57,11 @@ export default function ListItem({
       <div className={textClassName.join(' ')}>
         <div className="inline-block w-8">
           {thumb && (
-            <img src={thumb} alt="thumbnail" className="inline w-full" />
+            <img
+              src={thumb}
+              alt="thumbnail"
+              className="inline w-full rounded-md"
+            />
           )}
         </div>
         <div className="inline-block flex-grow pl-1">{value}</div>
@@ -64,6 +70,9 @@ export default function ListItem({
             {subtext}
           </div>
         )}
+        {domainIcon && (
+          <img src={domainIcon} alt="domain icon" className="inline w-6 pl-1" />
+        )}
       </div>
     </a>
   );

app/javascript/bundles/Main/components/PostHoverPreview.tsx

@@ -0,0 +1,121 @@
+import * as React from 'react';
+import { HoverPreview } from './HoverPreview';
+import {
+  getHeaderFooterClassName,
+  getContentClassName,
+  getHeaderTextClassName,
+  getSecondaryTextClassName,
+  getBorderClassName,
+  getIconClassName,
+  getAvatarImageClassName,
+} from '../utils/hoverPreviewUtils';
+
+interface PostHoverPreviewProps {
+  children: React.ReactNode;
+  postTitle: string;
+  postThumbnailPath?: string;
+  postThumbnailAlt?: string;
+  postDomainIcon?: string;
+  creatorName?: string;
+  creatorAvatarPath?: string;
+}
+
+export const PostHoverPreview: React.FC<PostHoverPreviewProps> = ({
+  children,
+  postTitle,
+  postThumbnailPath,
+  postThumbnailAlt,
+  postDomainIcon,
+  creatorName,
+  creatorAvatarPath,
+}) => {
+  // Force eager loading of images
+  React.useEffect(() => {
+    // Preload post thumbnail
+    if (postThumbnailPath) {
+      const thumbnailImg = new Image();
+      thumbnailImg.src = postThumbnailPath;
+    }
+
+    // Preload creator avatar
+    if (creatorAvatarPath) {
+      const avatarImg = new Image();
+      avatarImg.src = creatorAvatarPath;
+    }
+  }, [postThumbnailPath, creatorAvatarPath]);
+
+  // Add extra classes for PostHoverPreview's header/footer
+  const postHeaderFooterClassName = `${getHeaderFooterClassName()} justify-between p-3`;
+
+  const previewContent = (
+    <>
+      {/* Header: Title */}
+      <div className={postHeaderFooterClassName}>
+        <span className={getHeaderTextClassName()} title={postTitle}>
+          {postTitle}
+        </span>
+      </div>
+
+      {/* Image Content */}
+      <div
+        className={`flex items-center justify-center ${getContentClassName()} p-2`}
+      >
+        {postThumbnailPath ? (
+          <div className="transition-transform hover:scale-[1.02]">
+            <img
+              src={postThumbnailPath}
+              alt={postTitle}
+              className={`max-h-[250px] max-w-full rounded-md ${getBorderClassName()} object-contain shadow-md`}
+              loading="eager"
+            />
+          </div>
+        ) : (
+          <span className={`block px-4 py-6 ${getSecondaryTextClassName()}`}>
+            {postThumbnailAlt || 'No file available'}
+          </span>
+        )}
+      </div>
+
+      {/* Footer: Domain icon & Creator */}
+      {creatorName && (
+        <div className={postHeaderFooterClassName}>
+          <span className="flex items-center gap-2 justify-self-end">
+            {creatorAvatarPath && (
+              <img
+                src={creatorAvatarPath}
+                alt={creatorName}
+                className={getAvatarImageClassName('small')}
+              />
+            )}
+            <span
+              className="truncate text-xs font-medium text-slate-700"
+              title={creatorName}
+            >
+              {creatorName}
+            </span>
+          </span>
+          {(postDomainIcon && (
+            <img
+              src={postDomainIcon}
+              alt={postTitle}
+              className={getIconClassName()}
+            />
+          )) || (
+            <span className="h-6 w-6 grow rounded-md shadow-sm ring-blue-200"></span>
+          )}
+        </div>
+      )}
+    </>
+  );
+
+  return (
+    <HoverPreview
+      children={children}
+      previewContent={previewContent}
+      maxWidth="300px"
+      maxHeight="500px"
+    />
+  );
+};
+
+export default PostHoverPreview;

app/javascript/bundles/Main/components/PostHoverPreviewWrapper.tsx

@@ -0,0 +1,59 @@
+import * as React from 'react';
+import PostHoverPreview from './PostHoverPreview';
+import {
+  anchorClassNamesForVisualStyle,
+  iconClassNamesForSize,
+} from '../utils/hoverPreviewUtils';
+
+interface PostHoverPreviewWrapperProps {
+  visualStyle: 'sky-link' | 'description-section-link';
+  linkText: string;
+  postTitle: string;
+  postPath: string;
+  postThumbnailPath: string;
+  postThumbnailAlt: string;
+  postDomainIcon: string;
+  creatorName?: string;
+  creatorAvatarPath?: string;
+}
+
+export const PostHoverPreviewWrapper: React.FC<
+  PostHoverPreviewWrapperProps
+> = ({
+  visualStyle,
+  linkText,
+  postTitle,
+  postPath,
+  postThumbnailPath,
+  postThumbnailAlt,
+  postDomainIcon,
+  creatorName,
+  creatorAvatarPath,
+}) => {
+  return (
+    <PostHoverPreview
+      postTitle={postTitle}
+      postThumbnailPath={postThumbnailPath}
+      postThumbnailAlt={postThumbnailAlt}
+      postDomainIcon={postDomainIcon}
+      creatorName={creatorName}
+      creatorAvatarPath={creatorAvatarPath}
+    >
+      <a
+        href={postPath}
+        className={anchorClassNamesForVisualStyle(visualStyle, true)}
+      >
+        {visualStyle === 'description-section-link' && (
+          <img
+            src={postDomainIcon}
+            alt={postTitle}
+            className={iconClassNamesForSize('small')}
+          />
+        )}
+        {linkText}
+      </a>
+    </PostHoverPreview>
+  );
+};
+
+export default PostHoverPreviewWrapper;

app/javascript/bundles/Main/components/UserHoverPreview.tsx

@@ -0,0 +1,140 @@
+import * as React from 'react';
+import { HoverPreview } from './HoverPreview';
+import {
+  getHeaderFooterClassName,
+  getContentClassName,
+  getHeaderTextClassName,
+  getLabelTextClassName,
+  getValueTextClassName,
+  getSecondaryTextClassName,
+  getAvatarContainerClassName,
+  getAvatarImageClassName,
+  getIconClassName,
+} from '../utils/hoverPreviewUtils';
+
+interface UserHoverPreviewProps {
+  children: React.ReactNode;
+  userName: string;
+  userAvatarPath: string;
+  userAvatarAlt: string;
+  userDomainIcon?: string;
+  userRegisteredAt?: string;
+  userNumPosts?: number;
+  userNumFollowedBy?: number;
+  userNumFollowed?: number;
+}
+
+export const UserHoverPreview: React.FC<UserHoverPreviewProps> = ({
+  children,
+  userName,
+  userAvatarPath,
+  userAvatarAlt,
+  userDomainIcon,
+  userRegisteredAt,
+  userNumPosts,
+  userNumFollowedBy,
+  userNumFollowed,
+}) => {
+  // Force eager loading of the avatar image
+  React.useEffect(() => {
+    if (userAvatarPath) {
+      const img = new Image();
+      img.src = userAvatarPath;
+    }
+  }, [userAvatarPath]);
+
+  const previewContent = (
+    <>
+      {/* Header: User Name and Domain Icon */}
+      <div className={`${getHeaderFooterClassName()} justify-between`}>
+        <span className={getHeaderTextClassName()} title={userName}>
+          {userName}
+        </span>
+        {userDomainIcon && (
+          <img
+            src={userDomainIcon}
+            alt="Domain"
+            className={getIconClassName()}
+          />
+        )}
+      </div>
+
+      {/* Profile Content */}
+      <div className={getContentClassName()}>
+        <div className="flex items-center gap-4">
+          {/* Avatar */}
+          <div className="flex-shrink-0">
+            {userAvatarPath ? (
+              <div className={getAvatarContainerClassName()}>
+                <img
+                  src={userAvatarPath}
+                  alt={userAvatarAlt}
+                  className={getAvatarImageClassName('medium')}
+                  loading="eager"
+                />
+              </div>
+            ) : (
+              <span
+                className={`block h-[80px] w-[80px] rounded-md border border-slate-300 bg-slate-200 p-2 text-xs italic ${getSecondaryTextClassName()}`}
+              >
+                No avatar
+              </span>
+            )}
+          </div>
+
+          {/* User Stats */}
+          <div className="flex-grow">
+            <div className="grid grid-cols-2 gap-3">
+              <div className="text-center">
+                <span className={getValueTextClassName()}>
+                  {userNumPosts ?? '-'}
+                </span>
+                <span className={getLabelTextClassName()}>Posts</span>
+              </div>
+
+              <div className="text-center">
+                <span className={getValueTextClassName()}>
+                  {userNumFollowedBy ?? '-'}
+                </span>
+                <span className={getLabelTextClassName()}>Followers</span>
+              </div>
+
+              <div className="text-center">
+                <span className={getValueTextClassName()}>
+                  {userNumFollowed ?? '-'}
+                </span>
+                <span className={getLabelTextClassName()}>Following</span>
+              </div>
+
+              <div className="text-center">
+                {userRegisteredAt && (
+                  <>
+                    <span
+                      className={`text-2xs block font-medium text-slate-800`}
+                    >
+                      Member since
+                    </span>
+                    <span className={getLabelTextClassName()}>
+                      {userRegisteredAt}
+                    </span>
+                  </>
+                )}
+              </div>
+            </div>
+          </div>
+        </div>
+      </div>
+    </>
+  );
+
+  return (
+    <HoverPreview
+      children={children}
+      previewContent={previewContent}
+      maxWidth="400px"
+      maxHeight="280px"
+    />
+  );
+};
+
+export default UserHoverPreview;

app/javascript/bundles/Main/components/UserHoverPreviewWrapper.tsx

@@ -0,0 +1,78 @@
+import * as React from 'react';
+import { UserHoverPreview } from './UserHoverPreview';
+import {
+  anchorClassNamesForVisualStyle,
+  iconClassNamesForSize,
+} from '../utils/hoverPreviewUtils';
+
+type VisualStyle = 'sky-link' | 'description-section-link';
+type IconSize = 'small' | 'large';
+
+interface UserHoverPreviewWrapperProps {
+  visualStyle: VisualStyle;
+  iconSize: IconSize;
+  linkText: string;
+  userId: string;
+  userName: string;
+  userPath: string;
+  userAvatarPath: string;
+  userAvatarAlt: string;
+  userDomainIcon?: string;
+  userSmallAvatarPath?: string;
+  userRegisteredAt?: string;
+  userNumPosts?: number;
+  userNumFollowedBy?: number;
+  userNumFollowed?: number;
+}
+
+export const UserHoverPreviewWrapper: React.FC<
+  UserHoverPreviewWrapperProps
+> = ({
+  visualStyle,
+  iconSize,
+  linkText,
+  userName,
+  userPath,
+  userAvatarPath,
+  userAvatarAlt,
+  userDomainIcon,
+  userSmallAvatarPath,
+  userRegisteredAt,
+  userNumPosts,
+  userNumFollowedBy,
+  userNumFollowed,
+}) => {
+  return (
+    <UserHoverPreview
+      userName={userName}
+      userAvatarPath={userAvatarPath}
+      userAvatarAlt={userAvatarAlt}
+      userDomainIcon={userDomainIcon}
+      userRegisteredAt={userRegisteredAt}
+      userNumPosts={userNumPosts}
+      userNumFollowedBy={userNumFollowedBy}
+      userNumFollowed={userNumFollowed}
+    >
+      <a
+        href={userPath}
+        className={anchorClassNamesForVisualStyle(
+          visualStyle,
+          !!userSmallAvatarPath,
+        )}
+      >
+        {userSmallAvatarPath ? (
+          <img
+            src={userSmallAvatarPath}
+            alt={userAvatarAlt}
+            className={iconClassNamesForSize(iconSize)}
+          />
+        ) : (
+          <i className={iconClassNamesForSize(iconSize)}></i>
+        )}
+        {linkText}
+      </a>
+    </UserHoverPreview>
+  );
+};
+
+export default UserHoverPreviewWrapper;

app/javascript/bundles/Main/components/UserSearchBar.tsx

@@ -34,10 +34,12 @@ interface PropTypes {
 interface User {
   id: number;
   name: string;
-  url_name: string;
   thumb?: string;
   show_path: string;
-  num_posts: number;
+  num_posts?: number;
+  distance: number;
+  matched_name: string;
+  domain_icon: string;
 }
 
 interface ServerResponse {
@@ -90,7 +92,7 @@ export default function UserSearchBar({ isServerRendered }: PropTypes) {
       async function sendRequest() {
         try {
           let req = await fetch(
-            `${CONFIG.HOST}/api/fa/search_user_names?name=${userName}`,
+            `${CONFIG.HOST}/users/search_by_name.json?name=${userName}`,
             {
               signal: controller.signal,
             },
@@ -205,18 +207,21 @@ export default function UserSearchBar({ isServerRendered }: PropTypes) {
           />
         ) : null}
         {visibility.items
-          ? state.userList.map(({ name, thumb, show_path, num_posts }, idx) => (
-              <ListItem
-                key={'name-' + name}
-                isLast={idx == state.userList.length - 1}
-                selected={idx == state.selectedIdx}
-                style="item"
-                value={name}
-                thumb={thumb}
-                href={show_path}
-                subtext={`${num_posts.toString()} posts`}
-              />
-            ))
+          ? state.userList.map(
+              ({ name, thumb, show_path, num_posts, domain_icon }, idx) => (
+                <ListItem
+                  key={'name-' + name}
+                  isLast={idx == state.userList.length - 1}
+                  selected={idx == state.selectedIdx}
+                  style="item"
+                  value={name}
+                  thumb={thumb}
+                  href={show_path}
+                  subtext={num_posts ? `${num_posts.toString()} posts` : ''}
+                  domainIcon={domain_icon}
+                />
+              ),
+            )
           : null}
       </div>
     );
@@ -289,7 +294,7 @@ export default function UserSearchBar({ isServerRendered }: PropTypes) {
           ]
             .filter(Boolean)
             .join(' ')}
-          placeholder="Search FurAffinity Users?!?"
+          placeholder="Search Users"
           defaultValue={state.userName}
           onChange={(e) => {
             setState((s) => ({ ...s, typingSettled: false }));

app/javascript/bundles/Main/utils/hoverPreviewUtils.ts

@@ -0,0 +1,235 @@
+import {
+  RefObject,
+  CSSProperties,
+  useState,
+  useRef,
+  useEffect,
+  useMemo,
+} from 'react';
+
+/**
+ * Utility functions for hover preview styling
+ */
+
+export type IconSize = 'large' | 'small';
+export function iconClassNamesForSize(size: IconSize) {
+  switch (size) {
+    case 'large':
+      return 'h-8 w-8 flex-shrink-0 rounded-md';
+    case 'small':
+      return 'h-5 w-5 flex-shrink-0 rounded-sm';
+  }
+}
+
+// Base preview container styling
+export const getPreviewContainerClassName = (
+  maxWidth: string,
+  maxHeight: string,
+) => {
+  // calculate if the page is mobile
+  const isMobile = window.innerWidth < 640;
+  return [
+    isMobile ? 'hidden' : '',
+    `max-w-[${maxWidth}] max-h-[${maxHeight}] rounded-lg`,
+    'border border-slate-400 bg-slate-100',
+    'divide-y divide-slate-300',
+    'shadow-lg shadow-slate-500/30',
+  ].join(' ');
+};
+
+// Header/Footer styling
+export const getHeaderFooterClassName = () => {
+  return [
+    'flex items-center overflow-hidden',
+    'border-slate-400 bg-gradient-to-r from-slate-200 to-slate-300 p-2',
+  ].join(' ');
+};
+
+// Content area styling
+export const getContentClassName = () => {
+  return 'bg-slate-100 p-3';
+};
+
+// Text styling functions
+export const getHeaderTextClassName = () => {
+  return 'mr-2 min-w-0 truncate text-sm font-semibold text-slate-700';
+};
+
+export const getLabelTextClassName = () => {
+  return 'text-2xs text-slate-700';
+};
+
+export const getValueTextClassName = () => {
+  return 'block text-sm font-semibold text-slate-700';
+};
+
+export const getSecondaryTextClassName = () => {
+  return 'text-sm italic text-slate-600';
+};
+
+// Border and shadow styling
+export const getBorderClassName = () => {
+  return 'border border-slate-300';
+};
+
+// Avatar container styling
+export const getAvatarContainerClassName = () => {
+  return 'overflow-hidden rounded-md';
+};
+
+// Base avatar image styling
+export const getAvatarImageClassName = (size: string = 'medium') => {
+  const sizeClasses = {
+    small: 'h-6 w-6',
+    medium: 'h-[80px] w-[80px]',
+    large: 'h-20 w-20',
+  };
+
+  return `${sizeClasses[size as keyof typeof sizeClasses]} rounded-md ${getBorderClassName()} object-cover shadow-sm`;
+};
+
+// Icon styling
+export const getIconClassName = () => {
+  return 'h-8 w-8 rounded-md p-1';
+};
+
+/**
+ * Calculate the position for a hover preview popup
+ * @param containerRef The reference to the element containing the hover trigger
+ * @param previewRef The reference to the popup element
+ * @returns CSS properties for positioning the popup or undefined if refs are not available
+ */
+export const calculatePreviewPosition = (
+  containerRef: RefObject<HTMLElement>,
+  previewRef: RefObject<HTMLElement>,
+): CSSProperties | undefined => {
+  if (!containerRef.current || !previewRef.current) return undefined;
+
+  // Get dimensions
+  const linkRect = containerRef.current.getBoundingClientRect();
+  const previewRect = previewRef.current.getBoundingClientRect();
+  const viewport = {
+    width: window.innerWidth,
+    height: window.innerHeight,
+  };
+
+  // Determine horizontal position
+  const spaceRight = viewport.width - linkRect.right;
+  const spaceLeft = linkRect.left;
+  const showOnRight = spaceRight > spaceLeft;
+
+  // Calculate vertical center alignment
+  const linkCenter = linkRect.top + linkRect.height / 2;
+  const padding =
+    parseInt(getComputedStyle(document.documentElement).fontSize) * 2; // 2em
+
+  // Calculate preferred vertical position (centered with link)
+  let top = linkCenter - previewRect.height / 2;
+
+  // Adjust if too close to viewport edges
+  if (top < padding) top = padding;
+  if (top + previewRect.height > viewport.height - padding) {
+    top = viewport.height - previewRect.height - padding;
+  }
+
+  // Return position
+  return {
+    position: 'fixed',
+    zIndex: 9999,
+    top: `${top}px`,
+    overflow: 'auto',
+    transition: 'opacity 0.2s ease-in-out',
+    ...(showOnRight
+      ? { left: `${linkRect.right + 10}px` }
+      : { right: `${viewport.width - linkRect.left + 10}px` }),
+  };
+};
+
+/**
+ * Get initial offscreen position style for measurement
+ */
+export const getOffscreenMeasurementStyle = (): CSSProperties => {
+  return {
+    position: 'fixed',
+    left: '-9999px',
+    top: '0',
+    opacity: '0',
+    transition: 'opacity 0.2s ease-in-out',
+  };
+};
+
+/**
+ * Custom hook for handling hover preview functionality
+ */
+export const useHoverPreview = () => {
+  const [showPreview, setShowPreview] = useState(false);
+  const [previewStyle, setPreviewStyle] = useState<CSSProperties>({});
+  const containerRef = useRef<HTMLDivElement>(null);
+  const previewRef = useRef<HTMLDivElement>(null);
+
+  // Create portal container once
+  const portalContainer = useMemo(() => document.createElement('div'), []);
+
+  // Setup portal container
+  useEffect(() => {
+    document.body.appendChild(portalContainer);
+    return () => {
+      document.body.removeChild(portalContainer);
+    };
+  }, [portalContainer]);
+
+  // Position and display the preview
+  const updatePosition = () => {
+    const newPosition = calculatePreviewPosition(containerRef, previewRef);
+    if (newPosition) {
+      setPreviewStyle(newPosition);
+    }
+  };
+
+  // Two-step approach: first measure offscreen, then show
+  const handleMouseEnter = () => {
+    // Step 1: Render offscreen for measurement
+    setPreviewStyle(getOffscreenMeasurementStyle());
+    setShowPreview(true);
+
+    // Step 2: Position properly after a very short delay
+    setTimeout(() => {
+      updatePosition();
+    }, 20);
+  };
+
+  const handleMouseLeave = () => {
+    setShowPreview(false);
+  };
+
+  // Handle window resize
+  useEffect(() => {
+    if (!showPreview) return;
+
+    window.addEventListener('resize', updatePosition);
+    return () => {
+      window.removeEventListener('resize', updatePosition);
+    };
+  }, [showPreview]);
+
+  return {
+    containerRef,
+    previewRef,
+    portalContainer,
+    showPreview,
+    previewStyle,
+    handleMouseEnter,
+    handleMouseLeave,
+  };
+};
+
+export function anchorClassNamesForVisualStyle(
+  visualStyle: string,
+  hasIcon: boolean = false,
+) {
+  let classNames = ['truncate', 'gap-1'];
+  if (hasIcon) {
+    classNames.push('flex items-center');
+  }
+  return classNames.join(' ');
+}

app/javascript/bundles/UI/collapsibleSections.ts

@@ -0,0 +1,181 @@
+/**
+ * Collapsible Section Controls
+ *
+ * This module contains functions to control collapsible sections with font size controls.
+ * Features include:
+ * - Toggling between expanded and collapsed view
+ * - Increasing and decreasing font size
+ */
+
+/**
+ * Class representing a collapsible section with font size controls
+ */
+class CollapsibleSection {
+  private readonly section: Element;
+  private readonly content: HTMLElement;
+  private readonly toggleTextElement: Element | null;
+  private readonly toggleIconElement: HTMLElement | null;
+  private readonly fontDisplayElement: Element | null;
+  private readonly baseFontSize: number;
+  private readonly initiallyCollapsed: boolean;
+
+  /**
+   * Create a new collapsible section
+   * @param rootElement The root DOM element for this section
+   */
+  constructor(rootElement: Element) {
+    this.section = rootElement;
+    this.initiallyCollapsed = rootElement.hasAttribute(
+      'data-initially-collapsed',
+    );
+
+    // Find and cache all necessary DOM elements
+    const contentElement = rootElement.querySelector(
+      '[data-collapsable="content"]',
+    );
+    if (!(contentElement instanceof HTMLElement)) {
+      throw new Error(`Section is missing a content element`);
+    }
+
+    this.content = contentElement;
+
+    // Get single elements instead of arrays
+    this.toggleTextElement = rootElement.querySelector(
+      '[data-collapsable="toggle-text"]',
+    );
+
+    const iconElement = rootElement.querySelector(
+      '[data-collapsable="toggle-icon"]',
+    );
+    this.toggleIconElement =
+      iconElement instanceof HTMLElement ? iconElement : null;
+
+    this.fontDisplayElement = rootElement.querySelector(
+      '[data-collapsable="font-display"]',
+    );
+
+    // Initialize font size info
+    this.baseFontSize = parseFloat(
+      window.getComputedStyle(this.content).fontSize,
+    );
+    this.updateFontSizeDisplay();
+  }
+
+  /**
+   * Initialize event listeners for this section
+   */
+  initEventListeners(): void {
+    // Setup handlers for all interactive elements
+    this.section
+      .querySelectorAll('[data-collapsable="toggle"]')
+      .forEach((button) => {
+        button.addEventListener('click', (e) => {
+          e.preventDefault();
+          this.toggle();
+        });
+      });
+
+    this.section
+      .querySelectorAll('[data-collapsable="decrease"]')
+      .forEach((button) => {
+        button.addEventListener('click', (e) => {
+          e.preventDefault();
+          this.decreaseFontSize();
+        });
+      });
+
+    this.section
+      .querySelectorAll('[data-collapsable="increase"]')
+      .forEach((button) => {
+        button.addEventListener('click', (e) => {
+          e.preventDefault();
+          this.increaseFontSize();
+        });
+      });
+  }
+
+  /**
+   * Toggle between expanded and collapsed view
+   */
+  toggle(): void {
+    // Determine current state from DOM
+    const isCollapsed = this.content.classList.contains('line-clamp-6');
+
+    // Toggle line-clamp class
+    this.content.classList.toggle('line-clamp-6');
+
+    // Update toggle text
+    if (this.toggleTextElement) {
+      this.toggleTextElement.textContent = isCollapsed
+        ? 'Show Less'
+        : 'Show More';
+    }
+
+    // Update toggle icon
+    if (this.toggleIconElement) {
+      this.toggleIconElement.style.transform = isCollapsed
+        ? 'rotate(180deg)'
+        : 'rotate(0deg)';
+    }
+  }
+
+  /**
+   * Adjust font size of content within min/max bounds
+   */
+  adjustFontSize(delta: number, min = 12, max = 22): void {
+    const currentSize = parseFloat(
+      window.getComputedStyle(this.content).fontSize,
+    );
+    const newSize = Math.min(Math.max(currentSize + delta, min), max);
+
+    this.content.style.fontSize = `${newSize}px`;
+    this.updateFontSizeDisplay();
+  }
+
+  /**
+   * Increase font size (max 22px)
+   */
+  increaseFontSize(): void {
+    this.adjustFontSize(2);
+  }
+
+  /**
+   * Decrease font size (min 12px)
+   */
+  decreaseFontSize(): void {
+    this.adjustFontSize(-2);
+  }
+
+  /**
+   * Update the font size percentage display
+   */
+  updateFontSizeDisplay(): void {
+    if (!this.fontDisplayElement) return;
+
+    const currentSize = parseFloat(
+      window.getComputedStyle(this.content).fontSize,
+    );
+    const percentage = Math.round((currentSize / this.baseFontSize) * 100);
+
+    this.fontDisplayElement.textContent = `${percentage}%`;
+  }
+}
+
+/**
+ * Initialize all collapsible sections on the page
+ */
+export function initCollapsibleSections(): void {
+  document
+    .querySelectorAll('[data-collapsable="root"]')
+    .forEach((sectionElement) => {
+      try {
+        // Create a new section instance
+        const section = new CollapsibleSection(sectionElement);
+
+        // Set up event handlers
+        section.initEventListeners();
+      } catch (error) {
+        console.error('Failed to initialize section:', error);
+      }
+    });
+}

app/javascript/bundles/UI/components/IpAddressInput.tsx

@@ -0,0 +1,129 @@
+import * as React from 'react';
+import { useState, useEffect } from 'react';
+import {
+  validateIpAddress,
+  type ValidationResult,
+} from '../utils/ipValidation';
+
+interface IpAddressInputProps {
+  initialValue?: string;
+  name: string;
+  id?: string;
+  placeholder?: string;
+  onChange?: (value: string, isValid: boolean) => void;
+  className?: string;
+}
+
+const IpAddressInput: React.FC<IpAddressInputProps> = ({
+  initialValue = '',
+  name = 'ip_address_role[ip_address]',
+  id,
+  placeholder = 'Example: 192.168.1.1, 2001:db8::1, or 10.0.0.0/24',
+  onChange,
+  className = '',
+}) => {
+  const [value, setValue] = useState(initialValue);
+  const [validation, setValidation] = useState<ValidationResult>({
+    isValid: true,
+    message: '',
+    type: 'none',
+  });
+  const [isFocused, setIsFocused] = useState(false);
+
+  // Update validation when value changes
+  useEffect(() => {
+    const result = validateIpAddress(value);
+    setValidation(result);
+
+    // Call onChange callback if provided
+    if (onChange) {
+      onChange(value, result.isValid);
+    }
+  }, [value, onChange]);
+
+  // Get border color based on validation state
+  const getBorderColorClass = () => {
+    if (!isFocused) return 'border-slate-300';
+    if (value === '') return 'border-sky-500';
+    return validation.isValid ? 'border-emerald-500' : 'border-red-500';
+  };
+
+  return (
+    <div className="relative">
+      <input
+        type="text"
+        value={value}
+        onChange={(e) => setValue(e.target.value)}
+        onFocus={() => setIsFocused(true)}
+        onBlur={() => setIsFocused(false)}
+        placeholder={placeholder}
+        className={`block w-full rounded-md font-mono shadow-sm focus:ring-sky-500 sm:text-sm ${getBorderColorClass()} ${className}`}
+        id={id || 'ip_address_input'}
+      />
+
+      {/* This is a direct input that will be properly included in form submission */}
+      <input
+        type="text"
+        name={name}
+        value={value}
+        readOnly
+        style={{ display: 'none' }}
+      />
+
+      {/* Validation feedback */}
+      {value !== '' && (
+        <div
+          className={`mt-1 text-sm ${validation.isValid ? 'text-emerald-600' : 'text-red-600'}`}
+        >
+          <div className="flex items-center">
+            {validation.isValid ? (
+              <svg
+                className="mr-1 h-4 w-4"
+                fill="currentColor"
+                viewBox="0 0 20 20"
+              >
+                <path
+                  fillRule="evenodd"
+                  d="M10 18a8 8 0 100-16 8 8 0 000 16zm3.707-9.293a1 1 0 00-1.414-1.414L9 10.586 7.707 9.293a1 1 0 00-1.414 1.414l2 2a1 1 0 001.414 0l4-4z"
+                  clipRule="evenodd"
+                />
+              </svg>
+            ) : (
+              <svg
+                className="mr-1 h-4 w-4"
+                fill="currentColor"
+                viewBox="0 0 20 20"
+              >
+                <path
+                  fillRule="evenodd"
+                  d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z"
+                  clipRule="evenodd"
+                />
+              </svg>
+            )}
+            <span>{validation.message}</span>
+          </div>
+        </div>
+      )}
+
+      {/* Additional helpful information */}
+      <div className="mt-2 min-w-[400px] text-sm text-slate-500">
+        <div className="flex items-center">
+          {validation.type === 'cidr-v4' || validation.type === 'cidr-v6' ? (
+            <span>
+              CIDR notation represents an IP range (e.g., 10.0.0.0/24 for IPv4
+              or 2001:db8::/32 for IPv6)
+            </span>
+          ) : (
+            <span>
+              Enter a single IP address (IPv4: 192.168.1.1 or IPv6: 2001:db8::1)
+              or an IP range in CIDR notation
+            </span>
+          )}
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default IpAddressInput;

app/javascript/bundles/UI/components/index.ts

@@ -0,0 +1 @@
+export { default as IpAddressInput } from './IpAddressInput';

app/javascript/bundles/UI/startup/IpAddressInputRegistry.tsx

@@ -0,0 +1,8 @@
+import * as React from 'react';
+import ReactOnRails from 'react-on-rails';
+import { IpAddressInput } from '../components';
+
+// This is how react_on_rails can see the component in the browser.
+ReactOnRails.register({
+  IpAddressInput,
+});

app/javascript/bundles/UI/utils/ipValidation.ts

@@ -0,0 +1,128 @@
+export type ValidationResult = {
+  isValid: boolean;
+  message: string;
+  type: 'ipv4' | 'ipv6' | 'cidr-v4' | 'cidr-v6' | 'none';
+};
+
+// Utility functions for IP address validation
+const isIPv4Segment = (segment: string): boolean => {
+  const num = Number(segment);
+  return !isNaN(num) && num >= 0 && num <= 255 && segment === num.toString();
+};
+
+const isIPv4Address = (address: string): boolean => {
+  const segments = address.split('.');
+  return segments.length === 4 && segments.every(isIPv4Segment);
+};
+
+const isIPv6Segment = (segment: string): boolean => {
+  return segment.length <= 4 && /^[0-9a-fA-F]*$/.test(segment);
+};
+
+const isIPv6Address = (address: string): boolean => {
+  // Handle the :: compression
+  const parts = address.split('::');
+  if (parts.length > 2) return false; // More than one :: is invalid
+
+  if (parts.length === 2) {
+    const [left, right] = parts;
+    const leftSegments = left ? left.split(':') : [];
+    const rightSegments = right ? right.split(':') : [];
+
+    // Total segments should be 8 after decompression
+    if (leftSegments.length + rightSegments.length > 7) return false;
+
+    // Validate each segment
+    return (
+      leftSegments.every(isIPv6Segment) && rightSegments.every(isIPv6Segment)
+    );
+  }
+
+  // No compression, should be exactly 8 segments
+  const segments = address.split(':');
+  return segments.length === 8 && segments.every(isIPv6Segment);
+};
+
+const isCIDRPrefix = (prefix: string, isIPv6: boolean): boolean => {
+  const num = Number(prefix);
+  return !isNaN(num) && num >= 0 && num <= (isIPv6 ? 128 : 32);
+};
+
+const validateCIDR = (input: string): ValidationResult | null => {
+  const [address, prefix] = input.split('/');
+  if (!prefix) return null;
+
+  // Check if it's IPv6 CIDR
+  if (address.includes(':')) {
+    if (!isIPv6Address(address) || !isCIDRPrefix(prefix, true)) {
+      return {
+        isValid: false,
+        message: 'Invalid IPv6 CIDR range format',
+        type: 'none',
+      };
+    }
+    return {
+      isValid: true,
+      message: 'Valid IPv6 CIDR range',
+      type: 'cidr-v6',
+    };
+  }
+
+  // Check if it's IPv4 CIDR
+  if (!isIPv4Address(address) || !isCIDRPrefix(prefix, false)) {
+    return {
+      isValid: false,
+      message: 'Invalid IPv4 CIDR range format',
+      type: 'none',
+    };
+  }
+  return {
+    isValid: true,
+    message: 'Valid IPv4 CIDR range',
+    type: 'cidr-v4',
+  };
+};
+
+export const validateIpAddress = (input: string): ValidationResult => {
+  if (!input) {
+    return {
+      isValid: false,
+      message: 'IP address is required',
+      type: 'none',
+    };
+  }
+
+  if (input.includes('/')) {
+    const cidrResult = validateCIDR(input);
+    if (cidrResult) return cidrResult;
+
+    return {
+      isValid: false,
+      message: 'Invalid CIDR range format',
+      type: 'none',
+    };
+  }
+
+  // Single IP address validation
+  if (isIPv4Address(input)) {
+    return {
+      isValid: true,
+      message: 'Valid IPv4 address',
+      type: 'ipv4',
+    };
+  }
+
+  if (isIPv6Address(input)) {
+    return {
+      isValid: true,
+      message: 'Valid IPv6 address',
+      type: 'ipv6',
+    };
+  }
+
+  return {
+    isValid: false,
+    message: 'Invalid IP address format',
+    type: 'none',
+  };
+};

app/javascript/packs/application-bundle.js

@@ -2,9 +2,21 @@ import ReactOnRails from 'react-on-rails';
 
 import UserSearchBar from '../bundles/Main/components/UserSearchBar';
 import { UserMenu } from '../bundles/Main/components/UserMenu';
+import { PostHoverPreviewWrapper } from '../bundles/Main/components/PostHoverPreviewWrapper';
+import { UserHoverPreviewWrapper } from '../bundles/Main/components/UserHoverPreviewWrapper';
+import { initCollapsibleSections } from '../bundles/UI/collapsibleSections';
+import { IpAddressInput } from '../bundles/UI/components';
 
 // This is how react_on_rails can see the components in the browser.
 ReactOnRails.register({
   UserSearchBar,
   UserMenu,
+  PostHoverPreviewWrapper,
+  UserHoverPreviewWrapper,
+  IpAddressInput,
+});
+
+// Initialize collapsible sections
+document.addEventListener('DOMContentLoaded', function () {
+  initCollapsibleSections();
 });

app/javascript/packs/server-bundle.js

@@ -2,8 +2,13 @@ import ReactOnRails from 'react-on-rails';
 
 import UserSearchBar from '../bundles/Main/components/UserSearchBarServer';
 import { UserMenu } from '../bundles/Main/components/UserMenu';
+import { PostHoverPreviewWrapper } from '../bundles/Main/components/PostHoverPreviewWrapper';
+import { UserHoverPreviewWrapper } from '../bundles/Main/components/UserHoverPreviewWrapper';
+
 // This is how react_on_rails can see the UserSearchBar in the browser.
 ReactOnRails.register({
   UserMenu,
   UserSearchBar,
+  PostHoverPreviewWrapper,
+  UserHoverPreviewWrapper,
 });

app/jobs/application_job.rb

@@ -26,4 +26,22 @@ class ApplicationJob < ActiveJob::Base
     @ignore_signature_args.concat(args)
     @ignore_signature_args
   end
+
+  # collect all ignore_signature_args from all superclasses
+  sig { returns(T::Array[Symbol]) }
+  def self.gather_ignore_signature_args
+    args = T.let(%i[_aj_symbol_keys _aj_ruby2_keywords], T::Array[Symbol])
+
+    klass = T.let(self, T.class_of(ApplicationJob))
+    loop do
+      args += klass.ignore_signature_args
+      if (superklass = klass.superclass) && superklass < ApplicationJob
+        klass = superklass
+      else
+        break
+      end
+    end
+
+    args.uniq.sort
+  end
 end

app/jobs/domain/e621/job/base.rb

@@ -7,13 +7,23 @@ class Domain::E621::Job::Base < Scraper::JobBase
     :get_e621_http_client
   end
 
-  sig { returns(Domain::E621::User) }
+  sig { returns(Domain::User::E621User) }
   def user_from_args!
     T.must(user_from_args)
   end
 
-  sig { returns(T.nilable(Domain::E621::User)) }
+  sig { returns(T.nilable(Domain::User::E621User)) }
   def user_from_args
-    T.cast(arguments[0][:user], T.nilable(Domain::E621::User))
+    T.cast(arguments[0][:user], T.nilable(Domain::User::E621User))
+  end
+
+  sig { returns(Domain::Post::E621Post) }
+  def post_from_args!
+    T.must(post_from_args)
+  end
+
+  sig { returns(T.nilable(Domain::Post::E621Post)) }
+  def post_from_args
+    T.cast(arguments[0][:post], T.nilable(Domain::Post::E621Post))
   end
 end

app/jobs/domain/e621/job/posts_index_job.rb

@@ -2,10 +2,9 @@
 class Domain::E621::Job::PostsIndexJob < Domain::E621::Job::Base
   queue_as :e621
 
-  sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
+  sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
     response = http_client.get("https://e621.net/posts.json")
-    log_entry = response.log_entry
 
     if response.status_code != 200
       fatal_error(
@@ -13,13 +12,14 @@ class Domain::E621::Job::PostsIndexJob < Domain::E621::Job::Base
       )
     end
 
-    json = JSON.parse(response.body)
-    if json["posts"].nil?
-      fatal_error("no posts in response, hle #{log_entry.id}")
-    end
+    posts_json =
+      T.cast(
+        JSON.parse(response.body)["posts"],
+        T::Array[T::Hash[String, T.untyped]],
+      )
 
     e621_posts =
-      json["posts"].map do |post_json|
+      posts_json.map do |post_json|
         Domain::E621::TagUtil.initialize_or_update_post(
           post_json: post_json,
           caused_by_entry: causing_log_entry,
@@ -37,13 +37,6 @@ class Domain::E621::Job::PostsIndexJob < Domain::E621::Job::Base
       e621_post.save!
     end
 
-    (created_posts + updated_posts).uniq.each do |post|
-      logger.info(
-        "[e621_id: #{post.e621_id.to_s.bold}] enqueueing static file job",
-      )
-      defer_job(Domain::E621::Job::StaticFileJob, { post: post })
-    end
-
     logger.info(
       "#{updated_posts.count} updated, #{created_posts.count} created, #{seen_posts.count} seen",
     )

app/jobs/domain/e621/job/scan_post_favs_job.rb

@@ -5,14 +5,16 @@ class Domain::E621::Job::ScanPostFavsJob < Domain::E621::Job::Base
   MAX_USERS_PER_SLICE = 1000
 
   class UserRow < T::Struct
+    include T::Struct::ActsAsComparable
+
     const :e621_id, Integer
     const :name, String
     const :num_other_favs, Integer
   end
 
-  sig { override.params(args: T.untyped).void }
+  sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
-    post = T.cast(args[:post], Domain::E621::Post)
+    post = T.cast(args[:post], Domain::Post::E621Post)
     page = 1
     breaker = 0
     total_created_users = 0
@@ -32,9 +34,9 @@ class Domain::E621::Job::ScanPostFavsJob < Domain::E621::Job::Base
       rows = html.css("tbody tr")
       rows.each do |row_elem|
         user_member_elem = row_elem.css("td:first-child a")&.first
-        e621_user_id = user_member_elem["href"].split("/").last.to_i
-        e621_id_to_user_row[e621_user_id] = UserRow.new(
-          e621_id: e621_user_id,
+        e621_id = user_member_elem["href"].split("/").last.to_i
+        e621_id_to_user_row[e621_id] = UserRow.new(
+          e621_id: e621_id,
           name: user_member_elem.text,
           num_other_favs: row_elem.css("td:last-child").text.to_i,
         )
@@ -43,16 +45,16 @@ class Domain::E621::Job::ScanPostFavsJob < Domain::E621::Job::Base
       ReduxApplicationRecord.transaction do
         e621_id_to_user =
           T.cast(
-            Domain::E621::User.where(
-              e621_user_id: e621_id_to_user_row.keys,
-            ).index_by(&:e621_user_id),
-            T::Hash[Integer, Domain::E621::User],
+            Domain::User::E621User.where(
+              e621_id: e621_id_to_user_row.keys,
+            ).index_by(&:e621_id),
+            T::Hash[Integer, Domain::User::E621User],
           )
         e621_id_to_user_row.values.each do |user_row|
           user =
             e621_id_to_user[user_row.e621_id] ||
-              Domain::E621::User.new(
-                e621_user_id: user_row.e621_id,
+              Domain::User::E621User.new(
+                e621_id: user_row.e621_id,
                 name: user_row.name,
               )
           user.num_other_favs_cached = user_row.num_other_favs

app/jobs/domain/e621/job/scan_post_job.rb

@@ -2,48 +2,46 @@
 class Domain::E621::Job::ScanPostJob < Domain::E621::Job::Base
   queue_as :e621
 
-  sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
+  sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
-    post = T.let(args[:post] || raise("no post provided"), Domain::E621::Post)
+    post = post_from_args!
+    logger.push_tags(make_arg_tag(post))
+    file = post.file
+    logger.push_tags(make_arg_tag(file)) if file.present?
 
-    logger.prefix =
-      proc { "[e621_id #{post.e621_id.to_s.bold} / #{post.state&.bold}]" }
-
-    if post.file.present?
-      logger.warn("Post #{post.e621_id} already has a file")
+    if file.present? && file.state == "ok" && file.log_entry_id.present?
+      logger.warn("post already has file with 'ok' state, skipping")
       return
     end
 
-    if post.file_url_str.present?
-      logger.error("Post #{post.e621_id} already has a file URL")
-      return
-    end
+    logger.info("scanning post")
 
-    logger.info("Scanning post #{post.e621_id}")
     response = http_client.get("https://e621.net/posts/#{post.e621_id}.json")
-    log_entry = response.log_entry
+    post.scan_log_entry = response.log_entry
+    post.last_submission_log_entry = response.log_entry
+
     if response.status_code != 200
-      post.state_detail["scan_log_entry_id"] = log_entry.id
-      post.state = :scan_error
-      post.state_detail[
-        "scan_error"
-      ] = "Error scanning post #{post.e621_id}: #{response.status_code}"
+      post.state = "scan_error"
+      post.scan_error =
+        "Error scanning post #{post.e621_id}: #{response.status_code}"
       post.save!
       fatal_error(
         "Error scanning post #{post.e621_id}: #{response.status_code}",
       )
+    else
+      logger.info("scanned post")
     end
 
     post_json = JSON.parse(response.body)["post"]
     post =
       Domain::E621::TagUtil.initialize_or_update_post(
         post_json: post_json,
-        caused_by_entry: log_entry,
+        caused_by_entry: causing_log_entry,
       )
-    post.save!
-
-    unless post.file.present?
-      defer_job(Domain::E621::Job::StaticFileJob, { post: post })
-    end
+    post.scan_log_entry = response.log_entry
+    post.last_submission_log_entry = response.log_entry
+  ensure
+    post.save! if post
+    file.save! if file
   end
 end

app/jobs/domain/e621/job/scan_user_favs_job.rb

@@ -2,14 +2,22 @@
 class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
   MAX_PAGES_BEFORE_BREAK = 2400
   MAX_PER_PAGE = T.let(Rails.env.test? ? 4 : 320, Integer)
-  include HasMeasureDuration
 
-  sig { override.params(args: T.untyped).void }
+  sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
     user = user_from_args!
-    if user.scanned_favs_status == "error" && !args[:force]
-      logger.info("[user #{user.e621_user_id} has error status, skipping]")
-      return
+    logger.push_tags(make_arg_tag(user))
+    logger.info("server indicates #{user.num_other_favs_cached} favs")
+
+    if user.scanned_favs_error?
+      if force_scan?
+        logger.info(
+          "scanned favs status is error, but force scan is true, continuing",
+        )
+      else
+        logger.warn("scanned favs status is error, skipping")
+        return
+      end
     end
 
     last_e621_post_id = T.let(nil, T.nilable(Integer))
@@ -17,36 +25,22 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
     post_ids = T.let([], T::Array[Integer])
     total_new_posts = 0
 
-    prefix = [
-      "[e621 user id: #{user.e621_user_id&.to_s&.bold}]",
-      "[username: #{user.name&.bold}]",
-    ].join(" ")
-
-    logger.info("#{prefix} [cached favs: #{user.num_other_favs_cached}]")
-
     loop do
       breaker += 1
       if breaker > MAX_PAGES_BEFORE_BREAK
-        logger.warn(
-          "#{prefix} [breaker is too big] [last e621 post id: #{last_e621_post_id}]",
-        )
+        logger.error("breaker is too big (#{breaker})")
         break
       end
 
       url =
         "https://e621.net/posts.json?tags=status:any+fav:#{user.url_name}+order:id_desc&limit=#{MAX_PER_PAGE}"
-      if last_e621_post_id
-        limiter = "before #{last_e621_post_id.to_s.bold}"
-        url += "&page=b#{last_e621_post_id.to_s}"
-      else
-        limiter = "(none)"
-      end
+      url += "&page=b#{last_e621_post_id.to_s}" if last_e621_post_id
       response = http_client.get(url)
 
       if response.status_code == 403 &&
            response.body.include?("This users favorites are hidden")
         user.favs_are_hidden = true
-        user.scanned_favs_at = Time.current
+        user.scanned_favs_at = DateTime.current
         user.save!
         break
       end
@@ -63,9 +57,7 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
           T::Array[T::Hash[String, T.untyped]],
         )
       if posts_json.empty?
-        logger.info(
-          "#{prefix} [limiter: #{limiter}] [req: #{breaker}] [no posts found] ",
-        )
+        logger.info("no posts found on page #{breaker}")
         break
       end
 
@@ -76,55 +68,49 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
           end
           .to_h
 
-      measure(
-        "#{prefix} [finding favs: #{posts_json.size}] [req: #{breaker}]",
-      ) do
-        e621_id_to_post_id = T.let({}, T::Hash[Integer, Integer])
-        e621_post_id_to_post_json
-          .keys
-          .each_slice(1000) do |e621_post_id_slice|
-            e621_id_to_post_id.merge!(
-              Domain::E621::Post
-                .where(e621_id: e621_post_id_slice)
-                .pluck(:e621_id, :id)
-                .to_h,
-            )
-          end
-        missing_e621_ids =
-          e621_post_id_to_post_json.keys - e621_id_to_post_id.keys
+      logger.info "found #{posts_json.size} favs on page #{breaker}"
+      e621_id_to_post_id = T.let({}, T::Hash[Integer, Integer])
+      e621_post_id_to_post_json
+        .keys
+        .each_slice(1000) do |e621_post_id_slice|
+          e621_id_to_post_id.merge!(
+            Domain::Post::E621Post
+              .where(e621_id: e621_post_id_slice)
+              .pluck(:e621_id, :id)
+              .to_h,
+          )
+        end
+      missing_e621_ids =
+        e621_post_id_to_post_json.keys - e621_id_to_post_id.keys
 
-        if missing_e621_ids.any?
-          measure("#{prefix} [creating posts: #{missing_e621_ids.size}]") do
-            missing_e621_ids.each do |e621_post_id|
-              post_json = T.must(e621_post_id_to_post_json[e621_post_id])
-              post =
-                Domain::E621::TagUtil.initialize_or_update_post(
-                  post_json: post_json,
-                  caused_by_entry: causing_log_entry,
-                )
-              was_new = post.new_record?
-              post.save!
-              e621_id_to_post_id[e621_post_id] = T.must(post.id)
-              if was_new
-                logger.info(
-                  "#{prefix} [created post: e621 id #{post.e621_id} / id #{post.id}]",
-                )
-                total_new_posts += 1
-                defer_job(Domain::E621::Job::StaticFileJob, post: post)
-              end
-            end
+      if missing_e621_ids.any?
+        logger.info "creating #{missing_e621_ids.size} posts"
+        missing_e621_ids.each do |e621_post_id|
+          post_json = T.must(e621_post_id_to_post_json[e621_post_id])
+          post =
+            Domain::E621::TagUtil.initialize_or_update_post(
+              post_json: post_json,
+              caused_by_entry: causing_log_entry,
+            )
+          was_new = post.new_record?
+          post.set_index_page_entry(response.log_entry)
+          post.save!
+          e621_id_to_post_id[e621_post_id] = T.must(post.id)
+          if was_new
+            logger.info("created post #{make_arg_tag(post).join(" ")}")
+            total_new_posts += 1
           end
         end
-
-        post_ids.concat(e621_id_to_post_id.values)
-        logger.info(
-          "#{prefix} [req: #{breaker}] [total posts: #{post_ids.size}] [total created: #{total_new_posts}]",
-        )
       end
 
+      post_ids.concat(e621_id_to_post_id.values)
+      logger.info(
+        "[total posts: #{post_ids.size}] [total created: #{total_new_posts}]",
+      )
+
       if posts_json.size < MAX_PER_PAGE
         logger.info(
-          "#{prefix} [fewer than limit; breaking] [limiter: #{limiter}] [req: #{breaker}]",
+          "number of posts #{posts_json.size} < MAX_PER_PAGE (#{MAX_PER_PAGE}), breaking",
         )
         break
       end
@@ -132,37 +118,35 @@ class Domain::E621::Job::ScanUserFavsJob < Domain::E621::Job::Base
       last_e621_post_id = T.cast(T.must(posts_json.last)["id"].to_i, Integer)
     end
 
-    measure("#{prefix} [upserting favs: #{post_ids.size}]") do
-      post_ids.each_slice(1000) do |slice|
-        ReduxApplicationRecord.transaction do
-          Domain::E621::Fav.upsert_all(
-            slice.map { |post_id| { user_id: user.id, post_id: post_id } },
-            unique_by: :index_domain_e621_favs_on_user_id_and_post_id,
-          )
-        end
+    logger.info "upserting #{post_ids.size} favs"
+    post_ids.each_slice(1000) do |slice|
+      ReduxApplicationRecord.transaction do
+        Domain::UserPostFav.upsert_all(
+          slice.map { |post_id| { user_id: user.id, post_id: post_id } },
+          unique_by: :index_domain_user_post_favs_on_user_id_and_post_id,
+        )
       end
     end
 
     logger.info(
-      "#{prefix} " +
-        [
-          "[favs scanned: #{post_ids.size.to_s.bold}]",
-          "[posts created: #{total_new_posts.to_s.bold}]",
-          "[total requests: #{breaker}]",
-          "[done]",
-        ].join(" "),
+      [
+        "[favs scanned: #{post_ids.size.to_s.bold}]",
+        "[posts created: #{total_new_posts.to_s.bold}]",
+        "[total requests: #{breaker}]",
+        "done",
+      ].join(" "),
     )
 
-    user.scanned_favs_status = "ok"
-    user.scanned_favs_at = Time.current
+    user.scanned_favs_ok!
+    user.scanned_favs_at = DateTime.current
     user.save!
   rescue StandardError
-    logger.error("error scanning user favs: #{user&.e621_user_id}")
+    logger.error("error scanning user favs: #{user&.e621_id}")
     user = user_from_args
-    if user
-      user.scanned_favs_status = "error"
-      user.save!
-    end
+    user.scanned_favs_error! if user
     raise
+  ensure
+    user.save! if user
+    logger.pop_tags
   end
 end

app/jobs/domain/e621/job/scan_users_job.rb

@@ -1,6 +1,6 @@
 # typed: strict
 class Domain::E621::Job::ScanUsersJob < Domain::E621::Job::Base
-  sig { override.params(args: T.untyped).void }
+  sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
     after = T.let(args[:after_e621_id], T.nilable(String))
     breaker = 0
@@ -27,8 +27,8 @@ class Domain::E621::Job::ScanUsersJob < Domain::E621::Job::Base
       ReduxApplicationRecord.transaction do
         users_json.each do |user_json|
           user =
-            Domain::E621::User.find_or_initialize_by(
-              e621_user_id: user_json["id"],
+            Domain::User::E621User.find_or_initialize_by(
+              e621_id: user_json["id"],
             ) { |user| user.name = user_json["name"] }
           is_new = user.new_record?
           num_new_users += 1 if is_new

app/jobs/domain/e621/job/static_file_job.rb

@@ -1,52 +1,30 @@
 # typed: strict
 class Domain::E621::Job::StaticFileJob < Domain::E621::Job::Base
+  include Domain::StaticFileJobHelper
   queue_as :static_file
 
-  sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
+  sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
-    post =
-      T.let(args[:post] || fatal_error("post is required"), Domain::E621::Post)
-
-    logger.prefix = proc { "[e621_id #{post.e621_id.to_s.bold}]" }
-
-    file_url_str = post.file_url_str
-    if file_url_str.blank?
-      logger.warn("post has no file_url_str, enqueueing for scan")
-      defer_job(Domain::E621::Job::ScanPostJob, { post: post })
-      return
-    end
-
-    if post.state == "file_error"
-      retry_count = post.state_detail&.[]("file_error")&.[]("retry_count") || 0
-      if retry_count >= 3
-        logger.error("file has been retried 3 times, giving up")
-        return
-      end
-    end
-
-    response = http_client.get(file_url_str)
-
-    if response.status_code != 200
-      post.state = :file_error
-      fe = (post.state_detail["file_error"] ||= {})
-      fe["status_code"] = response.status_code
-      fe["log_entry_id"] = response.log_entry.id
-      fe["retry_count"] ||= 0
-      fe["retry_count"] += 1
-      post.save!
-
-      if response.status_code == 404
-        logger.error("#{response.status_code}, not retrying download")
+    file =
+      if file = (args[:file] || args[:post_file])
+        T.cast(file, Domain::PostFile)
+      elsif (post = args[:post]) && post.is_a?(Domain::Post::E621Post)
+        T.must(post.file)
+      elsif (post = args[:post]) && post.is_a?(Domain::E621::Post)
+        post =
+          Domain::Post::E621Post.find_by(e621_id: post.e621_id) ||
+            fatal_error(
+              format_tags(
+                "post with not found",
+                make_tag("e621_id", post.e621_id),
+              ),
+            )
+        T.must(post.file)
       else
-        fatal_error("#{response.status_code}, will retry later")
+        fatal_error(":file or :post is required")
       end
 
-      return
-    end
-
-    post.state = :ok
-    post.file = response.log_entry
-    post.save!
-    logger.info "downloaded file"
+    logger.push_tags(make_arg_tag(file), make_arg_tag(file.post))
+    download_post_file(file)
   end
 end

app/jobs/domain/fa/job/base.rb

@@ -1,95 +1,118 @@
 # typed: strict
 class Domain::Fa::Job::Base < Scraper::JobBase
+  abstract!
   discard_on ActiveJob::DeserializationError
+  include HasBulkEnqueueJobs
 
   sig { override.returns(Symbol) }
   def self.http_factory_method
     :get_fa_http_client
   end
 
-  sig { params(args: T.untyped).void }
-  def initialize(*args)
-    super(*T.unsafe(args))
-    @force_scan = T.let(false, T::Boolean)
-    @user = T.let(nil, T.nilable(Domain::Fa::User))
-    @created_user = T.let(false, T::Boolean)
-    @posts_enqueued_for_scan = T.let(Set.new, T::Set[Integer])
-  end
-
   protected
 
-  sig do
-    params(
-      args: T.untyped,
-      build_user: T::Boolean,
-      require_user_exists: T::Boolean,
-    ).returns(T.nilable(Domain::Fa::User))
-  end
-  def init_from_args!(args, build_user: true, require_user_exists: false)
-    @force_scan = !!args[:force_scan]
+  BUGGY_USER_URL_NAMES = T.let(["click here", "..", "."], T::Array[String])
 
-    if build_user
-      @user = find_or_build_user_from_args(args)
-    else
-      @user = find_user_from_args(args)
+  sig { params(user: Domain::User::FaUser).returns(T::Boolean) }
+  def buggy_user?(user)
+    if BUGGY_USER_URL_NAMES.include?(user.url_name)
+      logger.error(
+        format_tags("buggy user", make_tag("url_name", user.url_name)),
+      )
+      return true
     end
 
-    logger.prefix =
-      "[user #{(@user&.url_name || @user&.name || args[:url_name])&.bold} / #{@user&.state&.bold}]"
+    false
+  end
 
-    return nil unless @user
-    if @user.new_record?
-      if require_user_exists
-        fatal_error("user must already exist")
+  sig { returns(T::Boolean) }
+  def skip_enqueue_found_links?
+    !!arguments[0][:skip_enqueue_found_links]
+  end
+
+  sig { params(build_post: T::Boolean).returns(Domain::Post::FaPost) }
+  def post_from_args!(build_post: false)
+    args = arguments[0]
+    post = args[:post]
+    if post.is_a?(Domain::Post::FaPost)
+      return post
+    elsif post.is_a?(Domain::Fa::Post)
+      return Domain::Post::FaPost.find_by!(fa_id: post.fa_id)
+    elsif fa_id = args[:fa_id]
+      if build_post
+        Domain::Post::FaPost.find_or_initialize_by(fa_id: fa_id)
       else
-        @user.save!
-        @created_user = true
+        Domain::Post::FaPost.find_by!(fa_id: fa_id)
       end
+    else
+      fatal_error("arg 'post' must be a Domain::Post::FaPost or an Integer")
     end
-
-    @user
   end
 
-  sig { params(args: T.untyped).returns(Domain::Fa::User) }
-  def find_or_build_user_from_args(args)
-    find_user_from_args(args) ||
-      begin
-        url_name = Domain::Fa::User.name_to_url_name(args[:url_name])
-        user = Domain::Fa::User.new
-        user.url_name = url_name
-        user.name = url_name
-        user.state_detail ||= {}
-        if cle = causing_log_entry
-          user.state_detail["first_seen_entry"] = cle.id
+  sig { returns(Domain::UserAvatar) }
+  def avatar_from_args!
+    args = arguments[0]
+    avatar = args[:avatar]
+    user = args[:user]
+    if avatar.is_a?(Domain::UserAvatar)
+      return avatar
+    elsif user.is_a?(Domain::User::FaUser)
+      return T.must(user.avatar)
+    elsif user.is_a?(Domain::Fa::User)
+      user = Domain::User::FaUser.find_by(url_name: user.url_name)
+      return T.must(user&.avatar)
+    else
+      fatal_error(
+        "arg 'avatar' must be a Domain::UserAvatar or user must be a Domain::Fa::User",
+      )
+    end
+  end
+
+  sig { params(create_if_missing: T::Boolean).returns(Domain::User::FaUser) }
+  def user_from_args!(create_if_missing: true)
+    args = arguments[0]
+    user = args[:user]
+    if user.is_a?(Domain::User::FaUser)
+      user
+    elsif user.is_a?(Domain::Fa::User)
+      Domain::User::FaUser.find_by!(url_name: user.url_name)
+    elsif url_name = args[:url_name]
+      if create_if_missing
+        user =
+          Domain::User::FaUser.find_or_initialize_by(url_name:) do |user|
+            user.name = url_name
+          end
+
+        if user.new_record?
+          user.save!
+          defer_job(
+            Domain::Fa::Job::UserPageJob,
+            { user:, caused_by_entry: causing_log_entry },
+          )
         end
+
         user
+      else
+        Domain::User::FaUser.find_by!(url_name:)
       end
+    else
+      fatal_error(
+        "arg 'user' must be a Domain::User::FaUser or Domain::Fa::User, or url_name must be provided",
+      )
+    end
   end
 
-  sig { params(args: T.untyped).returns(T.nilable(Domain::Fa::User)) }
-  def find_user_from_args(args)
-    args[:user] ||
-      begin
-        if args[:url_name].blank?
-          fatal_error("arg 'url_name' is required if arg 'user' is nil")
-        end
-        url_name = Domain::Fa::User.name_to_url_name(args[:url_name])
-        Domain::Fa::User.find_by(url_name: url_name)
-      end
-  end
-
-  sig { params(scan_type: Symbol).returns(T::Boolean) }
-  def user_due_for_scan?(scan_type)
-    raise("user is nil") unless @user
-    unless @user.scan_due?(scan_type)
-      if @force_scan
+  sig { params(user: Domain::User::FaUser).returns(T::Boolean) }
+  def user_due_for_favs_scan?(user)
+    unless user.favs_scan.due?
+      if force_scan?
         logger.warn(
-          "scanned #{@user.scanned_ago_in_words(scan_type).bold} - force scanning",
+          "scanned favs #{user.favs_scan.ago_in_words.bold} ago - force scanning",
         )
         return true
       else
         logger.warn(
-          "scanned #{@user.scanned_ago_in_words(scan_type).bold} - skipping",
+          "scanned favs #{user.favs_scan.ago_in_words.bold} ago - skipping",
         )
         return false
       end
@@ -98,243 +121,475 @@ class Domain::Fa::Job::Base < Scraper::JobBase
     return true
   end
 
-  ListingsPageScanStats = Struct.new(:new_seen, :total_seen, :last_was_new)
+  class ListingPageScanStats < T::Struct
+    include T::Struct::ActsAsComparable
+    const :new_posts, T::Array[Domain::Post::FaPost]
+    const :all_posts, T::Array[Domain::Post::FaPost]
+  end
+
+  module ListingPageType
+    extend T::Sig
+
+    class BrowsePage < T::Struct
+      extend T::Sig
+      const :page_number, Integer
+    end
+
+    class GalleryPage < T::Struct
+      const :page_number, Integer
+      const :folder, String
+    end
+
+    class FavsPage < T::Struct
+      const :page_number, T.nilable(String)
+      const :user, Domain::User::FaUser
+    end
+
+    Type = T.type_alias { T.any(BrowsePage, GalleryPage, FavsPage) }
+
+    sig { params(page_type: Type).returns(String) }
+    def self.describe(page_type)
+      case page_type
+      when BrowsePage
+        "browse"
+      when GalleryPage
+        "folder '#{page_type.folder}'"
+      when FavsPage
+        "favs"
+      end
+    end
+  end
 
   sig do
     params(
-      job_type: Symbol,
-      page: T.untyped,
-      enqueue_posts_pri: Symbol,
-      enqueue_page_scan: T::Boolean,
-      enqueue_gallery_scan: T::Boolean,
-      page_desc: T.nilable(String),
-      fill_id_gaps: T::Boolean,
-      continue_for: T.nilable(Integer),
-    ).returns(ListingsPageScanStats)
+      page_type: ListingPageType::Type,
+      page_parser: Domain::Fa::Parser::Page,
+      for_user: T.nilable(Domain::User::FaUser),
+    ).returns(ListingPageScanStats)
   end
   def update_and_enqueue_posts_from_listings_page(
-    job_type,
-    page,
-    enqueue_posts_pri:,
-    enqueue_page_scan: true,
-    enqueue_gallery_scan: true,
-    page_desc: nil,
-    fill_id_gaps: false,
-    continue_for: nil
+    page_type,
+    page_parser:,
+    for_user: nil
   )
-    fatal_error("not a listings page") unless page.probably_listings_page?
-    submissions = page.submissions_parsed
+    fatal_error("not a listing page") unless page_parser.probably_listings_page?
 
-    fa_ids_to_manually_enqueue = Set.new
-    fa_ids = Set.new(submissions.map(&:id))
+    all_posts = T.let([], T::Array[Domain::Post::FaPost])
+    new_posts = T.let([], T::Array[Domain::Post::FaPost])
+    posts_to_save = T.let([], T::Array[Domain::Post::FaPost])
 
-    create_unseen_posts = false
+    page_parser.submissions_parsed.each do |submission|
+      post =
+        Domain::Post::FaPost.find_or_initialize_by_submission_parser(
+          submission,
+          first_seen_log_entry: last_log_entry,
+        )
 
-    if fill_id_gaps && submissions.any?
-      create_unseen_posts = true
-      max_fa_id, min_fa_id = fa_ids.max, fa_ids.min
-      # sanity check so we don't enqueue too many post jobs
-      if max_fa_id - min_fa_id <= 250
-        (min_fa_id..max_fa_id).each do |fa_id|
-          fa_ids_to_manually_enqueue << fa_id unless fa_ids.include?(fa_id)
-        end
+      case page_type
+      when ListingPageType::BrowsePage
+        post.first_browse_page ||= last_log_entry
+      when ListingPageType::GalleryPage
+        post.first_gallery_page ||= last_log_entry
+      end
+
+      all_posts << post
+      new_posts << post if post.new_record?
+
+      if post.new_record? || !post.state_ok? || post.file.blank? ||
+           post.file&.state_terminal_error?
+        post.state_ok!
+        posts_to_save << post
+        defer_job(Domain::Fa::Job::ScanPostJob, { post: })
+      end
+
+      if (post_file = post.file) && post_file.url_str.present? &&
+           post_file.log_entry.nil? && !post_file.state_terminal_error?
+        defer_job(Domain::Fa::Job::ScanFileJob, { post_file: })
+      end
+
+      if creator = post.creator
+        creator.state_ok!
+        creator.save!
+        enqueue_user_scan(creator, at_most_one_scan: true)
       end
     end
 
-    if continue_for && submissions.any?
-      max_fa_id = fa_ids.max
-      min_fa_id = [max_fa_id - continue_for, 0].max
-      fa_ids_to_manually_enqueue = Set.new(min_fa_id..max_fa_id)
-      fa_ids_to_manually_enqueue.subtract(fa_ids)
-      existing =
-        Domain::Fa::Post.where(
-          "fa_id >= ? AND fa_id <= ?",
-          min_fa_id,
-          max_fa_id,
-        ).pluck(:fa_id)
-      fa_ids_to_manually_enqueue.subtract(existing)
+    if for_user && (user_page = page_parser.user_page) &&
+         (url = user_page.profile_thumb_url)
+      enqueue_user_avatar(for_user, url)
     end
 
-    page_desc = (page_desc ? "page #{page_desc.to_s.bold}" : "page")
+    posts_to_save.each(&:save!)
 
-    listing_page_stats = ListingsPageScanStats.new(0, 0, false)
-    submissions.each do |submission|
-      post = Domain::Fa::Post.find_or_initialize_by(fa_id: submission.id)
-      listing_page_stats.last_was_new = post.new_record?
-      listing_page_stats.new_seen += 1 if post.new_record?
-      listing_page_stats.total_seen += 1
+    logger.info(
+      format_tags(
+        make_tag("page_number", page_type.page_number),
+        make_tag("page_type", ListingPageType.describe(page_type)),
+        make_tag("all_posts.count", all_posts.count),
+        make_tag("new_posts.count", new_posts.count),
+      ),
+    )
 
-      update_and_save_post_from_listings_page(job_type, post, submission)
-      if post.creator
-        enqueue_user_scan(
-          T.must(post.creator),
-          enqueue_page_scan: enqueue_page_scan,
-          enqueue_gallery_scan: enqueue_gallery_scan,
+    ListingPageScanStats.new(new_posts:, all_posts:)
+  end
+
+  sig { params(user: Domain::User::FaUser, at_most_one_scan: T::Boolean).void }
+  def enqueue_user_scan(user, at_most_one_scan:)
+    skip_page_enqueue = !!ENV["SKIP_PAGE_ENQUEUE"]
+    skip_favs_enqueue = !!ENV["SKIP_FAVS_ENQUEUE"]
+    skip_follows_enqueue = !!ENV["SKIP_FOLLOWS_ENQUEUE"]
+    skip_gallery_enqueue = !!ENV["SKIP_GALLERY_ENQUEUE"]
+
+    logger.tagged(make_arg_tag(user)) do
+      args =
+        if user.persisted?
+          { user: user }
+        else
+          unless user.url_name
+            logger.warn(
+              format_tags("url_name or id, skipping enqueue_user_scan"),
+            )
+            return
+          end
+
+          { url_name: user.url_name }
+        end
+
+      if (
+           user.page_scan.at.nil? || (!skip_page_enqueue && user.page_scan.due?)
+         ) && defer_job(Domain::Fa::Job::UserPageJob, args)
+        logger.info(
+          format_tags(
+            "enqueue user page job",
+            make_tag("last page scan", user.page_scan.ago_in_words),
+          ),
         )
       end
 
-      case post.state&.to_sym
-      when :ok
-        enqueue_post_scan(post, enqueue_posts_pri)
-      when :removed
-        logger.info "(todo) removed post seen in listing page, enqueue scan for fa_id #{post.fa_id}"
-      when :scan_error
-        logger.info "(todo) scan_error'd post seen in listing page for fa_id #{post.fa_id}"
-      when :file_error
-        logger.info "(todo) file_error'd post seen in listing page for fa_id #{post.fa_id}"
-      else
-        logger.info "unknown post state `#{post.state}` for fa_id #{post.fa_id}"
+      # don't enqueue any other jobs if the user page hasn't been scanned yet
+      return if at_most_one_scan && user.page_scan.due?
+
+      if (
+           user.favs_scan.at.nil? || (!skip_favs_enqueue && user.favs_scan.due?)
+         ) && defer_job(Domain::Fa::Job::FavsJob, args)
+        logger.info(
+          format_tags(
+            "enqueue user favs job",
+            make_tag("last favs scan", user.favs_scan.ago_in_words),
+          ),
+        )
+      end
+
+      return if at_most_one_scan && user.favs_scan.due?
+
+      if (
+           user.follows_scan.at.nil? ||
+             (!skip_follows_enqueue && user.follows_scan.due?)
+         ) && defer_job(Domain::Fa::Job::UserFollowsJob, args)
+        logger.info(
+          format_tags(
+            "enqueue user follows job",
+            make_tag("last follows scan", user.follows_scan.ago_in_words),
+          ),
+        )
+      end
+
+      return if at_most_one_scan && user.follows_scan.due?
+
+      if (
+           user.gallery_scan.at.nil? ||
+             (!skip_gallery_enqueue && user.gallery_scan.due?)
+         ) && defer_job(Domain::Fa::Job::UserGalleryJob, args)
+        logger.info(
+          format_tags(
+            "enqueue user gallery job",
+            make_tag("last gallery scan", user.gallery_scan.ago_in_words),
+          ),
+        )
       end
     end
-
-    fa_ids_to_manually_enqueue.to_a.sort.reverse.each do |fa_id|
-      if create_unseen_posts
-        # when filling gaps, only enqueue if the post wasn't found
-        post = Domain::Fa::Post.find_or_initialize_by(fa_id: fa_id)
-        if post.new_record?
-          post.save!
-          enqueue_post_scan(post, enqueue_posts_pri)
-        end
-      else
-        enqueue_fa_id_scan(fa_id, enqueue_posts_pri)
-      end
-    end
-
-    logger.info "#{page_desc} has #{submissions.count.to_s.bold} posts, " +
-                  "#{listing_page_stats.new_seen.to_s.bold} new"
-
-    listing_page_stats
   end
 
   sig do
-    params(job_type: Symbol, post: Domain::Fa::Post, submission: T.untyped).void
+    params(fa_ids: T::Array[Integer]).returns(T::Array[Domain::Post::FaPost])
   end
-  def update_and_save_post_from_listings_page(job_type, post, submission)
-    if job_type == :browse_page
-      post.log_entry_detail["first_browse_page_id"] ||= causing_log_entry&.id
-    elsif job_type == :gallery_page
-      post.log_entry_detail["first_gallery_page_id"] ||= causing_log_entry&.id
-    else
-      fatal_error("unhandled job_type: #{job_type}")
+  def find_or_create_posts_by_fa_ids(fa_ids)
+    posts = Domain::Post::FaPost.where(fa_id: fa_ids).to_a
+    missing_post_fa_ids = fa_ids - posts.map(&:fa_id)
+
+    ReduxApplicationRecord.transaction do
+      missing_post_fa_ids.each do |fa_id|
+        post = Domain::Post::FaPost.create!(fa_id: fa_id)
+        defer_job(Domain::Fa::Job::ScanPostJob, { post: post })
+        posts << post
+      end
     end
 
-    post.creator ||=
-      Domain::Fa::User.find_or_build_from_submission_parser(submission)
-    post.title = submission.title || fatal_error("blank title")
-    post.thumbnail_uri =
-      submission.thumb_path || fatal_error("blank thumb_path")
-    post.save!
+    posts = posts.index_by(&:fa_id)
+    fa_ids.map { |fa_id| posts[fa_id] }
   end
 
   sig do
     params(
-      user: Domain::Fa::User,
-      enqueue_page_scan: T::Boolean,
-      enqueue_gallery_scan: T::Boolean,
-      enqueue_favs_scan: T::Boolean,
-    ).void
+      recent_users: T::Array[Domain::Fa::Parser::UserPageHelper::RecentUser],
+    ).returns(T::Array[Domain::User::FaUser])
   end
-  def enqueue_user_scan(
-    user,
-    enqueue_page_scan: true,
-    enqueue_gallery_scan: true,
-    enqueue_favs_scan: true
-  )
-    users_enqueued_for_page_scan ||= Set.new
-    users_enqueued_for_gallery_scan ||= Set.new
-    users_enqueued_for_favs_scan ||= Set.new
+  def find_or_create_users_by_recent_users(recent_users)
+    users =
+      Domain::User::FaUser.where(url_name: recent_users.map(&:url_name)).to_a
 
-    args =
-      if user.persisted?
-        { user: user }
-      else
-        unless user.url_name
-          logger.warn "user does not have a url name and is not persisted, skipping (#{user.name})"
-          return
+    missing_recent_users =
+      recent_users.reject do |recent_user|
+        users.any? { |u| u.url_name == recent_user.url_name }
+      end
+
+    ReduxApplicationRecord.transaction do
+      missing_recent_users.each do |recent_user|
+        user =
+          Domain::User::FaUser.create!(
+            url_name: recent_user.url_name,
+            name: recent_user.name,
+          )
+        defer_job(Domain::Fa::Job::UserPageJob, { user: user })
+        users << user
+      end
+    end
+
+    users_by_url_name =
+      T.cast(users.index_by(&:url_name), T::Hash[String, Domain::User::FaUser])
+
+    # return user models in the same order as the input
+    recent_users.map { |name| T.must(users_by_url_name[name.url_name]) }
+  end
+
+  sig do
+    params(
+      user: Domain::User::FaUser,
+      response: Scraper::HttpClient::Response,
+    ).returns(T.nilable(Domain::Fa::Parser::Page))
+  end
+  def update_user_from_user_page(user, response)
+    disabled_or_not_found = user_disabled_or_not_found?(user, response)
+    user.scanned_page_at = Time.current
+    user.last_user_page_log_entry = response.log_entry
+    return nil if disabled_or_not_found
+
+    page = Domain::Fa::Parser::Page.new(response.body)
+    return nil unless page.probably_user_page?
+
+    user_page = page.user_page
+    user.state_ok!
+    user.name = user_page.name
+    user.registered_at = user_page.registered_since
+    user.num_pageviews = user_page.num_pageviews
+    user.num_submissions = user_page.num_submissions
+    user.num_comments_recieved = user_page.num_comments_recieved
+    user.num_comments_given = user_page.num_comments_given
+    user.num_journals = user_page.num_journals
+    user.num_favorites = user_page.num_favorites
+    user.num_watched_by = user_page.num_watched_by
+    user.num_watching = user_page.num_watching
+    user.account_status = user_page.account_status&.to_s
+    user.profile_html =
+      user_page.profile_html.encode("UTF-8", invalid: :replace, undef: :replace)
+    if url = user_page.profile_thumb_url
+      enqueue_user_avatar(user, url)
+    end
+
+    page
+  end
+
+  sig { params(user: Domain::User::FaUser, avatar_url_str: String).void }
+  def enqueue_user_avatar(user, avatar_url_str)
+    match = avatar_url_str.match(%r{/([^/]+)\.gif})
+    if match.nil?
+      logger.warn(
+        format_tags("invalid avatar url", make_tag("url", avatar_url_str)),
+      )
+      return
+    end
+
+    expected_url_name = match[1]
+    if user.url_name != expected_url_name
+      logger.warn(
+        format_tags(
+          "invalid avatar url",
+          make_tag("url", avatar_url_str),
+          make_tag("expected", expected_url_name),
+          make_tag("actual", user.url_name),
+        ),
+      )
+      return
+    end
+
+    uri = Addressable::URI.parse(avatar_url_str)
+    uri.scheme ||= "https"
+
+    avatar = user.avatar
+    if avatar.nil? || (avatar.url_str.present? && avatar.url_str != uri.to_s)
+      logger.info(format_tags("creating new avatar", make_tag("url", uri.to_s)))
+      avatar = user.avatars.build(url_str: uri.to_s)
+    elsif avatar.url_str.blank?
+      logger.info(format_tags("updating avatar", make_tag("url", uri.to_s)))
+      avatar.url_str = uri.to_s
+    end
+
+    if avatar.changed?
+      avatar.state_pending!
+      avatar.save!
+      defer_job(Domain::Fa::Job::UserAvatarJob, { avatar: })
+      user.association(:avatar).reload
+    end
+  end
+
+  FoundLink = Scraper::LinkFinder::FoundLink
+
+  sig { params(log_entry: HttpLogEntry).void }
+  def enqueue_jobs_from_found_links(log_entry)
+    return if skip_enqueue_found_links?
+
+    logger.tagged("link-finder") do
+      start_time = Time.now
+      unless PERMITTED_CONTENT_TYPES.any? { |ct|
+               ct.match(log_entry.content_type)
+             }
+        raise("unsupported content type: #{log_entry.content_type}")
+      end
+      document = log_entry.response_bytes || return
+      link_finder =
+        Scraper::LinkFinder.new(T.must(log_entry.uri_host), document)
+      link_finder.logger.level = :error
+      links = link_finder.find_links
+
+      url_names =
+        links.filter_map do |link|
+          link.is_a?(FoundLink::FaUser) ? link.url_name : nil
+        end
+      url_name_to_fa_user =
+        T.let(
+          Domain::User::FaUser.where(url_name: url_names).index_by(&:url_name),
+          T::Hash[String, Domain::User::FaUser],
+        )
+
+      fa_ids =
+        links.filter_map do |link|
+          link.is_a?(FoundLink::FaPost) ? link.fa_id : nil
+        end
+      fa_id_to_fa_post =
+        T.cast(
+          Domain::Post::FaPost.where(fa_id: fa_ids).index_by(&:fa_id),
+          T::Hash[Integer, Domain::Post::FaPost],
+        )
+
+      links
+        .filter_map do |link|
+          if link.is_a?(FoundLink::FaUser) || link.is_a?(FoundLink::FaPost)
+            link
+          else
+            nil
+          end
+        end
+        .each do |link|
+          case link
+          when FoundLink::FaUser
+            url_name = link.url_name
+            user =
+              url_name_to_fa_user[url_name] ||
+                Domain::User::FaUser.create!(url_name:) do |user|
+                  user.name ||= url_name
+                end
+            enqueue_user_scan(user, at_most_one_scan: true)
+          when FoundLink::FaPost
+            fa_id = link.fa_id
+            post =
+              fa_id_to_fa_post[fa_id] ||
+                Domain::Post::FaPost.build(fa_id:) do |post|
+                  post.first_seen_entry_id = log_entry.id
+                end
+            if post.new_record?
+              post.save!
+              defer_job(Domain::Fa::Job::ScanPostJob, { post: })
+            end
+          end
         end
 
-        { url_name: user.url_name }
-      end
+      duration_ms = (1000 * (Time.now - start_time)).to_i.to_s
+      logger.info(format_tags(make_tag("duration", "#{duration_ms} ms")))
+    end
+  rescue StandardError => e
+    logger.error(
+      format_tags(
+        make_tag("error.class", e.class.name),
+        make_tag("error.message", e.message),
+      ),
+    )
+  end
 
-    if enqueue_page_scan && users_enqueued_for_page_scan.add?(user.url_name)
-      if user.due_for_page_scan?
-        logger.info(
-          "enqueue user page job for #{T.must(user.url_name).bold}, " +
-            "last scanned #{time_ago_in_words(user.scanned_page_at)}",
-        )
-        defer_job(Domain::Fa::Job::UserPageJob, args)
-      end
+  DISABLED_PAGE_PATTERNS =
+    T.let(
+      [
+        /User ".+" has voluntarily disabled access/,
+        /The page you are trying to reach is currently pending deletion/,
+      ],
+      T::Array[Regexp],
+    )
+
+  NOT_FOUND_PAGE_PATTERNS =
+    T.let(
+      [
+        /User ".+" was not found in our database\./,
+        /The username ".+" could not be found\./,
+        %r{This user cannot be found.<br/><br/>},
+      ],
+      T::Array[Regexp],
+    )
+
+  module DisabledOrNotFoundResult
+    class Stop < T::Struct
+      include T::Struct::ActsAsComparable
+
+      const :message, String
     end
 
-    if enqueue_gallery_scan &&
-         users_enqueued_for_gallery_scan.add?(user.url_name)
-      if user.due_for_gallery_scan?
-        logger.info(
-          "enqueue user gallery job for #{T.must(user.url_name).bold}, " +
-            "last scanned #{time_ago_in_words(user.scanned_gallery_at)}",
-        )
-        defer_job(Domain::Fa::Job::UserGalleryJob, args)
-      end
-    end
+    class Ok < T::Struct
+      include T::Struct::ActsAsComparable
 
-    if enqueue_favs_scan && users_enqueued_for_favs_scan.add?(user.url_name)
-      if user.due_for_favs_scan?
-        logger.info(
-          "enqueue user favs job for #{T.must(user.url_name).bold}, " +
-            "last scanned #{time_ago_in_words(user.scanned_favs_at)}",
-        )
-        defer_job(Domain::Fa::Job::FavsJob, args)
-      end
+      const :page, Domain::Fa::Parser::Page
     end
   end
 
-  sig { params(enqueue_pri: T.nilable(Symbol)).returns(Integer) }
-  def self.normalize_enqueue_pri(enqueue_pri)
-    case enqueue_pri
-    when :low
-      -5
-    when :high
-      -15
-    else
-      -10
-    end
+  sig do
+    params(
+      user: Domain::User::FaUser,
+      response: Scraper::HttpClient::Response,
+    ).returns(T::Boolean)
   end
-
-  sig { params(fa_id: Integer, enqueue_pri: T.nilable(Symbol)).void }
-  def enqueue_fa_id_scan(fa_id, enqueue_pri = nil)
-    enqueue_pri = self.class.normalize_enqueue_pri(enqueue_pri)
-
-    if @posts_enqueued_for_scan.add?(fa_id)
-      logger.info "enqueue post scan for fa_id #{fa_id}"
-      defer_job(
-        Domain::Fa::Job::ScanPostJob,
-        { fa_id: fa_id },
-        { priority: enqueue_pri },
+  def user_disabled_or_not_found?(user, response)
+    if response.status_code != 200
+      fatal_error(
+        "http #{response.status_code}, log entry #{response.log_entry.id}",
       )
     end
-  end
 
-  sig { params(post: Domain::Fa::Post, enqueue_pri: T.nilable(Symbol)).void }
-  def enqueue_post_scan(post, enqueue_pri = nil)
-    enqueue_pri = self.class.normalize_enqueue_pri(enqueue_pri)
-
-    if @posts_enqueued_for_scan.add?(T.must(post.fa_id))
-      fa_id_str = (post.fa_id || "(nil)").to_s.bold
-      if !post.scanned?
-        logger.info "enqueue post scan for fa_id #{fa_id_str}"
-        defer_job(
-          Domain::Fa::Job::ScanPostJob,
-          { post: post },
-          { priority: enqueue_pri },
-        )
-      elsif !post.have_file?
-        logger.info "enqueue file scan for fa_id #{fa_id_str}"
-        defer_job(
-          Domain::Fa::Job::ScanFileJob,
-          { post: post },
-          { priority: enqueue_pri },
-        )
+    suppress_user_jobs =
+      Kernel.lambda do |user|
+        suppress_deferred_job(Domain::Fa::Job::UserPageJob, { user: })
+        suppress_deferred_job(Domain::Fa::Job::FavsJob, { user: })
+        suppress_deferred_job(Domain::Fa::Job::UserGalleryJob, { user: })
+        suppress_deferred_job(Domain::Fa::Job::UserFollowsJob, { user: })
       end
+
+    if DISABLED_PAGE_PATTERNS.any? { |pattern| response.body =~ pattern }
+      user.state_account_disabled!
+      user.is_disabled = true
+      suppress_user_jobs.call(user)
+      true
+    elsif NOT_FOUND_PAGE_PATTERNS.any? { |pattern| response.body =~ pattern }
+      user.state_error!
+      suppress_user_jobs.call(user)
+      true
+    else
+      false
     end
   end
 end

app/jobs/domain/fa/job/browse_page_job.rb

@@ -1,6 +1,7 @@
 # typed: strict
 class Domain::Fa::Job::BrowsePageJob < Domain::Fa::Job::Base
   queue_as :fa_browse_page
+  MAX_PAGE_NUMBER = T.let(Rails.env.development? ? 2 : 150, Integer)
 
   sig { params(args: T.untyped).void }
   def initialize(*args)
@@ -10,21 +11,31 @@ class Domain::Fa::Job::BrowsePageJob < Domain::Fa::Job::Base
     @total_num_posts_seen = T.let(0, Integer)
   end
 
-  sig { override.params(args: T::Hash[Symbol, T.untyped]).void }
+  sig { override.params(args: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
   def perform(args)
     while true
       break unless scan_browse_page
-      break if @page_number > 150
+
+      if @page_number > MAX_PAGE_NUMBER
+        logger.warn(
+          format_tags(
+            make_tag("page_number", @page_number),
+            make_tag("max_page_number", MAX_PAGE_NUMBER),
+            "exceeded max page number",
+          ),
+        )
+        break
+      end
       @page_number += 1
     end
 
     logger.info(
-      [
-        "[finished]",
-        "[total new: #{@total_num_new_posts_seen.to_s.bold}]",
-        "[total seen: #{@total_num_posts_seen.to_s.bold}]",
-        "[pages: #{@page_number.to_s.bold}]",
-      ].join(" "),
+      format_tags(
+        make_tag("total new posts", @total_num_new_posts_seen),
+        make_tag("total seen posts", @total_num_posts_seen),
+        make_tag("pages", @page_number),
+        "finished",
+      ),
     )
   end
 
@@ -46,18 +57,17 @@ class Domain::Fa::Job::BrowsePageJob < Domain::Fa::Job::Base
       )
     end
 
+    enqueue_jobs_from_found_links(response.log_entry)
+
     page = Domain::Fa::Parser::Page.new(response.body)
     listing_page_stats =
       update_and_enqueue_posts_from_listings_page(
-        :browse_page,
-        page,
-        enqueue_posts_pri: :high,
-        page_desc: "Browse@#{@page_number}",
-        fill_id_gaps: true,
+        ListingPageType::BrowsePage.new(page_number: @page_number),
+        page_parser: page,
       )
 
-    @total_num_new_posts_seen += listing_page_stats.new_seen
-    @total_num_posts_seen += listing_page_stats.total_seen
-    listing_page_stats.new_seen > 0
+    @total_num_new_posts_seen += listing_page_stats.new_posts.count
+    @total_num_posts_seen += listing_page_stats.all_posts.count
+    listing_page_stats.new_posts.count > 0
   end
 end