redux-scraper/app/controllers/domain/fa/api_controller.rb

# typed: true
class Domain::Fa::ApiController < ApplicationController
  skip_before_action :authenticate_user!
  before_action :validate_api_token!

  skip_before_action :verify_authenticity_token,
                     only: %i[enqueue_objects object_statuses]

  skip_before_action :validate_api_token!, only: %i[search_user_names]

  def search_user_names
    name = params[:name]
    limit = (params[:limit] || 5).to_i.clamp(0, 15)
    users = users_for_name(name, limit: limit)
    if !Rails.env.production? && name == "error"
      render status: 500, json: { error: "an error!" }
    else
      render json: { users: users }
    end
  end

  def object_statuses
    fa_ids = (params[:fa_ids] || []).map(&:to_i)
    url_names = (params[:url_names] || [])

    jobs_async =
      GoodJob::Job
        .select(:id, :queue_name, :serialized_params)
        .where(queue_name: "manual", finished_at: nil)
        .where(
          [
            "(serialized_params->'exception_executions' = '{}')",
            "(serialized_params->'exception_executions' is null)",
          ].join(" OR "),
        )
        .load_async

    users_async = Domain::Fa::User.where(url_name: url_names).load_async

    fa_id_to_post =
      Domain::Fa::Post
        .includes(:file)
        .where(fa_id: fa_ids)
        .map { |post| [post.fa_id, post] }
        .to_h

    posts_response = {}
    users_response = {}

    fa_ids.each do |fa_id|
      post = fa_id_to_post[fa_id]

      post_response =
        T.let(
          {
            terminal_state: false,
            seen_at: time_ago_or_never(post&.created_at),
            scanned_at: "never",
            downloaded_at: "never",
          },
          T::Hash[Symbol, T.untyped],
        )

      if post
        post_response[:info_url] = domain_fa_post_url(fa_id: post.fa_id)
        post_response[:scanned_at] = time_ago_or_never(post.scanned_at)

        if post.file.present?
          post_response[:downloaded_at] = time_ago_or_never(
            post.file&.created_at,
          )
          post_response[:state] = "have_file"
          post_response[:terminal_state] = true
        elsif post.scanned?
          post_response[:state] = "scanned_post"
        else
          post_response[:state] = post.state
        end
      else
        post_response[:state] = "not_seen"
      end

      posts_response[fa_id] = post_response
    end

    url_name_to_user = users_async.map { |user| [user.url_name, user] }.to_h

    url_names.each do |url_name|
      user = url_name_to_user[url_name]

      if user
        user_response = {
          created_at: time_ago_or_never(user.created_at),
          scanned_gallery_at: time_ago_or_never(user.scanned_gallery_at),
          scanned_page_at: time_ago_or_never(user.scanned_page_at),
        }
        states = []
        states << "page" unless user.due_for_page_scan?
        states << "gallery" unless user.due_for_gallery_scan?
        states << "seen" if states.empty?

        user_response[:state] = states.join(",")

        if user.scanned_gallery_at && user.scanned_page_at
          user_response[:terminal_state] = true
        end
      else
        user_response = { state: "not_seen", terminal_state: false }
      end
      users_response[url_name] = user_response
    end

    queue_depths = Hash.new { |hash, key| hash[key] = 0 }

    jobs_async.each do |job|
      queue_depths[job.serialized_params["job_class"]] += 1
    end

    queue_depths =
      queue_depths
        .map do |key, value|
          [
            key
              .delete_prefix("Domain::Fa::Job::")
              .split("::")
              .last
              .underscore
              .delete_suffix("_job")
              .gsub("_", " "),
            value,
          ]
        end
        .to_h

    render json: {
             posts: posts_response,
             users: users_response,
             queues: {
               total_depth: queue_depths.values.sum,
               depths: queue_depths,
             },
           }
  end

  def enqueue_objects
    @enqueue_counts ||= Hash.new { |h, k| h[k] = 0 }

    fa_ids = (params[:fa_ids] || []).map(&:to_i)
    url_names = (params[:url_names] || [])
    url_names_to_enqueue = Set.new(params[:url_names_to_enqueue] || [])

    fa_id_to_post =
      Domain::Fa::Post
        .includes(:file)
        .where(fa_id: fa_ids)
        .map { |post| [post.fa_id, post] }
        .to_h

    url_name_to_user =
      Domain::Fa::User
        .where(url_name: url_names)
        .map { |user| [user.url_name, user] }
        .to_h

    fa_ids.each do |fa_id|
      post = fa_id_to_post[fa_id]
      defer_post_scan(post, fa_id)
    end

    url_names.each do |url_name|
      user = url_name_to_user[url_name]
      defer_user_scan(user, url_name, url_names_to_enqueue.include?(url_name))
    end

    enqueue_deferred!

    render json: {
             post_scans: @enqueue_counts[Domain::Fa::Job::ScanPostJob],
             post_files: @enqueue_counts[Domain::Fa::Job::ScanFileJob],
             user_pages: @enqueue_counts[Domain::Fa::Job::UserPageJob],
             user_galleries: @enqueue_counts[Domain::Fa::Job::UserGalleryJob],
           }
  end

  def similar_users
    url_name = params[:url_name]
    exclude_url_name = params[:exclude_url_name]

    user = Domain::Fa::User.find_by(url_name: url_name)
    if user.nil?
      render status: 404,
             json: {
               error: "user '#{url_name}' not found",
               error_type: "user_not_found",
             }
      return
    end

    all_similar_users = helpers.similar_users_by_followed(user, limit: 10)

    if all_similar_users.nil?
      render status: 500,
             json: {
               error:
                 "user '#{url_name}' has not had recommendations computed yet",
               error_type: "recs_not_computed",
             }
      return
    end

    all_similar_users = users_list_to_similar_list(all_similar_users)

    not_followed_similar_users = nil
    if exclude_url_name
      exclude_folowed_by_user =
        Domain::Fa::User.find_by(url_name: exclude_url_name)
      not_followed_similar_users =
        if exclude_folowed_by_user.nil?
          # TODO - enqueue a manual UserFollowsJob for this user and have client
          # re-try the request later
          {
            error: "user '#{exclude_url_name}' not found",
            error_type: "exclude_user_not_found",
          }
        elsif exclude_folowed_by_user.scanned_follows_at.nil?
          {
            error:
              "user '#{exclude_url_name}' followers list hasn't been scanned",
            error_type: "exclude_user_not_scanned",
          }
        else
          users_list_to_similar_list(
            helpers.similar_users_by_followed(
              user,
              limit: 10,
              exclude_followed_by: exclude_folowed_by_user,
            ),
          )
        end
    end

    render json: {
             all: all_similar_users,
             not_followed: not_followed_similar_users,
           }
  end

  private

  def get_best_user_page_http_log_entry_for(user)
    for_path =
      proc do |uri_path|
        HttpLogEntry
          .where(
            uri_scheme: "https",
            uri_host: "www.furaffinity.net",
            uri_path: uri_path,
          )
          .order(created_at: :desc)
          .first
          &.response
      end

    for_hle_id =
      proc { |hle_id| hle_id && HttpLogEntry.find_by(id: hle_id)&.response }

    # older versions don't end in a trailing slash
    hle_id = user.log_entry_detail && user.log_entry_detail["last_user_page_id"]
    for_hle_id.call(hle_id) || for_path.call("/user/#{user.url_name}/") ||
      for_path.call("/user/#{user.url_name}")
  end

  def defer_post_scan(post, fa_id)
    if !post || !post.scanned?
      defer_manual(Domain::Fa::Job::ScanPostJob, { fa_id: fa_id }, -17)
    end

    if post && post.file_uri && !post.file.present?
      return(
        defer_manual(
          Domain::Fa::Job::ScanFileJob,
          { post: post },
          -15,
          "static_file",
        )
      )
    end
  end

  def defer_user_scan(user, url_name, highpri)
    if !user || user.due_for_page_scan?
      defer_manual(
        Domain::Fa::Job::UserPageJob,
        { url_name: url_name },
        highpri ? -16 : -6,
      )
      return
    end

    if !user || user.due_for_gallery_scan?
      defer_manual(
        Domain::Fa::Job::UserGalleryJob,
        { url_name: url_name },
        highpri ? -14 : -4,
      )
      return
    end

    false
  end

  def defer_manual(klass, args, priority, queue = "manual")
    @@enqueue_deduper ||= Set.new
    return unless @@enqueue_deduper.add?([klass, args, priority])

    @deferred_jobs ||= []
    @deferred_jobs << [klass, args, priority, queue]
    @enqueue_counts[klass] += 1
  end

  def enqueue_deferred!
    GoodJob::Bulk.enqueue do
      while job = (@deferred_jobs || []).shift
        klass, args, priority, queue = job
        klass.set(priority: priority, queue: queue).perform_later(args)
      end
    end
  end

  def time_ago_or_never(time)
    if time
      helpers.time_ago_in_words(time, include_seconds: true) + " ago"
    else
      "never"
    end
  end

  def users_for_name(name, limit: 10)
    users =
      Domain::Fa::User
        .where(
          [
            "(name ilike :name) OR (url_name ilike :name)",
            { name: "#{ReduxApplicationRecord.sanitize_sql_like(name)}%" },
          ],
        )
        .includes(:avatar)
        .select(:id, :state, :state_detail, :log_entry_detail, :name, :url_name)
        .select(
          "(SELECT COUNT(*) FROM domain_fa_posts WHERE creator_id = domain_fa_users.id) as num_posts",
        )
        .order(name: :asc)
        .limit(limit)

    users.map do |user|
      {
        id: user.id,
        name: user.name,
        url_name: user.url_name,
        thumb: helpers.fa_user_avatar_path(user, thumb: "64-avatar"),
        show_path: domain_fa_user_path(user.url_name),
        # `num_posts` is a manually added column, so we need to use T.unsafe to
        # access it
        num_posts: T.unsafe(user).num_posts,
      }
    end
  end

  def users_list_to_similar_list(users_list)
    users_list.map do |user|
      profile_thumb_url = user.avatar&.file_uri&.to_s
      profile_thumb_url ||
        begin
          profile_page_response = get_best_user_page_http_log_entry_for(user)
          if profile_page_response
            parser =
              Domain::Fa::Parser::Page.new(
                profile_page_response.contents,
                require_logged_in: false,
              )
            profile_thumb_url = parser.user_page.profile_thumb_url
          else
            if user.due_for_follows_scan?
              Domain::Fa::Job::UserFollowsJob.set(
                { priority: -20 },
              ).perform_later({ user: user })
            end
            if user.due_for_page_scan?
              Domain::Fa::Job::UserPageJob.set({ priority: -20 }).perform_later(
                { user: user },
              )
            end
          end
        rescue StandardError
          logger.error("error getting profile_thumb_url: #{$!.message}")
        end

      {
        name: user.name,
        url_name: user.url_name,
        profile_thumb_url: profile_thumb_url,
        url: "https://www.furaffinity.net/user/#{user.url_name}/",
      }
    end
  end

  API_TOKENS = {
    "a4eb03ac-b33c-439c-9b51-a834d1c5cf48" => "dymk",
    "56cc81fe-8c00-4436-8981-4580eab00e66" => "taargus",
    "9c38727f-f11d-41de-b775-0effd86d520c" => "xjal",
    "e38c568f-a24d-4f26-87f0-dfcd898a359d" => "fyacin",
    "41fa1144-d4cd-11ed-afa1-0242ac120002" => "soft_fox_lad",
    "9b3cf444-5913-4efb-9935-bf26501232ff" => "syfaro",
  }

  def validate_api_token!
    api_token = request.params[:api_token]
    api_user_name = API_TOKENS[api_token]
    return if api_user_name
    return if VpnOnlyRouteConstraint.new.matches?(request)
    render status: 403, json: { error: "not authenticated" }
  end
end