add typed where possible
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
# typed: strict
|
||||
module ApplicationCable
|
||||
class Channel < ActionCable::Channel::Base
|
||||
end
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: strict
|
||||
module ApplicationCable
|
||||
class Connection < ActionCable::Connection::Base
|
||||
end
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
# typed: true
|
||||
class ApplicationController < ActionController::Base
|
||||
include T::Sig
|
||||
include T::Helpers
|
||||
include Pundit::Authorization
|
||||
|
||||
before_action do
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class BlobEntriesController < ApplicationController
|
||||
skip_before_action :authenticate_user!, only: [:show]
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::E621::PostsController < ApplicationController
|
||||
def show
|
||||
@post = Domain::E621::Post.find_by!(e621_id: params[:e621_id])
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::ApiController < ApplicationController
|
||||
skip_before_action :authenticate_user!
|
||||
before_action :validate_api_token!
|
||||
@@ -279,7 +280,6 @@ class Domain::Fa::ApiController < ApplicationController
|
||||
"static_file",
|
||||
)
|
||||
)
|
||||
return
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
#typed: true
|
||||
|
||||
class Domain::Fa::PostsController < ApplicationController
|
||||
before_action :set_ivfflat_probes!, only: %i[show]
|
||||
before_action :set_domain_fa_post, only: %i[show scan_post]
|
||||
@@ -27,7 +29,7 @@ class Domain::Fa::PostsController < ApplicationController
|
||||
|
||||
# GET /domain/fa/posts/:fa_id/favorites
|
||||
def favorites
|
||||
@post = Domain::Fa::Post.find_by_fa_id!(params[:fa_id])
|
||||
@post = Domain::Fa::Post.find_by!(fa_id: params[:fa_id])
|
||||
end
|
||||
|
||||
def scan_post
|
||||
@@ -40,12 +42,12 @@ class Domain::Fa::PostsController < ApplicationController
|
||||
|
||||
def try_scan_post
|
||||
fa_id = params[:fa_id]&.to_i || raise("need fa_id parameter")
|
||||
post = Domain::Fa::Post.find_by_fa_id(fa_id)
|
||||
post = Domain::Fa::Post.find_by(fa_id: fa_id)
|
||||
enqueued = try_enqueue_post_scan(post, fa_id)
|
||||
|
||||
if post && post.file.present?
|
||||
if post && (file = post.file).present?
|
||||
state_string =
|
||||
"downloaded #{helpers.time_ago_in_words(post.file.created_at, include_seconds: true)} ago"
|
||||
"downloaded #{helpers.time_ago_in_words(file.created_at, include_seconds: true)} ago"
|
||||
elsif post && post.scanned?
|
||||
state_string =
|
||||
"scanned #{helpers.time_ago_in_words(post.scanned_at, include_seconds: true)} ago"
|
||||
@@ -62,7 +64,7 @@ class Domain::Fa::PostsController < ApplicationController
|
||||
enqueued: enqueued,
|
||||
title: post&.title,
|
||||
state: state_string,
|
||||
is_terminal_state: post&.scanned? && post&.file&.present? || false,
|
||||
is_terminal_state: post&.scanned? && post.file&.present? || false,
|
||||
}
|
||||
end
|
||||
|
||||
@@ -130,8 +132,8 @@ class Domain::Fa::PostsController < ApplicationController
|
||||
# Use callbacks to share common setup or constraints between actions.
|
||||
def set_domain_fa_post
|
||||
@post =
|
||||
Domain::Fa::Post.includes(:creator, file: :response).find_by_fa_id!(
|
||||
params[:fa_id],
|
||||
Domain::Fa::Post.includes(:creator, file: :response).find_by!(
|
||||
fa_id: params[:fa_id],
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
#typed: true
|
||||
|
||||
class Domain::Fa::UsersController < ApplicationController
|
||||
before_action :set_ivfflat_probes!, only: %i[show]
|
||||
before_action :set_user, only: %i[show]
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Inkbunny::PostsController < ApplicationController
|
||||
skip_before_action :authenticate_user!, only: %i[show index]
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Inkbunny::UsersController < ApplicationController
|
||||
def show
|
||||
@user = Domain::Inkbunny::User.find_by(name: params[:name])
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Twitter::ApiController < ApplicationController
|
||||
skip_before_action :verify_authenticity_token, only: %i[enqueue_objects]
|
||||
|
||||
@@ -16,7 +17,7 @@ class Domain::Twitter::ApiController < ApplicationController
|
||||
defer_manual(
|
||||
Domain::Twitter::Job::UserTimelineTweetsJob,
|
||||
{ name: name },
|
||||
highpri ? -16 : -6
|
||||
highpri ? -16 : -6,
|
||||
)
|
||||
end
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class GlobalStatesController < ApplicationController
|
||||
before_action :set_global_state, only: %i[edit update destroy]
|
||||
after_action :verify_authorized
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class IndexedPostsController < ApplicationController
|
||||
def index
|
||||
@posts = IndexedPost.all
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class LogEntriesController < ApplicationController
|
||||
def index
|
||||
@uri_filter = Addressable::URI.parse(params[:filter]) if params[:filter]
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class PagesController < ApplicationController
|
||||
skip_before_action :authenticate_user!, only: [:root]
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class UserScriptsController < ApplicationController
|
||||
skip_before_action :authenticate_user!, only: [:get]
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Users::RegistrationsController < Devise::RegistrationsController
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: strict
|
||||
# frozen_string_literal: true
|
||||
|
||||
class Users::SessionsController < Devise::SessionsController
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
# typed: strict
|
||||
module ApplicationHelper
|
||||
end
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module Domain::E621::PostsHelper
|
||||
def icon_asset_for_url(url)
|
||||
domain = extract_domain(url)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module Domain::Fa::PostsHelper
|
||||
def post_state_string(post)
|
||||
if post.have_file?
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module Domain::Fa::UsersHelper
|
||||
def avatar_url(sha256, thumb: "32-avatar")
|
||||
blob_path(HexUtil.bin2hex(sha256), format: "jpg", thumb: thumb)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module IndexablePostsHelper
|
||||
def show_path(indexed_post)
|
||||
case indexed_post.postable_type
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module LogEntriesHelper
|
||||
def is_send_data_content_type?(content_type)
|
||||
is_renderable_image_type?(content_type) ||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module SourceHelper
|
||||
def self.source_name_to_class_name
|
||||
{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class ApplicationJob < ActiveJob::Base
|
||||
include GoodJob::ActiveJobExtensions::Concurrency
|
||||
include HasColorLogger
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::E621::Job::Base < Scraper::JobBase
|
||||
discard_on ActiveJob::DeserializationError
|
||||
def self.http_factory_method
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module Domain::E621::Job
|
||||
class PostsIndexJob < Base
|
||||
queue_as :e621
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module Domain::E621::Job
|
||||
class ScanPostJob < Base
|
||||
queue_as :e621
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module Domain::E621::Job
|
||||
class StaticFileJob < Base
|
||||
queue_as :static_file
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::Job::Base < Scraper::JobBase
|
||||
discard_on ActiveJob::DeserializationError
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Job::BrowsePageJob < Domain::Fa::Job::Base
|
||||
queue_as :fa_browse_page
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Job::FavsJob < Domain::Fa::Job::Base
|
||||
include HasBulkEnqueueJobs
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Job::HomePageJob < Domain::Fa::Job::Base
|
||||
queue_as :fa_browse_page
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::Job::ScanFileJob < Domain::Fa::Job::Base
|
||||
queue_as :static_file
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Job::ScanPostJob < Domain::Fa::Job::Base
|
||||
queue_as :fa_post
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module Domain::Fa::Job
|
||||
class ScanPostUtils
|
||||
def self.find_or_create_by_fa_ids(fa_ids, caused_by_entry: nil)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module Domain::Fa::Job
|
||||
class ScanUserUtils
|
||||
DISABLED_PAGE_PATTERNS = [
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Job::UserAvatarJob < Domain::Fa::Job::Base
|
||||
queue_as :static_file
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
# Gather and record all the users that this user follows
|
||||
# This will be used to create an index of follower -> followed
|
||||
# of a specific user, for recommender training
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::Job::UserGalleryJob < Domain::Fa::Job::Base
|
||||
queue_as :fa_user_gallery
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module Domain::Fa::Job
|
||||
class UserIncrementalJob < Base
|
||||
queue_as :fa_user_page
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Job::UserPageJob < Domain::Fa::Job::Base
|
||||
queue_as :fa_user_page
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Inkbunny::Job::Base < Scraper::JobBase
|
||||
discard_on ActiveJob::DeserializationError
|
||||
queue_as :inkbunny
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module Domain::Inkbunny::Job
|
||||
class FileJob < Base
|
||||
queue_as :static_file
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module Domain::Inkbunny::Job
|
||||
class LatestPostsJob < Base
|
||||
def perform(args)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module Domain::Inkbunny::Job
|
||||
class UpdatePostsJob < Base
|
||||
def perform(args)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module Domain::Inkbunny::Job
|
||||
class UserAvatarJob < Base
|
||||
queue_as :static_file
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module Domain::Inkbunny::Job
|
||||
class UserGalleryJob < Base
|
||||
def perform(args)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Twitter::Job::MediaJob < Domain::Twitter::Job::TwitterJobBase
|
||||
queue_as :static_file
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Twitter::Job::TwitterJobBase < Scraper::JobBase
|
||||
discard_on ActiveJob::DeserializationError
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Twitter::Job::UserTimelineTweetsJob < Domain::Twitter::Job::TwitterJobBase
|
||||
GDLClient = Scraper::GalleryDlClient
|
||||
queue_as :twitter_timeline_tweets
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Scraper::JobBase < ApplicationJob
|
||||
ignore_signature_args :caused_by_entry
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class ColorLogger
|
||||
@quiet = Concurrent::ThreadLocalVar.new { 0 }
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class DbSampler
|
||||
SCHEMA = {
|
||||
::Domain::Fa::User => %i[avatar posts disco follower_joins fav_post_joins],
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module Domain::E621::TagUtil
|
||||
include HasColorLogger
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module Domain::Fa::HasCountFailedInQueue
|
||||
extend ActiveSupport::Concern
|
||||
included do
|
||||
@@ -7,13 +8,13 @@ module Domain::Fa::HasCountFailedInQueue
|
||||
queue_name: queue_name,
|
||||
finished_at: nil,
|
||||
performed_at: nil,
|
||||
error: nil
|
||||
error: nil,
|
||||
)
|
||||
.where(
|
||||
[
|
||||
"(serialized_params->'exception_executions' = '{}')",
|
||||
"(serialized_params->'exception_executions' is null)"
|
||||
].join(" OR ")
|
||||
"(serialized_params->'exception_executions' is null)",
|
||||
].join(" OR "),
|
||||
)
|
||||
.count
|
||||
end
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Parser::Base
|
||||
attr_reader :page_version
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Parser::ListedSubmissionParserHelper
|
||||
attr_accessor :debug
|
||||
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
# typed: strict
|
||||
class Domain::Fa::Parser::NotLoggedInError < StandardError
|
||||
end
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "nokogiri"
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Parser::SubmissionParserHelper < Domain::Fa::Parser::Base
|
||||
VERSION_0 = Domain::Fa::Parser::Page::VERSION_0
|
||||
VERSION_1 = Domain::Fa::Parser::Page::VERSION_1
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Fa::Parser::UserListParserHelper
|
||||
User = Struct.new(:name, :url_name, :href, keyword_init: true)
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::Parser::UserPageHelper < Domain::Fa::Parser::Base
|
||||
VERSION_0 = Domain::Fa::Parser::Page::VERSION_0
|
||||
VERSION_1 = Domain::Fa::Parser::Page::VERSION_1
|
||||
@@ -14,7 +15,7 @@ class Domain::Fa::Parser::UserPageHelper < Domain::Fa::Parser::Base
|
||||
"!", # suspended
|
||||
"-", # banned (permanent)
|
||||
"@", # admin
|
||||
"∞" # deceased
|
||||
"∞", # deceased
|
||||
]
|
||||
|
||||
def name
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::PostEnqueuer
|
||||
include HasBulkEnqueueJobs
|
||||
include HasColorLogger
|
||||
@@ -37,13 +38,13 @@ class Domain::Fa::PostEnqueuer
|
||||
if already_enqueued <= @low_water_mark
|
||||
to_enqueue = @high_water_mark - already_enqueued
|
||||
logger.info(
|
||||
"enqueuing #{to_enqueue.to_s.bold} more posts - #{already_enqueued.to_s.bold} already enqueued"
|
||||
"enqueuing #{to_enqueue.to_s.bold} more posts - #{already_enqueued.to_s.bold} already enqueued",
|
||||
)
|
||||
rows =
|
||||
measure(
|
||||
proc do |p|
|
||||
p && "gathered #{p.length.to_s.bold} posts" || "gathering posts..."
|
||||
end
|
||||
end,
|
||||
) do
|
||||
to_enqueue
|
||||
.times
|
||||
@@ -77,7 +78,7 @@ class Domain::Fa::PostEnqueuer
|
||||
else
|
||||
logger.info(
|
||||
"#{already_enqueued.to_s.bold} already enqueued (max #{@high_water_mark.to_s.bold}) - " +
|
||||
"waiting to fall below #{@low_water_mark.to_s.bold}"
|
||||
"waiting to fall below #{@low_water_mark.to_s.bold}",
|
||||
)
|
||||
:sleep
|
||||
end
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::SqliteExporter
|
||||
include HasMeasureDuration
|
||||
|
||||
@@ -16,12 +17,12 @@ class Domain::Fa::SqliteExporter
|
||||
%w[num_comments_given int],
|
||||
%w[num_journals int],
|
||||
%w[num_favorites int],
|
||||
%w[registered_at text]
|
||||
%w[registered_at text],
|
||||
],
|
||||
indexes: [{ on: "id", unique: true }, { on: "url_name", unique: true }],
|
||||
batch_size: 512,
|
||||
# format registered_at column
|
||||
each_row: ->(row) { row[11] = row[11]&.iso8601 }
|
||||
each_row: ->(row) { row[11] = row[11]&.iso8601 },
|
||||
},
|
||||
fa_follows: {
|
||||
model: Domain::Fa::Follow,
|
||||
@@ -29,9 +30,9 @@ class Domain::Fa::SqliteExporter
|
||||
indexes: [{ on: %w[follower_id followed_id], unique: true }],
|
||||
fk: {
|
||||
follower_id: %w[fa_users id],
|
||||
followed_id: %w[fa_users id]
|
||||
followed_id: %w[fa_users id],
|
||||
},
|
||||
batch_size: 4096
|
||||
batch_size: 4096,
|
||||
},
|
||||
fa_favs: {
|
||||
model: Domain::Fa::Fav,
|
||||
@@ -39,9 +40,9 @@ class Domain::Fa::SqliteExporter
|
||||
indexes: [{ on: %w[user_id post_id], unique: true }],
|
||||
fk: {
|
||||
user_id: %w[fa_users id],
|
||||
post_id: %w[fa_posts id]
|
||||
post_id: %w[fa_posts id],
|
||||
},
|
||||
batch_size: 4096
|
||||
batch_size: 4096,
|
||||
},
|
||||
fa_posts: {
|
||||
model: Domain::Fa::Post.where("file_url_str is not null"),
|
||||
@@ -58,16 +59,16 @@ class Domain::Fa::SqliteExporter
|
||||
%w[num_views int],
|
||||
%w[num_comments int],
|
||||
%w[num_favorites int],
|
||||
%w[posted_at text]
|
||||
%w[posted_at text],
|
||||
],
|
||||
batch_size: 4096,
|
||||
indexes: [{ on: "id", unique: true }, { on: "fa_id", unique: true }],
|
||||
fk: {
|
||||
creator_id: %w[fa_users id]
|
||||
creator_id: %w[fa_users id],
|
||||
},
|
||||
# format posted_at column
|
||||
each_row: ->(row) { row[12] = row[12]&.iso8601 }
|
||||
}
|
||||
each_row: ->(row) { row[12] = row[12]&.iso8601 },
|
||||
},
|
||||
}
|
||||
|
||||
def initialize(db, sample, tables)
|
||||
@@ -87,14 +88,14 @@ class Domain::Fa::SqliteExporter
|
||||
measure(
|
||||
proc do |num|
|
||||
num && "dumped #{table}, #{num} rows" || "dumping #{table}..."
|
||||
end
|
||||
end,
|
||||
) do
|
||||
dump_table_common(
|
||||
table: table,
|
||||
model: config[:model],
|
||||
columns: config[:columns],
|
||||
batch_size: config[:batch_size],
|
||||
each_row: config[:each_row]
|
||||
each_row: config[:each_row],
|
||||
)
|
||||
end
|
||||
end
|
||||
@@ -133,7 +134,7 @@ class Domain::Fa::SqliteExporter
|
||||
fk.map do |name, foreign|
|
||||
foreign_table, foreign_column = foreign
|
||||
"foreign key (#{name}) references #{foreign_table}(#{foreign_column})"
|
||||
end
|
||||
end,
|
||||
].flatten.join(",\n")
|
||||
|
||||
sql = <<-SQL
|
||||
@@ -195,7 +196,7 @@ class Domain::Fa::SqliteExporter
|
||||
"[#{(num_models / dump_duration).round(0).to_s.bold}/sec] " +
|
||||
"[load: #{load_duration.round(2).to_s.bold} sec] " +
|
||||
"[map: #{map_duration.round(2).to_s.bold} sec] " +
|
||||
"[insert: #{insert_duration.round(2).to_s.bold} sec]"
|
||||
"[insert: #{insert_duration.round(2).to_s.bold} sec]",
|
||||
)
|
||||
num_models
|
||||
ensure
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::UserEnqueuer
|
||||
include HasBulkEnqueueJobs
|
||||
include HasColorLogger
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::UserFactorCalculator
|
||||
include HasMeasureDuration
|
||||
|
||||
@@ -12,7 +13,7 @@ class Domain::Fa::UserFactorCalculator
|
||||
measure(
|
||||
proc do |r|
|
||||
r && "loaded #{r.length.to_s.bold} follows" || "loading follows"
|
||||
end
|
||||
end,
|
||||
) do
|
||||
Domain::Fa::Follow
|
||||
.all
|
||||
@@ -53,13 +54,13 @@ class Domain::Fa::UserFactorCalculator
|
||||
.each_slice(20_000) do |chunk|
|
||||
total += chunk.size
|
||||
measure(
|
||||
" -> write #{chunk.size.to_s.bold} factors - (#{total.to_s.bold} total)"
|
||||
" -> write #{chunk.size.to_s.bold} factors - (#{total.to_s.bold} total)",
|
||||
) do
|
||||
Domain::Fa::UserFactor.upsert_all(
|
||||
chunk,
|
||||
unique_by: :user_id,
|
||||
update_only: factors_col_name,
|
||||
returning: :id
|
||||
returning: :id,
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class ForkFuture
|
||||
def initialize(&block)
|
||||
read, write = ::IO.pipe
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
module HasBulkEnqueueJobs
|
||||
extend ActiveSupport::Concern
|
||||
included do
|
||||
@@ -16,7 +17,7 @@ module HasBulkEnqueueJobs
|
||||
ReduxApplicationRecord.transaction do
|
||||
existing_keys =
|
||||
GoodJob::Job.where(concurrency_key: key_to_job.keys).pluck(
|
||||
:concurrency_key
|
||||
:concurrency_key,
|
||||
)
|
||||
existing_keys.each { |key| key_to_job.delete(key) }
|
||||
GoodJob::Bulk.enqueue(key_to_job.values)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
require "active_support/concern"
|
||||
|
||||
module HasColorLogger
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module HasMeasureDuration
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class HexUtil
|
||||
def self.hex2bin(str)
|
||||
[str].pack("H*")
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
# typed: strict
|
||||
module LogWatcher
|
||||
end
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Scraper::ClientFactory
|
||||
@http_clients = Concurrent::ThreadLocalVar.new() { {} }
|
||||
@gallery_dl_clients = Concurrent::ThreadLocalVar.new(nil)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Scraper::CurlHttpPerformer
|
||||
Response =
|
||||
Struct.new(:response_code, :response_headers, :response_time_ms, :body)
|
||||
@@ -59,7 +60,7 @@ class Scraper::CurlHttpPerformer
|
||||
"#{self.class.humansize(body_str.size).to_s.bold}",
|
||||
(response_time_ms.to_s + "ms").light_blue.bold + "]",
|
||||
cf_cache_status,
|
||||
"#{url.bold}"
|
||||
"#{url.bold}",
|
||||
].reject(&:nil?).join(" ")
|
||||
|
||||
Response.new(response_code, response_headers, response_time_ms, body_str)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Scraper::E621HttpClientConfig < Scraper::HttpClientConfig
|
||||
DEFAULT_ALLOWED_DOMAINS = %w[e621.net *.e621.net]
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Scraper::FaHttpClientConfig < Scraper::HttpClientConfig
|
||||
DEFAULT_ALLOWED_DOMAINS = %w[*.furaffinity.net *.facdn.net ipinfo.io]
|
||||
UUID_PATTERN =
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Scraper::GalleryDlClient
|
||||
include HasColorLogger
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Scraper::HttpClient
|
||||
include HasColorLogger
|
||||
attr_reader :config
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Scraper::HttpClientConfig
|
||||
include HasColorLogger
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Scraper::InkbunnyHttpClientConfig < Scraper::HttpClientConfig
|
||||
DEFAULT_ALLOWED_DOMAINS = %w[inkbunny.net *.ib.metapix.net]
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Scraper::LinkFinder
|
||||
include HasColorLogger
|
||||
|
||||
@@ -8,7 +9,7 @@ class Scraper::LinkFinder
|
||||
|
||||
RAW_REGEXES = [
|
||||
URI.regexp,
|
||||
%r{(?:(?:https?://)?(?:[-\w\.]+)\.\w+(?:/(?:[\w\-/_\.…]*(?:\?\S+)?)?))}
|
||||
%r{(?:(?:https?://)?(?:[-\w\.]+)\.\w+(?:/(?:[\w\-/_\.…]*(?:\?\S+)?)?))},
|
||||
]
|
||||
|
||||
def find_links
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Scraper::TwitterHttpClientConfig < Scraper::HttpClientConfig
|
||||
DEFAULT_ALLOWED_DOMAINS = %w[*.twimg.com ipinfo.io]
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Sha256AttributeMapper
|
||||
def self.map_to(attr_value)
|
||||
HexUtil.bin2hex(attr_value)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class VpnOnlyRouteConstraint
|
||||
def matches?(request)
|
||||
if request.params[:force_vpn_off] == "1"
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: strict
|
||||
class ApplicationMailer < ActionMailer::Base
|
||||
default from: "from@example.com"
|
||||
layout "mailer"
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class BlobEntry < ReduxApplicationRecord
|
||||
self.table_name = "blob_entries_p"
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class BlobFile < ReduxApplicationRecord
|
||||
self.table_name = "blob_files"
|
||||
|
||||
@@ -18,7 +19,7 @@ class BlobFile < ReduxApplicationRecord
|
||||
self.primary_key = :sha256
|
||||
EMPTY_FILE_SHA256 =
|
||||
HexUtil.hex2bin(
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
)
|
||||
|
||||
enum :version, { v1: 1 }
|
||||
@@ -30,7 +31,7 @@ class BlobFile < ReduxApplicationRecord
|
||||
length: {
|
||||
minimum: 0,
|
||||
allow_nil: false,
|
||||
message: "can't be nil"
|
||||
message: "can't be nil",
|
||||
}
|
||||
|
||||
before_save do
|
||||
@@ -59,7 +60,7 @@ class BlobFile < ReduxApplicationRecord
|
||||
sha256: blob_entry.sha256,
|
||||
content_type: blob_entry.content_type,
|
||||
content_bytes: blob_entry.contents,
|
||||
created_at: blob_entry.created_at
|
||||
created_at: blob_entry.created_at,
|
||||
)
|
||||
end
|
||||
|
||||
@@ -93,8 +94,8 @@ class BlobFile < ReduxApplicationRecord
|
||||
self.version,
|
||||
*self.class.path_segments(
|
||||
FILE_PATH_PATTERNS[self.version.to_sym],
|
||||
HexUtil.bin2hex(self.sha256)
|
||||
)
|
||||
HexUtil.bin2hex(self.sha256),
|
||||
),
|
||||
]
|
||||
file_path_parts.join("/")
|
||||
end
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
module ImmutableModel
|
||||
extend ActiveSupport::Concern
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::User < ReduxApplicationRecord
|
||||
self.table_name = "domain_fa_users"
|
||||
include Pundit::Authorization
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Fa::UserAvatar < ReduxApplicationRecord
|
||||
self.table_name = "domain_fa_user_avatars"
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: strict
|
||||
class Domain::Fa::UserFactor < ReduxApplicationRecord
|
||||
self.table_name = "domain_fa_user_factors"
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class Domain::Twitter::Media < ReduxApplicationRecord
|
||||
self.table_name = "domain_twitter_medias"
|
||||
self.primary_key = :id
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: strict
|
||||
class Domain::Twitter::Tweet < ReduxApplicationRecord
|
||||
self.table_name = "domain_twitter_tweets"
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class Domain::Twitter::User < ReduxApplicationRecord
|
||||
self.table_name = "domain_twitter_users"
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class FlatSstEntry < ReduxApplicationRecord
|
||||
self.primary_key = :key
|
||||
# key - bytea
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
class GlobalState < ReduxApplicationRecord
|
||||
validates :key, presence: true, uniqueness: true
|
||||
validates :value, presence: true
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: true
|
||||
class HttpLogEntryHeader < ReduxApplicationRecord
|
||||
validates_presence_of(:sha256)
|
||||
validates :sha256, length: { is: 32 }
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
# typed: false
|
||||
require "xdiff"
|
||||
require "digest"
|
||||
|
||||
@@ -25,7 +26,7 @@ class LogStoreSstEntry < ReduxApplicationRecord
|
||||
contents_digest = Digest::SHA256.digest(contents)
|
||||
if contents_digest != key
|
||||
raise RuntimeError(
|
||||
"digest mismatch: #{HexUtil.bin2hex(contents_digest)} != #{hex_key}"
|
||||
"digest mismatch: #{HexUtil.bin2hex(contents_digest)} != #{hex_key}",
|
||||
)
|
||||
end
|
||||
|
||||
@@ -46,7 +47,7 @@ class LogStoreSstEntry < ReduxApplicationRecord
|
||||
patched_value = XDiff.patch(parent_value, patch_value)
|
||||
if patched_value.length != value_length
|
||||
raise RuntimeError.new(
|
||||
"length mismatch: #{patched_value.length} != #{value_length}"
|
||||
"length mismatch: #{patched_value.length} != #{value_length}",
|
||||
)
|
||||
end
|
||||
else
|
||||
@@ -60,7 +61,7 @@ class LogStoreSstEntry < ReduxApplicationRecord
|
||||
value_digest = Digest::SHA256.digest(patched_value)
|
||||
if value_digest != key
|
||||
raise RuntimeError(
|
||||
"digest mismatch: #{HexUtil.bin2hex(value_digest)} != #{hex_key}"
|
||||
"digest mismatch: #{HexUtil.bin2hex(value_digest)} != #{hex_key}",
|
||||
)
|
||||
end
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user