dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move entirely to BlobFile

Browse Source
This commit is contained in:
Dylan Knutson
2025-02-25 19:51:58 +00:00
parent 5ad0828aa8
commit a296688123
38 changed files with 177 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
app/assets/images/refurrer-logo-icon.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.2 KiB

BIN
app/assets/images/refurrer-logo-md.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 17 KiB

71
app/controllers/blob_entries_controller.rb
View File

@@ -1,7 +1,8 @@
# typed: true
# typed: strict
class BlobEntriesController < ApplicationController
skip_before_action :authenticate_user!, only: [:show]
sig { void }
def show
thumb = params[:thumb]
raise("invalid thumb #{thumb}") if thumb.present? && !thumb_params(thumb)
@@ -15,14 +16,10 @@ class BlobEntriesController < ApplicationController
etag += "-#{thumb}" if thumb
return unless stale?(last_modified: Time.at(0), strong_etag: etag)
sha256bin = HexUtil.hex2bin(sha256)
if show_blob_file(sha256, thumb)
return
elsif BlobFile.migrate_sha256!(sha256) && show_blob_file(sha256, thumb)
return
elsif blob_entry = BlobEntry.find_by(sha256: sha256bin)
show_blob_entry(blob_entry, thumb)
else
raise ActiveRecord::RecordNotFound
end
@@ -37,13 +34,16 @@ class BlobEntriesController < ApplicationController
filename = T.must(filename[..File.extname(filename).length])
filename += ".jpeg"
cache_key = "vips:thumbnail:#{sha256}:#{thumb}"
width, height = thumb_params(thumb)
thumb_params = thumb_params(thumb)
if thumb_params.nil?
raise ActionController::BadRequest.new("invalid thumbnail: #{thumb}")
end
width, height = thumb_params
thumb_data =
Rack::MiniProfiler.step("vips: load from cache") do
Rails
.cache
.fetch(cache_key, expires_in: 1.hour) do
.fetch(cache_key, expires_in: 1.day) do
blob_file = BlobFile.find_by(sha256: HexUtil.hex2bin(sha256))
if blob_file
content_type =
@@ -134,58 +134,7 @@ class BlobEntriesController < ApplicationController
end
end
sig { params(blob_entry: BlobEntry, thumb: T.nilable(String)).void }
def show_blob_entry(blob_entry, thumb)
sha256 = HexUtil.bin2hex(T.must(blob_entry.sha256))
content_type = blob_entry.content_type || "application/octet-stream"
# images, videos, etc
if helpers.is_send_data_content_type?(content_type)
if !thumb.blank? && helpers.is_renderable_image_type?(content_type)
filename = "thumb-#{thumb}-#{sha256}"
filename = T.must(filename[..File.extname(filename).length])
filename += ".jpeg"
width, height = thumb_params(thumb)
image =
T.unsafe(Vips::Image).thumbnail_buffer(
blob_entry.contents,
width,
height: height,
)
resized_image_contents = image.jpegsave_buffer
send_data(
resized_image_contents,
type: "image/jpg",
disposition: "inline",
filename: filename,
)
elsif !thumb.blank? && helpers.is_renderable_video_type?(content_type)
render plain: "no thumbnail"
else
ext = helpers.ext_for_content_type(content_type)
ext = ".#{ext}" if ext
send_data(
T.must(blob_entry.contents),
type: content_type,
disposition: "inline",
filename: "data#{ext}",
)
end
elsif content_type =~ %r{text/plain}
render plain: blob_entry.contents
elsif content_type.starts_with? "text/html"
render html: blob_entry.contents&.html_safe
elsif content_type.starts_with? "application/json"
pretty_json =
JSON.pretty_generate(JSON.parse(T.must(blob_entry.contents)))
render html:
"<html><body><pre>#{pretty_json}</pre></body></html>".html_safe
else
render plain: "no renderer for #{content_type}"
end
end
sig { params(thumb: String).returns(T.nilable([Integer, Integer])) }
def thumb_params(thumb)
case thumb
when "32-avatar"
@@ -198,6 +147,8 @@ class BlobEntriesController < ApplicationController
[400, 300]
when "medium"
[800, 600]
when "content-container"
[2048, 2048]
end
end
end

2
app/controllers/log_entries_controller.rb
View File

@@ -56,7 +56,7 @@ class LogEntriesController < ApplicationController
HttpLogEntry
.joins(:response)
.includes(:response)
.select("http_log_entries.*, blob_entries_p.size")
.select("http_log_entries.*, blob_files.size_bytes")
.find_each(batch_size: 100, order: :desc) do |log_entry|
break if log_entry.created_at < @time_window.ago
@last_window_count += 1

2
app/helpers/log_entries_helper.rb
View File

@@ -87,7 +87,7 @@ module LogEntriesHelper
sig { params(log_entry: HttpLogEntry).returns(T.nilable(String)) }
def render_msword_content(log_entry)
docx_body = log_entry.response&.contents
docx_body = log_entry.response_bytes
return nil if docx_body.blank?
# Invoke abiword to convert doc / docx to html
# Run abiword conversion with pipes

2
app/jobs/domain/fa/job/base.rb
View File

@@ -419,7 +419,7 @@ class Domain::Fa::Job::Base < Scraper::JobBase
}
raise("unsupported content type: #{log_entry.content_type}")
end
document = log_entry.response&.contents || return
document = log_entry.response_bytes || return
link_finder =
Scraper::LinkFinder.new(T.must(log_entry.uri_host), document)
link_finder.logger.level = :error

2
app/jobs/domain/twitter/job/media_job.rb
View File

@@ -17,7 +17,7 @@ class Domain::Twitter::Job::MediaJob < Domain::Twitter::Job::TwitterJobBase
response = http_client.get(@media.url_str)
logger.debug "#{HexUtil.humansize(T.must(response.log_entry.response&.size))} / " +
logger.debug "#{HexUtil.humansize(T.must(response.log_entry.response&.size_bytes))} / " +
"#{response.log_entry.content_type} / " +
"#{response.log_entry.response_time_ms} ms"

2
app/lib/domain/e621/tag_util.rb
View File

@@ -8,7 +8,7 @@ class Domain::E621::TagUtil
sig do
params(
post_json: T::Hash[String, T.untyped],
caused_by_entry: T.nilable(ReduxApplicationRecord),
caused_by_entry: T.nilable(HttpLogEntry),
force_update: T::Boolean,
).returns(Domain::Post::E621Post)
end

2
app/lib/domain/e621/task/fix_e621_post_missing_files.rb
View File

@@ -11,7 +11,7 @@ class Domain::E621::Task::FixE621PostMissingFiles
return
end
index_page_contents = index_page.response&.contents
index_page_contents = index_page.response_bytes
unless index_page_contents
logger.error("no index page contents for post #{post.id}")
return

4
app/lib/fa_backfill_favs.rb
View File

@@ -48,10 +48,10 @@ class FaBackfillFavs
)
entries.each do |entry|
response = T.let(entry.response, T.nilable(BlobEntry))
response = T.let(entry.response, T.nilable(BlobFile))
next unless response
contents = T.let(response.contents, T.nilable(String))
contents = T.let(response.content_bytes, T.nilable(String))
next unless contents
page =

19
app/lib/scraper/http_client.rb
View File

@@ -79,7 +79,7 @@ class Scraper::HttpClient
from_cache ? "[" + "CACHED".light_green.bold + "]" : nil,
"[entry #{log_entry.id.to_s.bold} /",
"GET #{response_code_colorized} /",
"#{HexUtil.humansize(T.must(response_blob_entry.bytes_stored)).bold} / #{HexUtil.humansize(T.must(response_blob_entry.size)).bold}]",
"#{HexUtil.humansize(T.must(response_blob_entry.size_bytes)).bold}]",
"[#{response_time_ms.to_s.bold} ms / #{total_time_ms.to_s.bold} ms]",
log_entry.uri.to_s.black,
].compact.join(" "),
@@ -112,7 +112,7 @@ class Scraper::HttpClient
if use_http_cache
if (cached_response = HttpLogEntry.find_by_uri(uri)) &&
(status_code = cached_response.status_code) &&
(body = cached_response.response&.contents)
(body = cached_response.response&.content_bytes)
print_request_performed_log_line(
from_cache: true,
log_entry: cached_response,
@@ -175,10 +175,9 @@ class Scraper::HttpClient
total_time_ms = -1
begin
response_blob_entry =
BlobEntry.find_or_build(
content_type: content_type,
contents: response_body,
)
BlobFile.find_or_initialize_from_contents(response_body) do |blob_file|
blob_file.content_type = content_type
end
scrubbed_uri = @config.scrub_stored_uri(uri)
log_entry =
@@ -200,9 +199,6 @@ class Scraper::HttpClient
},
)
# double write blob_file while migrating
response_blob_file =
BlobFile.find_or_initialize_from_blob_entry(response_blob_entry)
total_time_ms = ((Time.now - requested_at) * 1000).to_i
Scraper::Metrics::HttpClientMetrics.observe_request_finish(
@@ -215,11 +211,6 @@ class Scraper::HttpClient
)
log_entry.save!
begin
response_blob_file.save unless response_blob_file.persisted?
rescue => e
puts "error saving blob file #{HexUtil.bin2hex(T.must(response_blob_file.sha256))}: #{e}"
end
rescue StandardError
retries += 1
retry if retries < 2

19
app/models/blob_file.rb
View File

@@ -68,6 +68,23 @@ class BlobFile < ReduxApplicationRecord
end
end
sig do
params(
contents: String,
block: T.proc.params(blob_file: BlobFile).void,
).returns(BlobFile)
end
def self.find_or_initialize_from_contents(contents, &block)
sha256 = Digest::SHA256.digest(contents)
BlobFile.migrate_sha256!(sha256) ||
BlobFile.find_or_initialize_by(
sha256: Digest::SHA256.digest(contents),
) do |blob_file|
blob_file.content_bytes = contents
block.call(blob_file)
end
end
sig { params(blob_entry: BlobEntry).returns(BlobFile) }
def self.initialize_from_blob_entry(blob_entry)
BlobFile.new(
@@ -82,6 +99,8 @@ class BlobFile < ReduxApplicationRecord
def self.migrate_sha256!(sha256)
# convert to binary if hex formatted
sha256 = HexUtil.hex2bin(sha256) if sha256.length == 64
blob_file = BlobFile.find_by(sha256: sha256)
return blob_file if blob_file
blob_entry = BlobEntry.find_by(sha256: sha256)
return nil unless blob_entry
blob_file = BlobFile.find_or_initialize_from_blob_entry(blob_entry)

2
app/models/domain/fa/post.rb
View File

@@ -153,7 +153,7 @@ class Domain::Fa::Post < ReduxApplicationRecord
pa = posted_at
return pa if pa
begin
contents = guess_last_submission_page&.response&.contents
contents = guess_last_submission_page&.response_bytes
if contents
parser = Domain::Fa::Parser::Page.new(contents)
parser.submission.posted_date if parser.probably_submission?

4
app/models/domain/fa/user_avatar.rb
View File

@@ -17,12 +17,12 @@ class Domain::Fa::UserAvatar < ReduxApplicationRecord
belongs_to :user, class_name: "::Domain::Fa::User"
belongs_to :file,
foreign_key: :file_sha256,
class_name: "::BlobEntry",
class_name: "::BlobFile",
optional: true
belongs_to :log_entry, class_name: "::HttpLogEntry", optional: true
def file
@file_model ||= BlobEntry.ensure(file_sha256) if file_sha256
@file_model ||= BlobFile.migrate_sha256!(file_sha256) if file_sha256
end
def file_uri

2
app/models/domain/post/fa_post.rb
View File

@@ -171,7 +171,7 @@ class Domain::Post::FaPost < Domain::Post
pa = super
return pa if pa
begin
contents = guess_last_submission_log_entry&.response&.contents
contents = guess_last_submission_log_entry&.response_bytes
if contents
parser = Domain::Fa::Parser::Page.new(contents)
parser.submission.posted_date if parser.probably_submission?

3
app/models/domain/post/inkbunny_post.rb
View File

@@ -120,8 +120,7 @@ class Domain::Post::InkbunnyPost < Domain::Post
# TODO - this is a mess, need to backfill descriptions on inkbunny posts
description || self.ib_detail_raw&.dig("submission_json", "description") ||
begin
contents = self.deep_update_log_entry&.response&.contents
if contents.present?
if contents = self.deep_update_log_entry&.response_bytes
json = JSON.parse(contents)
json["submissions"]
.find { |s| s["submission_id"] == self.ib_id&.to_s }

13
app/models/domain/post_file.rb
View File

@@ -6,7 +6,7 @@ class Domain::PostFile < ReduxApplicationRecord
belongs_to :post, foreign_key: :post_id, class_name: "::Domain::Post"
belongs_to :log_entry, class_name: "::HttpLogEntry", optional: true
belongs_to :blob,
class_name: "::BlobEntry",
class_name: "::BlobFile",
optional: true,
foreign_key: :blob_sha256
@@ -34,6 +34,17 @@ class Domain::PostFile < ReduxApplicationRecord
self.type ||= self.class.name if new_record?
end
sig { returns(T.nilable(BlobFile)) }
def blob
super ||
begin
@blob_file_model = T.let(@blob_file_model, T.nilable(BlobFile))
@blob_file_model ||=
((sha256 = self.blob_sha256) ? BlobFile.migrate_sha256!(sha256) : nil)
@blob_file_model
end
end
sig { params(le: T.nilable(HttpLogEntry)).returns(T.nilable(HttpLogEntry)) }
def log_entry=(le)
self.blob_sha256 ||= le.response_sha256 if le.present?

2
app/models/domain/user/fa_user.rb
View File

@@ -109,7 +109,7 @@ class Domain::User::FaUser < Domain::User
account_status ||
begin
if (hle = guess_last_user_page_log_entry) &&
(response = hle.response) && (contents = response.contents)
(contents = hle.response_bytes)
parser =
Domain::Fa::Parser::Page.new(contents, require_logged_in: false)
parser.user_page.account_status.to_s if parser.probably_user_page?

25
app/models/http_log_entry.rb
View File

@@ -10,7 +10,7 @@ class HttpLogEntry < ReduxApplicationRecord
belongs_to :response,
foreign_key: :response_sha256,
class_name: "::BlobEntry",
class_name: "::BlobFile",
autosave: true
belongs_to :request_headers, class_name: "::HttpLogEntryHeader"
@@ -78,11 +78,24 @@ class HttpLogEntry < ReduxApplicationRecord
sig { returns(T.nilable(Integer)) }
def response_size
if association(:response).loaded?
T.must(self.response).size
else
BlobEntry.where(sha256: response_sha256).pick(:size)
end
response&.size_bytes
end
sig { returns(T.nilable(String)) }
def response_bytes
response&.content_bytes
end
sig { returns(T.nilable(BlobFile)) }
def response
super ||
begin
@response_blob_file ||= T.let(nil, T.nilable(BlobFile))
if sha256 = response_sha256
@response_blob_file ||= BlobFile.migrate_sha256!(sha256)
end
@response_blob_file
end
end
sig { params(uri: T.any(String, Addressable::URI)).void }

4
app/views/layouts/application.html.erb
View File

@@ -12,7 +12,7 @@
window.MiniProfiler.patchesApplied = true;
</script>
<% end %>
<%= favicon_link_tag "refurrer-logo.png" %>
<%= favicon_link_tag "refurrer-logo-icon.png" %>
<%= csrf_meta_tags %>
<%= csp_meta_tag %>
<%= javascript_pack_tag "application-bundle" %>
@@ -25,7 +25,7 @@
<div class="mx-auto max-w-5xl py-6 px-6 sm:px-8 flex items-baseline">
<h1 class="text-4xl sm:text-5xl font-bold text-slate-900">
<%= link_to root_path, class: "flex items-center" do %>
<%= image_tag asset_path("refurrer-logo.png"), class: "w-12 h-12 mr-2" %>
<%= image_tag asset_path("refurrer-logo-md.png"), class: "w-12 h-12 mr-2" %>
<span class="text-4xl sm:text-5xl font-bold text-slate-900">
ReFurrer
</span>

1
app/views/log_entries/renderers/_image.html.erb
View File

@@ -1 +1,2 @@
<% path = blob_path(HexUtil.bin2hex(log_entry.response_sha256), format: "jpg", thumb: "content-container") %>
<img alt="image" src="<%= path %>" class="md:rounded-md" />

22
config/initializers/prometheus_exporter.rb
View File

@@ -40,6 +40,28 @@ else
# connect to a Prometheus server, which we can't rely on in a test environment.
$stderr.puts "PrometheusExporter is disabled in test, console, and rake task environments"
module PrometheusExporter
module Instrumentation
class PeriodicStats
class << self
extend T::Sig
sig do
params(
args: T.untyped,
frequency: T.untyped,
client: T.untyped,
kwargs: T.untyped,
).void
end
def start(*args, frequency:, client: T.unsafe(nil), **kwargs)
end
sig { params(blk: T.proc.void).void }
def worker_loop(&blk)
end
end
end
end
class Client
class RemoteMetric
end

12
sorbet/rbi/dsl/domain/fa/user_avatar.rbi generated
View File

@@ -452,7 +452,7 @@ class Domain::Fa::UserAvatar
end
module GeneratedAssociationMethods
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def build_file(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -461,10 +461,10 @@ class Domain::Fa::UserAvatar
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Fa::User) }
def build_user(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_file(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_file!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -479,10 +479,10 @@ class Domain::Fa::UserAvatar
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Fa::User) }
def create_user!(*args, &blk); end
sig { returns(T.nilable(::BlobEntry)) }
sig { returns(T.nilable(::BlobFile)) }
def file; end
sig { params(value: T.nilable(::BlobEntry)).void }
sig { params(value: T.nilable(::BlobFile)).void }
def file=(value); end
sig { returns(T::Boolean) }
@@ -503,7 +503,7 @@ class Domain::Fa::UserAvatar
sig { returns(T::Boolean) }
def log_entry_previously_changed?; end
sig { returns(T.nilable(::BlobEntry)) }
sig { returns(T.nilable(::BlobFile)) }
def reload_file; end
sig { returns(T.nilable(::HttpLogEntry)) }

12
sorbet/rbi/dsl/domain/post_file.rbi generated
View File

@@ -453,10 +453,10 @@ class Domain::PostFile
end
module GeneratedAssociationMethods
sig { returns(T.nilable(::BlobEntry)) }
sig { returns(T.nilable(::BlobFile)) }
def blob; end
sig { params(value: T.nilable(::BlobEntry)).void }
sig { params(value: T.nilable(::BlobFile)).void }
def blob=(value); end
sig { returns(T::Boolean) }
@@ -465,7 +465,7 @@ class Domain::PostFile
sig { returns(T::Boolean) }
def blob_previously_changed?; end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def build_blob(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -474,10 +474,10 @@ class Domain::PostFile
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Post) }
def build_post(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_blob(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_blob!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -516,7 +516,7 @@ class Domain::PostFile
sig { returns(T::Boolean) }
def post_previously_changed?; end
sig { returns(T.nilable(::BlobEntry)) }
sig { returns(T.nilable(::BlobFile)) }
def reload_blob; end
sig { returns(T.nilable(::HttpLogEntry)) }

12
sorbet/rbi/dsl/domain/post_file/inkbunny_post_file.rbi generated
View File

@@ -454,13 +454,13 @@ class Domain::PostFile::InkbunnyPostFile
module EnumMethodsModule; end
module GeneratedAssociationMethods
sig { returns(T.nilable(::BlobEntry)) }
sig { returns(T.nilable(::BlobFile)) }
def blob; end
sig { params(value: T.nilable(::BlobEntry)).void }
sig { params(value: T.nilable(::BlobFile)).void }
def blob=(value); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def build_blob(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -469,10 +469,10 @@ class Domain::PostFile::InkbunnyPostFile
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Post) }
def build_post(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_blob(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_blob!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -499,7 +499,7 @@ class Domain::PostFile::InkbunnyPostFile
sig { params(value: T.nilable(::Domain::Post)).void }
def post=(value); end
sig { returns(T.nilable(::BlobEntry)) }
sig { returns(T.nilable(::BlobFile)) }
def reload_blob; end
sig { returns(T.nilable(::HttpLogEntry)) }

24
sorbet/rbi/dsl/good_job/job.rbi generated
View File

@@ -526,9 +526,6 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def active_job_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def adapter_class(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def advisory_lock(*args, &blk); end
@@ -550,12 +547,6 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def arel_columns(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def bind_value(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def coalesce_scheduled_at_created_at(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def create_with(*args, &blk); end
@@ -674,9 +665,6 @@ class GoodJob::Job
end
def page(num = nil); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def params_execution_count(*args, &blk); end
sig do
params(
num: Integer
@@ -2176,9 +2164,6 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def active_job_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def adapter_class(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def advisory_lock(*args, &blk); end
@@ -2200,12 +2185,6 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def arel_columns(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def bind_value(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def coalesce_scheduled_at_created_at(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def create_with(*args, &blk); end
@@ -2324,9 +2303,6 @@ class GoodJob::Job
end
def page(num = nil); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def params_execution_count(*args, &blk); end
sig do
params(
num: Integer

12
sorbet/rbi/dsl/http_log_entry.rbi generated
View File

@@ -453,7 +453,7 @@ class HttpLogEntry
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntryHeader) }
def build_request_headers(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def build_response(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntryHeader) }
@@ -483,10 +483,10 @@ class HttpLogEntry
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntryHeader) }
def create_request_headers!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_response(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_response!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntryHeader) }
@@ -501,7 +501,7 @@ class HttpLogEntry
sig { returns(T.nilable(::HttpLogEntryHeader)) }
def reload_request_headers; end
sig { returns(T.nilable(::BlobEntry)) }
sig { returns(T.nilable(::BlobFile)) }
def reload_response; end
sig { returns(T.nilable(::HttpLogEntryHeader)) }
@@ -531,10 +531,10 @@ class HttpLogEntry
sig { void }
def reset_response_headers; end
sig { returns(T.nilable(::BlobEntry)) }
sig { returns(T.nilable(::BlobFile)) }
def response; end
sig { params(value: T.nilable(::BlobEntry)).void }
sig { params(value: T.nilable(::BlobFile)).void }
def response=(value); end
sig { returns(T::Boolean) }

1
sorbet/tapioca/require.rb
View File

@@ -32,5 +32,6 @@ require "prometheus_exporter/client"
require "prometheus_exporter/metric"
require "prometheus_exporter/instrumentation"
require "prometheus_exporter/instrumentation/active_record"
require "prometheus_exporter/instrumentation/periodic_stats"
require "prometheus_exporter/instrumentation/good_job"
require "prometheus_exporter/middleware"

10
spec/factories/blob_file.rb Normal file
View File

@@ -0,0 +1,10 @@
# typed: false
FactoryBot.define do
factory :blob_file do
transient { contents { "test content #{SecureRandom.alphanumeric(10)}" } }
content_bytes { contents }
size_bytes { contents.bytesize }
content_type { "text/plain" }
sha256 { Digest::SHA256.digest(contents) }
end
end

2
spec/factories/domain/fa/users.rb
View File

@@ -9,7 +9,7 @@ FactoryBot.define do
trait :with_avatar do
after(:create) do |user|
create(:domain_fa_user_avatar, user: user, file: create(:blob_entry))
create(:domain_fa_user_avatar, user: user, file: create(:blob_file))
end
end
end

2
spec/factories/http_log_entries.rb
View File

@@ -15,7 +15,7 @@ FactoryBot.define do
performed_by { "direct" }
# Create associated records
association :response, factory: :blob_entry
association :response, factory: :blob_file
association :request_headers, factory: :http_log_entry_header
association :response_headers, factory: :http_log_entry_header

16
spec/helpers/http_client_mock_helpers.rb
View File

@@ -30,11 +30,11 @@ class HttpClientMockHelpers
request_headers: build(:http_log_entry_header),
response_headers: build(:http_log_entry_header),
response:
BlobEntry.find_by(sha256: sha256) ||
BlobFile.find_by(sha256: sha256) ||
build(
:blob_entry,
:blob_file,
content_type: request[:content_type],
content: request[:contents],
contents: request[:contents],
),
)
log_entry.save!
@@ -58,7 +58,7 @@ class HttpClientMockHelpers
logger.info "[mock http client] [#{method}] [#{uri}] [#{opts.inspect.truncate(80)}]"
Scraper::HttpClient::Response.new(
status_code: log_entry.status_code,
body: log_entry.response.contents,
body: log_entry.response.content_bytes,
log_entry: log_entry,
)
end,
@@ -83,11 +83,11 @@ class HttpClientMockHelpers
performed_by: "direct",
response_time_ms: rand(20..100),
response:
BlobEntry.find_by(sha256: sha256) ||
BlobFile.find_by(sha256: sha256) ||
build(
:blob_entry,
:blob_file,
content_type: request[:content_type],
content: request[:contents],
contents: request[:contents],
),
)
[request[:uri], log_entry]
@@ -103,7 +103,7 @@ class HttpClientMockHelpers
logger.info "[mock http client] [get] [#{uri}] [#{opts.inspect.truncate(80)}]"
Scraper::HttpClient::Response.new(
status_code: log_entry.status_code,
body: log_entry.response.contents,
body: log_entry.response.content_bytes,
log_entry: log_entry,
)
end

2
spec/helpers/log_entries_helper_spec.rb
View File

@@ -7,7 +7,7 @@ RSpec.describe LogEntriesHelper, type: :helper do
:http_log_entry,
response:
build(
:blob_entry,
:blob_file,
content_type: "application/msword",
contents:
File.binread(

2
spec/jobs/domain/e621/job/posts_index_job_spec.rb
View File

@@ -23,7 +23,7 @@ describe Domain::E621::Job::PostsIndexJob do
],
)
described_class.perform_now({ caused_by_entry: file })
perform_now({ caused_by_entry: file })
expect(Domain::Post::E621Post.count).to eq(5)
post = Domain::Post::E621Post.find_by(e621_id: 4_247_443)

8
spec/jobs/domain/fa/job/user_avatar_job_spec.rb
View File

@@ -103,8 +103,8 @@ describe Domain::Fa::Job::UserAvatarJob do
:http_log_entry,
response:
create(
:blob_entry,
content: avatar_fixture_file_2,
:blob_file,
contents: avatar_fixture_file_2,
content_type: "image/gif",
),
)
@@ -143,8 +143,8 @@ describe Domain::Fa::Job::UserAvatarJob do
:http_log_entry,
response:
create(
:blob_entry,
content: avatar_fixture_file,
:blob_file,
contents: avatar_fixture_file,
content_type: "image/gif",
),
)

24
spec/lib/fa_backfill_favs_spec.rb
View File

@@ -52,13 +52,7 @@ describe FaBackfillFavs do
requested_at: Time.current,
request_headers: empty_headers,
response_headers: empty_headers,
response:
BlobEntry.create!(
contents: iiszed_html,
content_type: "text/html",
sha256: Digest::SHA256.digest(iiszed_html),
size: iiszed_html.bytesize,
),
response: create(:blob_file, contents: iiszed_html),
)
HttpLogEntry.create!(
@@ -73,13 +67,7 @@ describe FaBackfillFavs do
requested_at: Time.current,
request_headers: empty_headers,
response_headers: empty_headers,
response:
BlobEntry.create!(
contents: renamonzeo_html,
content_type: "text/html",
sha256: Digest::SHA256.digest(renamonzeo_html),
size: renamonzeo_html.bytesize,
),
response: create(:blob_file, contents: renamonzeo_html),
)
HttpLogEntry.create!(
@@ -94,13 +82,7 @@ describe FaBackfillFavs do
requested_at: Time.current,
request_headers: empty_headers,
response_headers: empty_headers,
response:
BlobEntry.create!(
contents: stickiest_html,
content_type: "text/html",
sha256: Digest::SHA256.digest(stickiest_html),
size: stickiest_html.bytesize,
),
response: create(:blob_file, contents: stickiest_html),
)
end

30
spec/lib/scraper/http_client_spec.rb
View File

@@ -24,7 +24,7 @@ describe Scraper::HttpClient do
client =
Scraper::HttpClient.new(
TestHttpClientConfig.new,
SpecUtil.mock_http_performer(:get, "https://example.com")
SpecUtil.mock_http_performer(:get, "https://example.com"),
)
assert_raises(Scraper::HttpClient::InvalidURLError) do
client.get("https://foobar.com")
@@ -39,32 +39,32 @@ describe Scraper::HttpClient do
:get,
"https://www.example.com/",
request_headers: {
"cookie" => ""
"cookie" => "",
},
response_code: 200,
response_time_ms: 15,
response_headers: {
"content-type" => "text/plain"
"content-type" => "text/plain",
},
response_body: "the response " + SpecUtil.random_string(16)
)
response_body: "the response " + SpecUtil.random_string(16),
),
)
client.logger.level = :error
# note the lack of trailing slash - http client should set path to '/'
response = client.get("https://www.example.com")
assert_equal 200, response.status_code
assert_match /the response /, response.body
assert_equal(200, response.status_code)
assert_match(/the response /, response.body)
log_entry = response.log_entry
assert log_entry.persisted?
assert_equal "text/plain", log_entry.content_type
assert_in_delta Time.now, log_entry.requested_at, 50
assert_equal 15, log_entry.response_time_ms
assert_equal "get", log_entry.verb
assert_equal 200, log_entry.status_code
assert_equal "text/plain", log_entry.response.content_type
assert_match /the response/, log_entry.response.contents
assert(log_entry.persisted?)
assert_equal("text/plain", log_entry.content_type)
assert_in_delta(Time.now, log_entry.requested_at, 50)
assert_equal(15, log_entry.response_time_ms)
assert_equal("get", log_entry.verb)
assert_equal(200, log_entry.status_code)
assert_equal("text/plain", log_entry.response.content_type)
assert_match(/the response/, log_entry.response.content_bytes)
end
end

6
spec/models/http_log_entry_spec.rb
View File

@@ -19,7 +19,7 @@ RSpec.describe HttpLogEntry, type: :model do
end
describe "associations" do
it { should belong_to(:response).class_name("::BlobEntry") }
it { should belong_to(:response).class_name("::BlobFile") }
it { should belong_to(:request_headers).class_name("::HttpLogEntryHeader") }
it do
should belong_to(:response_headers).class_name("::HttpLogEntryHeader")
@@ -144,14 +144,14 @@ RSpec.describe HttpLogEntry, type: :model do
context "when response association is loaded" do
it "returns size from response object" do
test_content = "test content"
entry.response = build(:blob_entry, content: test_content)
entry.response = build(:blob_file, contents: test_content)
expect(entry.response_size).to eq(test_content.bytesize)
end
end
context "when response association is not loaded" do
it "queries size directly from database" do
size = entry.response.size
size = entry.response.size_bytes
entry.association(:response).reset
expect(entry.response_size).to eq(size)
end