diff --git a/app/assets/images/refurrer-logo-icon.png b/app/assets/images/refurrer-logo-icon.png
new file mode 100644
index 00000000..c5fbc1bd
Binary files /dev/null and b/app/assets/images/refurrer-logo-icon.png differ
diff --git a/app/assets/images/refurrer-logo-md.png b/app/assets/images/refurrer-logo-md.png
new file mode 100644
index 00000000..d66aa085
Binary files /dev/null and b/app/assets/images/refurrer-logo-md.png differ
diff --git a/app/controllers/blob_entries_controller.rb b/app/controllers/blob_entries_controller.rb
index 94ccd24b..bea8ce07 100644
--- a/app/controllers/blob_entries_controller.rb
+++ b/app/controllers/blob_entries_controller.rb
@@ -1,7 +1,8 @@
-# typed: true
+# typed: strict
class BlobEntriesController < ApplicationController
skip_before_action :authenticate_user!, only: [:show]
+ sig { void }
def show
thumb = params[:thumb]
raise("invalid thumb #{thumb}") if thumb.present? && !thumb_params(thumb)
@@ -15,14 +16,10 @@ class BlobEntriesController < ApplicationController
etag += "-#{thumb}" if thumb
return unless stale?(last_modified: Time.at(0), strong_etag: etag)
- sha256bin = HexUtil.hex2bin(sha256)
-
if show_blob_file(sha256, thumb)
return
elsif BlobFile.migrate_sha256!(sha256) && show_blob_file(sha256, thumb)
return
- elsif blob_entry = BlobEntry.find_by(sha256: sha256bin)
- show_blob_entry(blob_entry, thumb)
else
raise ActiveRecord::RecordNotFound
end
@@ -37,13 +34,16 @@ class BlobEntriesController < ApplicationController
filename = T.must(filename[..File.extname(filename).length])
filename += ".jpeg"
cache_key = "vips:thumbnail:#{sha256}:#{thumb}"
- width, height = thumb_params(thumb)
-
+ thumb_params = thumb_params(thumb)
+ if thumb_params.nil?
+ raise ActionController::BadRequest.new("invalid thumbnail: #{thumb}")
+ end
+ width, height = thumb_params
thumb_data =
Rack::MiniProfiler.step("vips: load from cache") do
Rails
.cache
- .fetch(cache_key, expires_in: 1.hour) do
+ .fetch(cache_key, expires_in: 1.day) do
blob_file = BlobFile.find_by(sha256: HexUtil.hex2bin(sha256))
if blob_file
content_type =
@@ -134,58 +134,7 @@ class BlobEntriesController < ApplicationController
end
end
- sig { params(blob_entry: BlobEntry, thumb: T.nilable(String)).void }
- def show_blob_entry(blob_entry, thumb)
- sha256 = HexUtil.bin2hex(T.must(blob_entry.sha256))
- content_type = blob_entry.content_type || "application/octet-stream"
- # images, videos, etc
- if helpers.is_send_data_content_type?(content_type)
- if !thumb.blank? && helpers.is_renderable_image_type?(content_type)
- filename = "thumb-#{thumb}-#{sha256}"
- filename = T.must(filename[..File.extname(filename).length])
- filename += ".jpeg"
-
- width, height = thumb_params(thumb)
- image =
- T.unsafe(Vips::Image).thumbnail_buffer(
- blob_entry.contents,
- width,
- height: height,
- )
- resized_image_contents = image.jpegsave_buffer
-
- send_data(
- resized_image_contents,
- type: "image/jpg",
- disposition: "inline",
- filename: filename,
- )
- elsif !thumb.blank? && helpers.is_renderable_video_type?(content_type)
- render plain: "no thumbnail"
- else
- ext = helpers.ext_for_content_type(content_type)
- ext = ".#{ext}" if ext
- send_data(
- T.must(blob_entry.contents),
- type: content_type,
- disposition: "inline",
- filename: "data#{ext}",
- )
- end
- elsif content_type =~ %r{text/plain}
- render plain: blob_entry.contents
- elsif content_type.starts_with? "text/html"
- render html: blob_entry.contents&.html_safe
- elsif content_type.starts_with? "application/json"
- pretty_json =
- JSON.pretty_generate(JSON.parse(T.must(blob_entry.contents)))
- render html:
- "#{pretty_json}".html_safe
- else
- render plain: "no renderer for #{content_type}"
- end
- end
-
+ sig { params(thumb: String).returns(T.nilable([Integer, Integer])) }
def thumb_params(thumb)
case thumb
when "32-avatar"
@@ -198,6 +147,8 @@ class BlobEntriesController < ApplicationController
[400, 300]
when "medium"
[800, 600]
+ when "content-container"
+ [2048, 2048]
end
end
end
diff --git a/app/controllers/log_entries_controller.rb b/app/controllers/log_entries_controller.rb
index 4e520448..c5dbf12c 100644
--- a/app/controllers/log_entries_controller.rb
+++ b/app/controllers/log_entries_controller.rb
@@ -56,7 +56,7 @@ class LogEntriesController < ApplicationController
HttpLogEntry
.joins(:response)
.includes(:response)
- .select("http_log_entries.*, blob_entries_p.size")
+ .select("http_log_entries.*, blob_files.size_bytes")
.find_each(batch_size: 100, order: :desc) do |log_entry|
break if log_entry.created_at < @time_window.ago
@last_window_count += 1
diff --git a/app/helpers/log_entries_helper.rb b/app/helpers/log_entries_helper.rb
index 310fb28f..8c1f72a9 100644
--- a/app/helpers/log_entries_helper.rb
+++ b/app/helpers/log_entries_helper.rb
@@ -87,7 +87,7 @@ module LogEntriesHelper
sig { params(log_entry: HttpLogEntry).returns(T.nilable(String)) }
def render_msword_content(log_entry)
- docx_body = log_entry.response&.contents
+ docx_body = log_entry.response_bytes
return nil if docx_body.blank?
# Invoke abiword to convert doc / docx to html
# Run abiword conversion with pipes
diff --git a/app/jobs/domain/fa/job/base.rb b/app/jobs/domain/fa/job/base.rb
index 89467ba6..a44a93d2 100644
--- a/app/jobs/domain/fa/job/base.rb
+++ b/app/jobs/domain/fa/job/base.rb
@@ -419,7 +419,7 @@ class Domain::Fa::Job::Base < Scraper::JobBase
}
raise("unsupported content type: #{log_entry.content_type}")
end
- document = log_entry.response&.contents || return
+ document = log_entry.response_bytes || return
link_finder =
Scraper::LinkFinder.new(T.must(log_entry.uri_host), document)
link_finder.logger.level = :error
diff --git a/app/jobs/domain/twitter/job/media_job.rb b/app/jobs/domain/twitter/job/media_job.rb
index 5f8e363b..f6ab6452 100644
--- a/app/jobs/domain/twitter/job/media_job.rb
+++ b/app/jobs/domain/twitter/job/media_job.rb
@@ -17,7 +17,7 @@ class Domain::Twitter::Job::MediaJob < Domain::Twitter::Job::TwitterJobBase
response = http_client.get(@media.url_str)
- logger.debug "#{HexUtil.humansize(T.must(response.log_entry.response&.size))} / " +
+ logger.debug "#{HexUtil.humansize(T.must(response.log_entry.response&.size_bytes))} / " +
"#{response.log_entry.content_type} / " +
"#{response.log_entry.response_time_ms} ms"
diff --git a/app/lib/domain/e621/tag_util.rb b/app/lib/domain/e621/tag_util.rb
index 12a3dcb9..ce729965 100644
--- a/app/lib/domain/e621/tag_util.rb
+++ b/app/lib/domain/e621/tag_util.rb
@@ -8,7 +8,7 @@ class Domain::E621::TagUtil
sig do
params(
post_json: T::Hash[String, T.untyped],
- caused_by_entry: T.nilable(ReduxApplicationRecord),
+ caused_by_entry: T.nilable(HttpLogEntry),
force_update: T::Boolean,
).returns(Domain::Post::E621Post)
end
diff --git a/app/lib/domain/e621/task/fix_e621_post_missing_files.rb b/app/lib/domain/e621/task/fix_e621_post_missing_files.rb
index 6931c121..adb400b0 100644
--- a/app/lib/domain/e621/task/fix_e621_post_missing_files.rb
+++ b/app/lib/domain/e621/task/fix_e621_post_missing_files.rb
@@ -11,7 +11,7 @@ class Domain::E621::Task::FixE621PostMissingFiles
return
end
- index_page_contents = index_page.response&.contents
+ index_page_contents = index_page.response_bytes
unless index_page_contents
logger.error("no index page contents for post #{post.id}")
return
diff --git a/app/lib/fa_backfill_favs.rb b/app/lib/fa_backfill_favs.rb
index b1a6b21d..106e071a 100644
--- a/app/lib/fa_backfill_favs.rb
+++ b/app/lib/fa_backfill_favs.rb
@@ -48,10 +48,10 @@ class FaBackfillFavs
)
entries.each do |entry|
- response = T.let(entry.response, T.nilable(BlobEntry))
+ response = T.let(entry.response, T.nilable(BlobFile))
next unless response
- contents = T.let(response.contents, T.nilable(String))
+ contents = T.let(response.content_bytes, T.nilable(String))
next unless contents
page =
diff --git a/app/lib/scraper/http_client.rb b/app/lib/scraper/http_client.rb
index 61c177fc..2a53ac07 100644
--- a/app/lib/scraper/http_client.rb
+++ b/app/lib/scraper/http_client.rb
@@ -79,7 +79,7 @@ class Scraper::HttpClient
from_cache ? "[" + "CACHED".light_green.bold + "]" : nil,
"[entry #{log_entry.id.to_s.bold} /",
"GET #{response_code_colorized} /",
- "#{HexUtil.humansize(T.must(response_blob_entry.bytes_stored)).bold} / #{HexUtil.humansize(T.must(response_blob_entry.size)).bold}]",
+ "#{HexUtil.humansize(T.must(response_blob_entry.size_bytes)).bold}]",
"[#{response_time_ms.to_s.bold} ms / #{total_time_ms.to_s.bold} ms]",
log_entry.uri.to_s.black,
].compact.join(" "),
@@ -112,7 +112,7 @@ class Scraper::HttpClient
if use_http_cache
if (cached_response = HttpLogEntry.find_by_uri(uri)) &&
(status_code = cached_response.status_code) &&
- (body = cached_response.response&.contents)
+ (body = cached_response.response&.content_bytes)
print_request_performed_log_line(
from_cache: true,
log_entry: cached_response,
@@ -175,10 +175,9 @@ class Scraper::HttpClient
total_time_ms = -1
begin
response_blob_entry =
- BlobEntry.find_or_build(
- content_type: content_type,
- contents: response_body,
- )
+ BlobFile.find_or_initialize_from_contents(response_body) do |blob_file|
+ blob_file.content_type = content_type
+ end
scrubbed_uri = @config.scrub_stored_uri(uri)
log_entry =
@@ -200,9 +199,6 @@ class Scraper::HttpClient
},
)
- # double write blob_file while migrating
- response_blob_file =
- BlobFile.find_or_initialize_from_blob_entry(response_blob_entry)
total_time_ms = ((Time.now - requested_at) * 1000).to_i
Scraper::Metrics::HttpClientMetrics.observe_request_finish(
@@ -215,11 +211,6 @@ class Scraper::HttpClient
)
log_entry.save!
- begin
- response_blob_file.save unless response_blob_file.persisted?
- rescue => e
- puts "error saving blob file #{HexUtil.bin2hex(T.must(response_blob_file.sha256))}: #{e}"
- end
rescue StandardError
retries += 1
retry if retries < 2
diff --git a/app/models/blob_file.rb b/app/models/blob_file.rb
index bce15657..4fbaae9d 100644
--- a/app/models/blob_file.rb
+++ b/app/models/blob_file.rb
@@ -68,6 +68,23 @@ class BlobFile < ReduxApplicationRecord
end
end
+ sig do
+ params(
+ contents: String,
+ block: T.proc.params(blob_file: BlobFile).void,
+ ).returns(BlobFile)
+ end
+ def self.find_or_initialize_from_contents(contents, &block)
+ sha256 = Digest::SHA256.digest(contents)
+ BlobFile.migrate_sha256!(sha256) ||
+ BlobFile.find_or_initialize_by(
+ sha256: Digest::SHA256.digest(contents),
+ ) do |blob_file|
+ blob_file.content_bytes = contents
+ block.call(blob_file)
+ end
+ end
+
sig { params(blob_entry: BlobEntry).returns(BlobFile) }
def self.initialize_from_blob_entry(blob_entry)
BlobFile.new(
@@ -82,6 +99,8 @@ class BlobFile < ReduxApplicationRecord
def self.migrate_sha256!(sha256)
# convert to binary if hex formatted
sha256 = HexUtil.hex2bin(sha256) if sha256.length == 64
+ blob_file = BlobFile.find_by(sha256: sha256)
+ return blob_file if blob_file
blob_entry = BlobEntry.find_by(sha256: sha256)
return nil unless blob_entry
blob_file = BlobFile.find_or_initialize_from_blob_entry(blob_entry)
diff --git a/app/models/domain/fa/post.rb b/app/models/domain/fa/post.rb
index 9fa63c14..3898edac 100644
--- a/app/models/domain/fa/post.rb
+++ b/app/models/domain/fa/post.rb
@@ -153,7 +153,7 @@ class Domain::Fa::Post < ReduxApplicationRecord
pa = posted_at
return pa if pa
begin
- contents = guess_last_submission_page&.response&.contents
+ contents = guess_last_submission_page&.response_bytes
if contents
parser = Domain::Fa::Parser::Page.new(contents)
parser.submission.posted_date if parser.probably_submission?
diff --git a/app/models/domain/fa/user_avatar.rb b/app/models/domain/fa/user_avatar.rb
index ef8e7749..582b3682 100644
--- a/app/models/domain/fa/user_avatar.rb
+++ b/app/models/domain/fa/user_avatar.rb
@@ -17,12 +17,12 @@ class Domain::Fa::UserAvatar < ReduxApplicationRecord
belongs_to :user, class_name: "::Domain::Fa::User"
belongs_to :file,
foreign_key: :file_sha256,
- class_name: "::BlobEntry",
+ class_name: "::BlobFile",
optional: true
belongs_to :log_entry, class_name: "::HttpLogEntry", optional: true
def file
- @file_model ||= BlobEntry.ensure(file_sha256) if file_sha256
+ @file_model ||= BlobFile.migrate_sha256!(file_sha256) if file_sha256
end
def file_uri
diff --git a/app/models/domain/post/fa_post.rb b/app/models/domain/post/fa_post.rb
index 42a1c5be..23ccac60 100644
--- a/app/models/domain/post/fa_post.rb
+++ b/app/models/domain/post/fa_post.rb
@@ -171,7 +171,7 @@ class Domain::Post::FaPost < Domain::Post
pa = super
return pa if pa
begin
- contents = guess_last_submission_log_entry&.response&.contents
+ contents = guess_last_submission_log_entry&.response_bytes
if contents
parser = Domain::Fa::Parser::Page.new(contents)
parser.submission.posted_date if parser.probably_submission?
diff --git a/app/models/domain/post/inkbunny_post.rb b/app/models/domain/post/inkbunny_post.rb
index 9ed153d6..cacf17b9 100644
--- a/app/models/domain/post/inkbunny_post.rb
+++ b/app/models/domain/post/inkbunny_post.rb
@@ -120,8 +120,7 @@ class Domain::Post::InkbunnyPost < Domain::Post
# TODO - this is a mess, need to backfill descriptions on inkbunny posts
description || self.ib_detail_raw&.dig("submission_json", "description") ||
begin
- contents = self.deep_update_log_entry&.response&.contents
- if contents.present?
+ if contents = self.deep_update_log_entry&.response_bytes
json = JSON.parse(contents)
json["submissions"]
.find { |s| s["submission_id"] == self.ib_id&.to_s }
diff --git a/app/models/domain/post_file.rb b/app/models/domain/post_file.rb
index b9712c41..7eb87c65 100644
--- a/app/models/domain/post_file.rb
+++ b/app/models/domain/post_file.rb
@@ -6,7 +6,7 @@ class Domain::PostFile < ReduxApplicationRecord
belongs_to :post, foreign_key: :post_id, class_name: "::Domain::Post"
belongs_to :log_entry, class_name: "::HttpLogEntry", optional: true
belongs_to :blob,
- class_name: "::BlobEntry",
+ class_name: "::BlobFile",
optional: true,
foreign_key: :blob_sha256
@@ -34,6 +34,17 @@ class Domain::PostFile < ReduxApplicationRecord
self.type ||= self.class.name if new_record?
end
+ sig { returns(T.nilable(BlobFile)) }
+ def blob
+ super ||
+ begin
+ @blob_file_model = T.let(@blob_file_model, T.nilable(BlobFile))
+ @blob_file_model ||=
+ ((sha256 = self.blob_sha256) ? BlobFile.migrate_sha256!(sha256) : nil)
+ @blob_file_model
+ end
+ end
+
sig { params(le: T.nilable(HttpLogEntry)).returns(T.nilable(HttpLogEntry)) }
def log_entry=(le)
self.blob_sha256 ||= le.response_sha256 if le.present?
diff --git a/app/models/domain/user/fa_user.rb b/app/models/domain/user/fa_user.rb
index 581162a4..b5bdb7f3 100644
--- a/app/models/domain/user/fa_user.rb
+++ b/app/models/domain/user/fa_user.rb
@@ -109,7 +109,7 @@ class Domain::User::FaUser < Domain::User
account_status ||
begin
if (hle = guess_last_user_page_log_entry) &&
- (response = hle.response) && (contents = response.contents)
+ (contents = hle.response_bytes)
parser =
Domain::Fa::Parser::Page.new(contents, require_logged_in: false)
parser.user_page.account_status.to_s if parser.probably_user_page?
diff --git a/app/models/http_log_entry.rb b/app/models/http_log_entry.rb
index f9820f45..fc38d356 100644
--- a/app/models/http_log_entry.rb
+++ b/app/models/http_log_entry.rb
@@ -10,7 +10,7 @@ class HttpLogEntry < ReduxApplicationRecord
belongs_to :response,
foreign_key: :response_sha256,
- class_name: "::BlobEntry",
+ class_name: "::BlobFile",
autosave: true
belongs_to :request_headers, class_name: "::HttpLogEntryHeader"
@@ -78,11 +78,24 @@ class HttpLogEntry < ReduxApplicationRecord
sig { returns(T.nilable(Integer)) }
def response_size
- if association(:response).loaded?
- T.must(self.response).size
- else
- BlobEntry.where(sha256: response_sha256).pick(:size)
- end
+ response&.size_bytes
+ end
+
+ sig { returns(T.nilable(String)) }
+ def response_bytes
+ response&.content_bytes
+ end
+
+ sig { returns(T.nilable(BlobFile)) }
+ def response
+ super ||
+ begin
+ @response_blob_file ||= T.let(nil, T.nilable(BlobFile))
+ if sha256 = response_sha256
+ @response_blob_file ||= BlobFile.migrate_sha256!(sha256)
+ end
+ @response_blob_file
+ end
end
sig { params(uri: T.any(String, Addressable::URI)).void }
diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb
index 1b827be8..d2aef3f6 100644
--- a/app/views/layouts/application.html.erb
+++ b/app/views/layouts/application.html.erb
@@ -12,7 +12,7 @@
window.MiniProfiler.patchesApplied = true;
<% end %>
- <%= favicon_link_tag "refurrer-logo.png" %>
+ <%= favicon_link_tag "refurrer-logo-icon.png" %>
<%= csrf_meta_tags %>
<%= csp_meta_tag %>
<%= javascript_pack_tag "application-bundle" %>
@@ -25,7 +25,7 @@
<%= link_to root_path, class: "flex items-center" do %>
- <%= image_tag asset_path("refurrer-logo.png"), class: "w-12 h-12 mr-2" %>
+ <%= image_tag asset_path("refurrer-logo-md.png"), class: "w-12 h-12 mr-2" %>
ReFurrer
diff --git a/app/views/log_entries/renderers/_image.html.erb b/app/views/log_entries/renderers/_image.html.erb
index 50db1777..3170eb30 100644
--- a/app/views/log_entries/renderers/_image.html.erb
+++ b/app/views/log_entries/renderers/_image.html.erb
@@ -1 +1,2 @@
+<% path = blob_path(HexUtil.bin2hex(log_entry.response_sha256), format: "jpg", thumb: "content-container") %>
diff --git a/config/initializers/prometheus_exporter.rb b/config/initializers/prometheus_exporter.rb
index ffdf732e..468eba7e 100644
--- a/config/initializers/prometheus_exporter.rb
+++ b/config/initializers/prometheus_exporter.rb
@@ -40,6 +40,28 @@ else
# connect to a Prometheus server, which we can't rely on in a test environment.
$stderr.puts "PrometheusExporter is disabled in test, console, and rake task environments"
module PrometheusExporter
+ module Instrumentation
+ class PeriodicStats
+ class << self
+ extend T::Sig
+ sig do
+ params(
+ args: T.untyped,
+ frequency: T.untyped,
+ client: T.untyped,
+ kwargs: T.untyped,
+ ).void
+ end
+ def start(*args, frequency:, client: T.unsafe(nil), **kwargs)
+ end
+
+ sig { params(blk: T.proc.void).void }
+ def worker_loop(&blk)
+ end
+ end
+ end
+ end
+
class Client
class RemoteMetric
end
diff --git a/sorbet/rbi/dsl/domain/fa/user_avatar.rbi b/sorbet/rbi/dsl/domain/fa/user_avatar.rbi
index c283a1a2..163e19d2 100644
--- a/sorbet/rbi/dsl/domain/fa/user_avatar.rbi
+++ b/sorbet/rbi/dsl/domain/fa/user_avatar.rbi
@@ -452,7 +452,7 @@ class Domain::Fa::UserAvatar
end
module GeneratedAssociationMethods
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def build_file(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -461,10 +461,10 @@ class Domain::Fa::UserAvatar
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Fa::User) }
def build_user(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_file(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_file!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -479,10 +479,10 @@ class Domain::Fa::UserAvatar
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Fa::User) }
def create_user!(*args, &blk); end
- sig { returns(T.nilable(::BlobEntry)) }
+ sig { returns(T.nilable(::BlobFile)) }
def file; end
- sig { params(value: T.nilable(::BlobEntry)).void }
+ sig { params(value: T.nilable(::BlobFile)).void }
def file=(value); end
sig { returns(T::Boolean) }
@@ -503,7 +503,7 @@ class Domain::Fa::UserAvatar
sig { returns(T::Boolean) }
def log_entry_previously_changed?; end
- sig { returns(T.nilable(::BlobEntry)) }
+ sig { returns(T.nilable(::BlobFile)) }
def reload_file; end
sig { returns(T.nilable(::HttpLogEntry)) }
diff --git a/sorbet/rbi/dsl/domain/post_file.rbi b/sorbet/rbi/dsl/domain/post_file.rbi
index a23f5d66..fc848950 100644
--- a/sorbet/rbi/dsl/domain/post_file.rbi
+++ b/sorbet/rbi/dsl/domain/post_file.rbi
@@ -453,10 +453,10 @@ class Domain::PostFile
end
module GeneratedAssociationMethods
- sig { returns(T.nilable(::BlobEntry)) }
+ sig { returns(T.nilable(::BlobFile)) }
def blob; end
- sig { params(value: T.nilable(::BlobEntry)).void }
+ sig { params(value: T.nilable(::BlobFile)).void }
def blob=(value); end
sig { returns(T::Boolean) }
@@ -465,7 +465,7 @@ class Domain::PostFile
sig { returns(T::Boolean) }
def blob_previously_changed?; end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def build_blob(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -474,10 +474,10 @@ class Domain::PostFile
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Post) }
def build_post(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_blob(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_blob!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -516,7 +516,7 @@ class Domain::PostFile
sig { returns(T::Boolean) }
def post_previously_changed?; end
- sig { returns(T.nilable(::BlobEntry)) }
+ sig { returns(T.nilable(::BlobFile)) }
def reload_blob; end
sig { returns(T.nilable(::HttpLogEntry)) }
diff --git a/sorbet/rbi/dsl/domain/post_file/inkbunny_post_file.rbi b/sorbet/rbi/dsl/domain/post_file/inkbunny_post_file.rbi
index b00ff569..1b11fa45 100644
--- a/sorbet/rbi/dsl/domain/post_file/inkbunny_post_file.rbi
+++ b/sorbet/rbi/dsl/domain/post_file/inkbunny_post_file.rbi
@@ -454,13 +454,13 @@ class Domain::PostFile::InkbunnyPostFile
module EnumMethodsModule; end
module GeneratedAssociationMethods
- sig { returns(T.nilable(::BlobEntry)) }
+ sig { returns(T.nilable(::BlobFile)) }
def blob; end
- sig { params(value: T.nilable(::BlobEntry)).void }
+ sig { params(value: T.nilable(::BlobFile)).void }
def blob=(value); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def build_blob(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -469,10 +469,10 @@ class Domain::PostFile::InkbunnyPostFile
sig { params(args: T.untyped, blk: T.untyped).returns(::Domain::Post) }
def build_post(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_blob(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_blob!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntry) }
@@ -499,7 +499,7 @@ class Domain::PostFile::InkbunnyPostFile
sig { params(value: T.nilable(::Domain::Post)).void }
def post=(value); end
- sig { returns(T.nilable(::BlobEntry)) }
+ sig { returns(T.nilable(::BlobFile)) }
def reload_blob; end
sig { returns(T.nilable(::HttpLogEntry)) }
diff --git a/sorbet/rbi/dsl/good_job/job.rbi b/sorbet/rbi/dsl/good_job/job.rbi
index 1923e936..ef6a7a2c 100644
--- a/sorbet/rbi/dsl/good_job/job.rbi
+++ b/sorbet/rbi/dsl/good_job/job.rbi
@@ -526,9 +526,6 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def active_job_id(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
- def adapter_class(*args, &blk); end
-
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def advisory_lock(*args, &blk); end
@@ -550,12 +547,6 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def arel_columns(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
- def bind_value(*args, &blk); end
-
- sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
- def coalesce_scheduled_at_created_at(*args, &blk); end
-
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def create_with(*args, &blk); end
@@ -674,9 +665,6 @@ class GoodJob::Job
end
def page(num = nil); end
- sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
- def params_execution_count(*args, &blk); end
-
sig do
params(
num: Integer
@@ -2176,9 +2164,6 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def active_job_id(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
- def adapter_class(*args, &blk); end
-
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def advisory_lock(*args, &blk); end
@@ -2200,12 +2185,6 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def arel_columns(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
- def bind_value(*args, &blk); end
-
- sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
- def coalesce_scheduled_at_created_at(*args, &blk); end
-
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def create_with(*args, &blk); end
@@ -2324,9 +2303,6 @@ class GoodJob::Job
end
def page(num = nil); end
- sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
- def params_execution_count(*args, &blk); end
-
sig do
params(
num: Integer
diff --git a/sorbet/rbi/dsl/http_log_entry.rbi b/sorbet/rbi/dsl/http_log_entry.rbi
index cfc72543..8f942516 100644
--- a/sorbet/rbi/dsl/http_log_entry.rbi
+++ b/sorbet/rbi/dsl/http_log_entry.rbi
@@ -453,7 +453,7 @@ class HttpLogEntry
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntryHeader) }
def build_request_headers(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def build_response(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntryHeader) }
@@ -483,10 +483,10 @@ class HttpLogEntry
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntryHeader) }
def create_request_headers!(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_response(*args, &blk); end
- sig { params(args: T.untyped, blk: T.untyped).returns(::BlobEntry) }
+ sig { params(args: T.untyped, blk: T.untyped).returns(::BlobFile) }
def create_response!(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(::HttpLogEntryHeader) }
@@ -501,7 +501,7 @@ class HttpLogEntry
sig { returns(T.nilable(::HttpLogEntryHeader)) }
def reload_request_headers; end
- sig { returns(T.nilable(::BlobEntry)) }
+ sig { returns(T.nilable(::BlobFile)) }
def reload_response; end
sig { returns(T.nilable(::HttpLogEntryHeader)) }
@@ -531,10 +531,10 @@ class HttpLogEntry
sig { void }
def reset_response_headers; end
- sig { returns(T.nilable(::BlobEntry)) }
+ sig { returns(T.nilable(::BlobFile)) }
def response; end
- sig { params(value: T.nilable(::BlobEntry)).void }
+ sig { params(value: T.nilable(::BlobFile)).void }
def response=(value); end
sig { returns(T::Boolean) }
diff --git a/sorbet/tapioca/require.rb b/sorbet/tapioca/require.rb
index f3896ec7..41e61d85 100644
--- a/sorbet/tapioca/require.rb
+++ b/sorbet/tapioca/require.rb
@@ -32,5 +32,6 @@ require "prometheus_exporter/client"
require "prometheus_exporter/metric"
require "prometheus_exporter/instrumentation"
require "prometheus_exporter/instrumentation/active_record"
+require "prometheus_exporter/instrumentation/periodic_stats"
require "prometheus_exporter/instrumentation/good_job"
require "prometheus_exporter/middleware"
diff --git a/spec/factories/blob_file.rb b/spec/factories/blob_file.rb
new file mode 100644
index 00000000..ff8b8e94
--- /dev/null
+++ b/spec/factories/blob_file.rb
@@ -0,0 +1,10 @@
+# typed: false
+FactoryBot.define do
+ factory :blob_file do
+ transient { contents { "test content #{SecureRandom.alphanumeric(10)}" } }
+ content_bytes { contents }
+ size_bytes { contents.bytesize }
+ content_type { "text/plain" }
+ sha256 { Digest::SHA256.digest(contents) }
+ end
+end
diff --git a/spec/factories/domain/fa/users.rb b/spec/factories/domain/fa/users.rb
index 4fb78e5c..52c30f17 100644
--- a/spec/factories/domain/fa/users.rb
+++ b/spec/factories/domain/fa/users.rb
@@ -9,7 +9,7 @@ FactoryBot.define do
trait :with_avatar do
after(:create) do |user|
- create(:domain_fa_user_avatar, user: user, file: create(:blob_entry))
+ create(:domain_fa_user_avatar, user: user, file: create(:blob_file))
end
end
end
diff --git a/spec/factories/http_log_entries.rb b/spec/factories/http_log_entries.rb
index 862d32b2..0c0d7d12 100644
--- a/spec/factories/http_log_entries.rb
+++ b/spec/factories/http_log_entries.rb
@@ -15,7 +15,7 @@ FactoryBot.define do
performed_by { "direct" }
# Create associated records
- association :response, factory: :blob_entry
+ association :response, factory: :blob_file
association :request_headers, factory: :http_log_entry_header
association :response_headers, factory: :http_log_entry_header
diff --git a/spec/helpers/http_client_mock_helpers.rb b/spec/helpers/http_client_mock_helpers.rb
index 2c99adbc..93cdfe4f 100644
--- a/spec/helpers/http_client_mock_helpers.rb
+++ b/spec/helpers/http_client_mock_helpers.rb
@@ -30,11 +30,11 @@ class HttpClientMockHelpers
request_headers: build(:http_log_entry_header),
response_headers: build(:http_log_entry_header),
response:
- BlobEntry.find_by(sha256: sha256) ||
+ BlobFile.find_by(sha256: sha256) ||
build(
- :blob_entry,
+ :blob_file,
content_type: request[:content_type],
- content: request[:contents],
+ contents: request[:contents],
),
)
log_entry.save!
@@ -58,7 +58,7 @@ class HttpClientMockHelpers
logger.info "[mock http client] [#{method}] [#{uri}] [#{opts.inspect.truncate(80)}]"
Scraper::HttpClient::Response.new(
status_code: log_entry.status_code,
- body: log_entry.response.contents,
+ body: log_entry.response.content_bytes,
log_entry: log_entry,
)
end,
@@ -83,11 +83,11 @@ class HttpClientMockHelpers
performed_by: "direct",
response_time_ms: rand(20..100),
response:
- BlobEntry.find_by(sha256: sha256) ||
+ BlobFile.find_by(sha256: sha256) ||
build(
- :blob_entry,
+ :blob_file,
content_type: request[:content_type],
- content: request[:contents],
+ contents: request[:contents],
),
)
[request[:uri], log_entry]
@@ -103,7 +103,7 @@ class HttpClientMockHelpers
logger.info "[mock http client] [get] [#{uri}] [#{opts.inspect.truncate(80)}]"
Scraper::HttpClient::Response.new(
status_code: log_entry.status_code,
- body: log_entry.response.contents,
+ body: log_entry.response.content_bytes,
log_entry: log_entry,
)
end
diff --git a/spec/helpers/log_entries_helper_spec.rb b/spec/helpers/log_entries_helper_spec.rb
index 71bf2ea1..3d39f8a5 100644
--- a/spec/helpers/log_entries_helper_spec.rb
+++ b/spec/helpers/log_entries_helper_spec.rb
@@ -7,7 +7,7 @@ RSpec.describe LogEntriesHelper, type: :helper do
:http_log_entry,
response:
build(
- :blob_entry,
+ :blob_file,
content_type: "application/msword",
contents:
File.binread(
diff --git a/spec/jobs/domain/e621/job/posts_index_job_spec.rb b/spec/jobs/domain/e621/job/posts_index_job_spec.rb
index bbc67007..07fe38bb 100644
--- a/spec/jobs/domain/e621/job/posts_index_job_spec.rb
+++ b/spec/jobs/domain/e621/job/posts_index_job_spec.rb
@@ -23,7 +23,7 @@ describe Domain::E621::Job::PostsIndexJob do
],
)
- described_class.perform_now({ caused_by_entry: file })
+ perform_now({ caused_by_entry: file })
expect(Domain::Post::E621Post.count).to eq(5)
post = Domain::Post::E621Post.find_by(e621_id: 4_247_443)
diff --git a/spec/jobs/domain/fa/job/user_avatar_job_spec.rb b/spec/jobs/domain/fa/job/user_avatar_job_spec.rb
index 566ffe18..80df780d 100644
--- a/spec/jobs/domain/fa/job/user_avatar_job_spec.rb
+++ b/spec/jobs/domain/fa/job/user_avatar_job_spec.rb
@@ -103,8 +103,8 @@ describe Domain::Fa::Job::UserAvatarJob do
:http_log_entry,
response:
create(
- :blob_entry,
- content: avatar_fixture_file_2,
+ :blob_file,
+ contents: avatar_fixture_file_2,
content_type: "image/gif",
),
)
@@ -143,8 +143,8 @@ describe Domain::Fa::Job::UserAvatarJob do
:http_log_entry,
response:
create(
- :blob_entry,
- content: avatar_fixture_file,
+ :blob_file,
+ contents: avatar_fixture_file,
content_type: "image/gif",
),
)
diff --git a/spec/lib/fa_backfill_favs_spec.rb b/spec/lib/fa_backfill_favs_spec.rb
index d6e7cea7..abe8ce85 100644
--- a/spec/lib/fa_backfill_favs_spec.rb
+++ b/spec/lib/fa_backfill_favs_spec.rb
@@ -52,13 +52,7 @@ describe FaBackfillFavs do
requested_at: Time.current,
request_headers: empty_headers,
response_headers: empty_headers,
- response:
- BlobEntry.create!(
- contents: iiszed_html,
- content_type: "text/html",
- sha256: Digest::SHA256.digest(iiszed_html),
- size: iiszed_html.bytesize,
- ),
+ response: create(:blob_file, contents: iiszed_html),
)
HttpLogEntry.create!(
@@ -73,13 +67,7 @@ describe FaBackfillFavs do
requested_at: Time.current,
request_headers: empty_headers,
response_headers: empty_headers,
- response:
- BlobEntry.create!(
- contents: renamonzeo_html,
- content_type: "text/html",
- sha256: Digest::SHA256.digest(renamonzeo_html),
- size: renamonzeo_html.bytesize,
- ),
+ response: create(:blob_file, contents: renamonzeo_html),
)
HttpLogEntry.create!(
@@ -94,13 +82,7 @@ describe FaBackfillFavs do
requested_at: Time.current,
request_headers: empty_headers,
response_headers: empty_headers,
- response:
- BlobEntry.create!(
- contents: stickiest_html,
- content_type: "text/html",
- sha256: Digest::SHA256.digest(stickiest_html),
- size: stickiest_html.bytesize,
- ),
+ response: create(:blob_file, contents: stickiest_html),
)
end
diff --git a/spec/lib/scraper/http_client_spec.rb b/spec/lib/scraper/http_client_spec.rb
index 0c9ba7f9..e2f960ee 100644
--- a/spec/lib/scraper/http_client_spec.rb
+++ b/spec/lib/scraper/http_client_spec.rb
@@ -24,7 +24,7 @@ describe Scraper::HttpClient do
client =
Scraper::HttpClient.new(
TestHttpClientConfig.new,
- SpecUtil.mock_http_performer(:get, "https://example.com")
+ SpecUtil.mock_http_performer(:get, "https://example.com"),
)
assert_raises(Scraper::HttpClient::InvalidURLError) do
client.get("https://foobar.com")
@@ -39,32 +39,32 @@ describe Scraper::HttpClient do
:get,
"https://www.example.com/",
request_headers: {
- "cookie" => ""
+ "cookie" => "",
},
response_code: 200,
response_time_ms: 15,
response_headers: {
- "content-type" => "text/plain"
+ "content-type" => "text/plain",
},
- response_body: "the response " + SpecUtil.random_string(16)
- )
+ response_body: "the response " + SpecUtil.random_string(16),
+ ),
)
client.logger.level = :error
# note the lack of trailing slash - http client should set path to '/'
response = client.get("https://www.example.com")
- assert_equal 200, response.status_code
- assert_match /the response /, response.body
+ assert_equal(200, response.status_code)
+ assert_match(/the response /, response.body)
log_entry = response.log_entry
- assert log_entry.persisted?
- assert_equal "text/plain", log_entry.content_type
- assert_in_delta Time.now, log_entry.requested_at, 50
- assert_equal 15, log_entry.response_time_ms
- assert_equal "get", log_entry.verb
- assert_equal 200, log_entry.status_code
- assert_equal "text/plain", log_entry.response.content_type
- assert_match /the response/, log_entry.response.contents
+ assert(log_entry.persisted?)
+ assert_equal("text/plain", log_entry.content_type)
+ assert_in_delta(Time.now, log_entry.requested_at, 50)
+ assert_equal(15, log_entry.response_time_ms)
+ assert_equal("get", log_entry.verb)
+ assert_equal(200, log_entry.status_code)
+ assert_equal("text/plain", log_entry.response.content_type)
+ assert_match(/the response/, log_entry.response.content_bytes)
end
end
diff --git a/spec/models/http_log_entry_spec.rb b/spec/models/http_log_entry_spec.rb
index 8de3e58f..767ced29 100644
--- a/spec/models/http_log_entry_spec.rb
+++ b/spec/models/http_log_entry_spec.rb
@@ -19,7 +19,7 @@ RSpec.describe HttpLogEntry, type: :model do
end
describe "associations" do
- it { should belong_to(:response).class_name("::BlobEntry") }
+ it { should belong_to(:response).class_name("::BlobFile") }
it { should belong_to(:request_headers).class_name("::HttpLogEntryHeader") }
it do
should belong_to(:response_headers).class_name("::HttpLogEntryHeader")
@@ -144,14 +144,14 @@ RSpec.describe HttpLogEntry, type: :model do
context "when response association is loaded" do
it "returns size from response object" do
test_content = "test content"
- entry.response = build(:blob_entry, content: test_content)
+ entry.response = build(:blob_file, contents: test_content)
expect(entry.response_size).to eq(test_content.bytesize)
end
end
context "when response association is not loaded" do
it "queries size directly from database" do
- size = entry.response.size
+ size = entry.response.size_bytes
entry.association(:response).reset
expect(entry.response_size).to eq(size)
end