dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add GoodJob logging enhancements and custom styles

Browse Source 
- Introduced a new `good_job_custom.css` file for custom styling of GoodJob logs.
- Added a new `pixiv.png` icon for domain-specific logging in the `e621` posts helper.
- Enhanced the `GoodJobHelper` module to parse ANSI escape codes for better log formatting.
- Implemented a new `GoodJobExecutionLogLinesCollection` model to store log lines associated with job executions.
- Updated views to display job execution details and logs with improved formatting and styling.
- Refactored `ColorLogger` to support log line accumulation for better log management.

These changes aim to improve the logging experience and visual representation of job execution details in the GoodJob dashboard.
This commit is contained in:
Dylan Knutson
2025-01-03 05:38:47 +00:00
parent ed299a404d
commit 4af584fffd
37 changed files with 2061 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Procfile.production
View File

@@ -1,3 +1,2 @@
rails: RAILS_ENV=production bundle exec rails s -b 0.0.0.0 -p 3000
tail: tail -f log/production.log
stats: bundle exec rake metrics:report_all

1
app/assets/config/manifest.js
View File

@@ -2,3 +2,4 @@
//= link_tree ../../javascript .js
//= link_tree ../../../vendor/javascript .js
//= link_tree ../builds
//= link good_job_custom.css

BIN
app/assets/images/domain-icons/pixiv.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 678 B

87
app/assets/stylesheets/good_job_custom.css Normal file
View File

@@ -0,0 +1,87 @@
/* ANSI Colors */
.ansi-bold {
font-weight: bold;
}
.ansi-black {
color: #000000;
}
.ansi-red {
color: #cd0000;
}
.ansi-green {
color: #00cd00;
}
.ansi-yellow {
color: #cdcd00;
}
.ansi-blue {
color: #0000ee;
}
.ansi-magenta {
color: #cd00cd;
}
.ansi-cyan {
color: #00cdcd;
}
.ansi-white {
color: #e5e5e5;
}
/* Bright variants */
.ansi-bright-black {
color: #7f7f7f;
}
.ansi-bright-red {
color: #ff0000;
}
.ansi-bright-green {
color: #00ff00;
}
.ansi-bright-yellow {
color: #ffff00;
}
.ansi-bright-blue {
color: #5c5cff;
}
.ansi-bright-magenta {
color: #ff00ff;
}
.ansi-bright-cyan {
color: #00ffff;
}
.ansi-bright-white {
color: #ffffff;
}
.log-uuid {
max-width: 100px;
overflow: hidden;
/* white-space: nowrap; */
text-overflow: ellipsis;
}
/* Log line container */
.log-line {
font-family: monospace;
font-size: 0.8rem;
line-height: 1;
margin: 2px 0;
padding: 2px 4px;
display: flex;
}
.log-line > span {
display: inline-block;
white-space: pre;
}
.good-job-execution-log {
background: #3d3d3d;
}
.text-truncate-link {
display: inline-block;
max-width: 300px;
overflow: hidden;
text-overflow: ellipsis;
}

1
app/helpers/domain/e621/posts_helper.rb
View File

@@ -15,6 +15,7 @@ module Domain::E621::PostsHelper
"inkbunny.png",
%w[*.newgrounds.com newgrounds.com] => "newgrounds.png",
%w[*.patreon.com patreon.com] => "patreon.png",
%w[*.pixiv.net pixiv.net *.pximg.net pximg.net] => "pixiv.png",
}
domain_patterns.each do |patterns, icon|

128
app/helpers/good_job_helper.rb Normal file
View File

@@ -0,0 +1,128 @@
# typed: strict
# frozen_string_literal: true
module GoodJobHelper
extend T::Sig
extend T::Helpers
extend self
class AnsiSegment < T::Struct
const :text, String
const :class_names, T::Array[String]
end
# ANSI escape code pattern
ANSI_PATTERN = /\e\[([0-9;]*)m/
UUID_REGEX = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
sig { params(text: String).returns(T::Array[AnsiSegment]) }
def parse_ansi(text)
segments = []
current_classes = T::Array[String].new
# Split the text into parts based on ANSI codes
parts = text.split(ANSI_PATTERN)
# Process each part and its corresponding ANSI codes
parts.each_with_index do |part, index|
if index.even?
# This is text content
segments << AnsiSegment.new(
text: part,
class_names: current_classes.dup,
)
else
# This is an ANSI code
codes = part.split(";").map(&:to_i)
if codes == [0]
current_classes.clear
else
codes.each do |code|
class_name = ansi_code_to_class(code)
current_classes << class_name if class_name
end
end
end
end
# go through segments and detect UUIDs, splitting the segment at the uuid
# and adding them to the segments array. Should result in a <before>, <uuid>,
# <after> tuple.
segments.flat_map do |segment|
if segment.text.match?(UUID_REGEX)
idx = segment.text.index(UUID_REGEX)
[
AnsiSegment.new(
text: segment.text[0...idx],
class_names: segment.class_names,
),
AnsiSegment.new(
text: segment.text[idx...idx + 36],
class_names: ["log-uuid"],
),
AnsiSegment.new(
text: segment.text[idx + 36..],
class_names: segment.class_names,
),
]
else
[segment]
end
end
end
sig { params(job: GoodJob::Job).returns(T::Hash[String, T.untyped]) }
def arguments_for_job(job)
deserialized =
T.cast(
ActiveJob::Arguments.deserialize(job.serialized_params).to_h,
T::Hash[String, T.untyped],
)
args = deserialized["arguments"].first
args.sort_by { |key, _| key.to_s }.to_h
end
private
sig { params(code: Integer).returns(T.nilable(String)) }
def ansi_code_to_class(code)
case code
when 1
"ansi-bold"
when 30
"ansi-black"
when 31
"ansi-red"
when 32
"ansi-green"
when 33
"ansi-yellow"
when 34
"ansi-blue"
when 35
"ansi-magenta"
when 36
"ansi-cyan"
when 37
"ansi-white"
when 90
"ansi-bright-black"
when 91
"ansi-bright-red"
when 92
"ansi-bright-green"
when 93
"ansi-bright-yellow"
when 94
"ansi-bright-blue"
when 95
"ansi-bright-magenta"
when 96
"ansi-bright-cyan"
when 97
"ansi-bright-white"
else
nil
end
end
end

2
app/jobs/domain/fa/job/scan_file_job.rb
View File

@@ -72,7 +72,7 @@ class Domain::Fa::Job::ScanFileJob < Domain::Fa::Job::Base
post.state_detail["404_count"] += 1
fof_count = (post.state_detail["404_count"] || 0)
if fof_count > 2
if fof_count > 1
post.state = :file_error
post.state_detail["file_error"] = "too many 404s"
post.save!

14
app/jobs/scraper/job_base.rb
View File

@@ -1,5 +1,8 @@
# typed: strict
class Scraper::JobBase < ApplicationJob
# used to store the last job execution (GoodJob::Execution)
thread_mattr_accessor :last_good_job_execution
abstract!
ignore_signature_args :caused_by_entry
@@ -248,6 +251,8 @@ class Scraper::JobBase < ApplicationJob
end
around_perform do |job, block|
log_lines = T.let([], T::Array[String])
ColorLogger.log_line_accumulator = proc { |line| log_lines << line }
block.call
rescue Net::ReadTimeout, Errno::ECONNREFUSED => e
logger.error "#{e.class.name} - sleep for a bit"
@@ -256,7 +261,16 @@ class Scraper::JobBase < ApplicationJob
rescue => e
raise e
ensure
ColorLogger.log_line_accumulator = nil
ColorLogger.quiet { job.enqueue_deferred_jobs! }
log_lines = T.must(log_lines)
good_job = GoodJob::CurrentThread.job
last_execution = Scraper::JobBase.last_good_job_execution
if good_job && last_execution && good_job == last_execution.job &&
log_lines.any?
Scraper::JobBase.last_good_job_execution = nil
last_execution.create_log_lines_collection!(log_lines: log_lines)
end
end
sig do

88
app/lib/color_logger.rb
View File

@@ -2,12 +2,28 @@
class ColorLogger < Logger
extend T::Sig
sig { returns(T.any(IO, StringIO)) }
attr_reader :sink
@quiet =
T.let(Concurrent::ThreadLocalVar.new { 0 }, Concurrent::ThreadLocalVar)
@log_line_accumulator =
T.let(Concurrent::ThreadLocalVar.new { nil }, Concurrent::ThreadLocalVar)
sig { params(sink: T.any(IO, StringIO)).returns(ColorLogger) }
def self.make(sink)
logger = ColorLogger.new(sink, @log_line_accumulator)
ActiveSupport::TaggedLogging.new(logger)
end
# @param log_line_accumulator a proc that is called with a String and returns void
sig do
params(
log_line_accumulator: T.nilable(T.proc.params(line: String).void),
).void
end
def self.log_line_accumulator=(log_line_accumulator)
@log_line_accumulator.value = log_line_accumulator
end
sig { params(blk: T.proc.void).void }
def self.quiet(&blk)
@quiet.value += 1
@@ -30,64 +46,40 @@ class ColorLogger < Logger
@quiet.value > 0
end
sig do
params(sink: T.any(IO, StringIO), klass_name: T.nilable(String)).returns(
ColorLogger,
)
end
def self.make(sink, klass_name = nil)
logger = ColorLogger.new(sink, klass_name)
ActiveSupport::TaggedLogging.new(logger)
end
sig { returns(T.nilable(T.any(String, T.proc.returns(String)))) }
attr_reader :prefix
sig { params(prefix: T.any(String, Proc)).void }
sig { params(prefix: T.any(String, T.proc.returns(String))).void }
def prefix=(prefix)
@prefix = prefix
end
private
sig { params(sink: T.any(IO, StringIO), klass_name: T.nilable(String)).void }
def initialize(sink, klass_name = nil)
sig do
params(
sink: T.any(IO, StringIO),
log_line_accumulator: Concurrent::ThreadLocalVar,
).void
end
def initialize(sink, log_line_accumulator)
super(sink)
@sink = sink
@prefix = T.let(nil, T.nilable(T.any(String, Proc)))
klass_name =
(
if klass_name
ColorLogger.remove_common_prefix(klass_name.dup)
else
"(Anonymous)"
end
)
@prefix = T.let(nil, T.nilable(T.any(String, T.proc.returns(String))))
@log_lines = T.let([], T::Array[String])
this = self
self.formatter =
proc do |severity, datetime, progname, msg|
color =
case severity
when "ERROR"
:red
when "WARN"
:yellow
else
:light_blue
end
prefix = this.prefix
prefix = prefix.call if prefix.is_a?(Proc)
klass_name_str = "[#{klass_name.send(color)}]".ljust(32)
prefix = @prefix.call if @prefix.is_a?(Proc)
if ColorLogger.quiet?
""
else
[klass_name_str, prefix, msg].reject(&:blank?).join(" ") + "\n"
line = [prefix, msg].reject(&:blank?).join(" ")
if (lla = log_line_accumulator.value)
lla.call(line)
end
line += "\n"
line = "" if ColorLogger.quiet?
line
end
end
sig { params(klass_name: String).returns(String) }
def self.remove_common_prefix(klass_name)
klass_name.delete_prefix!("Domain::")
klass_name
end
end

9
app/lib/has_color_logger.rb
View File

@@ -11,16 +11,11 @@ module HasColorLogger
extend ActiveSupport::Concern
included do
define_method(:logger) do
@logger ||=
T.let(
ColorLogger.make(sink, self.class.name),
T.nilable(ColorLogger),
)
@logger ||= T.let(ColorLogger.make(sink), T.nilable(ColorLogger))
end
define_singleton_method(:logger) do
@logger ||=
T.let(ColorLogger.make(sink, self.name), T.nilable(ColorLogger))
@logger ||= T.let(ColorLogger.make(sink), T.nilable(ColorLogger))
end
end
end

6
app/models/good_job_execution_log_lines_collection.rb Normal file
View File

@@ -0,0 +1,6 @@
# typed: strict
class GoodJobExecutionLogLinesCollection < ReduxApplicationRecord
belongs_to :good_job_execution,
class_name: "::GoodJob::Execution",
inverse_of: :log_lines_collection
end

21
app/views/good_job/_custom_execution_details.html.erb Normal file
View File

@@ -0,0 +1,21 @@
<div class="good-job-execution-log">
<% if log_lines = execution.log_lines_collection&.log_lines %>
<div class="log-lines">
<% log_lines.each do |line| %>
<div class="log-line">
<% segments = GoodJobHelper.parse_ansi(line) %>
<% segments.each do |segment| %>
<% class_names = segment.class_names %>
<span
title="<%= segment.text %>"
class="<%= class_names.join(" ") %>"
><%= segment.text %></span
>
<% end %>
</div>
<% end %>
</div>
<% else %>
No log lines
<% end %>
</div>

42
app/views/good_job/_custom_job_details.html.erb Normal file
View File

@@ -0,0 +1,42 @@
<div class="card mb-3">
<div class="card-header">
<h5 class="card-title small mb-0">
<i class="bi bi-list-ul me-2"></i>Job Arguments
</h5>
</div>
<div class="card-body p-0">
<div class="list-group list-group-flush">
<% GoodJobHelper
.arguments_for_job(job)
.each do |key, value| %>
<div class="list-group-item py-2">
<div class="row align-items-center">
<div class="col-md-2 fw-bold text-muted small"><%= key %></div>
<div class="col-md-10">
<% case value %>
<% when HttpLogEntry %>
<%= render "good_job/arguments/http_log_entry", log_entry: value %>
<% when Domain::Fa::Post %>
<%= render "good_job/arguments/domain_fa_post", post: value %>
<% when Domain::Fa::User %>
<%= render "good_job/arguments/domain_fa_user", user: value %>
<% when Domain::Inkbunny::User %>
<%= render "good_job/arguments/domain_inkbunny_user", user: value %>
<% when Domain::Inkbunny::File %>
<%= render "good_job/arguments/domain_inkbunny_file", file: value %>
<% when Domain::E621::Post %>
<%= render "good_job/arguments/domain_e621_post", post: value %>
<% else %>
<div class="text-truncate">
<code class="small" title="<%= value.inspect %>"
><%= value.inspect %></code
>
</div>
<% end %>
</div>
</div>
</div>
<% end %>
</div>
</div>
</div>

68
app/views/good_job/arguments/_domain_e621_post.html.erb Normal file
View File

@@ -0,0 +1,68 @@
<div class="d-flex align-items-center gap-2">
<%= link_to Rails.application.routes.url_helpers.domain_e621_post_path(post),
class: "badge bg-primary",
target: "_blank" do %>
<i class="fa-solid fa-cat me-1"></i>Domain::E621::Post #<%= post.id %>
<% end %>
<div class="d-flex align-items-center ms-auto gap-2">
<% if post.file_url_str.present? %>
<%= link_to post.file_url_str,
class: "badge bg-secondary text-truncate-link",
target: "_blank",
rel: "noopener noreferrer nofollow" do %>
<i class="fa-solid fa-link me-1"></i><%= post.file_url_str %>
<% end %>
<% end %>
<% if post.file.present? %>
<%= link_to Rails.application.routes.url_helpers.log_entry_path(post.file),
class: "badge bg-secondary",
target: "_blank" do %>
<i class="fa-solid fa-file me-1"></i>HttpLogEntry #<%= post.file.id %>
<% end %>
<span
class="badge <%= post.file.status_code.to_i < 400 ? "bg-success" : "bg-danger" %>"
>
<i class="fa-solid fa-signal me-1"></i><%= post.file.status_code %>
</span>
<% else %>
<span class="badge bg-warning text-dark">
<i class="fa-solid fa-file me-1"></i>Not present
</span>
<% end %>
<span
class="badge <%= case post.state
when "ok"
"bg-success"
when "removed"
"bg-danger"
when "scan_error", "file_error"
"bg-warning text-dark"
end %>"
>
<i
class="<%= case post.state
when "ok"
"fa-solid fa-check"
when "removed"
"fa-solid fa-trash"
when "scan_error"
"fa-solid fa-magnifying-glass-exclamation"
when "file_error"
"fa-solid fa-file-circle-exclamation"
end %> me-1"
></i
><%= post.state %>
</span>
<span
class="badge bg-light text-dark"
title="<%= time_ago_in_words(post.created_at) %> ago"
>
<i class="fa-regular fa-clock me-1"></i
><%= post.created_at.strftime("%Y-%m-%d %H:%M:%S") %>
</span>
</div>
</div>

55
app/views/good_job/arguments/_domain_fa_post.html.erb Normal file
View File

@@ -0,0 +1,55 @@
<div class="d-flex align-items-center gap-2">
<%= link_to Rails.application.routes.url_helpers.domain_fa_post_path(post),
class: "badge bg-primary ",
target: "_blank" do %>
<i class="fa-solid fa-paw me-1"></i>Domain::Fa::Post #<%= post.id %>
<% end %>
<div class="d-flex align-items-center ms-auto gap-2">
<span
class="badge <%= case post.state
when "ok"
"bg-success"
when "removed"
"bg-danger"
when "scan_error", "file_error"
"bg-warning text-dark"
end %>"
>
<i
class="<%= case post.state
when "ok"
"fa-solid fa-check"
when "removed"
"fa-solid fa-trash"
when "scan_error"
"fa-solid fa-magnifying-glass-exclamation"
when "file_error"
"fa-solid fa-file-circle-exclamation"
end %> me-1"
></i
><%= post.state %>
</span>
<span class="badge bg-secondary" title="<%= post.title %>">
<i class="fa-regular fa-image me-1"></i><%= post.title %>
</span>
<% if post.creator.present? %>
<%= link_to Rails.application.routes.url_helpers.domain_fa_user_path(post.creator),
class: "badge bg-light text-dark" do %>
<i class="fa-solid fa-user me-1"></i><%= post.creator.url_name %>
<% end %>
<% else %>
<span class="badge bg-light text-dark">
<i class="fa-solid fa-user me-1"></i>No Creator
</span>
<% end %>
<span
class="badge bg-light text-dark"
title="<%= time_ago_in_words(post.posted_at) %> ago"
>
<i class="fa-regular fa-clock me-1"></i
><%= post.posted_at.strftime("%Y-%m-%d %H:%M:%S") %>
</span>
</div>
</div>

37
app/views/good_job/arguments/_domain_fa_user.html.erb Normal file
View File

@@ -0,0 +1,37 @@
<div class="d-flex align-items-center gap-2">
<%= link_to Rails.application.routes.url_helpers.domain_fa_user_path(user),
class: "badge bg-primary",
target: "_blank" do %>
<i class="fa-solid fa-paw me-1"></i>Domain::Fa::User #<%= user.id %>
<% end %>
<div class="d-flex align-items-center ms-auto gap-2">
<span
class="badge <%= user.state == "ok" ? "bg-success" : "bg-warning text-dark" %>"
>
<i
class="<%= if user.state == "ok"
"fa-solid fa-check"
else
"fa-solid fa-magnifying-glass-exclamation"
end %> me-1"
></i
><%= user.state %>
</span>
<span class="badge bg-secondary">
<i class="fa-solid fa-at me-1"></i><%= user.url_name %>
</span>
<% if user.name.present? && user.name != user.url_name %>
<span class="badge bg-success">
<i class="fa-solid fa-signature me-1"></i><%= user.name %>
</span>
<% end %>
<span
class="badge bg-light text-dark"
title="<%= time_ago_in_words(user.created_at) %> ago"
>
<i class="fa-regular fa-clock me-1"></i
><%= user.created_at.strftime("%Y-%m-%d %H:%M:%S") %>
</span>
</div>
</div>

66
app/views/good_job/arguments/_domain_inkbunny_file.html.erb Normal file
View File

@@ -0,0 +1,66 @@
<div class="d-flex align-items-center gap-2">
<%= link_to Rails.application.routes.url_helpers.domain_inkbunny_post_path(
file.post,
),
class: "badge bg-primary",
target: "_blank" do %>
<i class="fa-solid fa-paw me-1"></i>Domain::Inkbunny::File #<%= file.id %>
<% end %>
<div class="d-flex align-items-center ms-auto gap-2">
<% if file.url_str.present? %>
<%= link_to file.url_str,
class: "badge bg-secondary text-truncate-link",
target: "_blank",
rel: "noopener noreferrer nofollow" do %>
<i class="fa-solid fa-link me-1"></i><%= file.url_str %>
<% end %>
<% end %>
<% if file.log_entry.present? %>
<%= link_to Rails.application.routes.url_helpers.log_entry_path(file.log_entry),
class: "badge bg-secondary",
target: "_blank" do %>
<i class="fa-solid fa-file me-1"></i>HttpLogEntry
#<%= file.log_entry.id %>
<% end %>
<% else %>
<!-- be not present -->
<span class="badge bg-warning text-dark">
<i class="fa-solid fa-file me-1"></i>Not present
</span>
<% end %>
<span
class="badge <%= case file.state
when "ok"
"bg-success"
when "removed"
"bg-danger"
when "scan_error", "file_error"
"bg-warning text-dark"
end %>"
>
<i
class="<%= case file.state
when "ok"
"fa-solid fa-check"
when "removed"
"fa-solid fa-trash"
when "scan_error"
"fa-solid fa-magnifying-glass-exclamation"
when "file_error"
"fa-solid fa-file-circle-exclamation"
end %> me-1"
></i
><%= file.state %>
</span>
<span
class="badge bg-light text-dark"
title="<%= time_ago_in_words(file.created_at) %> ago"
>
<i class="fa-regular fa-clock me-1"></i
><%= file.created_at.strftime("%Y-%m-%d %H:%M:%S") %>
</span>
</div>
</div>

21
app/views/good_job/arguments/_domain_inkbunny_user.html.erb Normal file
View File

@@ -0,0 +1,21 @@
<div class="d-flex align-items-center gap-2">
<%= link_to Rails.application.routes.url_helpers.domain_inkbunny_user_path(user),
class: "badge bg-primary",
target: "_blank" do %>
<i class="fa-solid fa-rabbit me-1"></i>Domain::Inkbunny::User
#<%= user.id %>
<% end %>
<div class="d-flex align-items-center ms-auto gap-2">
<span class="badge bg-secondary">
<i class="fa-solid fa-at me-1"></i><%= user.name %>
</span>
<span
class="badge bg-light text-dark"
title="<%= time_ago_in_words(user.created_at) %> ago"
>
<i class="fa-regular fa-clock me-1"></i
><%= user.created_at.strftime("%Y-%m-%d %H:%M:%S") %>
</span>
</div>
</div>

30
app/views/good_job/arguments/_http_log_entry.html.erb Normal file
View File

@@ -0,0 +1,30 @@
<div class="d-flex align-items-center gap-2">
<%= link_to Rails.application.routes.url_helpers.log_entry_path(log_entry),
class: "badge bg-primary",
target: "_blank" do %>
<i class="fa-regular fa-file-lines me-1"></i>HttpLogEntry
#<%= log_entry.id %>
<% end %>
<div class="d-flex align-items-center ms-auto gap-2">
<%= link_to log_entry.uri.to_s,
class: "badge bg-secondary text-truncate-link",
target: "_blank",
rel: "noopener noreferrer nofollow" do %>
<i class="fa-solid fa-link me-1"></i>
<%= log_entry.uri %>
<% end %>
<span
class="badge <%= log_entry.status_code.to_i < 400 ? "bg-success" : "bg-danger" %>"
>
<i class="fa-solid fa-signal me-1"></i><%= log_entry.status_code %>
</span>
<span
class="badge bg-light text-dark"
title="<%= time_ago_in_words(log_entry.created_at) %> ago"
>
<i class="fa-regular fa-clock me-1"></i
><%= log_entry.created_at.strftime("%Y-%m-%d %H:%M:%S") %>
</span>
</div>
</div>

77
app/views/layouts/good_job/application.html.erb Normal file
View File

@@ -0,0 +1,77 @@
<!DOCTYPE html>
<html lang="<%= I18n.locale %>" data-bs-theme="auto">
<head>
<title>Good Job Dashboard</title>
<meta charset="utf-8" />
<meta
content="width=device-width, initial-scale=1, shrink-to-fit=no"
name="viewport"
/>
<%= csrf_meta_tags %>
<%= csp_meta_tag %>
<%# Bootstrap Color Modes
"It is suggested to include the JavaScript at the top of your page
to reduce potential screen flickering during reloading of your site."
https://getbootstrap.com/docs/5.3/customize/color-modes/#javascript %>
<script nonce="<%= content_security_policy_nonce %>">
let theme = localStorage.getItem('good_job-theme');
if (!['light', 'dark'].includes(theme)) {
theme = window.matchMedia('(prefers-color-scheme: dark)').matches
? 'dark'
: 'light';
}
document.documentElement.setAttribute('data-bs-theme', theme);
</script>
<%# Do not use asset tag helpers to avoid paths being overriden by config.asset_host %>
<%= tag.link rel: "stylesheet",
href: frontend_static_path(:bootstrap, format: :css, locale: nil),
nonce: content_security_policy_nonce %>
<%= tag.link rel: "stylesheet",
href: frontend_static_path(:style, format: :css, locale: nil),
nonce: content_security_policy_nonce %>
<%= tag.script "",
src: frontend_static_path(:bootstrap, format: :js, locale: nil),
nonce: content_security_policy_nonce %>
<%= tag.script "",
src: frontend_static_path(:chartjs, format: :js, locale: nil),
nonce: content_security_policy_nonce %>
<%= tag.script "",
src: frontend_static_path(:rails_ujs, format: :js, locale: nil),
nonce: content_security_policy_nonce %>
<%= tag.script "",
src:
frontend_static_path(:es_module_shims, format: :js, locale: nil),
async: true,
nonce: content_security_policy_nonce %>
<% importmaps =
GoodJob::FrontendsController.js_modules.keys.index_with do |module_name|
frontend_module_path(module_name, format: :js, locale: nil)
end %>
<%= tag.script(
{ imports: importmaps }.to_json.html_safe,
type: "importmap",
nonce: content_security_policy_nonce,
) %>
<%= tag.script "", type: "module", nonce: content_security_policy_nonce do %>
import "application";
<% end %>
<%= stylesheet_link_tag "good_job_custom", nonce: content_security_policy_nonce %>
<link
rel="stylesheet"
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/css/all.min.css"
nonce="<%= content_security_policy_nonce %>"
/>
</head>
<body>
<div class="d-flex flex-column min-vh-100">
<%= render "good_job/shared/navbar" %>
<%= render "good_job/shared/secondary_navbar" %>
<div class="container-fluid flex-grow-1 relative">
<%= render "good_job/shared/alert" %>
<%= yield %>
</div>
<%= render "good_job/shared/footer" %>
</div>
</body>
</html>

2
config/application.rb
View File

@@ -16,7 +16,7 @@ Bundler.require(*Rails.groups)
module ReduxScraper
class Application < Rails::Application
config.session_store :cookie_store, key: "_refurrer_session"
config.assets.precompile << "delayed/web/application.css"
config.assets.precompile << "good_job_custom.css"
# Initialize configuration defaults for originally generated Rails version.
config.load_defaults 7.0

24
config/initializers/good_job.rb
View File

@@ -1,4 +1,4 @@
# typed: false
# typed: true
Rails.application.configure do
# GoodJob configuration - applies to all environments (including test)
@@ -15,7 +15,7 @@ Rails.application.configure do
config.good_job.logger = Logger.new(STDOUT)
config.good_job.logger.level = :info
if Rails.env.worker?
if Rails.env == "worker"
config.good_job.execution_mode = :async
config.good_job.on_thread_error = ->(exception) do
Rails.logger.error("GoodJob exception: #{exception}")
@@ -24,3 +24,23 @@ Rails.application.configure do
config.good_job.execution_mode = :external
end
end
ActiveSupport.on_load(:good_job_application_controller) do
T.bind(self, T.class_of(ActionController::Base))
content_security_policy do |policy|
policy.font_src :self, :https, :data, "cdnjs.cloudflare.com"
policy.style_src :self, :https, "cdnjs.cloudflare.com"
policy.style_src_elem :self, :https, "cdnjs.cloudflare.com"
end
end
ActiveSupport.on_load(:good_job_base_record) do
class GoodJob::Execution
has_one :log_lines_collection,
class_name: "::GoodJobExecutionLogLinesCollection",
dependent: :destroy,
inverse_of: :good_job_execution
after_create { Scraper::JobBase.last_good_job_execution = self }
end
end

17
db/migrate/20250102185501_good_job_execution_log_lines_collections.rb Normal file
View File

@@ -0,0 +1,17 @@
class GoodJobExecutionLogLinesCollections < ActiveRecord::Migration[7.2]
def change
create_table :good_job_execution_log_lines_collections do |t|
t.uuid :good_job_execution_id, null: false
t.jsonb :log_lines
t.timestamps
end
add_index :good_job_execution_log_lines_collections,
:good_job_execution_id,
unique: true
add_foreign_key :good_job_execution_log_lines_collections,
:good_job_executions,
column: :good_job_execution_id,
on_delete: :cascade
end
end

12
db/schema.rb generated
View File

@@ -1,4 +1,3 @@
# typed: false
# This file is auto-generated from the current state of the database. Instead
# of editing this file, please use the migrations feature of Active Record to
# incrementally modify your database, and then regenerate this schema definition.
@@ -11,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema[7.2].define(version: 2025_01_01_014121) do
ActiveRecord::Schema[7.2].define(version: 2025_01_02_185501) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_prewarm"
enable_extension "pg_stat_statements"
@@ -1646,6 +1645,14 @@ ActiveRecord::Schema[7.2].define(version: 2025_01_01_014121) do
t.datetime "finished_at"
end
create_table "good_job_execution_log_lines_collections", force: :cascade do |t|
t.uuid "good_job_execution_id", null: false
t.jsonb "log_lines"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["good_job_execution_id"], name: "idx_on_good_job_execution_id_685ddb5560", unique: true
end
create_table "good_job_executions", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
@@ -1837,6 +1844,7 @@ ActiveRecord::Schema[7.2].define(version: 2025_01_01_014121) do
add_foreign_key "domain_twitter_medias", "domain_twitter_tweets", column: "tweet_id"
add_foreign_key "domain_twitter_medias", "http_log_entries", column: "file_id"
add_foreign_key "domain_twitter_tweets", "domain_twitter_users", column: "author_id", primary_key: "tw_id", name: "on_author_id"
add_foreign_key "good_job_execution_log_lines_collections", "good_job_executions", on_delete: :cascade
add_foreign_key "http_log_entries", "http_log_entries", column: "caused_by_id"
add_foreign_key "http_log_entries", "http_log_entry_headers", column: "request_headers_id"
add_foreign_key "http_log_entries", "http_log_entry_headers", column: "response_headers_id"

4
sorbet/rbi/dsl/domain/e621/job/posts_index_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::E621::Job::PostsIndexJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::E621::Job::PostsIndexJob).void)
).returns(T.any(Domain::E621::Job::PostsIndexJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

4
sorbet/rbi/dsl/domain/e621/job/scan_post_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::E621::Job::ScanPostJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::E621::Job::ScanPostJob).void)
).returns(T.any(Domain::E621::Job::ScanPostJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

4
sorbet/rbi/dsl/domain/e621/job/static_file_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::E621::Job::StaticFileJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::E621::Job::StaticFileJob).void)
).returns(T.any(Domain::E621::Job::StaticFileJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

4
sorbet/rbi/dsl/domain/fa/job/browse_page_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::Fa::Job::BrowsePageJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::Fa::Job::BrowsePageJob).void)
).returns(T.any(Domain::Fa::Job::BrowsePageJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

4
sorbet/rbi/dsl/domain/fa/job/favs_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::Fa::Job::FavsJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::Fa::Job::FavsJob).void)
).returns(T.any(Domain::Fa::Job::FavsJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

4
sorbet/rbi/dsl/domain/fa/job/scan_file_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::Fa::Job::ScanFileJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::Fa::Job::ScanFileJob).void)
).returns(T.any(Domain::Fa::Job::ScanFileJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

4
sorbet/rbi/dsl/domain/fa/job/scan_post_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::Fa::Job::ScanPostJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::Fa::Job::ScanPostJob).void)
).returns(T.any(Domain::Fa::Job::ScanPostJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

4
sorbet/rbi/dsl/domain/fa/job/user_follows_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::Fa::Job::UserFollowsJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::Fa::Job::UserFollowsJob).void)
).returns(T.any(Domain::Fa::Job::UserFollowsJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

4
sorbet/rbi/dsl/domain/inkbunny/job/latest_posts_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::Inkbunny::Job::LatestPostsJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::Inkbunny::Job::LatestPostsJob).void)
).returns(T.any(Domain::Inkbunny::Job::LatestPostsJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

4
sorbet/rbi/dsl/domain/inkbunny/job/update_posts_job.rbi generated
View File

@@ -15,13 +15,13 @@ class Domain::Inkbunny::Job::UpdatePostsJob
sig do
params(
args: T.untyped,
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Domain::Inkbunny::Job::UpdatePostsJob).void)
).returns(T.any(Domain::Inkbunny::Job::UpdatePostsJob, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T.untyped).returns(T.untyped) }
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

1265
sorbet/rbi/dsl/good_job_execution_log_lines_collection.rbi generated Normal file
View File

File diff suppressed because it is too large Load Diff

11
sorbet/rbi/dsl/scraper/job_base.rbi generated
View File

@@ -12,5 +12,16 @@ class Scraper::JobBase
class << self
sig { returns(ColorLogger) }
def logger; end
sig do
params(
args: T::Hash[::Symbol, T.untyped],
block: T.nilable(T.proc.params(job: Scraper::JobBase).void)
).returns(T.any(Scraper::JobBase, FalseClass))
end
def perform_later(args, &block); end
sig { params(args: T::Hash[::Symbol, T.untyped]).void }
def perform_now(args); end
end
end

47
spec/lib/color_logger_spec.rb
View File

@@ -3,7 +3,6 @@ require "rails_helper"
describe ColorLogger do
let(:sink) { StringIO.new }
let(:prefix) { "[#{"(Anonymous)".light_blue}] " }
let(:read_sink) do
proc do
sink.rewind
@@ -19,22 +18,20 @@ describe ColorLogger do
it "logs contents with color", quiet: false do
logger.info("foo")
expect(read_sink.call).to eq("#{prefix} foo\n")
expect(read_sink.call).to eq("foo\n")
line1 = "bar!"
logger.info(line1)
line2 = "#{"yes".red}, #{"no".blue}"
logger.info(line2)
expect(read_sink.call).to eq(
["#{prefix} #{line1}\n", "#{prefix} #{line2}\n"].join(""),
)
expect(read_sink.call).to eq(["#{line1}\n", "#{line2}\n"].join(""))
end
it "respects the 'quiet' wrapper", quiet: false do
ColorLogger.quiet { logger.info("don't log me") }
logger.info("but do log this")
expect(read_sink.call).to eq("#{prefix} but do log this\n")
expect(read_sink.call).to eq("but do log this\n")
end
it "by default, rspec logs are quiet" do
@@ -43,20 +40,20 @@ describe ColorLogger do
end
end
it "uses stdout by default when included" do
inst =
Class
.new do
def self.name
"TestClass"
end
include HasColorLogger
end
.new
expect(inst.logger.sink).to be($stdout)
end
# it "uses stdout by default when included" do
# inst =
# Class
# .new do
# def self.name
# "TestClass"
# end
# include HasColorLogger
# end
# .new
# expect(inst.logger.sink).to be($stdout)
# end
it "can have other sink injected when including HasColorLogger" do
it "can have a sink injected" do
s = sink
klass =
Class.new do
@@ -65,10 +62,7 @@ describe ColorLogger do
end
include HasColorLogger[s]
end
expect(klass.logger.sink).to be(sink)
inst = klass.new
expect(inst.logger.sink).to be(sink)
# quiet by default
inst.logger.info("don't log")
@@ -77,15 +71,10 @@ describe ColorLogger do
# unquiet works
ColorLogger.unquiet do
inst.logger.info("foo bar")
# right class name is used
expect(read_sink.call).to eq(
"[#{"TestClass".light_blue}] foo bar\n",
)
expect(read_sink.call).to eq("foo bar\n")
klass.logger.info("baz quux")
expect(read_sink.call).to eq(
"[#{"TestClass".light_blue}] baz quux\n",
)
expect(read_sink.call).to eq("baz quux\n")
end
end
end