dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

env controls for enqeueuing jobs, link uuids in execution details

Browse Source
This commit is contained in:
Dylan Knutson
2025-02-28 02:21:19 +00:00
parent ec7cd52a76
commit ed525ee142
4 changed files with 69 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
app/helpers/domain/users_helper.rb
View File

@@ -113,18 +113,18 @@ module Domain::UsersHelper
if user.has_faved_posts?
rows << StatRow.new(name: "Favorites", value: user.user_post_favs.count)
end
if user.has_followed_users?
rows << StatRow.new(
name: "Following",
value: user.user_user_follows_from.count,
)
end
if user.has_followed_by_users?
rows << StatRow.new(
name: "Followed by",
value: user.user_user_follows_to.count,
)
end
if user.has_followed_users?
rows << StatRow.new(
name: "Following",
value: user.user_user_follows_from.count,
)
end
can_view_timestamps = policy(user).view_page_scanned_at_timestamps?
can_view_log_entries = policy(user).view_log_entries?
@@ -134,30 +134,6 @@ module Domain::UsersHelper
end
if user.is_a?(Domain::User::FaUser) && can_view_timestamps
rows << StatRow.new(
name: "Favs scanned",
value: user.favs_scan,
fa_icon_class: icon_for.call(user.favs_scan.due?),
hover_title: user.favs_scan.interval.inspect,
)
rows << StatRow.new(
name: "Gallery scanned",
value: user.gallery_scan,
fa_icon_class: icon_for.call(user.gallery_scan.due?),
hover_title: user.gallery_scan.interval.inspect,
)
rows << StatRow.new(
name: "Followers scanned",
value: user.followed_by_scan,
fa_icon_class: icon_for.call(user.followed_by_scan.due?),
hover_title: user.followed_by_scan.interval.inspect,
)
rows << StatRow.new(
name: "Followed scanned",
value: user.follows_scan,
fa_icon_class: icon_for.call(user.follows_scan.due?),
hover_title: user.follows_scan.interval.inspect,
)
if can_view_log_entries && hle = user.guess_last_user_page_log_entry
rows << StatRow.new(
name: "Page scanned",
@@ -168,12 +144,37 @@ module Domain::UsersHelper
)
else
rows << StatRow.new(
name: "Page scanned",
name: "Page",
value: user.page_scan,
fa_icon_class: icon_for.call(user.page_scan.due?),
hover_title: user.page_scan.interval.inspect,
)
end
rows << StatRow.new(
name: "Favs",
value: user.favs_scan,
fa_icon_class: icon_for.call(user.favs_scan.due?),
hover_title: user.favs_scan.interval.inspect,
)
rows << StatRow.new(
name: "Followed by",
value: user.followed_by_scan,
fa_icon_class: icon_for.call(user.followed_by_scan.due?),
hover_title: user.followed_by_scan.interval.inspect,
)
rows << StatRow.new(
name: "Following",
value: user.follows_scan,
fa_icon_class: icon_for.call(user.follows_scan.due?),
hover_title: user.follows_scan.interval.inspect,
)
rows << StatRow.new(
name: "Gallery",
value: user.gallery_scan,
fa_icon_class: icon_for.call(user.gallery_scan.due?),
hover_title: user.gallery_scan.interval.inspect,
)
elsif user.is_a?(Domain::User::E621User) && can_view_timestamps
if user.favs_are_hidden
rows << StatRow.new(name: "Favorites hidden", value: "yes")
@@ -184,7 +185,7 @@ module Domain::UsersHelper
)
end
rows << StatRow.new(
name: "Favorites scanned",
name: "Favorites",
value: user.favs_scan,
fa_icon_class: icon_for.call(user.favs_scan.due?),
hover_title: user.favs_scan.interval.inspect,

2
app/helpers/good_job_helper.rb
View File

@@ -11,6 +11,7 @@ module GoodJobHelper
const :text, String
const :class_names, T::Array[String]
const :url, T.nilable(String)
end
# ANSI escape code pattern
@@ -61,6 +62,7 @@ module GoodJobHelper
AnsiSegment.new(
text: segment.text[idx...idx + 36],
class_names: ["log-uuid"],
url: "/jobs/jobs/#{segment.text[idx...idx + 36]}",
),
AnsiSegment.new(
text: segment.text[idx + 36..],

25
app/jobs/domain/fa/job/base.rb
View File

@@ -235,6 +235,11 @@ class Domain::Fa::Job::Base < Scraper::JobBase
sig { params(user: Domain::User::FaUser, at_most_one_scan: T::Boolean).void }
def enqueue_user_scan(user, at_most_one_scan:)
skip_page_enqueue = !!ENV["SKIP_PAGE_ENQUEUE"]
skip_favs_enqueue = !!ENV["SKIP_FAVS_ENQUEUE"]
skip_follows_enqueue = !!ENV["SKIP_FOLLOWS_ENQUEUE"]
skip_gallery_enqueue = !!ENV["SKIP_GALLERY_ENQUEUE"]
logger.tagged(make_arg_tag(user)) do
args =
if user.persisted?
@@ -250,7 +255,9 @@ class Domain::Fa::Job::Base < Scraper::JobBase
{ url_name: user.url_name }
end
if user.page_scan.due? && defer_job(Domain::Fa::Job::UserPageJob, args)
if (
user.page_scan.at.nil? || (!skip_page_enqueue && user.page_scan.due?)
) && defer_job(Domain::Fa::Job::UserPageJob, args)
logger.info(
format_tags(
"enqueue user page job",
@@ -262,7 +269,9 @@ class Domain::Fa::Job::Base < Scraper::JobBase
# don't enqueue any other jobs if the user page hasn't been scanned yet
return if at_most_one_scan && user.page_scan.due?
if user.favs_scan.due? && defer_job(Domain::Fa::Job::FavsJob, args)
if (
user.favs_scan.at.nil? || (!skip_favs_enqueue && user.favs_scan.due?)
) && defer_job(Domain::Fa::Job::FavsJob, args)
logger.info(
format_tags(
"enqueue user favs job",
@@ -273,8 +282,10 @@ class Domain::Fa::Job::Base < Scraper::JobBase
return if at_most_one_scan && user.favs_scan.due?
if user.follows_scan.due? &&
defer_job(Domain::Fa::Job::UserFollowsJob, args)
if (
user.follows_scan.at.nil? ||
(!skip_follows_enqueue && user.follows_scan.due?)
) && defer_job(Domain::Fa::Job::UserFollowsJob, args)
logger.info(
format_tags(
"enqueue user follows job",
@@ -285,8 +296,10 @@ class Domain::Fa::Job::Base < Scraper::JobBase
return if at_most_one_scan && user.follows_scan.due?
if user.gallery_scan.due? &&
defer_job(Domain::Fa::Job::UserGalleryJob, args)
if (
user.gallery_scan.at.nil? ||
(!skip_gallery_enqueue && user.gallery_scan.due?)
) && defer_job(Domain::Fa::Job::UserGalleryJob, args)
logger.info(
format_tags(
"enqueue user gallery job",

20
app/views/good_job/_custom_execution_details.html.erb
View File

@@ -6,11 +6,21 @@
<% segments = parse_ansi(line) %>
<% segments.each do |segment| %>
<% class_names = segment.class_names %>
<span
title="<%= segment.text %>"
class="<%= class_names.join(" ") %>"
><%= segment.text %></span
>
<% if segment.url %>
<%= link_to(
segment.text,
segment.url,
class: class_names.join(" "),
title: segment.text,
target: "_blank",
) %>
<% else %>
<span
title="<%= segment.text %>"
class="<%= class_names.join(" ") %>"
><%= segment.text %></span
>
<% end %>
<% end %>
</div>
<% end %>