dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

create domain factors, similar posts/users sections implemented

Browse Source
This commit is contained in:
Dylan Knutson
2025-02-27 07:05:51 +00:00
parent 41a8dab3d3
commit 8c2593b414
37 changed files with 5781 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Rakefile
View File

@@ -113,7 +113,7 @@ task migrate_to_domain: :environment do
if only_domains.include?("fa")
# migrator.migrate_fa_users(only_user: only_user)
# migrator.migrate_fa_posts(only_user: only_user)
migrator.migrate_fa_users_favs(only_user: only_user)
# migrator.migrate_fa_users_favs(only_user: only_user)
migrator.migrate_fa_users_followed_users(only_user: only_user)
end

2
app/assets/stylesheets/application.tailwind.css
View File

@@ -7,7 +7,7 @@
}
.sky-section {
@apply divide-y divide-slate-300 overflow-hidden border border-slate-300 bg-slate-100 md:rounded-lg;
@apply divide-y divide-slate-300 overflow-hidden border border-slate-300 bg-slate-100 sm:rounded-lg;
}
.section-header {

49
app/assets/stylesheets/good_job_custom.css
View File

@@ -60,17 +60,28 @@
text-overflow: ellipsis;
}
/* Log line container */
.log-line {
/* All log lines container */
.good-job-log-lines {
overflow-x: auto;
}
/* Single log line container */
.good-job-log-line {
font-family: monospace;
font-size: 0.8rem;
line-height: 1;
margin: 2px 0;
padding: 2px 4px;
display: flex;
white-space: nowrap;
width: max-content; /* Make width match the content width */
}
.log-line > span {
.good-job-log-line:hover {
background-color: #ccc;
}
.good-job-log-line > span {
display: inline-block;
white-space: pre;
}
@@ -86,3 +97,35 @@
overflow: hidden;
text-overflow: ellipsis;
}
.good-job-arg-name {
white-space: nowrap;
}
.good-job-arg-grid {
display: grid;
grid-template-columns: auto 1fr;
}
.good-job-arg-value,
.good-job-arg-name {
padding: 0.35em 0.4em;
}
.good-job-arg-name,
.good-job-arg-value {
border-bottom: 1px solid #e0e0e0;
}
.good-job-arg-row {
display: contents;
}
.good-job-arg-row:hover > * {
background-color: #ccc;
}
/* This ensures the last row doesn't have a bottom border */
.good-job-arg-grid .good-job-arg-row:last-child * {
border-bottom: none;
}

12
app/helpers/domain/posts_helper.rb
View File

@@ -211,7 +211,7 @@ module Domain::PostsHelper
IB_CDN_HOSTS = %w[*.ib.metapix.net ib.metapix.net]
URL_SUFFIX_QUERY = T.let(<<-SQL.strip.chomp.freeze, String)
reverse(lower(json_attributes->>'url_str')) LIKE (reverse(lower(?)) || '%')
lower(json_attributes->>'url_str') = lower(?)
SQL
MATCHERS =
@@ -236,9 +236,13 @@ module Domain::PostsHelper
post_file =
Domain::PostFile.where(
"#{URL_SUFFIX_QUERY} OR #{URL_SUFFIX_QUERY}",
"d.furaffinity.net#{url.path}",
"facdn.net#{url.path}",
"lower(json_attributes->>'url_str') IN (?, ?, ?, ?, ?, ?)",
"d.furaffinity.net#{url.host}/#{url.path}",
"//d.furaffinity.net#{url.host}/#{url.path}",
"https://d.furaffinity.net#{url.host}/#{url.path}",
"d.facdn.net#{url.host}/#{url.path}",
"//d.facdn.net#{url.host}/#{url.path}",
"https://d.facdn.net#{url.host}/#{url.path}",
).first
if post_file && (post = post_file.post)

6
app/helpers/domain/users_helper.rb
View File

@@ -13,7 +13,11 @@ module Domain::UsersHelper
end
def domain_user_avatar_img_src_path(avatar, thumb: nil)
if (sha256 = avatar&.log_entry&.response_sha256)
blob_path(HexUtil.bin2hex(sha256), format: "jpg", thumb: thumb)
Rails.application.routes.url_helpers.blob_path(
HexUtil.bin2hex(sha256),
format: "jpg",
thumb: thumb,
)
else
# default / 'not found' avatar image
# "/blobs/9080fd4e7e23920eb2dccfe2d86903fc3e748eebb2e5aa8c657bbf6f3d941cdc/contents.jpg"

87
app/helpers/helpers_interface.rb
View File

@@ -2,7 +2,6 @@
module HelpersInterface
extend T::Sig
extend T::Helpers
abstract!
sig do
params(timestamp: T.nilable(ActiveSupport::TimeWithZone)).returns(String)
@@ -10,90 +9,4 @@ module HelpersInterface
def time_ago_or_never(timestamp)
timestamp ? time_ago_in_words(timestamp) + " ago" : "never"
end
sig do
abstract
.params(sha256: String, format: String, thumb: T.nilable(String))
.returns(String)
end
def blob_path(sha256, format:, thumb: nil)
end
sig { abstract.params(hle: T.nilable(HttpLogEntry)).returns(String) }
def log_entry_path(hle)
end
sig { abstract.params(path: String).returns(String) }
def asset_path(path)
end
sig { abstract.params(value: String).returns(String) }
def raw(value)
end
# sig do
# abstract.params(param: T.any(String, Domain::User::FaUser)).returns(String)
# end
# def domain_user_fa_users_path(param)
# end
#
sig { abstract.params(param: T.untyped, options: T.untyped).returns(String) }
def polymorphic_path(param, options = {})
end
sig do
abstract
.params(
timestamp: ActiveSupport::TimeWithZone,
include_seconds: T::Boolean,
)
.returns(String)
end
def time_ago_in_words(timestamp, include_seconds: false)
end
sig { abstract.params(size: Integer).returns(String) }
def number_to_human_size(size)
end
sig { abstract.params(user: Domain::User).returns(Domain::UserPolicy) }
def policy(user)
end
sig do
abstract
.params(
model: T.any(Domain::Post, Domain::User),
partial: String,
locals: T::Hash[Symbol, T.untyped],
)
.returns(String)
end
def render(model = nil, partial:, locals:)
end
sig { abstract.returns(ActionView::LookupContext) }
def lookup_context
end
sig { abstract.returns(ActionDispatch::Request) }
def request
end
sig do
abstract
.params(
link_text: String,
link_url: String,
options: T.untyped,
block: T.nilable(T.proc.returns(String)),
)
.returns(String)
end
def link_to(link_text, link_url, options = {}, &block)
end
sig { abstract.params(src: String, options: T.untyped).returns(String) }
def image_tag(src, options = {})
end
end

6
app/lib/domain/migrate_to_domain.rb
View File

@@ -445,11 +445,11 @@ class Domain::MigrateToDomain
format: "%t: %c/%C %B %p%% %a %e",
output: @pb_sink,
)
query.find_each do |user|
query.find_in_batches(batch_size: 10) do |batch|
ReduxApplicationRecord.transaction do
migrate_fa_user_followed_users(user)
batch.each { |user| migrate_fa_user_followed_users(user) }
end
pb.progress = [pb.progress + 1, pb.total].min
pb.progress = [pb.progress + batch.size, pb.total].min
end
end

5
app/models/domain/factors.rb Normal file
View File

@@ -0,0 +1,5 @@
# typed: strict
class Domain::Factors < ReduxApplicationRecord
self.abstract_class = true
has_neighbors :embedding
end

6
app/models/domain/factors/user_post_fav_post_factors.rb Normal file
View File

@@ -0,0 +1,6 @@
# typed: strict
class Domain::Factors::UserPostFavPostFactors < Domain::Factors
self.table_name = "domain_user_post_fav_post_factors"
self.primary_key = "post_id"
belongs_to :post, class_name: "::Domain::Post"
end

6
app/models/domain/factors/user_post_fav_user_factors.rb Normal file
View File

@@ -0,0 +1,6 @@
# typed: strict
class Domain::Factors::UserPostFavUserFactors < Domain::Factors
self.table_name = "domain_user_post_fav_user_factors"
self.primary_key = "user_id"
belongs_to :user, class_name: "::Domain::User"
end

6
app/models/domain/factors/user_user_follow_from_factors.rb Normal file
View File

@@ -0,0 +1,6 @@
# typed: strict
class Domain::Factors::UserUserFollowFromFactors < Domain::Factors
self.table_name = "domain_user_user_follow_from_factors"
self.primary_key = "user_id"
belongs_to :user, class_name: "::Domain::User"
end

6
app/models/domain/factors/user_user_follow_to_factors.rb Normal file
View File

@@ -0,0 +1,6 @@
# typed: strict
class Domain::Factors::UserUserFollowToFactors < Domain::Factors
self.table_name = "domain_user_user_follow_to_factors"
self.primary_key = "user_id"
belongs_to :user, class_name: "::Domain::User"
end

43
app/models/domain/post/e621_post.rb
View File

@@ -93,12 +93,51 @@ class Domain::Post::E621Post < Domain::Post
sig { override.returns(T.nilable(String)) }
def title
"E621 Post #{self.e621_id}"
self
.sources_array
.lazy
.filter_map do |source|
model =
T.cast(
T.unsafe(ApplicationController.helpers).link_for_source(source),
T.nilable(Domain::PostsHelper::LinkForSource),
)&.model
return model.title if model && model.is_a?(Domain::Post)
end
.first || "E621 Post #{self.e621_id}"
end
sig { override.returns(T.nilable(String)) }
def primary_creator_name_fallback_for_view
self.tags_array&.dig("artist")&.first || self.artists_array&.first
self
.sources_array
.lazy
.filter_map do |source|
model =
T.cast(
T.unsafe(ApplicationController.helpers).link_for_source(source),
T.nilable(Domain::PostsHelper::LinkForSource),
)&.model
if model && model.is_a?(Domain::Post) && model.class.has_creators?
return T.unsafe(model).creator&.name_for_view
elsif model && model.is_a?(Domain::User)
return model.name_for_view
end
end
.first ||
self.class.first_valid_artist_tag_in(self.tags_array&.dig("artist")) ||
self.class.first_valid_artist_tag_in(self.artists_array)
end
INVALID_ARTIST_TAGS = %w[unknown unknown_artist sound_warning].freeze
sig { params(list: T.nilable(T::Array[String])).returns(T.nilable(String)) }
def self.first_valid_artist_tag_in(list)
list &&
list
.lazy
.filter_map { |tag| INVALID_ARTIST_TAGS.include?(tag) ? nil : tag }
.first
end
sig { override.returns(T.nilable(T.any(String, Integer))) }

6
app/models/domain/user/fa_user.rb
View File

@@ -14,9 +14,9 @@ class Domain::User::FaUser < Domain::User
attr_json :num_comments_given, :integer
attr_json :num_journals, :integer
attr_json :num_favorites, :integer
attr_json_due_timestamp :scanned_gallery_at, 1.year
attr_json_due_timestamp :scanned_page_at, 1.month
attr_json_due_timestamp :scanned_follows_at, 1.month
attr_json_due_timestamp :scanned_gallery_at, 3.years
attr_json_due_timestamp :scanned_page_at, 3.months
attr_json_due_timestamp :scanned_follows_at, 3.months
attr_json_due_timestamp :scanned_favs_at, 1.month
attr_json_due_timestamp :scanned_incremental_at, 1.month
attr_json :registered_at, :datetime

18
app/views/domain/has_description_html/_section_description_sanitized.html.erb
View File

@@ -1,12 +1,10 @@
<section class="sky-section">
<div class="section-header"><%= description_title %></div>
<%# cache(model, expires_in: 12.hours) do %>
<% if (description_html = sanitize_description_html(model)) %>
<div class="bg-slate-800 p-4 text-slate-200">
<%= description_html %>
</div>
<% else %>
<div class="p-4 text-center text-slate-500"><%= no_description_text %></div>
<% end %>
<%# end %>
</section>
<% if (description_html = sanitize_description_html(model)) %>
<div class="bg-slate-800 p-4 text-slate-200">
<%= description_html %>
</div>
<% else %>
<div class="p-4 text-center text-slate-500"><%= no_description_text %></div>
<% end %>
</section>

8
app/views/domain/posts/_as_gallery_item.html.erb
View File

@@ -30,8 +30,12 @@
</h2>
<div class="px-4 pb-4 text-sm text-slate-500">
<div class="flex justify-between gap-2">
<% if @posts_index_view_config.show_creator_links && (creator = post.primary_creator_for_view) %>
<%= render "domain/users/by_inline_link", user: creator %>
<% if @posts_index_view_config.show_creator_links %>
<% if creator = post.primary_creator_for_view%>
<%= render "domain/users/by_inline_link", user: creator %>
<% elsif creator = post.primary_creator_name_fallback_for_view %>
<%= creator %>
<% end %>
<% end %>
<span class="flex-grow text-right">
<% if post.posted_at %>

34
app/views/domain/posts/default/_section_similar_posts.html.erb
View File

@@ -3,6 +3,38 @@
<div
class="grid grid-cols-[1fr_auto_auto] items-center divide-y divide-slate-300 bg-slate-100"
>
<div class="p-4 text-center text-slate-500">(TODO) No similar posts found</div>
<% factors = Domain::Factors::UserPostFavPostFactors.find_by(post: post) %>
<% if factors %>
<% nearest_neighbors = factors.nearest_neighbors(:embedding, distance: "cosine").includes(:post).limit(20) %>
<% nearest_neighbors.each do |factor| %>
<% post = factor.post %>
<% creator = post.class.has_creators? ? post.creator : nil %>
<div class="col-span-3 grid grid-cols-subgrid items-center">
<span class="text-md truncate px-4 py-2">
<%= link_to post.title, domain_post_path(post), class: "underline italic" %>
</span>
<% if creator %>
<a href="<%= domain_user_path(creator) %>" class="group flex items-center gap-2 mr-4">
<div class="flex items-center">
<% if creator.avatar %>
<%= image_tag domain_user_avatar_img_src_path(creator.avatar, thumb: "64-avatar"), class: "h-8 w-8 flex-shrink-0 rounded-md shadow-sm transition-all duration-200 group-hover:brightness-110 group-hover:scale-105 group-hover:shadow-md" %>
<% else %>
<div class="h-8 w-8 flex-shrink-0 rounded-md bg-gradient-to-br from-slate-300 to-slate-400 shadow-sm transition-all duration-200 group-hover:from-slate-400 group-hover:to-slate-500 group-hover:scale-105 group-hover:shadow-md"></div>
<% end %>
</div>
<span class="blue-link truncate transition-all duration-200 group-hover:underline">
<%= creator.url_name %>
</span>
</a>
<% else %>
<div class="px-2 py-2 flex items-center">
</div>
<span class="truncate px-4 py-2"><%= post.primary_creator_name_fallback_for_view %></span>
<% end %>
</div>
<% end %>
<% else %>
<div class="col-span-3 p-4 text-center text-slate-500">No similar posts found</div>
<% end %>
</div>
</section>

34
app/views/domain/posts/show.html.erb
View File

@@ -1,12 +1,26 @@
<div class="mx-auto mt-4 flex w-full max-w-2xl flex-col gap-4 pb-4">
<%= render_for_model(@post, "section_post_title", as: :post) %>
<%= render_for_model(@post, "section_post_groups", as: :post) %>
<%= render_for_model(@post, "section_primary_file", as: :post) %>
<%= render "domain/has_description_html/section_description_sanitized",
model: @post,
description_title: "Description",
no_description_text: "No description available" %>
<%= render_for_model(@post, "section_tags", as: :post) %>
<%= render_for_model(@post, "section_sources", as: :post) %>
<%= render_for_model(@post, "section_similar_posts", as: :post) %>
<% cache [@post, "section_post_title"], expires_in: 1.hour do %>
<%= render_for_model(@post, "section_post_title", as: :post) %>
<% end %>
<% cache [@post, "section_post_groups"], expires_in: 1.hour do %>
<%= render_for_model(@post, "section_post_groups", as: :post) %>
<% end %>
<% cache [@post, "section_primary_file"], expires_in: 1.hour do %>
<%= render_for_model(@post, "section_primary_file", as: :post) %>
<% end %>
<% cache [@post, "section_description_sanitized"], expires_in: 1.hour do %>
<%= render "domain/has_description_html/section_description_sanitized",
model: @post,
description_title: "Description",
no_description_text: "No description available" %>
<% end %>
<% cache [@post, "section_tags"], expires_in: 1.hour do %>
<%= render_for_model(@post, "section_tags", as: :post) %>
<% end %>
<% cache [@post, "section_sources"], expires_in: 1.hour do %>
<%= render_for_model(@post, "section_sources", as: :post) %>
<% end %>
<% cache [@post, "section_similar_posts"], expires_in: 1.hour do %>
<%= render_for_model(@post, "section_similar_posts", as: :post) %>
<% end %>
</div>

23
app/views/domain/users/_inline_link.html.erb
View File

@@ -1,14 +1,11 @@
<div class="flex items-center">
<%= link_to domain_user_path(user), class: "flex grow items-center gap-2" do %>
<img
src="<%= domain_user_avatar_img_src_path(user.avatar, thumb: "64-avatar") %>"
class="h-6 w-6 rounded-md"
/>
<span class="blue-link font-medium text-lg"> <%= user.name_for_view %> </span>
<% end %>
<% if !defined?(with_post_count) || with_post_count %>
<span class="ml-2 text-slate-500">
<%= pluralize(number_with_delimiter(user.posts.count, delimiter: ","), "post") %>
</span>
<% end %>
</div>
<%= link_to(domain_user_path(user), class: "flex grow items-center gap-2") do %>
<%= image_tag domain_user_avatar_img_src_path(user.avatar, thumb: "64-avatar"), class: "h-6 w-6 rounded-md" %>
<span class="blue-link font-medium text-lg"><%= user.name_for_view %></span>
<% end %>
<% if !defined?(with_post_count) || with_post_count %>
<span class="ml-2 text-slate-500">
<%= pluralize(number_with_delimiter(user.posts.count, delimiter: ","), "post") %>
</span>
<% end %>
</div>

13
app/views/domain/users/default/_profile_description.html.erb
View File

@@ -1,13 +0,0 @@
<section class="animated-shadow-sky sky-section">
<h2 class="section-header">Profile Description</h2>
<% cache(user, expires_in: 12.hours) do %>
<% raise %>
<% if (description_html = sanitize_description_html(user)) %>
<div class="bg-slate-800 p-4 text-slate-200">
<%= description_html %>
</div>
<% else %>
<div class="px-4 py-3 text-slate-500">No profile description available</div>
<% end %>
<% end %>
</section>

3
app/views/domain/users/default/_recent_faved_posts.html.erb
View File

@@ -1,5 +1,4 @@
<%# annoying postgres optimizer bug that causes an extremly bad plan if the limit is below 50 %>
<% fav_posts = user.faved_posts.limit(60).to_a[..5] %>
<% fav_posts = user.faved_posts.limit(5) %>
<section class="animated-shadow-sky sky-section">
<h2 class="section-header">
<span class="font-medium text-slate-900">Favorited Posts</span>

25
app/views/domain/users/default/_similar_users.html.erb
View File

@@ -0,0 +1,25 @@
<% factors = Domain::Factors::UserUserFollowToFactors.find_by(user: user) %>
<section class="animated-shadow-sky sky-section">
<h2 class="section-header">
<span class="font-medium text-slate-900">Similar Users</span>
</h2>
<% if factors %>
<% nearest_neighbors = factors.nearest_neighbors(:embedding, distance: "cosine").includes(:user).limit(10) %>
<% nearest_neighbors.each do |neighbor| %>
<% user = neighbor.user %>
<div class="flex items-center gap-2 whitespace-nowrap text-slate-600 justify-between w-full px-4 py-2">
<%= link_to(domain_user_path(user), class: "flex grow items-center gap-2") do %>
<%= image_tag domain_user_avatar_img_src_path(user.avatar, thumb: "64-avatar"), class: "h-6 w-6 rounded-md" %>
<span class="blue-link"><%= user.name_for_view %></span>
<% end %>
<% if !defined?(with_post_count) || with_post_count %>
<span class="ml-2 text-slate-500">
<%= pluralize(number_with_delimiter(user.posts.count, delimiter: ","), "post") %>
</span>
<% end %>
</div>
<% end %>
<% else %>
<div class="px-4 py-3 text-slate-500">No similar users</div>
<% end %>
</section>

4
app/views/domain/users/default/_stats.html.erb
View File

@@ -4,8 +4,8 @@
<% label = stat_row.name %>
<% value = stat_row.value %>
<% fa_icon_class = stat_row.fa_icon_class %>
<div class="flex items-center px-4 py-2">
<span class="grow text-slate-900"><%= label %></span>
<div class="flex items-center px-4 py-2 gap-2">
<span class="grow text-slate-900 truncate"><%= label %></span>
<span class="text-slate-500 relative group">
<% value_str = case value %>
<% when Integer %>

28
app/views/domain/users/show.html.erb
View File

@@ -1,21 +1,31 @@
<div class="mx-auto my-4 w-full space-y-4 md:max-w-2xl">
<%= render_for_model @user, "name_icon_and_status", as: :user %>
<div class="flex flex-col gap-4 sm:flex-row">
<div class="w-full sm:w-1/2">
<%= render_for_model @user, "stats", as: :user %>
<div class="flex flex-col gap-4 w-full sm:w-1/2">
<% cache [@user, "stats"], expires_in: 1.hour do %>
<%= render_for_model @user, "stats", as: :user %>
<% end %>
<% cache [@user, "similar_users"], expires_in: 1.hour do %>
<%= render_for_model @user, "similar_users", as: :user %>
<% end %>
</div>
<div class="flex flex-col gap-4 w-full sm:w-1/2">
<% if @user.has_created_posts? %>
<%= render_for_model @user, "recent_created_posts", as: :user %>
<% cache [@user, "recent_created_posts"], expires_in: 1.hour do %>
<%= render_for_model @user, "recent_created_posts", as: :user %>
<% end %>
<% end %>
<% if @user.has_faved_posts? %>
<%= render_for_model @user, "recent_faved_posts", as: :user %>
<% cache [@user, "recent_faved_posts"], expires_in: 1.hour do %>
<%= render_for_model @user, "recent_faved_posts", as: :user %>
<% end %>
<% end %>
</div>
</div>
<%= render "domain/has_description_html/section_description_sanitized",
model: @user,
description_title: "Profile",
no_description_text: "No description available" %>
<%= render_for_model @user, "similar_users", as: :user %>
<% cache [@user, "profile_description"], expires_in: 1.hour do %>
<%= render "domain/has_description_html/section_description_sanitized",
model: @user,
description_title: "Profile",
no_description_text: "No description available" %>
<% end %>
</div>

4
app/views/good_job/_custom_execution_details.html.erb
View File

@@ -1,8 +1,8 @@
<div class="good-job-execution-log">
<% if log_lines = execution.log_lines_collection&.log_lines %>
<div class="log-lines">
<div class="good-job-log-lines">
<% log_lines.each do |line| %>
<div class="log-line">
<div class="good-job-log-line">
<% segments = parse_ansi(line) %>
<% segments.each do |segment| %>
<% class_names = segment.class_names %>

78
app/views/good_job/_custom_job_details.html.erb
View File

@@ -5,49 +5,47 @@
</h5>
</div>
<div class="card-body p-0">
<div class="list-group list-group-flush">
<div class="list-group list-group-flush overflow-x-auto good-job-arg-grid">
<% arguments_for_job(job).each do |job_arg| %>
<div class="list-group-item py-2">
<div class="row align-items-center">
<% if job_arg.inferred %>
<div class="col-md-2 fw-medium text-muted small fst-italic"><%= job_arg.key %></div>
<div class='good-job-arg-row'>
<% if job_arg.inferred %>
<div class="fw-medium text-muted small good-job-arg-name fst-italic"><%= job_arg.key %></div>
<% else %>
<div class="fw-bold text-muted small good-job-arg-name"><%= job_arg.key %></div>
<% end %>
<div class="good-job-arg-value">
<% case job_arg.value %>
<% when HttpLogEntry %>
<%= render "good_job/arguments/http_log_entry", log_entry: job_arg.value %>
<% when Domain::Fa::Post %>
<%= render "good_job/arguments/domain_fa_post", post: job_arg.value %>
<% when Domain::Fa::User %>
<%= render "good_job/arguments/domain_fa_user", user: job_arg.value %>
<% when Domain::Inkbunny::User %>
<%= render "good_job/arguments/domain_inkbunny_user", user: job_arg.value %>
<% when Domain::Inkbunny::File %>
<%= render "good_job/arguments/domain_inkbunny_file", file: job_arg.value %>
<% when Domain::E621::Post %>
<%= render "good_job/arguments/domain_e621_post", post: job_arg.value %>
<% when Domain::PostFile %>
<%= render "good_job/arguments/domain_post_file", post_file: job_arg.value %>
<% when Domain::Post %>
<%= render "good_job/arguments/domain_post", post: job_arg.value %>
<% when Domain::User %>
<%= render "good_job/arguments/domain_user", user: job_arg.value %>
<% when Domain::UserAvatar %>
<%= render "good_job/arguments/domain_user_avatar", user_avatar: job_arg.value %>
<% when GoodJob::Job %>
<%= render "good_job/arguments/good_job_job", job: job_arg.value %>
<% else %>
<div class="col-md-2 fw-bold text-muted small"><%= job_arg.key %></div>
<div class="text-truncate">
<% if job_arg.inferred %>
<span class="small fst-italic" title="<%= job_arg.value.to_s %>"><%= job_arg.value.to_s %></span>
<% else %>
<code class="small" title="<%= job_arg.value.inspect %>"><%= job_arg.value.inspect %></code>
<% end%>
</div>
<% end %>
<div class="col-md-10">
<% case job_arg.value %>
<% when HttpLogEntry %>
<%= render "good_job/arguments/http_log_entry", log_entry: job_arg.value %>
<% when Domain::Fa::Post %>
<%= render "good_job/arguments/domain_fa_post", post: job_arg.value %>
<% when Domain::Fa::User %>
<%= render "good_job/arguments/domain_fa_user", user: job_arg.value %>
<% when Domain::Inkbunny::User %>
<%= render "good_job/arguments/domain_inkbunny_user", user: job_arg.value %>
<% when Domain::Inkbunny::File %>
<%= render "good_job/arguments/domain_inkbunny_file", file: job_arg.value %>
<% when Domain::E621::Post %>
<%= render "good_job/arguments/domain_e621_post", post: job_arg.value %>
<% when Domain::PostFile %>
<%= render "good_job/arguments/domain_post_file", post_file: job_arg.value %>
<% when Domain::Post %>
<%= render "good_job/arguments/domain_post", post: job_arg.value %>
<% when Domain::User %>
<%= render "good_job/arguments/domain_user", user: job_arg.value %>
<% when Domain::UserAvatar %>
<%= render "good_job/arguments/domain_user_avatar", user_avatar: job_arg.value %>
<% when GoodJob::Job %>
<%= render "good_job/arguments/good_job_job", job: job_arg.value %>
<% else %>
<div class="text-truncate">
<% if job_arg.inferred %>
<span class="small fst-italic" title="<%= job_arg.value.to_s %>"><%= job_arg.value.to_s %></span>
<% else %>
<code class="small" title="<%= job_arg.value.inspect %>"><%= job_arg.value.inspect %></code>
<% end%>
</div>
<% end %>
</div>
</div>
</div>
<% end %>

58
db/migrate/20250226003653_create_domain_post_factors.rb Normal file
View File

@@ -0,0 +1,58 @@
# typed: strict
class CreateDomainPostFactors < ActiveRecord::Migration[7.2]
extend T::Sig
sig { void }
def change
up_only { execute "SET DEFAULT_TABLESPACE = mirai" }
create_factor_table(
:domain_user_post_fav_post_factors,
:post,
:domain_posts,
)
create_factor_table(
:domain_user_post_fav_user_factors,
:user,
:domain_users,
)
create_factor_table(
:domain_user_user_follow_from_factors,
:user,
:domain_users,
)
create_factor_table(
:domain_user_user_follow_to_factors,
:user,
:domain_users,
)
end
sig do
params(
table_name: Symbol,
referenced_name: Symbol,
referenced_table_name: Symbol,
).void
end
def create_factor_table(table_name, referenced_name, referenced_table_name)
create_table table_name, id: false do |t|
t.references referenced_name,
null: false,
index: {
unique: true,
},
foreign_key: {
to_table: referenced_table_name,
validate: true,
}
t.vector :embedding, limit: 32
t.timestamps
end
add_index table_name, :embedding, using: :ivfflat, opclass: :vector_l2_ops
end
end

137
db/structure.sql
View File

@@ -2983,6 +2983,30 @@ CREATE TABLE public.domain_user_post_creations (
);
--
-- Name: domain_user_post_fav_post_factors; Type: TABLE; Schema: public; Owner: -; Tablespace: mirai
--
CREATE TABLE public.domain_user_post_fav_post_factors (
post_id bigint NOT NULL,
embedding public.vector(32),
created_at timestamp(6) without time zone NOT NULL,
updated_at timestamp(6) without time zone NOT NULL
);
--
-- Name: domain_user_post_fav_user_factors; Type: TABLE; Schema: public; Owner: -; Tablespace: mirai
--
CREATE TABLE public.domain_user_post_fav_user_factors (
user_id bigint NOT NULL,
embedding public.vector(32),
created_at timestamp(6) without time zone NOT NULL,
updated_at timestamp(6) without time zone NOT NULL
);
--
-- Name: domain_user_post_favs; Type: TABLE; Schema: public; Owner: -; Tablespace: mirai
--
@@ -3026,6 +3050,30 @@ CREATE SEQUENCE public.domain_user_search_names_id_seq
ALTER SEQUENCE public.domain_user_search_names_id_seq OWNED BY public.domain_user_search_names.id;
--
-- Name: domain_user_user_follow_from_factors; Type: TABLE; Schema: public; Owner: -; Tablespace: mirai
--
CREATE TABLE public.domain_user_user_follow_from_factors (
user_id bigint NOT NULL,
embedding public.vector(32),
created_at timestamp(6) without time zone NOT NULL,
updated_at timestamp(6) without time zone NOT NULL
);
--
-- Name: domain_user_user_follow_to_factors; Type: TABLE; Schema: public; Owner: -; Tablespace: mirai
--
CREATE TABLE public.domain_user_user_follow_to_factors (
user_id bigint NOT NULL,
embedding public.vector(32),
created_at timestamp(6) without time zone NOT NULL,
updated_at timestamp(6) without time zone NOT NULL
);
--
-- Name: domain_user_user_follows; Type: TABLE; Schema: public; Owner: -; Tablespace: mirai
--
@@ -7016,6 +7064,34 @@ CREATE INDEX index_domain_user_post_creations_on_post_id_and_user_id ON public.d
CREATE UNIQUE INDEX index_domain_user_post_creations_on_user_id_and_post_id ON public.domain_user_post_creations USING btree (user_id, post_id);
--
-- Name: index_domain_user_post_fav_post_factors_on_embedding; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE INDEX index_domain_user_post_fav_post_factors_on_embedding ON public.domain_user_post_fav_post_factors USING ivfflat (embedding);
--
-- Name: index_domain_user_post_fav_post_factors_on_post_id; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE UNIQUE INDEX index_domain_user_post_fav_post_factors_on_post_id ON public.domain_user_post_fav_post_factors USING btree (post_id);
--
-- Name: index_domain_user_post_fav_user_factors_on_embedding; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE INDEX index_domain_user_post_fav_user_factors_on_embedding ON public.domain_user_post_fav_user_factors USING ivfflat (embedding);
--
-- Name: index_domain_user_post_fav_user_factors_on_user_id; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE UNIQUE INDEX index_domain_user_post_fav_user_factors_on_user_id ON public.domain_user_post_fav_user_factors USING btree (user_id);
--
-- Name: index_domain_user_post_favs_on_post_id_and_user_id; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
@@ -7065,6 +7141,34 @@ CREATE INDEX index_domain_user_search_names_on_user_id ON public.domain_user_sea
CREATE UNIQUE INDEX index_domain_user_search_names_on_user_id_and_name ON public.domain_user_search_names USING btree (user_id, name);
--
-- Name: index_domain_user_user_follow_from_factors_on_embedding; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE INDEX index_domain_user_user_follow_from_factors_on_embedding ON public.domain_user_user_follow_from_factors USING ivfflat (embedding);
--
-- Name: index_domain_user_user_follow_from_factors_on_user_id; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE UNIQUE INDEX index_domain_user_user_follow_from_factors_on_user_id ON public.domain_user_user_follow_from_factors USING btree (user_id);
--
-- Name: index_domain_user_user_follow_to_factors_on_embedding; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE INDEX index_domain_user_user_follow_to_factors_on_embedding ON public.domain_user_user_follow_to_factors USING ivfflat (embedding);
--
-- Name: index_domain_user_user_follow_to_factors_on_user_id; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE UNIQUE INDEX index_domain_user_user_follow_to_factors_on_user_id ON public.domain_user_user_follow_to_factors USING btree (user_id);
--
-- Name: index_domain_user_user_follows_on_from_id_and_to_id; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
@@ -8386,6 +8490,14 @@ ALTER TABLE ONLY public.domain_inkbunny_favs
ADD CONSTRAINT fk_rails_61d7d623d3 FOREIGN KEY (user_id) REFERENCES public.domain_inkbunny_users(id);
--
-- Name: domain_user_user_follow_to_factors fk_rails_75a659b96f; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_user_follow_to_factors
ADD CONSTRAINT fk_rails_75a659b96f FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_fa_user_avatar_versions fk_rails_77fefb9ac3; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -8410,6 +8522,14 @@ ALTER TABLE ONLY public.domain_inkbunny_posts
ADD CONSTRAINT fk_rails_82ffd77f0c FOREIGN KEY (shallow_update_log_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_user_user_follow_from_factors fk_rails_83aa0dfb3a; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_user_follow_from_factors
ADD CONSTRAINT fk_rails_83aa0dfb3a FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_user_search_names fk_rails_8475fe75b5; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -8450,6 +8570,22 @@ ALTER TABLE ONLY public.domain_user_post_creations
ADD CONSTRAINT fk_rails_9f4b85bc57 FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_user_post_fav_post_factors fk_rails_a305b823b2; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_fav_post_factors
ADD CONSTRAINT fk_rails_a305b823b2 FOREIGN KEY (post_id) REFERENCES public.domain_posts(id);
--
-- Name: domain_user_post_fav_user_factors fk_rails_a719707033; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_user_post_fav_user_factors
ADD CONSTRAINT fk_rails_a719707033 FOREIGN KEY (user_id) REFERENCES public.domain_users(id);
--
-- Name: domain_user_user_follows fk_rails_b45e6e3979; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -8593,6 +8729,7 @@ ALTER TABLE ONLY public.domain_twitter_tweets
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20250226003653'),
('20250222035939'),
('20250206224121'),
('20250203235035'),

11
rake/e621.rake
View File

@@ -64,6 +64,17 @@ namespace :e621 do
desc "Show statistics about e621 favs"
task fav_stats: :environment do
puts "counting total users with cached favs..."
user_relation = Domain::User::E621User.where.not(num_other_favs_cached: nil)
total_users = user_relation.count
puts "total users: #{total_users}"
puts "counting how many of those users have favs scanned..."
users_with_favs_scanned =
user_relation.where.not(scanned_favs_at: nil).count
puts "users with favs scanned: #{users_with_favs_scanned}"
puts "percent scanned: #{((users_with_favs_scanned.to_f / total_users.to_f) * 100).round(1)}%"
puts "counting total cached..."
total_cached =
ReduxApplicationRecord.connection.execute(<<~SQL).first["total"]

16
sorbet/rbi/dsl/domain/factors.rbi generated Normal file
View File

@@ -0,0 +1,16 @@
# typed: true
# DO NOT EDIT MANUALLY
# This is an autogenerated file for dynamic methods in `Domain::Factors`.
# Please instead update this file by running `bin/tapioca dsl Domain::Factors`.
class Domain::Factors
sig { returns(ColorLogger) }
def logger; end
class << self
sig { returns(ColorLogger) }
def logger; end
end
end

1288
sorbet/rbi/dsl/domain/factors/user_post_fav_post_factors.rbi generated Normal file
View File

File diff suppressed because it is too large Load Diff

1288
sorbet/rbi/dsl/domain/factors/user_post_fav_user_factors.rbi generated Normal file
View File

File diff suppressed because it is too large Load Diff

1288
sorbet/rbi/dsl/domain/factors/user_user_follow_from_factors.rbi generated Normal file
View File

File diff suppressed because it is too large Load Diff

1288
sorbet/rbi/dsl/domain/factors/user_user_follow_to_factors.rbi generated Normal file
View File

File diff suppressed because it is too large Load Diff

24
sorbet/rbi/dsl/good_job/job.rbi generated
View File

@@ -526,6 +526,9 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def active_job_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def adapter_class(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def advisory_lock(*args, &blk); end
@@ -547,6 +550,12 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def arel_columns(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def bind_value(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def coalesce_scheduled_at_created_at(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def create_with(*args, &blk); end
@@ -665,6 +674,9 @@ class GoodJob::Job
end
def page(num = nil); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateAssociationRelation) }
def params_execution_count(*args, &blk); end
sig do
params(
num: Integer
@@ -2164,6 +2176,9 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def active_job_id(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def adapter_class(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def advisory_lock(*args, &blk); end
@@ -2185,6 +2200,12 @@ class GoodJob::Job
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def arel_columns(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def bind_value(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def coalesce_scheduled_at_created_at(*args, &blk); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def create_with(*args, &blk); end
@@ -2303,6 +2324,9 @@ class GoodJob::Job
end
def page(num = nil); end
sig { params(args: T.untyped, blk: T.untyped).returns(PrivateRelation) }
def params_execution_count(*args, &blk); end
sig do
params(
num: Integer

78
sorbet/rbi/shims/helpers.rbi Normal file
View File

@@ -0,0 +1,78 @@
# typed: strict
module HelpersInterface
extend T::Sig
sig do
params(sha256: String, format: String, thumb: T.nilable(String)).returns(
String,
)
end
def blob_path(sha256, format:, thumb: nil)
end
sig { params(hle: T.nilable(HttpLogEntry)).returns(String) }
def log_entry_path(hle)
end
sig { params(path: String).returns(String) }
def asset_path(path)
end
sig { params(value: String).returns(String) }
def raw(value)
end
sig { params(param: T.untyped, options: T.untyped).returns(String) }
def polymorphic_path(param, options = {})
end
sig do
params(
timestamp: ActiveSupport::TimeWithZone,
include_seconds: T::Boolean,
).returns(String)
end
def time_ago_in_words(timestamp, include_seconds: false)
end
sig { params(size: Integer).returns(String) }
def number_to_human_size(size)
end
sig { params(user: Domain::User).returns(Domain::UserPolicy) }
def policy(user)
end
sig do
params(
model: T.any(Domain::Post, Domain::User),
partial: String,
locals: T::Hash[Symbol, T.untyped],
).returns(String)
end
def render(model = nil, partial:, locals:)
end
sig { returns(ActionView::LookupContext) }
def lookup_context
end
sig { returns(ActionDispatch::Request) }
def request
end
sig do
params(
link_text: String,
link_url: String,
options: T.untyped,
block: T.nilable(T.proc.returns(String)),
).returns(String)
end
def link_to(link_text, link_url, options = {}, &block)
end
sig { params(src: String, options: T.untyped).returns(String) }
def image_tag(src, options = {})
end
end

35
spec/helpers/domain/posts_helper_spec.rb
View File

@@ -111,41 +111,6 @@ RSpec.describe Domain::PostsHelper, type: :helper do
end
end
%w[
https://ib.metapix.net/s/abc/3210.jpg
http://ib.metapix.net/s/abc/3210.jpg
http://www.ib.metapix.net/s/abc/3210.jpg
ib.metapix.net/s/abc/3210.jpg
gb2.ib.metapix.net/s/abc/3210.jpg
gb2.ib.metapix.net/s/abc/3210.jpg
https://gb2.ib.metapix.net/s/abc/3210.jpg
].each do |url|
it "works with inkbunny file URL #{url}" do
post =
create(
:domain_post_inkbunny_post,
ib_id: "123456",
title: "Post Title",
)
post.files.create!(url_str: url, ib_id: "78910")
expect(url).to eq_link_for_source(model: post, title: "Post Title")
end
end
%w[
https://d.facdn.net/art/dragonkai/1574894326/1574894326.dragonkai_tiefling_warlock_fortune.png
https://facdn.net/art/dragonkai/1574894326/1574894326.dragonkai_tiefling_warlock_fortune.png
www.facdn.net/art/dragonkai/1574894326/1574894326.dragonkai_tiefling_warlock_fortune.png
facdn.net/art/dragonkai/1574894326/1574894326.dragonkai_tiefling_warlock_fortune.png
].each do |url|
it "works with FA file URL #{url}" do
post =
create(:domain_post_fa_post, fa_id: "123456", title: "Post Title")
post.files.create!(url_str: url)
expect(url).to eq_link_for_source(model: post, title: "Post Title")
end
end
it "has the right avatar url" do
user =
create(