4 Commits

Author SHA1 Message Date
Dylan Knutson
4ed1c558b9 quick hack to optimize finding max valid fa_id 2025-07-27 20:24:06 +00:00
Dylan Knutson
c43d1ca197 migrate ib posts to aux table 2025-07-27 17:54:29 +00:00
Dylan Knutson
1f44ec2fa2 domain/users controller spec 2025-07-27 17:26:42 +00:00
Dylan Knutson
8b2ee14ef7 e621 user page fix for faved posts 2025-07-27 17:16:48 +00:00
17 changed files with 383 additions and 101 deletions

View File

@@ -5,3 +5,4 @@ tmp
log
public
.bundle
gems

View File

@@ -1,13 +1,13 @@
# How to use this codebase
- ALWAYS run `just tc` after making changes to ensure the codebase is typechecked.
- ALWAYS run `srb tc` after making changes to Ruby files to ensure the codebase is typechecked.
- ALWAYS run `bin/rspec <path_to_spec_file>` after a spec file is modified.
- Run `tapioca dsl` if models or concerns are modified.
- When modifying a specific file that has a corresponding spec file, run `bin/rspec <path_to_spec_file>` to run just that spec file.
- After modifying a file that has a corresponding spec file, run `bin/rspec <path_to_spec_file>` to run just that spec file.
- There are no view-specific tests, so if a view changes then run the controller tests instead.
- For instance, if you modify `app/models/domain/post.rb`, run `bin/rspec spec/models/domain/post_spec.rb`. If you modify `app/views/domain/users/index.html.erb`, run `bin/rspec spec/controllers/domain/users_controller_spec.rb`.
- At the end of a series of changes, ALWAYS run `just test` to run the entire test suite.
- If specs are failing, then fix the failures, and rerun with `bin/rspec <path_to_spec_file>`.
- ALWAYS construct ran commands ran with `run_terminal_cmd` to be compatible with fish shell.
- ALWAYS suffix commands ran with `run_terminal_cmd` with `2>&1 | cat` to avoid terminal bugs.
# === BACKLOG.MD GUIDELINES START ===

View File

@@ -470,9 +470,13 @@ module Domain::PostsHelper
post.keywords.map(&:strip).reject(&:blank?).compact
end
sig { params(post: Domain::Post::InkbunnyPost).returns(T::Array[String]) }
sig do
params(post: Domain::Post::InkbunnyPost).returns(
T.nilable(T::Array[String]),
)
end
def keywords_for_ib_post(post)
post.keywords.map { |keyword| keyword["keyword_name"] }.compact
post.keywords&.map { |keyword| keyword["keyword_name"] }&.compact
end
sig do

View File

@@ -1,6 +1,8 @@
# typed: strict
# frozen_string_literal: true
class Domain::Post::InkbunnyPost < Domain::Post
aux_table :ib, allow_redefining: :title
SUBMISSION_TYPE_MAP =
T.let(
{
@@ -29,25 +31,6 @@ class Domain::Post::InkbunnyPost < Domain::Post
T::Hash[Integer, String],
)
attr_json :ib_id, :integer
attr_json :state, :string
attr_json :rating, :string
attr_json :submission_type, :string
attr_json :title, :string
attr_json :writing, :string
attr_json :description, :string
attr_json :num_views, :integer
attr_json :num_files, :integer
attr_json :num_favs, :integer
attr_json :num_comments, :integer
attr_json :keywords, ActiveModel::Type::Value.new
attr_json :last_file_updated_at, ActiveModelUtcTimeValue.new
attr_json :deep_update_log_entry_id, :integer
attr_json :shallow_update_log_entry_id, :integer
attr_json :shallow_updated_at, ActiveModelUtcTimeValue.new
attr_json :deep_updated_at, ActiveModelUtcTimeValue.new
attr_json :ib_detail_raw, ActiveModel::Type::Value.new
has_multiple_files! Domain::PostFile::InkbunnyPostFile
has_single_creator! Domain::User::InkbunnyUser
has_faving_users! Domain::User::InkbunnyUser
@@ -105,11 +88,6 @@ class Domain::Post::InkbunnyPost < Domain::Post
self.description
end
sig { override.returns(T.nilable(String)) }
def title
super
end
sig { override.returns(T.nilable(Domain::User)) }
def primary_creator_for_view
self.creator

View File

@@ -1,4 +1 @@
<span>
<i class="fa-solid fa-tag mr-1"></i>
Rating: <%= post.rating_for_view %>
</span>
<%= render partial: "domain/posts/title_stat", locals: { label: "Rating", value: post.rating_for_view, icon_class: "fa-tag" } %>

View File

@@ -1,6 +1,6 @@
<%= sky_section_tag("Keywords") do %>
<% keywords = keywords_for_ib_post(post) %>
<% if keywords.any? %>
<% if keywords&.any? %>
<div class="flex flex-wrap gap-2">
<% keywords.each do |keyword| %>
<span

View File

@@ -1,12 +1,3 @@
<span>
<i class="fa-solid fa-eye mr-1"></i>
Views: <%= post.num_views %>
</span>
<span>
<i class="fa-solid fa-file-image mr-1"></i>
Files: <%= post.num_files %>
</span>
<span>
<i class="fa-solid fa-comment mr-1"></i>
Comments: <%= post.num_comments %>
</span>
<%= render partial: "domain/posts/title_stat", locals: { label: "Views", value: post.num_views, icon_class: "fa-eye" } %>
<%= render partial: "domain/posts/title_stat", locals: { label: "Files", value: post.num_files, icon_class: "fa-file-image" } %>
<%= render partial: "domain/posts/title_stat", locals: { label: "Comments", value: post.num_comments, icon_class: "fa-comment" } %>

View File

@@ -1,5 +1,9 @@
<%# nasty hack, otherwise postgres uses a bad query plan %>
<% fav_posts = user.faved_posts.includes(:creator).limit(5) %>
<% if user.is_a?(Domain::User::FaUser) || user.is_a?(Domain::User::InkbunnyUser) %>
<% fav_posts = user.faved_posts.includes(:creator).limit(5) %>
<% else %>
<% fav_posts = user.faved_posts.limit(5) %>
<% end%>
<% post_favs = Domain::UserPostFav.where(user: user, post: fav_posts).index_by(&:post_id) %>
<section class="animated-shadow-sky sky-section">
<h2 class="section-header">

View File

@@ -0,0 +1,72 @@
# typed: strict
# frozen_string_literal: true
class MigrateIbPostDataToAux < ActiveRecord::Migration[7.2]
sig { void }
def up
cols = [
[:ib_id, :integer, { index: true }],
[:state, :string, {}],
[:rating, :string, {}],
[:submission_type, :string, {}],
[:title, :string, {}],
[:writing, :string, {}],
[:description, :string, {}],
[:num_views, :integer, {}],
[:num_files, :integer, {}],
[:num_favs, :integer, {}],
[:num_comments, :integer, {}],
[:keywords, :jsonb, { default: [] }],
[:last_file_updated_at, :timestamp, {}],
[
:deep_update_log_entry,
:references,
{ foreign_key: { to_table: :http_log_entries }, index: false },
],
[
:shallow_update_log_entry,
:references,
{ foreign_key: { to_table: :http_log_entries }, index: false },
],
[:shallow_updated_at, :timestamp, {}],
[:deep_updated_at, :timestamp, {}],
[:ib_detail_raw, :jsonb, {}],
]
create_aux_table :domain_posts, :ib do |t|
cols.each { |name, type, opts| t.send(type, name, **opts) }
end
col_names =
cols.map do |name, type, opts|
type == :references ? "#{name}_id" : "#{name}"
end
col_selects =
cols.map do |name, type, opts|
if type == :references
"(json_attributes->>'#{name}_id')::integer as #{name}_id"
elsif type == :string
"(json_attributes->>'#{name}')::text as #{name}"
else
"(json_attributes->>'#{name}')::#{type} as #{name}"
end
end
execute <<~SQL
INSERT INTO domain_posts_ib_aux (
base_table_id,
#{col_names.join(",\n ")}
)
SELECT
id as base_table_id,
#{col_selects.join(",\n ")}
FROM domain_posts WHERE type = 'Domain::Post::InkbunnyPost'
SQL
end
sig { void }
def down
drop_table :domain_posts_ib_aux
end
end

View File

@@ -1441,6 +1441,52 @@ CREATE SEQUENCE public.domain_posts_fa_aux_base_table_id_seq
ALTER SEQUENCE public.domain_posts_fa_aux_base_table_id_seq OWNED BY public.domain_posts_fa_aux.base_table_id;
--
-- Name: domain_posts_ib_aux; Type: TABLE; Schema: public; Owner: -
--
CREATE TABLE public.domain_posts_ib_aux (
base_table_id bigint NOT NULL,
ib_id integer,
state character varying,
rating character varying,
submission_type character varying,
title character varying,
writing character varying,
description character varying,
num_views integer,
num_files integer,
num_favs integer,
num_comments integer,
keywords jsonb DEFAULT '[]'::jsonb,
last_file_updated_at timestamp without time zone,
deep_update_log_entry_id bigint,
shallow_update_log_entry_id bigint,
shallow_updated_at timestamp without time zone,
deep_updated_at timestamp without time zone,
ib_detail_raw jsonb
);
--
-- Name: domain_posts_ib_aux_base_table_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
CREATE SEQUENCE public.domain_posts_ib_aux_base_table_id_seq
START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
CACHE 1;
--
-- Name: domain_posts_ib_aux_base_table_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
--
ALTER SEQUENCE public.domain_posts_ib_aux_base_table_id_seq OWNED BY public.domain_posts_ib_aux.base_table_id;
--
-- Name: domain_posts_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
@@ -2878,6 +2924,13 @@ ALTER TABLE ONLY public.domain_posts_e621_aux ALTER COLUMN base_table_id SET DEF
ALTER TABLE ONLY public.domain_posts_fa_aux ALTER COLUMN base_table_id SET DEFAULT nextval('public.domain_posts_fa_aux_base_table_id_seq'::regclass);
--
-- Name: domain_posts_ib_aux base_table_id; Type: DEFAULT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_ib_aux ALTER COLUMN base_table_id SET DEFAULT nextval('public.domain_posts_ib_aux_base_table_id_seq'::regclass);
--
-- Name: domain_twitter_tweets id; Type: DEFAULT; Schema: public; Owner: -
--
@@ -3083,6 +3136,14 @@ ALTER TABLE ONLY public.domain_posts_fa_aux
ADD CONSTRAINT domain_posts_fa_aux_pkey PRIMARY KEY (base_table_id);
--
-- Name: domain_posts_ib_aux domain_posts_ib_aux_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_ib_aux
ADD CONSTRAINT domain_posts_ib_aux_pkey PRIMARY KEY (base_table_id);
--
-- Name: domain_posts domain_posts_pkey; Type: CONSTRAINT; Schema: public; Owner: -
--
@@ -4040,6 +4101,20 @@ CREATE INDEX index_domain_posts_fa_aux_on_base_table_id ON public.domain_posts_f
CREATE UNIQUE INDEX index_domain_posts_fa_aux_on_fa_id ON public.domain_posts_fa_aux USING btree (fa_id);
--
-- Name: index_domain_posts_ib_aux_on_base_table_id; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_posts_ib_aux_on_base_table_id ON public.domain_posts_ib_aux USING btree (base_table_id);
--
-- Name: index_domain_posts_ib_aux_on_ib_id; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_posts_ib_aux_on_ib_id ON public.domain_posts_ib_aux USING btree (ib_id);
--
-- Name: index_domain_posts_on_posted_at; Type: INDEX; Schema: public; Owner: -
--
@@ -5111,6 +5186,14 @@ ALTER TABLE ONLY public.domain_users_inkbunny_aux
ADD CONSTRAINT fk_rails_304ea0307f FOREIGN KEY (base_table_id) REFERENCES public.domain_users(id);
--
-- Name: domain_posts_ib_aux fk_rails_3762390d41; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_ib_aux
ADD CONSTRAINT fk_rails_3762390d41 FOREIGN KEY (shallow_update_log_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: http_log_entries fk_rails_42f35e9da0; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5135,6 +5218,14 @@ ALTER TABLE ONLY public.domain_user_user_follows
ADD CONSTRAINT fk_rails_4b2ab65400 FOREIGN KEY (from_id) REFERENCES public.domain_users(id);
--
-- Name: domain_posts_ib_aux fk_rails_5ee2c344bd; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_ib_aux
ADD CONSTRAINT fk_rails_5ee2c344bd FOREIGN KEY (deep_update_log_entry_id) REFERENCES public.http_log_entries(id);
--
-- Name: domain_twitter_medias fk_rails_5fffa41fa6; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5239,6 +5330,14 @@ ALTER TABLE ONLY public.domain_users_e621_aux
ADD CONSTRAINT fk_rails_b5bacbced6 FOREIGN KEY (base_table_id) REFERENCES public.domain_users(id);
--
-- Name: domain_posts_ib_aux fk_rails_b94b311254; Type: FK CONSTRAINT; Schema: public; Owner: -
--
ALTER TABLE ONLY public.domain_posts_ib_aux
ADD CONSTRAINT fk_rails_b94b311254 FOREIGN KEY (base_table_id) REFERENCES public.domain_posts(id);
--
-- Name: domain_posts_fa_aux fk_rails_be2be2e955; Type: FK CONSTRAINT; Schema: public; Owner: -
--
@@ -5350,6 +5449,7 @@ ALTER TABLE ONLY public.domain_twitter_tweets
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20250727173100'),
('20250726051748'),
('20250726051451'),
('20250725192431'),

View File

@@ -27,7 +27,8 @@ namespace :fa do
start_at = ENV["start_at"]
if start_at.is_a?(String) && start_at == "last"
start_at = Domain::Post::FaPost.where(state: :ok).maximum(:fa_id) - 1000
Domain::Post::FaPost.last
start_at = DomainPostsFaAux.where(state: :ok).maximum(:fa_id) - 1000
start_at = 0 if start_at < 0
stop_at =
ENV["stop_at"]&.to_i || GlobalState.get(global_state_key)&.to_i ||

9
sorbet/rbi/shims/aux.rbi Normal file
View File

@@ -0,0 +1,9 @@
# typed: strict
class DomainPostsFaAux < ActiveRecord::Base
extend T::Sig
sig { returns(T.nilable(Integer)) }
def fa_id
end
end

View File

@@ -11,7 +11,6 @@ end
RSpec.describe BlobEntriesController, type: :controller do
render_views
let(:blob_file) do
create(
:blob_file,

View File

@@ -0,0 +1,106 @@
# typed: false
require "rails_helper"
RSpec.describe Domain::UsersController, type: :controller do
render_views
# Create a real user with admin role for tests that require authentication
let(:admin_user) { create(:user, :admin) }
# Shared examples for common show behavior across all user types
shared_examples "show action for user type" do |user_factory, param_prefix, param_attr|
let(:domain_user) { create(user_factory) }
let(:composite_param) { "#{param_prefix}@#{domain_user.send(param_attr)}" }
context "when user exists" do
it "returns a successful response" do
get :show, params: { id: composite_param }
expect(response).to be_successful
end
it "renders the show template" do
get :show, params: { id: composite_param }
expect(response).to render_template(:show)
end
it "sets the @user instance variable" do
get :show, params: { id: composite_param }
expect(assigns(:user)).to eq(domain_user)
end
it "authorizes the user" do
expect(controller).to receive(:authorize).with(domain_user)
get :show, params: { id: composite_param }
end
it "does not require authentication" do
# Ensure no user is signed in
sign_out :user
get :show, params: { id: composite_param }
expect(response).to be_successful
end
end
context "when user does not exist" do
let(:invalid_param) { "#{param_prefix}@nonexistent" }
it "raises ActiveRecord::RecordNotFound" do
expect { get :show, params: { id: invalid_param } }.to raise_error(
ActiveRecord::RecordNotFound,
)
end
end
context "with invalid composite parameter format" do
it "raises ActionController::BadRequest for malformed param" do
expect { get :show, params: { id: "invalid_format" } }.to raise_error(
ActionController::BadRequest,
/invalid id/,
)
end
it "raises ActionController::BadRequest for unknown model type" do
expect { get :show, params: { id: "unknown@test" } }.to raise_error(
ActionController::BadRequest,
/unknown model type/,
)
end
end
end
describe "GET #show" do
before do
# Mock authorization to allow all actions for these tests
allow(controller).to receive(:authorize).and_return(true)
end
context "for Domain::User::FaUser" do
include_examples "show action for user type",
:domain_user_fa_user,
"fa",
:url_name
end
context "for Domain::User::E621User" do
include_examples "show action for user type",
:domain_user_e621_user,
"e621",
:name
end
context "for Domain::User::InkbunnyUser" do
include_examples "show action for user type",
:domain_user_inkbunny_user,
"ib",
:name
end
context "param configuration" do
it "uses the correct param config for user_id_param" do
param_config = described_class.param_config
expect(param_config.user_id_param).to eq(:id)
end
end
end
end

View File

@@ -12,6 +12,15 @@ end
describe Domain::Inkbunny::Job::StaticFileJob do
let(:http_client_mock) { instance_double("::Scraper::HttpClient") }
before { Scraper::ClientFactory.http_client_mock = http_client_mock }
let(:creator) do
create(:domain_user_inkbunny_user, ib_id: 12_345, name: "TheUser")
end
let(:post) do
create(:domain_post_inkbunny_post, ib_id: 67_891).tap do |p|
p.creator = creator
p.save!
end
end
let(:file) do
Domain::PostFile::InkbunnyPostFile.create!(
{
@@ -25,15 +34,7 @@ describe Domain::Inkbunny::Job::StaticFileJob do
md5s: {
initial_file_md5: FileJobSpec::AN_IMAGE_MD5,
},
post:
Domain::Post::InkbunnyPost.create!(
ib_id: 67_891,
creator:
Domain::User::InkbunnyUser.create!(
ib_id: 12_345,
name: "TheUser",
),
),
post: post,
},
)
end

View File

@@ -40,42 +40,54 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
end
let!(:post_3104202) do
Domain::Post::InkbunnyPost.create!(
ib_id: 3_104_202,
creator: user_thendyart,
title: "Phantom Touch - Page 25",
posted_at: Time.parse("2023-08-27 21:31:40.365597+02"),
last_file_updated_at: Time.parse("2023-08-27 21:30:06.222262+02"),
num_files: 1,
rating: "adult",
submission_type: "comic",
)
Domain::Post::InkbunnyPost
.create!(
ib_id: 3_104_202,
title: "Phantom Touch - Page 25",
posted_at: Time.parse("2023-08-27 21:31:40.365597+02"),
last_file_updated_at: Time.parse("2023-08-27 21:30:06.222262+02"),
num_files: 1,
rating: "adult",
submission_type: "comic",
)
.tap do |p|
p.creator = user_thendyart
p.save!
end
end
let!(:post_3104200) do
Domain::Post::InkbunnyPost.create!(
ib_id: 3_104_200,
creator: user_seff,
title: "Camp Pines Sketch Dump (Aug 2023)",
posted_at: Time.parse("2023-08-27 21:30:59.308046+02"),
last_file_updated_at: Time.parse("2023-08-27 21:26:14.049+02"),
num_files: 4,
rating: "adult",
submission_type: "picture_pinup",
)
Domain::Post::InkbunnyPost
.create!(
ib_id: 3_104_200,
title: "Camp Pines Sketch Dump (Aug 2023)",
posted_at: Time.parse("2023-08-27 21:30:59.308046+02"),
last_file_updated_at: Time.parse("2023-08-27 21:26:14.049+02"),
num_files: 4,
rating: "adult",
submission_type: "picture_pinup",
)
.tap do |p|
p.creator = user_seff
p.save!
end
end
let!(:post_3104197) do
Domain::Post::InkbunnyPost.create!(
ib_id: 3_104_197,
creator: user_soulcentinel,
title: "Comm - BJ bird",
posted_at: Time.parse("2023-08-27 21:29:37.995264+02"),
last_file_updated_at: Time.parse("2023-08-27 21:24:23.653306+02"),
num_files: 1,
rating: "adult",
submission_type: "picture_pinup",
)
Domain::Post::InkbunnyPost
.create!(
ib_id: 3_104_197,
title: "Comm - BJ bird",
posted_at: Time.parse("2023-08-27 21:29:37.995264+02"),
last_file_updated_at: Time.parse("2023-08-27 21:24:23.653306+02"),
num_files: 1,
rating: "adult",
submission_type: "picture_pinup",
)
.tap do |p|
p.creator = user_soulcentinel
p.save!
end
end
let(:ib_post_ids) { [3_104_202, 3_104_200, 3_104_197] }
@@ -349,16 +361,20 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
end
let!(:post_1047334) do
Domain::Post::InkbunnyPost.create!(
ib_id: 1_047_334,
creator: user_zzreg,
title: "New Submission",
posted_at: Time.parse("2016-03-13 22:18:52.32319+01"),
last_file_updated_at: Time.parse("2016-03-13 22:18:52.32319+01"),
num_files: 1,
rating: "general",
submission_type: "picture_pinup",
)
Domain::Post::InkbunnyPost
.create!(
ib_id: 1_047_334,
title: "New Submission",
posted_at: Time.parse("2016-03-13 22:18:52.32319+01"),
last_file_updated_at: Time.parse("2016-03-13 22:18:52.32319+01"),
num_files: 1,
rating: "general",
submission_type: "picture_pinup",
)
.tap do |p|
p.creator = user_zzreg
p.save!
end
end
it "updates post with new file information" do
@@ -440,9 +456,11 @@ describe Domain::Inkbunny::Job::UpdatePostsJob do
create(
:domain_post_inkbunny_post,
ib_id: 2_637_105,
creator: user_friar,
num_files: 5,
)
).tap do |p|
p.creator = user_friar
p.save!
end
end
it "handles files with null MD5 sums correctly" do

View File

@@ -80,6 +80,7 @@ RSpec.configure do |config|
# Add FactoryBot methods
config.include FactoryBot::Syntax::Methods
config.render_views
end
require "spec_util"