dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tests for source links

Browse Source
This commit is contained in:
Dylan Knutson
2025-02-22 04:48:14 +00:00
parent 091b0ef02d
commit 00d90312dc
9 changed files with 225 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
app/helpers/domain/posts_helper.rb
View File

@@ -365,7 +365,7 @@ module Domain::PostsHelper
include T::Struct::ActsAsComparable
const :hosts, T::Array[String]
const :pattern, Regexp
const :patterns, T::Array[Regexp]
const :find_proc, T.proc.params(id: String).returns(T.nilable(SourceResult))
end
@@ -375,18 +375,20 @@ module Domain::PostsHelper
MATCHERS =
T.let(
[
# Furaffinity posts
SourceMatcher.new(
hosts: FA_HOSTS,
pattern: %r{/view/(\d+)/?},
patterns: [%r{/view/(\d+)/?}],
find_proc: ->(id) do
if post = Domain::Post::FaPost.find_by(fa_id: id)
SourceResult.new(model: post, title: post.title_for_view)
end
end,
),
# Furaffinity users
SourceMatcher.new(
hosts: FA_HOSTS,
pattern: %r{/user/([^/]+)/?},
patterns: [%r{/user/([^/]+)/?}],
find_proc: ->(url_name) do
if user = Domain::User::FaUser.find_by(url_name: url_name)
SourceResult.new(
@@ -396,22 +398,24 @@ module Domain::PostsHelper
end
end,
),
# Inkbunny posts
SourceMatcher.new(
hosts: IB_HOSTS,
pattern: %r{/s/(\d+)/?},
patterns: [%r{/s/(\d+)/?}, %r{/submissionview\.php\?id=(\d+)/?}],
find_proc: ->(id) do
if post = Domain::Post::InkbunnyPost.find_by(ib_id: id)
SourceResult.new(model: post, title: post.title_for_view)
end
end,
),
# Inkbunny users
SourceMatcher.new(
hosts: IB_HOSTS,
pattern: %r{/([^/]+)/?},
patterns: [%r{/(\w+)/?$}],
find_proc: ->(name) do
if user =
Domain::User::InkbunnyUser.where(
"json_attributes->>'name' ILIKE ?",
"lower(json_attributes->>'name') = lower(?)",
name,
).first
SourceResult.new(
@@ -427,33 +431,43 @@ module Domain::PostsHelper
sig { params(source: String).returns(T.nilable(LinkForSource)) }
def link_for_source(source)
return nil if source.blank?
# normalize the source to a lowercase string with a protocol
source.downcase!
source = "https://" + source unless source.include?("://")
uri = URI.parse(source)
for matcher in MATCHERS
if matcher.hosts.include?(uri.host)
if (match = matcher.pattern.match(uri.path)) && (id = match[1])
object = matcher.find_proc.call(id)
return nil unless object
model = object.model
for pattern in matcher.patterns
if (match = pattern.match(uri.to_s)) && (id = match[1])
object = matcher.find_proc.call(id)
return nil unless object
model = object.model
if model.is_a?(Domain::Post)
model_path = domain_post_path(model)
elsif model.is_a?(Domain::User)
model_path = domain_user_path(model)
icon_path =
domain_user_avatar_img_src_path(model.avatar, thumb: "64-avatar")
else
model_path = "#"
end
if model.is_a?(Domain::Post)
model_path = domain_post_path(model)
elsif model.is_a?(Domain::User)
model_path = domain_user_path(model)
icon_path =
domain_user_avatar_img_src_path(
model.avatar,
thumb: "64-avatar",
)
else
model_path = "#"
end
return(
LinkForSource.new(
model:,
title: object.title,
model_path:,
icon_path:,
return(
LinkForSource.new(
model:,
title: object.title,
model_path:,
icon_path:,
)
)
)
end
end
end
end

2
app/models/domain/user/fa_user.rb
View File

@@ -170,7 +170,7 @@ class Domain::User::FaUser < Domain::User
sig { override.returns(T.nilable(String)) }
def name_for_view
url_name
name || url_name
end
sig { override.returns(Symbol) }

1
app/views/domain/posts/default/_section_title_details.erb
View File

@@ -0,0 +1 @@
<%# by default, no additional details are shown %>

20
config/initializers/migration_utilities.rb Normal file
View File

@@ -0,0 +1,20 @@
# typed: strict
class ActiveRecord::Migration
extend T::Sig
sig do
params(
table_name: T.any(String, Symbol),
column_name: T.any(String, Symbol),
options: T::Hash[Symbol, T.untyped],
).void
end
def add_json_index(table_name, column_name, options = {})
add_index(
table_name,
"(json_attributes->>'#{column_name}')",
name: "idx_#{table_name}_on_#{column_name}",
**options,
)
end
end

17
db/migrate/20250222035939_add_url_index_to_domain_post_files.rb Normal file
View File

@@ -0,0 +1,17 @@
# typed: strict
class AddUrlIndexToDomainPostFiles < ActiveRecord::Migration[7.2]
extend T::Sig
sig { void }
def change
add_json_index :domain_post_files, :url_str
add_index :domain_users,
"lower((json_attributes->>'name')::text)",
name: "idx_domain_users_inkbunny_on_name_lower",
where: "type = 'Domain::User::InkbunnyUser'"
add_index :domain_users,
"lower((json_attributes->>'name')::text)",
name: "idx_domain_users_e621_on_name_lower",
where: "type = 'Domain::User::E621User'"
end
end

26
db/structure.sql
View File

@@ -5560,6 +5560,31 @@ CREATE UNIQUE INDEX idx_domain_inkbunny_users_on_ib_id ON public.domain_users US
CREATE UNIQUE INDEX idx_domain_inkbunny_users_on_name ON public.domain_users USING btree (((json_attributes ->> 'name'::text))) WHERE (type = 'Domain::User::InkbunnyUser'::public.domain_user_type);
SET default_tablespace = '';
--
-- Name: idx_domain_post_files_on_url_str; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX idx_domain_post_files_on_url_str ON public.domain_post_files USING btree (((json_attributes ->> 'url_str'::text)));
--
-- Name: idx_domain_users_e621_on_name_lower; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX idx_domain_users_e621_on_name_lower ON public.domain_users USING btree (lower((json_attributes ->> 'name'::text))) WHERE (type = 'Domain::User::E621User'::public.domain_user_type);
--
-- Name: idx_domain_users_inkbunny_on_name_lower; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX idx_domain_users_inkbunny_on_name_lower ON public.domain_users USING btree (lower((json_attributes ->> 'name'::text))) WHERE (type = 'Domain::User::InkbunnyUser'::public.domain_user_type);
SET default_tablespace = mirai;
--
-- Name: idx_domain_users_on_migrated_user_favs_at; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
@@ -8565,6 +8590,7 @@ ALTER TABLE ONLY public.domain_twitter_tweets
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20250222035939'),
('20250206224121'),
('20250203235035'),
('20250131060105'),

96
spec/helpers/domain/posts_helper_spec.rb
View File

@@ -2,6 +2,8 @@
require "rails_helper"
RSpec.describe Domain::PostsHelper, type: :helper do
LinkForSource = Domain::PostsHelper::LinkForSource
describe "#page_str" do
context "when page is greater than 1" do
it "returns page string" do
@@ -366,4 +368,98 @@ RSpec.describe Domain::PostsHelper, type: :helper do
end
end
end
describe "#link_for_source" do
it "returns nil for invalid URLs" do
expect(helper.link_for_source("invalid-url")).to be_nil
end
it "returns nil for URLs that don't match any source matchers" do
expect(helper.link_for_source("https://www.google.com/")).to be_nil
end
describe "FA link handling" do
it "returns nil for FA URLs that are not found" do
expect(
helper.link_for_source("https://www.furaffinity.net/view/123456/"),
).to be_nil
end
%w[
https://www.furaffinity.net/view/123/
https://www.furaffinity.net/view/123
furaffinity.net/view/123/
furaffinity.net/view/123
www.furaffinity.net/view/123/
www.furaffinity.net/view/123
Furaffinity.net/view/123/
Furaffinity.net/view/123
].each do |url|
it "returns a link to FA post for #{url}" do
post = create(:domain_post_fa_post, fa_id: "123", title: "Post Title")
expect(url).to eq_link_for_source(model: post, title: "Post Title")
end
end
%w[
https://www.furaffinity.net/user/artistone/
https://www.furaffinity.net/user/artistone
https://furaffinity.net/user/artistone/
https://furaffinity.net/user/artistone
Furaffinity.net/user/artistone
furaffinity.net/user/artistone
www.furaffinity.net/user/artistone
].each do |url|
it "returns a link to FA user for #{url}" do
user =
create(
:domain_user_fa_user,
url_name: "artistone",
name: "Artist One",
)
expect(url).to eq_link_for_source(model: user, title: "Artist One")
end
end
%w[
inkbunny.net/s/123456-p3-#pictop
inkbunny.net/s/123456-p3-#pictop/
www.inkbunny.net/s/123456-p3-#pictop/
www.inkbunny.net/s/123456-p3-#pictop
Inkbunny.net/s/123456-p3-#pictop
inkbunny.net/submissionview.php?id=123456
http://inkbunny.net/submissionview.php?id=123456
http://www.inkbunny.net/submissionview.php?id=123456
].each do |url|
it "returns a link to inkbunny post for #{url}" do
post =
create(
:domain_post_inkbunny_post,
ib_id: "123456",
title: "Post Title",
)
expect(url).to eq_link_for_source(model: post, title: "Post Title")
end
end
%w[
https://www.inkbunny.net/user/artistone/
https://www.inkbunny.net/user/artistone
https://inkbunny.net/user/artistone/
https://inkbunny.net/user/artistone
http://www.inkbunny.net/user/ArtistOne
www.inkbunny.net/user/ArtistOne
Inkbunny.net/user/artistone
inkbunny.net/user/artistone
inkbunny.net/user/artistone/
www.inkbunny.net/user/artistone
www.inkbunny.net/user/artistone/
].each do |url|
it "returns a link to inkbunny user for #{url}" do
user = create(:domain_user_inkbunny_user, name: "artistone")
expect(url).to eq_link_for_source(model: user, title: "artistone")
end
end
end
end
end

1
spec/spec_helper.rb
View File

@@ -19,6 +19,7 @@ require "./spec/helpers/debug_helpers"
require "./spec/helpers/http_client_mock_helpers"
require "./spec/support/matchers/html_matchers"
require "./spec/support/matchers/job_matchers"
require "./spec/support/matchers/source_link_matchers"
require "rspec/sorbet"
RSpec::Sorbet.allow_doubles!
RSpec::Matchers.define_negated_matcher :not_change, :change

23
spec/support/matchers/source_link_matchers.rb Normal file
View File

@@ -0,0 +1,23 @@
# typed: false
RSpec::Matchers.define :eq_link_for_source do |model:, title:|
match do |actual_url|
actual = helper.link_for_source(actual_url)
expect(actual).to be_a(Domain::PostsHelper::LinkForSource)
expect(actual.model).to eq(model)
expect(actual.title).to eq(title)
end
failure_message do |actual_url|
actual = helper.link_for_source(actual_url)
if actual.nil?
"link for source was nil for url #{actual_url}"
elsif actual.model != model
"expected model #{model.to_gid.uri.to_s} to be #{actual.model.to_gid.uri.to_s} for url #{actual_url}"
elsif actual.title != title
"expected title #{title} to be #{actual.title} for url #{actual_url}"
else
"unknown failure"
end
end
end