dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

source link url suffix matching

Browse Source
This commit is contained in:
Dylan Knutson
2025-02-22 05:40:26 +00:00
parent 2e1922c68f
commit b470d1a669
5 changed files with 158 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
app/helpers/domain/posts_helper.rb
View File

@@ -366,11 +366,21 @@ module Domain::PostsHelper
const :hosts, T::Array[String]
const :patterns, T::Array[Regexp]
const :find_proc, T.proc.params(id: String).returns(T.nilable(SourceResult))
const :find_proc,
T
.proc
.params(match: MatchData, url: String)
.returns(T.nilable(SourceResult))
end
FA_HOSTS = %w[www.furaffinity.net furaffinity.net]
IB_HOSTS = %w[www.inkbunny.net inkbunny.net]
FA_HOSTS = %w[*.furaffinity.net furaffinity.net]
FA_CDN_HOSTS = %w[d.furaffinity.net *.facdn.net facdn.net]
IB_HOSTS = %w[*.inkbunny.net inkbunny.net]
IB_CDN_HOSTS = %w[*.ib.metapix.net ib.metapix.net]
URL_SUFFIX_QUERY = T.let(<<-SQL.strip.chomp.freeze, String)
reverse(lower(json_attributes->>'url_str')) LIKE (reverse(lower(?)) || '%')
SQL
MATCHERS =
T.let(
@@ -379,8 +389,27 @@ module Domain::PostsHelper
SourceMatcher.new(
hosts: FA_HOSTS,
patterns: [%r{/view/(\d+)/?}],
find_proc: ->(id) do
if post = Domain::Post::FaPost.find_by(fa_id: id)
find_proc: ->(match, _) do
if post = Domain::Post::FaPost.find_by(fa_id: match[1])
SourceResult.new(model: post, title: post.title_for_view)
end
end,
),
# Furaffinity posts via direct file URL
SourceMatcher.new(
hosts: FA_CDN_HOSTS,
patterns: [//],
find_proc: ->(_, url) do
url = Addressable::URI.parse(url)
post_file =
Domain::PostFile.where(
"#{URL_SUFFIX_QUERY} OR #{URL_SUFFIX_QUERY}",
"d.furaffinity.net#{url.path}",
"facdn.net#{url.path}",
).first
if post_file && (post = post_file.post)
SourceResult.new(model: post, title: post.title_for_view)
end
end,
@@ -389,8 +418,8 @@ module Domain::PostsHelper
SourceMatcher.new(
hosts: FA_HOSTS,
patterns: [%r{/user/([^/]+)/?}],
find_proc: ->(url_name) do
if user = Domain::User::FaUser.find_by(url_name: url_name)
find_proc: ->(match, _) do
if user = Domain::User::FaUser.find_by(url_name: match[1])
SourceResult.new(
model: user,
title: user.name_for_view || "unknown",
@@ -402,21 +431,38 @@ module Domain::PostsHelper
SourceMatcher.new(
hosts: IB_HOSTS,
patterns: [%r{/s/(\d+)/?}, %r{/submissionview\.php\?id=(\d+)/?}],
find_proc: ->(id) do
if post = Domain::Post::InkbunnyPost.find_by(ib_id: id)
find_proc: ->(match, _) do
if post = Domain::Post::InkbunnyPost.find_by(ib_id: match[1])
SourceResult.new(model: post, title: post.title_for_view)
end
end,
),
# Inkbunny posts via direct file URL
SourceMatcher.new(
hosts: IB_CDN_HOSTS,
patterns: [//],
find_proc: ->(_, url) do
url = Addressable::URI.parse(url)
if post_file =
Domain::PostFile.where(
"#{URL_SUFFIX_QUERY}",
"ib.metapix.net#{url.path}",
).first
if post = post_file.post
SourceResult.new(model: post, title: post.title_for_view)
end
end
end,
),
# Inkbunny users
SourceMatcher.new(
hosts: IB_HOSTS,
patterns: [%r{/(\w+)/?$}],
find_proc: ->(name) do
find_proc: ->(match, _) do
if user =
Domain::User::InkbunnyUser.where(
"lower(json_attributes->>'name') = lower(?)",
name,
match[1],
).first
SourceResult.new(
model: user,
@@ -437,12 +483,16 @@ module Domain::PostsHelper
source.downcase!
source = "https://" + source unless source.include?("://")
uri = URI.parse(source)
uri_host = uri.host
return nil if uri_host.blank?
for matcher in MATCHERS
if matcher.hosts.include?(uri.host)
if matcher.hosts.any? { |host|
File.fnmatch?(host, uri_host, File::FNM_PATHNAME)
}
for pattern in matcher.patterns
if (match = pattern.match(uri.to_s)) && (id = match[1])
object = matcher.find_proc.call(id)
if (match = pattern.match(uri.to_s))
object = matcher.find_proc.call(match, uri.to_s)
return nil unless object
model = object.model

4
app/views/domain/posts/e621/_source_link.html.erb
View File

@@ -2,7 +2,7 @@
<% source = "https://#{source}" %>
<% end %>
<div class="flex items-center justify-between gap-2 py-1 first:pt-0 last:pb-0">
<div class="flex items-center gap-2 justify-center h-full max-w-full">
<div class="flex items-center gap-2 justify-center h-full min-w-0 flex-shrink">
<% if icon_path = icon_asset_for_url(source) %>
<%= image_tag icon_path, class: "h-6 w-6 mt-1 flex-shrink-0" %>
<% else %>
@@ -16,7 +16,7 @@
<% if lfs = link_for_source(source) %>
<%= link_to lfs.model_path,
class:
"flex items-center rounded overflow-hidden bg-blue-600 text-sm text-white hover:bg-blue-700 h-8" do %>
"flex items-center rounded overflow-hidden bg-blue-600 text-sm text-white hover:bg-blue-700 h-8 flex-shrink-0" do %>
<% if lfs.icon_path %>
<%= image_tag lfs.icon_path, class: "h-full" %>
<% end %>

20
db/migrate/20250222035939_add_url_index_to_domain_post_files.rb
View File

@@ -4,13 +4,27 @@ class AddUrlIndexToDomainPostFiles < ActiveRecord::Migration[7.2]
sig { void }
def change
add_json_index :domain_post_files, :url_str
up_only { execute "SET default_tablespace = mirai" }
# prefix search file URLs
add_index :domain_post_files,
"lower((json_attributes->>'url_str')::text) text_pattern_ops",
name: "idx_domain_post_files_on_url_str_lower"
# suffix search file URLs
add_index :domain_post_files,
"reverse(lower((json_attributes->>'url_str')::text)) text_pattern_ops",
name: "idx_domain_post_files_on_url_str_lower_reverse"
# InkbunnyUser#name
add_index :domain_users,
"lower((json_attributes->>'name')::text)",
"lower((json_attributes->>'name')::text) text_pattern_ops",
name: "idx_domain_users_inkbunny_on_name_lower",
where: "type = 'Domain::User::InkbunnyUser'"
# E621User#name
add_index :domain_users,
"lower((json_attributes->>'name')::text)",
"lower((json_attributes->>'name')::text) text_pattern_ops",
name: "idx_domain_users_e621_on_name_lower",
where: "type = 'Domain::User::E621User'"
end

21
db/structure.sql
View File

@@ -5560,30 +5560,33 @@ CREATE UNIQUE INDEX idx_domain_inkbunny_users_on_ib_id ON public.domain_users US
CREATE UNIQUE INDEX idx_domain_inkbunny_users_on_name ON public.domain_users USING btree (((json_attributes ->> 'name'::text))) WHERE (type = 'Domain::User::InkbunnyUser'::public.domain_user_type);
SET default_tablespace = '';
--
-- Name: idx_domain_post_files_on_url_str; Type: INDEX; Schema: public; Owner: -
-- Name: idx_domain_post_files_on_url_str_lower; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE INDEX idx_domain_post_files_on_url_str ON public.domain_post_files USING btree (((json_attributes ->> 'url_str'::text)));
CREATE INDEX idx_domain_post_files_on_url_str_lower ON public.domain_post_files USING btree (lower((json_attributes ->> 'url_str'::text)) text_pattern_ops);
--
-- Name: idx_domain_users_e621_on_name_lower; Type: INDEX; Schema: public; Owner: -
-- Name: idx_domain_post_files_on_url_str_lower_reverse; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE INDEX idx_domain_users_e621_on_name_lower ON public.domain_users USING btree (lower((json_attributes ->> 'name'::text))) WHERE (type = 'Domain::User::E621User'::public.domain_user_type);
CREATE INDEX idx_domain_post_files_on_url_str_lower_reverse ON public.domain_post_files USING btree (reverse(lower((json_attributes ->> 'url_str'::text))) text_pattern_ops);
--
-- Name: idx_domain_users_inkbunny_on_name_lower; Type: INDEX; Schema: public; Owner: -
-- Name: idx_domain_users_e621_on_name_lower; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE INDEX idx_domain_users_inkbunny_on_name_lower ON public.domain_users USING btree (lower((json_attributes ->> 'name'::text))) WHERE (type = 'Domain::User::InkbunnyUser'::public.domain_user_type);
CREATE INDEX idx_domain_users_e621_on_name_lower ON public.domain_users USING btree (lower((json_attributes ->> 'name'::text)) text_pattern_ops) WHERE (type = 'Domain::User::E621User'::public.domain_user_type);
SET default_tablespace = mirai;
--
-- Name: idx_domain_users_inkbunny_on_name_lower; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai
--
CREATE INDEX idx_domain_users_inkbunny_on_name_lower ON public.domain_users USING btree (lower((json_attributes ->> 'name'::text)) text_pattern_ops) WHERE (type = 'Domain::User::InkbunnyUser'::public.domain_user_type);
--
-- Name: idx_domain_users_on_migrated_user_favs_at; Type: INDEX; Schema: public; Owner: -; Tablespace: mirai

63
spec/helpers/domain/posts_helper_spec.rb
View File

@@ -460,6 +460,69 @@ RSpec.describe Domain::PostsHelper, type: :helper do
expect(url).to eq_link_for_source(model: user, title: "artistone")
end
end
%w[
https://ib.metapix.net/s/abc/3210.jpg
http://ib.metapix.net/s/abc/3210.jpg
http://www.ib.metapix.net/s/abc/3210.jpg
ib.metapix.net/s/abc/3210.jpg
gb2.ib.metapix.net/s/abc/3210.jpg
gb2.ib.metapix.net/s/abc/3210.jpg
https://gb2.ib.metapix.net/s/abc/3210.jpg
].each do |url|
it "works with inkbunny file URL #{url}" do
post =
create(
:domain_post_inkbunny_post,
ib_id: "123456",
title: "Post Title",
)
post.files.create!(url_str: url, ib_id: "78910")
expect(url).to eq_link_for_source(model: post, title: "Post Title")
end
end
%w[
https://d.facdn.net/art/dragonkai/1574894326/1574894326.dragonkai_tiefling_warlock_fortune.png
https://facdn.net/art/dragonkai/1574894326/1574894326.dragonkai_tiefling_warlock_fortune.png
www.facdn.net/art/dragonkai/1574894326/1574894326.dragonkai_tiefling_warlock_fortune.png
facdn.net/art/dragonkai/1574894326/1574894326.dragonkai_tiefling_warlock_fortune.png
].each do |url|
it "works with FA file URL #{url}" do
post =
create(:domain_post_fa_post, fa_id: "123456", title: "Post Title")
post.files.create!(url_str: url)
expect(url).to eq_link_for_source(model: post, title: "Post Title")
end
end
it "has the right avatar url" do
user =
create(
:domain_user_fa_user,
url_name: "artistone",
name: "Artist One",
)
create(:domain_user_avatar, user:)
link_for_source =
helper.link_for_source("https://www.furaffinity.net/user/artistone/")
expect(link_for_source).to be_present
expect(link_for_source.icon_path).to eq(
helper.domain_user_avatar_img_src_path(
user.avatar,
thumb: "64-avatar",
),
)
end
it "has the right model path" do
create(:domain_post_fa_post, fa_id: "123456", title: "Post Title")
link_for_source =
helper.link_for_source("https://www.furaffinity.net/view/123456/")
expect(link_for_source).to be_present
expect(link_for_source.model_path).to eq("/posts/fa/123456")
end
end
end
end