dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

by descending post id

Browse Source
This commit is contained in:
Dylan Knutson
2025-08-13 08:18:47 +00:00
parent 6bb0b255fb
commit b33a267a83
13 changed files with 185 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/controllers/domain/posts_controller.rb
View File

@@ -281,10 +281,7 @@ class Domain::PostsController < DomainController
def posts_relation(starting_relation, skip_ordering: false)
relation = starting_relation
relation = T.unsafe(policy_scope(relation)).page(params[:page]).per(50)
relation =
relation.order(
"#{relation.klass.post_order_attribute} DESC, id DESC",
) unless skip_ordering
relation = relation.order("posted_at DESC NULLS LAST") unless skip_ordering
relation
end
end

10
app/helpers/domain/descriptions_helper.rb
View File

@@ -327,13 +327,15 @@ module Domain::DescriptionsHelper
link_text: String,
visual_style: String,
domain_icon: T::Boolean,
link_params: T::Hash[Symbol, T.untyped],
).returns(T::Hash[Symbol, T.untyped])
end
def props_for_post_hover_preview(
post,
link_text,
visual_style,
domain_icon: true
domain_icon: true,
link_params: {}
)
cache_key = [
post,
@@ -348,7 +350,11 @@ module Domain::DescriptionsHelper
linkText: link_text,
postId: post.to_param,
postTitle: post.title,
postPath: Rails.application.routes.url_helpers.domain_post_path(post),
postPath:
Rails.application.routes.url_helpers.domain_post_path(
post,
link_params,
),
postThumbnailPath: thumbnail_for_post_path(post),
postThumbnailAlt: "View on #{domain_name_for_model(post)}",
postDomainIcon: domain_icon ? domain_model_icon_path(post) : nil,

20
app/helpers/domain/posts_helper.rb
View File

@@ -366,6 +366,7 @@ module Domain::PostsHelper
IB_HOSTS = %w[*.inkbunny.net inkbunny.net]
IB_CDN_HOSTS = %w[*.ib.metapix.net ib.metapix.net]
E621_HOSTS = %w[www.e621.net e621.net]
BLUESKY_HOSTS = %w[bsky.app]
URL_SUFFIX_QUERY = T.let(<<-SQL.strip.chomp.freeze, String)
lower('url_str') = lower(?)
@@ -491,6 +492,25 @@ module Domain::PostsHelper
end
end,
),
# Bluesky posts
SourceMatcher.new(
hosts: BLUESKY_HOSTS,
patterns: [%r{/profile/([^/]+)/post/(\w+)}],
find_proc: ->(helper, match, _) do
handle_or_did = match[1]
post_rkey = match[2]
if handle_or_did.start_with?("did:")
did = handle_or_did
else
user = Domain::User::BlueskyUser.find_by(handle: handle_or_did)
did = user&.did
end
next unless did
at_uri = "at://#{did}/app.bsky.feed.post/#{post_rkey}"
post = Domain::Post::BlueskyPost.find_by(at_uri:)
SourceResult.new(model: post, title: post.title_for_view) if post
end,
),
],
T::Array[SourceMatcher],
)

3
app/lib/domain/fa/parser/page.rb
View File

@@ -122,6 +122,9 @@ class Domain::Fa::Parser::Page < Domain::Fa::Parser::Base
sig { returns(ActiveSupport::TimeZone) }
def logged_in_user_tz
# server default for unauthenticated requests
return ActiveSupport::TimeZone.new("America/New_York") unless logged_in?
case logged_in_user
when "zzreg", "cottoniq"
ActiveSupport::TimeZone.new("America/Los_Angeles")

2
app/models/domain/user/bluesky_user.rb
View File

@@ -72,7 +72,7 @@ class Domain::User::BlueskyUser < Domain::User
sig { override.returns(T.nilable(String)) }
def name_for_view
"@#{handle}"
display_name.present? ? display_name : "@#{handle}"
end
sig { override.returns(String) }

3
app/views/domain/has_description_html/_inline_link_domain_post.html.erb
View File

@@ -4,12 +4,13 @@
<% visual_style = local_assigns[:visual_style] || "sky-link" %>
<% link_text = local_assigns[:link_text] || post.title_for_view %>
<% domain_icon = local_assigns[:domain_icon].nil? ? true : local_assigns[:domain_icon] %>
<% link_params = local_assigns[:link_params] || {} %>
<%=
react_component(
"PostHoverPreviewWrapper",
{
prerender: false,
props: props_for_post_hover_preview(post, link_text, visual_style, domain_icon:),
props: props_for_post_hover_preview(post, link_text, visual_style, domain_icon:, link_params:),
html_options: {
class: link_classes_for_visual_style(visual_style)
}

5
app/views/domain/posts/default/_section_visualy_similar_posts.html.erb
View File

@@ -26,6 +26,7 @@
<% fprints.each_with_index do |similar_fingerprint, index| %>
<div class="grid grid-cols-subgrid col-span-full max-w-max py-1 px-2 gap-2">
<% post = similar_fingerprint.fingerprint.post_file.post %>
<% post_file_idx = post.files.index { |f| f.id == similar_fingerprint.fingerprint.post_file_id } %>
<% creator = post.class.has_creators? ? post.creator : post.primary_fallback_creator_for_view %>
<div class="text-md items-center flex justify-end">
<div class="w-full text-center font-medium <%= match_badge_classes(similar_fingerprint.similarity_percentage) %>">
@@ -33,7 +34,9 @@
</div>
</div>
<div class="text-md self-center">
<%= render "domain/has_description_html/inline_link_domain_post", post: post, visual_style: "sky-link" %>
<%= render "domain/has_description_html/inline_link_domain_post", post: post, visual_style: "sky-link", link_params: {
idx: post_file_idx != 0 ? post_file_idx : nil,
} %>
</div>
<% if creator %>
<div class="text-md items-center">

8
app/views/domain/users/bsky/_name.html.erb
View File

@@ -1,8 +1,14 @@
<div class="flex items-center gap-2">
<span class="text-slate-900 text-lg font-bold">
<% if user.display_name.present? %>
<span class="text-slate-800 text-lg font-semibold">
<%= user.display_name %>
</span>
<span class="text-sm font-normal text-slate-500" title="<%= user.did %>">
@<%= user.handle %>
</span>
<% else %>
<span class="text-slate-800 text-lg font-semibold" title="<%= user.did %>">
<%= user.handle %>
</span>
<% end %>
</div>

15
db/migrate/20250813000020_create_index_on_posts_posted_at_nulls_last.rb Normal file
View File

@@ -0,0 +1,15 @@
class CreateIndexOnPostsPostedAtNullsLast < ActiveRecord::Migration[7.2]
def up
execute <<-SQL
CREATE INDEX index_domain_posts_on_posted_at_desc_nulls_last
ON domain_posts
USING BTREE
(posted_at DESC NULLS LAST)
SQL
end
def down
execute <<-SQL
DROP INDEX index_domain_posts_on_posted_at_desc_nulls_last
SQL
end
end

8
db/structure.sql
View File

@@ -4453,6 +4453,13 @@ CREATE INDEX index_domain_posts_ib_aux_on_ib_id ON public.domain_posts_ib_aux US
CREATE INDEX index_domain_posts_on_posted_at ON public.domain_posts USING btree (posted_at);
--
-- Name: index_domain_posts_on_posted_at_desc_nulls_last; Type: INDEX; Schema: public; Owner: -
--
CREATE INDEX index_domain_posts_on_posted_at_desc_nulls_last ON public.domain_posts USING btree (posted_at DESC NULLS LAST);
--
-- Name: index_domain_posts_on_type; Type: INDEX; Schema: public; Owner: -
--
@@ -5885,6 +5892,7 @@ ALTER TABLE ONLY public.domain_twitter_tweets
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
('20250813000020'),
('20250812215907'),
('20250812214902'),
('20250812214415'),

18
rake/fa.rake
View File

@@ -288,4 +288,22 @@ namespace :fa do
Tasks::Fa::MigrateFaUserPostFavs.new.run_for_posts(start_at:, batch_size:)
end
end
desc "Backfill posted_at for FaPosts from url_str"
task backfill_posted_at_from_url_str: :environment do
progress = ProgressBar.create(total: nil, format: "%t: %c/%C %B %p%% %a %e")
Domain::Post::FaPost
.where(posted_at: nil)
.includes(:files)
.find_in_batches(batch_size: 100) do |batch|
ReduxApplicationRecord.transaction do
batch.each do |post|
if (posted_at = post.posted_at)
post.update(posted_at:)
progress.increment
end
end
end
end
end
end

93
spec/helpers/domain/posts_helper_spec.rb
View File

@@ -140,5 +140,98 @@ RSpec.describe Domain::PostsHelper, type: :helper do
expect(link_for_source.model_path).to eq("/posts/fa@123456")
end
end
describe "Bluesky link handling" do
it "returns nil for Bluesky URLs that are not found" do
expect(
helper.link_for_source(
"https://bsky.app/profile/user1.bsky.social/post/123456",
),
).to be_nil
end
%w[
https://bsky.app/profile/user1.bsky.social/post/123456
https://bsky.app/profile/user1.bsky.social/post/123456/
bsky.app/profile/user1.bsky.social/post/123456
bsky.app/profile/user1.bsky.social/post/123456/
Bsky.app/profile/user1.bsky.social/post/123456
Bsky.app/profile/user1.bsky.social/post/123456/
].each do |url|
it "returns a link to Bluesky post for #{url}" do
user = create(:domain_user_bluesky_user, handle: "user1.bsky.social")
post =
create(
:domain_post_bluesky_post,
creator: user,
rkey: "123456",
at_uri: "at://#{user.did}/app.bsky.feed.post/123456",
text: "Bluesky Post Title",
)
expect(url).to eq_link_for_source(
model: post,
title: "Bluesky Post Title",
)
end
end
%w[
https://bsky.app/profile/did:plc:1234567890/post/abcdef
https://bsky.app/profile/did:plc:1234567890/post/abcdef/
bsky.app/profile/did:plc:1234567890/post/abcdef
bsky.app/profile/did:plc:1234567890/post/abcdef/
Bsky.app/profile/did:plc:1234567890/post/abcdef
Bsky.app/profile/did:plc:1234567890/post/abcdef/
].each do |url|
it "returns a link to Bluesky post for DID-based URL #{url}" do
post =
create(
:domain_post_bluesky_post,
rkey: "abcdef",
at_uri: "at://did:plc:1234567890/app.bsky.feed.post/abcdef",
text: "DID Bluesky Post",
)
expect(url).to eq_link_for_source(
model: post,
title: "DID Bluesky Post",
)
end
end
it "has the right model path for Bluesky posts" do
user = create(:domain_user_bluesky_user, handle: "user1.bsky.social")
post =
create(
:domain_post_bluesky_post,
creator: user,
rkey: "123456",
at_uri: "at://#{user.did}/app.bsky.feed.post/123456",
text: "Bluesky Post Title",
)
link_for_source =
helper.link_for_source(
"https://bsky.app/profile/user1.bsky.social/post/123456",
)
expect(link_for_source).to be_present
expect(link_for_source.model_path).to eq("/posts/bsky@123456")
end
it "handles URLs with different case variations" do
user = create(:domain_user_bluesky_user, handle: "user1.bsky.social")
post =
create(
:domain_post_bluesky_post,
creator: user,
rkey: "123456",
at_uri: "at://#{user.did}/app.bsky.feed.post/123456",
text: "Case Test Post",
)
# Test that the method handles case variations in the host part
# but the handle part should match exactly as stored in the database
expect(
"BSKY.APP/profile/user1.bsky.social/post/123456",
).to eq_link_for_source(model: post, title: "Case Test Post")
end
end
end
end

1
spec/support/matchers/source_link_matchers.rb
View File

@@ -11,7 +11,6 @@ RSpec::Matchers.define :eq_link_for_source do |model:, title:|
failure_message do |actual_url|
actual = helper.link_for_source(actual_url)
if actual.nil?
binding.pry
"link for source was nil for url #{actual_url}"
elsif actual.model != model
"expected model #{model.to_gid.uri.to_s} to be #{actual.model.to_gid.uri.to_s} for url #{actual_url}"