dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip for migrator for users

Browse Source
This commit is contained in:
Dylan Knutson
2025-01-27 05:38:38 +00:00
parent 414b56ab69
commit dc5063bb23
7 changed files with 226 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
app/lib/domain/fa/migrate_users_to_graph.rb
View File

@@ -12,6 +12,12 @@ class Domain::Fa::MigrateUsersToGraph
Domain::Fa::User.find_each(start: @start_at_id) do |user|
migrate_user(user)
end
relation = Domain::Fa::Follow
relation = relation.where("follower_id >= ?", @start_at_id) if @start_at_id
relation
.order(follower_id: :asc)
.find_each { |follow| migrate_follow(follow) }
end
sig { params(user: Domain::Fa::User).void }
@@ -20,6 +26,23 @@ class Domain::Fa::MigrateUsersToGraph
id: user.id,
url_name: user.url_name,
name: user.name,
scanned_page_at: user.scanned_page_at,
scanned_gallery_at: user.scanned_gallery_at,
scanned_follows_at: user.scanned_follows_at,
scanned_favorite_posts_at: user.scanned_favs_at,
scanned_incremental_at: user.scanned_incremental_at,
)
end
sig { params(follow: Domain::Fa::Follow).void }
def migrate_follow(follow)
ReduxGraph::Edge::Fa::UserFollowsUser.upsert(
{
from_node_type: ReduxGraph::Node::Fa::User,
from_node_id: follow.follower_id,
to_node_type: ReduxGraph::Node::Fa::User,
to_node_id: follow.followed_id,
},
)
end
end

14
app/lib/graph/create_graph_table.rb
View File

@@ -67,13 +67,13 @@ module Graph
SQL
end
partition_table_name = "#{graph_name}_#{kind_plural}_#{partition_name}"
dir.down { execute <<-SQL }
-- ALTER TABLE #{graph_name}_#{kind_plural}
-- DETACH PARTITION #{partition_table_name};
DROP TABLE IF EXISTS #{partition_table_name};
SQL
# TODO: remove this once we have a way to drop partitions
# partition_table_name = "#{graph_name}_#{kind_plural}_#{partition_name}"
# dir.down { execute <<-SQL }
# -- ALTER TABLE #{graph_name}_#{kind_plural}
# -- DETACH PARTITION #{partition_table_name};
# -- DROP TABLE IF EXISTS #{partition_table_name};
# SQL
end
end

4
app/models/redux_graph/edge/fa/user_follows_user.rb Normal file
View File

@@ -0,0 +1,4 @@
# typed: strict
class ReduxGraph::Edge::Fa::UserFollowsUser < ReduxGraph::Edge
self.partition = :fa_user_follows_user
end

26
app/models/redux_graph/node/fa/user.rb
View File

@@ -6,12 +6,13 @@ module ReduxGraph::Node::Fa
has_one_node :profile_description,
edge_type: ReduxGraph::Edge::Common::NodeHasDescription,
node_type: ReduxGraph::Node::Common::ContentDescription
has_many_nodes :posts,
has_many_nodes :created_posts,
edge_type: ReduxGraph::Edge::Common::CreatorHasPost,
node_type: Post,
inverse_of: :creator
has_many_nodes :favorites,
has_many_nodes :favorite_posts,
edge_type: ReduxGraph::Edge::Common::UserHasFavorite,
node_type: Post,
inverse_of: :favorited_by
@@ -20,15 +21,26 @@ module ReduxGraph::Node::Fa
edge_type: ReduxGraph::Edge::Common::UserHasAvatar,
node_type: ReduxGraph::Node::Common::UserAvatar
has_many_nodes :users_being_followed,
edge_type: ReduxGraph::Edge::Fa::UserFollowsUser,
node_type: User,
inverse_of: :users_following_me
has_many_nodes :users_following_me,
edge_type: ReduxGraph::Edge::Fa::UserFollowsUser,
node_type: User,
inverse_of: :users_being_followed
virtual_column :url_name, Graph::ColumnType::String
virtual_column :name, Graph::ColumnType::String
virtual_column :user_page_scanned_at, Graph::ColumnType::Time
virtual_column :gallery_scanned_at, Graph::ColumnType::Time
virtual_column :favorite_posts_scanned_at, Graph::ColumnType::Time
virtual_column :users_being_followed_scanned_at, Graph::ColumnType::Time
virtual_column :users_following_me_scanned_at, Graph::ColumnType::Time
virtual_column :scanned_page_at, Graph::ColumnType::DateTime
virtual_column :scanned_gallery_at, Graph::ColumnType::DateTime
virtual_column :scanned_follows_at, Graph::ColumnType::DateTime
virtual_column :scanned_favorite_posts_at, Graph::ColumnType::DateTime
virtual_column :scanned_incremental_at, Graph::ColumnType::DateTime
validates :url_name, presence: true
validates_presence_of :name, :url_name
end
end

1
db/migrate/20250110040848_create_graph_domain_fa_enums.rb
View File

@@ -18,6 +18,7 @@ class CreateGraphDomainFaEnums < ActiveRecord::Migration[7.2]
register_type ReduxGraph::Edge::Common::NodeHasDescription
register_type ReduxGraph::Edge::Common::UserHasAvatar
register_type ReduxGraph::Edge::Common::UserHasFavorite
register_type ReduxGraph::Edge::Fa::UserFollowsUser
end
end
end

7
spec/factories/domain/fa/follows.rb Normal file
View File

@@ -0,0 +1,7 @@
# typed: false
FactoryBot.define do
factory :domain_fa_follow, class: "Domain::Fa::Follow" do
association :follower, factory: :domain_fa_user
association :followed, factory: :domain_fa_user
end
end

165
spec/lib/domain/fa/migrate_users_to_graph_spec.rb Normal file
View File

@@ -0,0 +1,165 @@
# typed: false
require "rails_helper"
RSpec.describe Domain::Fa::MigrateUsersToGraph do
describe "#run" do
it "migrates users to graph nodes" do
# Create test users
user1 =
create(
:domain_fa_user,
url_name: "artist1",
name: "Artist 1",
scanned_page_at: 1.day.ago,
scanned_gallery_at: 2.days.ago,
scanned_follows_at: 3.days.ago,
scanned_favs_at: 4.days.ago,
scanned_incremental_at: 5.days.ago,
)
user2 =
create(
:domain_fa_user,
url_name: "artist2",
name: "Artist 2",
scanned_page_at: 2.days.ago,
scanned_gallery_at: 3.days.ago,
scanned_follows_at: 4.days.ago,
scanned_favs_at: 5.days.ago,
scanned_incremental_at: 6.days.ago,
)
# Run migration
described_class.new.run
# Verify graph nodes were created correctly
graph_user1 = ReduxGraph::Node::Fa::User.find(user1.id)
expect(graph_user1.url_name).to eq("artist1")
expect(graph_user1.name).to eq("Artist 1")
expect(graph_user1.scanned_page_at).to be_within(1.second).of(
user1.scanned_page_at,
)
expect(graph_user1.scanned_gallery_at).to be_within(1.second).of(
user1.scanned_gallery_at,
)
expect(graph_user1.scanned_follows_at).to be_within(1.second).of(
user1.scanned_follows_at,
)
expect(graph_user1.scanned_favorite_posts_at).to be_within(1.second).of(
user1.scanned_favs_at,
)
expect(graph_user1.scanned_incremental_at).to be_within(1.second).of(
user1.scanned_incremental_at,
)
graph_user2 = ReduxGraph::Node::Fa::User.find(user2.id)
expect(graph_user2.url_name).to eq("artist2")
expect(graph_user2.name).to eq("Artist 2")
expect(graph_user2.scanned_page_at).to be_within(1.second).of(
user2.scanned_page_at,
)
expect(graph_user2.scanned_gallery_at).to be_within(1.second).of(
user2.scanned_gallery_at,
)
expect(graph_user2.scanned_follows_at).to be_within(1.second).of(
user2.scanned_follows_at,
)
expect(graph_user2.scanned_favorite_posts_at).to be_within(1.second).of(
user2.scanned_favs_at,
)
expect(graph_user2.scanned_incremental_at).to be_within(1.second).of(
user2.scanned_incremental_at,
)
end
it "migrates users starting from a specific ID" do
user1 = create(:domain_fa_user, url_name: "artist1", name: "Artist 1")
user2 = create(:domain_fa_user, url_name: "artist2", name: "Artist 2")
user3 = create(:domain_fa_user, url_name: "artist3", name: "Artist 3")
# Start migration from user2's ID
described_class.new(start_at_id: user2.id).run
# First user should not be migrated
expect { ReduxGraph::Node::Fa::User.find(user1.id) }.to raise_error(
ActiveRecord::RecordNotFound,
)
# Second and third users should be migrated
expect(ReduxGraph::Node::Fa::User.find(user2.id)).to be_present
expect(ReduxGraph::Node::Fa::User.find(user3.id)).to be_present
end
it "migrates user follows to graph edges" do
# Create users
user1 = create(:domain_fa_user, url_name: "artist1", name: "Artist 1")
user2 = create(:domain_fa_user, url_name: "artist2", name: "Artist 2")
user3 = create(:domain_fa_user, url_name: "artist3", name: "Artist 3")
# Create follow relationships
follow1 = create(:domain_fa_follow, follower: user1, followed: user2)
follow2 = create(:domain_fa_follow, follower: user2, followed: user3)
follow3 = create(:domain_fa_follow, follower: user3, followed: user1)
# Run migration
described_class.new.run
# Verify follows were migrated correctly
graph_user1 = ReduxGraph::Node::Fa::User.find(user1.id)
graph_user2 = ReduxGraph::Node::Fa::User.find(user2.id)
graph_user3 = ReduxGraph::Node::Fa::User.find(user3.id)
# Check users_being_followed relationships
expect(graph_user1.users_being_followed.pluck(:id)).to contain_exactly(
user2.id,
)
expect(graph_user2.users_being_followed.pluck(:id)).to contain_exactly(
user3.id,
)
expect(graph_user3.users_being_followed.pluck(:id)).to contain_exactly(
user1.id,
)
# Check users_following_me relationships
expect(graph_user1.users_following_me.pluck(:id)).to contain_exactly(
user3.id,
)
expect(graph_user2.users_following_me.pluck(:id)).to contain_exactly(
user1.id,
)
expect(graph_user3.users_following_me.pluck(:id)).to contain_exactly(
user2.id,
)
end
it "migrates follows starting from a specific user ID" do
user1 = create(:domain_fa_user, url_name: "artist1", name: "Artist 1")
user2 = create(:domain_fa_user, url_name: "artist2", name: "Artist 2")
user3 = create(:domain_fa_user, url_name: "artist3", name: "Artist 3")
follow1 = create(:domain_fa_follow, follower: user1, followed: user2)
follow2 = create(:domain_fa_follow, follower: user2, followed: user3)
follow3 = create(:domain_fa_follow, follower: user3, followed: user1)
# Start migration from user2's ID
described_class.new(start_at_id: user2.id).run
# Verify only follows from user2 and user3 were migrated
graph_user2 = ReduxGraph::Node::Fa::User.find(user2.id)
graph_user3 = ReduxGraph::Node::Fa::User.find(user3.id)
# Check follows were migrated correctly
expect(graph_user2.users_being_followed.pluck(:id)).to contain_exactly(
user3.id,
)
expect(graph_user3.users_being_followed.pluck(:id)).to contain_exactly(
user1.id,
)
# Check user1's follows were not migrated
expect { ReduxGraph::Node::Fa::User.find(user1.id) }.to raise_error(
ActiveRecord::RecordNotFound,
)
end
end
end