dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add pundit-matchers gem and enhance indexed post handling

Browse Source 
- Added `pundit-matchers` gem to improve policy testing capabilities.
- Updated `BlobsController` to support a new "tiny" size option for avatars.
- Enhanced `IndexablePostsHelper` with a `show_path` method for different postable types.
- Refactored `IndexedPost` model to include methods for retrieving artist information and external links.
- Modified `Domain::E621::Post` model to initialize `tags_array` as a hash.
- Updated views for indexed posts to support new display formats (gallery and table).
- Improved test coverage with new user factory and updated specs for controller and job behaviors.
This commit is contained in:
Dylan Knutson
2024-12-27 21:32:11 +00:00
parent e1b3fa4401
commit efccf79f64
24 changed files with 334 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Gemfile
View File

@@ -90,6 +90,7 @@ group :test do
gem "shoulda-matchers"
gem "factory_bot_rails"
gem "parallel_tests"
gem "pundit-matchers", "~> 4.0"
end
gem "xdiff", path: "/gems/xdiff-rb"

6
Gemfile.lock
View File

@@ -270,6 +270,11 @@ GEM
nio4r (~> 2.0)
pundit (2.4.0)
activesupport (>= 3.0.0)
pundit-matchers (4.0.0)
rspec-core (~> 3.12)
rspec-expectations (~> 3.12)
rspec-mocks (~> 3.12)
rspec-support (~> 3.12)
raabro (1.4.0)
racc (1.8.1)
rack (2.2.10)
@@ -480,6 +485,7 @@ DEPENDENCIES
pry-stack_explorer
puma (~> 5.0)
pundit (~> 2.4)
pundit-matchers (~> 4.0)
rack-cors
rack-mini-profiler (~> 3.3)
rails (~> 7.2)

BIN
app/assets/images/domain-icons/x-twitter.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.1 KiB

2
app/controllers/blobs_controller.rb
View File

@@ -74,6 +74,8 @@ class BlobsController < ApplicationController
[32, 32]
when "64-avatar"
[64, 64]
when "tiny"
[100, 100]
when "small"
[400, 300]
when "medium"

1
app/helpers/domain/e621/posts_helper.rb
View File

@@ -9,6 +9,7 @@ module Domain::E621::PostsHelper
%w[*.bsky.app bsky.app] => "bsky.png",
%w[*.itaku.ee itaku.ee] => "itaku.png",
%w[*.deviantart.com deviantart.com *.wixmp.com] => "deviantart.png",
%w[*.twitter.com twitter.com *.x.com x.com] => "x-twitter.png",
}
domain_patterns.each do |patterns, icon|

15
app/helpers/indexable_posts_helper.rb
View File

@@ -1,2 +1,17 @@
module IndexablePostsHelper
def show_path(indexed_post)
case indexed_post.postable_type
when "Domain::Fa::Post"
# need to use the helper here because the postable is not loaded
Rails.application.routes.url_helpers.domain_fa_post_path(
indexed_post.postable,
)
when "Domain::E621::Post"
Rails.application.routes.url_helpers.domain_e621_post_path(
indexed_post.postable,
)
else
raise("Unsupported postable type: #{indexed_post.postable_type}")
end
end
end

4
app/helpers/log_entries_helper.rb
View File

@@ -25,6 +25,8 @@ module LogEntriesHelper
"png"
when "image/gif"
"gif"
when "video/webm"
"webm"
else
nil
end
@@ -42,7 +44,7 @@ module LogEntriesHelper
end
def is_renderable_video_type?(content_type)
["video/mp4"].any? { |ct| content_type.starts_with?(ct) }
%w[video/mp4 video/webm].any? { |ct| content_type.starts_with?(ct) }
end
def is_flash_content_type?(content_type)

4
app/jobs/domain/e621/job/static_file_job.rb
View File

@@ -1,6 +1,6 @@
module Domain::E621::Job
class StaticFileJob < Base
queue_as :e621
queue_as :static_file
ignore_signature_args :caused_by_entry
def perform(args)
@@ -14,7 +14,7 @@ module Domain::E621::Job
logger.warn("post has no file_url_str, enqueueing for scan")
defer_job(
Domain::E621::Job::ScanPostJob,
{ post: post, caused_by_entry: caused_by_entry }
{ post: post, caused_by_entry: caused_by_entry },
)
return
end

2
app/lib/domain/e621/tag_util.rb
View File

@@ -57,7 +57,7 @@ module Domain::E621::TagUtil
post_json["flags"].to_a.select(&:second).map(&:first)
e621_post.pools_array = post_json["pools"]
e621_post.sources_array = post_json["sources"]
e621_post.tags_array = e621_tag_names
e621_post.tags_array = post_json["tags"]
e621_post
end

11
app/models/domain/e621/post.rb
View File

@@ -19,7 +19,7 @@ class Domain::E621::Post < ReduxApplicationRecord
self.pools_array ||= []
self.sources_array ||= []
self.artists_array ||= []
self.tags_array ||= []
self.tags_array ||= {}
end
has_many :taggings, class_name: "Domain::E621::Tagging", inverse_of: :post
@@ -36,6 +36,15 @@ class Domain::E621::Post < ReduxApplicationRecord
foreign_key: :e621_id,
optional: true
def to_param
self.e621_id.to_s
end
def tags_array
ta = super
ta.is_a?(Hash) ? ta : { "general" => ta }
end
def index_page_http_log_entry
if state_detail["last_index_page_id"].present?
HttpLogEntry.find_by(id: state_detail["last_index_page_id"])

63
app/models/indexed_post.rb
View File

@@ -4,13 +4,29 @@ class IndexedPost < ReduxApplicationRecord
belongs_to :postable, polymorphic: true, inverse_of: :indexed_post
has_one :file, through: :postable
def show_path
def artist_name
case postable_type
when "Domain::Fa::Post"
# need to use the helper here because the postable is not loaded
Rails.application.routes.url_helpers.domain_fa_post_path(postable)
postable&.creator&.name
when "Domain::E621::Post"
Rails.application.routes.url_helpers.domain_e621_post_path(postable)
array = postable&.tags_array
return unless array
array.is_a?(Hash) ? array["artist"].first : nil
else
raise("Unsupported postable type: #{postable_type}")
end
end
def artist_path
case postable_type
when "Domain::Fa::Post"
if postable&.creator
Rails.application.routes.url_helpers.domain_fa_user_path(
postable&.creator,
)
end
when "Domain::E621::Post"
return nil
else
raise("Unsupported postable type: #{postable_type}")
end
@@ -19,7 +35,7 @@ class IndexedPost < ReduxApplicationRecord
def title
case postable_type
when "Domain::Fa::Post"
postable&.title || "FA Post #{postable&.id}"
postable&.title || "FA Post #{postable&.fa_id}"
when "Domain::E621::Post"
"E621 Post #{postable&.e621_id}"
else
@@ -27,6 +43,17 @@ class IndexedPost < ReduxApplicationRecord
end
end
def posted_at
case postable_type
when "Domain::Fa::Post"
postable&.posted_at
when "Domain::E621::Post"
postable&.posted_at
else
raise("Unsupported postable type: #{postable_type}")
end
end
def file_sha256
case postable_type
when "Domain::Fa::Post", "Domain::E621::Post"
@@ -48,4 +75,30 @@ class IndexedPost < ReduxApplicationRecord
raise("Unsupported postable type: #{postable_type}")
end
end
def external_link_title
case postable_type
when "Domain::Fa::Post"
fa_id = postable&.fa_id
"FA #{fa_id}" if fa_id.present?
when "Domain::E621::Post"
e621_id = postable&.e621_id
"E621 #{e621_id}" if e621_id.present?
else
raise("Unsupported postable type: #{postable_type}")
end
end
def external_link_url
case postable_type
when "Domain::Fa::Post"
fa_id = postable&.fa_id
"https://www.furaffinity.net/view/#{fa_id}" if fa_id.present?
when "Domain::E621::Post"
e621_id = postable&.e621_id
"https://e621.net/posts/#{e621_id}" if e621_id.present?
else
raise("Unsupported postable type: #{postable_type}")
end
end
end

7
app/models/user.rb
View File

@@ -7,10 +7,5 @@ class User < ReduxApplicationRecord
:rememberable,
:validatable
enum role: {
user: "user",
admin: "admin",
moderator: "moderator",
},
_default: "user"
enum :role, %i[user admin moderator].index_with(&:to_s), default: :user
end

10
app/views/domain/e621/posts/_post.html.erb
View File

@@ -63,9 +63,17 @@
<div class="section-header">Tags</div>
<div class="bg-slate-100 p-4">
<% if @post.tags_array.any? %>
<% tags_array =
(
if @post.tags_array.is_a?(Hash)
@post.tags_array
else
{ "general" => @post.tags_array }
end
) %>
<div class="flex flex-wrap gap-2">
<% tag_category_order.each do |category| %>
<% (@post.tags_array[category.to_s] || []).each do |tag| %>
<% (tags_array[category.to_s] || []).each do |tag| %>
<span
class="<%= tag_category_tw_class(category) %> rounded px-2 py-1 text-sm text-slate-600"
>

21
app/views/indexed_posts/_indexed_post.html.erb → app/views/indexed_posts/_as_gallery_item.html.erb
View File

@@ -7,16 +7,15 @@
<div class="flex items-center justify-center p-4">
<% if post.file_sha256.present? %>
<%= link_to post.show_path do %>
<img
class="max-h-[300px] max-w-[300px] rounded-md border border-slate-300 object-contain shadow-md"
alt="<%= post.title %>"
src="<%= contents_blob_path(
HexUtil.bin2hex(post.file_sha256),
format: "jpg",
thumb: "small",
) %>"
/>
<%= link_to show_path(post) do %>
<%= image_tag contents_blob_path(
HexUtil.bin2hex(post.file_sha256),
format: "jpg",
thumb: "small",
),
class:
"max-h-[300px] max-w-[300px] rounded-md border border-slate-300 object-contain shadow-md",
alt: post.title %>
<% end %>
<% else %>
<span>No file available</span>
@@ -25,7 +24,7 @@
<div class="border-t border-slate-300">
<h2 class="p-4 text-center text-lg">
<%= link_to post.title, post.show_path, class: "sky-link" %>
<%= link_to post.title, show_path(post), class: "sky-link" %>
</h2>
<div class="px-4 pb-4 text-sm text-slate-600">
<div class="flex items-start justify-between">

40
app/views/indexed_posts/_as_table_row_item.html.erb Normal file
View File

@@ -0,0 +1,40 @@
<div class="grid-row contents">
<div class="grid-cell">
<% if post.file_sha256.present? %>
<%= image_tag contents_blob_path(
HexUtil.bin2hex(post.file_sha256),
format: "jpg",
thumb: "tiny",
),
class: "h-16 w-16 object-cover rounded",
alt: post.title %>
<% else %>
indexable post id: <%= post.id %>
<% end %>
</div>
<div class="grid-cell min-w-0">
<%= link_to post.title, show_path(post), class: "text-blue-600 hover:text-blue-800" %>
</div>
<div class="grid-cell text-center">
<% if post.artist_path.present? %>
<%= link_to post.artist_name,
post.artist_path,
class: "text-blue-600 hover:text-blue-800" %>
<% else %>
<%= post.artist_name %>
<% end %>
</div>
<div class="grid-cell text-center">
<%= link_to post.external_link_url,
class: "text-blue-600 hover:text-blue-800",
target: "_blank",
rel: "noreferrer" do %>
<%= post.external_link_title %>
<i class="fas fa-external-link-alt ml-1"></i>
<% end %>
</div>
<div class="grid-cell text-right">
<%= post.posted_at ? time_ago_in_words(post.posted_at) : "Unknown" %>
</div>
</div>
<div class="col-span-full border-b border-slate-300"></div>

126
app/views/indexed_posts/index.html.erb
View File

@@ -2,37 +2,103 @@
<h1 class="text-2xl">All Posts <%= page_str(params) %></h1>
</div>
<div class="mx-auto mb-4 mt-4 flex justify-center gap-4">
<% active_sources = (params[:sources] || SourceHelper.all_source_names).uniq %>
<% SourceHelper.all_source_names.each do |source| %>
<% is_active = active_sources.include?(source) %>
<% link_sources =
(
if is_active
active_sources - [source]
else
active_sources + [source]
end
) %>
<% posts_path_url =
if SourceHelper.has_all_sources?(link_sources)
indexed_posts_path
else
indexed_posts_path(sources: link_sources)
end %>
<%= link_to(
"#{source.titleize} #{is_active ? "(On)" : "(Off)"}",
posts_path_url,
class:
"px-4 py-2 rounded #{is_active ? "bg-blue-500 text-white" : "bg-gray-300"}",
) %>
<% end %>
<div class="mb-6 bg-white shadow">
<div class="mx-auto px-4 sm:px-6 lg:px-8">
<div
class="flex flex-col items-center justify-between gap-4 py-4 sm:flex-row"
>
<!-- Domain Filters -->
<div class="flex flex-wrap gap-2">
<span class="my-auto font-medium text-gray-700">Sources:</span>
<% active_sources = (params[:sources] || SourceHelper.all_source_names).uniq %>
<% SourceHelper.all_source_names.each do |source| %>
<% is_active = active_sources.include?(source) %>
<% link_sources =
(
if is_active
active_sources - [source]
else
active_sources + [source]
end
) %>
<% posts_path_url =
if SourceHelper.has_all_sources?(link_sources)
indexed_posts_path(view: params[:view])
else
indexed_posts_path(sources: link_sources, view: params[:view])
end %>
<%= link_to(
source.titleize,
posts_path_url,
class:
"px-3 py-1 rounded-full text-sm #{is_active ? "bg-blue-500 text-white" : "bg-gray-100 text-gray-700 hover:bg-gray-200"}",
) %>
<% end %>
</div>
<!-- View Type Selector -->
<div class="flex items-center gap-2">
<span class="font-medium text-gray-700">View:</span>
<%= link_to(
indexed_posts_path(view: "gallery", sources: params[:sources]),
class:
"px-3 py-1 rounded-full text-sm #{params[:view] != "table" ? "bg-blue-500 text-white" : "bg-gray-100 text-gray-700 hover:bg-gray-200"}",
) do %>
<i class="fas fa-th-large mr-1"></i> Gallery
<% end %>
<%= link_to(
indexed_posts_path(view: "table", sources: params[:sources]),
class:
"px-3 py-1 rounded-full text-sm #{params[:view] == "table" ? "bg-blue-500 text-white" : "bg-gray-100 text-gray-700 hover:bg-gray-200"}",
) do %>
<i class="fas fa-list mr-1"></i> Table
<% end %>
</div>
</div>
</div>
</div>
<%= render partial: "shared/pagination_controls", locals: { collection: @posts } %>
<% content_for :head do %>
<style>
.grid-cell {
padding: 0.25rem;
border-right: 1px solid #e2e8f0;
}
.grid-cell:last-child {
padding-left: 0;
padding-right: 1rem;
border-right: none;
}
.grid-cell:first-child {
padding-left: 1rem;
}
.grid-row:hover .grid-cell {
background-color: #f1f5f9;
}
</style>
<% end %>
<% if params[:view] == "table" %>
<div
class="mx-auto grid grid-cols-[auto_1fr_auto_auto_auto] border-b border-slate-300 text-sm"
>
<div class="grid-row contents">
<div class="grid-cell text-center font-semibold">Thumbnail</div>
<div class="grid-cell text-left font-semibold">Title</div>
<div class="grid-cell text-center font-semibold">Artist</div>
<div class="grid-cell text-center font-semibold">Source</div>
<div class="grid-cell text-right font-semibold">Posted</div>
</div>
<div class="col-span-full border-b border-slate-300"></div>
<div class="mx-auto flex flex-wrap justify-center">
<% @posts.each do |post| %>
<%= render partial: "indexed_post", locals: { post: post } %>
<% end %>
</div>
<% @posts.each do |post| %>
<%= render partial: "as_table_row_item", locals: { post: post } %>
<% end %>
</div>
<% else %>
<div class="mx-auto flex flex-wrap justify-center">
<% @posts.each do |post| %>
<%= render partial: "as_gallery_item", locals: { post: post } %>
<% end %>
</div>
<% end %>

1
app/views/log_entries/_content_container.html.erb
View File

@@ -4,6 +4,7 @@
<img alt="image" src="<%= contents_path %>" class="rounded-md" />
<% elsif is_renderable_video_type?(log_entry.content_type) %>
<video
class="rounded-md"
alt="video"
controls="controls"
loop="loop"

40
spec/controllers/log_entries_controller_spec.rb
View File

@@ -1,18 +1,40 @@
require "rails_helper"
RSpec.describe LogEntriesController, type: :controller do
describe "GET #index" do
it "returns filtered log entries" do
get :index, params: { filter: "example.com/test" }
expect(response).to be_successful
let(:user) { create(:user, :admin) }
context "when user is not signed in" do
describe "GET #index" do
it "redirects to sign in" do
get :index, params: { filter: "example.com/test" }
expect(response).to redirect_to(new_user_session_path)
end
end
describe "GET #stats" do
it "redirects to sign in" do
get :stats, params: { seconds: 3600 }
expect(response).to redirect_to(new_user_session_path)
end
end
end
describe "GET #stats" do
it "returns statistics in the specified time window" do
get :stats, params: { seconds: 3600 }
expect(response).to be_successful
expect(assigns(:time_window)).to eq 3600.seconds
context "when user is signed in" do
before { sign_in user }
describe "GET #index" do
it "returns filtered log entries" do
get :index, params: { filter: "example.com/test" }
expect(response).to be_successful
end
end
describe "GET #stats" do
it "returns statistics in the specified time window" do
get :stats, params: { seconds: 3600 }
expect(response).to be_successful
expect(assigns(:time_window)).to eq 3600.seconds
end
end
end
end

16
spec/factories/users.rb Normal file
View File

@@ -0,0 +1,16 @@
FactoryBot.define do
factory :user do
sequence(:email) { |n| "user#{n}@example.com" }
password { "password123" }
password_confirmation { "password123" }
role { :user }
trait :admin do
role { :admin }
end
trait :moderator do
role { :moderator }
end
end
end

15
spec/jobs/domain/e621/job/posts_index_job_spec.rb
View File

@@ -21,7 +21,7 @@ describe Domain::E621::Job::PostsIndexJob do
describe "#perform" do
it "creates new posts" do
expect do perform_now({}) end.to change(Domain::E621::Post, :count).by(5)
expect { perform_now({}) }.to change(Domain::E621::Post, :count).by(5)
end
it "updates existing posts" do
@@ -33,14 +33,19 @@ describe Domain::E621::Job::PostsIndexJob do
tags_array: ["some_tag"],
},
)
expect(post.tags_array).to eq(["some_tag"])
expect do perform_now({}) end.to change(Domain::E621::Post, :count).by(4)
expect(post.tags_array).to eq({ "general" => ["some_tag"] })
expect { perform_now({}) }.to change(Domain::E621::Post, :count).by(4)
post.reload
expect(post.file_url_str).to eq(
"https://static1.e621.net/data/1c/61/1c6169aa51668681e9697a48144d7c78.jpg",
)
expect(post.tags_array).to include("alcohol", "beach_ball", "wide_hips")
expect(post.tags_array).not_to include("some_tag")
expect(post.tags_array).to match(
hash_including(
"general" => array_including("alcohol", "beach_ball", "wide_hips"),
"species" => array_including("mammal", "procyonid", "raccoon"),
),
)
expect(post.tags_array.values.flatten).not_to include("some_tag")
end
# it "fixes tags to reflect reality" do

4
spec/jobs/domain/e621/job/scan_post_job_spec.rb
View File

@@ -34,7 +34,9 @@ describe Domain::E621::Job::ScanPostJob do
"https://static1.e621.net/data/c0/fa/c0fa5293f1d1440c2d3f2c3e027d3c36.jpg",
)
expect(post.md5).to eq("c0fa5293f1d1440c2d3f2c3e027d3c36")
expect(post.tags_array).to include("black_nose", "facial_tuft")
expect(post.tags_array).to match(
hash_including("general" => array_including("black_nose", "facial_tuft")),
)
expect(
SpecUtil.enqueued_jobs(Domain::E621::Job::StaticFileJob).length,

11
spec/policies/domain/fa/user_policy_spec.rb
View File

@@ -10,15 +10,20 @@ RSpec.describe Domain::Fa::UserPolicy, type: :policy do
let(:policy) { described_class.new(user, fa_user) }
it { expect(policy).to permit_action(:show) }
it { expect(policy).to forbid_action(:view_scraper_metadata) }
it { expect(policy).to forbid_action(:view_scraped_at_timestamps) }
end
context "for an admin" do
let(:user) { build(:user, :admin) }
let(:policy) { described_class.new(user, fa_user) }
before do
puts "Debug: User role is #{user.role.inspect}"
puts "Debug: User admin? #{user.admin?.inspect}"
end
it { expect(policy).to permit_action(:show) }
it { expect(policy).to permit_action(:view_scraper_metadata) }
it { expect(policy).to permit_action(:view_scraped_at_timestamps) }
end
context "for a regular user" do
@@ -26,6 +31,6 @@ RSpec.describe Domain::Fa::UserPolicy, type: :policy do
let(:policy) { described_class.new(user, fa_user) }
it { expect(policy).to permit_action(:show) }
it { expect(policy).to forbid_action(:view_scraper_metadata) }
it { expect(policy).to forbid_action(:view_scraped_at_timestamps) }
end
end

13
spec/rails_helper.rb
View File

@@ -1,12 +1,12 @@
# This file is copied to spec/ when you run 'rails generate rspec:install'
require "spec_helper"
ENV["RAILS_ENV"] ||= "test"
# ENV["RAILS_ENV"] ||= "test"
require_relative "../config/environment"
require "spec_helper"
require "rspec/rails"
# Prevent database truncation if the environment is production
if Rails.env.production?
abort("The Rails environment is running in production mode!")
end
require "rspec/rails"
# Add additional requires below this line. Rails is not loaded until this point!
# Requires supporting ruby files with custom matchers and macros, etc, in
@@ -36,6 +36,10 @@ RSpec.configure do |config|
# This will use the defaults of :js and :server_rendering meta tags
ReactOnRails::TestHelper.configure_rspec_to_compile_assets(config)
# Add Devise test helpers
config.include Devise::Test::ControllerHelpers, type: :controller
config.include Devise::Test::IntegrationHelpers, type: :request
# Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
# config.fixture_path = "#{::Rails.root}/spec/fixtures"
@@ -73,6 +77,9 @@ end
require "spec_util"
# Add Pundit matchers configuration
require "pundit/matchers"
Shoulda::Matchers.configure do |config|
config.integrate do |with|
with.test_framework :rspec

11
test/fixtures/users.yml vendored
View File

@@ -1,11 +0,0 @@
# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
# This model initially had no columns defined. If you add columns to the
# model remove the "{}" from the fixture names and add the columns immediately
# below each fixture, per the syntax in the comments below
#
one: {}
# column: value
#
two: {}
# column: value