dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance Gemfile, update styles, and improve log entry handling

Browse Source 
- Added `db-query-matchers` gem for improved query testing capabilities.
- Updated `sanitize` gem version to `~> 6.1` for better security and features.
- Refactored styles in `application.tailwind.css` for better responsiveness.
- Improved `LogEntriesController` to utilize `response_size` for more accurate data handling.
- Added a new `favorites` action in `Domain::Fa::PostsController` for better user experience.
- Enhanced `fa_post_description_sanitized` method in `Domain::Fa::PostsHelper` for improved HTML sanitization.
- Updated views for `Domain::Fa::Posts` to streamline layout and improve user interaction.
- Improved pagination controls for better navigation across post listings.
This commit is contained in:
Dylan Knutson
2024-12-29 20:30:10 +00:00
parent ca914dbe25
commit 3cfa166b4a
29 changed files with 962 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Gemfile
View File

@@ -91,6 +91,7 @@ group :test do
gem "factory_bot_rails"
gem "parallel_tests"
gem "pundit-matchers", "~> 4.0"
gem "db-query-matchers", "~> 0.14"
end
gem "xdiff", path: "/gems/xdiff-rb"
@@ -136,7 +137,7 @@ end
gem "rack-cors"
gem "react_on_rails"
gem "sanitize"
gem "sanitize", "~> 6.1"
gem "shakapacker"
group :development do

12
Gemfile.lock
View File

@@ -136,6 +136,9 @@ GEM
curb (1.0.5)
daemons (1.4.1)
date (3.4.1)
db-query-matchers (0.14.0)
activesupport (>= 4.0, < 8.1)
rspec (>= 3.0)
debug (1.10.0)
irb (~> 1.10)
reline (>= 0.3.8)
@@ -350,6 +353,10 @@ GEM
rexml (3.2.5)
rice (4.0.4)
ripcord (2.0.0)
rspec (3.13.0)
rspec-core (~> 3.13.0)
rspec-expectations (~> 3.13.0)
rspec-mocks (~> 3.13.0)
rspec-core (3.13.2)
rspec-support (~> 3.13.0)
rspec-expectations (3.13.3)
@@ -374,7 +381,7 @@ GEM
ffi (~> 1.12)
rubyzip (2.3.2)
rufo (0.15.1)
sanitize (6.0.2)
sanitize (6.1.3)
crass (~> 1.0.2)
nokogiri (>= 1.12.0)
sd_notify (0.1.1)
@@ -459,6 +466,7 @@ DEPENDENCIES
cssbundling-rails (~> 1.4)
curb
daemons
db-query-matchers (~> 0.14)
debug (~> 1.10)
devise (~> 4.9)
diffy
@@ -500,7 +508,7 @@ DEPENDENCIES
ruby-prof-speedscope
ruby-vips
rufo
sanitize
sanitize (~> 6.1)
sd_notify
selenium-webdriver
shakapacker

5
app/assets/stylesheets/application.tailwind.css
View File

@@ -7,7 +7,7 @@
}
.sky-section {
@apply overflow-hidden rounded-lg border border-slate-300 bg-slate-100 divide-y divide-slate-300;
@apply divide-y divide-slate-300 overflow-hidden md:rounded-lg border border-slate-300 bg-slate-100;
}
.section-header {
@@ -15,5 +15,6 @@
}
.sky-link {
@apply text-sky-600 underline decoration-dotted hover:text-sky-800 transition-colors;
@apply text-sky-600 underline decoration-dotted transition-colors hover:text-sky-800;
}

5
app/controllers/domain/fa/posts_controller.rb
View File

@@ -25,6 +25,11 @@ class Domain::Fa::PostsController < ApplicationController
def show
end
# GET /domain/fa/posts/1/favorites
def favorites
@post = Domain::Fa::Post.find_by_fa_id!(params[:fa_id])
end
def scan_post
if try_enqueue_post_scan(@post, @post.fa_id)
redirect_to domain_fa_post_path(@post.fa_id), notice: "Enqueued for scan"

19
app/controllers/log_entries_controller.rb
View File

@@ -13,7 +13,7 @@ class LogEntriesController < ApplicationController
query =
query.where("uri_path = ?", @uri_filter.path).where(
"uri_query like ?",
@uri_filter.query + "%"
@uri_filter.query + "%",
)
else
query = query.where("uri_path like ?", @uri_filter.path + "%")
@@ -53,25 +53,20 @@ class LogEntriesController < ApplicationController
end
HttpLogEntry
.joins(:response)
.includes(:response)
.select("http_log_entries.*, blob_entries_p.size")
.find_each(batch_size: 100, order: :desc) do |log_entry|
break if log_entry.created_at < @time_window.ago
@last_window_count += 1
@last_window_bytes += log_entry.response.size
@last_window_bytes_stored += log_entry.response.bytes_stored
@last_window_bytes += log_entry.response_size
content_type = log_entry.content_type.split(";").first
@content_type_counts[content_type][:count] += 1
@content_type_counts[content_type][:bytes] += log_entry.response.size
@content_type_counts[content_type][
:bytes_stored
] += log_entry.response.bytes_stored
@content_type_counts[content_type][:bytes] += log_entry.response_size
@by_domain_counts[log_entry.uri_host][:count] += 1
@by_domain_counts[log_entry.uri_host][:bytes] += log_entry.response.size
@by_domain_counts[log_entry.uri_host][
:bytes_stored
] += log_entry.response.bytes_stored
@by_domain_counts[log_entry.uri_host][:bytes] += log_entry.response_size
end
end
@@ -80,7 +75,7 @@ class LogEntriesController < ApplicationController
HttpLogEntry.includes(
:caused_by_entry,
:triggered_entries,
response: :base
response: :base,
).find(params[:id])
end
end

10
app/helpers/domain/e621/posts_helper.rb
View File

@@ -74,6 +74,16 @@ module Domain::E621::PostsHelper
end
end
def fa_post_for_source(source)
uri = URI.parse(source)
return unless %w[www.furaffinity.net furaffinity.net].include?(uri.host)
fa_id = uri.path.match(%r{/view/(\d+)})[1]
return unless fa_id
Domain::Fa::Post.find_by(fa_id: fa_id)
rescue StandardError
nil
end
private
def extract_domain(url)

118
app/helpers/domain/fa/posts_helper.rb
View File

@@ -1,8 +1,4 @@
module Domain::Fa::PostsHelper
def can_see_hosted_post?(post)
VpnOnlyRouteConstraint.new.matches?(request)
end
def post_state_string(post)
if post.have_file?
"file"
@@ -39,16 +35,108 @@ module Domain::Fa::PostsHelper
end
def fa_post_description_sanitized(html)
raw Sanitize.fragment(
html,
elements: %w[br img b i span strong],
attributes: {
"span" => %w[style],
"a" => []
},
css: {
properties: %w[font-size color]
}
)
fa_post_id_to_node = {}
fa_user_url_name_to_node = {}
sanitizer =
Sanitize.new(
elements: %w[br img b i span strong],
attributes: {
"span" => %w[style],
},
css: {
properties: %w[font-size color],
},
transformers: [
lambda do |env|
# Only allow and transform FA links
if env[:node_name] == "a"
node = env[:node]
# by default, assume the host is www.furaffinity.net
href = node["href"]&.downcase || ""
href = "//" + href if href.match?(/^(www\.)?furaffinity\.net/)
uri = URI.parse(href)
uri.host ||= "www.furaffinity.net"
path = uri.path
fa_host_matcher = /^(www\.)?furaffinity\.net$/
fa_post_matcher = %r{^/view/(\d+)/?$}
fa_user_matcher = %r{^/user/(\w+)/?$}
if fa_host_matcher.match?(uri.host) && path
if path.match?(fa_post_matcher)
fa_id = path.match(fa_post_matcher)[1].to_i
fa_post_id_to_node[fa_id] = node
next { node_whitelist: [node] }
elsif path.match?(fa_user_matcher)
fa_url_name = path.match(fa_user_matcher)[1]
fa_user_url_name_to_node[fa_url_name] = node
next { node_whitelist: [node] }
end
end
# Don't allow any other links
node.replace(node.children)
end
end,
],
)
fragment = Nokogiri::HTML5.fragment(sanitizer.send(:preprocess, html))
sanitizer.node!(fragment)
if fa_post_id_to_node.any?
# Batch load posts and their titles, ensuring fa_post_ids are strings
posts_by_id =
Domain::Fa::Post.where(fa_id: fa_post_id_to_node.keys).index_by(&:fa_id)
# Replace the link text with post titles if available
fa_post_id_to_node.each do |fa_id, node|
if (post = posts_by_id[fa_id])
node.replace(
Nokogiri::HTML5.fragment(
render(
partial: "domain/fa/posts/description_inline_link_fa_post",
locals: {
post: post,
},
),
),
)
else
node.replace(node.children)
end
end
end
if fa_user_url_name_to_node.any?
# Batch load users and their names, ensuring fa_user_url_names are strings
users_by_url_name =
Domain::Fa::User
.where(url_name: fa_user_url_name_to_node.keys)
.includes(:avatar)
.index_by(&:url_name)
# Replace the link text with user names if available
fa_user_url_name_to_node.each do |fa_url_name, node|
if (user = users_by_url_name[fa_url_name])
node.replace(
Nokogiri::HTML5.fragment(
render(
partial: "domain/fa/posts/description_inline_link_fa_user",
locals: {
user: user,
},
),
),
)
else
node.replace(node.children)
end
end
end
raw fragment.to_html(preserve_newline: true)
end
end

16
app/models/domain/fa/post.rb
View File

@@ -148,6 +148,22 @@ class Domain::Fa::Post < ReduxApplicationRecord
end
end
def description
content = super
return nil if content.nil? || content.blank?
# this is a hack to remove the first two lines of the description, which are
# always empty and a <br><br>
lines = content.lines.map(&:strip).map(&:chomp)
if lines.length > 3
if lines[0] == "" && lines[1].start_with?("<a href=") &&
lines[2] == "<br><br>"
return lines[3..].join("\n")
end
end
content
end
def have_file?
self.file_id.present?
end

16
app/views/domain/e621/posts/_post.html.erb
View File

@@ -1,8 +1,8 @@
<div
id="<%= dom_id post %>"
class="mx-auto mt-4 flex w-full max-w-2xl flex-col gap-4 pb-4"
class="mx-auto mt-4 flex w-full flex-col gap-4 pb-4 md:max-w-2xl"
>
<section class="rounded-md border border-slate-300 bg-slate-50 p-4">
<section class="border border-slate-300 bg-slate-50 p-4 md:rounded-md">
<div class="flex items-center justify-between">
<div>
<span class="text-md italic">
@@ -98,17 +98,7 @@
<div class="bg-slate-100 p-4">
<div class="divide-y divide-slate-200">
<% @post.sources_array.each do |source| %>
<div class="flex items-start gap-2 py-1 first:pt-0 last:pb-0">
<% if icon_path = icon_asset_for_url(source) %>
<%= image_tag icon_path, class: "h-4 w-4 mt-1 flex-shrink-0" %>
<% else %>
<i class="fa-solid fa-link mt-1 h-4 w-4 flex-shrink-0"></i>
<% end %>
<%= link_to source,
source,
class: "text-blue-600 hover:underline truncate",
target: "_blank" %>
</div>
<%= render partial: "domain/e621/posts/source_link", locals: { source: source } %>
<% end %>
</div>
</div>

19
app/views/domain/e621/posts/_source_link.html.erb Normal file
View File

@@ -0,0 +1,19 @@
<div class="flex items-start gap-2 py-1 first:pt-0 last:pb-0">
<% if icon_path = icon_asset_for_url(source) %>
<%= image_tag icon_path, class: "h-4 w-4 mt-1 flex-shrink-0" %>
<% else %>
<i class="fa-solid fa-link mt-1 h-4 w-4 flex-shrink-0"></i>
<% end %>
<%= link_to source,
source,
class: "text-blue-600 hover:underline truncate",
target: "_blank" %>
<% fa_post = fa_post_for_source(source) %>
<% if fa_post %>
<%= link_to domain_fa_post_path(fa_post.fa_id),
class:
"float-right inline-flex items-center gap-2 rounded bg-blue-600 px-3 py-1 text-sm text-white hover:bg-blue-700" do %>
<span class="truncate"><%= fa_post.title %></span>
<% end %>
<% end %>
</div>

6
app/views/domain/fa/posts/_description_inline_link_fa_post.html.erb Normal file
View File

@@ -0,0 +1,6 @@
<%= link_to domain_fa_post_path(post),
class:
"text-sky-200 transition-all hover:text-sky-800 inline-flex items-center hover:bg-gray-100 rounded-md gap-1 px-1" do %>
<i class="fa-regular fa-image h-4 w-4 flex-shrink-0"></i>
<span><%= post.title %></span>
<% end %>

10
app/views/domain/fa/posts/_description_inline_link_fa_user.html.erb Normal file
View File

@@ -0,0 +1,10 @@
<%= link_to domain_fa_user_path(user),
class:
"text-sky-200 transition-all hover:text-sky-800 inline-flex items-center hover:bg-gray-100 rounded-md gap-1 px-1 align-bottom" do %>
<img
src="<%= fa_user_avatar_path(user, thumb: "32-avatar") %>"
class="inline-block h-4 w-4 flex-shrink-0 rounded-sm object-cover"
alt="<%= user.name %>'s avatar"
/>
<span><%= user.name %></span>
<% end %>

85
app/views/domain/fa/posts/_post.html.erb
View File

@@ -3,19 +3,24 @@
class="mx-auto mt-4 flex w-full max-w-2xl flex-col gap-4 pb-4"
>
<section class="rounded-md border border-slate-300 bg-slate-50 p-4">
<div class="flex items-center justify-between">
<div>
<span class="text-md italic">
<%= link_to @post.title,
"https://www.furaffinity.net/view/#{@post.fa_id}",
class: "text-blue-600 hover:underline",
target: "_blank" %>
</span>
<span class="ml-2 italic">
by <%= render "domain/fa/users/inline_link", user: @post.creator %>
</span>
<div class="flex items-center justify-between gap-4">
<div class="flex min-w-0 items-center gap-4">
<div class="flex min-w-0 items-center gap-2">
<span class="truncate text-lg font-medium">
<%= link_to @post.title,
"https://www.furaffinity.net/view/#{@post.fa_id}",
class: "text-blue-600 hover:underline",
target: "_blank" %>
</span>
<i class="fa-solid fa-arrow-up-right-from-square text-slate-400"></i>
</div>
<div class="flex items-center gap-2 whitespace-nowrap text-slate-600">
by
<%= render "domain/fa/users/inline_link",
user: @post.creator,
with_post_count: false %>
</div>
</div>
<i class="fa-solid fa-arrow-up-right-from-square text-slate-400"></i>
</div>
<div class="mt-2 flex flex-wrap gap-x-4 text-sm text-slate-600">
<span>
@@ -34,6 +39,11 @@
<span>
<i class="fa-solid fa-heart mr-1"></i>
Favorites: <%= @post.num_favorites %>
<% if policy(@post).view_scraper_metadata? %>
(<%= link_to pluralize(@post.faved_by.count, "fav"),
favorites_domain_fa_post_path(@post),
class: "text-blue-600 hover:underline" %>)
<% end %>
</span>
</div>
<% if policy(@post).view_scraper_metadata? %>
@@ -77,54 +87,7 @@
</section>
<% end %>
<section class="sky-section">
<% if (post_description_html = @post.description) %>
<div class="section-header">Post Description</div>
<div class="bg-slate-800 p-4 text-slate-200">
<%= fa_post_description_sanitized(post_description_html) %>
</div>
<% else %>
<div>(No post description)</div>
<% end %>
</section>
<section class="sky-section">
<div class="section-header">Similar Posts</div>
<div class="bg-slate-100">
<% cache(@post.disco, expires_in: 12.hours) do %>
<% similar =
@post
.disco
&.nearest_neighbors(:for_favorite, distance: "cosine")
&.limit(10)
&.includes(:post) %>
<% if similar %>
<% similar.each do |factor| %>
<% post = factor.post %>
<div
class="flex flex-row items-center justify-between border-b px-2 py-1 last:border-b-0"
>
<div class="flex flex-row items-center">
<span class="text-md italic"
><%= link_to post.title, domain_fa_post_path(post.fa_id), class: "underline" %></span
>
<span class="ml-2 italic"
>by
<%= render "domain/fa/users/inline_link", user: post.creator %></span
>
</div>
<span class="ml-2 text-sm text-slate-500">
(distance:
<%= number_with_precision(factor.neighbor_distance, precision: 5) %>)
</span>
</div>
<% end %>
<% else %>
<div class="p-4 text-center italic text-slate-400">
No similar posts
</div>
<% end %>
<% end %>
</div>
</section>
<%= render "section_description", { post: @post } %>
<%= render "section_similar_posts", { post: @post } %>
</div>

10
app/views/domain/fa/posts/_section_description.html.erb Normal file
View File

@@ -0,0 +1,10 @@
<section class="sky-section">
<% if (post_description_html = @post.description) %>
<div class="section-header">Post Description</div>
<div class="bg-slate-800 p-4 text-slate-200">
<%= fa_post_description_sanitized(post_description_html) %>
</div>
<% else %>
<div>(No post description)</div>
<% end %>
</section>

42
app/views/domain/fa/posts/_section_similar_posts.html.erb Normal file
View File

@@ -0,0 +1,42 @@
<section class="sky-section">
<div class="section-header">Similar Posts</div>
<div
class="grid grid-cols-[1fr_auto_auto] items-center divide-y divide-slate-300 bg-slate-100"
>
<% cache(post.disco, expires_in: 12.hours) do %>
<% similar =
post
.disco
&.nearest_neighbors(:for_favorite, distance: "cosine")
&.limit(5)
&.includes(:post) %>
<% if similar %>
<% similar
.map(&:post)
.each do |post| %>
<% creator = post.creator %>
<div class="col-span-3 grid grid-cols-subgrid">
<span class="text-md truncate px-4 py-2">
<%= link_to post.title, domain_fa_post_path(post.fa_id), class: "underline italic" %>
</span>
<a href="<%= domain_fa_user_path(creator) %>" class="contents">
<div class="px-2 py-2">
<img
src="<%= fa_user_avatar_path(creator, thumb: "64-avatar") %>"
class="h-8 w-8 flex-shrink-0 rounded-md"
/>
</div>
<span class="sky-link truncate px-4 py-2">
<%= creator.url_name %>
</span>
</a>
</div>
<% end %>
<% else %>
<div class="col-span-3 p-4 text-center italic text-slate-400">
No similar posts
</div>
<% end %>
<% end %>
</div>
</section>

51
app/views/domain/fa/posts/favorites.html.erb Normal file
View File

@@ -0,0 +1,51 @@
<div class="mx-auto mt-4 flex w-full max-w-2xl flex-col gap-4 pb-4">
<section class="rounded-md border border-slate-300 bg-slate-50 p-4">
<div class="flex items-center justify-between gap-4">
<div class="flex min-w-0 items-center gap-2">
<h1 class="text-lg font-medium">
Users who favorited
<%= link_to @post.title,
domain_fa_post_path(@post),
class: "text-blue-600 hover:underline" %>
</h1>
</div>
</div>
</section>
<% favs = @post.faved_by.includes(:avatar).to_a %>
<% if favs.any? %>
<section
class="overflow-hidden rounded-md border border-slate-300 bg-slate-50"
>
<div class="divide-y divide-slate-200">
<% favs.each do |user| %>
<%= link_to domain_fa_user_path(user),
class: "flex items-center gap-4 p-4 hover:bg-slate-100" do %>
<% if user.avatar&.file_sha256.present? %>
<%= image_tag fa_user_avatar_path(user, thumb: "64-avatar"),
class: "h-12 w-12 rounded-md border object-cover",
alt: user.name %>
<% else %>
<div
class="flex h-12 w-12 items-center justify-center rounded-full bg-slate-200"
>
<i class="bi bi-person text-slate-400"></i>
</div>
<% end %>
<div class="min-w-0">
<div class="font-medium text-slate-900"><%= user.name %></div>
<div class="text-sm text-slate-500">@<%= user.url_name %></div>
</div>
<% end %>
<% end %>
</div>
</section>
<% else %>
<section
class="rounded-md border border-slate-300 bg-slate-50 p-8 text-center"
>
<i class="bi bi-heart mb-3 block text-4xl text-slate-400"></i>
<p class="text-slate-600">No users have favorited this post yet.</p>
</section>
<% end %>
</div>

5
app/views/domain/fa/posts/index.html.erb
View File

@@ -17,10 +17,10 @@
<% end %>
</div>
<%= render partial: "shared/pagination_controls", locals: { collection: @posts } %>
<div class="mx-auto flex flex-row flex-wrap justify-center">
<div class="mx-auto flex max-w-full flex-row flex-wrap justify-center">
<% @posts.each do |post| %>
<div
class="m-4 flex flex-shrink-0 flex-col rounded-lg border border-slate-300 bg-slate-50 shadow-sm"
class="m-4 flex flex-col rounded-lg border border-slate-300 bg-slate-50 shadow-sm"
>
<% if policy(post).view_file? %>
<div
@@ -76,3 +76,4 @@
</div>
<% end %>
</div>
<%= render partial: "shared/pagination_controls", locals: { collection: @posts } %>

8
app/views/domain/fa/users/_inline_link.html.erb
View File

@@ -9,7 +9,9 @@
/>
<span class="sky-link"> <%= user.url_name %> </span>
</a>
<span class="text-slate-500"
><%= pluralize(number_with_delimiter(user.posts.count, delimiter: ","), "post") %></span
>
<% if !defined?(with_post_count) || with_post_count %>
<span class="ml-2 text-slate-500">
<%= pluralize(number_with_delimiter(user.posts.count, delimiter: ","), "post") %>
</span>
<% end %>
</div>

8
app/views/domain/fa/users/_user.html.erb
View File

@@ -1,10 +1,10 @@
<div id="<%= dom_id user %>" class="mx-auto my-4 w-full max-w-2xl space-y-4">
<div id="<%= dom_id user %>" class="mx-auto my-4 w-full space-y-4 md:max-w-2xl">
<%= render "domain/fa/users/show_sections/name_icon_and_status", user: user %>
<div class="flex flex-row gap-4">
<div class="w-1/2">
<div class="flex flex-col gap-4 sm:flex-row">
<div class="w-full sm:w-1/2">
<%= render "domain/fa/users/show_sections/stats", user: user %>
</div>
<div class="w-1/2">
<div class="w-full sm:w-1/2">
<%= render "domain/fa/users/show_sections/recent_posts", user: user %>
</div>
</div>

2
app/views/domain/fa/users/show_sections/_recent_posts.html.erb
View File

@@ -10,7 +10,7 @@
<% if user.posts.any? %>
<% user
.posts
.order(created_at: :desc)
.order(fa_id: :desc)
.limit(5)
.each do |post| %>
<div class="flex items-center px-4 py-2">

41
app/views/indexed_posts/index.html.erb
View File

@@ -1,3 +1,23 @@
<% content_for :head do %>
<style>
.grid-cell {
padding: 0.25rem;
border-right: 1px solid #e2e8f0;
}
.grid-cell:last-child {
padding-left: 0;
padding-right: 1rem;
border-right: none;
}
.grid-cell:first-child {
padding-left: 1rem;
}
.grid-row:hover .grid-cell {
background-color: #f1f5f9;
}
</style>
<% end %>
<div class="mx-auto mt-4 text-center sm:mt-6">
<h1 class="text-2xl">All Posts <%= page_str(params) %></h1>
</div>
@@ -57,27 +77,7 @@
</div>
</div>
</div>
<%= render partial: "shared/pagination_controls", locals: { collection: @posts } %>
<% content_for :head do %>
<style>
.grid-cell {
padding: 0.25rem;
border-right: 1px solid #e2e8f0;
}
.grid-cell:last-child {
padding-left: 0;
padding-right: 1rem;
border-right: none;
}
.grid-cell:first-child {
padding-left: 1rem;
}
.grid-row:hover .grid-cell {
background-color: #f1f5f9;
}
</style>
<% end %>
<% if params[:view] == "table" %>
<div
class="mx-auto grid grid-cols-[auto_1fr_auto_auto_auto] border-b border-slate-300 text-sm"
@@ -102,3 +102,4 @@
<% end %>
</div>
<% end %>
<%= render partial: "shared/pagination_controls", locals: { collection: @posts } %>

4
app/views/log_entries/_content_container.html.erb
View File

@@ -1,10 +1,10 @@
<% contents_path = contents_blob_path(HexUtil.bin2hex(log_entry.response_sha256)) %>
<section class="flex grow justify-center overflow-clip">
<% if is_renderable_image_type?(log_entry.content_type) %>
<img alt="image" src="<%= contents_path %>" class="rounded-md" />
<img alt="image" src="<%= contents_path %>" class="md:rounded-md" />
<% elsif is_renderable_video_type?(log_entry.content_type) %>
<video
class="rounded-md"
class="md:rounded-md"
alt="video"
controls="controls"
loop="loop"

177
app/views/log_entries/stats.html.erb
View File

@@ -1,72 +1,121 @@
<% content_for :head do %>
<style type="text/css" data-turbolinks-track>
table {
border-collapse: collapse;
.grid-cell {
padding: 0.25rem;
border-right: 1px solid #e2e8f0;
}
table td {
border-right: 1px solid black;
text-align: right;
padding: 0 0.5em;
}
table td:last-child {
.grid-cell:last-child {
padding-left: 0;
padding-right: 1rem;
border-right: none;
}
.grid-cell:first-child {
padding-left: 1rem;
}
.grid-row:hover .grid-cell {
background-color: #f1f5f9;
}
</style>
<% end %>
<h1>Http Request Log Stats (<%= link_to "Index", log_entries_path %>)</h1>
<h2>
<% time_ago = proc do |time_window|
if time_window < 1.minute
pluralize(time_window.seconds, "second")
else
time_ago_in_words(time_window.ago)
end
end %>
<% stats_link = proc do |time_window| %>
<% if @time_window == time_window %>
<%= time_ago.call(time_window) %>
<% else %>
<%= link_to time_ago.call(time_window), stats_log_entries_path(seconds: time_window.in_seconds) %>
<div class="mx-auto mt-4 text-center sm:mt-6">
<h1 class="text-2xl">Http Request Log Stats</h1>
<div class="mt-2 text-lg">
<%= link_to "Back to Index",
log_entries_path,
class: "text-blue-600 hover:text-blue-800" %>
</div>
</div>
<div class="mx-auto mt-4 text-center">
<div class="space-x-2 text-lg">
<% [10.seconds, 30.seconds, 1.minute, 5.minutes, 30.minutes].each do |time_window| %>
<% if @time_window == time_window %>
<span class="rounded bg-blue-100 px-2 py-1">
<%= if time_window < 1.minute
pluralize(time_window.seconds, "second")
else
time_ago_in_words(time_window.ago)
end %>
</span>
<% else %>
<%= link_to(
(
if time_window < 1.minute
pluralize(time_window.seconds, "second")
else
time_ago_in_words(time_window.ago)
end
),
stats_log_entries_path(seconds: time_window.in_seconds),
class: "text-blue-600 hover:text-blue-800 hover:bg-blue-50 px-2 py-1 rounded",
) %>
<% end %>
<% end %>
<% end %>
<%= stats_link.call(10.seconds) %>
<%= stats_link.call(30.seconds) %>
<%= stats_link.call(1.minute) %>
<%= stats_link.call(5.minutes) %>
<%= stats_link.call(30.minutes) %>
</h2>
<h2><%= @last_window_count %> requests in last <%= time_ago.call(@time_window.seconds) %> (<%= (@last_window_count.to_f / @time_window.in_seconds).round(1) %>/sec)</h2>
<% if @last_window_bytes > 0 && @last_window_bytes_stored > 0
stored_ratio = (@last_window_bytes_stored.to_f / @last_window_bytes).round(2)
else
stored_ratio = "none"
end %>
<h3>
<%= HexUtil.humansize(@last_window_bytes) %> bytes transferred -
<%= HexUtil.humansize(@last_window_bytes_stored) %> bytes stored (<%= stored_ratio %>x) -
<%= HexUtil.humansize(@last_window_bytes / @time_window.in_seconds) %>/sec
</h3>
<h2>By content type</h2>
<table>
<% @content_type_counts.sort_by { |_ignore, stats| -stats[:count] }.each do |content_type, stats| %>
<tr>
<td><%= content_type %></td>
<td><%= stats[:count] %> requests</td>
<td><%= HexUtil.humansize(stats[:bytes]) %> transferred</td>
<td><%= HexUtil.humansize(stats[:bytes_stored]) %> stored</td>
<td><%= (stats[:bytes_stored].to_f / stats[:bytes]).round(2) %>x storage ratio</td>
</tr>
<% end %>
</table>
<h2>By domain</h2>
<table>
<% @by_domain_counts.sort_by { |_ignore, stats| -stats[:bytes] }.each do |domain, stats| %>
<tr>
<td><%= domain %></td>
<td><%= stats[:count] %> requests</td>
<td><%= HexUtil.humansize(stats[:bytes]) %> transferred</td>
<td><%= HexUtil.humansize(stats[:bytes_stored]) %> stored</td>
<td><%= (stats[:bytes_stored].to_f / stats[:bytes]).round(2) %>x storage ratio</td>
</tr>
<% end %>
</table>
</div>
<div class="mt-4 text-lg">
<%= @last_window_count %> requests in last
<%= if @time_window < 1.minute
pluralize(@time_window.seconds, "second")
else
time_ago_in_words(@time_window.ago)
end %>
<span class="text-slate-600">
(<%= (@last_window_count.to_f / @time_window.in_seconds).round(1) %>/sec)
</span>
</div>
<div class="mt-2 text-slate-600">
<%= HexUtil.humansize(@last_window_bytes) %> bytes transferred •
<%= HexUtil.humansize(@last_window_bytes / @time_window.in_seconds) %>/sec
</div>
</div>
<div class="mx-auto mt-6">
<h2 class="mb-2 text-xl">By content type</h2>
<div class="grid grid-cols-[1fr_auto_auto] border-b border-slate-300 text-sm">
<div class="grid-row contents">
<div class="grid-cell font-semibold">Content Type</div>
<div class="grid-cell text-right font-semibold">Requests</div>
<div class="grid-cell text-right font-semibold">Transferred</div>
</div>
<div class="col-span-full border-b border-slate-300"></div>
<% @content_type_counts
.sort_by { |_ignore, stats| -stats[:count] }
.each do |content_type, stats| %>
<div class="grid-row contents">
<div class="grid-cell"><%= content_type %></div>
<div class="grid-cell text-right"><%= stats[:count] %></div>
<div class="grid-cell text-right">
<%= HexUtil.humansize(stats[:bytes]) %>
</div>
</div>
<div class="col-span-full border-b border-slate-300"></div>
<% end %>
</div>
</div>
<div class="mx-auto mt-6">
<h2 class="mb-2 text-xl">By domain</h2>
<div class="grid grid-cols-[1fr_auto_auto] border-b border-slate-300 text-sm">
<div class="grid-row contents">
<div class="grid-cell font-semibold">Domain</div>
<div class="grid-cell text-right font-semibold">Requests</div>
<div class="grid-cell text-right font-semibold">Transferred</div>
</div>
<div class="col-span-full border-b border-slate-300"></div>
<% @by_domain_counts
.sort_by { |_ignore, stats| -stats[:bytes] }
.each do |domain, stats| %>
<div class="grid-row contents">
<div class="grid-cell"><%= domain %></div>
<div class="grid-cell text-right"><%= stats[:count] %></div>
<div class="grid-cell text-right">
<%= HexUtil.humansize(stats[:bytes]) %>
</div>
</div>
<div class="col-span-full border-b border-slate-300"></div>
<% end %>
</div>
</div>

8
app/views/shared/_pagination_controls.html.erb
View File

@@ -6,10 +6,10 @@
<%= link_to_previous_page collection,
raw("← Previous Page (#{collection.current_page - 1})"),
class:
"sky-link px-6 py-2 text-center border-slate-300 border-r-2 bg-slate-100 flex-1 flex items-center justify-center" %>
"sky-link py-2 text-center border-slate-300 border-r-2 bg-slate-100 flex-1 flex items-center justify-center" %>
<% else %>
<div
class="flex flex-1 items-center justify-center border-r border-slate-300 bg-slate-50 px-6 py-2 text-center text-slate-400"
class="flex flex-1 items-center justify-center border-r border-slate-300 bg-slate-50 py-2 text-center text-slate-400"
>
← Previous Page
</div>
@@ -18,10 +18,10 @@
<%= link_to_next_page collection,
raw("Next Page (#{collection.current_page + 1}) →"),
class:
"sky-link px-6 py-2 text-center bg-slate-100 flex-1 flex items-center justify-center" %>
"sky-link py-2 text-center bg-slate-100 flex-1 flex items-center justify-center" %>
<% else %>
<div
class="flex flex-1 items-center justify-center bg-slate-50 px-6 py-2 text-center text-slate-400"
class="flex flex-1 items-center justify-center bg-slate-50 py-2 text-center text-slate-400"
>
Next Page →
</div>

1
config/routes.rb
View File

@@ -29,6 +29,7 @@ Rails.application.routes.draw do
end
resources :posts, param: :fa_id, only: [:show] do
post :scan_post, on: :member
get :favorites, on: :member
end
end
namespace :e621 do

431
spec/helpers/domain/fa/posts_helper_spec.rb Normal file
View File

@@ -0,0 +1,431 @@
require "rails_helper"
RSpec.describe Domain::Fa::PostsHelper, type: :helper do
describe "#post_state_string" do
let(:post) { build(:domain_fa_post) }
context "when post has a file" do
before { allow(post).to receive(:have_file?).and_return(true) }
it 'returns "file"' do
expect(helper.post_state_string(post)).to eq("file")
end
end
context "when post is scanned but has no file" do
before do
allow(post).to receive(:have_file?).and_return(false)
allow(post).to receive(:scanned?).and_return(true)
end
it 'returns "scanned"' do
expect(helper.post_state_string(post)).to eq("scanned")
end
end
context "when post is neither scanned nor has file" do
before do
allow(post).to receive(:have_file?).and_return(false)
allow(post).to receive(:scanned?).and_return(false)
allow(post).to receive(:state).and_return("pending")
end
it "returns the post state" do
expect(helper.post_state_string(post)).to eq("pending")
end
end
end
describe "#page_str" do
context "when page is greater than 1" do
it "returns page string" do
expect(helper.page_str(page: "2")).to eq("(page 2)")
end
end
context "when page is 1 or not specified" do
it "returns nil" do
expect(helper.page_str(page: "1")).to be_nil
expect(helper.page_str({})).to be_nil
end
end
end
describe "#scanned_and_file_description" do
let(:post) { build(:domain_fa_post) }
let(:file) { double("file", created_at: 1.day.ago) }
context "when post is scanned with known time" do
before do
allow(post).to receive(:scanned?).and_return(true)
allow(post).to receive(:scanned_at).and_return(2.hours.ago)
end
it "includes scan time" do
allow(post).to receive(:file).and_return(nil)
expect(helper.scanned_and_file_description(post)).to include(
"Scanned about 2 hours ago",
)
end
end
context "when post is scanned but time unknown" do
before do
allow(post).to receive(:scanned?).and_return(true)
allow(post).to receive(:scanned_at).and_return(nil)
end
it "shows unknown scan time" do
allow(post).to receive(:file).and_return(nil)
expect(helper.scanned_and_file_description(post)).to include(
"Scanned (unknown) ago",
)
end
end
context "when post has file" do
before do
allow(post).to receive(:scanned?).and_return(false)
allow(post).to receive(:file).and_return(file)
end
it "includes file time" do
expect(helper.scanned_and_file_description(post)).to include(
"file 1 day ago",
)
end
end
context "when post has neither scan nor file" do
before do
allow(post).to receive(:scanned?).and_return(false)
allow(post).to receive(:file).and_return(nil)
end
it "shows appropriate message" do
expect(helper.scanned_and_file_description(post)).to eq(
"Not scanned, no file",
)
end
end
end
describe "#fa_post_description_sanitized" do
describe "basic HTML sanitization" do
it "works" do
sanitized =
helper.fa_post_description_sanitized(
'<b>Bold</b> <i>Italic</i> <span style="color: red; font-size: 12px;">Styled</span> <script>alert("bad")</script>',
)
[
"<b>Bold</b>",
"<i>Italic</i>",
'<span style="color: red; font-size: 12px;">Styled</span>',
].each { |text| expect(sanitized).to include(text) }
["<script>"].each { |text| expect(sanitized).not_to include(text) }
end
end
describe "FA post link handling" do
%w[
https://www.furaffinity.net/view/123456/
https://furaffinity.net/view/123456/
https://www.furaffinity.net/view/123456
https://furaffinity.net/view/123456
furaffinity.net/view/123456
furaffinity.net/view/123456/
www.furaffinity.net/view/123456/
www.furaffinity.net/view/123456
www.furaffinity.net/view/123456/
www.furaffinity.net/view/123456
Furaffinity.net/view/123456
].each do |url|
it "processes #{url}" do
post = create(:domain_fa_post, fa_id: "123456", title: "Post Title")
html = %(<a href="#{url}">FA Link</a>)
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to eq_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_post",
locals: {
post: post,
},
),
)
end
end
it "sanitizes nested content within valid FA links" do
html =
'<a href="https://www.furaffinity.net/view/123456/"><b>Bold</b> <script>bad</script> <span style="color: red; background: blue;">Text</span></a>'
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to eq_html(
'<b>Bold</b> <span style="color: red; ">Text</span>',
)
end
describe "post title lookup" do
let!(:post1) do
create(:domain_fa_post, fa_id: "123", title: "First Post")
end
let!(:post2) do
create(:domain_fa_post, fa_id: "456", title: "Second Post")
end
it "replaces link text with post titles when posts exist" do
html =
'
<a href="https://www.furaffinity.net/view/123/">Original Text 1</a>
<a href="https://www.furaffinity.net/view/456/">Original Text 2</a>
'
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to include_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_post",
locals: {
post: post1,
},
),
)
expect(sanitized).to include_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_post",
locals: {
post: post2,
},
),
)
end
it "removes the link when the post doesn't exist" do
html =
'<a href="https://www.furaffinity.net/view/789/">Original Text</a>'
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to eq_html("Original Text")
end
it "replaces nested elements when replacing titles" do
html =
'<a href="https://www.furaffinity.net/view/123/"><b>Bold</b> <i>Text</i></a>'
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to eq_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_post",
locals: {
post: post1,
},
),
)
end
it "performs a single database query for multiple links" do
html =
'
<a href="https://www.furaffinity.net/view/123/">Link 1</a>
<a href="https://www.furaffinity.net/view/456/">Link 2</a>
<a href="https://www.furaffinity.net/view/789/">Link 3</a>
'
expect {
helper.fa_post_description_sanitized(html)
}.to make_database_queries(count: 1)
end
end
end
describe "non-FA link handling" do
{
'<a href="https://www.google.com/">Google</a>' => "Google",
'<a href="https://www.furaffinity.net/user/">Invalid User Link</a>' =>
"Invalid User Link",
'<a href="https://www.furaffinity.net/view/">Invalid Post Link</a>' =>
"Invalid Post Link",
'<a href="bfuraffinity.net/view/">Not actually an FA Link</a>' =>
"Not actually an FA Link",
"<a>No Href Link</a>" => "No Href Link",
'<a href="https://www.google.com/"><b>Bold</b> and <i>italic</i> text</a>' =>
"<b>Bold</b> and <i>italic</i> text",
}.each do |input, expected|
it "processes '#{input}' correctly" do
sanitized = helper.fa_post_description_sanitized(input)
expect(sanitized).to eq_html(expected)
end
end
end
describe "CSS property handling" do
it "only allows specified CSS properties" do
input =
'<span style="font-size: 12px; color: red; background: blue; position: absolute;">Test</span>'
sanitized = helper.fa_post_description_sanitized(input)
expect(sanitized).to include("font-size")
expect(sanitized).to include("color")
expect(sanitized).not_to include("background")
expect(sanitized).not_to include("position")
end
end
describe "nested element handling" do
{
"valid nested elements" => {
input: '<b><i><span style="color: red;">Nested</span></i></b>',
expected: '<b><i><span style="color: red;">Nested</span></i></b>',
},
"invalid nested elements" => {
input: "<b><script>bad</script><i>good</i></b>",
expected: "<b><i>good</i></b>",
},
}.each do |description, test_case|
it "handles #{description}" do
sanitized = helper.fa_post_description_sanitized(test_case[:input])
expect(sanitized).to eq(test_case[:expected])
end
end
end
describe "multiple link handling" do
it "correctly processes multiple links of different types" do
post1 = create(:domain_fa_post, fa_id: "123", title: "Post Title")
user1 =
create(:domain_fa_user, url_name: "username1", name: "User Name 1")
html =
'
<a href="https://www.furaffinity.net/view/123/">FA Post</a>
<a href="https://www.furaffinity.net/user/username1/">FA User</a>
<a href="https://google.com">Google</a>
<a>No href</a>
'
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to include_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_post",
locals: {
post: post1,
},
),
)
expect(sanitized).to include_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_user",
locals: {
user: user1,
},
),
)
expect(sanitized).not_to include("FA Post")
expect(sanitized).not_to include("FA User")
expect(sanitized).to include("Google")
expect(sanitized).to include("No href")
expect(sanitized.scan(/<a/).length).to eq(2)
end
end
describe "FA user link handling" do
let!(:user1) do
create(:domain_fa_user, url_name: "artistone", name: "Artist One")
end
let!(:user2) do
create(:domain_fa_user, url_name: "artisttwo", name: "Artist Two")
end
let!(:user3) do
create(:domain_fa_user, url_name: "artistthree", name: "Artist Three")
end
it "replaces link text with user names when users exist" do
html =
'
<a href="https://www.furaffinity.net/user/artistone/">Original Text 1</a>
<a href="https://www.furaffinity.net/user/artisttwo/">Original Text 2</a>
'
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to include_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_user",
locals: {
user: user1,
},
),
)
expect(sanitized).to include_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_user",
locals: {
user: user2,
},
),
)
end
it "Removes the link when the user doesn't exist" do
html =
'<a href="https://www.furaffinity.net/user/nonexistent/">Original Text</a>'
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to eq_html("Original Text")
end
it "replaces nested elements when replacing names" do
html =
'<a href="https://www.furaffinity.net/user/artistone/"><b>Bold</b> <i>Text</i></a>'
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to include_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_user",
locals: {
user: user1,
},
),
)
end
it "performs minimal database queries for multiple links" do
html =
'
<a href="https://www.furaffinity.net/user/artistone/">Link 1</a>
<a href="https://www.furaffinity.net/user/artisttwo/">Link 2</a>
<a href="https://www.furaffinity.net/user/artistthree/">Link 3</a>
<a href="https://www.furaffinity.net/user/nonexistent/">Link 4</a>
'
expect {
helper.fa_post_description_sanitized(html)
}.to make_database_queries(count: 2)
end
%w[
https://www.furaffinity.net/user/artistone/
https://furaffinity.net/user/artistone/
https://www.furaffinity.net/user/artistone
https://furaffinity.net/user/artistone
furaffinity.net/user/artistone
furaffinity.net/user/artistone/
www.furaffinity.net/user/artistone/
www.furaffinity.net/user/artistone
Furaffinity.net/user/artistone
].each do |url|
it "processes #{url}" do
html = %(<a href="#{url}">FA User Link</a>)
sanitized = helper.fa_post_description_sanitized(html)
expect(sanitized).to eq_html(
render(
partial: "domain/fa/posts/description_inline_link_fa_user",
locals: {
user: user1,
},
),
)
end
end
end
end
end

2
spec/rails_helper.rb
View File

@@ -3,6 +3,8 @@ ENV["RAILS_ENV"] ||= "test"
require_relative "../config/environment"
require "spec_helper"
require "rspec/rails"
require "pundit/matchers"
require "db-query-matchers"
# Prevent database truncation if the environment is production
if Rails.env.production?
abort("The Rails environment is running in production mode!")

1
spec/spec_helper.rb
View File

@@ -15,6 +15,7 @@
# See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
require "./spec/helpers/spec_helpers"
require "./spec/helpers/debug_helpers"
require "./spec/support/matchers/html_matchers"
RSpec.configure do |config|
config.include SpecHelpers

53
spec/support/matchers/html_matchers.rb Normal file
View File

@@ -0,0 +1,53 @@
normalize_html = lambda { |html| Nokogiri::HTML5.fragment(html.to_s) }
nodes_equal =
lambda do |a, b|
return true if a == b
return false unless a.name == b.name
stype = a.node_type
otype = b.node_type
return false unless stype == otype
sa = a.attributes
oa = b.attributes
return false unless sa.length == oa.length
sa = sa.sort.map { |n, a| [n, a.value, a.namespace && a.namespace.href] }
oa = oa.sort.map { |n, a| [n, a.value, a.namespace && a.namespace.href] }
return false unless sa == oa
skids = a.children
okids = b.children
return false unless skids.length == okids.length
if stype == Nokogiri::XML::Node::TEXT_NODE &&
(a.content.strip != b.content.strip)
return false
end
skids.to_enum.with_index.all? { |ski, i| nodes_equal.call(ski, okids[i]) }
end
RSpec::Matchers.define :eq_html do |expected|
match do |actual|
nodes_equal.call(normalize_html.call(expected), normalize_html.call(actual))
end
failure_message do |actual|
"expected HTML to match.\nExpected: #{normalize_html.call(expected)}\nGot: #{normalize_html.call(actual)}"
end
end
RSpec::Matchers.define :include_html do |expected|
match do |actual|
actual = normalize_html.call(actual)
expected = normalize_html.call(expected).children.first
node_or_children_equal =
lambda do |node|
nodes_equal.call(node, expected) ||
node.children.any? { |child| node_or_children_equal.call(child) }
end
node_or_children_equal.call(actual)
end
failure_message do |actual|
"expected HTML to include fragment.\nExpected fragment:\n#{normalize_html.call(expected)}\nIn:\n#{normalize_html.call(actual)}"
end
end