Update blob entry handling and enhance staging configuration

- Changed the staging server port from 3000 to 3001 in the Procfile for better port management.
- Introduced a new BlobEntry model to replace BlobEntryP, ensuring a more consistent data structure across the application.
- Updated various controllers and views to utilize the new BlobEntry model, enhancing data retrieval and rendering processes.
- Added a new BlobEntriesController to manage blob entries, including a show action for retrieving content based on SHA256.
- Enhanced the Rakefile to enqueue periodic jobs for updating posts, improving background processing capabilities.
- Updated routes to reflect the new BlobEntry model and ensure proper resource handling.
- Improved tests for blob entry functionality, ensuring robust coverage and reliability in data handling.
This commit is contained in:
Dylan Knutson
2024-12-30 19:35:27 +00:00
parent 2681502c3e
commit 15c11b2b89
41 changed files with 608 additions and 168 deletions

View File

@@ -54,6 +54,19 @@ RUN \
uuid-dev \
zlib1g-dev
# Install & configure delta diff tool
RUN wget -O- https://github.com/dandavison/delta/releases/download/0.18.2/git-delta_0.18.2_amd64.deb > /tmp/git-delta.deb && \
sudo dpkg -i /tmp/git-delta.deb && \
rm /tmp/git-delta.deb
RUN su vscode -c 'git config --global core.pager "delta"' && \
su vscode -c 'git config --global interactive.diffFilter "delta --color-only"' && \
su vscode -c 'git config --global delta.navigate true' && \
su vscode -c 'git config --global delta.dark true' && \
su vscode -c 'git config --global delta.side-by-side true' && \
su vscode -c 'git config --global merge.conflictstyle "zdiff3"'
# Install native gems
COPY --from=native-gems /usr/src/app/gems/xdiff-rb /gems/xdiff-rb
COPY --from=native-gems /usr/src/app/gems/rb-bsdiff /gems/rb-bsdiff

View File

@@ -1,25 +1,26 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
// README at: https://github.com/devcontainers/templates/tree/main/src/ruby-rails-postgres
{
"name": "Ruby on Rails & Postgres",
"dockerComposeFile": "docker-compose.yml",
"service": "app",
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
"features": {
"ghcr.io/meaningful-ooo/devcontainer-features/fish:1": {},
"ghcr.io/nikobockerman/devcontainer-features/fish-persistent-data:2": {}
},
// Use 'forwardPorts' to make a list of ports inside the container available locally.
// This can be used to network with other containers or the host.
// "forwardPorts": [3000, 5432],
// Use 'postCreateCommand' to run commands after the container is created.
// "postCreateCommand": "bundle install && rake db:setup",
"postCreateCommand": ".devcontainer/post-create.sh",
"forwardPorts": [
8080, // pgadmin
3000 // rails
]
// Configure tool-specific properties.
// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "root"
}
"name": "Ruby on Rails & Postgres",
"dockerComposeFile": "docker-compose.yml",
"service": "app",
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
"features": {
"ghcr.io/meaningful-ooo/devcontainer-features/fish:1": {},
"ghcr.io/nikobockerman/devcontainer-features/fish-persistent-data:2": {}
},
// Use 'forwardPorts' to make a list of ports inside the container available locally.
// This can be used to network with other containers or the host.
// "forwardPorts": [3000, 5432],
// Use 'postCreateCommand' to run commands after the container is created.
// "postCreateCommand": "bundle install && rake db:setup",
"postCreateCommand": ".devcontainer/post-create.sh",
"forwardPorts": [
8080, // pgadmin
3000, // rails development
3001 // rails staging
]
// Configure tool-specific properties.
// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "root"
}

View File

@@ -1,4 +1,4 @@
rails: RAILS_ENV=staging ./bin/rails s -p 3000
rails: RAILS_ENV=staging ./bin/rails s -p 3001
wp-client: RAILS_ENV=development HMR=true ./bin/webpacker-dev-server
wp-server: RAILS_ENV=development HMR=true SERVER_BUNDLE_ONLY=yes ./bin/webpacker --watch
css: RAILS_ENV=development yarn build:css[debug] --watch

View File

@@ -42,6 +42,10 @@ task periodic_tasks: %i[environment set_logger_stdout] do
Rake::Task["fa:browse_page_job"].execute
Rake::Task["fa:home_page_job"].execute
Rake::Task["e621:posts_index_job"].execute
Domain::Inkbunny::Job::UpdatePostsJob.set(
queue: "inkbunny",
priority: -20,
).perform_later({})
puts "enqueue periodic jobs"
sleep 1.minute
end

View File

@@ -3,3 +3,5 @@
- [ ] Add bookmarking feature for posts across different domains
- [ ] Add search feature to search FA descriptions, tags, E621 descriptions, tags
- [ ] Get inkbunny index scan job working
- [ ] Attach logs to jobs, page to view jobs and their logs
- [ ] Standardize all the embeddings tables to use the same schema (item_id, embedding)

View File

@@ -1,12 +1,7 @@
class BlobsController < ApplicationController
skip_before_action :authenticate_user!, only: [:contents]
class BlobEntriesController < ApplicationController
skip_before_action :authenticate_user!, only: [:show]
def show
sha256 = HexUtil.hex2bin(params[:sha256])
@blob = BlobEntryP.ensure(sha256: sha256)
end
def contents
thumb = params[:thumb]
raise("invalid thumb #{thumb}") if thumb.present? && !thumb_params(thumb)
@@ -14,13 +9,13 @@ class BlobsController < ApplicationController
response.headers["Expires"] = expires_dur.from_now.httpdate
expires_in expires_dur, public: true
sha256 = params[:id]
sha256 = params[:sha256]
etag = sha256
etag += "-#{thumb}" if thumb
return unless stale?(last_modified: Time.at(0), strong_etag: etag)
# images, videos, etc
blob_entry = BlobEntryP.find(HexUtil.hex2bin(sha256))
blob_entry = BlobEntry.find(HexUtil.hex2bin(sha256))
if helpers.is_send_data_content_type?(blob_entry.content_type)
if !thumb.blank? &&
helpers.is_thumbable_content_type?(blob_entry.content_type)

View File

@@ -7,26 +7,25 @@ class Domain::Fa::PostsController < ApplicationController
skip_before_action :authenticate_user!, only: %i[show index]
# This action is always scoped to a user, so the :user_url_name parameter is required.
# GET /domain/fa/users/:user_url_name/posts
def index
@user =
Domain::Fa::User.find_by(url_name: params[:user_url_name]) ||
raise(ActiveRecord::RecordNotFound)
@user = Domain::Fa::User.find_by!(url_name: params[:user_url_name])
relation = policy_scope(@user.posts)
@posts =
relation
.includes(:creator, :file)
.order(fa_id: :desc)
.page(params[:page])
.per(50)
.order(fa_id: :desc)
.without_count
end
# GET /domain/fa/posts/1
# GET /domain/fa/posts/:fa_id
def show
end
# GET /domain/fa/posts/1/favorites
# GET /domain/fa/posts/:fa_id/favorites
def favorites
@post = Domain::Fa::Post.find_by_fa_id!(params[:fa_id])
end

View File

@@ -1,8 +1,15 @@
class Domain::Inkbunny::PostsController < ApplicationController
skip_before_action :authenticate_user!, only: [:show]
skip_before_action :authenticate_user!, only: %i[show index]
def index
@posts = Domain::Inkbunny::Post.page(params[:page])
relation = Domain::Inkbunny::Post.includes(:creator, :files)
if params[:user_id].present?
@user = Domain::Inkbunny::User.find(params[:user_id])
relation = relation.where(creator: @user)
end
@posts = relation.order(ib_post_id: :desc).page(params[:page]).per(50)
end
def show

View File

@@ -1,11 +1,15 @@
module Domain::Fa::UsersHelper
def avatar_url(sha256, thumb: "32-avatar")
blob_path(HexUtil.bin2hex(sha256), format: "jpg", thumb: thumb)
end
def fa_user_avatar_path(user, thumb: nil)
if (sha256 = user.avatar&.file_sha256)
contents_blob_path(HexUtil.bin2hex(sha256), format: "jpg", thumb: thumb)
blob_path(HexUtil.bin2hex(sha256), format: "jpg", thumb: thumb)
else
# default / 'not found' avatar image
# "/blobs/9080fd4e7e23920eb2dccfe2d86903fc3e748eebb2e5aa8c657bbf6f3d941cdc/contents.jpg"
image_path("user-circle.svg")
asset_path("user-circle.svg")
end
end

View File

@@ -6,7 +6,7 @@ class DbSampler
::Domain::Fa::Follow => %i[follower followed],
::Domain::Fa::Fav => %i[user post],
::Domain::Fa::UserFactor => [],
::BlobEntryP => [:base],
::BlobEntry => [:base],
::HttpLogEntry => %i[
request_headers
response_headers

View File

@@ -122,7 +122,7 @@ class Scraper::GalleryDlClient
retries = 0
begin
response_blob_entry =
BlobEntryP.find_or_build(
BlobEntry.find_or_build(
content_type: content_type,
contents: http_event.body,
)

View File

@@ -83,7 +83,7 @@ class Scraper::HttpClient
retries = 0
begin
response_blob_entry =
BlobEntryP.find_or_build(
BlobEntry.find_or_build(
content_type: content_type,
contents: response_body,
)

View File

@@ -1,4 +1,4 @@
class BlobEntryP < ReduxApplicationRecord
class BlobEntry < ReduxApplicationRecord
self.table_name = "blob_entries_p"
include ImmutableModel
@@ -7,17 +7,17 @@ class BlobEntryP < ReduxApplicationRecord
self.primary_key = :sha256
EMPTY_FILE_SHA256 =
HexUtil.hex2bin(
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
)
belongs_to :base,
optional: true,
foreign_key: :base_sha256,
class_name: "::BlobEntryP"
class_name: "::BlobEntry"
after_create do
actual_sha256 = Digest::SHA256.digest(contents)
raise("digest mismatch for BlobEntryP") if sha256 != actual_sha256
raise("digest mismatch for BlobEntry") if sha256 != actual_sha256
end
def base
@@ -29,7 +29,7 @@ class BlobEntryP < ReduxApplicationRecord
length: {
minimum: 0,
allow_nil: false,
message: "can't be nil"
message: "can't be nil",
}
validates :sha256, length: { is: 32 }
validates :base_sha256, length: { is: 32 }, if: :base_sha256
@@ -57,12 +57,12 @@ class BlobEntryP < ReduxApplicationRecord
def self.find_or_build(content_type:, contents:)
sha256 = Digest::SHA256.digest(contents)
BlobEntryP.find_by(sha256: sha256) ||
BlobEntry.find_by(sha256: sha256) ||
begin
build_record(
content_type: content_type,
sha256: sha256,
contents: contents
contents: contents,
)
end
end
@@ -76,7 +76,7 @@ class BlobEntryP < ReduxApplicationRecord
sha256: sha256,
content_type: content_type,
size: contents.size,
contents: contents
contents: contents,
)
record
end

View File

@@ -4,8 +4,8 @@ class Domain::Fa::UserAvatar < ReduxApplicationRecord
schema_version: 1,
separate_versions_table: true,
map_attribute: {
file_sha256: ::Sha256AttributeMapper
}
file_sha256: ::Sha256AttributeMapper,
},
)
enum :state,
@@ -13,7 +13,7 @@ class Domain::Fa::UserAvatar < ReduxApplicationRecord
:ok, # got the file, no problem
:download_error, # other error processing the file
:no_file_on_guessed_user_page_error,
:file_not_found # 404 from server
:file_not_found, # 404 from server
]
after_initialize do
self.state ||= :ok
@@ -23,12 +23,12 @@ class Domain::Fa::UserAvatar < ReduxApplicationRecord
belongs_to :user, class_name: "::Domain::Fa::User"
belongs_to :file,
foreign_key: :file_sha256,
class_name: "::BlobEntryP",
class_name: "::BlobEntry",
optional: true
belongs_to :log_entry, class_name: "::HttpLogEntry", optional: true
def file
@file_model ||= BlobEntryP.ensure(file_sha256) if file_sha256
@file_model ||= BlobEntry.ensure(file_sha256) if file_sha256
end
before_validation { file_uri = Addressable::URI.parse(file_url_str) }
@@ -53,7 +53,7 @@ class Domain::Fa::UserAvatar < ReduxApplicationRecord
page =
Domain::Fa::Parser::Page.new(
hle.response.contents,
require_logged_in: false
require_logged_in: false,
)
if page.probably_user_page? && (url = page.user_page.profile_thumb_url)
return :user_page, url

View File

@@ -4,7 +4,7 @@ class Domain::Inkbunny::File < ReduxApplicationRecord
belongs_to :post, class_name: "::Domain::Inkbunny::Post", inverse_of: :files
belongs_to :blob_entry,
class_name: "::BlobEntryP",
class_name: "::BlobEntry",
foreign_key: :blob_entry_sha256,
optional: true
@@ -26,6 +26,6 @@ class Domain::Inkbunny::File < ReduxApplicationRecord
md5_initial
md5_full
md5s
]
],
)
end

View File

@@ -9,7 +9,7 @@ class HttpLogEntry < ReduxApplicationRecord
belongs_to :response,
foreign_key: :response_sha256,
class_name: "::BlobEntryP",
class_name: "::BlobEntry",
autosave: true
belongs_to :request_headers, class_name: "::HttpLogEntryHeader"
@@ -48,7 +48,7 @@ class HttpLogEntry < ReduxApplicationRecord
if association(:response).loaded?
response.size
else
BlobEntryP.where(sha256: response_sha256).pick(:size)
BlobEntry.where(sha256: response_sha256).pick(:size)
end
end

View File

@@ -31,7 +31,7 @@
<img
class="max-h-[300px] max-w-[300px] rounded-md border border-slate-300 object-contain shadow-md"
alt="<%= post.title %>"
src="<%= contents_blob_path(
src="<%= blob_path(
HexUtil.bin2hex(post.file.response_sha256),
format: "jpg",
thumb: "small",

View File

@@ -1,32 +1,63 @@
<div class='mx-auto mt-4 sm:mt-6 text-center'>
<div class="mx-auto mt-4 text-center sm:mt-6">
<% if @user %>
<h1 class='text-2xl'><%= link_to(@user.name, @user, class: "underline") %>'s posts</h1>
<h1 class="text-2xl">
<%= link_to(@user.name, @user, class: "underline") %>'s posts
</h1>
<% else %>
<h1 class='text-2xl'>Inkbunny Posts (<%= @posts.total_count %>), page <%= page_str(params) %></h1>
<h1 class="text-2xl">
Inkbunny Posts (<%= @posts.total_count %>), page <%= page_str(params) %>
</h1>
<% end %>
</div>
<div class='mx-auto w-full border-2 border-slate-300 sm:max-w-md rounded-md mt-4 sm:mt-6 mb-4 sm:mb-6'>
<div class='flex justify-center items-stretch h-full'>
<%= link_to "Previous page", path_to_prev_page(@posts), class: 'hover:underline hover:bg-slate-200 hover:shadow-sm px-6 py-2 text-center border-slate-300 border-r-2 bg-slate-100 flex-1 flex items-center justify-center' %>
<%= link_to "Next page", path_to_next_page(@posts), class: 'hover:underline hover:bg-slate-200 hover:shadow-sm px-6 py-2 text-center bg-slate-100 flex-1 flex items-center justify-center' %>
<div
class="mx-auto mb-4 mt-4 w-full rounded-md border-2 border-slate-300 sm:mb-6 sm:mt-6 sm:max-w-md"
>
<div class="flex h-full items-stretch justify-center">
<%= link_to "Previous page",
path_to_prev_page(@posts),
class:
"hover:underline hover:bg-slate-200 hover:shadow-sm px-6 py-2 text-center border-slate-300 border-r-2 bg-slate-100 flex-1 flex items-center justify-center" %>
<%= link_to "Next page",
path_to_next_page(@posts),
class:
"hover:underline hover:bg-slate-200 hover:shadow-sm px-6 py-2 text-center bg-slate-100 flex-1 flex items-center justify-center" %>
</div>
</div>
<div class='flex-row mx-2'>
<div class="mx-2 flex-row">
<% @posts.each do |post| %>
<div class='mx-auto border-slate-300 border-2 rounded-md mb-4 p-2 bg-slate-100 max-w-6xl'>
<div class='text-slate-800 mb-2 flex justify-between'>
<div
class="mx-auto mb-4 max-w-6xl rounded-md border-2 border-slate-300 bg-slate-100 p-2"
>
<div class="mb-2 flex justify-between text-slate-800">
<div>
<%= link_to post.title, post, class: 'hover:underline' %>
<span class="text-slate-500 text-sm ml-2">by <%= link_to post.creator.name, post.creator, class: 'hover:underline' %></span>
<%= link_to post.title, post, class: "hover:underline" %>
<span class="ml-2 text-sm text-slate-500"
>by
<%= link_to post.creator.name, post.creator, class: "hover:underline" %></span
>
</div>
<div class="text-sm text-slate-400">
(ID: <%= post.ib_post_id %>,
<%= pluralize(post.files.count, "file") %>)
</div>
<div class='text-slate-400 text-sm'>(ID: <%= post.ib_post_id %>, <%= pluralize(post.files.count, 'file') %>)</div>
</div>
<div class='flex flex-row gap-2'>
<div class="flex flex-row gap-2">
<% post.files.each do |file| %>
<% img_src_path = contents_blob_path(HexUtil.bin2hex(file.blob_entry_sha256), format: "jpg", thumb: 'small') %>
<div class="rounded-md overflow-hidden">
<img class='my-2 first:pl-0 last:pr-0 h-32 border-2 border-slate-400 shadow-md rounded-md' alt='<%= post.title %>' src='<%= img_src_path %>' />
</div>
<% if policy(post).view_file? %>
<% img_src_path =
blob_path(
HexUtil.bin2hex(file.blob_entry_sha256),
format: "jpg",
thumb: "small",
) %>
<div class="overflow-hidden rounded-md">
<img
class="my-2 h-32 rounded-md border-2 border-slate-400 shadow-md first:pl-0 last:pr-0"
alt="<%= post.title %>"
src="<%= img_src_path %>"
/>
</div>
<% end %>
<% end %>
</div>
</div>

View File

@@ -52,7 +52,7 @@
<img
class="h-auto w-full"
alt="<%= @post.title %>"
src="<%= contents_blob_path(HexUtil.bin2hex(file.blob_entry_sha256), format: "jpg") %>"
src="<%= blob_path(HexUtil.bin2hex(file.blob_entry_sha256), format: "jpg") %>"
/>
</div>
<% else %>

View File

@@ -1,22 +1,32 @@
<div class='mx-auto'>
<h1 class='text-2xl'><%= @user.name %>'s Profile</h1>
<div class="mx-auto">
<h1 class="text-2xl"><%= @user.name %>'s Profile</h1>
</div>
<div class='mx-auto'>
<h2 class='text-xl mt-4 mb-2'>Posts (<%= @user.posts.count %>)</h2>
<div class='flex-row mx-2'>
<div class="mx-auto">
<h2 class="mb-2 mt-4 text-xl">Posts (<%= @user.posts.count %>)</h2>
<div class="mx-2 flex-row">
<% @user.posts.each do |post| %>
<div class='border-stone-00 border-2 rounded-md mb-4 p-4'>
<div class='text-stone-800 mb-2 flex justify-between'>
<div>
<%= link_to post.title, post, class: 'hover:underline' %>
<div class="border-stone-00 mb-4 rounded-md border-2 p-4">
<div class="mb-2 flex justify-between text-stone-800">
<div><%= link_to post.title, post, class: "hover:underline" %></div>
<div class="text-sm text-stone-400">
(ID: <%= post.ib_post_id %>,
<%= pluralize(post.files.count, "file") %>)
</div>
<div class='text-stone-400 text-sm'>(ID: <%= post.ib_post_id %>, <%= pluralize(post.files.count, 'file') %>)</div>
</div>
<div class='flex flex-row gap-2'>
<div class="flex flex-row gap-2">
<% post.files.each do |file| %>
<% img_src_path = contents_blob_path(HexUtil.bin2hex(file.blob_entry_sha256), format: "jpg", thumb: 'small') %>
<div class="rounded-md overflow-hidden">
<img class='p-2 first:pl-0 last:pr-0 h-32' alt='<%= post.title %>' src='<%= img_src_path %>' />
<% img_src_path =
blob_path(
HexUtil.bin2hex(file.blob_entry_sha256),
format: "jpg",
thumb: "small",
) %>
<div class="overflow-hidden rounded-md">
<img
class="h-32 p-2 first:pl-0 last:pr-0"
alt="<%= post.title %>"
src="<%= img_src_path %>"
/>
</div>
<% end %>
</div>

View File

@@ -8,7 +8,7 @@
<div class="flex items-center justify-center p-4">
<% if post.file_sha256.present? %>
<%= link_to show_path(post) do %>
<%= image_tag contents_blob_path(
<%= image_tag blob_path(
HexUtil.bin2hex(post.file_sha256),
format: "jpg",
thumb: "small",

View File

@@ -1,7 +1,7 @@
<div class="grid-row contents">
<div class="grid-cell">
<% if post.file_sha256.present? %>
<%= image_tag contents_blob_path(
<%= image_tag blob_path(
HexUtil.bin2hex(post.file_sha256),
format: "jpg",
thumb: "tiny",

View File

@@ -1,27 +1,23 @@
<% contents_path = contents_blob_path(HexUtil.bin2hex(log_entry.response_sha256)) %>
<% path = blob_path(HexUtil.bin2hex(log_entry.response_sha256)) %>
<section class="flex grow justify-center overflow-clip">
<% if is_renderable_image_type?(log_entry.content_type) %>
<img alt="image" src="<%= contents_path %>" class="md:rounded-md" />
<img alt="image" src="<%= path %>" class="md:rounded-md" />
<% elsif is_renderable_video_type?(log_entry.content_type) %>
<video
class="md:rounded-md"
alt="video"
controls="controls"
loop="loop"
src="<%= contents_path %>"
src="<%= path %>"
></video>
<% elsif is_flash_content_type?(log_entry.content_type) %>
<embed
alt="embed"
type="<%= log_entry.content_type %>"
src="<%= contents_path %>"
/>
<embed alt="embed" type="<%= log_entry.content_type %>" src="<%= path %>" />
<% else %>
<iframe
sandbox
title="log entry contents"
referrerpolicy="no-referrer"
src="<%= contents_path %>"
src="<%= path %>"
></iframe>
<% end %>
</section>

View File

@@ -25,9 +25,9 @@ Rails.application.routes.draw do
constraints: {
url_name: %r{[^/]+},
} do
resources :posts, controller: "/domain/fa/posts"
resources :posts, controller: "/domain/fa/posts", only: %i[index]
end
resources :posts, param: :fa_id, only: [:show] do
resources :posts, param: :fa_id, only: %i[show] do
post :scan_post, on: :member
get :favorites, on: :member
end
@@ -37,30 +37,31 @@ Rails.application.routes.draw do
end
namespace :inkbunny, path: "ib" do
resources :users, param: :name, only: [:show] do
resources :posts, controller: "/domain/inkbunny/posts", only: %i[index]
end
resources :posts, param: :ib_post_id, only: %i[show]
resources :users, param: :name, only: [:show]
end
end
resources :blobs, only: [], slug: :sha256 do
get :contents, on: :member
end
namespace :domain do
namespace :fa do
resources :users, param: :url_name, only: [] do
end
resources :posts, param: :fa_id, only: [:index] do
end
end
end
resources :blobs, controller: :blob_entries, only: [:show], param: :sha256
resources :indexed_posts, only: [:index], path: "posts"
resources :blobs, only: [:show], slug: :sha256
get "us/:script", to: "user_scripts#get", constraints: { script: /.*/ }
resources :global_states, path: "state" do
collection do
get "fa-cookies", to: "global_states#fa_cookies"
get "fa-cookies/edit", to: "global_states#edit_fa_cookies"
patch "fa-cookies", to: "global_states#update_fa_cookies"
get "ib-cookies", to: "global_states#ib_cookies"
get "ib-cookies/edit", to: "global_states#edit_ib_cookies"
patch "ib-cookies", to: "global_states#update_ib_cookies"
end
end
scope constraints: VpnOnlyRouteConstraint.new do
mount PgHero::Engine => "pghero"
mount GoodJob::Engine => "jobs"
@@ -88,16 +89,4 @@ Rails.application.routes.draw do
}
end
end
resources :global_states, path: "state" do
collection do
get "fa-cookies", to: "global_states#fa_cookies"
get "fa-cookies/edit", to: "global_states#edit_fa_cookies"
patch "fa-cookies", to: "global_states#update_fa_cookies"
get "ib-cookies", to: "global_states#ib_cookies"
get "ib-cookies/edit", to: "global_states#edit_ib_cookies"
patch "ib-cookies", to: "global_states#update_ib_cookies"
end
end
end

View File

@@ -15,11 +15,11 @@ namespace :blob_file do
num_migrated = 0
num_processed = 0
start_time = Time.now
BlobEntryP.in_batches(
BlobEntry.in_batches(
of: batch_size,
start: HexUtil.hex2bin(start_at),
order: :asc,
use_ranges: true
use_ranges: true,
) do |batch|
batch_migrated = insert_blob_entries_batch(batch)
num_migrated += batch_migrated
@@ -29,7 +29,7 @@ namespace :blob_file do
"[migrated: #{ActiveSupport::NumberHelper.number_to_delimited(num_migrated).rjust(8)}]",
"[processed: #{ActiveSupport::NumberHelper.number_to_delimited(num_processed).rjust(8)}]",
"[rate: #{rate.round(1).to_s.rjust(5)}/second]",
"[last: '#{HexUtil.bin2hex(batch.last.sha256)}']"
"[last: '#{HexUtil.bin2hex(batch.last.sha256)}']",
].join(" ")
start_time = Time.now
end
@@ -45,7 +45,7 @@ namespace :blob_file do
missing_sha256s = blob_entry_sha256s - blob_file_sha256s
BlobFile.transaction do
BlobEntryP
BlobEntry
.where(sha256: missing_sha256s)
.each do |blob_entry|
blob_file = BlobFile.initialize_from_blob_entry(blob_entry)

View File

@@ -0,0 +1,120 @@
require "rails_helper"
RSpec.describe BlobEntriesController, type: :controller do
let(:blob_entry) { create(:blob_entry) }
let(:sha256_hex) { HexUtil.bin2hex(blob_entry.sha256) }
describe "GET #show" do
context "with plain text content" do
let(:blob_entry) do
create(:blob_entry, content_type: "text/plain", content: "Hello World")
end
it "renders content as plain text" do
get :show, params: { sha256: sha256_hex }
expect(response).to have_http_status(:success)
expect(response.body).to eq("Hello World")
expect(response.content_type).to eq("text/plain; charset=utf-8")
end
end
context "with HTML content" do
let(:blob_entry) do
create(:blob_entry, content_type: "text/html", content: "<p>Hello</p>")
end
it "renders content as HTML" do
get :show, params: { sha256: sha256_hex }
expect(response).to have_http_status(:success)
expect(response.body).to include("<p>Hello</p>")
expect(response.content_type).to eq("text/html; charset=utf-8")
end
end
context "with JSON content" do
let(:blob_entry) do
create(
:blob_entry,
content_type: "application/json",
content: '{"hello":"world"}',
)
end
it "renders pretty JSON in HTML" do
get :show, params: { sha256: sha256_hex }
expect(response).to have_http_status(:success)
expect(response.body).to include("{\n \"hello\": \"world\"\n}")
expect(response.content_type).to eq("text/html; charset=utf-8")
end
end
context "with image content" do
let(:blob_entry) do
create(
:blob_entry,
content_type: "image/jpeg",
content: "fake-image-data",
)
end
it "sends data with correct headers" do
get :show, params: { sha256: sha256_hex }
expect(response).to have_http_status(:success)
expect(response.content_type).to eq("image/jpeg")
expect(response.headers["Content-Disposition"]).to include("inline")
expect(response.body).to eq("fake-image-data")
end
context "with thumbnail request" do
before do
allow(Vips::Image).to receive(:thumbnail_buffer).and_return(
double(jpegsave_buffer: "thumbnail-data"),
)
end
it "generates thumbnail for valid size" do
get :show, params: { sha256: sha256_hex, thumb: "tiny" }
expect(response).to have_http_status(:success)
expect(response.content_type).to eq("image/jpg")
expect(response.headers["Content-Disposition"]).to include("inline")
expect(response.body).to eq("thumbnail-data")
end
it "raises error for invalid thumb size" do
expect {
get :show, params: { sha256: sha256_hex, thumb: "invalid" }
}.to raise_error(/invalid thumb/)
end
end
end
context "with unknown content type" do
let(:blob_entry) do
create(:blob_entry, content_type: "application/unknown")
end
it "renders error message" do
get :show, params: { sha256: sha256_hex }
expect(response).to have_http_status(:success)
expect(response.body).to include("no renderer for application/unknown")
end
end
it "sets cache headers" do
get :show, params: { sha256: sha256_hex }
expect(response.headers["Expires"]).to be_present
expect(response.headers["Cache-Control"]).to include("public")
end
it "handles ETags" do
get :show, params: { sha256: sha256_hex }
etag = response.headers["ETag"]
expect(etag).to be_present
# Subsequent request with same ETag should return not modified
request.headers["If-None-Match"] = etag
get :show, params: { sha256: sha256_hex }
expect(response).to have_http_status(:not_modified)
end
end
end

View File

@@ -0,0 +1,69 @@
require "rails_helper"
RSpec.describe Domain::Fa::PostsController, type: :controller do
render_views
include Domain::Fa::UsersHelper
describe "GET #index" do
let(:user) { create(:domain_fa_user, :with_avatar) }
let!(:posts) do
[
create(:domain_fa_post, creator: user, title: "Test Post 1"),
create(:domain_fa_post, creator: user, title: "Test Post 2"),
]
end
before do
allow_any_instance_of(Domain::Fa::PostPolicy).to receive(
:view_file?,
).and_return(true)
end
it "renders index template with posts" do
get :index, params: { user_url_name: user.url_name }
expect(response).to be_successful
expect(response).to render_template(:index)
expect(response.body).to include(user.name)
expect(response.body).to include("posts")
posts.each do |post|
expect(response.body).to include(post.title)
if post.file
expect(response.body).to include(
blob_path(
HexUtil.bin2hex(post.file.response_sha256),
format: "jpg",
thumb: "small",
),
)
end
end
end
context "when viewing a specific user's posts" do
it "shows the user's posts with avatar" do
get :index, params: { user_url_name: user.url_name }
expect(response).to be_successful
expect(response.body).to include(user.name)
expect(response.body).to include("posts")
expect(response.body).to include(
blob_path(
HexUtil.bin2hex(user.avatar.file_sha256),
format: "jpg",
thumb: "64-avatar",
),
)
posts.each { |post| expect(response.body).to include(post.title) }
end
end
end
describe "GET #show" do
let(:user) { create(:domain_fa_user) }
let(:post) { create(:domain_fa_post, creator: user) }
it "returns http success" do
get :show, params: { user_url_name: user.url_name, fa_id: post.fa_id }
expect(response).to be_successful
end
end
end

View File

@@ -1,31 +1,76 @@
require "rails_helper"
RSpec.describe Domain::Inkbunny::PostsController, type: :controller do
render_views
describe "GET #show" do
let(:post) { create(:domain_inkbunny_post) }
let(:file) { create(:domain_inkbunny_file, post: post) }
context "when user is not logged in" do
it "shows post details but not file content or scraper metadata" do
file # Create the file
get :show, params: { ib_post_id: post.ib_post_id }
it "returns http success" do
get :show, params: { ib_post_id: post.ib_post_id }
expect(response).to be_successful
end
end
describe "GET #index" do
let(:user) { create(:domain_inkbunny_user, name: "Test User") }
let!(:posts) do
[
create(:domain_inkbunny_post, creator: user, title: "Test Post 1"),
create(:domain_inkbunny_post, creator: user, title: "Test Post 2"),
]
end
let!(:files) do
posts.map { |post| create(:domain_inkbunny_file, post: post) }
end
context "when user is not an admin" do
it "renders index template with posts but without thumbnails" do
get :index, params: { user_name: user.name }
expect(response).to be_successful
expect(response).to render_template(:show)
expect(response).to render_template(:index)
expect(response.body).to include(posts[0].title)
expect(response.body).to include(posts[1].title)
expect(response.body).to include(user.name)
files.each { |file| expect(response.body).not_to include("<img") }
end
end
context "when user is an admin" do
let(:user) { create(:user, :admin) }
let(:admin) { create(:user, :admin) }
before { sign_in admin }
before do
sign_in user
file # Create the file
it "renders index template with posts and thumbnails" do
get :index, params: { user_name: user.name }
expect(response).to be_successful
expect(response).to render_template(:index)
expect(response.body).to include(posts[0].title)
expect(response.body).to include(posts[1].title)
expect(response.body).to include(user.name)
files.each do |file|
expect(response.body).to include(
blob_path(
HexUtil.bin2hex(file.blob_entry_sha256),
format: "jpg",
thumb: "small",
),
)
end
end
it "shows file content and scraper metadata" do
get :show, params: { ib_post_id: post.ib_post_id }
it "requires a user_name parameter" do
expect { get :index }.to raise_error(
ActionController::UrlGenerationError,
)
end
end
context "when viewing a specific user's posts" do
it "shows the user's posts" do
get :index, params: { user_name: user.name }
expect(response).to be_successful
expect(response).to render_template(:show)
expect(response.body).to include("#{user.name}")
posts.each { |post| expect(response.body).to include(post.title) }
end
end
end

View File

@@ -0,0 +1,71 @@
require "rails_helper"
RSpec.describe IndexedPostsController, type: :controller do
render_views
let(:admin_user) { create(:user, role: :admin) }
describe "GET #index" do
context "when user is not signed in" do
it "redirects to the sign in page" do
get :index
expect(response).to redirect_to(new_user_session_path)
end
end
context "when admin user is signed in" do
before { sign_in admin_user }
let(:file) { create(:http_log_entry) }
let!(:fa_post) { create(:domain_fa_post, file: file, title: "Test Post") }
context "with gallery view" do
it "renders gallery view with thumbnails" do
get :index, params: { view: "gallery" }
expect(response).to be_successful
expect(response).to render_template(:index)
expect(response.body).to include(fa_post.title)
expect(response.body).to include(
blob_path(
HexUtil.bin2hex(file.response_sha256),
format: "jpg",
thumb: "small",
),
)
end
end
context "with table view" do
it "renders table view with thumbnails" do
get :index, params: { view: "table" }
expect(response).to be_successful
expect(response).to render_template(:index)
expect(response.body).to include(fa_post.title)
expect(response.body).to include(
blob_path(
HexUtil.bin2hex(file.response_sha256),
format: "jpg",
thumb: "tiny",
),
)
end
end
context "when post has no file" do
let!(:fa_post) do
create(:domain_fa_post, file: nil, title: "Test Post")
end
it "shows appropriate message in gallery view" do
get :index, params: { view: "gallery" }
expect(response.body).to include("No file available")
end
it "shows appropriate message in table view" do
get :index, params: { view: "table" }
expect(response.body).to include("(none)")
end
end
end
end
end

View File

@@ -1,5 +1,5 @@
FactoryBot.define do
factory :blob_entry_p do
factory :blob_entry do
transient { content { "test content" } }
content_type { "text/plain" }

View File

@@ -0,0 +1,9 @@
FactoryBot.define do
factory :domain_fa_user_avatar, class: "Domain::Fa::UserAvatar" do
association :user, factory: :domain_fa_user
association :file, factory: :http_log_entry
state { :ok }
state_detail { {} }
file_url_str { "https://example.com/avatar.jpg" }
end
end

View File

@@ -2,5 +2,14 @@ FactoryBot.define do
factory :domain_fa_user, class: "Domain::Fa::User" do
sequence(:url_name) { |n| "user#{n}" }
sequence(:name) { |n| "User #{n}" }
state { :ok }
state_detail { {} }
log_entry_detail { {} }
trait :with_avatar do
after(:create) do |user|
create(:domain_fa_user_avatar, user: user, file: create(:blob_entry))
end
end
end
end

View File

@@ -13,7 +13,7 @@ FactoryBot.define do
performed_by { "direct" }
# Create associated records
association :response, factory: :blob_entry_p
association :response, factory: :blob_entry
association :request_headers, factory: :http_log_entry_header
association :response_headers, factory: :http_log_entry_header

View File

@@ -0,0 +1,13 @@
FactoryBot.define do
factory :indexed_post do
association :postable, factory: :domain_fa_post
trait :with_e621_post do
association :postable, factory: :domain_e621_post
end
trait :with_inkbunny_post do
association :postable, factory: :domain_inkbunny_post
end
end
end

View File

@@ -0,0 +1,53 @@
require "rails_helper"
RSpec.describe Domain::Fa::UsersHelper, type: :helper do
describe "#avatar_url" do
it "returns the correct blob path with thumb parameter" do
sha256 = SecureRandom.bytes(32)
hex = HexUtil.bin2hex(sha256)
expect(helper.avatar_url(sha256)).to eq(
blob_path(hex, format: "jpg", thumb: "32-avatar"),
)
expect(helper.avatar_url(sha256, thumb: "64-avatar")).to eq(
blob_path(hex, format: "jpg", thumb: "64-avatar"),
)
end
end
describe "#fa_user_avatar_path" do
context "when user has an avatar" do
let(:user) { create(:domain_fa_user, :with_avatar) }
it "returns the correct blob path" do
expect(helper.fa_user_avatar_path(user)).to eq(
blob_path(
HexUtil.bin2hex(user.avatar.file.sha256),
format: "jpg",
thumb: nil,
),
)
end
it "respects the thumb parameter" do
expect(helper.fa_user_avatar_path(user, thumb: "64-avatar")).to eq(
blob_path(
HexUtil.bin2hex(user.avatar.file.sha256),
format: "jpg",
thumb: "64-avatar",
),
)
end
end
context "when user has no avatar" do
let(:user) { create(:domain_fa_user) }
it "returns the default avatar image path" do
expect(helper.fa_user_avatar_path(user)).to match(
%r{/assets/user-circle-[a-f0-9]+\.svg},
)
end
end
end
end

View File

@@ -101,7 +101,7 @@ describe Domain::Fa::Job::UserAvatarJob do
avatar.state = :ok
avatar.file =
create(
:blob_entry_p,
:blob_entry,
content: avatar_fixture_file_2,
content_type: "image/gif",
)
@@ -149,7 +149,7 @@ describe Domain::Fa::Job::UserAvatarJob do
avatar.state = :ok
avatar.file =
create(
:blob_entry_p,
:blob_entry,
content: avatar_fixture_file,
content_type: "image/gif",
)

View File

@@ -18,7 +18,7 @@ RSpec.describe HttpLogEntry, type: :model do
end
describe "associations" do
it { should belong_to(:response).class_name("::BlobEntryP") }
it { should belong_to(:response).class_name("::BlobEntry") }
it { should belong_to(:request_headers).class_name("::HttpLogEntryHeader") }
it do
should belong_to(:response_headers).class_name("::HttpLogEntryHeader")
@@ -143,7 +143,7 @@ RSpec.describe HttpLogEntry, type: :model do
context "when response association is loaded" do
it "returns size from response object" do
test_content = "test content"
entry.response = build(:blob_entry_p, content: test_content)
entry.response = build(:blob_entry, content: test_content)
expect(entry.response_size).to eq(test_content.bytesize)
end
end

View File

@@ -87,7 +87,7 @@ class SpecUtil
end
def self.build_blob_entry(content_type: "text/plain", contents: nil)
BlobEntryP.find_or_build(
BlobEntry.find_or_build(
content_type: content_type,
contents: contents || random_string(1024),
)

View File

@@ -1,6 +1,6 @@
require "test_helper"
class BlobEntryPTest < ActiveSupport::TestCase
class BlobEntryTest < ActiveSupport::TestCase
test "building a blob works" do
blob = TestUtil.build_blob_entry
assert blob.valid?
@@ -20,17 +20,17 @@ class BlobEntryPTest < ActiveSupport::TestCase
assert_raises(ActiveRecord::ReadOnlyRecord) { model.destroy }
end
test "model dual-writes a BlobEntryP model" do
test "model dual-writes a BlobEntry model" do
model = TestUtil.build_blob_entry
model.save!
model_p = BlobEntryP.find_by(sha256: model.sha256)
model_p = BlobEntry.find_by(sha256: model.sha256)
assert_same_blob_entry model, model_p
end
test "ensure works for creating a blob entry" do
model = TestUtil.build_blob_entry
model.save!
model_p = BlobEntryP.ensure(model.sha256)
model_p = BlobEntry.ensure(model.sha256)
assert_same_blob_entry model, model_p
end

View File

@@ -53,14 +53,14 @@ class BlobFileTest < ActiveSupport::TestCase
["abcd1234", [2], %w[ab abcd1234]],
["abcd1234", [4, 2], %w[abcd 12 abcd1234]],
["abcd1234", [4, 2, 2], %w[abcd 12 34 abcd1234]],
["abcd1234", [2, 2, 1], %w[ab cd 1 abcd1234]]
["abcd1234", [2, 2, 1], %w[ab cd 1 abcd1234]],
]
test_cases.each do |sha256_hex, pattern, expected|
assert_equal BlobFile.path_segments(pattern, sha256_hex), expected
end
end
test "from an initialized BlobEntryP" do
test "from an initialized BlobEntry" do
blob_entry = TestUtil.build_blob_entry
blob_file = BlobFile.initialize_from_blob_entry(blob_entry)
assert blob_file.save

View File

@@ -31,7 +31,7 @@ module TestUtil
end
def self.build_blob_entry(content_type: "text/plain", contents: nil)
BlobEntryP.find_or_build(
BlobEntry.find_or_build(
content_type: content_type,
contents: contents || random_string(1024),
)