upgrade to rails 7.2, ui improvements
This commit is contained in:
Dylan Knutson
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
@@ -24,5 +24,5 @@
|
||||
"comments": "off",
|
||||
"strings": "on"
|
||||
},
|
||||
"sqliteViewer.maxFileSize": 1024
|
||||
"sqliteViewer.maxFileSize": 4000
|
||||
}
|
||||
9
Gemfile
9
Gemfile
@@ -5,7 +5,7 @@ ruby "3.2.6"
|
||||
# ruby "3.0.3"
|
||||
|
||||
# Bundle edge Rails instead: gem "rails", github: "rails/rails", branch: "main"
|
||||
gem "rails", "~> 7.0.4", ">= 7.0.4.2"
|
||||
gem "rails", "~> 7.2"
|
||||
|
||||
# The original asset pipeline for Rails [https://github.com/rails/sprockets-rails]
|
||||
gem "sprockets-rails"
|
||||
@@ -55,7 +55,7 @@ gem "bootsnap", require: false
|
||||
|
||||
group :development, :test, :staging do
|
||||
# See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
|
||||
gem "debug", platforms: %i[mri mingw x64_mingw]
|
||||
gem "debug", "~> 1.10", platforms: %i[mri mingw x64_mingw]
|
||||
end
|
||||
|
||||
group :development, :staging do
|
||||
@@ -81,7 +81,7 @@ end
|
||||
group :test do
|
||||
# Use system testing [https://guides.rubyonrails.org/testing.html#system-testing]
|
||||
gem "capybara"
|
||||
gem "rspec-rails"
|
||||
gem "rspec-rails", "~> 7.0"
|
||||
gem "selenium-webdriver"
|
||||
gem "webdrivers"
|
||||
end
|
||||
@@ -94,7 +94,6 @@ gem "rb-bsdiff", path: "/gems/rb-bsdiff"
|
||||
|
||||
gem "addressable"
|
||||
gem "colorize"
|
||||
gem "composite_primary_keys", "~> 14.0"
|
||||
gem "concurrent-ruby-edge", require: "concurrent-edge"
|
||||
gem "concurrent-ruby-ext", require: "concurrent"
|
||||
gem "curb"
|
||||
@@ -121,7 +120,7 @@ gem "neighbor"
|
||||
gem "progressbar"
|
||||
|
||||
group :production, :staging do
|
||||
gem "rails_semantic_logger"
|
||||
gem "rails_semantic_logger", "~> 4.17"
|
||||
end
|
||||
|
||||
group :production do
|
||||
|
||||
250
Gemfile.lock
250
Gemfile.lock
@@ -30,73 +30,82 @@ PATH
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
actioncable (7.0.4.2)
|
||||
actionpack (= 7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
actioncable (7.2.2.1)
|
||||
actionpack (= 7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
nio4r (~> 2.0)
|
||||
websocket-driver (>= 0.6.1)
|
||||
actionmailbox (7.0.4.2)
|
||||
actionpack (= 7.0.4.2)
|
||||
activejob (= 7.0.4.2)
|
||||
activerecord (= 7.0.4.2)
|
||||
activestorage (= 7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
mail (>= 2.7.1)
|
||||
net-imap
|
||||
net-pop
|
||||
net-smtp
|
||||
actionmailer (7.0.4.2)
|
||||
actionpack (= 7.0.4.2)
|
||||
actionview (= 7.0.4.2)
|
||||
activejob (= 7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
mail (~> 2.5, >= 2.5.4)
|
||||
net-imap
|
||||
net-pop
|
||||
net-smtp
|
||||
rails-dom-testing (~> 2.0)
|
||||
actionpack (7.0.4.2)
|
||||
actionview (= 7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
rack (~> 2.0, >= 2.2.0)
|
||||
zeitwerk (~> 2.6)
|
||||
actionmailbox (7.2.2.1)
|
||||
actionpack (= 7.2.2.1)
|
||||
activejob (= 7.2.2.1)
|
||||
activerecord (= 7.2.2.1)
|
||||
activestorage (= 7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
mail (>= 2.8.0)
|
||||
actionmailer (7.2.2.1)
|
||||
actionpack (= 7.2.2.1)
|
||||
actionview (= 7.2.2.1)
|
||||
activejob (= 7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
mail (>= 2.8.0)
|
||||
rails-dom-testing (~> 2.2)
|
||||
actionpack (7.2.2.1)
|
||||
actionview (= 7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
nokogiri (>= 1.8.5)
|
||||
racc
|
||||
rack (>= 2.2.4, < 3.2)
|
||||
rack-session (>= 1.0.1)
|
||||
rack-test (>= 0.6.3)
|
||||
rails-dom-testing (~> 2.0)
|
||||
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
||||
actiontext (7.0.4.2)
|
||||
actionpack (= 7.0.4.2)
|
||||
activerecord (= 7.0.4.2)
|
||||
activestorage (= 7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
rails-dom-testing (~> 2.2)
|
||||
rails-html-sanitizer (~> 1.6)
|
||||
useragent (~> 0.16)
|
||||
actiontext (7.2.2.1)
|
||||
actionpack (= 7.2.2.1)
|
||||
activerecord (= 7.2.2.1)
|
||||
activestorage (= 7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
globalid (>= 0.6.0)
|
||||
nokogiri (>= 1.8.5)
|
||||
actionview (7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
actionview (7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
builder (~> 3.1)
|
||||
erubi (~> 1.4)
|
||||
rails-dom-testing (~> 2.0)
|
||||
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
||||
activejob (7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
erubi (~> 1.11)
|
||||
rails-dom-testing (~> 2.2)
|
||||
rails-html-sanitizer (~> 1.6)
|
||||
activejob (7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
globalid (>= 0.3.6)
|
||||
activemodel (7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
activerecord (7.0.4.2)
|
||||
activemodel (= 7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
activestorage (7.0.4.2)
|
||||
actionpack (= 7.0.4.2)
|
||||
activejob (= 7.0.4.2)
|
||||
activerecord (= 7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
activemodel (7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
activerecord (7.2.2.1)
|
||||
activemodel (= 7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
timeout (>= 0.4.0)
|
||||
activestorage (7.2.2.1)
|
||||
actionpack (= 7.2.2.1)
|
||||
activejob (= 7.2.2.1)
|
||||
activerecord (= 7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
marcel (~> 1.0)
|
||||
mini_mime (>= 1.1.0)
|
||||
activesupport (7.0.4.2)
|
||||
concurrent-ruby (~> 1.0, >= 1.0.2)
|
||||
activesupport (7.2.2.1)
|
||||
base64
|
||||
benchmark (>= 0.3)
|
||||
bigdecimal
|
||||
concurrent-ruby (~> 1.0, >= 1.3.1)
|
||||
connection_pool (>= 2.2.5)
|
||||
drb
|
||||
i18n (>= 1.6, < 2)
|
||||
logger (>= 1.4.2)
|
||||
minitest (>= 5.1)
|
||||
tzinfo (~> 2.0)
|
||||
securerandom (>= 0.3)
|
||||
tzinfo (~> 2.0, >= 2.0.5)
|
||||
addressable (2.8.1)
|
||||
public_suffix (>= 2.0.2, < 6.0)
|
||||
base64 (0.2.0)
|
||||
benchmark (0.4.0)
|
||||
bigdecimal (3.1.8)
|
||||
bindex (0.8.1)
|
||||
binding_of_caller (1.0.0)
|
||||
debug_inspector (>= 0.0.1)
|
||||
@@ -114,8 +123,6 @@ GEM
|
||||
xpath (~> 3.2)
|
||||
coderay (1.1.3)
|
||||
colorize (0.8.1)
|
||||
composite_primary_keys (14.0.6)
|
||||
activerecord (~> 7.0.2)
|
||||
concurrent-ruby (1.3.4)
|
||||
concurrent-ruby-edge (0.7.1)
|
||||
concurrent-ruby (~> 1.3)
|
||||
@@ -127,12 +134,12 @@ GEM
|
||||
railties (>= 6.0.0)
|
||||
curb (1.0.5)
|
||||
daemons (1.4.1)
|
||||
date (3.3.3)
|
||||
debug (1.7.1)
|
||||
irb (>= 1.5.0)
|
||||
reline (>= 0.3.1)
|
||||
date (3.4.1)
|
||||
debug (1.10.0)
|
||||
irb (~> 1.10)
|
||||
reline (>= 0.3.8)
|
||||
debug_inspector (1.1.0)
|
||||
diff-lcs (1.5.0)
|
||||
diff-lcs (1.5.1)
|
||||
diffy (3.4.2)
|
||||
discard (1.2.1)
|
||||
activerecord (>= 4.2, < 8)
|
||||
@@ -141,6 +148,7 @@ GEM
|
||||
numo-narray
|
||||
domain_name (0.5.20190701)
|
||||
unf (>= 0.0.5, < 1.0.0)
|
||||
drb (2.2.1)
|
||||
erubi (1.13.0)
|
||||
et-orbi (1.2.11)
|
||||
tzinfo
|
||||
@@ -168,8 +176,9 @@ GEM
|
||||
i18n (1.14.6)
|
||||
concurrent-ruby (~> 1.0)
|
||||
io-console (0.6.0)
|
||||
irb (1.6.2)
|
||||
reline (>= 0.3.0)
|
||||
irb (1.14.3)
|
||||
rdoc (>= 4.0.0)
|
||||
reline (>= 0.4.2)
|
||||
jbuilder (2.11.5)
|
||||
actionview (>= 5.0.0)
|
||||
activesupport (>= 5.0.0)
|
||||
@@ -190,15 +199,16 @@ GEM
|
||||
listen (3.9.0)
|
||||
rb-fsevent (~> 0.10, >= 0.10.3)
|
||||
rb-inotify (~> 0.9, >= 0.9.10)
|
||||
logger (1.6.4)
|
||||
loofah (2.23.1)
|
||||
crass (~> 1.0.2)
|
||||
nokogiri (>= 1.12.0)
|
||||
mail (2.8.0.1)
|
||||
mail (2.8.1)
|
||||
mini_mime (>= 0.1.1)
|
||||
net-imap
|
||||
net-pop
|
||||
net-smtp
|
||||
marcel (1.0.2)
|
||||
marcel (1.0.4)
|
||||
matrix (0.4.2)
|
||||
memory_profiler (1.0.1)
|
||||
method_source (1.1.0)
|
||||
@@ -208,14 +218,14 @@ GEM
|
||||
msgpack (1.6.0)
|
||||
neighbor (0.2.2)
|
||||
activerecord (>= 5.2)
|
||||
net-imap (0.3.4)
|
||||
net-imap (0.5.3)
|
||||
date
|
||||
net-protocol
|
||||
net-pop (0.1.2)
|
||||
net-protocol
|
||||
net-protocol (0.2.1)
|
||||
net-protocol (0.2.2)
|
||||
timeout
|
||||
net-smtp (0.3.3)
|
||||
net-smtp (0.5.0)
|
||||
net-protocol
|
||||
nio4r (2.7.4)
|
||||
nokogiri (1.17.2)
|
||||
@@ -236,6 +246,9 @@ GEM
|
||||
pry-stack_explorer (0.6.1)
|
||||
binding_of_caller (~> 1.0)
|
||||
pry (~> 0.13)
|
||||
psych (5.2.2)
|
||||
date
|
||||
stringio
|
||||
public_suffix (5.0.1)
|
||||
puma (5.6.5)
|
||||
nio4r (~> 2.0)
|
||||
@@ -248,22 +261,27 @@ GEM
|
||||
rack (>= 1.2.0)
|
||||
rack-proxy (0.7.6)
|
||||
rack
|
||||
rack-session (1.0.2)
|
||||
rack (< 3)
|
||||
rack-test (2.1.0)
|
||||
rack (>= 1.3)
|
||||
rails (7.0.4.2)
|
||||
actioncable (= 7.0.4.2)
|
||||
actionmailbox (= 7.0.4.2)
|
||||
actionmailer (= 7.0.4.2)
|
||||
actionpack (= 7.0.4.2)
|
||||
actiontext (= 7.0.4.2)
|
||||
actionview (= 7.0.4.2)
|
||||
activejob (= 7.0.4.2)
|
||||
activemodel (= 7.0.4.2)
|
||||
activerecord (= 7.0.4.2)
|
||||
activestorage (= 7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
rackup (1.0.1)
|
||||
rack (< 3)
|
||||
webrick
|
||||
rails (7.2.2.1)
|
||||
actioncable (= 7.2.2.1)
|
||||
actionmailbox (= 7.2.2.1)
|
||||
actionmailer (= 7.2.2.1)
|
||||
actionpack (= 7.2.2.1)
|
||||
actiontext (= 7.2.2.1)
|
||||
actionview (= 7.2.2.1)
|
||||
activejob (= 7.2.2.1)
|
||||
activemodel (= 7.2.2.1)
|
||||
activerecord (= 7.2.2.1)
|
||||
activestorage (= 7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
bundler (>= 1.15.0)
|
||||
railties (= 7.0.4.2)
|
||||
railties (= 7.2.2.1)
|
||||
rails-dom-testing (2.2.0)
|
||||
activesupport (>= 5.0.0)
|
||||
minitest
|
||||
@@ -271,22 +289,25 @@ GEM
|
||||
rails-html-sanitizer (1.6.2)
|
||||
loofah (~> 2.21)
|
||||
nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
|
||||
rails_semantic_logger (4.12.0)
|
||||
rails_semantic_logger (4.17.0)
|
||||
rack
|
||||
railties (>= 5.1)
|
||||
semantic_logger (~> 4.13)
|
||||
railties (7.0.4.2)
|
||||
actionpack (= 7.0.4.2)
|
||||
activesupport (= 7.0.4.2)
|
||||
method_source
|
||||
semantic_logger (~> 4.16)
|
||||
railties (7.2.2.1)
|
||||
actionpack (= 7.2.2.1)
|
||||
activesupport (= 7.2.2.1)
|
||||
irb (~> 1.13)
|
||||
rackup (>= 1.0.0)
|
||||
rake (>= 12.2)
|
||||
thor (~> 1.0)
|
||||
zeitwerk (~> 2.5)
|
||||
thor (~> 1.0, >= 1.2.2)
|
||||
zeitwerk (~> 2.6)
|
||||
rainbow (3.1.1)
|
||||
rake (13.2.1)
|
||||
rb-fsevent (0.11.2)
|
||||
rb-inotify (0.11.1)
|
||||
ffi (~> 1.0)
|
||||
rdoc (6.10.0)
|
||||
psych (>= 4.0.0)
|
||||
react_on_rails (13.3.3)
|
||||
addressable
|
||||
connection_pool
|
||||
@@ -294,28 +315,28 @@ GEM
|
||||
rails (>= 5.2)
|
||||
rainbow (~> 3.0)
|
||||
regexp_parser (2.6.2)
|
||||
reline (0.3.2)
|
||||
reline (0.6.0)
|
||||
io-console (~> 0.5)
|
||||
rexml (3.2.5)
|
||||
rice (4.0.4)
|
||||
ripcord (2.0.0)
|
||||
rspec-core (3.12.1)
|
||||
rspec-support (~> 3.12.0)
|
||||
rspec-expectations (3.12.2)
|
||||
rspec-core (3.13.2)
|
||||
rspec-support (~> 3.13.0)
|
||||
rspec-expectations (3.13.3)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.12.0)
|
||||
rspec-mocks (3.12.3)
|
||||
rspec-support (~> 3.13.0)
|
||||
rspec-mocks (3.13.2)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.12.0)
|
||||
rspec-rails (6.0.1)
|
||||
actionpack (>= 6.1)
|
||||
activesupport (>= 6.1)
|
||||
railties (>= 6.1)
|
||||
rspec-core (~> 3.11)
|
||||
rspec-expectations (~> 3.11)
|
||||
rspec-mocks (~> 3.11)
|
||||
rspec-support (~> 3.11)
|
||||
rspec-support (3.12.0)
|
||||
rspec-support (~> 3.13.0)
|
||||
rspec-rails (7.1.0)
|
||||
actionpack (>= 7.0)
|
||||
activesupport (>= 7.0)
|
||||
railties (>= 7.0)
|
||||
rspec-core (~> 3.13)
|
||||
rspec-expectations (~> 3.13)
|
||||
rspec-mocks (~> 3.13)
|
||||
rspec-support (~> 3.13)
|
||||
rspec-support (3.13.2)
|
||||
ruby-prof (1.4.5)
|
||||
ruby-prof-speedscope (0.3.0)
|
||||
ruby-prof (~> 1.0)
|
||||
@@ -327,11 +348,12 @@ GEM
|
||||
crass (~> 1.0.2)
|
||||
nokogiri (>= 1.12.0)
|
||||
sd_notify (0.1.1)
|
||||
securerandom (0.4.1)
|
||||
selenium-webdriver (4.8.0)
|
||||
rexml (~> 3.2, >= 3.2.5)
|
||||
rubyzip (>= 1.2.2, < 3.0)
|
||||
websocket (~> 1.0)
|
||||
semantic_logger (4.13.0)
|
||||
semantic_logger (4.16.1)
|
||||
concurrent-ruby (~> 1.0)
|
||||
semantic_range (3.0.0)
|
||||
shakapacker (6.6.0)
|
||||
@@ -351,6 +373,7 @@ GEM
|
||||
stackprof (0.2.24)
|
||||
stimulus-rails (1.2.1)
|
||||
railties (>= 6.0.0)
|
||||
stringio (3.1.2)
|
||||
syntax_tree (6.2.0)
|
||||
prettier_print (>= 1.2.0)
|
||||
table_print (1.5.7)
|
||||
@@ -359,7 +382,7 @@ GEM
|
||||
tailwindcss-ruby
|
||||
tailwindcss-ruby (3.4.17)
|
||||
thor (1.3.2)
|
||||
timeout (0.3.1)
|
||||
timeout (0.4.3)
|
||||
turbo-rails (1.3.3)
|
||||
actionpack (>= 6.0.0)
|
||||
activejob (>= 6.0.0)
|
||||
@@ -369,6 +392,7 @@ GEM
|
||||
unf (0.1.4)
|
||||
unf_ext
|
||||
unf_ext (0.0.8.2)
|
||||
useragent (0.16.11)
|
||||
web-console (4.2.0)
|
||||
actionview (>= 6.0.0)
|
||||
activemodel (>= 6.0.0)
|
||||
@@ -378,6 +402,7 @@ GEM
|
||||
nokogiri (~> 1.6)
|
||||
rubyzip (>= 1.3.0)
|
||||
selenium-webdriver (~> 4.0)
|
||||
webrick (1.9.1)
|
||||
websocket (1.2.9)
|
||||
websocket-driver (0.7.6)
|
||||
websocket-extensions (>= 0.1.0)
|
||||
@@ -395,13 +420,12 @@ DEPENDENCIES
|
||||
bootsnap
|
||||
capybara
|
||||
colorize
|
||||
composite_primary_keys (~> 14.0)
|
||||
concurrent-ruby-edge
|
||||
concurrent-ruby-ext
|
||||
cssbundling-rails (~> 1.4)
|
||||
curb
|
||||
daemons
|
||||
debug
|
||||
debug (~> 1.10)
|
||||
diffy
|
||||
discard
|
||||
disco
|
||||
@@ -424,13 +448,13 @@ DEPENDENCIES
|
||||
puma (~> 5.0)
|
||||
rack-cors
|
||||
rack-mini-profiler
|
||||
rails (~> 7.0.4, >= 7.0.4.2)
|
||||
rails (~> 7.2)
|
||||
rails_live_reload!
|
||||
rails_semantic_logger
|
||||
rails_semantic_logger (~> 4.17)
|
||||
rb-bsdiff!
|
||||
react_on_rails
|
||||
ripcord
|
||||
rspec-rails
|
||||
rspec-rails (~> 7.0)
|
||||
ruby-prof
|
||||
ruby-prof-speedscope
|
||||
ruby-vips
|
||||
|
||||
@@ -1,13 +1,8 @@
|
||||
class ApplicationController < ActionController::Base
|
||||
before_action :validate_api_token
|
||||
before_action do
|
||||
if Rails.env.development? || Rails.env.staging?
|
||||
@site_title = "TheTitle"
|
||||
@site_subtitle = "Placeholder Tagline Here"
|
||||
else
|
||||
@site_title = "ReFurrer"
|
||||
@site_subtitle = "Furry Swiss Army Knife"
|
||||
end
|
||||
@site_title = "ReFurrer"
|
||||
@site_subtitle = "Furry Swiss Army Knife"
|
||||
end
|
||||
before_action do
|
||||
if Rails.env.development? || Rails.env.staging?
|
||||
@@ -18,7 +13,6 @@ class ApplicationController < ActionController::Base
|
||||
API_TOKENS = {
|
||||
"a4eb03ac-b33c-439c-9b51-a834d1c5cf48" => "dymk",
|
||||
"56cc81fe-8c00-4436-8981-4580eab00e66" => "taargus",
|
||||
"a36f0d68-5262-4b62-9e2d-dfe648d70f35" => "vilk",
|
||||
"9c38727f-f11d-41de-b775-0effd86d520c" => "xjal",
|
||||
"e38c568f-a24d-4f26-87f0-dfcd898a359d" => "fyacin",
|
||||
"41fa1144-d4cd-11ed-afa1-0242ac120002" => "soft_fox_lad",
|
||||
|
||||
@@ -5,6 +5,16 @@ module LogEntriesHelper
|
||||
is_flash_content_type?(content_type)
|
||||
end
|
||||
|
||||
def path_iterative_parts(uri_path)
|
||||
path_parts = uri_path.split("/")
|
||||
(1...path_parts.length).map do |i|
|
||||
[
|
||||
path_parts[i],
|
||||
path_parts[0..i].join("/") + (i == path_parts.length - 1 ? "" : "/")
|
||||
]
|
||||
end
|
||||
end
|
||||
|
||||
def ext_for_content_type(content_type)
|
||||
case content_type
|
||||
when "image/jpeg"
|
||||
|
||||
@@ -4,7 +4,7 @@ class ApplicationJob < ActiveJob::Base
|
||||
|
||||
retry_on(
|
||||
StandardError,
|
||||
wait: :exponentially_longer,
|
||||
wait: :polynomially_longer,
|
||||
attempts: Float::INFINITY
|
||||
) do |job, exception|
|
||||
job.logger.error(
|
||||
|
||||
@@ -186,20 +186,12 @@ class Scraper::JobBase < ApplicationJob
|
||||
end
|
||||
|
||||
around_perform do |job, block|
|
||||
error = nil
|
||||
start = Time.now
|
||||
begin
|
||||
block.call
|
||||
ensure
|
||||
duration_ms = (Time.now - start) * 1000
|
||||
end
|
||||
block.call
|
||||
rescue Net::ReadTimeout, Errno::ECONNREFUSED => e
|
||||
logger.error "#{e.class.name} - sleep for a bit"
|
||||
sleep rand(2.0..7.0)
|
||||
error = e
|
||||
raise e
|
||||
rescue => e
|
||||
error = e
|
||||
raise e
|
||||
end
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ class Domain::Fa::PostEnqueuer
|
||||
.where("id >= ?", start_at)
|
||||
.where("file_id is null")
|
||||
.where(state: "ok")
|
||||
.pluck_each(:id, :fa_id, :file_url_str) { |p| e << p }
|
||||
.find_each { |p| e << [p.id, p.fa_id, p.file_url_str] }
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -287,7 +287,7 @@ class Domain::Fa::SqliteExporter
|
||||
start_time = Time.now
|
||||
models_in_measure = 0
|
||||
|
||||
relation.in_batches(of: batch_size) do |batch|
|
||||
relation.in_batches(of: batch_size, use_ranges: true) do |batch|
|
||||
batch = batch.pluck(*cols.map(&:first).map(&:to_sym)).to_a
|
||||
yield batch
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class Scraper::ClientFactory
|
||||
end
|
||||
|
||||
def self.get_fa_http_client
|
||||
if Rails.env.test? || Rails.env.development?
|
||||
if Rails.env.test?
|
||||
@http_client_mock || raise("no http client mock set")
|
||||
else
|
||||
_http_client_impl(:fa, Scraper::FaHttpClientConfig)
|
||||
|
||||
@@ -7,7 +7,8 @@ class Scraper::FaHttpClientConfig < Scraper::HttpClientConfig
|
||||
end
|
||||
|
||||
def ratelimit
|
||||
[["d.furaffinity.net", :none], ["*.facdn.net", :none], ["*", 0.25]]
|
||||
# number represents minimum delay in seconds between requests to the same domain
|
||||
[["d.furaffinity.net", :none], ["*.facdn.net", :none], ["*", 1]]
|
||||
end
|
||||
|
||||
def allowed_domains
|
||||
|
||||
@@ -21,7 +21,7 @@ class BlobFile < ReduxApplicationRecord
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
)
|
||||
|
||||
enum version: { v1: 1 }
|
||||
enum :version, { v1: 1 }
|
||||
after_initialize { self.version ||= :v1 }
|
||||
|
||||
validates_presence_of(:sha256, :content_type, :size_bytes)
|
||||
|
||||
@@ -7,8 +7,8 @@ class Domain::E621::Post < ReduxApplicationRecord
|
||||
default_scope -> { kept }
|
||||
|
||||
# see state_detail for scan_error/file_error
|
||||
enum state: %i[ok scan_error, file_error]
|
||||
enum rating: %i[s q e]
|
||||
enum :state, %i[ok scan_error file_error]
|
||||
enum :rating, %i[s q e]
|
||||
|
||||
validates_presence_of(:e621_id, :state)
|
||||
after_initialize do
|
||||
|
||||
@@ -3,7 +3,8 @@ class Domain::E621::Tagging < ReduxApplicationRecord
|
||||
belongs_to :post, class_name: "Domain::E621::Post"
|
||||
belongs_to :tag, class_name: "Domain::E621::Tag"
|
||||
|
||||
enum category: %i[
|
||||
enum :category,
|
||||
%i[
|
||||
cat_general
|
||||
cat_artist
|
||||
cat_copyright
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
class Domain::Fa::Fav < ReduxApplicationRecord
|
||||
self.table_name = "domain_fa_favs"
|
||||
self.primary_keys = :user_id, :post_id
|
||||
self.primary_key = %i[user_id post_id]
|
||||
|
||||
belongs_to :user, class_name: "::Domain::Fa::User"
|
||||
belongs_to :post, class_name: "::Domain::Fa::Post"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
class Domain::Fa::Follow < ReduxApplicationRecord
|
||||
self.table_name = "domain_fa_follows"
|
||||
self.primary_keys = :follower_id, :followed_id
|
||||
self.primary_key = %i[follower_id followed_id]
|
||||
|
||||
belongs_to :follower, class_name: "::Domain::Fa::User"
|
||||
belongs_to :followed, class_name: "::Domain::Fa::User"
|
||||
|
||||
@@ -2,7 +2,7 @@ class Domain::Twitter::Media < ReduxApplicationRecord
|
||||
self.table_name = "domain_twitter_medias"
|
||||
self.primary_key = :id
|
||||
|
||||
enum state: %i[ok error]
|
||||
enum :state, %i[ok error]
|
||||
after_initialize do
|
||||
self.state_detail ||= {}
|
||||
self.raw_data ||= {}
|
||||
|
||||
@@ -8,7 +8,7 @@ class Domain::Twitter::Tweet < ReduxApplicationRecord
|
||||
|
||||
has_many :medias, class_name: "Domain::Twitter::Media", foreign_key: :tweet_id
|
||||
|
||||
enum state: %i[ok error]
|
||||
enum :state, %i[ok error]
|
||||
after_initialize do
|
||||
self.state ||= "ok"
|
||||
self.state_detail ||= {}
|
||||
|
||||
@@ -9,7 +9,7 @@ class Domain::Twitter::User < ReduxApplicationRecord
|
||||
|
||||
has_many :medias, through: :tweets
|
||||
|
||||
enum state: %i[ok error]
|
||||
enum :state, %i[ok error]
|
||||
validates_presence_of(:name)
|
||||
after_initialize do
|
||||
self.state ||= "ok"
|
||||
|
||||
@@ -2,16 +2,10 @@ class HttpLogEntry < ReduxApplicationRecord
|
||||
include ImmutableModel
|
||||
before_destroy { raise ActiveRecord::ReadOnlyRecord }
|
||||
|
||||
enum verb: %i[get post], _prefix: true
|
||||
enum performed_by: %i[
|
||||
direct
|
||||
legacy
|
||||
proxy-1
|
||||
dedipath-1
|
||||
direct-gdl
|
||||
serverhost-1
|
||||
],
|
||||
_prefix: true
|
||||
enum :verb, %i[get post], prefix: true
|
||||
enum :performed_by,
|
||||
%i[direct legacy proxy-1 dedipath-1 direct-gdl serverhost-1],
|
||||
prefix: true
|
||||
|
||||
belongs_to :response,
|
||||
foreign_key: :response_sha256,
|
||||
|
||||
@@ -42,8 +42,8 @@ class Legacy::E621::Post < LegacyApplicationRecord
|
||||
:score
|
||||
validates_uniqueness_of :md5, :e621_id
|
||||
|
||||
serialize :sources, Array
|
||||
serialize :artists, Array
|
||||
serialize :sources, coder: JSON
|
||||
serialize :artists, coder: JSON
|
||||
|
||||
belongs_to :blob_entry, class_name: "Legacy::BlobEntry"
|
||||
|
||||
@@ -51,19 +51,14 @@ class Legacy::E621::Post < LegacyApplicationRecord
|
||||
# we've checked for the existance of its file on the
|
||||
# disk and it isn't there: :should_download
|
||||
# we've made an attempt to download its file: :processed
|
||||
enum status: %i[
|
||||
not_processed
|
||||
should_download
|
||||
processed
|
||||
processed_404
|
||||
processed_err
|
||||
]
|
||||
enum :status,
|
||||
%i[not_processed should_download processed processed_404 processed_err]
|
||||
validates_inclusion_of :status, in: statuses.keys
|
||||
|
||||
has_many :taggings, class_name: "Legacy::E621::Tagging"
|
||||
has_many :tags, through: :taggings
|
||||
|
||||
enum rating: %i[s q e]
|
||||
enum :rating, %i[s q e]
|
||||
validates_inclusion_of :rating, in: ratings.keys
|
||||
|
||||
def file_relative_path
|
||||
|
||||
@@ -41,11 +41,12 @@ class Legacy::Fa::Post < LegacyApplicationRecord
|
||||
validates_presence_of :fa_id
|
||||
|
||||
# array of [{be: blob_entry_id, file_url: old_file_url}]
|
||||
serialize :old_files, Array
|
||||
serialize :keywords, Array
|
||||
serialize :in_folders, Array
|
||||
serialize :old_files, coder: JSON
|
||||
serialize :keywords, coder: JSON
|
||||
serialize :in_folders, coder: JSON
|
||||
|
||||
enum state: [
|
||||
enum :state,
|
||||
[
|
||||
:seen_listing, # have seen a reference to this post on a listing page
|
||||
:scanned_submission, # have scanned the actual submission page
|
||||
:scan_error, # error scanning the submission page
|
||||
@@ -54,7 +55,7 @@ class Legacy::Fa::Post < LegacyApplicationRecord
|
||||
] # error getting the static asset
|
||||
validates_inclusion_of :state, in: Legacy::Fa::Post.states.keys
|
||||
|
||||
serialize :state_error
|
||||
# serialize :state_error
|
||||
validates_presence_of :state_error, if: -> { scan_error? || static_error? }
|
||||
|
||||
belongs_to :creator, class_name: "::Legacy::Fa::User"
|
||||
|
||||
@@ -66,13 +66,13 @@ class Legacy::HttpLogEntry < LegacyApplicationRecord
|
||||
:response_size
|
||||
)
|
||||
|
||||
enum verb: %i[get post]
|
||||
enum :verb, %i[get post]
|
||||
validates_inclusion_of :verb, in: Legacy::HttpLogEntry.verbs.keys
|
||||
|
||||
# text: use Diffy diffing
|
||||
# binary: use BSDiff
|
||||
# native: use the native LogStore server to store the entry
|
||||
enum diff_type: %i[text binary native]
|
||||
enum :diff_type, %i[text binary native]
|
||||
validates_inclusion_of :diff_type, in: Legacy::HttpLogEntry.diff_types.keys
|
||||
after_initialize { self.diff_type = "native" if new_record? }
|
||||
|
||||
|
||||
@@ -5,12 +5,7 @@
|
||||
<h1 class='text-2xl'>All FurAffinity posts, page <%= page_str(params) || 1 %></h1>
|
||||
<% end %>
|
||||
</div>
|
||||
<div class='mx-auto w-full border-2 border-slate-300 sm:max-w-md rounded-md mt-4 sm:mt-6 mb-4 sm:mb-6'>
|
||||
<div class='flex justify-center items-stretch h-full'>
|
||||
<%= link_to "Previous page", path_to_prev_page(@posts), class: 'hover:underline hover:bg-slate-200 hover:shadow-sm px-6 py-2 text-center border-slate-300 border-r-2 bg-slate-100 flex-1 flex items-center justify-center' %>
|
||||
<%= link_to "Next page", path_to_next_page(@posts), class: 'hover:underline hover:bg-slate-200 hover:shadow-sm px-6 py-2 text-center bg-slate-100 flex-1 flex items-center justify-center' %>
|
||||
</div>
|
||||
</div>
|
||||
<%= render partial: "shared/pagination_controls", locals: { collection: @posts } %>
|
||||
<div class='flex flex-row flex-wrap'>
|
||||
<% @posts.each do |post| %>
|
||||
<div class='border border-gray-300 rounded flex-shrink-0 bg-slate-50 m-4 flex flex-col'>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<!DOCTYPE html>
|
||||
<html class=''>
|
||||
<html class='h-full'>
|
||||
<head>
|
||||
<title><%= @site_title %></title>
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
@@ -18,7 +18,7 @@
|
||||
<%= stylesheet_link_tag "tailwind", "inter-font", "data-turbo-track": "reload" %>
|
||||
<%= yield :head %>
|
||||
</head>
|
||||
<body class="mx-0 flex flex-col">
|
||||
<body class="mx-0 flex flex-col h-full">
|
||||
<header class="bg-slate-100 border-slate-200 border-b-2">
|
||||
<div class="mx-auto max-w-5xl py-6 px-6 sm:px-8 flex items-baseline">
|
||||
<h1 class="text-4xl sm:text-5xl font-bold text-slate-900">
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<% contents_path = contents_blob_path(HexUtil.bin2hex(log_entry.response_sha256)) %>
|
||||
<section class='my-2 border-slate-300 border-2 overflow-clip rounded-md'>
|
||||
<section class='overflow-clip rounded-md grow'>
|
||||
<% if is_renderable_image_type?(log_entry.content_type) %>
|
||||
<img alt="image" src="<%= contents_path %>"/>
|
||||
<% elsif is_renderable_video_type?(log_entry.content_type) %>
|
||||
|
||||
@@ -1,112 +1,101 @@
|
||||
<% content_for :head do %>
|
||||
<style type="text/css" data-turbolinks-track>
|
||||
.hle-table {
|
||||
width: 100%;
|
||||
<style>
|
||||
.grid-cell {
|
||||
padding: 0.25rem;
|
||||
border-right: 1px solid #e2e8f0;
|
||||
}
|
||||
.grid-cell:last-child {
|
||||
padding-left: 0;
|
||||
padding-right: 1rem;
|
||||
border-right: none;
|
||||
}
|
||||
.grid-cell:first-child {
|
||||
padding-left: 1rem;
|
||||
}
|
||||
.grid-row:hover .grid-cell {
|
||||
background-color: #f1f5f9;
|
||||
}
|
||||
.hle-table {
|
||||
font-family: monospace;
|
||||
border-collapse: collapse
|
||||
}
|
||||
.hle-table tr td {
|
||||
padding: 0.25em 0.2em;
|
||||
}
|
||||
.hle-table tr td {
|
||||
border-bottom: 1px solid black;
|
||||
}
|
||||
.hle-table tr:first-child td {
|
||||
border-top: 1px solid black;
|
||||
}
|
||||
.hle-table a {
|
||||
color: black;
|
||||
text-decoration: underline;
|
||||
}
|
||||
.hle-table .col-id a {
|
||||
font-weight: bold;
|
||||
}
|
||||
.hle-table .leftb {
|
||||
border-left: 1px solid black;
|
||||
}
|
||||
.hle-table .rightb {
|
||||
border-right: 1px solid black;
|
||||
}
|
||||
.hle-table .col-code.good {
|
||||
color: blue;
|
||||
}
|
||||
.hle-table .col-code.bad {
|
||||
color: red;
|
||||
}
|
||||
.hle-table .part {
|
||||
background-color: rgba(black, 0.25);
|
||||
}
|
||||
.hle-table .uri-td {
|
||||
word-break: break-all;
|
||||
}
|
||||
.hle-table tr:hover {
|
||||
background-color: #cecece;
|
||||
}
|
||||
.floatr {
|
||||
float: right;
|
||||
}
|
||||
.col-size {
|
||||
width: 140px;
|
||||
}
|
||||
.col-time {
|
||||
text-align: right;
|
||||
width: 100px;
|
||||
}
|
||||
.col-id {
|
||||
width: 70px;
|
||||
text-align: center;
|
||||
}
|
||||
.col-code {
|
||||
width: 25px;
|
||||
text-align: center;
|
||||
}
|
||||
</style>
|
||||
<% end %>
|
||||
<h1>
|
||||
Http Request Log (<%= link_to "Stats", stats_log_entries_path(seconds: 60) %>)
|
||||
<div class='mx-auto mt-4 sm:mt-6 text-center'>
|
||||
<h1 class='text-2xl'>HTTP Log Entries</h1>
|
||||
<div class='mt-2 text-lg flex items-center justify-center gap-2'>
|
||||
<%= link_to stats_log_entries_path(seconds: 60), class: "inline-flex items-center gap-1 text-blue-600 hover:text-blue-800" do %>
|
||||
<%= render partial: "shared/icons/chart_bars", locals: { class_name: "w-5 h-5" } %>
|
||||
View Statistics
|
||||
<% end %>
|
||||
<% if @uri_filter %>
|
||||
<span class="text-slate-400">|</span>
|
||||
<%= link_to "Back to All Entries", log_entries_path, class: "text-blue-600 hover:text-blue-800" %>
|
||||
<% end %>
|
||||
</div>
|
||||
<% if @uri_filter %>
|
||||
(<%= link_to "Index", log_entries_path %>)
|
||||
<div class='mt-2 text-slate-600'>
|
||||
Currently filtering: <code class="bg-slate-100 px-2 py-1 rounded font-mono text-sm"><%= @uri_filter %></code>
|
||||
</div>
|
||||
<% end %>
|
||||
</h1>
|
||||
<nav>
|
||||
<%= link_to_previous_page @log_entries, "Previous Page" %> <%= link_to_next_page @log_entries, "Next Page" %>
|
||||
</nav>
|
||||
<table class='hle-table'>
|
||||
</div>
|
||||
<%= render partial: "shared/pagination_controls", locals: { collection: @log_entries } %>
|
||||
<div class="grid grid-cols-[auto_auto_auto_auto_1fr_auto_auto_auto] border-b border-slate-300 text-sm">
|
||||
<div class='grid-row contents'>
|
||||
<div class="grid-cell text-center font-semibold">ID</div>
|
||||
<div class="grid-cell text-right font-semibold">Size</div>
|
||||
<div class="grid-cell text-center font-semibold">Time</div>
|
||||
<div class="grid-cell text-center font-semibold">Status</div>
|
||||
<div class="grid-cell text-center font-semibold">URI</div>
|
||||
<div class="grid-cell text-center font-semibold"></div>
|
||||
<div class="grid-cell text-center font-semibold">Type</div>
|
||||
<div class="grid-cell text-center font-semibold">Response</div>
|
||||
</div>
|
||||
<div class="col-span-full border-b border-slate-300"></div>
|
||||
<% @log_entries.each do |hle| %>
|
||||
<tr>
|
||||
<td class="col-size leftb">
|
||||
<%= HexUtil.humansize(hle.response.size) %> <br>
|
||||
(<%= (hle.response.bytes_stored.to_f / hle.response.size).round(2) %>, <%= hle.performed_by %>)
|
||||
</td>
|
||||
<td class="col-time leftb">
|
||||
<div class="grid-row contents">
|
||||
<div class="grid-cell text-center">
|
||||
<%= link_to hle.id, log_entry_path(hle.id), class: "text-blue-600 hover:text-blue-800" %>
|
||||
</div>
|
||||
<div class="grid-cell text-right">
|
||||
<%= HexUtil.humansize(hle.response.size) %>
|
||||
</div>
|
||||
<div class="grid-cell text-right">
|
||||
<%= time_ago_in_words(hle.created_at, include_seconds: true) %> ago
|
||||
</td>
|
||||
<td class="col-id leftb rightb">
|
||||
<%= link_to hle.id, log_entry_path(hle.id) %>
|
||||
</td>
|
||||
<% status_code_class = hle.status_code == 200 ? "good" : "bad" %>
|
||||
<td class="col-code leftb <%= status_code_class %>"><%= hle.status_code %></td>
|
||||
<td class="path leftb uri-td">
|
||||
<% path_parts = hle.uri_path.split("/")
|
||||
iterative_parts = (1...path_parts.length).map do |i|
|
||||
[path_parts[i], path_parts[0..i].join("/") + (i == path_parts.length - 1 ? "" : "/")]
|
||||
end %>
|
||||
<a class='part host' href="/log_entries/filter/<%= hle.uri_host %>">
|
||||
<%= hle.uri_scheme %>://<%= hle.uri_host %><!--
|
||||
--></a><!--
|
||||
--><% iterative_parts.each do |part, up_to| %><!--
|
||||
--><a class='part path' href="/log_entries/filter/<%= hle.uri_host %><%= up_to %>"><!--
|
||||
-->/<%= part %><!--
|
||||
--></a><% end %><!--
|
||||
--><% if hle.uri_query %><!--
|
||||
--><span title="<%= hle.uri_query %>"><%= "?#{hle.uri_query.truncate(120 - hle.uri.to_s.length)}" %></span>
|
||||
</div>
|
||||
<div class="grid-cell text-center">
|
||||
<span class="<%= hle.status_code == 200 ? 'text-green-600' : 'text-red-600' %>">
|
||||
<%= hle.status_code %>
|
||||
</span>
|
||||
</div>
|
||||
<div class="grid-cell min-w-0">
|
||||
<% iterative_parts = path_iterative_parts(hle.uri_path) %>
|
||||
<div class="flex truncate whitespace-nowrap overflow-hidden ">
|
||||
<a class="bg-slate-100 rounded-l hover:bg-slate-200" href="/log_entries/filter/<%= hle.uri_host %>">
|
||||
<%= hle.uri_scheme %>://<%= hle.uri_host %>
|
||||
</a>
|
||||
<%- iterative_parts.each_with_index do |(part, up_to), index| -%>
|
||||
<% uri_and_up_to = hle.uri_host + up_to %>
|
||||
<a
|
||||
class="bg-slate-100 <%= index == iterative_parts.length-1 ? 'rounded-r' : '' %> hover:bg-slate-200"
|
||||
href="/log_entries/filter/<%= uri_and_up_to %>"
|
||||
title="<%= hle.uri_scheme + "://" + uri_and_up_to %>"
|
||||
>/<%= part %></a>
|
||||
<%- end -%>
|
||||
<%- if hle.uri_query -%>
|
||||
<span class="text-slate-600 min-w-0" title="<%= hle.uri_query %>">
|
||||
<%= "?#{hle.uri_query.truncate(120 - hle.uri.to_s.length)}" %>
|
||||
</span><%- end -%>
|
||||
</div>
|
||||
</div>
|
||||
<div class="grid-cell text-center">
|
||||
<%= link_to hle.uri.to_s, class: "text-blue-600 hover:text-blue-800", target: "_blank", rel: "noreferrer" do %>
|
||||
<%= render partial: "shared/icons/external_link", locals: { class_name: "w-4 h-4 inline" } %>
|
||||
<% end %>
|
||||
<a class="floatr" rel="noreferrer" target="_blank" href="<%= hle.uri_str %>">(link)</a>
|
||||
</td>
|
||||
<td class="type leftb"><%= hle.content_type %></td>
|
||||
<td class="time leftb rightb"><%= hle.response_time_ms %>ms</td>
|
||||
</tr>
|
||||
</div>
|
||||
<div class="grid-cell max-w-24 truncate">
|
||||
<span title="<%= hle.content_type %>"> <%= hle.content_type %> </span>
|
||||
</div>
|
||||
<div class="grid-cell text-right">
|
||||
<%= hle.response_time_ms %>ms
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-span-full border-b border-slate-300"></div>
|
||||
<% end %>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
@@ -3,135 +3,91 @@
|
||||
<script src="https://unpkg.com/@ruffle-rs/ruffle"></script>
|
||||
<% end %>
|
||||
<style type="text/css" data-turbolinks-track>
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
}
|
||||
table tr {
|
||||
border-bottom: 1px solid black;
|
||||
}
|
||||
table.log-entry-details tr:hover {
|
||||
background-color: #ececec;
|
||||
}
|
||||
table tr:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
table td {
|
||||
padding: 0.2em;
|
||||
padding-left: 0.2em;
|
||||
border-right: 1px solid black;
|
||||
}
|
||||
table td:last-child {
|
||||
border-right: none;
|
||||
}
|
||||
table td:first-child {
|
||||
text-align: right;
|
||||
}
|
||||
|
||||
iframe {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
border: none;
|
||||
box-shadow: 0 0 5px 1px black;
|
||||
}
|
||||
|
||||
embed, ruffle-embed {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.log-entry-triggers-container {
|
||||
max-height: 100px;
|
||||
overflow: scroll;
|
||||
overflow-x: hidden;
|
||||
border-top: 1px solid black;
|
||||
border-bottom: 1px solid black;
|
||||
}
|
||||
.hle-mini-status-code {
|
||||
width: 0em;
|
||||
}
|
||||
.hle-mini-id-link {
|
||||
width: 0em;
|
||||
}
|
||||
.hle-mini-uri {}
|
||||
</style>
|
||||
<% end %>
|
||||
<%= link_to "← Log Entries".html_safe, log_entries_path %>
|
||||
<h3>
|
||||
<span title="<%= @log_entry.uri.to_s %>"><%= @log_entry.uri.to_s.truncate(120) %></span> -
|
||||
<%= @log_entry.status_code %>
|
||||
</h3>
|
||||
<table class='log-entry-details'>
|
||||
<tr>
|
||||
<% rtms = @log_entry.response_time_ms %>
|
||||
<td>response time</td>
|
||||
<td><%= rtms == -1 ? "(not recorded)" : "#{rtms}ms" %></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>performed by</td>
|
||||
<td><%= @log_entry.performed_by %> - <%= time_ago_in_words(@log_entry.requested_at, include_seconds: true) %> ago</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>content type</td>
|
||||
<td><%= @log_entry.content_type %></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>content size</td>
|
||||
<td><%= HexUtil.humansize(@log_entry.response.size) %> (<%= HexUtil.humansize(@log_entry.response.bytes_stored) %> stored)</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>headers (subset)</td>
|
||||
<td>
|
||||
<table>
|
||||
<% ["cf-ray", "cf-cache-status"].each do |header_name| %>
|
||||
<tr>
|
||||
<td><%= header_name %></td>
|
||||
<td><%= @log_entry.response_headers.headers[header_name] || "(empty)" %></td>
|
||||
</tr>
|
||||
<% end %>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<% if @log_entry.response.base %>
|
||||
<tr>
|
||||
<td>base log entry</td>
|
||||
<% base_hle = HttpLogEntry.find_by(response: @log_entry.response.base) %>
|
||||
<td>
|
||||
<table>
|
||||
<% if base_hle %>
|
||||
<%= render partial: "log_entry_table_row_mini", locals: { entry: base_hle } %>
|
||||
<% else %>
|
||||
<tr><td><i>HLE not found...</i></td></tr>
|
||||
<div class="bg-slate-50">
|
||||
<div class="flex flex-row max-w-screen-md mx-auto py-4">
|
||||
<div class="flex items-center ml-4 pr-4 min-w-[150px]">
|
||||
<%= link_to log_entries_path, class: "px-4 py-2 text-sm font-medium text-slate-700 bg-white border border-slate-300 rounded-md hover:bg-slate-50" do %>
|
||||
<span class="mr-1">←</span>
|
||||
<span>Log Entries</span>
|
||||
<% end %>
|
||||
</div>
|
||||
<div class="bg-white shadow rounded-lg p-4">
|
||||
<div class="grid grid-cols-[auto_1fr] gap-x-4 gap-y-2 items-center">
|
||||
<div class="text-slate-500 text-sm text-right">URL</div>
|
||||
<div>
|
||||
<div class="flex items-center">
|
||||
<%= link_to(@log_entry.uri.to_s, class: "text-blue-600 hover:text-blue-800 underline decoration-dotted", target: "_blank", rel: "noopener") do %>
|
||||
<span class="truncate max-w-[40rem] inline-block align-middle"><%= @log_entry.uri.to_s %></span>
|
||||
<%= render partial: "shared/icons/external_link", locals: {class_name: "inline h-4 w-4 ml-0.5"} %>
|
||||
<% end %>
|
||||
<span class="ml-2 px-2 py-0.5 text-sm rounded-full <%= @log_entry.status_code >= 400 ? 'bg-red-100 text-red-800' : 'bg-green-100 text-green-800' %>">
|
||||
<%= @log_entry.status_code %>
|
||||
</span>
|
||||
</div>
|
||||
<div class="mt-1 text-sm text-slate-500">
|
||||
<% rtms = @log_entry.response_time_ms %>
|
||||
<span><%= rtms == -1 ? "Response time not recorded" : "#{rtms}ms" %></span>
|
||||
<span class="mx-2">•</span>
|
||||
<span><%= HexUtil.humansize(@log_entry.response.size) %> (<%= HexUtil.humansize(@log_entry.response.bytes_stored) %> stored)</span>
|
||||
<span class="mx-2">•</span>
|
||||
<span>Performed by <%= @log_entry.performed_by %></span>
|
||||
<span class="mx-2">•</span>
|
||||
<span><%= time_ago_in_words(@log_entry.requested_at, include_seconds: true) %> ago</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="text-slate-500 text-sm text-right">Content</div>
|
||||
<div class="text-sm">
|
||||
<span class="font-mono"><%= @log_entry.content_type %></span>
|
||||
</div>
|
||||
<div class="text-slate-500 text-sm text-right">Headers</div>
|
||||
<div class="text-sm font-mono grid grid-cols-2 gap-x-4">
|
||||
<% ["cf-ray", "cf-cache-status"].each do |header_name| %>
|
||||
<div class="text-slate-500"><%= header_name %></div>
|
||||
<div><%= @log_entry.response_headers.headers[header_name] || "(empty)" %></div>
|
||||
<% end %>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<% end %>
|
||||
<% if @log_entry.caused_by_entry %>
|
||||
<% caused_by = @log_entry.caused_by_entry %>
|
||||
<tr>
|
||||
<td>caused by</td>
|
||||
<td>
|
||||
<table>
|
||||
<%= render partial: "log_entry_table_row_mini", locals: { entry: caused_by } %>
|
||||
</table>
|
||||
</td>
|
||||
</tr>
|
||||
<% end %>
|
||||
<% triggered = @log_entry.triggered_entries %>
|
||||
<% if triggered.length > 0 %>
|
||||
<tr>
|
||||
<td>triggered <%= pluralize(triggered.length, "entry") %></td>
|
||||
<td>
|
||||
<div class='log-entry-triggers-container'>
|
||||
<table class='log-entry-triggers'>
|
||||
</div>
|
||||
<% if @log_entry.response.base %>
|
||||
<div class="text-slate-500 text-sm text-right">Base Entry</div>
|
||||
<div>
|
||||
<% base_hle = HttpLogEntry.find_by(response: @log_entry.response.base) %>
|
||||
<% if base_hle %>
|
||||
<%= render partial: "log_entry_table_row_mini", locals: { entry: base_hle } %>
|
||||
<% else %>
|
||||
<span class="text-slate-500 italic">HLE not found...</span>
|
||||
<% end %>
|
||||
</div>
|
||||
<% end %>
|
||||
<% if @log_entry.caused_by_entry %>
|
||||
<div class="text-slate-500 text-sm text-right">Caused By</div>
|
||||
<div>
|
||||
<%= render partial: "log_entry_table_row_mini", locals: { entry: @log_entry.caused_by_entry } %>
|
||||
</div>
|
||||
<% end %>
|
||||
<% triggered = @log_entry.triggered_entries %>
|
||||
<% if triggered.length > 0 %>
|
||||
<div class="text-slate-500 text-sm text-right">Triggered</div>
|
||||
<div class="max-h-24 overflow-y-auto border border-slate-200 rounded">
|
||||
<% triggered.each do |entry| %>
|
||||
<%= render partial: "log_entry_table_row_mini", locals: { entry: entry } %>
|
||||
<% end %>
|
||||
</table>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
<% end %>
|
||||
</table>
|
||||
<%= render partial: "content_container", locals: { log_entry: @log_entry } %>
|
||||
</div>
|
||||
<% end %>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="border-t w-full flex grow shadow-inner">
|
||||
<%= render partial: "content_container", locals: { log_entry: @log_entry } %>
|
||||
</div>
|
||||
|
||||
18
app/views/shared/_pagination_controls.html.erb
Normal file
18
app/views/shared/_pagination_controls.html.erb
Normal file
@@ -0,0 +1,18 @@
|
||||
<div class='mx-auto w-full border-2 border-slate-300 sm:max-w-md rounded-md mt-4 sm:mt-6 mb-4 sm:mb-6'>
|
||||
<div class='flex justify-center items-stretch h-full'>
|
||||
<% if collection.prev_page %>
|
||||
<%= link_to_previous_page collection, raw("← Previous Page (#{collection.current_page - 1})"), class: 'hover:underline hover:bg-slate-200 hover:shadow-sm px-6 py-2 text-center border-slate-300 border-r-2 bg-slate-100 flex-1 flex items-center justify-center' %>
|
||||
<% else %>
|
||||
<div class='px-6 py-2 text-center border-slate-300 border-r-2 bg-slate-50 text-slate-400 flex-1 flex items-center justify-center'>
|
||||
← Previous Page
|
||||
</div>
|
||||
<% end %>
|
||||
<% if collection.next_page %>
|
||||
<%= link_to_next_page collection, raw("Next Page (#{collection.current_page + 1}) →"), class: 'hover:underline hover:bg-slate-200 hover:shadow-sm px-6 py-2 text-center bg-slate-100 flex-1 flex items-center justify-center' %>
|
||||
<% else %>
|
||||
<div class='px-6 py-2 text-center bg-slate-50 text-slate-400 flex-1 flex items-center justify-center'>
|
||||
Next Page →
|
||||
</div>
|
||||
<% end %>
|
||||
</div>
|
||||
</div>
|
||||
3
app/views/shared/icons/_chart_bars.html.erb
Normal file
3
app/views/shared/icons/_chart_bars.html.erb
Normal file
@@ -0,0 +1,3 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="<%= local_assigns[:class_name] %>">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" d="M3 13.125C3 12.504 3.504 12 4.125 12h2.25c.621 0 1.125.504 1.125 1.125v6.75C7.5 20.496 6.996 21 6.375 21h-2.25A1.125 1.125 0 013 19.875v-6.75zM9.75 8.625c0-.621.504-1.125 1.125-1.125h2.25c.621 0 1.125.504 1.125 1.125v11.25c0 .621-.504 1.125-1.125 1.125h-2.25a1.125 1.125 0 01-1.125-1.125V8.625zM16.5 4.125c0-.621.504-1.125 1.125-1.125h2.25C20.496 3 21 3.504 21 4.125v15.75c0 .621-.504 1.125-1.125 1.125h-2.25a1.125 1.125 0 01-1.125-1.125V4.125z" />
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 669 B |
@@ -11,7 +11,6 @@ module ReduxScraper
|
||||
config.assets.precompile << "delayed/web/application.css"
|
||||
# Initialize configuration defaults for originally generated Rails version.
|
||||
config.load_defaults 7.0
|
||||
config.active_record.legacy_connection_handling = false
|
||||
config.autoload_paths << config.root.join("app/lib")
|
||||
|
||||
# all environments use good_job (in external execution mode)
|
||||
|
||||
@@ -83,10 +83,7 @@ testcookies: &testcookies
|
||||
path: /
|
||||
|
||||
development:
|
||||
direct: *testcookies
|
||||
proxy-1: *testcookies
|
||||
dedipath-1: *testcookies
|
||||
serverhost-1: *testcookies
|
||||
direct: *vipvillageworker
|
||||
|
||||
production:
|
||||
direct: *ddwhatnow
|
||||
|
||||
9642
package-lock.json
generated
9642
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -18,13 +18,19 @@ namespace :blob_file do
|
||||
BlobEntryP.in_batches(
|
||||
of: batch_size,
|
||||
start: HexUtil.hex2bin(start_at),
|
||||
order: :asc
|
||||
order: :asc,
|
||||
use_ranges: true
|
||||
) do |batch|
|
||||
batch_migrated = insert_blob_entries_batch(batch)
|
||||
num_migrated += batch_migrated
|
||||
num_processed += batch.size
|
||||
rate = batch_migrated.to_f / (Time.now - start_time)
|
||||
puts "migrated #{batch_migrated}, processed #{num_processed} @ #{rate.round(1)}/second [last: #{HexUtil.bin2hex(batch.last.sha256)}]"
|
||||
puts [
|
||||
"[migrated: #{ActiveSupport::NumberHelper.number_to_delimited(num_migrated).rjust(8)}]",
|
||||
"[processed: #{ActiveSupport::NumberHelper.number_to_delimited(num_processed).rjust(8)}]",
|
||||
"[rate: #{rate.round(1).to_s.rjust(5)}/second]",
|
||||
"[last: '#{HexUtil.bin2hex(batch.last.sha256)}']"
|
||||
].join(" ")
|
||||
start_time = Time.now
|
||||
end
|
||||
num_migrated
|
||||
|
||||
@@ -152,6 +152,40 @@ describe Domain::Fa::Job::BrowsePageJob do
|
||||
end
|
||||
end
|
||||
|
||||
context "duplicates found on the page" do
|
||||
let! :log_entries do
|
||||
SpecUtil.init_http_client_mock(
|
||||
http_client_mock,
|
||||
[
|
||||
{
|
||||
uri: "https://www.furaffinity.net/browse/",
|
||||
status_code: 200,
|
||||
content_type: "text/html",
|
||||
contents:
|
||||
SpecUtil.read_fixture_file(
|
||||
"domain/fa/job/browse_page_duplicates.html"
|
||||
),
|
||||
caused_by_entry_idx: nil
|
||||
},
|
||||
{
|
||||
uri: "https://www.furaffinity.net/browse/2/",
|
||||
status_code: 200,
|
||||
content_type: "text/html",
|
||||
contents:
|
||||
SpecUtil.read_fixture_file(
|
||||
"domain/fa/job/browse_page_no_submissions.html"
|
||||
),
|
||||
caused_by_entry_idx: 0
|
||||
}
|
||||
]
|
||||
)
|
||||
end
|
||||
|
||||
it "succeeds" do
|
||||
expect { perform_now({}) }.not_to raise_error
|
||||
end
|
||||
end
|
||||
|
||||
context "with one unseen post" do
|
||||
include_context "user and post getters"
|
||||
let! :log_entries do
|
||||
|
||||
@@ -37,7 +37,7 @@ RSpec.configure do |config|
|
||||
ReactOnRails::TestHelper.configure_rspec_to_compile_assets(config)
|
||||
|
||||
# Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
|
||||
config.fixture_path = "#{::Rails.root}/spec/fixtures"
|
||||
# config.fixture_path = "#{::Rails.root}/spec/fixtures"
|
||||
|
||||
# If you're not using ActiveRecord, or you'd prefer not to run each of your
|
||||
# examples within a transaction, remove the following line or assign false
|
||||
|
||||
1316
test/fixtures/files/domain/fa/job/browse_page_duplicates.html
vendored
Normal file
1316
test/fixtures/files/domain/fa/job/browse_page_duplicates.html
vendored
Normal file
File diff suppressed because one or more lines are too long
@@ -1,9 +1,9 @@
|
||||
class Domain::Fa::PostTest < ActiveSupport::TestCase
|
||||
test "remove buggy prefixes" do
|
||||
# TODO - implement this
|
||||
# Some posts have a title prefixed with "Font size adjustment: smallerlarger"
|
||||
# which should be removed
|
||||
# Legacy::Fa::Post.where("title like ?", "Font size adjustment: smallerlarger%").count
|
||||
# => 7056
|
||||
end
|
||||
# test "remove buggy prefixes" do
|
||||
# TODO - implement this
|
||||
# Some posts have a title prefixed with "Font size adjustment: smallerlarger"
|
||||
# which should be removed
|
||||
# Legacy::Fa::Post.where("title like ?", "Font size adjustment: smallerlarger%").count
|
||||
# => 7056
|
||||
# end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user