add e621 to periodic tasks

This commit is contained in:
Dylan Knutson
2023-08-22 17:47:39 -07:00
parent ea5a2a7d6c
commit 2e0c2fdf51
2 changed files with 2 additions and 80 deletions

View File

@@ -48,7 +48,7 @@ task :periodic_tasks => [:environment, :set_logger_stdout] do
loop do
Rake::Task["fa:browse_page_job"].execute
Rake::Task["fa:home_page_job"].execute
# Rake::Task["e621:posts_index_job"].execute
Rake::Task["e621:posts_index_job"].execute
puts "enqueue periodic jobs"
sleep 1.minute
end
@@ -111,7 +111,7 @@ task :good_job do
"GOOD_JOB_MAX_THREADS" => "4",
"GOOD_JOB_QUEUES" => [
"manual:4",
"fa_post:2",
"fa_post,e621:2",
"*:6",
].reject(&:nil?).join(";"),
}

View File

@@ -1,78 +0,0 @@
PROXY = ENV["proxy"]
VALID_PROXIES = ["direct", "proxy-1", "dedipath-1", "serverhost-1"]
raise("'proxy' env must be one of #{VALID_PROXIES}") unless VALID_PROXIES.include?(PROXY)
# This runs in the master process after it preloads the app
after_preload_app do
puts "Master #{Process.pid} preloaded app"
# Don't hang on to database connections from the master after we've
# completed initialization
ActiveRecord::Base.connection_pool.disconnect!
end
# This runs in the worker processes after it has been forked
on_worker_boot do |worker_info|
ActiveJob::Base.logger = Logger.new("/dev/null")
Delayed::Worker.logger = Logger.new(STDOUT)
Delayed::Worker.logger.level = :error
Rails.logger = Logger.new(STDOUT)
puts "Worker #{Process.pid} started"
sleep rand(1..5)
# Reconnect to the database
ActiveRecord::Base.establish_connection
end
# This runs in the master process after a worker starts
after_worker_boot do |worker_info|
puts "Master #{Process.pid} booted worker #{worker_info.name} with " \
"process id #{worker_info.process_id}"
end
# This runs in the master process after a worker shuts down
after_worker_shutdown do |worker_info|
puts "Master #{Process.pid} detected dead worker #{worker_info.name} " \
"with process id #{worker_info.process_id}"
end
PRIORITIZE_STATIC = ENV["prioritize_static"]&.to_i == 1
worker_configs = [
{
name: "manual",
queues: [:manual, :link_finder],
workers: 3,
},
{
name: "twitter / fa galleries / e621",
queues: [
:twitter_timeline_tweets,
:fa_user_page,
:fa_user_gallery,
:e621,
],
workers: PRIORITIZE_STATIC ? 1 : 2,
},
{
name: "fa posts",
queues: [:fa_post],
workers: PRIORITIZE_STATIC ? 1 : 2,
},
{
name: "static files",
queues: [:static_file, :twitter_static_file],
workers: PRIORITIZE_STATIC ? 8 : 4,
},
]
worker_configs.each do |config|
worker_group("#{PROXY} - #{config[:name]}") do |g|
g.read_ahead = 8
g.sleep_delay = 5
g.workers = config[:workers]
g.queues = config[:queues].map(&:to_s)
end
end
preload_app