tweaks to queue sizes

Rakefile

@@ -61,7 +61,7 @@ end
 task :workoff_failed_jobs => [:environment, :set_ar_stdout, :set_logger_stdout] do
   worker = Delayed::Worker.new
   Delayed::Job.where(
-    "last_error is not null and attempts < 2"
+    "last_error is not null and attempts <= 2"
   ).find_each(batch_size: 1) do |job|
     worker.run(job)
   end

app/jobs/scraper/job_base.rb

@@ -30,7 +30,7 @@ class Scraper::JobBase < ApplicationJob
   ]
 
   def enqueue_jobs_from_found_links(log_entry, suppress_jobs: [])
-    # return
+    return unless ENV["enqueue_jobs_from_found_links"] == "1"
 
     start_time = Time.now
     unless PERMITTED_CONTENT_TYPES.any? { |ct| ct.match(log_entry.content_type) }
@@ -107,7 +107,7 @@ class Scraper::JobBase < ApplicationJob
 
         job_defs << job_def.merge(
           job: Domain::Fa::Job::ScanFileJob,
-        ) if post && post.state == "ok" && post.scanned?
+        ) if post && post.state == "ok" && post.scanned? && !post.have_file?
       when :twitter_user
         name = link[:name]
         user = name_to_twitter_user[name]

app/lib/metrics/estimate_db_rows_reporter.rb

@@ -5,6 +5,7 @@ class Metrics::EstimateDbRowsReporter < Metrics::Reporter
       "http_log_entry_headers",
       "versions",
       "delayed_jobs",
+      "blob_entries",
     ]
     extra_tables_sql = extra_tables.map { |t| "'#{t}'" }.join(",")
     row_estimates = ReduxApplicationRecord.connection.exec_query(

app/lib/scraper/fa_http_client.rb

@@ -20,7 +20,7 @@ class Scraper::FaHttpClient < Scraper::BaseHttpClient
     [
       ["d.furaffinity.net", :none],
       ["*.facdn.net", :none],
-      ["*", 1],
+      ["*", 0.25],
     ]
   end
 

config/database.yml

@@ -17,7 +17,7 @@ redux_prod: &redux_prod
   username: scraper_redux
   password: pdkFLqRmQwPUPaDDC4pX
   migrations_paths: db/redux_migrate
-  pool: 2
+  pool: 4
 
 redux_test: &redux_test
   adapter: postgresql

pool_combined.rb

@@ -37,7 +37,7 @@ after_worker_shutdown do |worker_info|
        "with process id #{worker_info.process_id}"
 end
 
-PRIORITIZE_STATIC = ENV["prioritize_static"]&.to_i != 0
+PRIORITIZE_STATIC = ENV["prioritize_static"]&.to_i == 1
 worker_configs = [
   {
     name: "manual",
@@ -52,12 +52,12 @@ worker_configs = [
   {
     name: "fa posts",
     queues: [:fa_post],
-    workers: PRIORITIZE_STATIC ? 1 : 3,
+    workers: PRIORITIZE_STATIC ? 1 : 2,
   },
   {
     name: "static files",
     queues: [:static_file, :twitter_static_file],
-    workers: PRIORITIZE_STATIC ? 8 : 6,
+    workers: PRIORITIZE_STATIC ? 8 : 4,
   },
 ]