add logging / debugging to jobs

app/jobs/application_job.rb

@@ -1,4 +1,16 @@
 class ApplicationJob < ActiveJob::Base
+  include GoodJob::ActiveJobExtensions::Concurrency
+  include HasColorLogger
+
+  retry_on(
+    StandardError,
+    wait: :exponentially_longer,
+    attempts: Float::INFINITY,
+  ) do |job, exception|
+    job.logger.error("error: #{exception.message}\n#{exception.backtrace.join("\n")}")
+    binding.pry
+  end
+
   # Automatically retry jobs that encountered a deadlock
   # retry_on ActiveRecord::Deadlocked
 

app/jobs/domain/fa/job/scan_file_job.rb

@@ -3,8 +3,6 @@ class Domain::Fa::Job::ScanFileJob < Domain::Fa::Job::FaJobBase
   ignore_signature_args :caused_by_entry
 
   def perform(args)
-    logger.level = :warn
-
     @post = args[:post]
     @caused_by_entry = args[:caused_by_entry]
     @force_scan = !!args[:force_scan]

app/jobs/scraper/job_base.rb

@@ -1,6 +1,4 @@
 class Scraper::JobBase < ApplicationJob
-  include GoodJob::ActiveJobExtensions::Concurrency
-  include HasColorLogger
   attr_reader :http_client
 
   def initialize(...)
@@ -21,6 +19,7 @@ class Scraper::JobBase < ApplicationJob
 
       ignore_signature_args = self.class.ignore_signature_args
       ignore_signature_args << :_aj_symbol_keys
+      ignore_signature_args << :_aj_ruby2_keywords
       sig_arguments = first_argument.reject do |key, value|
         ignore_signature_args.include?(key.to_sym)
       end.to_h
@@ -39,6 +38,7 @@ class Scraper::JobBase < ApplicationJob
     level = job.logger.level
     job.logger.level = :info
     block.call
+  ensure
     job.logger.level = level
   end
 
@@ -189,7 +189,7 @@ class Scraper::JobBase < ApplicationJob
     sleep rand(2.0..7.0)
     error = e
     raise e
-  rescue Exception => e
+  rescue => e
     error = e
     raise e
   ensure

config/application.rb

@@ -17,7 +17,6 @@ module ReduxScraper
     # GoodJob configuration - applies to all environments (including test)
     config.active_job.queue_adapter = :good_job
     config.good_job.inline_execution_respects_schedule = true
-    config.good_job.execution_mode = :async
     config.good_job.active_record_parent_class = "ReduxApplicationRecord"
     config.good_job.retry_on_unhandled_error = true
     config.good_job.smaller_number_is_higher_priority = true
@@ -27,6 +26,17 @@ module ReduxScraper
     config.good_job.cleanup_preserved_jobs_before_seconds_ago = 1.day
     config.good_job.cleanup_interval_jobs = 100_000
     config.good_job.cleanup_interval_seconds = 4.hours
+    config.good_job.logger = Logger.new(STDOUT)
+    config.good_job.logger.level = :warn
+
+    if Rails.env.worker? || Rails.env.test?
+      config.good_job.execution_mode = :async
+      config.good_job.on_thread_error = ->(exception) {
+        binding.pry
+      }
+    else
+      config.good_job.execution_mode = :external
+    end
 
     # Configuration for the application, engines, and railties goes here.
     #

config/cookies/fa.yml

@@ -88,7 +88,7 @@ development:
   dedipath-1: *blazeandwish
   serverhost-1: *cottoniq
 
-async_worker:
+worker:
   direct: *ddwhatnow
   proxy-1: *vipvillageworker
   dedipath-1: *blazeandwish

config/database.yml

@@ -71,7 +71,7 @@ production:
   legacy:
     <<: *legacy_prod
 
-async_worker:
+worker:
   redux:
     <<: *redux_prod
     pool: 16

config/environments/worker.rb

@@ -0,0 +1,85 @@
+require "active_support/core_ext/integer/time"
+
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # Code is not reloaded between requests.
+  config.cache_classes = true
+
+  # Eager load code on boot. This eager loads most of Rails and
+  # your application in memory, allowing both threaded web servers
+  # and those relying on copy on write to perform better.
+  # Rake tasks automatically ignore this option for performance.
+  config.eager_load = true
+
+  # Full error reports are disabled and caching is turned on.
+  config.consider_all_requests_local = false
+  config.action_controller.perform_caching = true
+
+  # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"]
+  # or in config/master.key. This key is used to decrypt credentials (and other encrypted files).
+  # config.require_master_key = true
+
+  config.public_file_server.enabled = false
+
+  # Compress CSS using a preprocessor.
+  # config.assets.css_compressor = :sass
+
+  # Do not fallback to assets pipeline if a precompiled asset is missed.
+  config.assets.compile = false
+
+  # Enable serving of images, stylesheets, and JavaScripts from an asset server.
+  # config.asset_host = "http://assets.example.com"
+
+  # Specifies the header that your server uses for sending files.
+  # config.action_dispatch.x_sendfile_header = "X-Sendfile" # for Apache
+  # config.action_dispatch.x_sendfile_header = "X-Accel-Redirect" # for NGINX
+
+  # Store uploaded files on the local file system (see config/storage.yml for options).
+  config.active_storage.service = :local
+
+  # Mount Action Cable outside main process or domain.
+  # config.action_cable.mount_path = nil
+  # config.action_cable.url = "wss://example.com/cable"
+  # config.action_cable.allowed_request_origins = [ "http://example.com", /http:\/\/example.*/ ]
+
+  # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
+  # config.force_ssl = true
+
+  # Include generic and useful information about system operation, but avoid logging too much
+  # information to avoid inadvertent exposure of personally identifiable information (PII).
+  config.log_level = :info
+
+  # Prepend all log lines with the following tags.
+  config.log_tags = [:request_id]
+
+  # Use a different cache store in production.
+  # config.cache_store = :mem_cache_store
+
+  # Use a real queuing backend for Active Job (and separate queues per environment).
+  # config.active_job.queue_adapter     = :resque
+  # config.active_job.queue_name_prefix = "legacy_explorer_production"
+
+  config.action_mailer.perform_caching = false
+
+  # Ignore bad email addresses and do not raise email delivery errors.
+  # Set this to true and configure the email server for immediate delivery to raise delivery errors.
+  # config.action_mailer.raise_delivery_errors = false
+
+  # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
+  # the I18n.default_locale when a translation cannot be found).
+  config.i18n.fallbacks = true
+
+  # Don't log any deprecations.
+  config.active_support.report_deprecations = false
+
+  # Use default logging formatter so that PID and timestamp are not suppressed.
+  config.log_formatter = ::Logger::Formatter.new
+
+  logger = ActiveSupport::Logger.new(STDOUT)
+  logger.formatter = config.log_formatter
+  config.logger = ActiveSupport::TaggedLogging.new(logger)
+
+  # Do not dump schema after migrations.
+  config.active_record.dump_schema_after_migration = false
+end

config/influxdb.yml

@@ -10,6 +10,12 @@ production:
   org: "primary"
   bucket: "redux-scraper"
 
+worker:
+  host: "http://grafana.local:8086"
+  token: "W2ikhmpQxQHZAStgSEK6s6aJxnOqeD4Zz2MI1m_lnD1JX57V9Esqm0zXb3DWbN7Gnj2GdmF_YrcvE8cy6NbIqQ=="
+  org: "primary"
+  bucket: "redux-scraper"
+
 test:
   host: "http://grafana.local:8086"
   token: "P0hzfg-vKiBTmpE-wPMJKkU8wj8VjUN1_OMxNVFhKRqxSdq3msQZ0ZWpVCT9MPAzsvoR40Caaxd2kq_LEIJpxQ=="

config/proxies.yml

@@ -21,6 +21,9 @@ production:
 development:
   <<: *default
 
+worker:
+  <<: *default
+
 test:
   direct: {}
   proxy-1: {}