fix up metrics reporting

Procfile.production

@@ -1,2 +1,3 @@
 rails: RAILS_ENV=production bundle exec rails s -b 0.0.0.0 -p 3000
 tail: tail -f log/production.log
+stats: bundle exec rake metrics:report_all

Procfile.worker

@@ -1,3 +1,4 @@
+stats: RAILS_ENV=worker bundle exec rake metrics:report_all
 cron: RAILS_ENV=worker bundle exec rake periodic_tasks
 direct: RAILS_ENV=worker bundle exec rake good_job proxy=direct
 proxy-1: RAILS_ENV=worker bundle exec rake good_job proxy=proxy-1

app/lib/metrics/delayed_job_reporter.rb

@@ -1,32 +0,0 @@
-class Metrics::DelayedJobReporter < Metrics::Reporter
-  def report_impl
-    total = Delayed::Job.count
-    by_queue = Delayed::Job.group("queue").count
-    num_working = Delayed::Job.where("locked_at is not null").count
-    num_failed = Delayed::Job.where("last_error is not null").count
-    num_pending = total - num_working - num_failed
-    by_status = {
-      "working" => num_working,
-      "failed" => num_failed,
-      "pending" => num_pending,
-    }
-
-    write_point(
-      "delayed_jobs",
-      tags: { aggregation: "total" },
-      fields: { "total" => total },
-    )
-
-    write_point(
-      "delayed_jobs",
-      tags: { aggregation: "queue" },
-      fields: by_queue,
-    )
-
-    write_point(
-      "delayed_jobs",
-      tags: { aggregation: "status" },
-      fields: by_status,
-    )
-  end
-end

app/lib/metrics/estimate_db_rows_reporter.rb

@@ -5,7 +5,7 @@ class Metrics::EstimateDbRowsReporter < Metrics::Reporter
       "http_log_entry_headers",
       "versions",
       "delayed_jobs",
-      "blob_entries",
+      "blob_entries_p",
     ]
     extra_tables_sql = extra_tables.map { |t| "'#{t}'" }.join(",")
     row_estimates = ReduxApplicationRecord.connection.exec_query(
@@ -14,6 +14,7 @@ class Metrics::EstimateDbRowsReporter < Metrics::Reporter
         "FROM pg_stat_all_tables",
         "WHERE relname IN (#{extra_tables_sql})",
         "OR relname like 'domain_%'",
+        "OR relname like 'blob_entries_p_%'",
       ].join(" ")
     ).rows.to_h
 

app/lib/metrics/good_job_reporter.rb

@@ -0,0 +1,26 @@
+class Metrics::GoodJobReporter < Metrics::Reporter
+  def report_impl
+    total = GoodJob::Job.count
+    by_queue = GoodJob::Job.group("queue_name").count
+    by_state = GoodJob::JobsFilter.new({}).states
+    logger.info "write metrics: total=#{total}, by_queue=#{by_queue}, by_state=#{by_state}"
+
+    write_point(
+      "job_queues",
+      tags: { aggregation: "total" },
+      fields: { "total" => total },
+    )
+
+    write_point(
+      "job_queues",
+      tags: { aggregation: "queue" },
+      fields: by_queue,
+    )
+
+    write_point(
+      "job_queues",
+      tags: { aggregation: "state" },
+      fields: by_state,
+    )
+  end
+end

app/lib/metrics/reporter.rb

@@ -1,7 +1,7 @@
 class Metrics::Reporter
-  REPORT = !Rails.env.test? && !Rails.env.worker?
+  include HasColorLogger
 
-  attr_reader :logger
+  REPORT = !Rails.env.test?
 
   def self.singleton
     @singleton ||= Metrics::Reporter.new
@@ -11,14 +11,16 @@ class Metrics::Reporter
   end
 
   def initialize(host: nil, token: nil, org: nil, bucket: nil, default_tags: {})
-    return unless REPORT
+    unless REPORT
+      logger.warn "!!! not reporting for this environment !!!"
+      return
+    end
 
     host ||= Rails.application.config.x.influxdb.host || raise("no host")
     token ||= Rails.application.config.x.influxdb.token || raise("no token")
     org ||= Rails.application.config.x.influxdb.org || raise("no org")
     bucket ||= Rails.application.config.x.influxdb.bucket || raise("no bucket")
 
-    @logger = ColorLogger.make($stdout, self)
     @client = InfluxDB2::Client.new(
       host, token,
       org: org,

config/influxdb.yml

@@ -1,6 +1,6 @@
 development:
-  host: "http://grafana.local:8086"
-  token: "W2ikhmpQxQHZAStgSEK6s6aJxnOqeD4Zz2MI1m_lnD1JX57V9Esqm0zXb3DWbN7Gnj2GdmF_YrcvE8cy6NbIqQ=="
+  host: "http://grafana:8086"
+  token: "cq3ytOdINIktgUynWj0tDfkpFha8iQHy0njLmpx87Rlko4bo2Gn8oxdk6dLL6y-X3jlZqXJjZ2xtpzFvk1BSMw=="
   org: "primary"
   bucket: "redux-scraper-dev"
 

rake/metrics.rake

@@ -2,8 +2,8 @@ namespace :metrics do
   desc "run reporters periodically"
   task :report_all => [:environment, :set_logger_stdout] do
     schedule = {
-      Rake::Task["metrics:delayed_job"] => 10.seconds,
-      Rake::Task["metrics:estimate_db_rows"] => 10.seconds,
+      Rake::Task["metrics:jobs"] => 60.seconds,
+      Rake::Task["metrics:estimate_db_rows"] => 60.seconds,
     }
 
     last_ran = {}
@@ -25,9 +25,9 @@ namespace :metrics do
     end
   end
 
-  desc "Report DelayedJob queue metrics"
-  task :delayed_job => :environment do
-    Metrics::DelayedJobReporter.new.report
+  desc "Report job queue metrics"
+  task :jobs => :environment do
+    Metrics::GoodJobReporter.new.report
   end
 
   desc "Report estimated db row metrics"