db sampler script for creating realistic development environment

Rakefile

@@ -74,6 +74,25 @@ namespace :blob_entries do
   end
 end
 
+namespace :db_sampler do
+  task :export => :environment do
+    url_names = ENV["url_names"] || raise("need 'url_names' (comma-separated)")
+    outfile_path = ENV["outfile"] || raise("need 'outfile' (file path)")
+    outfile = File.open(outfile_path, "wb")
+    DbSampler.new(outfile).export(url_names.split(","))
+  ensure
+    outfile.close if outfile
+  end
+
+  task :import => [:environment] do
+    infile_path = ENV["infile"] || raise("need 'infile' (file path)")
+    infile = File.open(infile_path, "rb")
+    DbSampler.new(infile).import
+  ensure
+    infile.close if infile
+  end
+end
+
 task :good_job do
   proxies = ["direct", "proxy-1", "dedipath-1", "serverhost-1"]
   proxy = ENV["proxy"]

app/lib/db_sampler.rb

@@ -0,0 +1,121 @@
+class DbSampler
+  include HasColorLogger
+  include HasMeasureDuration
+
+  SCHEMA = {
+    ::Domain::Fa::User => [
+      :avatar,
+      :posts,
+      :disco,
+      :follower_joins,
+      :fav_post_joins,
+    ],
+    ::Domain::Fa::UserAvatar => [:file, :log_entry],
+    ::Domain::Fa::Post => [:file, :creator],
+    ::Domain::Fa::Follow => [:follower, :followed],
+    ::Domain::Fa::Fav => [:user, :post],
+    ::Domain::Fa::UserFactor => [],
+    ::BlobEntryP => [:base],
+    ::HttpLogEntry => [
+      :request_headers,
+      :response_headers,
+      :response,
+      :caused_by_entry,
+    ],
+    ::HttpLogEntryHeader => [],
+  }
+
+  def initialize(file)
+    @file = file
+    @handled = Set.new
+  end
+
+  def export(url_names)
+    Domain::Fa::User.includes({
+      avatar: [:file, :log_entry],
+      follower_joins: [:follower, :followed],
+      posts: { file: :response },
+    }).where(url_name: url_names).each do |user|
+      handle_model(user, 0, 0)
+    end
+  end
+
+  def import
+    puts "reading file..."
+    deferred = []
+    while (line = @file.gets)
+      line.chomp!
+      model = Marshal.load(Base64.strict_decode64(line))
+
+      begin
+        import_model(model)
+      rescue ActiveRecord::InvalidForeignKey
+        puts("defer #{model_id(model)}")
+        deferred.push(model)
+      end
+    end
+
+    deferred.each do |model|
+      import_model(model)
+    end
+  end
+
+  private
+
+  def model_id(model)
+    name = model.class.name
+    pk = model.class.primary_key.to_sym
+    id = model.send(pk)
+    id_fmt = id.to_s
+    id_fmt = HexUtil.bin2hex(id) if pk == :sha256
+    id_fmt = "#{name} / #{id_fmt}"
+  end
+
+  def import_model(model)
+    pk = model.class.primary_key.to_sym
+    id = model.send(pk)
+    exists = model.class.exists?(pk => id)
+
+    if exists
+      puts("skipped existing #{model_id(model)}")
+    else
+      model2 = model.class.new
+      model.attribute_names.map(&:to_sym).each do |attr|
+        model2.write_attribute(attr, model.read_attribute(attr))
+      end
+      model2.save(validate: false)
+      puts("imported #{model_id(model)}")
+    end
+  end
+
+  def handle_model(model, level, user_depth)
+    return unless model
+    is_user = model.is_a?(Domain::Fa::User)
+    user_depth += 1 if is_user
+    return unless @handled.add?(model)
+
+    unless user_depth > 1 && is_user
+      assocs = SCHEMA[model.class] || raise("invalid: #{model.class.name}")
+      assocs.each do |assoc|
+        model2 = model.send(assoc)
+        next unless model2
+        if model2.respond_to? :each
+          model2.each do |model3|
+            handle_model(model3, level + 1, user_depth)
+          end
+        else
+          handle_model(model2, level + 1, user_depth)
+        end
+      end
+    end
+
+    dump(model, level)
+  end
+
+  def dump(model, level)
+    @file.puts(Base64.strict_encode64(Marshal.dump(model)))
+    id = model.send(model.class.primary_key)
+    id = HexUtil.bin2hex(id) if model.class.primary_key == "sha256"
+    puts ("-" * level) + " dumped #{model.class.name}/#{id}"
+  end
+end

app/views/log_entries/show.html.erb

@@ -98,7 +98,11 @@
       <% base_hle = HttpLogEntry.find_by(response: @log_entry.response.base) %>
       <td>
         <table>
-          <%= render partial: "log_entry_table_row_mini", locals: { entry: base_hle } %>
+          <% if base_hle %>
+            <%= render partial: "log_entry_table_row_mini", locals: { entry: base_hle } %>
+          <% else %>
+            <tr><td><i>HLE not found...</i></td></tr>
+          <% end %>
         </table>
       </td>
     </tr>