more progress on visual search

This commit is contained in:
Dylan Knutson
2025-03-04 14:56:42 +00:00
parent fdffd40277
commit 67de25a2c2
11 changed files with 563 additions and 3 deletions

View File

@@ -1,10 +1,22 @@
# typed: true
require "open-uri"
require "tempfile"
require "base64"
class Domain::PostsController < DomainController
extend T::Sig
extend T::Helpers
skip_before_action :authenticate_user!,
only: %i[show index user_favorite_posts user_created_posts]
only: %i[
show
index
user_favorite_posts
user_created_posts
visual_search
visual_results
]
before_action :set_post!, only: %i[show]
before_action :set_user!, only: %i[user_favorite_posts user_created_posts]
before_action :set_post_group!, only: %i[posts_in_group]
@@ -99,8 +111,141 @@ class Domain::PostsController < DomainController
render :index
end
# GET /posts/visual_search
sig(:final) { void }
def visual_search
authorize Domain::Post
end
sig { params(content_type: T.nilable(String)).returns(T::Boolean) }
def check_content_type!(content_type)
return false unless content_type
ret =
Domain::PostFileFingerprint::VALID_CONTENT_TYPES.any? do |type|
content_type.match?(type)
end
unless ret
flash.now[:error] = "The uploaded file is not a valid image format."
render :visual_search
end
ret
end
# POST /posts/visual_search
sig(:final) { void }
def visual_results
authorize Domain::Post
# Process the uploaded image or URL
image_result = process_image_input
return unless image_result
image_path, content_type = image_result
# Create thumbnail for the view if possible
@uploaded_image_data_uri = create_thumbnail(image_path, content_type)
@uploaded_hash_value = generate_fingerprint(image_path)
@post_file_fingerprints = find_similar_fingerprints(@uploaded_hash_value)
@posts = @post_file_fingerprints.map(&:post_file).compact.map(&:post)
ensure
# Clean up any temporary files
if @temp_file
@temp_file.unlink
@temp_file = nil
end
end
private
# Process the uploaded file or URL and return [image_path, content_type] or nil on failure
sig { returns(T.nilable([String, String])) }
def process_image_input
if params[:image_file].present?
process_uploaded_file
elsif params[:image_url].present?
process_image_url
else
flash.now[:error] = "Please upload an image or provide an image URL."
render :visual_search
nil
end
end
# Process an uploaded file and return [image_path, content_type] or nil on failure
sig { returns(T.nilable([String, String])) }
def process_uploaded_file
image_file = params[:image_file]
content_type = T.must(image_file.content_type)
return nil unless check_content_type!(content_type)
image_path = T.must(image_file.tempfile.path)
[image_path, content_type]
end
# Process an image URL and return [image_path, content_type] or nil on failure
sig { returns(T.nilable([String, String])) }
def process_image_url
# Download the image to a temporary file
image_url = params[:image_url]
image_io = URI.open(image_url)
if image_io.nil?
flash.now[:error] = "The URL does not point to a valid image format."
render :visual_search
return nil
end
content_type = T.must(T.unsafe(image_io).content_type)
return nil unless check_content_type!(content_type)
# Save to temp file
extension = helpers.extension_for_content_type(content_type) || "jpg"
@temp_file = Tempfile.new(["image", ".#{extension}"])
@temp_file.binmode
image_data = image_io.read
@temp_file.write(image_data)
@temp_file.close
image_path = T.must(@temp_file.path)
[image_path, content_type]
rescue StandardError => e
Rails.logger.error("Error processing image URL: #{e.message}")
flash.now[:error] = "Error downloading search image"
render :visual_search
nil
end
# Create a thumbnail from the image and return the data URI
sig do
params(image_path: String, content_type: String).returns(T.nilable(String))
end
def create_thumbnail(image_path, content_type)
helpers.create_image_thumbnail_data_uri(image_path, content_type)
end
# Generate a fingerprint from the image path
sig { params(image_path: String).returns(String) }
def generate_fingerprint(image_path)
::DHashVips::IDHash.fingerprint(image_path).to_s(2).rjust(256, "0")
end
# Find similar images based on the fingerprint
sig { params(hash_value: String).returns(ActiveRecord::Relation) }
def find_similar_fingerprints(hash_value)
Domain::PostFileFingerprint
.includes(post_file: :post)
.order(
Arel.sql(
"(hash_value <~> '#{ActiveRecord::Base.connection.quote_string(hash_value)}') ASC",
),
)
.limit(10)
end
sig { override.returns(DomainController::DomainParamConfig) }
def self.param_config
DomainController::DomainParamConfig.new(

View File

@@ -9,6 +9,7 @@ module Domain::PostsHelper
include Domain::DomainsHelper
include Domain::DomainModelHelper
include Pundit::Authorization
require "base64"
abstract!
class DomainData < T::Struct
@@ -92,6 +93,52 @@ module Domain::PostsHelper
end
end
# Create a data URI thumbnail from an image file
sig do
params(file_path: String, content_type: String, max_size: Integer).returns(
T.nilable(String),
)
end
def create_image_thumbnail_data_uri(file_path, content_type, max_size = 180)
# Load the Vips library properly instead of using require directly
begin
# Load the image
image = ::Vips::Image.new_from_file(file_path)
# Calculate the scaling factor to keep within max_size
scale = [max_size.to_f / image.width, max_size.to_f / image.height].min
# Only scale down, not up
scale = 1.0 if scale > 1.0
# Resize the image (use nearest neighbor for speed as this is just a thumbnail)
thumbnail = image.resize(scale)
# Get the image data in the original format
# For JPEG use quality 85 for a good balance of quality vs size
output_format = content_type.split("/").last
case output_format
when "jpeg", "jpg"
image_data = thumbnail.write_to_buffer(".jpg", Q: 85)
when "png"
image_data = thumbnail.write_to_buffer(".png", compression: 6)
when "gif"
image_data = thumbnail.write_to_buffer(".gif")
else
# Default to JPEG if format not recognized
image_data = thumbnail.write_to_buffer(".jpg", Q: 85)
content_type = "image/jpeg"
end
# Create the data URI
"data:#{content_type};base64,#{Base64.strict_encode64(image_data)}"
rescue => e
Rails.logger.error("Error creating thumbnail: #{e.message}")
nil
end
end
sig { params(content_type: String).returns(String) }
def pretty_content_type(content_type)
case content_type

View File

@@ -0,0 +1,60 @@
module Domain
module VisualSearchHelper
# Calculate the similarity percentage between two fingerprint hash values
# @param hash_value [String] The hash value to compare
# @param reference_hash_value [String] The reference hash value to compare against
# @return [Float] The similarity percentage between 0 and 100
def calculate_similarity_percentage(hash_value, reference_hash_value)
# Calculate hamming distance between the two hash values
distance =
hash_value
.split("")
.zip(reference_hash_value.split(""))
.map { |a, b| a.to_i ^ b.to_i }
.sum
# Maximum possible distance for a 256-bit hash
max_distance = 256
# Calculate similarity percentage based on distance
((max_distance - distance) / max_distance.to_f * 100).round(1)
end
# Determine the background color class based on similarity percentage
# @param similarity_percentage [Float] The similarity percentage between 0 and 100
# @return [String] The Tailwind CSS background color class
def match_badge_bg_color(similarity_percentage)
case similarity_percentage
when 90..100
"bg-green-600"
when 70..89
"bg-blue-600"
when 50..69
"bg-amber-500"
else
"bg-slate-700"
end
end
# Determine the text color class based on similarity percentage
# @param similarity_percentage [Float] The similarity percentage between 0 and 100
# @return [String] The Tailwind CSS text color class
def match_text_color(similarity_percentage)
case similarity_percentage
when 90..100
"text-green-700"
when 70..89
"text-blue-700"
when 50..69
"text-amber-700"
else
"text-slate-700"
end
end
# Get the CSS classes for the match percentage badge
# @param similarity_percentage [Float] The similarity percentage between 0 and 100
# @return [String] The complete CSS classes for the match percentage badge
def match_badge_classes(similarity_percentage)
"#{match_badge_bg_color(similarity_percentage)} text-white font-semibold text-xs rounded-full px-3 py-1 shadow-md"
end
end
end

View File

@@ -107,7 +107,6 @@ class BlobFile < ReduxApplicationRecord
return nil unless blob_entry
blob_file = BlobFile.find_or_initialize_from_blob_entry(blob_entry)
return nil unless blob_file.save
logger.info("migrated sha256 #{HexUtil.bin2hex(sha256)} to blob_file")
blob_file
rescue ActiveRecord::RecordNotUnique => e
raise e if retried

View File

@@ -35,4 +35,54 @@ class Domain::PostFileFingerprint < ReduxApplicationRecord
fingerprint = DHashVips::IDHash.fingerprint(path)
self.hash_value = fingerprint.to_s(2).rjust(256, "0")
end
# Calculate the Hamming distance between this fingerprint and another fingerprint
# @param other_fingerprint [Domain::PostFileFingerprint] The fingerprint to compare with
# @return [Integer, nil] The Hamming distance (number of differing bits) or nil if either fingerprint is invalid
sig do
params(other_fingerprint: T.nilable(Domain::PostFileFingerprint)).returns(
T.nilable(Integer),
)
end
def hamming_distance_to(other_fingerprint)
this_hash = hash_value
other_hash = other_fingerprint&.hash_value
return nil if this_hash.blank? || other_hash.blank?
self.class.hamming_distance(this_hash, other_hash)
end
# Calculate the Hamming distance between two hash values
# @param hash_value1 [String] The first hash value
# @param hash_value2 [String] The second hash value
# @return [Integer, nil] The Hamming distance (number of differing bits) or nil if either hash value is invalid
sig do
params(hash_value1: String, hash_value2: String).returns(T.nilable(Integer))
end
def self.hamming_distance(hash_value1, hash_value2)
hash_value1
.split("")
.zip(hash_value2.split(""))
.map { |a, b| a.to_i ^ b.to_i }
.sum
end
# Calculate the similarity percentage between this fingerprint and another fingerprint
# @param other_fingerprint [Domain::PostFileFingerprint] The fingerprint to compare with
# @return [Float, nil] The similarity percentage between 0 and 100 or nil if either fingerprint is invalid
sig do
params(other_fingerprint: T.nilable(Domain::PostFileFingerprint)).returns(
T.nilable(Float),
)
end
def similarity_percentage_to(other_fingerprint)
distance = hamming_distance_to(other_fingerprint)
return nil unless distance
# Maximum possible distance for a 256-bit hash
max_distance = 256
# Calculate similarity percentage based on distance
result = ((max_distance - distance) / max_distance.to_f * 100).round(1)
# Ensure the return type is Float
Float(result)
end
end

View File

@@ -12,6 +12,16 @@ class Domain::PostPolicy < ApplicationPolicy
true
end
sig { returns(T::Boolean) }
def visual_search?
is_role_user?
end
sig { returns(T::Boolean) }
def visual_results?
is_role_user?
end
sig { returns(T::Boolean) }
def view_file?
is_role_admin? || is_role_moderator? || is_role_user?

View File

@@ -20,6 +20,20 @@
<div class="w-full max-w-2xl mx-auto mt-4 text-center sm:mt-6">
<% index_type_header_partial = "domain/posts/index_type_headers/#{@posts_index_view_config.index_type_header}" %>
<%= render partial: index_type_header_partial, locals: { user: @user, params: params, posts: @posts } %>
<div class="mt-4 flex justify-center gap-4">
<% if params[:view] == "table" %>
<%= link_to domain_posts_path(view: "gallery"), class: "text-blue-600 hover:text-blue-800" do %>
<i class="fas fa-th-large"></i> Gallery View
<% end %>
<% else %>
<%= link_to domain_posts_path(view: "table"), class: "text-blue-600 hover:text-blue-800" do %>
<i class="fas fa-table"></i> Table View
<% end %>
<% end %>
<%= link_to visual_search_domain_posts_path, class: "text-blue-600 hover:text-blue-800" do %>
<i class="fas fa-search"></i> Visual Search
<% end %>
</div>
</div>
<% if @posts_index_view_config.show_domain_filters %>
<%= render partial: "domain_filter_controls" %>

View File

@@ -0,0 +1,83 @@
<div class="mx-auto flex flex-wrap justify-center mb-4 px-6">
<div class="text-center w-full max-w-lg mx-auto">
<h1 class="text-2xl font-bold mb-1 text-gray-900">Visual Search Results</h1>
<div class="flex justify-between items-center mt-2 mb-4">
<span class="text-gray-600 text-sm font-medium">
<% if @post_file_fingerprints.any? %>
<span class="bg-blue-100 text-blue-700 px-2 py-0.5 rounded-md font-semibold">
<%= @post_file_fingerprints.size %>
</span> similar images found
<% else %>
No matches found
<% end %>
</span>
<%= link_to "New Search", visual_search_domain_posts_path, class: "text-white hover:text-white transition-colors duration-200 text-sm font-semibold bg-blue-600 hover:bg-blue-700 px-3 py-1.5 rounded-md shadow-sm" %>
</div>
<% if @uploaded_image_data_uri.present? %>
<div class="mx-auto mb-6 bg-white p-3 rounded-lg border border-gray-300 shadow-sm text-center">
<h3 class="text-sm font-medium text-gray-700 mb-2">Your search image:</h3>
<div class="flex justify-center items-center">
<img src="<%= @uploaded_image_data_uri %>" alt="Uploaded image" class="max-h-[180px] rounded-lg border border-gray-300 object-contain shadow-sm" />
</div>
</div>
<% end %>
<% if @post_file_fingerprints.empty? %>
<div class="bg-blue-50 text-blue-800 p-4 rounded-lg border border-blue-300 shadow-sm mx-auto max-w-lg">
<p class="mb-1 font-medium">No similar images found</p>
<p class="text-sm text-blue-700">Try adjusting the similarity threshold to a higher value.</p>
</div>
<% end %>
</div>
<% if @post_file_fingerprints.any? %>
<div class="mx-2">
<div class="flex flex-wrap gap-3 justify-center">
<% @post_file_fingerprints.each do |post_file_fingerprint| %>
<% post_file = post_file_fingerprint.post_file %>
<% post = post_file.post %>
<% similarity_percentage = calculate_similarity_percentage(post_file_fingerprint.hash_value, @uploaded_hash_value) %>
<div class="flex flex-col h-fit rounded-md border border-gray-300 bg-white shadow hover:shadow-md transition-shadow duration-300 overflow-hidden w-full sm:w-[calc(50%-0.75rem)] md:w-[calc(33.333%-0.75rem)] lg:w-[calc(25%-0.75rem)]">
<div class="flex justify-between items-center border-b border-gray-200 p-2 bg-gray-50">
<div class="flex items-center">
<%= render "domain/posts/inline_postable_domain_link", post: post %>
</div>
<div class="<%= match_badge_classes(similarity_percentage) %> ml-auto rounded-full font-bold text-xs px-2 py-1 shadow-sm">
<%= similarity_percentage %>%
</div>
</div>
<% if thumbnail = thumbnail_for_post_path(post) %>
<div class="flex items-center justify-center p-2 border-b border-gray-200 bg-white w-full">
<%= link_to domain_post_path(post), class: "transition-transform duration-300 " do %>
<%= image_tag thumbnail,
class: "rounded-lg border max-h-[180px] max-w-[180px] border-gray-300 object-contain shadow-sm",
alt: post.title.presence || "Untitled Post" %>
<% end %>
</div>
<% end %>
<div class="flex flex-col justify-between">
<h2 class="p-2 text-center text-base font-medium text-gray-800 truncate">
<%= link_to post.title.presence || "Untitled Post", domain_post_path(post), class: "text-blue-700 hover:text-blue-800 transition-colors duration-200" %>
</h2>
<div class="px-2 pb-2 text-xs text-gray-600 mt-auto">
<div class="flex justify-between items-center flex-wrap gap-1">
<span class="flex items-center gap-0.5">
<% if creator = post.primary_creator_for_view %>
<span class="text-gray-500 italic">by</span>
<%= render "domain/has_description_html/inline_link_domain_user", user: creator, visual_style: "sky-link", size: "small" %>
<% elsif creator = post.primary_creator_name_fallback_for_view %>
<span class="text-gray-500 italic">by</span> <%= creator %>
<% end %>
</span>
<span class="font-medium">
<% if post.created_at %>
<span class="text-gray-500"><i class="far fa-clock mr-1"></i></span> <%= time_ago_in_words(post.created_at) %> ago
<% end %>
</span>
</div>
</div>
</div>
</div>
<% end %>
</div>
</div>
<% end %>
</div>

View File

@@ -0,0 +1,118 @@
<div class="mx-auto mt-4 w-full max-w-2xl flex flex-col gap-4 pb-4">
<div class="text-center">
<h1 class="text-2xl font-semibold mb-1">Visual Search</h1>
<p class="text-slate-600">Find images similar to the one you provide.</p>
</div>
<% if flash[:error] %>
<div class="bg-red-50 text-red-700 p-4 rounded-md border border-red-300">
<%= flash[:error] %>
</div>
<% end %>
<div class="bg-white rounded-lg border border-slate-300 shadow-sm overflow-hidden">
<div class="p-4 sm:p-6">
<%= form_with url: visual_results_domain_posts_path, method: :post, multipart: true, class: "flex flex-col gap-4" do |form| %>
<div id="drag-drop-area" class="border-2 border-dashed border-slate-300 rounded-lg p-6 text-center mb-4 transition-colors duration-200">
<div class="flex flex-col items-center justify-center gap-2">
<svg xmlns="http://www.w3.org/2000/svg" class="h-10 w-10 text-slate-400" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z" />
</svg>
<p class="text-slate-600 font-medium">Drag and drop an image here</p>
<p class="text-xs text-slate-500">or use one of the options below</p>
</div>
</div>
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
<div class="flex flex-col gap-2">
<h3 class="text-lg font-medium">Option 1: Upload an image</h3>
<div class="flex flex-col gap-1">
<%= form.label :image_file, "Choose an image file", class: "text-sm font-medium text-slate-700" %>
<%= form.file_field :image_file, accept: "image/png,image/jpeg,image/jpg,image/gif,image/webp", class: "w-full rounded-md border-slate-300 text-sm", id: "image-file-input" %>
<p class="text-xs text-slate-500">Supported formats: JPG, PNG, GIF, WebP</p>
</div>
</div>
<div class="flex flex-col gap-2">
<h3 class="text-lg font-medium">Option 2: Provide image URL</h3>
<div class="flex flex-col gap-1">
<%= form.label :image_url, "Image URL", class: "text-sm font-medium text-slate-700" %>
<%= form.url_field :image_url, class: "w-full rounded-md border-slate-300 text-sm", placeholder: "https://example.com/image.jpg" %>
<p class="text-xs text-slate-500">Enter the direct URL to an image</p>
</div>
</div>
</div>
<div class="border-t border-slate-200 my-4"></div>
<div class="mt-4">
<%= form.submit "Search for Similar Images", class: "w-full bg-blue-600 hover:bg-blue-700 text-white py-2 px-4 rounded-md text-sm font-medium transition-colors" %>
</div>
<% end %>
</div>
</div>
</div>
<script>
document.addEventListener('DOMContentLoaded', function() {
const dragDropArea = document.getElementById('drag-drop-area');
const fileInput = document.getElementById('image-file-input');
// Prevent default drag behaviors
['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => {
dragDropArea.addEventListener(eventName, preventDefaults, false);
document.body.addEventListener(eventName, preventDefaults, false);
});
// Highlight drop area when item is dragged over it
['dragenter', 'dragover'].forEach(eventName => {
dragDropArea.addEventListener(eventName, highlight, false);
});
// Remove highlight when item is dragged out or dropped
['dragleave', 'drop'].forEach(eventName => {
dragDropArea.addEventListener(eventName, unhighlight, false);
});
// Handle dropped files
dragDropArea.addEventListener('drop', handleDrop, false);
function preventDefaults(e) {
e.preventDefault();
e.stopPropagation();
}
function highlight() {
dragDropArea.classList.add('border-blue-500');
dragDropArea.classList.add('bg-blue-50');
}
function unhighlight() {
dragDropArea.classList.remove('border-blue-500');
dragDropArea.classList.remove('bg-blue-50');
}
function handleDrop(e) {
const dt = e.dataTransfer;
const files = dt.files;
if (files.length) {
// Check if the dropped file is an image
const file = files[0];
if (file.type.match('image.*')) {
// Update the file input with the dropped file
fileInput.files = files;
// Show file name as feedback
const fileName = document.createElement('p');
fileName.textContent = `Selected: ${file.name}`;
fileName.className = 'text-sm text-blue-600 mt-2';
// Remove any previous file name
const previousFileName = dragDropArea.querySelector('.text-blue-600');
if (previousFileName) {
previousFileName.remove();
}
dragDropArea.appendChild(fileName);
} else {
// Alert if file is not an image
alert('Please drop an image file.');
}
}
}
});
</script>

View File

@@ -40,6 +40,14 @@ Rails.application.routes.draw do
as: :domain_posts,
only: %i[index show],
controller: "domain/posts" do
collection do
# show the search page to find similar posts
get :visual_search
# display visually similar posts results
post :visual_search,
as: :visual_results,
to: "domain/posts#visual_results"
end
resources :users, only: %i[], controller: "domain/users", path: "" do
get :faved_by, on: :collection, action: :users_faving_post
end

View File

@@ -10,7 +10,7 @@ namespace :fingerprint do
end
end
task for_ids: :environment do
task for_posts: :environment do
ids = ENV["IDS"].split(",")
pb = ProgressBar.create(total: ids.size)
ids.each do |id|
@@ -19,4 +19,30 @@ namespace :fingerprint do
pb.progress = [pb.progress + 1, pb.total].min
end
end
task for_users: :environment do
ids = ENV["IDS"].split(",")
ids.each do |id|
user = DomainController.find_model_from_param(Domain::User, id)
puts "migrating #{id}: #{user.posts.count} posts"
pb =
ProgressBar.create(
total: user.posts.count,
format: "%t: %c/%C %B %p%% %a %e",
)
user
.posts
.includes(:files)
.find_in_batches(batch_size: 10) do |batch|
ReduxApplicationRecord.transaction do
batch.each do |post|
post.files.each do |file|
file.create_fingerprint unless file.fingerprint
end
pb.progress = [pb.progress + 1, pb.total].min
end
end
end
end
end
end