dymk/redux-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

visual search fixes

Browse Source
This commit is contained in:
Dylan Knutson
2025-08-14 19:29:28 +00:00
parent 2a8d631b29
commit 62f14d10d4
6 changed files with 180 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/controllers/domain/posts_controller.rb
View File

@@ -172,7 +172,9 @@ class Domain::PostsController < DomainController
before = Time.now
similar_fingerprints =
helpers.find_similar_fingerprints(thumbs_and_fingerprints).take(10)
helpers.find_similar_fingerprints(
thumbs_and_fingerprints.map(&:to_fingerprint_and_detail),
).take(10)
@time_taken = Time.now - before

20
app/helpers/domain/visual_search_helper.rb
View File

@@ -69,14 +69,21 @@ module Domain
end
class SimilarFingerprintResult < T::Struct
include T::Struct::ActsAsComparable
const :fingerprint, Domain::PostFile::BitFingerprint
const :similarity_percentage, Float
end
class FingerprintAndDetail < T::Struct
include T::Struct::ActsAsComparable
const :fingerprint, String
const :detail_fingerprint, String
end
# Find similar images based on the fingerprint
sig do
params(
fingerprints: T::Array[GenerateFingerprintsResult],
fingerprints: T::Array[FingerprintAndDetail],
limit: Integer,
oversearch: Integer,
includes: T.untyped,
@@ -124,9 +131,20 @@ module Domain
end
class GenerateFingerprintsResult < T::Struct
extend T::Sig
include T::Struct::ActsAsComparable
const :thumb_path, String
const :fingerprint, String
const :detail_fingerprint, String
sig { returns(FingerprintAndDetail) }
def to_fingerprint_and_detail
FingerprintAndDetail.new(
fingerprint: fingerprint,
detail_fingerprint: detail_fingerprint,
)
end
end
# Generate a fingerprint from the image path

190
app/javascript/bundles/Main/components/VisualSearchForm.tsx
View File

@@ -15,6 +15,8 @@ interface ImageState {
file: File;
previewUrl: string;
originalFileSize: number | null;
thumbnailFile?: File; // For video thumbnails
isVideo?: boolean;
}
const ACCEPTED_IMAGE_TYPES = [
@@ -23,10 +25,11 @@ const ACCEPTED_IMAGE_TYPES = [
'image/jpg',
'image/gif',
'image/webp',
'video/mp4',
];
const ACCEPTED_EXTENSIONS =
'image/png,image/jpeg,image/jpg,image/gif,image/webp';
'image/png,image/jpeg,image/jpg,image/gif,image/webp,video/mp4';
// Feedback Message Component
interface FeedbackMessageProps {
@@ -126,23 +129,98 @@ async function resizeImageIfNeeded(file: File): Promise<File> {
});
}
// Helper function to generate thumbnail from video file
async function generateVideoThumbnail(file: File): Promise<File> {
return new Promise((resolve) => {
const video = document.createElement('video');
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d')!;
video.onloadedmetadata = () => {
// Set canvas dimensions to match video
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
// Seek to 1 second into the video (or 10% through if shorter)
const seekTime = Math.min(1, video.duration * 0.1);
video.currentTime = seekTime;
};
video.onseeked = () => {
// Draw the current frame to canvas
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
// Convert to blob as JPEG
canvas.toBlob(
(blob) => {
if (blob) {
// Create a new file with thumbnail data but keep original video file name base
const thumbnailName =
file.name.replace(/\.[^/.]+$/, '') + '_thumbnail.jpg';
const thumbnailFile = new File([blob], thumbnailName, {
type: 'image/jpeg',
lastModified: Date.now(),
});
resolve(thumbnailFile);
} else {
resolve(file); // Fallback to original file
}
},
'image/jpeg',
0.8, // Quality setting (80%)
);
};
video.onerror = () => {
resolve(file); // Fallback to original file if video processing fails
};
// Load the video file
video.src = URL.createObjectURL(file);
video.load();
});
}
function ImagePreview({ imageState, onRemove }: ImagePreviewProps) {
const isVideo = imageState.isVideo;
return (
<div className="flex items-center gap-4">
<img
src={imageState.previewUrl}
alt="Selected image thumbnail"
className="max-h-32 max-w-32 flex-shrink-0 rounded-md object-cover shadow-sm"
/>
<div className="relative max-h-32 max-w-32 flex-shrink-0">
<img
src={imageState.previewUrl}
alt={isVideo ? 'Video thumbnail' : 'Selected image thumbnail'}
className="max-h-32 max-w-32 rounded-md object-cover shadow-sm"
/>
{isVideo && (
<div className="absolute inset-0 flex items-center justify-center">
<div className="rounded-full bg-black bg-opacity-70 p-2">
<svg
className="h-4 w-4 text-white"
fill="currentColor"
viewBox="0 0 20 20"
>
<path d="M6.3 2.841A1.5 1.5 0 004 4.11V15.89a1.5 1.5 0 002.3 1.269l9.344-5.89a1.5 1.5 0 000-2.538L6.3 2.84z" />
</svg>
</div>
</div>
)}
</div>
<div className="flex min-w-0 flex-1 flex-col justify-center gap-1">
<h3 className="text-sm font-medium text-green-700">Selected Image</h3>
<h3 className="text-sm font-medium text-green-700">
{isVideo ? 'Selected Video' : 'Selected Image'}
</h3>
<p
className="max-w-32 truncate text-xs text-green-600"
title={imageState.file.name}
>
{imageState.file.name}
</p>
{imageState.originalFileSize ? (
{isVideo ? (
<p className="text-xs text-slate-500">
{formatFileSize(imageState.file.size)} (thumbnail generated)
</p>
) : imageState.originalFileSize ? (
<div className="text-xs text-slate-500">
<div>Original: {formatFileSize(imageState.originalFileSize)}</div>
<div>Resized: {formatFileSize(imageState.file.size)}</div>
@@ -159,9 +237,9 @@ function ImagePreview({ imageState, onRemove }: ImagePreviewProps) {
onRemove();
}}
className="mt-1 self-start rounded bg-slate-600 px-2 py-1 text-xs font-medium text-slate-100 transition-colors hover:bg-red-600 focus:bg-red-600 focus:outline-none"
title="Clear image"
title={isVideo ? 'Clear video' : 'Clear image'}
>
Remove Image
{isVideo ? 'Remove Video' : 'Remove Image'}
</button>
</div>
</div>
@@ -191,10 +269,10 @@ function EmptyDropZone({ isMobile }: EmptyDropZoneProps) {
/>
</svg>
<p className="hidden font-medium text-slate-600 sm:block">
Drag and drop an image here
Drag and drop an image or video here
</p>
<p className="block font-medium text-slate-600 sm:hidden">
tap here to paste an image from the clipboard
tap here to paste an image or video from the clipboard
</p>
<p className="text-xs text-slate-500">or use one of the options below</p>
</div>
@@ -284,13 +362,15 @@ function FileUploadSection({
}: FileUploadSectionProps) {
return (
<div className="flex flex-1 flex-col items-center justify-center">
<h3 className="text-lg font-medium text-slate-500">Upload an image</h3>
<h3 className="text-lg font-medium text-slate-500">
Upload an image or video
</h3>
<div className="flex flex-col gap-1">
<label
htmlFor="image-file-input"
className="text-sm font-medium text-slate-700"
>
Choose an image file
Choose an image or video file
</label>
<input
ref={fileInputRef}
@@ -305,7 +385,7 @@ function FileUploadSection({
}}
/>
<p className="text-xs text-slate-500">
Supported formats: JPG, PNG, GIF, WebP
Supported formats: JPG, PNG, GIF, WebP, MP4
</p>
</div>
</div>
@@ -321,13 +401,15 @@ interface UrlUploadSectionProps {
function UrlUploadSection({ imageUrl, onUrlChange }: UrlUploadSectionProps) {
return (
<div className="flex flex-1 flex-col items-center justify-center">
<h3 className="text-lg font-medium text-slate-500">Provide image URL</h3>
<h3 className="text-lg font-medium text-slate-500">
Provide image or video URL
</h3>
<div className="flex flex-col gap-1">
<label
htmlFor="image-url-input"
className="text-sm font-medium text-slate-700"
>
Image URL
Image or Video URL
</label>
<input
id="image-url-input"
@@ -336,10 +418,10 @@ function UrlUploadSection({ imageUrl, onUrlChange }: UrlUploadSectionProps) {
value={imageUrl}
onChange={(e) => onUrlChange(e.target.value)}
className="w-full rounded-md border-slate-300 text-sm"
placeholder="https://example.com/image.jpg"
placeholder="https://example.com/image.jpg or https://example.com/video.mp4"
/>
<p className="text-xs text-slate-500">
Enter the direct URL to an image
Enter the direct URL to an image or video
</p>
</div>
</div>
@@ -460,7 +542,7 @@ export default function VisualSearchForm({
async (file: File) => {
if (!ACCEPTED_IMAGE_TYPES.includes(file.type)) {
showFeedback(
'Please select a valid image file (JPG, PNG, GIF, WebP)',
'Please select a valid image or video file (JPG, PNG, GIF, WebP, MP4)',
'error',
);
return;
@@ -471,34 +553,57 @@ export default function VisualSearchForm({
URL.revokeObjectURL(imageState.previewUrl);
}
// Show processing message for large files
// Show processing message for large files or videos
const originalSize = file.size;
const isLargeFile = originalSize > 2 * 1024 * 1024; // 2MB
const isVideo = file.type.startsWith('video/');
if (isLargeFile) {
showFeedback('Processing large image...', 'warning');
if (isLargeFile || isVideo) {
showFeedback(
isVideo
? 'Generating video thumbnail...'
: 'Processing large image...',
'warning',
);
}
try {
// Resize image if needed
const processedFile = await resizeImageIfNeeded(file);
let processedFile: File;
let thumbnailFile: File | undefined;
let previewUrl: string;
if (isVideo) {
// For video files, generate thumbnail for preview but keep original for upload
thumbnailFile = await generateVideoThumbnail(file);
processedFile = file; // Keep original video for upload
previewUrl = URL.createObjectURL(thumbnailFile);
// Set the original video file in the file input for form submission
const dataTransfer = new DataTransfer();
dataTransfer.items.add(file);
fileInputRef.current!.files = dataTransfer.files;
} else {
// For image files, process as before
processedFile = await resizeImageIfNeeded(file);
previewUrl = URL.createObjectURL(processedFile);
const dataTransfer = new DataTransfer();
dataTransfer.items.add(processedFile);
fileInputRef.current!.files = dataTransfer.files;
}
clearFeedback();
const dataTransfer = new DataTransfer();
dataTransfer.items.add(processedFile);
fileInputRef.current!.files = dataTransfer.files;
// Track original size if image was resized
const wasResized = processedFile.size < originalSize;
// Create preview URL for the thumbnail
const previewUrl = URL.createObjectURL(processedFile);
const wasResized = !isVideo && processedFile.size < originalSize;
// Set all image state at once
setImageState({
file: processedFile,
previewUrl,
originalFileSize: wasResized ? originalSize : null,
thumbnailFile,
isVideo,
});
// Visual feedback
@@ -506,7 +611,9 @@ export default function VisualSearchForm({
setTimeout(() => setIsDragOver(false), 1000);
} catch (error) {
showFeedback(
'Error processing image. Please try another file.',
isVideo
? 'Error processing video. Please try another file.'
: 'Error processing image. Please try another file.',
'error',
);
}
@@ -553,10 +660,13 @@ export default function VisualSearchForm({
const files = e.dataTransfer.files;
if (files.length > 0) {
const file = files[0];
if (file.type.match('image.*')) {
if (file.type.match('image.*') || file.type.match('video.*')) {
await handleImageFile(file);
} else {
showFeedback(`Please drop an image file (got ${file.type})`, 'error');
showFeedback(
`Please drop an image or video file (got ${file.type})`,
'error',
);
}
}
},
@@ -585,7 +695,7 @@ export default function VisualSearchForm({
}
showFeedback(
'No image found in clipboard. Copy an image first, then try again.',
'No image or video found in clipboard. Copy an image or video first, then try again.',
'warning',
);
return false;
@@ -621,6 +731,10 @@ export default function VisualSearchForm({
imageFile = item.getAsFile();
break;
}
if (item.type.indexOf('video') !== -1) {
imageFile = item.getAsFile();
break;
}
}
if (imageFile) {
@@ -629,7 +743,7 @@ export default function VisualSearchForm({
dragDropRef.current?.focus();
} else {
showFeedback(
'No image found in clipboard. Copy an image first, then paste here.',
'No image or video found in clipboard. Copy an image or video first, then paste here.',
'warning',
);
}

2
app/lib/tasks/telegram_bot_task.rb
View File

@@ -236,7 +236,7 @@ module Tasks
search_stopwatch = Stopwatch.start
similar_results =
find_similar_fingerprints(
fingerprints,
fingerprints.map(&:to_fingerprint_and_detail),
limit: 10,
oversearch: 3,
includes: {

6
app/views/domain/posts/default/_section_visualy_similar_posts.html.erb
View File

@@ -7,8 +7,10 @@
<% fprint_value = fprint&.fingerprint_value %>
<% fprint_detail_value = fprint&.fingerprint_detail_value %>
<% fprints = fprint && fprint_value && fprint_detail_value && find_similar_fingerprints(
fingerprint_value: fprint_value,
fingerprint_detail_value: fprint_detail_value,
[Domain::VisualSearchHelper::FingerprintAndDetail.new(
fingerprint: fprint_value,
detail_fingerprint: fprint_detail_value,
)],
limit: 5,
oversearch: 5,
includes: {post_file: :post},

2
spec/controllers/domain/posts_controller_spec.rb
View File

@@ -161,7 +161,7 @@ RSpec.describe Domain::PostsController, type: :controller do
# Mock the similar fingerprints search
expect(controller.helpers).to receive(:find_similar_fingerprints).with(
mock_generate_fingerprints_results,
mock_generate_fingerprints_results.map(&:to_fingerprint_and_detail),
).and_return(mock_similar_fingerprints)
# Mock the thumbnail data URI creation