From 7fb1dfa95af47ab866e5287e59d1857773de6fbc Mon Sep 17 00:00:00 2001 From: imeepos Date: Thu, 17 Jul 2025 15:29:59 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E4=BE=BF=E6=8D=B7?= =?UTF-8?q?=E5=B0=8F=E5=B7=A5=E5=85=B7=E9=A1=B5=E9=9D=A2=20-=20AI=E6=A3=80?= =?UTF-8?q?=E7=B4=A2=E5=9B=BE=E7=89=87/=E6=95=B0=E6=8D=AE=E6=B8=85?= =?UTF-8?q?=E6=B4=97=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增便捷工具页面 (/tools)，提供AI检索图片/数据清洗功能 - 支持JSONL格式数据的URI匹配去重处理 - 实现实时进度显示和批量数据处理 - 添加完整的错误处理和用户反馈机制 - 遵循Tauri开发规范和UI/UX设计标准 - 包含测试数据和功能文档 技术实现： - 后端: Rust异步处理，流式文件读取，进度事件发送 - 前端: React + TypeScript，文件选择对话框，进度条显示 - 导航: 新增便捷工具菜单项，集成到主导航栏 --- apps/desktop/TOOLS_FEATURE.md | 100 +++++ apps/desktop/src-tauri/src/lib.rs | 4 +- .../src/presentation/commands/mod.rs | 1 + .../presentation/commands/tools_commands.rs | 222 +++++++++++ apps/desktop/src/App.tsx | 2 + apps/desktop/src/components/Navigation.tsx | 9 +- apps/desktop/src/pages/Tools.tsx | 354 ++++++++++++++++++ apps/desktop/test_data/sample_all_data.jsonl | 5 + .../test_data/sample_remove_data.jsonl | 2 + 9 files changed, 697 insertions(+), 2 deletions(-) create mode 100644 apps/desktop/TOOLS_FEATURE.md create mode 100644 apps/desktop/src-tauri/src/presentation/commands/tools_commands.rs create mode 100644 apps/desktop/src/pages/Tools.tsx create mode 100644 apps/desktop/test_data/sample_all_data.jsonl create mode 100644 apps/desktop/test_data/sample_remove_data.jsonl diff --git a/apps/desktop/TOOLS_FEATURE.md b/apps/desktop/TOOLS_FEATURE.md new file mode 100644 index 0000000..8450dde --- /dev/null +++ b/apps/desktop/TOOLS_FEATURE.md @@ -0,0 +1,100 @@ +# 便捷小工具功能 - v0.1.33 + +## 功能概述 + +新增便捷小工具页面，提供AI检索图片/数据清洗功能，支持JSONL格式数据的去重处理。 + +## 功能特性 + +### AI检索图片/数据清洗工具 + +- **文件格式支持**: JSONL (JSON Lines) 格式 +- **数据去重**: 基于URI字段进行精确匹配去重 +- **进度显示**: 实时显示处理进度和状态 +- **批量处理**: 支持大数据量文件处理 +- **结果统计**: 显示原始数据量、去除数量、最终结果数量 + +## 使用方法 + +1. **选择全部数据文件**: 包含所有数据的JSONL文件 +2. **选择要去除的数据文件**: 包含需要去除数据的JSONL文件 +3. **选择输出文件**: 指定处理结果的保存位置 +4. **开始处理**: 点击"开始处理"按钮执行数据清洗 + +## 数据格式说明 + +### 全部数据文件格式 +```json +{ + "kind": "storage#object", + "id": "...", + "uri": "gs://bucket/path/to/file.jpg", + // 其他字段... +} +``` + +### 要去除的数据文件格式 +```json +{ + "id": "uuid", + "schema_id": "default_schema", + "json_data": "{\"uri\":\"gs://bucket/path/to/file.jpg\", ...}" +} +``` + +## 处理逻辑 + +1. **解析要去除的数据**: 从`json_data`字段中提取`uri`值 +2. **构建URI集合**: 将所有要去除的URI存储在HashSet中 +3. **过滤全部数据**: 遍历全部数据文件，过滤掉URI匹配的记录 +4. **保存结果**: 将过滤后的数据保存到输出文件 + +## 技术实现 + +### 后端 (Rust) +- **命令**: `clean_jsonl_data` +- **文件**: `src-tauri/src/presentation/commands/tools_commands.rs` +- **特性**: + - 异步处理 + - 进度事件发送 + - 内存优化的流式处理 + - 错误处理和恢复 + +### 前端 (React + TypeScript) +- **页面**: `src/pages/Tools.tsx` +- **路由**: `/tools` +- **特性**: + - 文件选择对话框 + - 实时进度显示 + - 结果统计展示 + - 错误处理和用户反馈 + +## 导航集成 + +在主导航栏中新增"便捷工具"菜单项，使用扳手图标，提供快速访问入口。 + +## 测试数据 + +项目包含测试数据文件： +- `test_data/sample_all_data.jsonl`: 示例全部数据文件 +- `test_data/sample_remove_data.jsonl`: 示例要去除的数据文件 + +## 性能优化 + +- **流式处理**: 逐行读取文件，避免内存溢出 +- **进度更新**: 每处理100行更新一次进度，平衡性能和用户体验 +- **错误容错**: 解析失败的行会被跳过并记录警告 + +## 开发规范遵循 + +- 遵循Tauri开发规范的四层架构设计 +- 使用TypeScript确保类型安全 +- 遵循UI/UX设计标准，提供优雅的用户界面 +- 实现完整的错误处理和用户反馈机制 + +## 未来扩展 + +- 支持更多数据格式 (CSV, JSON等) +- 添加更多数据清洗规则 +- 支持自定义匹配字段 +- 添加数据预览功能 diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 137b7ae..9c40492 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -253,7 +253,9 @@ pub fn run() { commands::test_commands::test_logging, // 调试命令 commands::debug_commands::test_parse_draft_file, - commands::debug_commands::validate_template_structure + commands::debug_commands::validate_template_structure, + // 便捷工具命令 + commands::tools_commands::clean_jsonl_data ]) .setup(|app| { // 初始化日志系统 diff --git a/apps/desktop/src-tauri/src/presentation/commands/mod.rs b/apps/desktop/src-tauri/src/presentation/commands/mod.rs index d914261..67618b5 100644 --- a/apps/desktop/src-tauri/src/presentation/commands/mod.rs +++ b/apps/desktop/src-tauri/src/presentation/commands/mod.rs @@ -16,3 +16,4 @@ pub mod material_matching_commands; pub mod template_matching_result_commands; pub mod export_record_commands; pub mod video_generation_commands; +pub mod tools_commands; diff --git a/apps/desktop/src-tauri/src/presentation/commands/tools_commands.rs b/apps/desktop/src-tauri/src/presentation/commands/tools_commands.rs new file mode 100644 index 0000000..621396d --- /dev/null +++ b/apps/desktop/src-tauri/src/presentation/commands/tools_commands.rs @@ -0,0 +1,222 @@ +use tauri::{command, Emitter}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::fs::File; +use std::io::{BufRead, BufReader, Write}; +use std::path::Path; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DataCleaningProgress { + pub current: usize, + pub total: usize, + pub percentage: f64, + pub status: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DataCleaningResult { + pub success: bool, + pub message: String, + pub original_count: usize, + pub removed_count: usize, + pub final_count: usize, + pub output_file: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct StorageObject { + pub uri: String, + // 其他字段可以忽略，我们只关心 uri +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ProcessedData { + pub json_data: String, +} + +/// AI检索图片/数据清洗 - JSONL格式处理 +#[command] +pub async fn clean_jsonl_data( + all_data_file: String, + remove_data_file: String, + output_file: String, + window: tauri::Window, +) -> Result { + // 验证文件存在 + if !Path::new(&all_data_file).exists() { + return Err(format!("全部数据文件不存在: {}", all_data_file)); + } + + if !Path::new(&remove_data_file).exists() { + return Err(format!("要去除的数据文件不存在: {}", remove_data_file)); + } + + // 发送开始处理的进度 + let _ = window.emit("data-cleaning-progress", DataCleaningProgress { + current: 0, + total: 0, + percentage: 0.0, + status: "开始处理...".to_string(), + }); + + // 第一步：读取要去除的数据文件，提取所有URI + let _ = window.emit("data-cleaning-progress", DataCleaningProgress { + current: 0, + total: 0, + percentage: 10.0, + status: "读取要去除的数据文件...".to_string(), + }); + + let remove_uris = match extract_remove_uris(&remove_data_file).await { + Ok(uris) => uris, + Err(e) => return Err(format!("读取要去除的数据文件失败: {}", e)), + }; + + let _ = window.emit("data-cleaning-progress", DataCleaningProgress { + current: 0, + total: 0, + percentage: 30.0, + status: format!("找到 {} 个要去除的URI", remove_uris.len()), + }); + + // 第二步：处理全部数据文件 + let _ = window.emit("data-cleaning-progress", DataCleaningProgress { + current: 0, + total: 0, + percentage: 40.0, + status: "开始处理全部数据文件...".to_string(), + }); + + let result = match process_all_data_file(&all_data_file, &remove_uris, &output_file, &window).await { + Ok(result) => result, + Err(e) => return Err(format!("处理全部数据文件失败: {}", e)), + }; + + let _ = window.emit("data-cleaning-progress", DataCleaningProgress { + current: result.final_count, + total: result.original_count, + percentage: 100.0, + status: "处理完成".to_string(), + }); + + Ok(result) +} + +/// 从要去除的数据文件中提取所有URI +async fn extract_remove_uris(file_path: &str) -> Result, String> { + let file = File::open(file_path) + .map_err(|e| format!("无法打开文件: {}", e))?; + + let reader = BufReader::new(file); + let mut uris = HashSet::new(); + + for line in reader.lines() { + let line = line.map_err(|e| format!("读取行失败: {}", e))?; + + if line.trim().is_empty() { + continue; + } + + // 解析JSON + let processed_data: ProcessedData = serde_json::from_str(&line) + .map_err(|e| format!("解析JSON失败: {} - 行内容: {}", e, line))?; + + // 从json_data字段中解析内部JSON + let inner_json: serde_json::Value = serde_json::from_str(&processed_data.json_data) + .map_err(|e| format!("解析内部JSON失败: {}", e))?; + + // 提取uri字段 + if let Some(uri) = inner_json.get("uri").and_then(|v| v.as_str()) { + uris.insert(uri.to_string()); + } + } + + Ok(uris) +} + +/// 处理全部数据文件，过滤掉要去除的URI +async fn process_all_data_file( + input_file: &str, + remove_uris: &HashSet, + output_file: &str, + window: &tauri::Window, +) -> Result { + let input_file_handle = File::open(input_file) + .map_err(|e| format!("无法打开输入文件: {}", e))?; + + let reader = BufReader::new(input_file_handle); + + let mut output_file_handle = File::create(output_file) + .map_err(|e| format!("无法创建输出文件: {}", e))?; + + let mut original_count = 0; + let mut removed_count = 0; + let mut final_count = 0; + + // 首先计算总行数用于进度显示 + let total_lines = count_lines(input_file)?; + + for (line_number, line) in reader.lines().enumerate() { + let line = line.map_err(|e| format!("读取行失败: {}", e))?; + + if line.trim().is_empty() { + continue; + } + + original_count += 1; + + // 解析JSON + let storage_object: StorageObject = match serde_json::from_str(&line) { + Ok(obj) => obj, + Err(e) => { + // 如果解析失败，跳过这一行但记录警告 + eprintln!("警告: 解析JSON失败，跳过行 {}: {} - 内容: {}", line_number + 1, e, line); + continue; + } + }; + + // 检查URI是否在要去除的列表中 + if remove_uris.contains(&storage_object.uri) { + removed_count += 1; + } else { + // 写入输出文件 + writeln!(output_file_handle, "{}", line) + .map_err(|e| format!("写入输出文件失败: {}", e))?; + final_count += 1; + } + + // 每处理100行发送一次进度更新 + if line_number % 100 == 0 { + let percentage = 40.0 + (line_number as f64 / total_lines as f64) * 50.0; + let _ = window.emit("data-cleaning-progress", DataCleaningProgress { + current: line_number, + total: total_lines, + percentage, + status: format!("已处理 {} 行，已去除 {} 个重复项", line_number, removed_count), + }); + } + } + + // 确保文件被正确写入 + output_file_handle.flush() + .map_err(|e| format!("刷新输出文件失败: {}", e))?; + + Ok(DataCleaningResult { + success: true, + message: "数据清洗完成".to_string(), + original_count, + removed_count, + final_count, + output_file: output_file.to_string(), + }) +} + +/// 计算文件行数 +fn count_lines(file_path: &str) -> Result { + let file = File::open(file_path) + .map_err(|e| format!("无法打开文件计算行数: {}", e))?; + + let reader = BufReader::new(file); + let count = reader.lines().count(); + Ok(count) +} diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index 9b20475..9074617 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -8,6 +8,7 @@ import ModelDetail from './pages/ModelDetail'; import AiClassificationSettings from './pages/AiClassificationSettings'; import TemplateManagement from './pages/TemplateManagement'; import { MaterialModelBinding } from './pages/MaterialModelBinding'; +import Tools from './pages/Tools'; import Navigation from './components/Navigation'; import { NotificationSystem, useNotifications } from './components/NotificationSystem'; import { useProjectStore } from './store/projectStore'; @@ -78,6 +79,7 @@ function App() { } /> } /> } /> + } /> diff --git a/apps/desktop/src/components/Navigation.tsx b/apps/desktop/src/components/Navigation.tsx index 945b573..f209a26 100644 --- a/apps/desktop/src/components/Navigation.tsx +++ b/apps/desktop/src/components/Navigation.tsx @@ -5,7 +5,8 @@ import { UserGroupIcon, CpuChipIcon, DocumentDuplicateIcon, - LinkIcon + LinkIcon, + WrenchScrewdriverIcon } from '@heroicons/react/24/outline'; const Navigation: React.FC = () => { @@ -41,6 +42,12 @@ const Navigation: React.FC = () => { href: '/ai-classification-settings', icon: CpuChipIcon, description: '管理AI视频分类规则' + }, + { + name: '便捷工具', + href: '/tools', + icon: WrenchScrewdriverIcon, + description: 'AI检索图片/数据清洗工具' } ]; diff --git a/apps/desktop/src/pages/Tools.tsx b/apps/desktop/src/pages/Tools.tsx new file mode 100644 index 0000000..7e34714 --- /dev/null +++ b/apps/desktop/src/pages/Tools.tsx @@ -0,0 +1,354 @@ +import React, { useState } from 'react'; +import { + Wrench, + FileText, + Upload, + Download, + Loader2, + CheckCircle, + AlertCircle, + Info, + Trash2 +} from 'lucide-react'; +import { invoke } from '@tauri-apps/api/core'; +import { open, save } from '@tauri-apps/plugin-dialog'; +import { listen } from '@tauri-apps/api/event'; +import { useNotifications } from '../components/NotificationSystem'; + +interface DataCleaningProgress { + current: number; + total: number; + percentage: number; + status: string; +} + +interface DataCleaningResult { + success: boolean; + message: string; + original_count: number; + removed_count: number; + final_count: number; + output_file: string; +} + +/** + * 便捷小工具页面 + * 遵循 Tauri 开发规范和 UI/UX 设计标准 + */ +const Tools: React.FC = () => { + const [allDataFile, setAllDataFile] = useState(''); + const [removeDataFile, setRemoveDataFile] = useState(''); + const [outputFile, setOutputFile] = useState(''); + const [isProcessing, setIsProcessing] = useState(false); + const [progress, setProgress] = useState(null); + const [result, setResult] = useState(null); + + const { success, error } = useNotifications(); + + // 选择全部数据文件 + const selectAllDataFile = async () => { + try { + const selected = await open({ + multiple: false, + filters: [{ + name: 'JSONL Files', + extensions: ['jsonl'] + }] + }); + + if (selected && typeof selected === 'string') { + setAllDataFile(selected); + } + } catch (err) { + error('文件选择失败', '无法选择全部数据文件'); + } + }; + + // 选择要去除的数据文件 + const selectRemoveDataFile = async () => { + try { + const selected = await open({ + multiple: false, + filters: [{ + name: 'JSONL Files', + extensions: ['jsonl'] + }] + }); + + if (selected && typeof selected === 'string') { + setRemoveDataFile(selected); + } + } catch (err) { + error('文件选择失败', '无法选择要去除的数据文件'); + } + }; + + // 选择输出文件 + const selectOutputFile = async () => { + try { + const selected = await save({ + filters: [{ + name: 'JSONL Files', + extensions: ['jsonl'] + }], + defaultPath: 'cleaned_data.jsonl' + }); + + if (selected) { + setOutputFile(selected); + } + } catch (err) { + error('文件选择失败', '无法选择输出文件'); + } + }; + + // 开始数据清洗 + const startDataCleaning = async () => { + if (!allDataFile || !removeDataFile || !outputFile) { + error('参数错误', '请选择所有必需的文件'); + return; + } + + setIsProcessing(true); + setProgress(null); + setResult(null); + + try { + // 监听进度事件 + const unlisten = await listen('data-cleaning-progress', (event) => { + setProgress(event.payload); + }); + + // 调用后端命令 + const cleaningResult = await invoke('clean_jsonl_data', { + allDataFile, + removeDataFile, + outputFile + }); + + setResult(cleaningResult); + + if (cleaningResult.success) { + success('数据清洗完成', + `原始数据: ${cleaningResult.original_count} 条，` + + `去除重复: ${cleaningResult.removed_count} 条，` + + `最终结果: ${cleaningResult.final_count} 条` + ); + } else { + error('数据清洗失败', cleaningResult.message); + } + + // 清理事件监听器 + unlisten(); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : '未知错误'; + error('数据清洗失败', errorMessage); + } finally { + setIsProcessing(false); + } + }; + + // 重置表单 + const resetForm = () => { + setAllDataFile(''); + setRemoveDataFile(''); + setOutputFile(''); + setProgress(null); + setResult(null); + }; + + return ( +
+ {/* 页面标题 */} +
+
+ +
+
+

便捷小工具

+

AI检索图片/数据清洗工具集

+
+
+ + {/* AI检索图片/数据清洗工具 */} +
+
+
+ +
+

AI检索图片/数据清洗

+

JSONL格式数据去重处理

+
+
+
+ +
+ {/* 使用说明 */} +
+
+ +
+

使用说明：

+
    +
  • 选择包含所有数据的JSONL文件（全部数据）
    • +
  • 选择包含要去除数据的JSONL文件（要去除的数据）
    • +
  • 系统将根据URI字段进行匹配，去除重复项
    • +
  • 处理结果将保存到指定的输出文件
    • +
+
+
+
+ + {/* 文件选择区域 */} +
+ {/* 全部数据文件 */} +
+ +
+ +
+ {allDataFile && ( +

+ {allDataFile} +

+ )} +
+ + {/* 要去除的数据文件 */} +
+ +
+ +
+ {removeDataFile && ( +

+ {removeDataFile} +

+ )} +
+
+ + {/* 输出文件选择 */} +
+ +
+ +
+ {outputFile && ( +

+ {outputFile} +

+ )} +
+ + {/* 进度显示 */} + {progress && ( +
+
+ + 处理中... +
+
+
+ {progress.status} + {progress.percentage.toFixed(1)}% +
+
+
+
+ {progress.total > 0 && ( +
+ {progress.current.toLocaleString()} / {progress.total.toLocaleString()} +
+ )} +
+
+ )} + + {/* 结果显示 */} + {result && ( +
+
+ {result.success ? ( + + ) : ( + + )} +
+

+ {result.message} +

+ {result.success && ( +
+

原始数据条数: {result.original_count.toLocaleString()}

+

去除重复条数: {result.removed_count.toLocaleString()}

+

最终结果条数: {result.final_count.toLocaleString()}

+

输出文件: {result.output_file}

+
+ )} +
+
+
+ )} + + {/* 操作按钮 */} +
+ + + +
+
+
+
+ ); +}; + +export default Tools; diff --git a/apps/desktop/test_data/sample_all_data.jsonl b/apps/desktop/test_data/sample_all_data.jsonl new file mode 100644 index 0000000..3fafd63 --- /dev/null +++ b/apps/desktop/test_data/sample_all_data.jsonl @@ -0,0 +1,5 @@ +{"kind":"storage#object","id":"fashion_image_block/gallery_v2/models/test_image_1.jpg/1752665521922040","selfLink":"https://www.googleapis.com/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_1.jpg","mediaLink":"https://storage.googleapis.com/download/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_1.jpg?generation=1752665521922040&alt=media","name":"gallery_v2/models/test_image_1.jpg","bucket":"fashion_image_block","generation":"1752665521922040","metageneration":"1","contentType":"image/jpeg","storageClass":"STANDARD","size":310505,"md5Hash":"WmzNIJHZxALLVSfn0esl5Q==","crc32c":"6skf3A==","etag":"CPi/4O6jwY4DEAE=","timeCreated":"2025-07-16T11:32:01.981000Z","updated":"2025-07-16T11:32:01.981000Z","uri":"gs://fashion_image_block/gallery_v2/models/test_image_1.jpg"} +{"kind":"storage#object","id":"fashion_image_block/gallery_v2/models/test_image_2.jpg/1752665524250767","selfLink":"https://www.googleapis.com/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_2.jpg","mediaLink":"https://storage.googleapis.com/download/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_2.jpg?generation=1752665524250767&alt=media","name":"gallery_v2/models/test_image_2.jpg","bucket":"fashion_image_block","generation":"1752665524250767","metageneration":"1","contentType":"image/jpeg","storageClass":"STANDARD","size":488328,"md5Hash":"ksnQY8Wv6g5Q+LkmaxBV+A==","crc32c":"u4zm9A==","etag":"CI/R7u+jwY4DEAE=","timeCreated":"2025-07-16T11:32:04.317000Z","updated":"2025-07-16T11:32:04.317000Z","uri":"gs://fashion_image_block/gallery_v2/models/test_image_2.jpg"} +{"kind":"storage#object","id":"fashion_image_block/gallery_v2/models/test_image_3.jpg/1752665522362206","selfLink":"https://www.googleapis.com/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_3.jpg","mediaLink":"https://storage.googleapis.com/download/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_3.jpg?generation=1752665522362206&alt=media","name":"gallery_v2/models/test_image_3.jpg","bucket":"fashion_image_block","generation":"1752665522362206","metageneration":"1","contentType":"image/jpeg","storageClass":"STANDARD","size":719213,"md5Hash":"fPmRuc/nFZnN6g5u9pJ99g==","crc32c":"r4eLmg==","etag":"CN6u++6jwY4DEAE=","timeCreated":"2025-07-16T11:32:02.420000Z","updated":"2025-07-16T11:32:02.420000Z","uri":"gs://fashion_image_block/gallery_v2/models/test_image_3.jpg"} +{"kind":"storage#object","id":"fashion_image_block/gallery_v2/models/test_image_4.jpg/1752665523165046","selfLink":"https://www.googleapis.com/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_4.jpg","mediaLink":"https://storage.googleapis.com/download/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_4.jpg?generation=1752665523165046&alt=media","name":"gallery_v2/models/test_image_4.jpg","bucket":"fashion_image_block","generation":"1752665523165046","metageneration":"1","contentType":"image/jpeg","storageClass":"STANDARD","size":463800,"md5Hash":"G0EjX9ZdkMEXoDR3h+zNJQ==","crc32c":"+wH9Bg==","etag":"CPaurO+jwY4DEAE=","timeCreated":"2025-07-16T11:32:03.230000Z","updated":"2025-07-16T11:32:03.230000Z","uri":"gs://fashion_image_block/gallery_v2/models/test_image_4.jpg"} +{"kind":"storage#object","id":"fashion_image_block/gallery_v2/models/test_image_5.jpg/1752665521595009","selfLink":"https://www.googleapis.com/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_5.jpg","mediaLink":"https://storage.googleapis.com/download/storage/v1/b/fashion_image_block/o/gallery_v2%2Fmodels%2Ftest_image_5.jpg?generation=1752665521595009&alt=media","name":"gallery_v2/models/test_image_5.jpg","bucket":"fashion_image_block","generation":"1752665521595009","metageneration":"1","contentType":"image/jpeg","storageClass":"STANDARD","size":435049,"md5Hash":"vw3s4DXgN06HPdSnizhPrQ==","crc32c":"kHGLOA==","etag":"CIHFzO6jwY4DEAE=","timeCreated":"2025-07-16T11:32:01.663000Z","updated":"2025-07-16T11:32:01.663000Z","uri":"gs://fashion_image_block/gallery_v2/models/test_image_5.jpg"} diff --git a/apps/desktop/test_data/sample_remove_data.jsonl b/apps/desktop/test_data/sample_remove_data.jsonl new file mode 100644 index 0000000..66f714c --- /dev/null +++ b/apps/desktop/test_data/sample_remove_data.jsonl @@ -0,0 +1,2 @@ +{"id":"0fc38c97-e8a7-4fc2-8228-7598d1b993e9","schema_id":"default_schema","json_data":"{\"dress_color_pattern\":{\"Hue\":0.08,\"Saturation\":0.05,\"Value\":0.95,\"rgb_hex\":\"#f2ebe6\"},\"style_description\":\"测试样例1\",\"environment_tags\":[\"Indoor\",\"Test\"],\"products\":[{\"category\":\"测试\",\"color_pattern\":{\"Hue\":0.08,\"Saturation\":0.05,\"Value\":0.95,\"rgb_hex\":\"#f2ebe6\"},\"design_styles\":[\"Test\"],\"design_styles_embedding\":[]}],\"uri\":\"gs://fashion_image_block/gallery_v2/models/test_image_2.jpg\",\"description\":\"\",\"runtime\":\"\",\"releaseDate\":\"2025-07-16T11:31:48.092583Z\",\"title\":\"model\",\"categories\":[\"测试\"]}"} +{"id":"e32fcc8d-166c-4da5-991d-422f7c5a47e7","schema_id":"default_schema","json_data":"{\"dress_color_pattern\":{\"Hue\":0.08,\"Saturation\":0.05,\"Value\":0.95,\"rgb_hex\":\"#f2ebe6\"},\"style_description\":\"测试样例2\",\"environment_tags\":[\"Indoor\",\"Test\"],\"products\":[{\"category\":\"测试\",\"color_pattern\":{\"Hue\":0.08,\"Saturation\":0.05,\"Value\":0.95,\"rgb_hex\":\"#f2ebe6\"},\"design_styles\":[\"Test\"],\"design_styles_embedding\":[]}],\"uri\":\"gs://fashion_image_block/gallery_v2/models/test_image_4.jpg\",\"description\":\"\",\"runtime\":\"\",\"releaseDate\":\"2025-07-16T11:31:48.092583Z\",\"title\":\"model\",\"categories\":[\"测试\"]}"}