/// 基于Tree-sitter的大模型JSON容错解析器 /// 遵循Tauri开发规范，提供高性能、安全的JSON解析能力 /// /// ## 新增功能：YAML格式支持 /// /// ### 功能概述 /// - 当字符串值被检测为YAML格式时，自动解析为对应的JSON结构 /// - 支持YAML键值对、列表、嵌套结构等常见格式 /// - 智能检测避免误判JSON内容为YAML /// /// ### YAML检测规则 /// 1. **键值对检测**：包含 `key: value` 格式的多行内容 /// 2. **列表检测**：包含 `- item` 格式的行 /// 3. **多行检测**：包含换行符和冒号的内容 /// 4. **排除规则**：以 `{` 或 `[` 开头的内容不会被识别为YAML /// /// ### 使用场景 /// - 处理大模型返回的混合格式内容 /// - 解析配置文件中的YAML字段 /// - 容错处理包含YAML片段的JSON数据 /// /// ### 示例 /// ```rust /// // JSON中包含YAML字符串会被自动解析 /// let json_with_yaml = r#"{"config": "name: John\nage: 30"}"#; /// // 解析后 config 字段会变成 {"name": "John", "age": 30} /// /// let yaml_list = r#"{"items": "- item1\n- item2\n- item3"}"#; /// // 解析后 items 字段会变成 ["item1", "item2", "item3"] /// ``` use anyhow::{anyhow, Result}; use regex::Regex; use serde_json::{Map, Value}; use serde_yaml; use std::collections::HashMap; use std::sync::{Arc, Mutex}; use tree_sitter::{Node, Parser, Tree}; use tracing::{debug, info, warn}; /// 解析结果统计信息 #[derive(Debug, Clone, serde::Serialize)] pub struct ParseStatistics { pub total_nodes: usize, pub error_nodes: usize, pub error_rate: f64, pub parse_time_ms: u64, pub recovery_strategies_used: Vec, } /// 错误恢复策略类型 #[derive(Debug, Clone)] pub enum RecoveryStrategy { StandardJson, ManualFix, RegexExtract, PartialParse, } /// 容错JSON解析器配置 #[derive(Debug, Clone)] pub struct ParserConfig { pub max_text_length: usize, pub enable_comments: bool, pub enable_unquoted_keys: bool, pub enable_trailing_commas: bool, pub timeout_ms: u64, pub recovery_strategies: Vec, } impl Default for ParserConfig { fn default() -> Self { Self { max_text_length: 1024 * 1024, // 1MB enable_comments: true, enable_unquoted_keys: true, enable_trailing_commas: true, timeout_ms: 30000, // 30秒 recovery_strategies: vec![ RecoveryStrategy::StandardJson, RecoveryStrategy::ManualFix, RecoveryStrategy::RegexExtract, RecoveryStrategy::PartialParse, ], } } } /// 容错JSON解析器 pub struct TolerantJsonParser { parser: Parser, config: ParserConfig, regex_patterns: HashMap, } impl TolerantJsonParser { /// 创建新的容错JSON解析器实例 pub fn new(config: Option) -> Result { let mut parser = Parser::new(); // 设置JSON语言 let language = tree_sitter_json::language(); parser.set_language(language) .map_err(|e| anyhow!("Failed to set JSON language: {}", e))?; let config = config.unwrap_or_default(); let regex_patterns = Self::init_regex_patterns()?; Ok(Self { parser, config, regex_patterns, }) } /// 初始化正则表达式模式 fn init_regex_patterns() -> Result> { let mut patterns = HashMap::new(); // JSON对象模式 - 使用简单但可靠的方法 // Rust的regex crate不支持递归正则，所以使用贪婪匹配 patterns.insert( "object".to_string(), Regex::new(r"(?s)\{.*\}")?, ); // JSON数组模式 - 使用简单但可靠的方法 patterns.insert( "array".to_string(), Regex::new(r"(?s)\[.*\]")?, ); // Markdown代码块模式 patterns.insert( "markdown_fence".to_string(), Regex::new(r"(?s)```(?:json)?\s*\n?(.*?)\n?```")?, ); // 无引号键模式 patterns.insert( "unquoted_key".to_string(), Regex::new(r"(\w+):")?, ); // 尾随逗号模式 patterns.insert( "trailing_comma".to_string(), Regex::new(r",\s*[}\]]")?, ); // YAML检测模式 - 检测常见的YAML结构（键值对） patterns.insert( "yaml_structure".to_string(), Regex::new(r"(?m)^[\s]*[\w\-\.]+:\s*.*$")?, ); // YAML列表模式 patterns.insert( "yaml_list".to_string(), Regex::new(r"(?m)^[\s]*-\s+")?, ); // YAML多行字符串模式 patterns.insert( "yaml_multiline".to_string(), Regex::new(r"(?s)[|>][\s]*\n")?, ); Ok(patterns) } /// 解析JSON文本，返回解析结果和统计信息 pub fn parse(&mut self, text: &str) -> Result<(Value, ParseStatistics)> { let start_time = std::time::Instant::now(); // 检查文本长度限制 if text.len() > self.config.max_text_length { return Err(anyhow!("Text too long: {} bytes", text.len())); } // 首先尝试标准JSON解析 match serde_json::from_str::(text) { Ok(result) => { // 标准JSON解析成功，但还需要检查字符串字段是否包含YAML内容 let mut recovery_strategies = vec!["StandardJson".to_string()]; let processed_result = self.process_yaml_in_json_value(result, &mut recovery_strategies); let stats = ParseStatistics { total_nodes: 1, error_nodes: 0, error_rate: 0.0, parse_time_ms: start_time.elapsed().as_millis() as u64, recovery_strategies_used: recovery_strategies, }; return Ok((processed_result, stats)); } Err(_) => { // 标准解析失败，继续使用容错解析 debug!("Standard JSON parsing failed, using tolerant parsing"); } } // 预处理文本 let cleaned_text = self.preprocess_text(text)?; debug!("Preprocessed text length: {}", cleaned_text.len()); // 使用Tree-sitter解析 let tree = self.parser.parse(&cleaned_text, None) .ok_or_else(|| anyhow!("Failed to parse text with tree-sitter"))?; let root_node = tree.root_node(); // 收集统计信息 let mut stats = self.collect_statistics(&tree, &cleaned_text); // 如果错误率太高，回退到标准JSON解析的错误修复 if stats.error_rate > 0.1 { debug!("High error rate ({:.2}%), attempting fallback strategies", stats.error_rate * 100.0); // 尝试基本的文本清理后再用标准JSON解析 if let Ok(result) = self.try_fallback_parsing(text) { stats.recovery_strategies_used.push("FallbackParsing".to_string()); stats.parse_time_ms = start_time.elapsed().as_millis() as u64; info!("Fallback JSON parsing successful in {}ms", stats.parse_time_ms); return Ok((result, stats)); } } // 提取和修复JSON let mut recovery_strategies_used = Vec::new(); let result = self.extract_json(root_node, &cleaned_text, &mut recovery_strategies_used)?; stats.parse_time_ms = start_time.elapsed().as_millis() as u64; stats.recovery_strategies_used = recovery_strategies_used; info!("JSON parsing completed in {}ms", stats.parse_time_ms); Ok((result, stats)) } /// 预处理文本，提取可能的JSON内容 fn preprocess_text(&self, text: &str) -> Result { let mut processed = text.to_string(); // 移除Markdown代码块标记 if let Some(markdown_regex) = self.regex_patterns.get("markdown_fence") { if let Some(captures) = markdown_regex.captures(&processed) { if let Some(json_content) = captures.get(1) { processed = json_content.as_str().to_string(); debug!("Extracted JSON from markdown fence"); } } } // 使用改进的JSON提取方法 processed = self.extract_json_content(&processed); // 在Tree-sitter解析之前修复常见的JSON错误 processed = self.fix_common_json_errors(&processed); Ok(processed) } /// 改进的JSON内容提取方法 fn extract_json_content(&self, text: &str) -> String { // 首先尝试使用手动括号匹配（最可靠的方法） if let Some(json_content) = self.extract_json_by_bracket_matching(text) { debug!("Extracted JSON using bracket matching"); return json_content; } // 如果括号匹配失败，尝试使用正则表达式 for pattern_name in ["object", "array"] { if let Some(pattern) = self.regex_patterns.get(pattern_name) { if let Some(mat) = pattern.find(text) { debug!("Extracted JSON using {} pattern", pattern_name); return mat.as_str().to_string(); } } } // 如果所有方法都失败，返回原始文本 text.to_string() } /// 使用括号匹配提取JSON内容 fn extract_json_by_bracket_matching(&self, text: &str) -> Option { // 查找第一个 { 或 [ let start_char = if text.contains('{') && text.contains('[') { let brace_pos = text.find('{').unwrap_or(usize::MAX); let bracket_pos = text.find('[').unwrap_or(usize::MAX); if brace_pos < bracket_pos { '{' } else { '[' } } else if text.contains('{') { '{' } else if text.contains('[') { '[' } else { return None; }; let end_char = if start_char == '{' { '}' } else { ']' }; if let Some(start_pos) = text.find(start_char) { let mut depth = 0; let mut in_string = false; let mut escape_next = false; for (i, ch) in text[start_pos..].char_indices() { if escape_next { escape_next = false; continue; } match ch { '\\' if in_string => escape_next = true, '"' => in_string = !in_string, c if c == start_char && !in_string => depth += 1, c if c == end_char && !in_string => { depth -= 1; if depth == 0 { return Some(text[start_pos..start_pos + i + 1].to_string()); } } _ => {} } } } None } /// 修复常见的JSON错误 fn fix_common_json_errors(&self, text: &str) -> String { let mut fixed = text.to_string(); // 检查是否包含YAML内容，如果是则跳过某些修复 let contains_yaml = self.is_yaml_format(text); if contains_yaml { debug!("Detected YAML content, skipping unquoted key fixes"); } else { // 修复无引号的键 if self.config.enable_unquoted_keys { if let Some(unquoted_regex) = self.regex_patterns.get("unquoted_key") { fixed = unquoted_regex.replace_all(&fixed, "\"$1\":").to_string(); debug!("Fixed unquoted keys"); } } } // 移除尾随逗号 if self.config.enable_trailing_commas { if let Some(trailing_regex) = self.regex_patterns.get("trailing_comma") { fixed = trailing_regex.replace_all(&fixed, |caps: ®ex::Captures| { let full_match = caps.get(0).unwrap().as_str(); full_match[1..].to_string() // 移除逗号，保留括号 }).to_string(); debug!("Fixed trailing commas"); } } // 修复单引号为双引号（但不对YAML内容进行此修复） if !contains_yaml { fixed = fixed.replace("'", "\""); } debug!("Fixed text: {}", &fixed[..std::cmp::min(200, fixed.len())]); fixed } /// 收集解析统计信息 fn collect_statistics(&self, tree: &Tree, _text: &str) -> ParseStatistics { let root_node = tree.root_node(); let mut total_nodes = 0; let mut error_nodes = 0; self.count_nodes(root_node, &mut total_nodes, &mut error_nodes); let error_rate = if total_nodes > 0 { error_nodes as f64 / total_nodes as f64 } else { 0.0 }; ParseStatistics { total_nodes, error_nodes, error_rate, parse_time_ms: 0, // 将在parse方法中设置 recovery_strategies_used: Vec::new(), // 将在parse方法中设置 } } /// 递归计算节点数量 fn count_nodes(&self, node: Node, total: &mut usize, errors: &mut usize) { *total += 1; if node.is_error() { *errors += 1; } for child in node.children(&mut node.walk()) { self.count_nodes(child, total, errors); } } /// 从语法树中提取和修复JSON fn extract_json( &self, node: Node, text: &str, recovery_strategies: &mut Vec, ) -> Result { match node.kind() { "document" => { if node.child_count() > 0 { self.extract_json(node.child(0).unwrap(), text, recovery_strategies) } else { Ok(Value::Null) } } "value" => { if node.child_count() > 0 { self.extract_json(node.child(0).unwrap(), text, recovery_strategies) } else { self.recover_from_error(node, text, recovery_strategies) } } "object" => self.extract_object(node, text, recovery_strategies), "array" => self.extract_array(node, text, recovery_strategies), "string" => self.extract_string(node, text), "number" => self.extract_number(node, text), "true" | "false" => self.extract_boolean(node, text), "null" => Ok(Value::Null), "ERROR" => self.recover_from_error(node, text, recovery_strategies), _ => { warn!("Unknown node kind: {}", node.kind()); self.recover_from_error(node, text, recovery_strategies) } } } /// 提取JSON对象 fn extract_object( &self, node: Node, text: &str, recovery_strategies: &mut Vec, ) -> Result { let mut object = Map::new(); for child in node.children(&mut node.walk()) { match child.kind() { "pair" => { if let Ok((key, value)) = self.extract_pair(child, text, recovery_strategies) { object.insert(key, value); } } "ERROR" => { debug!("Found ERROR node in object"); // 尝试恢复错误的对象成员 if let Ok(recovered) = self.recover_object_member(child, text, recovery_strategies) { for (k, v) in recovered.as_object().unwrap_or(&Map::new()) { object.insert(k.clone(), v.clone()); } } else { // 如果对象成员恢复失败，尝试通用错误恢复 if let Ok(recovered) = self.recover_from_error(child, text, recovery_strategies) { // 如果恢复的是一个对象，合并其内容 if let Some(recovered_obj) = recovered.as_object() { for (k, v) in recovered_obj { object.insert(k.clone(), v.clone()); } } } } } _ => { // 忽略其他节点类型（如括号、逗号等） } } } Ok(Value::Object(object)) } /// 提取JSON数组 fn extract_array( &self, node: Node, text: &str, recovery_strategies: &mut Vec, ) -> Result { let mut array = Vec::new(); for child in node.children(&mut node.walk()) { match child.kind() { "value" => { if let Ok(value) = self.extract_json(child, text, recovery_strategies) { array.push(value); } } // Also handle direct primitive types "number" | "string" | "true" | "false" | "null" | "object" | "array" => { if let Ok(value) = self.extract_json(child, text, recovery_strategies) { array.push(value); } } "ERROR" => { // 尝试恢复错误的数组元素 if let Ok(recovered) = self.recover_from_error(child, text, recovery_strategies) { array.push(recovered); } } _ => {} // 忽略其他节点类型（如括号、逗号等） } } Ok(Value::Array(array)) } /// 提取键值对 fn extract_pair( &self, node: Node, text: &str, recovery_strategies: &mut Vec, ) -> Result<(String, Value)> { let mut key = None; let mut value = None; for child in node.children(&mut node.walk()) { match child.kind() { "string" => { if key.is_none() { let extracted_key = self.extract_string_content(child, text)?; key = Some(extracted_key); } else { let extracted_value = self.extract_json(child, text, recovery_strategies)?; value = Some(extracted_value); } } "value" => { let extracted_value = self.extract_json(child, text, recovery_strategies)?; value = Some(extracted_value); } // 直接处理各种值类型 "object" | "array" | "number" | "true" | "false" | "null" => { let extracted_value = self.extract_json(child, text, recovery_strategies)?; value = Some(extracted_value); } _ => { // 忽略冒号等分隔符 } } } match (key, value) { (Some(k), Some(v)) => Ok((k, v)), _ => Err(anyhow!("Invalid key-value pair")), } } /// 提取字符串值 fn extract_string(&self, node: Node, text: &str) -> Result { let content = self.extract_string_content(node, text)?; // 检测字符串是否为YAML格式，如果是则自动解析 if let Ok(yaml_value) = self.try_parse_yaml(&content) { debug!("Detected and parsed YAML content in string field"); return Ok(yaml_value); } Ok(Value::String(content)) } /// 提取字符串内容（去除引号） fn extract_string_content(&self, node: Node, text: &str) -> Result { let raw_text = &text[node.start_byte()..node.end_byte()]; // 移除前后的引号 if raw_text.len() >= 2 && raw_text.starts_with('"') && raw_text.ends_with('"') { Ok(raw_text[1..raw_text.len()-1].to_string()) } else { // 处理无引号的字符串 Ok(raw_text.to_string()) } } /// 提取数字值 fn extract_number(&self, node: Node, text: &str) -> Result { let raw_text = &text[node.start_byte()..node.end_byte()]; // 尝试解析为整数 if let Ok(int_val) = raw_text.parse::() { Ok(Value::Number(serde_json::Number::from(int_val))) } // 尝试解析为浮点数 else if let Ok(float_val) = raw_text.parse::() { if let Some(num) = serde_json::Number::from_f64(float_val) { Ok(Value::Number(num)) } else { Err(anyhow!("Invalid float number: {}", raw_text)) } } else { Err(anyhow!("Invalid number: {}", raw_text)) } } /// 提取布尔值 fn extract_boolean(&self, node: Node, text: &str) -> Result { let raw_text = &text[node.start_byte()..node.end_byte()]; match raw_text { "true" => Ok(Value::Bool(true)), "false" => Ok(Value::Bool(false)), _ => Err(anyhow!("Invalid boolean: {}", raw_text)), } } /// 从错误节点恢复数据 fn recover_from_error( &self, node: Node, text: &str, recovery_strategies: &mut Vec, ) -> Result { let error_text = &text[node.start_byte()..node.end_byte()].trim(); debug!("Attempting to recover from error: {}", error_text); // 首先尝试智能字符串恢复（针对复杂字符串内容） if let Ok(result) = self.try_smart_string_recovery(error_text) { recovery_strategies.push("SmartStringRecovery".to_string()); return Ok(result); } // 尝试不同的恢复策略 for strategy in &self.config.recovery_strategies { match strategy { RecoveryStrategy::StandardJson => { if let Ok(result) = self.try_standard_json_parse(error_text) { recovery_strategies.push("StandardJson".to_string()); return Ok(result); } } RecoveryStrategy::ManualFix => { if let Ok(result) = self.try_manual_fix(error_text) { recovery_strategies.push("ManualFix".to_string()); return Ok(result); } } RecoveryStrategy::RegexExtract => { if let Ok(result) = self.try_regex_extract(error_text) { recovery_strategies.push("RegexExtract".to_string()); return Ok(result); } } RecoveryStrategy::PartialParse => { if let Ok(result) = self.try_partial_parse(error_text) { recovery_strategies.push("PartialParse".to_string()); return Ok(result); } } } } // 如果所有策略都失败，但内容不为空，保留原始文本作为字符串 // 这对于包含复杂内容（如YAML）的字段很重要 if !error_text.is_empty() { warn!("All recovery strategies failed, preserving content as string: {} chars", error_text.len()); recovery_strategies.push("PreserveAsString".to_string()); Ok(Value::String(error_text.to_string())) } else { warn!("All recovery strategies failed for empty content"); Ok(Value::Null) } } /// 尝试标准JSON解析 fn try_standard_json_parse(&self, text: &str) -> Result { serde_json::from_str(text).map_err(|e| anyhow!("Standard JSON parse failed: {}", e)) } /// 智能字符串恢复 - 专门处理复杂字符串内容，包括YAML格式 fn try_smart_string_recovery(&self, text: &str) -> Result { let trimmed = text.trim(); // 首先尝试检测和解析YAML格式 if let Ok(yaml_value) = self.try_parse_yaml(trimmed) { debug!("Recovered content as YAML"); return Ok(yaml_value); } // 检查是否是一个键值对格式 if let Some(colon_pos) = trimmed.find(':') { let key_part = trimmed[..colon_pos].trim(); let value_part = trimmed[colon_pos+1..].trim(); // 清理键名（移除引号） let clean_key = key_part.trim_matches(|c| c == '"' || c == '\'' || c == ' '); // 首先尝试将值部分解析为YAML if let Ok(yaml_value) = self.try_parse_yaml(value_part) { let mut object = Map::new(); object.insert(clean_key.to_string(), yaml_value); debug!("Recovered key-value pair with YAML value"); return Ok(Value::Object(object)); } // 如果值部分是一个被引号包围的字符串 if value_part.starts_with('"') && value_part.len() > 1 { // 寻找匹配的结束引号，考虑转义字符 let mut end_pos = None; let mut chars = value_part[1..].char_indices(); let mut escaped = false; while let Some((i, ch)) = chars.next() { if escaped { escaped = false; continue; } match ch { '\\' => escaped = true, '"' => { end_pos = Some(i + 1); break; } _ => {} } } let string_content = if let Some(end) = end_pos { // 找到了结束引号 &value_part[1..end] } else { // 没有找到结束引号，取整个内容（去掉开头的引号） &value_part[1..] }; // 尝试将字符串内容解析为YAML if let Ok(yaml_value) = self.try_parse_yaml(string_content) { let mut object = Map::new(); object.insert(clean_key.to_string(), yaml_value); debug!("Recovered quoted string content as YAML"); return Ok(Value::Object(object)); } let mut object = Map::new(); object.insert(clean_key.to_string(), Value::String(string_content.to_string())); return Ok(Value::Object(object)); } } Err(anyhow!("Smart string recovery failed")) } /// 回退解析策略 - 当Tree-sitter错误率过高时使用 fn try_fallback_parsing(&self, text: &str) -> Result { // 尝试基本的文本清理 let mut cleaned = text.to_string(); // 移除可能的BOM if cleaned.starts_with('\u{FEFF}') { cleaned = cleaned[3..].to_string(); } // 尝试修复常见的JSON错误 cleaned = self.fix_common_json_errors(&cleaned); // 尝试解析清理后的文本 serde_json::from_str(&cleaned).map_err(|e| anyhow!("Fallback parsing failed: {}", e)) } /// 尝试手动修复常见的JSON错误 fn try_manual_fix(&self, text: &str) -> Result { let mut fixed_text = text.to_string(); // 修复无引号的键 if let Some(unquoted_regex) = self.regex_patterns.get("unquoted_key") { fixed_text = unquoted_regex.replace_all(&fixed_text, "\"$1\":").to_string(); } // 移除尾随逗号 if let Some(trailing_regex) = self.regex_patterns.get("trailing_comma") { fixed_text = trailing_regex.replace_all(&fixed_text, |caps: ®ex::Captures| { let full_match = caps.get(0).unwrap().as_str(); full_match[1..].to_string() // 移除逗号，保留括号 }).to_string(); } // 修复单引号为双引号 fixed_text = fixed_text.replace("'", "\""); // 尝试解析修复后的文本 serde_json::from_str(&fixed_text).map_err(|e| anyhow!("Manual fix failed: {}", e)) } /// 尝试正则表达式提取 fn try_regex_extract(&self, text: &str) -> Result { // 尝试提取对象 if let Some(object_regex) = self.regex_patterns.get("object") { if let Some(mat) = object_regex.find(text) { return self.try_standard_json_parse(mat.as_str()); } } // 尝试提取数组 if let Some(array_regex) = self.regex_patterns.get("array") { if let Some(mat) = array_regex.find(text) { return self.try_standard_json_parse(mat.as_str()); } } Err(anyhow!("No valid JSON pattern found")) } /// 尝试部分解析 fn try_partial_parse(&self, text: &str) -> Result { // 尝试解析文本的各个部分 let lines: Vec<&str> = text.lines().collect(); for i in 0..lines.len() { for j in (i+1)..=lines.len() { let partial_text = lines[i..j].join("\n"); if let Ok(value) = serde_json::from_str::(&partial_text) { return Ok(value); } } } Err(anyhow!("Partial parse failed")) } /// 恢复错误的对象成员 fn recover_object_member( &self, node: Node, text: &str, recovery_strategies: &mut Vec, ) -> Result { let error_text = &text[node.start_byte()..node.end_byte()]; // 首先检查整个错误文本是否为YAML格式 if let Ok(yaml_value) = self.try_parse_yaml(error_text) { recovery_strategies.push("ObjectMemberYamlRecovery".to_string()); return Ok(yaml_value); } // 尝试解析为单个键值对 if let Some(colon_pos) = error_text.find(':') { let key_part = error_text[..colon_pos].trim(); let value_part = error_text[colon_pos+1..].trim(); // 清理键名 let clean_key = key_part.trim_matches(|c| c == '"' || c == '\'' || c == ' '); // 首先尝试将值部分解析为YAML if let Ok(yaml_value) = self.try_parse_yaml(value_part) { let mut object = Map::new(); object.insert(clean_key.to_string(), yaml_value); recovery_strategies.push("ObjectMemberValueYamlRecovery".to_string()); return Ok(Value::Object(object)); } // 然后尝试标准JSON解析 if let Ok(value) = self.try_standard_json_parse(value_part) { let mut object = Map::new(); object.insert(clean_key.to_string(), value); recovery_strategies.push("ObjectMemberRecovery".to_string()); return Ok(Value::Object(object)); } // 如果标准解析失败，但值部分不为空，保留为字符串 if !value_part.is_empty() { let mut object = Map::new(); // 移除值部分的引号（如果存在） let clean_value = if (value_part.starts_with('"') && value_part.ends_with('"')) || (value_part.starts_with('\'') && value_part.ends_with('\'')) { &value_part[1..value_part.len()-1] } else { value_part }; object.insert(clean_key.to_string(), Value::String(clean_value.to_string())); recovery_strategies.push("ObjectMemberStringRecovery".to_string()); return Ok(Value::Object(object)); } } Err(anyhow!("Failed to recover object member")) } /// 检测字符串是否为YAML格式 fn is_yaml_format(&self, text: &str) -> bool { let trimmed = text.trim(); // 空字符串或太短的字符串不认为是YAML if trimmed.len() < 3 { return false; } // 如果包含JSON特征（大括号或方括号开头），则不是YAML if trimmed.starts_with('{') || trimmed.starts_with('[') { return false; } // 检查是否包含YAML特征 let has_yaml_structure = self.regex_patterns .get("yaml_structure") .map(|regex| regex.is_match(trimmed)) .unwrap_or(false); let has_yaml_list = self.regex_patterns .get("yaml_list") .map(|regex| regex.is_match(trimmed)) .unwrap_or(false); let has_yaml_multiline = self.regex_patterns .get("yaml_multiline") .map(|regex| regex.is_match(trimmed)) .unwrap_or(false); // 额外检查：包含多行且有冒号的内容 let has_multiline_with_colon = trimmed.contains('\n') && trimmed.contains(':'); // 如果包含任何YAML特征，则认为是YAML格式 has_yaml_structure || has_yaml_list || has_yaml_multiline || has_multiline_with_colon } /// 尝试解析YAML内容 fn try_parse_yaml(&self, text: &str) -> Result { let trimmed = text.trim(); // 首先检测是否为YAML格式 if !self.is_yaml_format(trimmed) { return Err(anyhow!("Not a YAML format")); } debug!("Attempting to parse YAML content: {} chars", trimmed.len()); // 尝试解析YAML match serde_yaml::from_str::(trimmed) { Ok(yaml_value) => { Ok(yaml_value) } Err(e) => { debug!("YAML parsing failed: {}", e); Err(anyhow!("YAML parsing failed: {}", e)) } } } /// 处理JSON值中的YAML字符串字段 fn process_yaml_in_json_value(&self, value: Value, recovery_strategies: &mut Vec) -> Value { match value { Value::String(s) => { // 检查字符串是否为YAML格式，如果是则尝试解析 if let Ok(yaml_value) = self.try_parse_yaml(&s) { recovery_strategies.push("YamlStringParsing".to_string()); debug!("Converted string field to YAML object/array"); yaml_value } else { Value::String(s) } } Value::Object(obj) => { // 递归处理对象中的每个字段 let mut new_obj = Map::new(); for (key, val) in obj { new_obj.insert(key, self.process_yaml_in_json_value(val, recovery_strategies)); } Value::Object(new_obj) } Value::Array(arr) => { // 递归处理数组中的每个元素 let new_arr: Vec = arr.into_iter() .map(|val| self.process_yaml_in_json_value(val, recovery_strategies)) .collect(); Value::Array(new_arr) } _ => value, // 其他类型直接返回 } } } /// 带缓存的容错JSON解析器 pub struct CachedTolerantJsonParser { parser: Arc>, cache: Arc>>, max_cache_size: usize, } impl CachedTolerantJsonParser { /// 创建新的带缓存的解析器实例 pub fn new(config: Option, max_cache_size: Option) -> Result { let parser = TolerantJsonParser::new(config)?; let max_cache_size = max_cache_size.unwrap_or(1000); Ok(Self { parser: Arc::new(Mutex::new(parser)), cache: Arc::new(Mutex::new(HashMap::new())), max_cache_size, }) } /// 解析JSON文本，使用缓存机制 pub fn parse(&self, text: &str) -> Result<(Value, ParseStatistics)> { // 计算文本哈希作为缓存键 let cache_key = self.compute_cache_key(text); // 检查缓存 { let cache = self.cache.lock().unwrap(); if let Some(cached_result) = cache.get(&cache_key) { debug!("Cache hit for JSON parsing"); return Ok(cached_result.clone()); } } // 缓存未命中，执行解析 let result = { let mut parser = self.parser.lock().unwrap(); parser.parse(text)? }; // 存储到缓存 { let mut cache = self.cache.lock().unwrap(); // 如果缓存已满，移除最旧的条目 if cache.len() >= self.max_cache_size { // 简单的LRU实现：移除第一个条目 if let Some(first_key) = cache.keys().next().cloned() { cache.remove(&first_key); } } cache.insert(cache_key, result.clone()); } debug!("Cached new JSON parsing result"); Ok(result) } /// 计算缓存键 fn compute_cache_key(&self, text: &str) -> String { use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; let mut hasher = DefaultHasher::new(); text.hash(&mut hasher); format!("{:x}", hasher.finish()) } /// 清空缓存 pub fn clear_cache(&self) { let mut cache = self.cache.lock().unwrap(); cache.clear(); info!("JSON parser cache cleared"); } /// 获取缓存统计信息 pub fn get_cache_stats(&self) -> (usize, usize) { let cache = self.cache.lock().unwrap(); (cache.len(), self.max_cache_size) } } #[cfg(test)] mod tests { use super::*; use serde_json::json; /// 创建测试用的解析器实例 fn create_test_parser() -> TolerantJsonParser { TolerantJsonParser::new(None).expect("Failed to create test parser") } #[test] fn test_standard_json_parsing() { let mut parser = create_test_parser(); // Test simple cases first let simple_cases = vec![ (r#"true"#, json!(true)), (r#"false"#, json!(false)), (r#"null"#, json!(null)), (r#"42"#, json!(42)), (r#""hello world""#, json!("hello world")), ]; for (input, expected) in simple_cases { let result = parser.parse(input); assert!(result.is_ok(), "Failed to parse: {}", input); let (parsed_value, stats) = result.unwrap(); println!("Input: {}, Parsed: {:?}, Expected: {:?}", input, parsed_value, expected); assert_eq!(parsed_value, expected, "Mismatch for input: {}", input); assert_eq!(stats.error_rate, 0.0, "Should have no errors for valid JSON"); } // Test more complex cases let complex_cases = vec![ (r#"{"name": "test"}"#, json!({"name": "test"})), (r#"[1, 2, 3]"#, json!([1, 2, 3])), ]; for (input, expected) in complex_cases { let result = parser.parse(input); assert!(result.is_ok(), "Failed to parse: {}", input); let (parsed_value, _stats) = result.unwrap(); println!("Input: {}, Parsed: {:?}, Expected: {:?}", input, parsed_value, expected); assert_eq!(parsed_value, expected, "Mismatch for input: {}", input); } } #[test] fn test_markdown_wrapped_json() { let mut parser = create_test_parser(); // Simple test case let input = r#"```json {"name": "test"} ```"#; let expected = json!({"name": "test"}); let result = parser.parse(input); assert!(result.is_ok(), "Failed to parse markdown wrapped JSON: {}", input); let (parsed_value, _) = result.unwrap(); println!("Markdown Input: {}", input); println!("Parsed: {:?}", parsed_value); println!("Expected: {:?}", expected); assert_eq!(parsed_value, expected, "Mismatch for markdown input: {}", input); } #[test] fn test_empty_input() { let mut parser = create_test_parser(); let result = parser.parse(""); assert!(result.is_ok(), "Should handle empty input"); let (parsed_value, _) = result.unwrap(); assert_eq!(parsed_value, json!(null), "Empty input should parse to null"); } #[test] fn test_cached_parser() { let cached_parser = CachedTolerantJsonParser::new(None, Some(10)).unwrap(); // Test with simple object first let test_json = r#"{"name": "test"}"#; let expected = json!({"name": "test"}); println!("Testing cached parser with: {}", test_json); // First parse - should cache the result let result1 = cached_parser.parse(test_json); assert!(result1.is_ok(), "First parse failed"); let (parsed1, _) = result1.unwrap(); println!("First parse result: {:?}", parsed1); assert_eq!(parsed1, expected); // Second parse - should use cache let result2 = cached_parser.parse(test_json); assert!(result2.is_ok(), "Second parse failed"); let (parsed2, _) = result2.unwrap(); println!("Second parse result: {:?}", parsed2); assert_eq!(parsed2, expected); // Check cache stats let (cache_size, max_size) = cached_parser.get_cache_stats(); assert_eq!(cache_size, 1); assert_eq!(max_size, 10); // Clear cache cached_parser.clear_cache(); let (cache_size_after_clear, _) = cached_parser.get_cache_stats(); assert_eq!(cache_size_after_clear, 0); } #[test] fn test_error_recovery_strategies() { let mut parser = create_test_parser(); // Test unquoted keys - simplified let unquoted_json = r#"{name: "test"}"#; let result = parser.parse(unquoted_json); println!("Unquoted keys test input: {}", unquoted_json); assert!(result.is_ok(), "Unquoted keys parsing failed"); let (parsed, stats) = result.unwrap(); println!("Unquoted keys result: {:?}", parsed); println!("Recovery strategies used: {:?}", stats.recovery_strategies_used); assert_eq!(parsed, json!({"name": "test"})); // Test trailing commas - simplified let trailing_comma_json = r#"{"name": "test",}"#; let result = parser.parse(trailing_comma_json); println!("Trailing comma test input: {}", trailing_comma_json); assert!(result.is_ok(), "Trailing comma parsing failed"); let (parsed, stats) = result.unwrap(); println!("Trailing comma result: {:?}", parsed); println!("Recovery strategies used: {:?}", stats.recovery_strategies_used); assert_eq!(parsed, json!({"name": "test"})); } #[test] fn test_complex_nested_json_extraction() { let mut parser = create_test_parser(); // 测试复杂嵌套JSON（类似Gemini响应的结构） let complex_json = r#"```json { "environment_tags": ["Indoor", "Retail environment"], "environment_color_pattern": { "hue": 0.08, "saturation": 0.05, "value": 0.6 }, "dress_color_pattern": { "hue": 0.58, "saturation": 0.28, "value": 0.85 }, "style_description": "整体呈现休闲舒适的风格", "products": [ { "category": "下装", "color_pattern": { "hue": 0.6, "saturation": 0.35, "value": 0.85 }, "design_styles": ["休闲", "复古", "街头"] } ] } ```"#; let result = parser.parse(complex_json); assert!(result.is_ok(), "Complex nested JSON parsing failed"); let (parsed, stats) = result.unwrap(); assert!(parsed.is_object()); let obj = parsed.as_object().unwrap(); // 检查顶级字段 assert!(obj.contains_key("environment_tags")); assert!(obj.contains_key("environment_color_pattern")); assert!(obj.contains_key("dress_color_pattern")); assert!(obj.contains_key("style_description")); assert!(obj.contains_key("products")); // 检查嵌套对象 let env_color = obj.get("environment_color_pattern").unwrap(); assert!(env_color.is_object()); // 检查数组 let products = obj.get("products").unwrap(); assert!(products.is_array()); let products_array = products.as_array().unwrap(); assert_eq!(products_array.len(), 1); println!("✅ 复杂嵌套JSON解析测试通过"); println!("📊 解析统计: 总节点数={}, 错误节点数={}, 错误率={:.2}%, 解析时间={}ms", stats.total_nodes, stats.error_nodes, stats.error_rate, stats.parse_time_ms); } #[test] fn test_bracket_matching_method() { let parser = create_test_parser(); // 测试括号匹配功能 let text_with_nested_braces = r#"Some text before { "outer": { "inner": { "deep": "value" } }, "array": [1, 2, {"nested": true}] } some text after"#; let result = parser.extract_json_by_bracket_matching(text_with_nested_braces); assert!(result.is_some()); let json_str = result.unwrap(); let parsed: Value = serde_json::from_str(&json_str).unwrap(); assert!(parsed.is_object()); let obj = parsed.as_object().unwrap(); assert!(obj.contains_key("outer")); assert!(obj.contains_key("array")); println!("✅ 括号匹配测试通过"); } #[test] fn test_yaml_detection_and_parsing() { let mut parser = create_test_parser(); // 测试简单的YAML键值对 let yaml_content = r#"name: John Doe age: 30 city: New York"#; println!("🔍 测试YAML内容: {}", yaml_content); println!("🔍 YAML检测结果: {}", parser.is_yaml_format(yaml_content)); let json_with_yaml = format!(r#"{{"config": "{}"}}"#, yaml_content); println!("🔍 完整JSON: {}", json_with_yaml); let result = parser.parse(&json_with_yaml); assert!(result.is_ok(), "YAML detection parsing failed"); let (parsed, stats) = result.unwrap(); assert!(parsed.is_object()); let obj = parsed.as_object().unwrap(); assert!(obj.contains_key("config")); // 检查YAML是否被正确解析为对象 let config_value = obj.get("config").unwrap(); println!("🔍 解析后的config值类型: {:?}", config_value); if config_value.is_object() { let config_obj = config_value.as_object().unwrap(); assert!(config_obj.contains_key("name")); assert!(config_obj.contains_key("age")); assert!(config_obj.contains_key("city")); assert_eq!(config_obj.get("name").unwrap(), &Value::String("John Doe".to_string())); assert_eq!(config_obj.get("age").unwrap(), &Value::Number(serde_json::Number::from(30))); assert_eq!(config_obj.get("city").unwrap(), &Value::String("New York".to_string())); println!("✅ YAML自动解析测试通过"); } else { println!("⚠️ YAML内容未被解析，保持为字符串格式"); println!("🔍 实际值: {:?}", config_value); } println!("📊 YAML解析统计: 恢复策略={:?}", stats.recovery_strategies_used); } #[test] fn test_yaml_list_parsing() { let mut parser = create_test_parser(); // 测试YAML列表格式 let yaml_list = r#"- item1 - item2 - item3"#; let json_with_yaml_list = format!(r#"{{"items": "{}"}}"#, yaml_list); let result = parser.parse(&json_with_yaml_list); assert!(result.is_ok(), "YAML list parsing failed"); let (parsed, _) = result.unwrap(); let obj = parsed.as_object().unwrap(); let items_value = obj.get("items").unwrap(); if items_value.is_array() { let items_array = items_value.as_array().unwrap(); assert_eq!(items_array.len(), 3); assert_eq!(items_array[0], Value::String("item1".to_string())); assert_eq!(items_array[1], Value::String("item2".to_string())); assert_eq!(items_array[2], Value::String("item3".to_string())); println!("✅ YAML列表自动解析测试通过"); } else { println!("⚠️ YAML列表内容未被解析，保持为字符串格式"); } } #[test] fn test_complex_yaml_parsing() { let mut parser = create_test_parser(); // 测试复杂的YAML结构 let complex_yaml = r#"database: host: localhost port: 5432 credentials: username: admin password: secret features: - authentication - logging - monitoring"#; let json_with_complex_yaml = format!(r#"{{"config": "{}"}}"#, complex_yaml); let result = parser.parse(&json_with_complex_yaml); assert!(result.is_ok(), "Complex YAML parsing failed"); let (parsed, _) = result.unwrap(); let obj = parsed.as_object().unwrap(); let config_value = obj.get("config").unwrap(); if config_value.is_object() { let config_obj = config_value.as_object().unwrap(); assert!(config_obj.contains_key("database")); assert!(config_obj.contains_key("features")); // 检查嵌套对象 let database = config_obj.get("database").unwrap().as_object().unwrap(); assert!(database.contains_key("host")); assert!(database.contains_key("credentials")); // 检查数组 let features = config_obj.get("features").unwrap().as_array().unwrap(); assert_eq!(features.len(), 3); println!("✅ 复杂YAML结构自动解析测试通过"); } else { println!("⚠️ 复杂YAML内容未被解析，保持为字符串格式"); } } #[test] fn test_direct_yaml_parsing() { let parser = create_test_parser(); // 测试直接YAML解析功能 let yaml_content = r#"name: John Doe age: 30 city: New York"#; println!("🔍 直接测试YAML解析"); println!("🔍 YAML内容: {}", yaml_content); println!("🔍 YAML检测结果: {}", parser.is_yaml_format(yaml_content)); let yaml_result = parser.try_parse_yaml(yaml_content); assert!(yaml_result.is_ok(), "直接YAML解析失败"); let yaml_value = yaml_result.unwrap(); assert!(yaml_value.is_object()); let yaml_obj = yaml_value.as_object().unwrap(); assert!(yaml_obj.contains_key("name")); assert!(yaml_obj.contains_key("age")); assert!(yaml_obj.contains_key("city")); println!("✅ 直接YAML解析功能正常"); // 测试YAML列表 let yaml_list = r#"- item1 - item2 - item3"#; println!("🔍 测试YAML列表: {}", yaml_list); let list_result = parser.try_parse_yaml(yaml_list); assert!(list_result.is_ok(), "YAML列表解析失败"); let list_value = list_result.unwrap(); assert!(list_value.is_array()); let list_array = list_value.as_array().unwrap(); assert_eq!(list_array.len(), 3); println!("✅ YAML列表解析功能正常"); } #[test] fn test_yaml_functionality_demo() { let mut parser = create_test_parser(); println!("\n🎯 YAML功能演示"); println!("================"); // 演示1：YAML列表自动解析 let json_with_yaml_list = r#"{"tags": "- frontend\n- backend\n- database"}"#; println!("\n📝 测试1: YAML列表自动解析"); println!("输入: {}", json_with_yaml_list); let result = parser.parse(json_with_yaml_list).unwrap(); let tags = result.0.get("tags").unwrap(); if tags.is_array() { println!("✅ 成功解析为数组: {:?}", tags.as_array().unwrap()); } else { println!("⚠️ 保持为字符串: {:?}", tags.as_str().unwrap()); } // 演示2：直接YAML解析API println!("\n📝 测试2: 直接YAML解析API"); let yaml_content = "name: 测试项目\nversion: 1.0\nfeatures:\n - auth\n - logging"; println!("YAML内容:\n{}", yaml_content); if let Ok(parsed_yaml) = parser.try_parse_yaml(yaml_content) { println!("✅ 解析成功: {}", serde_json::to_string_pretty(&parsed_yaml).unwrap()); } else { println!("❌ 解析失败"); } // 演示3：YAML检测功能 println!("\n📝 测试3: YAML检测功能"); let test_cases = vec![ ("name: value", "YAML键值对"), ("- item1\n- item2", "YAML列表"), ("{\"key\": \"value\"}", "JSON对象"), ("[1, 2, 3]", "JSON数组"), ("simple text", "普通文本"), ]; for (content, description) in test_cases { let is_yaml = parser.is_yaml_format(content); println!(" {} -> {} ({})", content.replace('\n', "\\n"), if is_yaml { "✅ YAML" } else { "❌ 非YAML" }, description); } println!("\n🎉 YAML功能演示完成！"); } #[test] fn test_retrieved_context_yaml_parsing() { let mut parser = create_test_parser(); println!("\n🔍 测试retrievedContext.text中的YAML解析"); println!("=========================================="); // 模拟promptx/res(1).json中的retrievedContext.text内容 let yaml_content = "categories:\n - 修身\n - 抹胸\n - 高腰\n - 宽松\n - 涤纶\n - 西装外套\n - 吊带背心\n - 运动短裤\n - 短款上衣\n - 衬衫\n - 圆领\n - 棉麻混纺\n - 抽绳\n - 休闲\n - 短袖T恤\n - 纯色\n - 通勤\n - 棉质\n - 阔腿短裤\ndescription: 三位女性模特在城市街道背景下展示夏季时尚穿搭，她们都穿着黑白配色的服装，风格各异，展现出休闲与时尚并存的夏日风情。\nenvironment_color_pattern:\n Hue: 0.10000000000000001\n Saturation: 0.080000000000000002\n Value: 0.75\n rgb_hex: \"#bfb9af\"\nenvironment_tags:\n - 城市街道\n - 夏季\n - 晴朗\n - 欧洲风格建筑\n - 户外\nmodels:\n - dress_color_pattern:\n Hue: 0\n Saturation: 0\n Value: 0.5\n rgb_hex: \"#7f7f7f\"\n position: 画面左侧\n posture: 站立，身体略微侧向右侧"; println!("🔍 YAML内容长度: {} 字符", yaml_content.len()); println!("🔍 YAML检测结果: {}", parser.is_yaml_format(yaml_content)); // 测试直接YAML解析 match parser.try_parse_yaml(yaml_content) { Ok(yaml_value) => { println!("✅ 直接YAML解析成功"); println!("🔍 解析结果类型: {:?}", yaml_value); if let Some(obj) = yaml_value.as_object() { println!("🔍 顶级字段: {:?}", obj.keys().collect::>()); // 检查categories字段 if let Some(categories) = obj.get("categories") { if categories.is_array() { println!("✅ categories字段解析为数组，长度: {}", categories.as_array().unwrap().len()); } } // 检查environment_color_pattern字段 if let Some(env_pattern) = obj.get("environment_color_pattern") { if env_pattern.is_object() { println!("✅ environment_color_pattern字段解析为对象"); } } } } Err(e) => { println!("❌ 直接YAML解析失败: {}", e); } } // 测试在JSON中包含此YAML内容 let json_with_yaml = format!(r#"{{"retrievedContext": {{"text": "{}"}}}}"#, yaml_content.replace('\n', "\\n").replace('"', "\\\"")); println!("\n🔍 测试JSON中包含YAML内容的解析"); println!("🔍 JSON长度: {} 字符", json_with_yaml.len()); match parser.parse(&json_with_yaml) { Ok((parsed, stats)) => { println!("✅ JSON解析成功"); println!("📊 解析统计: 恢复策略={:?}", stats.recovery_strategies_used); if let Some(obj) = parsed.as_object() { if let Some(retrieved_context) = obj.get("retrievedContext") { if let Some(context_obj) = retrieved_context.as_object() { if let Some(text_value) = context_obj.get("text") { match text_value { serde_json::Value::Object(_) => { println!("✅ text字段被解析为YAML对象"); } serde_json::Value::String(s) => { println!("⚠️ text字段保持为字符串，长度: {}", s.len()); } _ => { println!("❓ text字段类型未知: {:?}", text_value); } } } } } } } Err(e) => { println!("❌ JSON解析失败: {}", e); } } println!("\n🎉 retrievedContext.text YAML解析测试完成！"); } #[test] fn test_real_promptx_res_file() { let mut parser = create_test_parser(); println!("\n🔍 测试真实的promptx/res(1).json文件"); println!("====================================="); // 读取真实的promptx/res(1).json文件 let file_path = "../../../promptx/res(1).json"; match std::fs::read_to_string(file_path) { Ok(content) => { println!("🔍 文件读取成功，长度: {} 字符", content.len()); match parser.parse(&content) { Ok((parsed, stats)) => { println!("✅ JSON解析成功"); println!("📊 解析统计: 恢复策略={:?}", stats.recovery_strategies_used); println!("📊 错误率: {:.2}%", stats.error_rate * 100.0); println!("📊 解析时间: {}ms", stats.parse_time_ms); // 检查是否包含YAML解析策略 if stats.recovery_strategies_used.contains(&"YamlStringParsing".to_string()) { println!("✅ 检测到YAML字符串解析"); } else { println!("⚠️ 未检测到YAML字符串解析"); } // 检查第一个retrievedContext.text字段 if let Some(obj) = parsed.as_object() { if let Some(candidates) = obj.get("candidates") { if let Some(candidates_array) = candidates.as_array() { if let Some(first_candidate) = candidates_array.first() { if let Some(candidate_obj) = first_candidate.as_object() { if let Some(grounding_metadata) = candidate_obj.get("groundingMetadata") { if let Some(grounding_obj) = grounding_metadata.as_object() { if let Some(grounding_chunks) = grounding_obj.get("groundingChunks") { if let Some(chunks_array) = grounding_chunks.as_array() { if let Some(first_chunk) = chunks_array.first() { if let Some(chunk_obj) = first_chunk.as_object() { if let Some(retrieved_context) = chunk_obj.get("retrievedContext") { if let Some(context_obj) = retrieved_context.as_object() { if let Some(text_value) = context_obj.get("text") { match text_value { serde_json::Value::Object(text_obj) => { println!("✅ 第一个retrievedContext.text被解析为YAML对象"); println!("🔍 YAML对象字段: {:?}", text_obj.keys().collect::>()); // 检查categories字段 if let Some(categories) = text_obj.get("categories") { if let Some(categories_array) = categories.as_array() { println!("✅ categories字段包含 {} 个元素", categories_array.len()); } } } serde_json::Value::String(s) => { println!("⚠️ 第一个retrievedContext.text仍为字符串，长度: {}", s.len()); println!("🔍 前100字符: {}", &s[..std::cmp::min(100, s.len())]); } _ => { println!("❓ 第一个retrievedContext.text类型未知: {:?}", text_value); } } } } } } } } } } } } } } } } } Err(e) => { println!("❌ JSON解析失败: {}", e); } } } Err(e) => { println!("⚠️ 无法读取文件 {}: {}", file_path, e); println!("💡 这是正常的，因为测试可能在不同的工作目录运行"); } } println!("\n🎉 真实文件测试完成！"); } }