1688 lines
64 KiB
Rust
1688 lines
64 KiB
Rust
/// 基于Tree-sitter的大模型JSON容错解析器
|
||
/// 遵循Tauri开发规范，提供高性能、安全的JSON解析能力
|
||
///
|
||
/// ## 新增功能：YAML格式支持
|
||
///
|
||
/// ### 功能概述
|
||
/// - 当字符串值被检测为YAML格式时，自动解析为对应的JSON结构
|
||
/// - 支持YAML键值对、列表、嵌套结构等常见格式
|
||
/// - 智能检测避免误判JSON内容为YAML
|
||
///
|
||
/// ### YAML检测规则
|
||
/// 1. **键值对检测**：包含 `key: value` 格式的多行内容
|
||
/// 2. **列表检测**：包含 `- item` 格式的行
|
||
/// 3. **多行检测**：包含换行符和冒号的内容
|
||
/// 4. **排除规则**：以 `{` 或 `[` 开头的内容不会被识别为YAML
|
||
///
|
||
/// ### 使用场景
|
||
/// - 处理大模型返回的混合格式内容
|
||
/// - 解析配置文件中的YAML字段
|
||
/// - 容错处理包含YAML片段的JSON数据
|
||
///
|
||
/// ### 示例
|
||
/// ```rust
|
||
/// // JSON中包含YAML字符串会被自动解析
|
||
/// let json_with_yaml = r#"{"config": "name: John\nage: 30"}"#;
|
||
/// // 解析后 config 字段会变成 {"name": "John", "age": 30}
|
||
///
|
||
/// let yaml_list = r#"{"items": "- item1\n- item2\n- item3"}"#;
|
||
/// // 解析后 items 字段会变成 ["item1", "item2", "item3"]
|
||
/// ```
|
||
use anyhow::{anyhow, Result};
|
||
use regex::Regex;
|
||
use serde_json::{Map, Value};
|
||
use serde_yaml;
|
||
use std::collections::HashMap;
|
||
use std::sync::{Arc, Mutex};
|
||
use tree_sitter::{Node, Parser, Tree};
|
||
use tracing::{debug, info, warn};
|
||
|
||
/// 解析结果统计信息
|
||
#[derive(Debug, Clone, serde::Serialize)]
|
||
pub struct ParseStatistics {
|
||
pub total_nodes: usize,
|
||
pub error_nodes: usize,
|
||
pub error_rate: f64,
|
||
pub parse_time_ms: u64,
|
||
pub recovery_strategies_used: Vec<String>,
|
||
}
|
||
|
||
/// 错误恢复策略类型
|
||
#[derive(Debug, Clone)]
|
||
pub enum RecoveryStrategy {
|
||
StandardJson,
|
||
ManualFix,
|
||
RegexExtract,
|
||
PartialParse,
|
||
}
|
||
|
||
/// 容错JSON解析器配置
|
||
#[derive(Debug, Clone)]
|
||
pub struct ParserConfig {
|
||
pub max_text_length: usize,
|
||
pub enable_comments: bool,
|
||
pub enable_unquoted_keys: bool,
|
||
pub enable_trailing_commas: bool,
|
||
pub timeout_ms: u64,
|
||
pub recovery_strategies: Vec<RecoveryStrategy>,
|
||
}
|
||
|
||
impl Default for ParserConfig {
|
||
fn default() -> Self {
|
||
Self {
|
||
max_text_length: 1024 * 1024, // 1MB
|
||
enable_comments: true,
|
||
enable_unquoted_keys: true,
|
||
enable_trailing_commas: true,
|
||
timeout_ms: 30000, // 30秒
|
||
recovery_strategies: vec![
|
||
RecoveryStrategy::StandardJson,
|
||
RecoveryStrategy::ManualFix,
|
||
RecoveryStrategy::RegexExtract,
|
||
RecoveryStrategy::PartialParse,
|
||
],
|
||
}
|
||
}
|
||
}
|
||
|
||
/// 容错JSON解析器
|
||
pub struct TolerantJsonParser {
|
||
parser: Parser,
|
||
config: ParserConfig,
|
||
regex_patterns: HashMap<String, Regex>,
|
||
}
|
||
|
||
impl TolerantJsonParser {
|
||
/// 创建新的容错JSON解析器实例
|
||
pub fn new(config: Option<ParserConfig>) -> Result<Self> {
|
||
let mut parser = Parser::new();
|
||
|
||
// 设置JSON语言
|
||
let language = tree_sitter_json::language();
|
||
parser.set_language(language)
|
||
.map_err(|e| anyhow!("Failed to set JSON language: {}", e))?;
|
||
|
||
let config = config.unwrap_or_default();
|
||
let regex_patterns = Self::init_regex_patterns()?;
|
||
|
||
Ok(Self {
|
||
parser,
|
||
config,
|
||
regex_patterns,
|
||
})
|
||
}
|
||
|
||
/// 初始化正则表达式模式
|
||
fn init_regex_patterns() -> Result<HashMap<String, Regex>> {
|
||
let mut patterns = HashMap::new();
|
||
|
||
// JSON对象模式 - 使用简单但可靠的方法
|
||
// Rust的regex crate不支持递归正则，所以使用贪婪匹配
|
||
patterns.insert(
|
||
"object".to_string(),
|
||
Regex::new(r"(?s)\{.*\}")?,
|
||
);
|
||
|
||
// JSON数组模式 - 使用简单但可靠的方法
|
||
patterns.insert(
|
||
"array".to_string(),
|
||
Regex::new(r"(?s)\[.*\]")?,
|
||
);
|
||
|
||
// Markdown代码块模式
|
||
patterns.insert(
|
||
"markdown_fence".to_string(),
|
||
Regex::new(r"(?s)```(?:json)?\s*\n?(.*?)\n?```")?,
|
||
);
|
||
|
||
// 无引号键模式
|
||
patterns.insert(
|
||
"unquoted_key".to_string(),
|
||
Regex::new(r"(\w+):")?,
|
||
);
|
||
|
||
// 尾随逗号模式
|
||
patterns.insert(
|
||
"trailing_comma".to_string(),
|
||
Regex::new(r",\s*[}\]]")?,
|
||
);
|
||
|
||
// YAML检测模式 - 检测常见的YAML结构（键值对）
|
||
patterns.insert(
|
||
"yaml_structure".to_string(),
|
||
Regex::new(r"(?m)^[\s]*[\w\-\.]+:\s*.*$")?,
|
||
);
|
||
|
||
// YAML列表模式
|
||
patterns.insert(
|
||
"yaml_list".to_string(),
|
||
Regex::new(r"(?m)^[\s]*-\s+")?,
|
||
);
|
||
|
||
// YAML多行字符串模式
|
||
patterns.insert(
|
||
"yaml_multiline".to_string(),
|
||
Regex::new(r"(?s)[|>][\s]*\n")?,
|
||
);
|
||
|
||
Ok(patterns)
|
||
}
|
||
|
||
/// 解析JSON文本，返回解析结果和统计信息
|
||
pub fn parse(&mut self, text: &str) -> Result<(Value, ParseStatistics)> {
|
||
let start_time = std::time::Instant::now();
|
||
|
||
// 检查文本长度限制
|
||
if text.len() > self.config.max_text_length {
|
||
return Err(anyhow!("Text too long: {} bytes", text.len()));
|
||
}
|
||
|
||
// 首先尝试标准JSON解析
|
||
match serde_json::from_str::<Value>(text) {
|
||
Ok(result) => {
|
||
// 标准JSON解析成功，但还需要检查字符串字段是否包含YAML内容
|
||
let mut recovery_strategies = vec!["StandardJson".to_string()];
|
||
let processed_result = self.process_yaml_in_json_value(result, &mut recovery_strategies);
|
||
|
||
let stats = ParseStatistics {
|
||
total_nodes: 1,
|
||
error_nodes: 0,
|
||
error_rate: 0.0,
|
||
parse_time_ms: start_time.elapsed().as_millis() as u64,
|
||
recovery_strategies_used: recovery_strategies,
|
||
};
|
||
return Ok((processed_result, stats));
|
||
}
|
||
Err(_) => {
|
||
// 标准解析失败，继续使用容错解析
|
||
debug!("Standard JSON parsing failed, using tolerant parsing");
|
||
}
|
||
}
|
||
|
||
// 预处理文本
|
||
let cleaned_text = self.preprocess_text(text)?;
|
||
debug!("Preprocessed text length: {}", cleaned_text.len());
|
||
|
||
// 使用Tree-sitter解析
|
||
let tree = self.parser.parse(&cleaned_text, None)
|
||
.ok_or_else(|| anyhow!("Failed to parse text with tree-sitter"))?;
|
||
|
||
let root_node = tree.root_node();
|
||
|
||
// 收集统计信息
|
||
let mut stats = self.collect_statistics(&tree, &cleaned_text);
|
||
|
||
// 如果错误率太高，回退到标准JSON解析的错误修复
|
||
if stats.error_rate > 0.1 {
|
||
debug!("High error rate ({:.2}%), attempting fallback strategies", stats.error_rate * 100.0);
|
||
|
||
// 尝试基本的文本清理后再用标准JSON解析
|
||
if let Ok(result) = self.try_fallback_parsing(text) {
|
||
stats.recovery_strategies_used.push("FallbackParsing".to_string());
|
||
stats.parse_time_ms = start_time.elapsed().as_millis() as u64;
|
||
info!("Fallback JSON parsing successful in {}ms", stats.parse_time_ms);
|
||
return Ok((result, stats));
|
||
}
|
||
}
|
||
|
||
// 提取和修复JSON
|
||
let mut recovery_strategies_used = Vec::new();
|
||
let result = self.extract_json(root_node, &cleaned_text, &mut recovery_strategies_used)?;
|
||
|
||
stats.parse_time_ms = start_time.elapsed().as_millis() as u64;
|
||
stats.recovery_strategies_used = recovery_strategies_used;
|
||
|
||
info!("JSON parsing completed in {}ms", stats.parse_time_ms);
|
||
|
||
Ok((result, stats))
|
||
}
|
||
|
||
/// 预处理文本，提取可能的JSON内容
|
||
fn preprocess_text(&self, text: &str) -> Result<String> {
|
||
let mut processed = text.to_string();
|
||
|
||
// 移除Markdown代码块标记
|
||
if let Some(markdown_regex) = self.regex_patterns.get("markdown_fence") {
|
||
if let Some(captures) = markdown_regex.captures(&processed) {
|
||
if let Some(json_content) = captures.get(1) {
|
||
processed = json_content.as_str().to_string();
|
||
debug!("Extracted JSON from markdown fence");
|
||
}
|
||
}
|
||
}
|
||
|
||
// 使用改进的JSON提取方法
|
||
processed = self.extract_json_content(&processed);
|
||
|
||
// 在Tree-sitter解析之前修复常见的JSON错误
|
||
processed = self.fix_common_json_errors(&processed);
|
||
|
||
Ok(processed)
|
||
}
|
||
|
||
/// 改进的JSON内容提取方法
|
||
fn extract_json_content(&self, text: &str) -> String {
|
||
// 首先尝试使用手动括号匹配（最可靠的方法）
|
||
if let Some(json_content) = self.extract_json_by_bracket_matching(text) {
|
||
debug!("Extracted JSON using bracket matching");
|
||
return json_content;
|
||
}
|
||
|
||
// 如果括号匹配失败，尝试使用正则表达式
|
||
for pattern_name in ["object", "array"] {
|
||
if let Some(pattern) = self.regex_patterns.get(pattern_name) {
|
||
if let Some(mat) = pattern.find(text) {
|
||
debug!("Extracted JSON using {} pattern", pattern_name);
|
||
return mat.as_str().to_string();
|
||
}
|
||
}
|
||
}
|
||
|
||
// 如果所有方法都失败，返回原始文本
|
||
text.to_string()
|
||
}
|
||
|
||
/// 使用括号匹配提取JSON内容
|
||
fn extract_json_by_bracket_matching(&self, text: &str) -> Option<String> {
|
||
// 查找第一个 { 或 [
|
||
let start_char = if text.contains('{') && text.contains('[') {
|
||
let brace_pos = text.find('{').unwrap_or(usize::MAX);
|
||
let bracket_pos = text.find('[').unwrap_or(usize::MAX);
|
||
if brace_pos < bracket_pos { '{' } else { '[' }
|
||
} else if text.contains('{') {
|
||
'{'
|
||
} else if text.contains('[') {
|
||
'['
|
||
} else {
|
||
return None;
|
||
};
|
||
|
||
let end_char = if start_char == '{' { '}' } else { ']' };
|
||
|
||
if let Some(start_pos) = text.find(start_char) {
|
||
let mut depth = 0;
|
||
let mut in_string = false;
|
||
let mut escape_next = false;
|
||
|
||
for (i, ch) in text[start_pos..].char_indices() {
|
||
if escape_next {
|
||
escape_next = false;
|
||
continue;
|
||
}
|
||
|
||
match ch {
|
||
'\\' if in_string => escape_next = true,
|
||
'"' => in_string = !in_string,
|
||
c if c == start_char && !in_string => depth += 1,
|
||
c if c == end_char && !in_string => {
|
||
depth -= 1;
|
||
if depth == 0 {
|
||
return Some(text[start_pos..start_pos + i + 1].to_string());
|
||
}
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
}
|
||
|
||
None
|
||
}
|
||
|
||
/// 修复常见的JSON错误
|
||
fn fix_common_json_errors(&self, text: &str) -> String {
|
||
let mut fixed = text.to_string();
|
||
|
||
// 检查是否包含YAML内容，如果是则跳过某些修复
|
||
let contains_yaml = self.is_yaml_format(text);
|
||
|
||
if contains_yaml {
|
||
debug!("Detected YAML content, skipping unquoted key fixes");
|
||
} else {
|
||
// 修复无引号的键
|
||
if self.config.enable_unquoted_keys {
|
||
if let Some(unquoted_regex) = self.regex_patterns.get("unquoted_key") {
|
||
fixed = unquoted_regex.replace_all(&fixed, "\"$1\":").to_string();
|
||
debug!("Fixed unquoted keys");
|
||
}
|
||
}
|
||
}
|
||
|
||
// 移除尾随逗号
|
||
if self.config.enable_trailing_commas {
|
||
if let Some(trailing_regex) = self.regex_patterns.get("trailing_comma") {
|
||
fixed = trailing_regex.replace_all(&fixed, |caps: ®ex::Captures| {
|
||
let full_match = caps.get(0).unwrap().as_str();
|
||
full_match[1..].to_string() // 移除逗号，保留括号
|
||
}).to_string();
|
||
debug!("Fixed trailing commas");
|
||
}
|
||
}
|
||
|
||
// 修复单引号为双引号（但不对YAML内容进行此修复）
|
||
if !contains_yaml {
|
||
fixed = fixed.replace("'", "\"");
|
||
}
|
||
|
||
debug!("Fixed text: {}", &fixed[..std::cmp::min(200, fixed.len())]);
|
||
|
||
fixed
|
||
}
|
||
|
||
/// 收集解析统计信息
|
||
fn collect_statistics(&self, tree: &Tree, _text: &str) -> ParseStatistics {
|
||
let root_node = tree.root_node();
|
||
let mut total_nodes = 0;
|
||
let mut error_nodes = 0;
|
||
|
||
self.count_nodes(root_node, &mut total_nodes, &mut error_nodes);
|
||
|
||
let error_rate = if total_nodes > 0 {
|
||
error_nodes as f64 / total_nodes as f64
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
ParseStatistics {
|
||
total_nodes,
|
||
error_nodes,
|
||
error_rate,
|
||
parse_time_ms: 0, // 将在parse方法中设置
|
||
recovery_strategies_used: Vec::new(), // 将在parse方法中设置
|
||
}
|
||
}
|
||
|
||
/// 递归计算节点数量
|
||
fn count_nodes(&self, node: Node, total: &mut usize, errors: &mut usize) {
|
||
*total += 1;
|
||
|
||
if node.is_error() {
|
||
*errors += 1;
|
||
}
|
||
|
||
for child in node.children(&mut node.walk()) {
|
||
self.count_nodes(child, total, errors);
|
||
}
|
||
}
|
||
|
||
/// 从语法树中提取和修复JSON
|
||
fn extract_json(
|
||
&self,
|
||
node: Node,
|
||
text: &str,
|
||
recovery_strategies: &mut Vec<String>,
|
||
) -> Result<Value> {
|
||
match node.kind() {
|
||
"document" => {
|
||
if node.child_count() > 0 {
|
||
self.extract_json(node.child(0).unwrap(), text, recovery_strategies)
|
||
} else {
|
||
Ok(Value::Null)
|
||
}
|
||
}
|
||
"value" => {
|
||
if node.child_count() > 0 {
|
||
self.extract_json(node.child(0).unwrap(), text, recovery_strategies)
|
||
} else {
|
||
self.recover_from_error(node, text, recovery_strategies)
|
||
}
|
||
}
|
||
"object" => self.extract_object(node, text, recovery_strategies),
|
||
"array" => self.extract_array(node, text, recovery_strategies),
|
||
"string" => self.extract_string(node, text),
|
||
"number" => self.extract_number(node, text),
|
||
"true" | "false" => self.extract_boolean(node, text),
|
||
"null" => Ok(Value::Null),
|
||
"ERROR" => self.recover_from_error(node, text, recovery_strategies),
|
||
_ => {
|
||
warn!("Unknown node kind: {}", node.kind());
|
||
self.recover_from_error(node, text, recovery_strategies)
|
||
}
|
||
}
|
||
}
|
||
|
||
/// 提取JSON对象
|
||
fn extract_object(
|
||
&self,
|
||
node: Node,
|
||
text: &str,
|
||
recovery_strategies: &mut Vec<String>,
|
||
) -> Result<Value> {
|
||
let mut object = Map::new();
|
||
|
||
for child in node.children(&mut node.walk()) {
|
||
match child.kind() {
|
||
"pair" => {
|
||
if let Ok((key, value)) = self.extract_pair(child, text, recovery_strategies) {
|
||
object.insert(key, value);
|
||
}
|
||
}
|
||
"ERROR" => {
|
||
debug!("Found ERROR node in object");
|
||
// 尝试恢复错误的对象成员
|
||
if let Ok(recovered) = self.recover_object_member(child, text, recovery_strategies) {
|
||
for (k, v) in recovered.as_object().unwrap_or(&Map::new()) {
|
||
object.insert(k.clone(), v.clone());
|
||
}
|
||
} else {
|
||
// 如果对象成员恢复失败，尝试通用错误恢复
|
||
if let Ok(recovered) = self.recover_from_error(child, text, recovery_strategies) {
|
||
// 如果恢复的是一个对象，合并其内容
|
||
if let Some(recovered_obj) = recovered.as_object() {
|
||
for (k, v) in recovered_obj {
|
||
object.insert(k.clone(), v.clone());
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
_ => {
|
||
// 忽略其他节点类型（如括号、逗号等）
|
||
}
|
||
}
|
||
}
|
||
|
||
Ok(Value::Object(object))
|
||
}
|
||
|
||
/// 提取JSON数组
|
||
fn extract_array(
|
||
&self,
|
||
node: Node,
|
||
text: &str,
|
||
recovery_strategies: &mut Vec<String>,
|
||
) -> Result<Value> {
|
||
let mut array = Vec::new();
|
||
|
||
for child in node.children(&mut node.walk()) {
|
||
match child.kind() {
|
||
"value" => {
|
||
if let Ok(value) = self.extract_json(child, text, recovery_strategies) {
|
||
array.push(value);
|
||
}
|
||
}
|
||
// Also handle direct primitive types
|
||
"number" | "string" | "true" | "false" | "null" | "object" | "array" => {
|
||
if let Ok(value) = self.extract_json(child, text, recovery_strategies) {
|
||
array.push(value);
|
||
}
|
||
}
|
||
"ERROR" => {
|
||
// 尝试恢复错误的数组元素
|
||
if let Ok(recovered) = self.recover_from_error(child, text, recovery_strategies) {
|
||
array.push(recovered);
|
||
}
|
||
}
|
||
_ => {} // 忽略其他节点类型（如括号、逗号等）
|
||
}
|
||
}
|
||
|
||
Ok(Value::Array(array))
|
||
}
|
||
|
||
/// 提取键值对
|
||
fn extract_pair(
|
||
&self,
|
||
node: Node,
|
||
text: &str,
|
||
recovery_strategies: &mut Vec<String>,
|
||
) -> Result<(String, Value)> {
|
||
let mut key = None;
|
||
let mut value = None;
|
||
|
||
for child in node.children(&mut node.walk()) {
|
||
match child.kind() {
|
||
"string" => {
|
||
if key.is_none() {
|
||
let extracted_key = self.extract_string_content(child, text)?;
|
||
key = Some(extracted_key);
|
||
} else {
|
||
let extracted_value = self.extract_json(child, text, recovery_strategies)?;
|
||
value = Some(extracted_value);
|
||
}
|
||
}
|
||
"value" => {
|
||
let extracted_value = self.extract_json(child, text, recovery_strategies)?;
|
||
value = Some(extracted_value);
|
||
}
|
||
// 直接处理各种值类型
|
||
"object" | "array" | "number" | "true" | "false" | "null" => {
|
||
let extracted_value = self.extract_json(child, text, recovery_strategies)?;
|
||
value = Some(extracted_value);
|
||
}
|
||
_ => {
|
||
// 忽略冒号等分隔符
|
||
}
|
||
}
|
||
}
|
||
|
||
match (key, value) {
|
||
(Some(k), Some(v)) => Ok((k, v)),
|
||
_ => Err(anyhow!("Invalid key-value pair")),
|
||
}
|
||
}
|
||
|
||
/// 提取字符串值
|
||
fn extract_string(&self, node: Node, text: &str) -> Result<Value> {
|
||
let content = self.extract_string_content(node, text)?;
|
||
|
||
// 检测字符串是否为YAML格式，如果是则自动解析
|
||
if let Ok(yaml_value) = self.try_parse_yaml(&content) {
|
||
debug!("Detected and parsed YAML content in string field");
|
||
return Ok(yaml_value);
|
||
}
|
||
|
||
Ok(Value::String(content))
|
||
}
|
||
|
||
/// 提取字符串内容（去除引号）
|
||
fn extract_string_content(&self, node: Node, text: &str) -> Result<String> {
|
||
let raw_text = &text[node.start_byte()..node.end_byte()];
|
||
|
||
// 移除前后的引号
|
||
if raw_text.len() >= 2 && raw_text.starts_with('"') && raw_text.ends_with('"') {
|
||
Ok(raw_text[1..raw_text.len()-1].to_string())
|
||
} else {
|
||
// 处理无引号的字符串
|
||
Ok(raw_text.to_string())
|
||
}
|
||
}
|
||
|
||
/// 提取数字值
|
||
fn extract_number(&self, node: Node, text: &str) -> Result<Value> {
|
||
let raw_text = &text[node.start_byte()..node.end_byte()];
|
||
|
||
// 尝试解析为整数
|
||
if let Ok(int_val) = raw_text.parse::<i64>() {
|
||
Ok(Value::Number(serde_json::Number::from(int_val)))
|
||
}
|
||
// 尝试解析为浮点数
|
||
else if let Ok(float_val) = raw_text.parse::<f64>() {
|
||
if let Some(num) = serde_json::Number::from_f64(float_val) {
|
||
Ok(Value::Number(num))
|
||
} else {
|
||
Err(anyhow!("Invalid float number: {}", raw_text))
|
||
}
|
||
} else {
|
||
Err(anyhow!("Invalid number: {}", raw_text))
|
||
}
|
||
}
|
||
|
||
/// 提取布尔值
|
||
fn extract_boolean(&self, node: Node, text: &str) -> Result<Value> {
|
||
let raw_text = &text[node.start_byte()..node.end_byte()];
|
||
match raw_text {
|
||
"true" => Ok(Value::Bool(true)),
|
||
"false" => Ok(Value::Bool(false)),
|
||
_ => Err(anyhow!("Invalid boolean: {}", raw_text)),
|
||
}
|
||
}
|
||
|
||
/// 从错误节点恢复数据
|
||
fn recover_from_error(
|
||
&self,
|
||
node: Node,
|
||
text: &str,
|
||
recovery_strategies: &mut Vec<String>,
|
||
) -> Result<Value> {
|
||
let error_text = &text[node.start_byte()..node.end_byte()].trim();
|
||
debug!("Attempting to recover from error: {}", error_text);
|
||
|
||
// 首先尝试智能字符串恢复（针对复杂字符串内容）
|
||
if let Ok(result) = self.try_smart_string_recovery(error_text) {
|
||
recovery_strategies.push("SmartStringRecovery".to_string());
|
||
return Ok(result);
|
||
}
|
||
|
||
// 尝试不同的恢复策略
|
||
for strategy in &self.config.recovery_strategies {
|
||
match strategy {
|
||
RecoveryStrategy::StandardJson => {
|
||
if let Ok(result) = self.try_standard_json_parse(error_text) {
|
||
recovery_strategies.push("StandardJson".to_string());
|
||
return Ok(result);
|
||
}
|
||
}
|
||
RecoveryStrategy::ManualFix => {
|
||
if let Ok(result) = self.try_manual_fix(error_text) {
|
||
recovery_strategies.push("ManualFix".to_string());
|
||
return Ok(result);
|
||
}
|
||
}
|
||
RecoveryStrategy::RegexExtract => {
|
||
if let Ok(result) = self.try_regex_extract(error_text) {
|
||
recovery_strategies.push("RegexExtract".to_string());
|
||
return Ok(result);
|
||
}
|
||
}
|
||
RecoveryStrategy::PartialParse => {
|
||
if let Ok(result) = self.try_partial_parse(error_text) {
|
||
recovery_strategies.push("PartialParse".to_string());
|
||
return Ok(result);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 如果所有策略都失败，但内容不为空，保留原始文本作为字符串
|
||
// 这对于包含复杂内容（如YAML）的字段很重要
|
||
if !error_text.is_empty() {
|
||
warn!("All recovery strategies failed, preserving content as string: {} chars", error_text.len());
|
||
recovery_strategies.push("PreserveAsString".to_string());
|
||
Ok(Value::String(error_text.to_string()))
|
||
} else {
|
||
warn!("All recovery strategies failed for empty content");
|
||
Ok(Value::Null)
|
||
}
|
||
}
|
||
|
||
/// 尝试标准JSON解析
|
||
fn try_standard_json_parse(&self, text: &str) -> Result<Value> {
|
||
serde_json::from_str(text).map_err(|e| anyhow!("Standard JSON parse failed: {}", e))
|
||
}
|
||
|
||
/// 智能字符串恢复 - 专门处理复杂字符串内容，包括YAML格式
|
||
fn try_smart_string_recovery(&self, text: &str) -> Result<Value> {
|
||
let trimmed = text.trim();
|
||
|
||
// 首先尝试检测和解析YAML格式
|
||
if let Ok(yaml_value) = self.try_parse_yaml(trimmed) {
|
||
debug!("Recovered content as YAML");
|
||
return Ok(yaml_value);
|
||
}
|
||
|
||
// 检查是否是一个键值对格式
|
||
if let Some(colon_pos) = trimmed.find(':') {
|
||
let key_part = trimmed[..colon_pos].trim();
|
||
let value_part = trimmed[colon_pos+1..].trim();
|
||
|
||
// 清理键名（移除引号）
|
||
let clean_key = key_part.trim_matches(|c| c == '"' || c == '\'' || c == ' ');
|
||
|
||
// 首先尝试将值部分解析为YAML
|
||
if let Ok(yaml_value) = self.try_parse_yaml(value_part) {
|
||
let mut object = Map::new();
|
||
object.insert(clean_key.to_string(), yaml_value);
|
||
debug!("Recovered key-value pair with YAML value");
|
||
return Ok(Value::Object(object));
|
||
}
|
||
|
||
// 如果值部分是一个被引号包围的字符串
|
||
if value_part.starts_with('"') && value_part.len() > 1 {
|
||
// 寻找匹配的结束引号，考虑转义字符
|
||
let mut end_pos = None;
|
||
let mut chars = value_part[1..].char_indices();
|
||
let mut escaped = false;
|
||
|
||
while let Some((i, ch)) = chars.next() {
|
||
if escaped {
|
||
escaped = false;
|
||
continue;
|
||
}
|
||
|
||
match ch {
|
||
'\\' => escaped = true,
|
||
'"' => {
|
||
end_pos = Some(i + 1);
|
||
break;
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
|
||
let string_content = if let Some(end) = end_pos {
|
||
// 找到了结束引号
|
||
&value_part[1..end]
|
||
} else {
|
||
// 没有找到结束引号，取整个内容（去掉开头的引号）
|
||
&value_part[1..]
|
||
};
|
||
|
||
// 尝试将字符串内容解析为YAML
|
||
if let Ok(yaml_value) = self.try_parse_yaml(string_content) {
|
||
let mut object = Map::new();
|
||
object.insert(clean_key.to_string(), yaml_value);
|
||
debug!("Recovered quoted string content as YAML");
|
||
return Ok(Value::Object(object));
|
||
}
|
||
|
||
let mut object = Map::new();
|
||
object.insert(clean_key.to_string(), Value::String(string_content.to_string()));
|
||
return Ok(Value::Object(object));
|
||
}
|
||
}
|
||
|
||
Err(anyhow!("Smart string recovery failed"))
|
||
}
|
||
|
||
/// 回退解析策略 - 当Tree-sitter错误率过高时使用
|
||
fn try_fallback_parsing(&self, text: &str) -> Result<Value> {
|
||
// 尝试基本的文本清理
|
||
let mut cleaned = text.to_string();
|
||
|
||
// 移除可能的BOM
|
||
if cleaned.starts_with('\u{FEFF}') {
|
||
cleaned = cleaned[3..].to_string();
|
||
}
|
||
|
||
// 尝试修复常见的JSON错误
|
||
cleaned = self.fix_common_json_errors(&cleaned);
|
||
|
||
// 尝试解析清理后的文本
|
||
serde_json::from_str(&cleaned).map_err(|e| anyhow!("Fallback parsing failed: {}", e))
|
||
}
|
||
|
||
/// 尝试手动修复常见的JSON错误
|
||
fn try_manual_fix(&self, text: &str) -> Result<Value> {
|
||
let mut fixed_text = text.to_string();
|
||
|
||
// 修复无引号的键
|
||
if let Some(unquoted_regex) = self.regex_patterns.get("unquoted_key") {
|
||
fixed_text = unquoted_regex.replace_all(&fixed_text, "\"$1\":").to_string();
|
||
}
|
||
|
||
// 移除尾随逗号
|
||
if let Some(trailing_regex) = self.regex_patterns.get("trailing_comma") {
|
||
fixed_text = trailing_regex.replace_all(&fixed_text, |caps: ®ex::Captures| {
|
||
let full_match = caps.get(0).unwrap().as_str();
|
||
full_match[1..].to_string() // 移除逗号，保留括号
|
||
}).to_string();
|
||
}
|
||
|
||
// 修复单引号为双引号
|
||
fixed_text = fixed_text.replace("'", "\"");
|
||
|
||
// 尝试解析修复后的文本
|
||
serde_json::from_str(&fixed_text).map_err(|e| anyhow!("Manual fix failed: {}", e))
|
||
}
|
||
|
||
/// 尝试正则表达式提取
|
||
fn try_regex_extract(&self, text: &str) -> Result<Value> {
|
||
// 尝试提取对象
|
||
if let Some(object_regex) = self.regex_patterns.get("object") {
|
||
if let Some(mat) = object_regex.find(text) {
|
||
return self.try_standard_json_parse(mat.as_str());
|
||
}
|
||
}
|
||
|
||
// 尝试提取数组
|
||
if let Some(array_regex) = self.regex_patterns.get("array") {
|
||
if let Some(mat) = array_regex.find(text) {
|
||
return self.try_standard_json_parse(mat.as_str());
|
||
}
|
||
}
|
||
|
||
Err(anyhow!("No valid JSON pattern found"))
|
||
}
|
||
|
||
/// 尝试部分解析
|
||
fn try_partial_parse(&self, text: &str) -> Result<Value> {
|
||
// 尝试解析文本的各个部分
|
||
let lines: Vec<&str> = text.lines().collect();
|
||
|
||
for i in 0..lines.len() {
|
||
for j in (i+1)..=lines.len() {
|
||
let partial_text = lines[i..j].join("\n");
|
||
if let Ok(value) = serde_json::from_str::<Value>(&partial_text) {
|
||
return Ok(value);
|
||
}
|
||
}
|
||
}
|
||
|
||
Err(anyhow!("Partial parse failed"))
|
||
}
|
||
|
||
/// 恢复错误的对象成员
|
||
fn recover_object_member(
|
||
&self,
|
||
node: Node,
|
||
text: &str,
|
||
recovery_strategies: &mut Vec<String>,
|
||
) -> Result<Value> {
|
||
let error_text = &text[node.start_byte()..node.end_byte()];
|
||
|
||
// 首先检查整个错误文本是否为YAML格式
|
||
if let Ok(yaml_value) = self.try_parse_yaml(error_text) {
|
||
recovery_strategies.push("ObjectMemberYamlRecovery".to_string());
|
||
return Ok(yaml_value);
|
||
}
|
||
|
||
// 尝试解析为单个键值对
|
||
if let Some(colon_pos) = error_text.find(':') {
|
||
let key_part = error_text[..colon_pos].trim();
|
||
let value_part = error_text[colon_pos+1..].trim();
|
||
|
||
// 清理键名
|
||
let clean_key = key_part.trim_matches(|c| c == '"' || c == '\'' || c == ' ');
|
||
|
||
// 首先尝试将值部分解析为YAML
|
||
if let Ok(yaml_value) = self.try_parse_yaml(value_part) {
|
||
let mut object = Map::new();
|
||
object.insert(clean_key.to_string(), yaml_value);
|
||
recovery_strategies.push("ObjectMemberValueYamlRecovery".to_string());
|
||
return Ok(Value::Object(object));
|
||
}
|
||
|
||
// 然后尝试标准JSON解析
|
||
if let Ok(value) = self.try_standard_json_parse(value_part) {
|
||
let mut object = Map::new();
|
||
object.insert(clean_key.to_string(), value);
|
||
recovery_strategies.push("ObjectMemberRecovery".to_string());
|
||
return Ok(Value::Object(object));
|
||
}
|
||
|
||
// 如果标准解析失败，但值部分不为空，保留为字符串
|
||
if !value_part.is_empty() {
|
||
let mut object = Map::new();
|
||
// 移除值部分的引号（如果存在）
|
||
let clean_value = if (value_part.starts_with('"') && value_part.ends_with('"')) ||
|
||
(value_part.starts_with('\'') && value_part.ends_with('\'')) {
|
||
&value_part[1..value_part.len()-1]
|
||
} else {
|
||
value_part
|
||
};
|
||
object.insert(clean_key.to_string(), Value::String(clean_value.to_string()));
|
||
recovery_strategies.push("ObjectMemberStringRecovery".to_string());
|
||
return Ok(Value::Object(object));
|
||
}
|
||
}
|
||
|
||
Err(anyhow!("Failed to recover object member"))
|
||
}
|
||
|
||
/// 检测字符串是否为YAML格式
|
||
fn is_yaml_format(&self, text: &str) -> bool {
|
||
let trimmed = text.trim();
|
||
|
||
// 空字符串或太短的字符串不认为是YAML
|
||
if trimmed.len() < 3 {
|
||
return false;
|
||
}
|
||
|
||
// 如果包含JSON特征（大括号或方括号开头），则不是YAML
|
||
if trimmed.starts_with('{') || trimmed.starts_with('[') {
|
||
return false;
|
||
}
|
||
|
||
// 检查是否包含YAML特征
|
||
let has_yaml_structure = self.regex_patterns
|
||
.get("yaml_structure")
|
||
.map(|regex| regex.is_match(trimmed))
|
||
.unwrap_or(false);
|
||
|
||
let has_yaml_list = self.regex_patterns
|
||
.get("yaml_list")
|
||
.map(|regex| regex.is_match(trimmed))
|
||
.unwrap_or(false);
|
||
|
||
let has_yaml_multiline = self.regex_patterns
|
||
.get("yaml_multiline")
|
||
.map(|regex| regex.is_match(trimmed))
|
||
.unwrap_or(false);
|
||
|
||
// 额外检查：包含多行且有冒号的内容
|
||
let has_multiline_with_colon = trimmed.contains('\n') && trimmed.contains(':');
|
||
|
||
// 如果包含任何YAML特征，则认为是YAML格式
|
||
has_yaml_structure || has_yaml_list || has_yaml_multiline || has_multiline_with_colon
|
||
}
|
||
|
||
/// 尝试解析YAML内容
|
||
fn try_parse_yaml(&self, text: &str) -> Result<Value> {
|
||
let trimmed = text.trim();
|
||
|
||
// 首先检测是否为YAML格式
|
||
if !self.is_yaml_format(trimmed) {
|
||
return Err(anyhow!("Not a YAML format"));
|
||
}
|
||
|
||
debug!("Attempting to parse YAML content: {} chars", trimmed.len());
|
||
|
||
// 尝试解析YAML
|
||
match serde_yaml::from_str::<Value>(trimmed) {
|
||
Ok(yaml_value) => {
|
||
Ok(yaml_value)
|
||
}
|
||
Err(e) => {
|
||
debug!("YAML parsing failed: {}", e);
|
||
Err(anyhow!("YAML parsing failed: {}", e))
|
||
}
|
||
}
|
||
}
|
||
|
||
/// 处理JSON值中的YAML字符串字段
|
||
fn process_yaml_in_json_value(&self, value: Value, recovery_strategies: &mut Vec<String>) -> Value {
|
||
match value {
|
||
Value::String(s) => {
|
||
// 检查字符串是否为YAML格式，如果是则尝试解析
|
||
if let Ok(yaml_value) = self.try_parse_yaml(&s) {
|
||
recovery_strategies.push("YamlStringParsing".to_string());
|
||
debug!("Converted string field to YAML object/array");
|
||
yaml_value
|
||
} else {
|
||
Value::String(s)
|
||
}
|
||
}
|
||
Value::Object(obj) => {
|
||
// 递归处理对象中的每个字段
|
||
let mut new_obj = Map::new();
|
||
for (key, val) in obj {
|
||
new_obj.insert(key, self.process_yaml_in_json_value(val, recovery_strategies));
|
||
}
|
||
Value::Object(new_obj)
|
||
}
|
||
Value::Array(arr) => {
|
||
// 递归处理数组中的每个元素
|
||
let new_arr: Vec<Value> = arr.into_iter()
|
||
.map(|val| self.process_yaml_in_json_value(val, recovery_strategies))
|
||
.collect();
|
||
Value::Array(new_arr)
|
||
}
|
||
_ => value, // 其他类型直接返回
|
||
}
|
||
}
|
||
}
|
||
|
||
/// 带缓存的容错JSON解析器
|
||
pub struct CachedTolerantJsonParser {
|
||
parser: Arc<Mutex<TolerantJsonParser>>,
|
||
cache: Arc<Mutex<HashMap<String, (Value, ParseStatistics)>>>,
|
||
max_cache_size: usize,
|
||
}
|
||
|
||
impl CachedTolerantJsonParser {
|
||
/// 创建新的带缓存的解析器实例
|
||
pub fn new(config: Option<ParserConfig>, max_cache_size: Option<usize>) -> Result<Self> {
|
||
let parser = TolerantJsonParser::new(config)?;
|
||
let max_cache_size = max_cache_size.unwrap_or(1000);
|
||
|
||
Ok(Self {
|
||
parser: Arc::new(Mutex::new(parser)),
|
||
cache: Arc::new(Mutex::new(HashMap::new())),
|
||
max_cache_size,
|
||
})
|
||
}
|
||
|
||
/// 解析JSON文本，使用缓存机制
|
||
pub fn parse(&self, text: &str) -> Result<(Value, ParseStatistics)> {
|
||
// 计算文本哈希作为缓存键
|
||
let cache_key = self.compute_cache_key(text);
|
||
|
||
// 检查缓存
|
||
{
|
||
let cache = self.cache.lock().unwrap();
|
||
if let Some(cached_result) = cache.get(&cache_key) {
|
||
debug!("Cache hit for JSON parsing");
|
||
return Ok(cached_result.clone());
|
||
}
|
||
}
|
||
|
||
// 缓存未命中，执行解析
|
||
let result = {
|
||
let mut parser = self.parser.lock().unwrap();
|
||
parser.parse(text)?
|
||
};
|
||
|
||
// 存储到缓存
|
||
{
|
||
let mut cache = self.cache.lock().unwrap();
|
||
|
||
// 如果缓存已满，移除最旧的条目
|
||
if cache.len() >= self.max_cache_size {
|
||
// 简单的LRU实现：移除第一个条目
|
||
if let Some(first_key) = cache.keys().next().cloned() {
|
||
cache.remove(&first_key);
|
||
}
|
||
}
|
||
|
||
cache.insert(cache_key, result.clone());
|
||
}
|
||
|
||
debug!("Cached new JSON parsing result");
|
||
Ok(result)
|
||
}
|
||
|
||
/// 计算缓存键
|
||
fn compute_cache_key(&self, text: &str) -> String {
|
||
use std::collections::hash_map::DefaultHasher;
|
||
use std::hash::{Hash, Hasher};
|
||
|
||
let mut hasher = DefaultHasher::new();
|
||
text.hash(&mut hasher);
|
||
format!("{:x}", hasher.finish())
|
||
}
|
||
|
||
/// 清空缓存
|
||
pub fn clear_cache(&self) {
|
||
let mut cache = self.cache.lock().unwrap();
|
||
cache.clear();
|
||
info!("JSON parser cache cleared");
|
||
}
|
||
|
||
/// 获取缓存统计信息
|
||
pub fn get_cache_stats(&self) -> (usize, usize) {
|
||
let cache = self.cache.lock().unwrap();
|
||
(cache.len(), self.max_cache_size)
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use serde_json::json;
|
||
|
||
/// 创建测试用的解析器实例
|
||
fn create_test_parser() -> TolerantJsonParser {
|
||
TolerantJsonParser::new(None).expect("Failed to create test parser")
|
||
}
|
||
|
||
#[test]
|
||
fn test_standard_json_parsing() {
|
||
let mut parser = create_test_parser();
|
||
|
||
// Test simple cases first
|
||
let simple_cases = vec![
|
||
(r#"true"#, json!(true)),
|
||
(r#"false"#, json!(false)),
|
||
(r#"null"#, json!(null)),
|
||
(r#"42"#, json!(42)),
|
||
(r#""hello world""#, json!("hello world")),
|
||
];
|
||
|
||
for (input, expected) in simple_cases {
|
||
let result = parser.parse(input);
|
||
assert!(result.is_ok(), "Failed to parse: {}", input);
|
||
|
||
let (parsed_value, stats) = result.unwrap();
|
||
println!("Input: {}, Parsed: {:?}, Expected: {:?}", input, parsed_value, expected);
|
||
assert_eq!(parsed_value, expected, "Mismatch for input: {}", input);
|
||
assert_eq!(stats.error_rate, 0.0, "Should have no errors for valid JSON");
|
||
}
|
||
|
||
// Test more complex cases
|
||
let complex_cases = vec![
|
||
(r#"{"name": "test"}"#, json!({"name": "test"})),
|
||
(r#"[1, 2, 3]"#, json!([1, 2, 3])),
|
||
];
|
||
|
||
for (input, expected) in complex_cases {
|
||
let result = parser.parse(input);
|
||
assert!(result.is_ok(), "Failed to parse: {}", input);
|
||
|
||
let (parsed_value, _stats) = result.unwrap();
|
||
println!("Input: {}, Parsed: {:?}, Expected: {:?}", input, parsed_value, expected);
|
||
assert_eq!(parsed_value, expected, "Mismatch for input: {}", input);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn test_markdown_wrapped_json() {
|
||
let mut parser = create_test_parser();
|
||
|
||
// Simple test case
|
||
let input = r#"```json
|
||
{"name": "test"}
|
||
```"#;
|
||
let expected = json!({"name": "test"});
|
||
|
||
let result = parser.parse(input);
|
||
assert!(result.is_ok(), "Failed to parse markdown wrapped JSON: {}", input);
|
||
|
||
let (parsed_value, _) = result.unwrap();
|
||
println!("Markdown Input: {}", input);
|
||
println!("Parsed: {:?}", parsed_value);
|
||
println!("Expected: {:?}", expected);
|
||
assert_eq!(parsed_value, expected, "Mismatch for markdown input: {}", input);
|
||
}
|
||
|
||
#[test]
|
||
fn test_empty_input() {
|
||
let mut parser = create_test_parser();
|
||
|
||
let result = parser.parse("");
|
||
assert!(result.is_ok(), "Should handle empty input");
|
||
|
||
let (parsed_value, _) = result.unwrap();
|
||
assert_eq!(parsed_value, json!(null), "Empty input should parse to null");
|
||
}
|
||
|
||
#[test]
|
||
fn test_cached_parser() {
|
||
let cached_parser = CachedTolerantJsonParser::new(None, Some(10)).unwrap();
|
||
|
||
// Test with simple object first
|
||
let test_json = r#"{"name": "test"}"#;
|
||
let expected = json!({"name": "test"});
|
||
|
||
println!("Testing cached parser with: {}", test_json);
|
||
|
||
// First parse - should cache the result
|
||
let result1 = cached_parser.parse(test_json);
|
||
assert!(result1.is_ok(), "First parse failed");
|
||
let (parsed1, _) = result1.unwrap();
|
||
println!("First parse result: {:?}", parsed1);
|
||
assert_eq!(parsed1, expected);
|
||
|
||
// Second parse - should use cache
|
||
let result2 = cached_parser.parse(test_json);
|
||
assert!(result2.is_ok(), "Second parse failed");
|
||
let (parsed2, _) = result2.unwrap();
|
||
println!("Second parse result: {:?}", parsed2);
|
||
assert_eq!(parsed2, expected);
|
||
|
||
// Check cache stats
|
||
let (cache_size, max_size) = cached_parser.get_cache_stats();
|
||
assert_eq!(cache_size, 1);
|
||
assert_eq!(max_size, 10);
|
||
|
||
// Clear cache
|
||
cached_parser.clear_cache();
|
||
let (cache_size_after_clear, _) = cached_parser.get_cache_stats();
|
||
assert_eq!(cache_size_after_clear, 0);
|
||
}
|
||
|
||
#[test]
|
||
fn test_error_recovery_strategies() {
|
||
let mut parser = create_test_parser();
|
||
|
||
// Test unquoted keys - simplified
|
||
let unquoted_json = r#"{name: "test"}"#;
|
||
let result = parser.parse(unquoted_json);
|
||
println!("Unquoted keys test input: {}", unquoted_json);
|
||
assert!(result.is_ok(), "Unquoted keys parsing failed");
|
||
let (parsed, stats) = result.unwrap();
|
||
println!("Unquoted keys result: {:?}", parsed);
|
||
println!("Recovery strategies used: {:?}", stats.recovery_strategies_used);
|
||
assert_eq!(parsed, json!({"name": "test"}));
|
||
|
||
// Test trailing commas - simplified
|
||
let trailing_comma_json = r#"{"name": "test",}"#;
|
||
let result = parser.parse(trailing_comma_json);
|
||
println!("Trailing comma test input: {}", trailing_comma_json);
|
||
assert!(result.is_ok(), "Trailing comma parsing failed");
|
||
let (parsed, stats) = result.unwrap();
|
||
println!("Trailing comma result: {:?}", parsed);
|
||
println!("Recovery strategies used: {:?}", stats.recovery_strategies_used);
|
||
assert_eq!(parsed, json!({"name": "test"}));
|
||
}
|
||
|
||
#[test]
|
||
fn test_complex_nested_json_extraction() {
|
||
let mut parser = create_test_parser();
|
||
|
||
// 测试复杂嵌套JSON（类似Gemini响应的结构）
|
||
let complex_json = r#"```json
|
||
{
|
||
"environment_tags": ["Indoor", "Retail environment"],
|
||
"environment_color_pattern": {
|
||
"hue": 0.08,
|
||
"saturation": 0.05,
|
||
"value": 0.6
|
||
},
|
||
"dress_color_pattern": {
|
||
"hue": 0.58,
|
||
"saturation": 0.28,
|
||
"value": 0.85
|
||
},
|
||
"style_description": "整体呈现休闲舒适的风格",
|
||
"products": [
|
||
{
|
||
"category": "下装",
|
||
"color_pattern": {
|
||
"hue": 0.6,
|
||
"saturation": 0.35,
|
||
"value": 0.85
|
||
},
|
||
"design_styles": ["休闲", "复古", "街头"]
|
||
}
|
||
]
|
||
}
|
||
```"#;
|
||
|
||
let result = parser.parse(complex_json);
|
||
assert!(result.is_ok(), "Complex nested JSON parsing failed");
|
||
|
||
let (parsed, stats) = result.unwrap();
|
||
assert!(parsed.is_object());
|
||
|
||
let obj = parsed.as_object().unwrap();
|
||
|
||
// 检查顶级字段
|
||
assert!(obj.contains_key("environment_tags"));
|
||
assert!(obj.contains_key("environment_color_pattern"));
|
||
assert!(obj.contains_key("dress_color_pattern"));
|
||
assert!(obj.contains_key("style_description"));
|
||
assert!(obj.contains_key("products"));
|
||
|
||
// 检查嵌套对象
|
||
let env_color = obj.get("environment_color_pattern").unwrap();
|
||
assert!(env_color.is_object());
|
||
|
||
// 检查数组
|
||
let products = obj.get("products").unwrap();
|
||
assert!(products.is_array());
|
||
let products_array = products.as_array().unwrap();
|
||
assert_eq!(products_array.len(), 1);
|
||
|
||
println!("✅ 复杂嵌套JSON解析测试通过");
|
||
println!("📊 解析统计: 总节点数={}, 错误节点数={}, 错误率={:.2}%, 解析时间={}ms",
|
||
stats.total_nodes, stats.error_nodes, stats.error_rate, stats.parse_time_ms);
|
||
}
|
||
|
||
#[test]
|
||
fn test_bracket_matching_method() {
|
||
let parser = create_test_parser();
|
||
|
||
// 测试括号匹配功能
|
||
let text_with_nested_braces = r#"Some text before {
|
||
"outer": {
|
||
"inner": {
|
||
"deep": "value"
|
||
}
|
||
},
|
||
"array": [1, 2, {"nested": true}]
|
||
} some text after"#;
|
||
|
||
let result = parser.extract_json_by_bracket_matching(text_with_nested_braces);
|
||
assert!(result.is_some());
|
||
|
||
let json_str = result.unwrap();
|
||
let parsed: Value = serde_json::from_str(&json_str).unwrap();
|
||
assert!(parsed.is_object());
|
||
|
||
let obj = parsed.as_object().unwrap();
|
||
assert!(obj.contains_key("outer"));
|
||
assert!(obj.contains_key("array"));
|
||
|
||
println!("✅ 括号匹配测试通过");
|
||
}
|
||
|
||
#[test]
|
||
fn test_yaml_detection_and_parsing() {
|
||
let mut parser = create_test_parser();
|
||
|
||
// 测试简单的YAML键值对
|
||
let yaml_content = r#"name: John Doe
|
||
age: 30
|
||
city: New York"#;
|
||
|
||
println!("🔍 测试YAML内容: {}", yaml_content);
|
||
println!("🔍 YAML检测结果: {}", parser.is_yaml_format(yaml_content));
|
||
|
||
let json_with_yaml = format!(r#"{{"config": "{}"}}"#, yaml_content);
|
||
println!("🔍 完整JSON: {}", json_with_yaml);
|
||
|
||
let result = parser.parse(&json_with_yaml);
|
||
assert!(result.is_ok(), "YAML detection parsing failed");
|
||
|
||
let (parsed, stats) = result.unwrap();
|
||
assert!(parsed.is_object());
|
||
|
||
let obj = parsed.as_object().unwrap();
|
||
assert!(obj.contains_key("config"));
|
||
|
||
// 检查YAML是否被正确解析为对象
|
||
let config_value = obj.get("config").unwrap();
|
||
println!("🔍 解析后的config值类型: {:?}", config_value);
|
||
|
||
if config_value.is_object() {
|
||
let config_obj = config_value.as_object().unwrap();
|
||
assert!(config_obj.contains_key("name"));
|
||
assert!(config_obj.contains_key("age"));
|
||
assert!(config_obj.contains_key("city"));
|
||
|
||
assert_eq!(config_obj.get("name").unwrap(), &Value::String("John Doe".to_string()));
|
||
assert_eq!(config_obj.get("age").unwrap(), &Value::Number(serde_json::Number::from(30)));
|
||
assert_eq!(config_obj.get("city").unwrap(), &Value::String("New York".to_string()));
|
||
|
||
println!("✅ YAML自动解析测试通过");
|
||
} else {
|
||
println!("⚠️ YAML内容未被解析，保持为字符串格式");
|
||
println!("🔍 实际值: {:?}", config_value);
|
||
}
|
||
|
||
println!("📊 YAML解析统计: 恢复策略={:?}", stats.recovery_strategies_used);
|
||
}
|
||
|
||
#[test]
|
||
fn test_yaml_list_parsing() {
|
||
let mut parser = create_test_parser();
|
||
|
||
// 测试YAML列表格式
|
||
let yaml_list = r#"- item1
|
||
- item2
|
||
- item3"#;
|
||
|
||
let json_with_yaml_list = format!(r#"{{"items": "{}"}}"#, yaml_list);
|
||
|
||
let result = parser.parse(&json_with_yaml_list);
|
||
assert!(result.is_ok(), "YAML list parsing failed");
|
||
|
||
let (parsed, _) = result.unwrap();
|
||
let obj = parsed.as_object().unwrap();
|
||
let items_value = obj.get("items").unwrap();
|
||
|
||
if items_value.is_array() {
|
||
let items_array = items_value.as_array().unwrap();
|
||
assert_eq!(items_array.len(), 3);
|
||
assert_eq!(items_array[0], Value::String("item1".to_string()));
|
||
assert_eq!(items_array[1], Value::String("item2".to_string()));
|
||
assert_eq!(items_array[2], Value::String("item3".to_string()));
|
||
|
||
println!("✅ YAML列表自动解析测试通过");
|
||
} else {
|
||
println!("⚠️ YAML列表内容未被解析，保持为字符串格式");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn test_complex_yaml_parsing() {
|
||
let mut parser = create_test_parser();
|
||
|
||
// 测试复杂的YAML结构
|
||
let complex_yaml = r#"database:
|
||
host: localhost
|
||
port: 5432
|
||
credentials:
|
||
username: admin
|
||
password: secret
|
||
features:
|
||
- authentication
|
||
- logging
|
||
- monitoring"#;
|
||
|
||
let json_with_complex_yaml = format!(r#"{{"config": "{}"}}"#, complex_yaml);
|
||
|
||
let result = parser.parse(&json_with_complex_yaml);
|
||
assert!(result.is_ok(), "Complex YAML parsing failed");
|
||
|
||
let (parsed, _) = result.unwrap();
|
||
let obj = parsed.as_object().unwrap();
|
||
let config_value = obj.get("config").unwrap();
|
||
|
||
if config_value.is_object() {
|
||
let config_obj = config_value.as_object().unwrap();
|
||
assert!(config_obj.contains_key("database"));
|
||
assert!(config_obj.contains_key("features"));
|
||
|
||
// 检查嵌套对象
|
||
let database = config_obj.get("database").unwrap().as_object().unwrap();
|
||
assert!(database.contains_key("host"));
|
||
assert!(database.contains_key("credentials"));
|
||
|
||
// 检查数组
|
||
let features = config_obj.get("features").unwrap().as_array().unwrap();
|
||
assert_eq!(features.len(), 3);
|
||
|
||
println!("✅ 复杂YAML结构自动解析测试通过");
|
||
} else {
|
||
println!("⚠️ 复杂YAML内容未被解析，保持为字符串格式");
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn test_direct_yaml_parsing() {
|
||
let parser = create_test_parser();
|
||
|
||
// 测试直接YAML解析功能
|
||
let yaml_content = r#"name: John Doe
|
||
age: 30
|
||
city: New York"#;
|
||
|
||
println!("🔍 直接测试YAML解析");
|
||
println!("🔍 YAML内容: {}", yaml_content);
|
||
println!("🔍 YAML检测结果: {}", parser.is_yaml_format(yaml_content));
|
||
|
||
let yaml_result = parser.try_parse_yaml(yaml_content);
|
||
assert!(yaml_result.is_ok(), "直接YAML解析失败");
|
||
|
||
let yaml_value = yaml_result.unwrap();
|
||
assert!(yaml_value.is_object());
|
||
|
||
let yaml_obj = yaml_value.as_object().unwrap();
|
||
assert!(yaml_obj.contains_key("name"));
|
||
assert!(yaml_obj.contains_key("age"));
|
||
assert!(yaml_obj.contains_key("city"));
|
||
|
||
println!("✅ 直接YAML解析功能正常");
|
||
|
||
// 测试YAML列表
|
||
let yaml_list = r#"- item1
|
||
- item2
|
||
- item3"#;
|
||
|
||
println!("🔍 测试YAML列表: {}", yaml_list);
|
||
let list_result = parser.try_parse_yaml(yaml_list);
|
||
assert!(list_result.is_ok(), "YAML列表解析失败");
|
||
|
||
let list_value = list_result.unwrap();
|
||
assert!(list_value.is_array());
|
||
|
||
let list_array = list_value.as_array().unwrap();
|
||
assert_eq!(list_array.len(), 3);
|
||
|
||
println!("✅ YAML列表解析功能正常");
|
||
}
|
||
|
||
#[test]
|
||
fn test_yaml_functionality_demo() {
|
||
let mut parser = create_test_parser();
|
||
|
||
println!("\n🎯 YAML功能演示");
|
||
println!("================");
|
||
|
||
// 演示1：YAML列表自动解析
|
||
let json_with_yaml_list = r#"{"tags": "- frontend\n- backend\n- database"}"#;
|
||
println!("\n📝 测试1: YAML列表自动解析");
|
||
println!("输入: {}", json_with_yaml_list);
|
||
|
||
let result = parser.parse(json_with_yaml_list).unwrap();
|
||
let tags = result.0.get("tags").unwrap();
|
||
if tags.is_array() {
|
||
println!("✅ 成功解析为数组: {:?}", tags.as_array().unwrap());
|
||
} else {
|
||
println!("⚠️ 保持为字符串: {:?}", tags.as_str().unwrap());
|
||
}
|
||
|
||
// 演示2：直接YAML解析API
|
||
println!("\n📝 测试2: 直接YAML解析API");
|
||
let yaml_content = "name: 测试项目\nversion: 1.0\nfeatures:\n - auth\n - logging";
|
||
println!("YAML内容:\n{}", yaml_content);
|
||
|
||
if let Ok(parsed_yaml) = parser.try_parse_yaml(yaml_content) {
|
||
println!("✅ 解析成功: {}", serde_json::to_string_pretty(&parsed_yaml).unwrap());
|
||
} else {
|
||
println!("❌ 解析失败");
|
||
}
|
||
|
||
// 演示3：YAML检测功能
|
||
println!("\n📝 测试3: YAML检测功能");
|
||
let test_cases = vec![
|
||
("name: value", "YAML键值对"),
|
||
("- item1\n- item2", "YAML列表"),
|
||
("{\"key\": \"value\"}", "JSON对象"),
|
||
("[1, 2, 3]", "JSON数组"),
|
||
("simple text", "普通文本"),
|
||
];
|
||
|
||
for (content, description) in test_cases {
|
||
let is_yaml = parser.is_yaml_format(content);
|
||
println!(" {} -> {} ({})", content.replace('\n', "\\n"), if is_yaml { "✅ YAML" } else { "❌ 非YAML" }, description);
|
||
}
|
||
|
||
println!("\n🎉 YAML功能演示完成！");
|
||
}
|
||
|
||
#[test]
|
||
fn test_retrieved_context_yaml_parsing() {
|
||
let mut parser = create_test_parser();
|
||
|
||
println!("\n🔍 测试retrievedContext.text中的YAML解析");
|
||
println!("==========================================");
|
||
|
||
// 模拟promptx/res(1).json中的retrievedContext.text内容
|
||
let yaml_content = "categories:\n - 修身\n - 抹胸\n - 高腰\n - 宽松\n - 涤纶\n - 西装外套\n - 吊带背心\n - 运动短裤\n - 短款上衣\n - 衬衫\n - 圆领\n - 棉麻混纺\n - 抽绳\n - 休闲\n - 短袖T恤\n - 纯色\n - 通勤\n - 棉质\n - 阔腿短裤\ndescription: 三位女性模特在城市街道背景下展示夏季时尚穿搭，她们都穿着黑白配色的服装，风格各异，展现出休闲与时尚并存的夏日风情。\nenvironment_color_pattern:\n Hue: 0.10000000000000001\n Saturation: 0.080000000000000002\n Value: 0.75\n rgb_hex: \"#bfb9af\"\nenvironment_tags:\n - 城市街道\n - 夏季\n - 晴朗\n - 欧洲风格建筑\n - 户外\nmodels:\n - dress_color_pattern:\n Hue: 0\n Saturation: 0\n Value: 0.5\n rgb_hex: \"#7f7f7f\"\n position: 画面左侧\n posture: 站立，身体略微侧向右侧";
|
||
|
||
println!("🔍 YAML内容长度: {} 字符", yaml_content.len());
|
||
println!("🔍 YAML检测结果: {}", parser.is_yaml_format(yaml_content));
|
||
|
||
// 测试直接YAML解析
|
||
match parser.try_parse_yaml(yaml_content) {
|
||
Ok(yaml_value) => {
|
||
println!("✅ 直接YAML解析成功");
|
||
println!("🔍 解析结果类型: {:?}", yaml_value);
|
||
|
||
if let Some(obj) = yaml_value.as_object() {
|
||
println!("🔍 顶级字段: {:?}", obj.keys().collect::<Vec<_>>());
|
||
|
||
// 检查categories字段
|
||
if let Some(categories) = obj.get("categories") {
|
||
if categories.is_array() {
|
||
println!("✅ categories字段解析为数组，长度: {}", categories.as_array().unwrap().len());
|
||
}
|
||
}
|
||
|
||
// 检查environment_color_pattern字段
|
||
if let Some(env_pattern) = obj.get("environment_color_pattern") {
|
||
if env_pattern.is_object() {
|
||
println!("✅ environment_color_pattern字段解析为对象");
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("❌ 直接YAML解析失败: {}", e);
|
||
}
|
||
}
|
||
|
||
// 测试在JSON中包含此YAML内容
|
||
let json_with_yaml = format!(r#"{{"retrievedContext": {{"text": "{}"}}}}"#,
|
||
yaml_content.replace('\n', "\\n").replace('"', "\\\""));
|
||
|
||
println!("\n🔍 测试JSON中包含YAML内容的解析");
|
||
println!("🔍 JSON长度: {} 字符", json_with_yaml.len());
|
||
|
||
match parser.parse(&json_with_yaml) {
|
||
Ok((parsed, stats)) => {
|
||
println!("✅ JSON解析成功");
|
||
println!("📊 解析统计: 恢复策略={:?}", stats.recovery_strategies_used);
|
||
|
||
if let Some(obj) = parsed.as_object() {
|
||
if let Some(retrieved_context) = obj.get("retrievedContext") {
|
||
if let Some(context_obj) = retrieved_context.as_object() {
|
||
if let Some(text_value) = context_obj.get("text") {
|
||
match text_value {
|
||
serde_json::Value::Object(_) => {
|
||
println!("✅ text字段被解析为YAML对象");
|
||
}
|
||
serde_json::Value::String(s) => {
|
||
println!("⚠️ text字段保持为字符串，长度: {}", s.len());
|
||
}
|
||
_ => {
|
||
println!("❓ text字段类型未知: {:?}", text_value);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("❌ JSON解析失败: {}", e);
|
||
}
|
||
}
|
||
|
||
println!("\n🎉 retrievedContext.text YAML解析测试完成！");
|
||
}
|
||
|
||
#[test]
|
||
fn test_real_promptx_res_file() {
|
||
let mut parser = create_test_parser();
|
||
|
||
println!("\n🔍 测试真实的promptx/res(1).json文件");
|
||
println!("=====================================");
|
||
|
||
// 读取真实的promptx/res(1).json文件
|
||
let file_path = "../../../promptx/res(1).json";
|
||
match std::fs::read_to_string(file_path) {
|
||
Ok(content) => {
|
||
println!("🔍 文件读取成功，长度: {} 字符", content.len());
|
||
|
||
match parser.parse(&content) {
|
||
Ok((parsed, stats)) => {
|
||
println!("✅ JSON解析成功");
|
||
println!("📊 解析统计: 恢复策略={:?}", stats.recovery_strategies_used);
|
||
println!("📊 错误率: {:.2}%", stats.error_rate * 100.0);
|
||
println!("📊 解析时间: {}ms", stats.parse_time_ms);
|
||
|
||
// 检查是否包含YAML解析策略
|
||
if stats.recovery_strategies_used.contains(&"YamlStringParsing".to_string()) {
|
||
println!("✅ 检测到YAML字符串解析");
|
||
} else {
|
||
println!("⚠️ 未检测到YAML字符串解析");
|
||
}
|
||
|
||
// 检查第一个retrievedContext.text字段
|
||
if let Some(obj) = parsed.as_object() {
|
||
if let Some(candidates) = obj.get("candidates") {
|
||
if let Some(candidates_array) = candidates.as_array() {
|
||
if let Some(first_candidate) = candidates_array.first() {
|
||
if let Some(candidate_obj) = first_candidate.as_object() {
|
||
if let Some(grounding_metadata) = candidate_obj.get("groundingMetadata") {
|
||
if let Some(grounding_obj) = grounding_metadata.as_object() {
|
||
if let Some(grounding_chunks) = grounding_obj.get("groundingChunks") {
|
||
if let Some(chunks_array) = grounding_chunks.as_array() {
|
||
if let Some(first_chunk) = chunks_array.first() {
|
||
if let Some(chunk_obj) = first_chunk.as_object() {
|
||
if let Some(retrieved_context) = chunk_obj.get("retrievedContext") {
|
||
if let Some(context_obj) = retrieved_context.as_object() {
|
||
if let Some(text_value) = context_obj.get("text") {
|
||
match text_value {
|
||
serde_json::Value::Object(text_obj) => {
|
||
println!("✅ 第一个retrievedContext.text被解析为YAML对象");
|
||
println!("🔍 YAML对象字段: {:?}", text_obj.keys().collect::<Vec<_>>());
|
||
|
||
// 检查categories字段
|
||
if let Some(categories) = text_obj.get("categories") {
|
||
if let Some(categories_array) = categories.as_array() {
|
||
println!("✅ categories字段包含 {} 个元素", categories_array.len());
|
||
}
|
||
}
|
||
}
|
||
serde_json::Value::String(s) => {
|
||
println!("⚠️ 第一个retrievedContext.text仍为字符串，长度: {}", s.len());
|
||
println!("🔍 前100字符: {}", &s[..std::cmp::min(100, s.len())]);
|
||
}
|
||
_ => {
|
||
println!("❓ 第一个retrievedContext.text类型未知: {:?}", text_value);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("❌ JSON解析失败: {}", e);
|
||
}
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("⚠️ 无法读取文件 {}: {}", file_path, e);
|
||
println!("💡 这是正常的，因为测试可能在不同的工作目录运行");
|
||
}
|
||
}
|
||
|
||
println!("\n🎉 真实文件测试完成！");
|
||
}
|
||
}
|