feat: 添加YAML智能解析支持到容错JSON解析器
修复tolerant_json_parser无法处理retrievedContext.text中YAML结构的问题 新增process_yaml_in_json_value方法,递归检查JSON字符串字段中的YAML内容 即使标准JSON解析成功,也会自动检测并解析YAML字符串字段 添加YamlStringParsing恢复策略,提供详细的解析统计信息 更新前端功能特性描述,添加YAML支持说明 新增完整的测试用例验证YAML解析功能 支持复杂的YAML嵌套结构、数组和对象解析 遵循promptx/tauri-desktop-app-expert开发规范
This commit is contained in:
@@ -24,6 +24,7 @@ tauri-plugin-fs = "2"
|
|||||||
tauri-plugin-dialog = "2"
|
tauri-plugin-dialog = "2"
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
serde_yaml = "0.9"
|
||||||
rusqlite = { version = "0.31", features = ["bundled", "chrono"] }
|
rusqlite = { version = "0.31", features = ["bundled", "chrono"] }
|
||||||
chrono = { version = "0.4", features = ["serde"] }
|
chrono = { version = "0.4", features = ["serde"] }
|
||||||
uuid = { version = "1.0", features = ["v4", "serde"] }
|
uuid = { version = "1.0", features = ["v4", "serde"] }
|
||||||
|
|||||||
@@ -1,8 +1,37 @@
|
|||||||
/// 基于Tree-sitter的大模型JSON容错解析器
|
/// 基于Tree-sitter的大模型JSON容错解析器
|
||||||
/// 遵循Tauri开发规范,提供高性能、安全的JSON解析能力
|
/// 遵循Tauri开发规范,提供高性能、安全的JSON解析能力
|
||||||
|
///
|
||||||
|
/// ## 新增功能:YAML格式支持
|
||||||
|
///
|
||||||
|
/// ### 功能概述
|
||||||
|
/// - 当字符串值被检测为YAML格式时,自动解析为对应的JSON结构
|
||||||
|
/// - 支持YAML键值对、列表、嵌套结构等常见格式
|
||||||
|
/// - 智能检测避免误判JSON内容为YAML
|
||||||
|
///
|
||||||
|
/// ### YAML检测规则
|
||||||
|
/// 1. **键值对检测**:包含 `key: value` 格式的多行内容
|
||||||
|
/// 2. **列表检测**:包含 `- item` 格式的行
|
||||||
|
/// 3. **多行检测**:包含换行符和冒号的内容
|
||||||
|
/// 4. **排除规则**:以 `{` 或 `[` 开头的内容不会被识别为YAML
|
||||||
|
///
|
||||||
|
/// ### 使用场景
|
||||||
|
/// - 处理大模型返回的混合格式内容
|
||||||
|
/// - 解析配置文件中的YAML字段
|
||||||
|
/// - 容错处理包含YAML片段的JSON数据
|
||||||
|
///
|
||||||
|
/// ### 示例
|
||||||
|
/// ```rust
|
||||||
|
/// // JSON中包含YAML字符串会被自动解析
|
||||||
|
/// let json_with_yaml = r#"{"config": "name: John\nage: 30"}"#;
|
||||||
|
/// // 解析后 config 字段会变成 {"name": "John", "age": 30}
|
||||||
|
///
|
||||||
|
/// let yaml_list = r#"{"items": "- item1\n- item2\n- item3"}"#;
|
||||||
|
/// // 解析后 items 字段会变成 ["item1", "item2", "item3"]
|
||||||
|
/// ```
|
||||||
use anyhow::{anyhow, Result};
|
use anyhow::{anyhow, Result};
|
||||||
use regex::Regex;
|
use regex::Regex;
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
|
use serde_yaml;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use tree_sitter::{Node, Parser, Tree};
|
use tree_sitter::{Node, Parser, Tree};
|
||||||
@@ -118,42 +147,96 @@ impl TolerantJsonParser {
|
|||||||
Regex::new(r",\s*[}\]]")?,
|
Regex::new(r",\s*[}\]]")?,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// YAML检测模式 - 检测常见的YAML结构(键值对)
|
||||||
|
patterns.insert(
|
||||||
|
"yaml_structure".to_string(),
|
||||||
|
Regex::new(r"(?m)^[\s]*[\w\-\.]+:\s*.*$")?,
|
||||||
|
);
|
||||||
|
|
||||||
|
// YAML列表模式
|
||||||
|
patterns.insert(
|
||||||
|
"yaml_list".to_string(),
|
||||||
|
Regex::new(r"(?m)^[\s]*-\s+")?,
|
||||||
|
);
|
||||||
|
|
||||||
|
// YAML多行字符串模式
|
||||||
|
patterns.insert(
|
||||||
|
"yaml_multiline".to_string(),
|
||||||
|
Regex::new(r"(?s)[|>][\s]*\n")?,
|
||||||
|
);
|
||||||
|
|
||||||
Ok(patterns)
|
Ok(patterns)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 解析JSON文本,返回解析结果和统计信息
|
/// 解析JSON文本,返回解析结果和统计信息
|
||||||
pub fn parse(&mut self, text: &str) -> Result<(Value, ParseStatistics)> {
|
pub fn parse(&mut self, text: &str) -> Result<(Value, ParseStatistics)> {
|
||||||
let start_time = std::time::Instant::now();
|
let start_time = std::time::Instant::now();
|
||||||
|
|
||||||
// 检查文本长度限制
|
// 检查文本长度限制
|
||||||
if text.len() > self.config.max_text_length {
|
if text.len() > self.config.max_text_length {
|
||||||
return Err(anyhow!("Text too long: {} bytes", text.len()));
|
return Err(anyhow!("Text too long: {} bytes", text.len()));
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Starting tolerant JSON parsing, text length: {}", text.len());
|
info!("Starting tolerant JSON parsing, text length: {}", text.len());
|
||||||
|
|
||||||
|
// 首先尝试标准JSON解析
|
||||||
|
match serde_json::from_str::<Value>(text) {
|
||||||
|
Ok(result) => {
|
||||||
|
// 标准JSON解析成功,但还需要检查字符串字段是否包含YAML内容
|
||||||
|
let mut recovery_strategies = vec!["StandardJson".to_string()];
|
||||||
|
let processed_result = self.process_yaml_in_json_value(result, &mut recovery_strategies);
|
||||||
|
|
||||||
|
let stats = ParseStatistics {
|
||||||
|
total_nodes: 1,
|
||||||
|
error_nodes: 0,
|
||||||
|
error_rate: 0.0,
|
||||||
|
parse_time_ms: start_time.elapsed().as_millis() as u64,
|
||||||
|
recovery_strategies_used: recovery_strategies,
|
||||||
|
};
|
||||||
|
info!("Standard JSON parsing successful in {}ms", stats.parse_time_ms);
|
||||||
|
return Ok((processed_result, stats));
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
// 标准解析失败,继续使用容错解析
|
||||||
|
debug!("Standard JSON parsing failed, using tolerant parsing");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 预处理文本
|
// 预处理文本
|
||||||
let cleaned_text = self.preprocess_text(text)?;
|
let cleaned_text = self.preprocess_text(text)?;
|
||||||
debug!("Preprocessed text length: {}", cleaned_text.len());
|
debug!("Preprocessed text length: {}", cleaned_text.len());
|
||||||
|
|
||||||
// 使用Tree-sitter解析
|
// 使用Tree-sitter解析
|
||||||
let tree = self.parser.parse(&cleaned_text, None)
|
let tree = self.parser.parse(&cleaned_text, None)
|
||||||
.ok_or_else(|| anyhow!("Failed to parse text with tree-sitter"))?;
|
.ok_or_else(|| anyhow!("Failed to parse text with tree-sitter"))?;
|
||||||
|
|
||||||
let root_node = tree.root_node();
|
let root_node = tree.root_node();
|
||||||
|
|
||||||
// 收集统计信息
|
// 收集统计信息
|
||||||
let mut stats = self.collect_statistics(&tree, &cleaned_text);
|
let mut stats = self.collect_statistics(&tree, &cleaned_text);
|
||||||
|
|
||||||
|
// 如果错误率太高,回退到标准JSON解析的错误修复
|
||||||
|
if stats.error_rate > 0.1 {
|
||||||
|
debug!("High error rate ({:.2}%), attempting fallback strategies", stats.error_rate * 100.0);
|
||||||
|
|
||||||
|
// 尝试基本的文本清理后再用标准JSON解析
|
||||||
|
if let Ok(result) = self.try_fallback_parsing(text) {
|
||||||
|
stats.recovery_strategies_used.push("FallbackParsing".to_string());
|
||||||
|
stats.parse_time_ms = start_time.elapsed().as_millis() as u64;
|
||||||
|
info!("Fallback JSON parsing successful in {}ms", stats.parse_time_ms);
|
||||||
|
return Ok((result, stats));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 提取和修复JSON
|
// 提取和修复JSON
|
||||||
let mut recovery_strategies_used = Vec::new();
|
let mut recovery_strategies_used = Vec::new();
|
||||||
let result = self.extract_json(root_node, &cleaned_text, &mut recovery_strategies_used)?;
|
let result = self.extract_json(root_node, &cleaned_text, &mut recovery_strategies_used)?;
|
||||||
|
|
||||||
stats.parse_time_ms = start_time.elapsed().as_millis() as u64;
|
stats.parse_time_ms = start_time.elapsed().as_millis() as u64;
|
||||||
stats.recovery_strategies_used = recovery_strategies_used;
|
stats.recovery_strategies_used = recovery_strategies_used;
|
||||||
|
|
||||||
info!("JSON parsing completed in {}ms", stats.parse_time_ms);
|
info!("JSON parsing completed in {}ms", stats.parse_time_ms);
|
||||||
|
|
||||||
Ok((result, stats))
|
Ok((result, stats))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -252,11 +335,18 @@ impl TolerantJsonParser {
|
|||||||
fn fix_common_json_errors(&self, text: &str) -> String {
|
fn fix_common_json_errors(&self, text: &str) -> String {
|
||||||
let mut fixed = text.to_string();
|
let mut fixed = text.to_string();
|
||||||
|
|
||||||
// 修复无引号的键
|
// 检查是否包含YAML内容,如果是则跳过某些修复
|
||||||
if self.config.enable_unquoted_keys {
|
let contains_yaml = self.is_yaml_format(text);
|
||||||
if let Some(unquoted_regex) = self.regex_patterns.get("unquoted_key") {
|
|
||||||
fixed = unquoted_regex.replace_all(&fixed, "\"$1\":").to_string();
|
if contains_yaml {
|
||||||
debug!("Fixed unquoted keys");
|
debug!("Detected YAML content, skipping unquoted key fixes");
|
||||||
|
} else {
|
||||||
|
// 修复无引号的键
|
||||||
|
if self.config.enable_unquoted_keys {
|
||||||
|
if let Some(unquoted_regex) = self.regex_patterns.get("unquoted_key") {
|
||||||
|
fixed = unquoted_regex.replace_all(&fixed, "\"$1\":").to_string();
|
||||||
|
debug!("Fixed unquoted keys");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -271,8 +361,10 @@ impl TolerantJsonParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 修复单引号为双引号
|
// 修复单引号为双引号(但不对YAML内容进行此修复)
|
||||||
fixed = fixed.replace("'", "\"");
|
if !contains_yaml {
|
||||||
|
fixed = fixed.replace("'", "\"");
|
||||||
|
}
|
||||||
|
|
||||||
debug!("Fixed text: {}", &fixed[..std::cmp::min(200, fixed.len())]);
|
debug!("Fixed text: {}", &fixed[..std::cmp::min(200, fixed.len())]);
|
||||||
|
|
||||||
@@ -361,14 +453,10 @@ impl TolerantJsonParser {
|
|||||||
let mut object = Map::new();
|
let mut object = Map::new();
|
||||||
|
|
||||||
for child in node.children(&mut node.walk()) {
|
for child in node.children(&mut node.walk()) {
|
||||||
debug!("Object child node kind: '{}', text: '{}'", child.kind(), &text[child.start_byte()..child.end_byte()]);
|
|
||||||
match child.kind() {
|
match child.kind() {
|
||||||
"pair" => {
|
"pair" => {
|
||||||
if let Ok((key, value)) = self.extract_pair(child, text, recovery_strategies) {
|
if let Ok((key, value)) = self.extract_pair(child, text, recovery_strategies) {
|
||||||
debug!("Successfully extracted pair: {} = {:?}", key, value);
|
|
||||||
object.insert(key, value);
|
object.insert(key, value);
|
||||||
} else {
|
|
||||||
debug!("Failed to extract pair from node");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
"ERROR" => {
|
"ERROR" => {
|
||||||
@@ -378,11 +466,21 @@ impl TolerantJsonParser {
|
|||||||
for (k, v) in recovered.as_object().unwrap_or(&Map::new()) {
|
for (k, v) in recovered.as_object().unwrap_or(&Map::new()) {
|
||||||
object.insert(k.clone(), v.clone());
|
object.insert(k.clone(), v.clone());
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
// 如果对象成员恢复失败,尝试通用错误恢复
|
||||||
|
if let Ok(recovered) = self.recover_from_error(child, text, recovery_strategies) {
|
||||||
|
// 如果恢复的是一个对象,合并其内容
|
||||||
|
if let Some(recovered_obj) = recovered.as_object() {
|
||||||
|
for (k, v) in recovered_obj {
|
||||||
|
object.insert(k.clone(), v.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
debug!("Ignoring object child node kind: '{}'", child.kind());
|
// 忽略其他节点类型(如括号、逗号等)
|
||||||
} // 忽略其他节点类型(如括号、逗号等)
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -434,35 +532,29 @@ impl TolerantJsonParser {
|
|||||||
let mut key = None;
|
let mut key = None;
|
||||||
let mut value = None;
|
let mut value = None;
|
||||||
|
|
||||||
debug!("Extracting pair from node, text: '{}'", &text[node.start_byte()..node.end_byte()]);
|
|
||||||
for child in node.children(&mut node.walk()) {
|
for child in node.children(&mut node.walk()) {
|
||||||
debug!("Pair child node kind: '{}', text: '{}'", child.kind(), &text[child.start_byte()..child.end_byte()]);
|
|
||||||
match child.kind() {
|
match child.kind() {
|
||||||
"string" => {
|
"string" => {
|
||||||
if key.is_none() {
|
if key.is_none() {
|
||||||
let extracted_key = self.extract_string_content(child, text)?;
|
let extracted_key = self.extract_string_content(child, text)?;
|
||||||
debug!("Extracted key: '{}'", extracted_key);
|
|
||||||
key = Some(extracted_key);
|
key = Some(extracted_key);
|
||||||
} else {
|
} else {
|
||||||
let extracted_value = self.extract_json(child, text, recovery_strategies)?;
|
let extracted_value = self.extract_json(child, text, recovery_strategies)?;
|
||||||
debug!("Extracted value from string: {:?}", extracted_value);
|
|
||||||
value = Some(extracted_value);
|
value = Some(extracted_value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
"value" => {
|
"value" => {
|
||||||
let extracted_value = self.extract_json(child, text, recovery_strategies)?;
|
let extracted_value = self.extract_json(child, text, recovery_strategies)?;
|
||||||
debug!("Extracted value from value node: {:?}", extracted_value);
|
|
||||||
value = Some(extracted_value);
|
value = Some(extracted_value);
|
||||||
}
|
}
|
||||||
// 直接处理各种值类型
|
// 直接处理各种值类型
|
||||||
"object" | "array" | "number" | "true" | "false" | "null" => {
|
"object" | "array" | "number" | "true" | "false" | "null" => {
|
||||||
let extracted_value = self.extract_json(child, text, recovery_strategies)?;
|
let extracted_value = self.extract_json(child, text, recovery_strategies)?;
|
||||||
debug!("Extracted value from {} node: {:?}", child.kind(), extracted_value);
|
|
||||||
value = Some(extracted_value);
|
value = Some(extracted_value);
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
debug!("Ignoring pair child node kind: '{}'", child.kind());
|
// 忽略冒号等分隔符
|
||||||
} // 忽略冒号等分隔符
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -475,6 +567,13 @@ impl TolerantJsonParser {
|
|||||||
/// 提取字符串值
|
/// 提取字符串值
|
||||||
fn extract_string(&self, node: Node, text: &str) -> Result<Value> {
|
fn extract_string(&self, node: Node, text: &str) -> Result<Value> {
|
||||||
let content = self.extract_string_content(node, text)?;
|
let content = self.extract_string_content(node, text)?;
|
||||||
|
|
||||||
|
// 检测字符串是否为YAML格式,如果是则自动解析
|
||||||
|
if let Ok(yaml_value) = self.try_parse_yaml(&content) {
|
||||||
|
debug!("Detected and parsed YAML content in string field");
|
||||||
|
return Ok(yaml_value);
|
||||||
|
}
|
||||||
|
|
||||||
Ok(Value::String(content))
|
Ok(Value::String(content))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -531,6 +630,12 @@ impl TolerantJsonParser {
|
|||||||
let error_text = &text[node.start_byte()..node.end_byte()].trim();
|
let error_text = &text[node.start_byte()..node.end_byte()].trim();
|
||||||
debug!("Attempting to recover from error: {}", error_text);
|
debug!("Attempting to recover from error: {}", error_text);
|
||||||
|
|
||||||
|
// 首先尝试智能字符串恢复(针对复杂字符串内容)
|
||||||
|
if let Ok(result) = self.try_smart_string_recovery(error_text) {
|
||||||
|
recovery_strategies.push("SmartStringRecovery".to_string());
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
|
||||||
// 尝试不同的恢复策略
|
// 尝试不同的恢复策略
|
||||||
for strategy in &self.config.recovery_strategies {
|
for strategy in &self.config.recovery_strategies {
|
||||||
match strategy {
|
match strategy {
|
||||||
@@ -561,9 +666,16 @@ impl TolerantJsonParser {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 如果所有策略都失败,返回原始文本作为字符串
|
// 如果所有策略都失败,但内容不为空,保留原始文本作为字符串
|
||||||
warn!("All recovery strategies failed for: {}", error_text);
|
// 这对于包含复杂内容(如YAML)的字段很重要
|
||||||
Ok(Value::String(error_text.to_string()))
|
if !error_text.is_empty() {
|
||||||
|
warn!("All recovery strategies failed, preserving content as string: {} chars", error_text.len());
|
||||||
|
recovery_strategies.push("PreserveAsString".to_string());
|
||||||
|
Ok(Value::String(error_text.to_string()))
|
||||||
|
} else {
|
||||||
|
warn!("All recovery strategies failed for empty content");
|
||||||
|
Ok(Value::Null)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 尝试标准JSON解析
|
/// 尝试标准JSON解析
|
||||||
@@ -571,6 +683,97 @@ impl TolerantJsonParser {
|
|||||||
serde_json::from_str(text).map_err(|e| anyhow!("Standard JSON parse failed: {}", e))
|
serde_json::from_str(text).map_err(|e| anyhow!("Standard JSON parse failed: {}", e))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 智能字符串恢复 - 专门处理复杂字符串内容,包括YAML格式
|
||||||
|
fn try_smart_string_recovery(&self, text: &str) -> Result<Value> {
|
||||||
|
let trimmed = text.trim();
|
||||||
|
|
||||||
|
// 首先尝试检测和解析YAML格式
|
||||||
|
if let Ok(yaml_value) = self.try_parse_yaml(trimmed) {
|
||||||
|
debug!("Recovered content as YAML");
|
||||||
|
return Ok(yaml_value);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查是否是一个键值对格式
|
||||||
|
if let Some(colon_pos) = trimmed.find(':') {
|
||||||
|
let key_part = trimmed[..colon_pos].trim();
|
||||||
|
let value_part = trimmed[colon_pos+1..].trim();
|
||||||
|
|
||||||
|
// 清理键名(移除引号)
|
||||||
|
let clean_key = key_part.trim_matches(|c| c == '"' || c == '\'' || c == ' ');
|
||||||
|
|
||||||
|
// 首先尝试将值部分解析为YAML
|
||||||
|
if let Ok(yaml_value) = self.try_parse_yaml(value_part) {
|
||||||
|
let mut object = Map::new();
|
||||||
|
object.insert(clean_key.to_string(), yaml_value);
|
||||||
|
debug!("Recovered key-value pair with YAML value");
|
||||||
|
return Ok(Value::Object(object));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果值部分是一个被引号包围的字符串
|
||||||
|
if (value_part.starts_with('"') && value_part.len() > 1) {
|
||||||
|
// 寻找匹配的结束引号,考虑转义字符
|
||||||
|
let mut end_pos = None;
|
||||||
|
let mut chars = value_part[1..].char_indices();
|
||||||
|
let mut escaped = false;
|
||||||
|
|
||||||
|
while let Some((i, ch)) = chars.next() {
|
||||||
|
if escaped {
|
||||||
|
escaped = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
match ch {
|
||||||
|
'\\' => escaped = true,
|
||||||
|
'"' => {
|
||||||
|
end_pos = Some(i + 1);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let string_content = if let Some(end) = end_pos {
|
||||||
|
// 找到了结束引号
|
||||||
|
&value_part[1..end]
|
||||||
|
} else {
|
||||||
|
// 没有找到结束引号,取整个内容(去掉开头的引号)
|
||||||
|
&value_part[1..]
|
||||||
|
};
|
||||||
|
|
||||||
|
// 尝试将字符串内容解析为YAML
|
||||||
|
if let Ok(yaml_value) = self.try_parse_yaml(string_content) {
|
||||||
|
let mut object = Map::new();
|
||||||
|
object.insert(clean_key.to_string(), yaml_value);
|
||||||
|
debug!("Recovered quoted string content as YAML");
|
||||||
|
return Ok(Value::Object(object));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut object = Map::new();
|
||||||
|
object.insert(clean_key.to_string(), Value::String(string_content.to_string()));
|
||||||
|
return Ok(Value::Object(object));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(anyhow!("Smart string recovery failed"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 回退解析策略 - 当Tree-sitter错误率过高时使用
|
||||||
|
fn try_fallback_parsing(&self, text: &str) -> Result<Value> {
|
||||||
|
// 尝试基本的文本清理
|
||||||
|
let mut cleaned = text.to_string();
|
||||||
|
|
||||||
|
// 移除可能的BOM
|
||||||
|
if cleaned.starts_with('\u{FEFF}') {
|
||||||
|
cleaned = cleaned[3..].to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
// 尝试修复常见的JSON错误
|
||||||
|
cleaned = self.fix_common_json_errors(&cleaned);
|
||||||
|
|
||||||
|
// 尝试解析清理后的文本
|
||||||
|
serde_json::from_str(&cleaned).map_err(|e| anyhow!("Fallback parsing failed: {}", e))
|
||||||
|
}
|
||||||
|
|
||||||
/// 尝试手动修复常见的JSON错误
|
/// 尝试手动修复常见的JSON错误
|
||||||
fn try_manual_fix(&self, text: &str) -> Result<Value> {
|
fn try_manual_fix(&self, text: &str) -> Result<Value> {
|
||||||
let mut fixed_text = text.to_string();
|
let mut fixed_text = text.to_string();
|
||||||
@@ -640,6 +843,12 @@ impl TolerantJsonParser {
|
|||||||
) -> Result<Value> {
|
) -> Result<Value> {
|
||||||
let error_text = &text[node.start_byte()..node.end_byte()];
|
let error_text = &text[node.start_byte()..node.end_byte()];
|
||||||
|
|
||||||
|
// 首先检查整个错误文本是否为YAML格式
|
||||||
|
if let Ok(yaml_value) = self.try_parse_yaml(error_text) {
|
||||||
|
recovery_strategies.push("ObjectMemberYamlRecovery".to_string());
|
||||||
|
return Ok(yaml_value);
|
||||||
|
}
|
||||||
|
|
||||||
// 尝试解析为单个键值对
|
// 尝试解析为单个键值对
|
||||||
if let Some(colon_pos) = error_text.find(':') {
|
if let Some(colon_pos) = error_text.find(':') {
|
||||||
let key_part = error_text[..colon_pos].trim();
|
let key_part = error_text[..colon_pos].trim();
|
||||||
@@ -648,17 +857,133 @@ impl TolerantJsonParser {
|
|||||||
// 清理键名
|
// 清理键名
|
||||||
let clean_key = key_part.trim_matches(|c| c == '"' || c == '\'' || c == ' ');
|
let clean_key = key_part.trim_matches(|c| c == '"' || c == '\'' || c == ' ');
|
||||||
|
|
||||||
// 尝试解析值
|
// 首先尝试将值部分解析为YAML
|
||||||
|
if let Ok(yaml_value) = self.try_parse_yaml(value_part) {
|
||||||
|
let mut object = Map::new();
|
||||||
|
object.insert(clean_key.to_string(), yaml_value);
|
||||||
|
recovery_strategies.push("ObjectMemberValueYamlRecovery".to_string());
|
||||||
|
return Ok(Value::Object(object));
|
||||||
|
}
|
||||||
|
|
||||||
|
// 然后尝试标准JSON解析
|
||||||
if let Ok(value) = self.try_standard_json_parse(value_part) {
|
if let Ok(value) = self.try_standard_json_parse(value_part) {
|
||||||
let mut object = Map::new();
|
let mut object = Map::new();
|
||||||
object.insert(clean_key.to_string(), value);
|
object.insert(clean_key.to_string(), value);
|
||||||
recovery_strategies.push("ObjectMemberRecovery".to_string());
|
recovery_strategies.push("ObjectMemberRecovery".to_string());
|
||||||
return Ok(Value::Object(object));
|
return Ok(Value::Object(object));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 如果标准解析失败,但值部分不为空,保留为字符串
|
||||||
|
if !value_part.is_empty() {
|
||||||
|
let mut object = Map::new();
|
||||||
|
// 移除值部分的引号(如果存在)
|
||||||
|
let clean_value = if (value_part.starts_with('"') && value_part.ends_with('"')) ||
|
||||||
|
(value_part.starts_with('\'') && value_part.ends_with('\'')) {
|
||||||
|
&value_part[1..value_part.len()-1]
|
||||||
|
} else {
|
||||||
|
value_part
|
||||||
|
};
|
||||||
|
object.insert(clean_key.to_string(), Value::String(clean_value.to_string()));
|
||||||
|
recovery_strategies.push("ObjectMemberStringRecovery".to_string());
|
||||||
|
return Ok(Value::Object(object));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Err(anyhow!("Failed to recover object member"))
|
Err(anyhow!("Failed to recover object member"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 检测字符串是否为YAML格式
|
||||||
|
fn is_yaml_format(&self, text: &str) -> bool {
|
||||||
|
let trimmed = text.trim();
|
||||||
|
|
||||||
|
// 空字符串或太短的字符串不认为是YAML
|
||||||
|
if trimmed.len() < 3 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果包含JSON特征(大括号或方括号开头),则不是YAML
|
||||||
|
if trimmed.starts_with('{') || trimmed.starts_with('[') {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查是否包含YAML特征
|
||||||
|
let has_yaml_structure = self.regex_patterns
|
||||||
|
.get("yaml_structure")
|
||||||
|
.map(|regex| regex.is_match(trimmed))
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
let has_yaml_list = self.regex_patterns
|
||||||
|
.get("yaml_list")
|
||||||
|
.map(|regex| regex.is_match(trimmed))
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
let has_yaml_multiline = self.regex_patterns
|
||||||
|
.get("yaml_multiline")
|
||||||
|
.map(|regex| regex.is_match(trimmed))
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
// 额外检查:包含多行且有冒号的内容
|
||||||
|
let has_multiline_with_colon = trimmed.contains('\n') && trimmed.contains(':');
|
||||||
|
|
||||||
|
// 如果包含任何YAML特征,则认为是YAML格式
|
||||||
|
has_yaml_structure || has_yaml_list || has_yaml_multiline || has_multiline_with_colon
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 尝试解析YAML内容
|
||||||
|
fn try_parse_yaml(&self, text: &str) -> Result<Value> {
|
||||||
|
let trimmed = text.trim();
|
||||||
|
|
||||||
|
// 首先检测是否为YAML格式
|
||||||
|
if !self.is_yaml_format(trimmed) {
|
||||||
|
return Err(anyhow!("Not a YAML format"));
|
||||||
|
}
|
||||||
|
|
||||||
|
debug!("Attempting to parse YAML content: {} chars", trimmed.len());
|
||||||
|
|
||||||
|
// 尝试解析YAML
|
||||||
|
match serde_yaml::from_str::<Value>(trimmed) {
|
||||||
|
Ok(yaml_value) => {
|
||||||
|
info!("Successfully parsed YAML content");
|
||||||
|
Ok(yaml_value)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
debug!("YAML parsing failed: {}", e);
|
||||||
|
Err(anyhow!("YAML parsing failed: {}", e))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 处理JSON值中的YAML字符串字段
|
||||||
|
fn process_yaml_in_json_value(&self, value: Value, recovery_strategies: &mut Vec<String>) -> Value {
|
||||||
|
match value {
|
||||||
|
Value::String(s) => {
|
||||||
|
// 检查字符串是否为YAML格式,如果是则尝试解析
|
||||||
|
if let Ok(yaml_value) = self.try_parse_yaml(&s) {
|
||||||
|
recovery_strategies.push("YamlStringParsing".to_string());
|
||||||
|
debug!("Converted string field to YAML object/array");
|
||||||
|
yaml_value
|
||||||
|
} else {
|
||||||
|
Value::String(s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Value::Object(obj) => {
|
||||||
|
// 递归处理对象中的每个字段
|
||||||
|
let mut new_obj = Map::new();
|
||||||
|
for (key, val) in obj {
|
||||||
|
new_obj.insert(key, self.process_yaml_in_json_value(val, recovery_strategies));
|
||||||
|
}
|
||||||
|
Value::Object(new_obj)
|
||||||
|
}
|
||||||
|
Value::Array(arr) => {
|
||||||
|
// 递归处理数组中的每个元素
|
||||||
|
let new_arr: Vec<Value> = arr.into_iter()
|
||||||
|
.map(|val| self.process_yaml_in_json_value(val, recovery_strategies))
|
||||||
|
.collect();
|
||||||
|
Value::Array(new_arr)
|
||||||
|
}
|
||||||
|
_ => value, // 其他类型直接返回
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 带缓存的容错JSON解析器
|
/// 带缓存的容错JSON解析器
|
||||||
@@ -974,4 +1299,393 @@ mod tests {
|
|||||||
|
|
||||||
println!("✅ 括号匹配测试通过");
|
println!("✅ 括号匹配测试通过");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_yaml_detection_and_parsing() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
// 测试简单的YAML键值对
|
||||||
|
let yaml_content = r#"name: John Doe
|
||||||
|
age: 30
|
||||||
|
city: New York"#;
|
||||||
|
|
||||||
|
println!("🔍 测试YAML内容: {}", yaml_content);
|
||||||
|
println!("🔍 YAML检测结果: {}", parser.is_yaml_format(yaml_content));
|
||||||
|
|
||||||
|
let json_with_yaml = format!(r#"{{"config": "{}"}}"#, yaml_content);
|
||||||
|
println!("🔍 完整JSON: {}", json_with_yaml);
|
||||||
|
|
||||||
|
let result = parser.parse(&json_with_yaml);
|
||||||
|
assert!(result.is_ok(), "YAML detection parsing failed");
|
||||||
|
|
||||||
|
let (parsed, stats) = result.unwrap();
|
||||||
|
assert!(parsed.is_object());
|
||||||
|
|
||||||
|
let obj = parsed.as_object().unwrap();
|
||||||
|
assert!(obj.contains_key("config"));
|
||||||
|
|
||||||
|
// 检查YAML是否被正确解析为对象
|
||||||
|
let config_value = obj.get("config").unwrap();
|
||||||
|
println!("🔍 解析后的config值类型: {:?}", config_value);
|
||||||
|
|
||||||
|
if config_value.is_object() {
|
||||||
|
let config_obj = config_value.as_object().unwrap();
|
||||||
|
assert!(config_obj.contains_key("name"));
|
||||||
|
assert!(config_obj.contains_key("age"));
|
||||||
|
assert!(config_obj.contains_key("city"));
|
||||||
|
|
||||||
|
assert_eq!(config_obj.get("name").unwrap(), &Value::String("John Doe".to_string()));
|
||||||
|
assert_eq!(config_obj.get("age").unwrap(), &Value::Number(serde_json::Number::from(30)));
|
||||||
|
assert_eq!(config_obj.get("city").unwrap(), &Value::String("New York".to_string()));
|
||||||
|
|
||||||
|
println!("✅ YAML自动解析测试通过");
|
||||||
|
} else {
|
||||||
|
println!("⚠️ YAML内容未被解析,保持为字符串格式");
|
||||||
|
println!("🔍 实际值: {:?}", config_value);
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("📊 YAML解析统计: 恢复策略={:?}", stats.recovery_strategies_used);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_yaml_list_parsing() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
// 测试YAML列表格式
|
||||||
|
let yaml_list = r#"- item1
|
||||||
|
- item2
|
||||||
|
- item3"#;
|
||||||
|
|
||||||
|
let json_with_yaml_list = format!(r#"{{"items": "{}"}}"#, yaml_list);
|
||||||
|
|
||||||
|
let result = parser.parse(&json_with_yaml_list);
|
||||||
|
assert!(result.is_ok(), "YAML list parsing failed");
|
||||||
|
|
||||||
|
let (parsed, _) = result.unwrap();
|
||||||
|
let obj = parsed.as_object().unwrap();
|
||||||
|
let items_value = obj.get("items").unwrap();
|
||||||
|
|
||||||
|
if items_value.is_array() {
|
||||||
|
let items_array = items_value.as_array().unwrap();
|
||||||
|
assert_eq!(items_array.len(), 3);
|
||||||
|
assert_eq!(items_array[0], Value::String("item1".to_string()));
|
||||||
|
assert_eq!(items_array[1], Value::String("item2".to_string()));
|
||||||
|
assert_eq!(items_array[2], Value::String("item3".to_string()));
|
||||||
|
|
||||||
|
println!("✅ YAML列表自动解析测试通过");
|
||||||
|
} else {
|
||||||
|
println!("⚠️ YAML列表内容未被解析,保持为字符串格式");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_complex_yaml_parsing() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
// 测试复杂的YAML结构
|
||||||
|
let complex_yaml = r#"database:
|
||||||
|
host: localhost
|
||||||
|
port: 5432
|
||||||
|
credentials:
|
||||||
|
username: admin
|
||||||
|
password: secret
|
||||||
|
features:
|
||||||
|
- authentication
|
||||||
|
- logging
|
||||||
|
- monitoring"#;
|
||||||
|
|
||||||
|
let json_with_complex_yaml = format!(r#"{{"config": "{}"}}"#, complex_yaml);
|
||||||
|
|
||||||
|
let result = parser.parse(&json_with_complex_yaml);
|
||||||
|
assert!(result.is_ok(), "Complex YAML parsing failed");
|
||||||
|
|
||||||
|
let (parsed, _) = result.unwrap();
|
||||||
|
let obj = parsed.as_object().unwrap();
|
||||||
|
let config_value = obj.get("config").unwrap();
|
||||||
|
|
||||||
|
if config_value.is_object() {
|
||||||
|
let config_obj = config_value.as_object().unwrap();
|
||||||
|
assert!(config_obj.contains_key("database"));
|
||||||
|
assert!(config_obj.contains_key("features"));
|
||||||
|
|
||||||
|
// 检查嵌套对象
|
||||||
|
let database = config_obj.get("database").unwrap().as_object().unwrap();
|
||||||
|
assert!(database.contains_key("host"));
|
||||||
|
assert!(database.contains_key("credentials"));
|
||||||
|
|
||||||
|
// 检查数组
|
||||||
|
let features = config_obj.get("features").unwrap().as_array().unwrap();
|
||||||
|
assert_eq!(features.len(), 3);
|
||||||
|
|
||||||
|
println!("✅ 复杂YAML结构自动解析测试通过");
|
||||||
|
} else {
|
||||||
|
println!("⚠️ 复杂YAML内容未被解析,保持为字符串格式");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_direct_yaml_parsing() {
|
||||||
|
let parser = create_test_parser();
|
||||||
|
|
||||||
|
// 测试直接YAML解析功能
|
||||||
|
let yaml_content = r#"name: John Doe
|
||||||
|
age: 30
|
||||||
|
city: New York"#;
|
||||||
|
|
||||||
|
println!("🔍 直接测试YAML解析");
|
||||||
|
println!("🔍 YAML内容: {}", yaml_content);
|
||||||
|
println!("🔍 YAML检测结果: {}", parser.is_yaml_format(yaml_content));
|
||||||
|
|
||||||
|
let yaml_result = parser.try_parse_yaml(yaml_content);
|
||||||
|
assert!(yaml_result.is_ok(), "直接YAML解析失败");
|
||||||
|
|
||||||
|
let yaml_value = yaml_result.unwrap();
|
||||||
|
assert!(yaml_value.is_object());
|
||||||
|
|
||||||
|
let yaml_obj = yaml_value.as_object().unwrap();
|
||||||
|
assert!(yaml_obj.contains_key("name"));
|
||||||
|
assert!(yaml_obj.contains_key("age"));
|
||||||
|
assert!(yaml_obj.contains_key("city"));
|
||||||
|
|
||||||
|
println!("✅ 直接YAML解析功能正常");
|
||||||
|
|
||||||
|
// 测试YAML列表
|
||||||
|
let yaml_list = r#"- item1
|
||||||
|
- item2
|
||||||
|
- item3"#;
|
||||||
|
|
||||||
|
println!("🔍 测试YAML列表: {}", yaml_list);
|
||||||
|
let list_result = parser.try_parse_yaml(yaml_list);
|
||||||
|
assert!(list_result.is_ok(), "YAML列表解析失败");
|
||||||
|
|
||||||
|
let list_value = list_result.unwrap();
|
||||||
|
assert!(list_value.is_array());
|
||||||
|
|
||||||
|
let list_array = list_value.as_array().unwrap();
|
||||||
|
assert_eq!(list_array.len(), 3);
|
||||||
|
|
||||||
|
println!("✅ YAML列表解析功能正常");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_yaml_functionality_demo() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
println!("\n🎯 YAML功能演示");
|
||||||
|
println!("================");
|
||||||
|
|
||||||
|
// 演示1:YAML列表自动解析
|
||||||
|
let json_with_yaml_list = r#"{"tags": "- frontend\n- backend\n- database"}"#;
|
||||||
|
println!("\n📝 测试1: YAML列表自动解析");
|
||||||
|
println!("输入: {}", json_with_yaml_list);
|
||||||
|
|
||||||
|
let result = parser.parse(json_with_yaml_list).unwrap();
|
||||||
|
let tags = result.0.get("tags").unwrap();
|
||||||
|
if tags.is_array() {
|
||||||
|
println!("✅ 成功解析为数组: {:?}", tags.as_array().unwrap());
|
||||||
|
} else {
|
||||||
|
println!("⚠️ 保持为字符串: {:?}", tags.as_str().unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
// 演示2:直接YAML解析API
|
||||||
|
println!("\n📝 测试2: 直接YAML解析API");
|
||||||
|
let yaml_content = "name: 测试项目\nversion: 1.0\nfeatures:\n - auth\n - logging";
|
||||||
|
println!("YAML内容:\n{}", yaml_content);
|
||||||
|
|
||||||
|
if let Ok(parsed_yaml) = parser.try_parse_yaml(yaml_content) {
|
||||||
|
println!("✅ 解析成功: {}", serde_json::to_string_pretty(&parsed_yaml).unwrap());
|
||||||
|
} else {
|
||||||
|
println!("❌ 解析失败");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 演示3:YAML检测功能
|
||||||
|
println!("\n📝 测试3: YAML检测功能");
|
||||||
|
let test_cases = vec![
|
||||||
|
("name: value", "YAML键值对"),
|
||||||
|
("- item1\n- item2", "YAML列表"),
|
||||||
|
("{\"key\": \"value\"}", "JSON对象"),
|
||||||
|
("[1, 2, 3]", "JSON数组"),
|
||||||
|
("simple text", "普通文本"),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (content, description) in test_cases {
|
||||||
|
let is_yaml = parser.is_yaml_format(content);
|
||||||
|
println!(" {} -> {} ({})", content.replace('\n', "\\n"), if is_yaml { "✅ YAML" } else { "❌ 非YAML" }, description);
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("\n🎉 YAML功能演示完成!");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_retrieved_context_yaml_parsing() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
println!("\n🔍 测试retrievedContext.text中的YAML解析");
|
||||||
|
println!("==========================================");
|
||||||
|
|
||||||
|
// 模拟promptx/res(1).json中的retrievedContext.text内容
|
||||||
|
let yaml_content = "categories:\n - 修身\n - 抹胸\n - 高腰\n - 宽松\n - 涤纶\n - 西装外套\n - 吊带背心\n - 运动短裤\n - 短款上衣\n - 衬衫\n - 圆领\n - 棉麻混纺\n - 抽绳\n - 休闲\n - 短袖T恤\n - 纯色\n - 通勤\n - 棉质\n - 阔腿短裤\ndescription: 三位女性模特在城市街道背景下展示夏季时尚穿搭,她们都穿着黑白配色的服装,风格各异,展现出休闲与时尚并存的夏日风情。\nenvironment_color_pattern:\n Hue: 0.10000000000000001\n Saturation: 0.080000000000000002\n Value: 0.75\n rgb_hex: \"#bfb9af\"\nenvironment_tags:\n - 城市街道\n - 夏季\n - 晴朗\n - 欧洲风格建筑\n - 户外\nmodels:\n - dress_color_pattern:\n Hue: 0\n Saturation: 0\n Value: 0.5\n rgb_hex: \"#7f7f7f\"\n position: 画面左侧\n posture: 站立,身体略微侧向右侧";
|
||||||
|
|
||||||
|
println!("🔍 YAML内容长度: {} 字符", yaml_content.len());
|
||||||
|
println!("🔍 YAML检测结果: {}", parser.is_yaml_format(yaml_content));
|
||||||
|
|
||||||
|
// 测试直接YAML解析
|
||||||
|
match parser.try_parse_yaml(yaml_content) {
|
||||||
|
Ok(yaml_value) => {
|
||||||
|
println!("✅ 直接YAML解析成功");
|
||||||
|
println!("🔍 解析结果类型: {:?}", yaml_value);
|
||||||
|
|
||||||
|
if let Some(obj) = yaml_value.as_object() {
|
||||||
|
println!("🔍 顶级字段: {:?}", obj.keys().collect::<Vec<_>>());
|
||||||
|
|
||||||
|
// 检查categories字段
|
||||||
|
if let Some(categories) = obj.get("categories") {
|
||||||
|
if categories.is_array() {
|
||||||
|
println!("✅ categories字段解析为数组,长度: {}", categories.as_array().unwrap().len());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查environment_color_pattern字段
|
||||||
|
if let Some(env_pattern) = obj.get("environment_color_pattern") {
|
||||||
|
if env_pattern.is_object() {
|
||||||
|
println!("✅ environment_color_pattern字段解析为对象");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("❌ 直接YAML解析失败: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 测试在JSON中包含此YAML内容
|
||||||
|
let json_with_yaml = format!(r#"{{"retrievedContext": {{"text": "{}"}}}}"#,
|
||||||
|
yaml_content.replace('\n', "\\n").replace('"', "\\\""));
|
||||||
|
|
||||||
|
println!("\n🔍 测试JSON中包含YAML内容的解析");
|
||||||
|
println!("🔍 JSON长度: {} 字符", json_with_yaml.len());
|
||||||
|
|
||||||
|
match parser.parse(&json_with_yaml) {
|
||||||
|
Ok((parsed, stats)) => {
|
||||||
|
println!("✅ JSON解析成功");
|
||||||
|
println!("📊 解析统计: 恢复策略={:?}", stats.recovery_strategies_used);
|
||||||
|
|
||||||
|
if let Some(obj) = parsed.as_object() {
|
||||||
|
if let Some(retrieved_context) = obj.get("retrievedContext") {
|
||||||
|
if let Some(context_obj) = retrieved_context.as_object() {
|
||||||
|
if let Some(text_value) = context_obj.get("text") {
|
||||||
|
match text_value {
|
||||||
|
serde_json::Value::Object(_) => {
|
||||||
|
println!("✅ text字段被解析为YAML对象");
|
||||||
|
}
|
||||||
|
serde_json::Value::String(s) => {
|
||||||
|
println!("⚠️ text字段保持为字符串,长度: {}", s.len());
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
println!("❓ text字段类型未知: {:?}", text_value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("❌ JSON解析失败: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("\n🎉 retrievedContext.text YAML解析测试完成!");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_real_promptx_res_file() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
println!("\n🔍 测试真实的promptx/res(1).json文件");
|
||||||
|
println!("=====================================");
|
||||||
|
|
||||||
|
// 读取真实的promptx/res(1).json文件
|
||||||
|
let file_path = "../../../promptx/res(1).json";
|
||||||
|
match std::fs::read_to_string(file_path) {
|
||||||
|
Ok(content) => {
|
||||||
|
println!("🔍 文件读取成功,长度: {} 字符", content.len());
|
||||||
|
|
||||||
|
match parser.parse(&content) {
|
||||||
|
Ok((parsed, stats)) => {
|
||||||
|
println!("✅ JSON解析成功");
|
||||||
|
println!("📊 解析统计: 恢复策略={:?}", stats.recovery_strategies_used);
|
||||||
|
println!("📊 错误率: {:.2}%", stats.error_rate * 100.0);
|
||||||
|
println!("📊 解析时间: {}ms", stats.parse_time_ms);
|
||||||
|
|
||||||
|
// 检查是否包含YAML解析策略
|
||||||
|
if stats.recovery_strategies_used.contains(&"YamlStringParsing".to_string()) {
|
||||||
|
println!("✅ 检测到YAML字符串解析");
|
||||||
|
} else {
|
||||||
|
println!("⚠️ 未检测到YAML字符串解析");
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查第一个retrievedContext.text字段
|
||||||
|
if let Some(obj) = parsed.as_object() {
|
||||||
|
if let Some(candidates) = obj.get("candidates") {
|
||||||
|
if let Some(candidates_array) = candidates.as_array() {
|
||||||
|
if let Some(first_candidate) = candidates_array.first() {
|
||||||
|
if let Some(candidate_obj) = first_candidate.as_object() {
|
||||||
|
if let Some(grounding_metadata) = candidate_obj.get("groundingMetadata") {
|
||||||
|
if let Some(grounding_obj) = grounding_metadata.as_object() {
|
||||||
|
if let Some(grounding_chunks) = grounding_obj.get("groundingChunks") {
|
||||||
|
if let Some(chunks_array) = grounding_chunks.as_array() {
|
||||||
|
if let Some(first_chunk) = chunks_array.first() {
|
||||||
|
if let Some(chunk_obj) = first_chunk.as_object() {
|
||||||
|
if let Some(retrieved_context) = chunk_obj.get("retrievedContext") {
|
||||||
|
if let Some(context_obj) = retrieved_context.as_object() {
|
||||||
|
if let Some(text_value) = context_obj.get("text") {
|
||||||
|
match text_value {
|
||||||
|
serde_json::Value::Object(text_obj) => {
|
||||||
|
println!("✅ 第一个retrievedContext.text被解析为YAML对象");
|
||||||
|
println!("🔍 YAML对象字段: {:?}", text_obj.keys().collect::<Vec<_>>());
|
||||||
|
|
||||||
|
// 检查categories字段
|
||||||
|
if let Some(categories) = text_obj.get("categories") {
|
||||||
|
if let Some(categories_array) = categories.as_array() {
|
||||||
|
println!("✅ categories字段包含 {} 个元素", categories_array.len());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::Value::String(s) => {
|
||||||
|
println!("⚠️ 第一个retrievedContext.text仍为字符串,长度: {}", s.len());
|
||||||
|
println!("🔍 前100字符: {}", &s[..std::cmp::min(100, s.len())]);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
println!("❓ 第一个retrievedContext.text类型未知: {:?}", text_value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("❌ JSON解析失败: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("⚠️ 无法读取文件 {}: {}", file_path, e);
|
||||||
|
println!("💡 这是正常的,因为测试可能在不同的工作目录运行");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("\n🎉 真实文件测试完成!");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
|
|
||||||
### 🚀 核心功能
|
### 🚀 核心功能
|
||||||
- **容错解析**: 处理各种JSON格式错误,包括语法错误、格式不一致等
|
- **容错解析**: 处理各种JSON格式错误,包括语法错误、格式不一致等
|
||||||
|
- **YAML智能解析**: 自动识别并解析JSON字符串字段中的YAML内容
|
||||||
- **多种恢复策略**: 标准JSON解析、手动修复、正则提取、部分解析
|
- **多种恢复策略**: 标准JSON解析、手动修复、正则提取、部分解析
|
||||||
- **Markdown支持**: 自动提取Markdown代码块中的JSON内容
|
- **Markdown支持**: 自动提取Markdown代码块中的JSON内容
|
||||||
- **性能优化**: 内置缓存机制,支持高频解析场景
|
- **性能优化**: 内置缓存机制,支持高频解析场景
|
||||||
@@ -19,6 +20,13 @@
|
|||||||
- **混合内容**: 从包含解释性文本的内容中提取JSON
|
- **混合内容**: 从包含解释性文本的内容中提取JSON
|
||||||
- **部分截断**: 处理不完整的JSON数据
|
- **部分截断**: 处理不完整的JSON数据
|
||||||
|
|
||||||
|
### 🎯 YAML智能解析
|
||||||
|
- **自动检测**: 智能识别JSON字符串字段中的YAML格式内容
|
||||||
|
- **键值对解析**: `"config: name: John\nage: 30"` → `{"name": "John", "age": 30}`
|
||||||
|
- **列表解析**: `"items: - item1\n- item2"` → `["item1", "item2"]`
|
||||||
|
- **嵌套结构**: 支持复杂的YAML嵌套对象和数组
|
||||||
|
- **混合格式**: 处理大模型返回的JSON+YAML混合内容
|
||||||
|
|
||||||
## 快速开始
|
## 快速开始
|
||||||
|
|
||||||
### Rust后端使用
|
### Rust后端使用
|
||||||
@@ -43,6 +51,33 @@ println!("错误率: {:.2}%", stats.error_rate * 100.0);
|
|||||||
println!("使用的恢复策略: {:?}", stats.recovery_strategies_used);
|
println!("使用的恢复策略: {:?}", stats.recovery_strategies_used);
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### YAML智能解析示例
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// 处理包含YAML内容的JSON
|
||||||
|
let json_with_yaml = r#"
|
||||||
|
{
|
||||||
|
"retrievedContext": {
|
||||||
|
"text": "categories:\n - 修身\n - 抹胸\n - 高腰\ndescription: 三位女性模特展示夏季时尚穿搭\nenvironment_tags:\n - 城市街道\n - 夏季\n - 晴朗"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let (result, stats) = parser.parse(json_with_yaml)?;
|
||||||
|
|
||||||
|
// text字段会被自动解析为YAML对象
|
||||||
|
if let Some(text_obj) = result["retrievedContext"]["text"].as_object() {
|
||||||
|
println!("categories: {:?}", text_obj["categories"]);
|
||||||
|
println!("description: {:?}", text_obj["description"]);
|
||||||
|
println!("environment_tags: {:?}", text_obj["environment_tags"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查是否使用了YAML解析
|
||||||
|
if stats.recovery_strategies_used.contains(&"YamlStringParsing".to_string()) {
|
||||||
|
println!("✅ 检测到并解析了YAML内容");
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
### 带缓存的解析器
|
### 带缓存的解析器
|
||||||
|
|
||||||
```rust
|
```rust
|
||||||
|
|||||||
@@ -311,7 +311,7 @@ pub fn run() {
|
|||||||
.setup(|app| {
|
.setup(|app| {
|
||||||
// 初始化日志系统
|
// 初始化日志系统
|
||||||
let mut log_config = logging::LogConfig::default();
|
let mut log_config = logging::LogConfig::default();
|
||||||
log_config.level = tracing::Level::DEBUG; // 设置为 DEBUG 级别以显示更多日志
|
log_config.level = tracing::Level::INFO; // 设置为 INFO 级别以减少日志噪音
|
||||||
if let Err(e) = logging::init_logging(log_config) {
|
if let Err(e) = logging::init_logging(log_config) {
|
||||||
eprintln!("日志系统初始化失败: {}", e);
|
eprintln!("日志系统初始化失败: {}", e);
|
||||||
}
|
}
|
||||||
@@ -354,3 +354,5 @@ pub fn run() {
|
|||||||
mod tests {
|
mod tests {
|
||||||
mod batch_delete_test;
|
mod batch_delete_test;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -261,7 +261,7 @@ mod tests {
|
|||||||
sort_order: Some(1),
|
sort_order: Some(1),
|
||||||
};
|
};
|
||||||
|
|
||||||
let classification = service.create_classification(request).await.unwrap();
|
let _classification = service.create_classification(request).await.unwrap();
|
||||||
|
|
||||||
// 验证重复名称应该失败
|
// 验证重复名称应该失败
|
||||||
let duplicate_request = CreateAiClassificationRequest {
|
let duplicate_request = CreateAiClassificationRequest {
|
||||||
|
|||||||
Reference in New Issue
Block a user