feat: 实现基于Tree-sitter的容错JSON解析器
- 添加TolerantJsonParser核心解析器，支持多种错误恢复策略 - 实现CachedTolerantJsonParser带缓存的解析器，提升性能 - 支持Markdown代码块提取、无引号键、尾随逗号等容错功能 - 添加完整的Tauri命令接口，支持前端调用 - 包含全面的单元测试和集成测试 - 提供详细的API文档和使用示例 主要功能: - 标准JSON解析、手动修复、正则提取、部分解析等恢复策略 - 支持处理大模型返回的不规范JSON数据 - 内置缓存机制，支持高频解析场景 - 详细的解析统计信息和错误报告 - 遵循Tauri开发规范的分层架构设计
This commit is contained in:
imeepos
@@ -0,0 +1,768 @@
|
||||
/// 基于Tree-sitter的大模型JSON容错解析器
|
||||
/// 遵循Tauri开发规范，提供高性能、安全的JSON解析能力
|
||||
use anyhow::{anyhow, Result};
|
||||
use regex::Regex;
|
||||
use serde_json::{Map, Value};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tree_sitter::{Node, Parser, Tree};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// 解析结果统计信息
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct ParseStatistics {
|
||||
pub total_nodes: usize,
|
||||
pub error_nodes: usize,
|
||||
pub error_rate: f64,
|
||||
pub parse_time_ms: u64,
|
||||
pub recovery_strategies_used: Vec<String>,
|
||||
}
|
||||
|
||||
/// 错误恢复策略类型
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum RecoveryStrategy {
|
||||
StandardJson,
|
||||
ManualFix,
|
||||
RegexExtract,
|
||||
PartialParse,
|
||||
}
|
||||
|
||||
/// 容错JSON解析器配置
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ParserConfig {
|
||||
pub max_text_length: usize,
|
||||
pub enable_comments: bool,
|
||||
pub enable_unquoted_keys: bool,
|
||||
pub enable_trailing_commas: bool,
|
||||
pub timeout_ms: u64,
|
||||
pub recovery_strategies: Vec<RecoveryStrategy>,
|
||||
}
|
||||
|
||||
impl Default for ParserConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_text_length: 1024 * 1024, // 1MB
|
||||
enable_comments: true,
|
||||
enable_unquoted_keys: true,
|
||||
enable_trailing_commas: true,
|
||||
timeout_ms: 30000, // 30秒
|
||||
recovery_strategies: vec![
|
||||
RecoveryStrategy::StandardJson,
|
||||
RecoveryStrategy::ManualFix,
|
||||
RecoveryStrategy::RegexExtract,
|
||||
RecoveryStrategy::PartialParse,
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 容错JSON解析器
|
||||
pub struct TolerantJsonParser {
|
||||
parser: Parser,
|
||||
config: ParserConfig,
|
||||
regex_patterns: HashMap<String, Regex>,
|
||||
}
|
||||
|
||||
impl TolerantJsonParser {
|
||||
/// 创建新的容错JSON解析器实例
|
||||
pub fn new(config: Option<ParserConfig>) -> Result<Self> {
|
||||
let mut parser = Parser::new();
|
||||
|
||||
// 设置JSON语言
|
||||
let language = tree_sitter_json::language();
|
||||
parser.set_language(language)
|
||||
.map_err(|e| anyhow!("Failed to set JSON language: {}", e))?;
|
||||
|
||||
let config = config.unwrap_or_default();
|
||||
let regex_patterns = Self::init_regex_patterns()?;
|
||||
|
||||
Ok(Self {
|
||||
parser,
|
||||
config,
|
||||
regex_patterns,
|
||||
})
|
||||
}
|
||||
|
||||
/// 初始化正则表达式模式
|
||||
fn init_regex_patterns() -> Result<HashMap<String, Regex>> {
|
||||
let mut patterns = HashMap::new();
|
||||
|
||||
// JSON对象模式
|
||||
patterns.insert(
|
||||
"object".to_string(),
|
||||
Regex::new(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}")?,
|
||||
);
|
||||
|
||||
// JSON数组模式
|
||||
patterns.insert(
|
||||
"array".to_string(),
|
||||
Regex::new(r"\[[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]")?,
|
||||
);
|
||||
|
||||
// Markdown代码块模式
|
||||
patterns.insert(
|
||||
"markdown_fence".to_string(),
|
||||
Regex::new(r"(?s)```(?:json)?\s*\n?(.*?)\n?```")?,
|
||||
);
|
||||
|
||||
// 无引号键模式
|
||||
patterns.insert(
|
||||
"unquoted_key".to_string(),
|
||||
Regex::new(r"(\w+):")?,
|
||||
);
|
||||
|
||||
// 尾随逗号模式
|
||||
patterns.insert(
|
||||
"trailing_comma".to_string(),
|
||||
Regex::new(r",\s*[}\]]")?,
|
||||
);
|
||||
|
||||
Ok(patterns)
|
||||
}
|
||||
|
||||
/// 解析JSON文本，返回解析结果和统计信息
|
||||
pub fn parse(&mut self, text: &str) -> Result<(Value, ParseStatistics)> {
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
// 检查文本长度限制
|
||||
if text.len() > self.config.max_text_length {
|
||||
return Err(anyhow!("Text too long: {} bytes", text.len()));
|
||||
}
|
||||
|
||||
info!("Starting tolerant JSON parsing, text length: {}", text.len());
|
||||
|
||||
// 预处理文本
|
||||
let cleaned_text = self.preprocess_text(text)?;
|
||||
debug!("Preprocessed text length: {}", cleaned_text.len());
|
||||
|
||||
// 使用Tree-sitter解析
|
||||
let tree = self.parser.parse(&cleaned_text, None)
|
||||
.ok_or_else(|| anyhow!("Failed to parse text with tree-sitter"))?;
|
||||
|
||||
let root_node = tree.root_node();
|
||||
|
||||
// 收集统计信息
|
||||
let mut stats = self.collect_statistics(&tree, &cleaned_text);
|
||||
|
||||
// 提取和修复JSON
|
||||
let mut recovery_strategies_used = Vec::new();
|
||||
let result = self.extract_json(root_node, &cleaned_text, &mut recovery_strategies_used)?;
|
||||
|
||||
stats.parse_time_ms = start_time.elapsed().as_millis() as u64;
|
||||
stats.recovery_strategies_used = recovery_strategies_used;
|
||||
|
||||
info!("JSON parsing completed in {}ms", stats.parse_time_ms);
|
||||
|
||||
Ok((result, stats))
|
||||
}
|
||||
|
||||
/// 预处理文本，提取可能的JSON内容
|
||||
fn preprocess_text(&self, text: &str) -> Result<String> {
|
||||
let mut processed = text.to_string();
|
||||
|
||||
// 移除Markdown代码块标记
|
||||
if let Some(markdown_regex) = self.regex_patterns.get("markdown_fence") {
|
||||
if let Some(captures) = markdown_regex.captures(&processed) {
|
||||
if let Some(json_content) = captures.get(1) {
|
||||
processed = json_content.as_str().to_string();
|
||||
debug!("Extracted JSON from markdown fence");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 查找JSON模式
|
||||
for pattern_name in ["object", "array"] {
|
||||
if let Some(pattern) = self.regex_patterns.get(pattern_name) {
|
||||
if let Some(mat) = pattern.find(&processed) {
|
||||
processed = mat.as_str().to_string();
|
||||
debug!("Extracted JSON using {} pattern", pattern_name);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(processed)
|
||||
}
|
||||
|
||||
/// 收集解析统计信息
|
||||
fn collect_statistics(&self, tree: &Tree, _text: &str) -> ParseStatistics {
|
||||
let root_node = tree.root_node();
|
||||
let mut total_nodes = 0;
|
||||
let mut error_nodes = 0;
|
||||
|
||||
self.count_nodes(root_node, &mut total_nodes, &mut error_nodes);
|
||||
|
||||
let error_rate = if total_nodes > 0 {
|
||||
error_nodes as f64 / total_nodes as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
ParseStatistics {
|
||||
total_nodes,
|
||||
error_nodes,
|
||||
error_rate,
|
||||
parse_time_ms: 0, // 将在parse方法中设置
|
||||
recovery_strategies_used: Vec::new(), // 将在parse方法中设置
|
||||
}
|
||||
}
|
||||
|
||||
/// 递归计算节点数量
|
||||
fn count_nodes(&self, node: Node, total: &mut usize, errors: &mut usize) {
|
||||
*total += 1;
|
||||
|
||||
if node.is_error() {
|
||||
*errors += 1;
|
||||
}
|
||||
|
||||
for child in node.children(&mut node.walk()) {
|
||||
self.count_nodes(child, total, errors);
|
||||
}
|
||||
}
|
||||
|
||||
/// 从语法树中提取和修复JSON
|
||||
fn extract_json(
|
||||
&self,
|
||||
node: Node,
|
||||
text: &str,
|
||||
recovery_strategies: &mut Vec<String>,
|
||||
) -> Result<Value> {
|
||||
match node.kind() {
|
||||
"document" => {
|
||||
if node.child_count() > 0 {
|
||||
self.extract_json(node.child(0).unwrap(), text, recovery_strategies)
|
||||
} else {
|
||||
Ok(Value::Null)
|
||||
}
|
||||
}
|
||||
"value" => {
|
||||
if node.child_count() > 0 {
|
||||
self.extract_json(node.child(0).unwrap(), text, recovery_strategies)
|
||||
} else {
|
||||
self.recover_from_error(node, text, recovery_strategies)
|
||||
}
|
||||
}
|
||||
"object" => self.extract_object(node, text, recovery_strategies),
|
||||
"array" => self.extract_array(node, text, recovery_strategies),
|
||||
"string" => self.extract_string(node, text),
|
||||
"number" => self.extract_number(node, text),
|
||||
"true" | "false" => self.extract_boolean(node, text),
|
||||
"null" => Ok(Value::Null),
|
||||
"ERROR" => self.recover_from_error(node, text, recovery_strategies),
|
||||
_ => {
|
||||
warn!("Unknown node kind: {}", node.kind());
|
||||
self.recover_from_error(node, text, recovery_strategies)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 提取JSON对象
|
||||
fn extract_object(
|
||||
&self,
|
||||
node: Node,
|
||||
text: &str,
|
||||
recovery_strategies: &mut Vec<String>,
|
||||
) -> Result<Value> {
|
||||
let mut object = Map::new();
|
||||
|
||||
for child in node.children(&mut node.walk()) {
|
||||
match child.kind() {
|
||||
"pair" => {
|
||||
if let Ok((key, value)) = self.extract_pair(child, text, recovery_strategies) {
|
||||
object.insert(key, value);
|
||||
}
|
||||
}
|
||||
"ERROR" => {
|
||||
// 尝试恢复错误的对象成员
|
||||
if let Ok(recovered) = self.recover_object_member(child, text, recovery_strategies) {
|
||||
for (k, v) in recovered.as_object().unwrap_or(&Map::new()) {
|
||||
object.insert(k.clone(), v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {} // 忽略其他节点类型（如括号、逗号等）
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Value::Object(object))
|
||||
}
|
||||
|
||||
/// 提取JSON数组
|
||||
fn extract_array(
|
||||
&self,
|
||||
node: Node,
|
||||
text: &str,
|
||||
recovery_strategies: &mut Vec<String>,
|
||||
) -> Result<Value> {
|
||||
let mut array = Vec::new();
|
||||
|
||||
for child in node.children(&mut node.walk()) {
|
||||
match child.kind() {
|
||||
"value" => {
|
||||
if let Ok(value) = self.extract_json(child, text, recovery_strategies) {
|
||||
array.push(value);
|
||||
}
|
||||
}
|
||||
// Also handle direct primitive types
|
||||
"number" | "string" | "true" | "false" | "null" | "object" | "array" => {
|
||||
if let Ok(value) = self.extract_json(child, text, recovery_strategies) {
|
||||
array.push(value);
|
||||
}
|
||||
}
|
||||
"ERROR" => {
|
||||
// 尝试恢复错误的数组元素
|
||||
if let Ok(recovered) = self.recover_from_error(child, text, recovery_strategies) {
|
||||
array.push(recovered);
|
||||
}
|
||||
}
|
||||
_ => {} // 忽略其他节点类型（如括号、逗号等）
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Value::Array(array))
|
||||
}
|
||||
|
||||
/// 提取键值对
|
||||
fn extract_pair(
|
||||
&self,
|
||||
node: Node,
|
||||
text: &str,
|
||||
recovery_strategies: &mut Vec<String>,
|
||||
) -> Result<(String, Value)> {
|
||||
let mut key = None;
|
||||
let mut value = None;
|
||||
|
||||
for child in node.children(&mut node.walk()) {
|
||||
match child.kind() {
|
||||
"string" => {
|
||||
if key.is_none() {
|
||||
key = Some(self.extract_string_content(child, text)?);
|
||||
} else {
|
||||
value = Some(self.extract_json(child, text, recovery_strategies)?);
|
||||
}
|
||||
}
|
||||
"value" => {
|
||||
value = Some(self.extract_json(child, text, recovery_strategies)?);
|
||||
}
|
||||
_ => {} // 忽略冒号等分隔符
|
||||
}
|
||||
}
|
||||
|
||||
match (key, value) {
|
||||
(Some(k), Some(v)) => Ok((k, v)),
|
||||
_ => Err(anyhow!("Invalid key-value pair")),
|
||||
}
|
||||
}
|
||||
|
||||
/// 提取字符串值
|
||||
fn extract_string(&self, node: Node, text: &str) -> Result<Value> {
|
||||
let content = self.extract_string_content(node, text)?;
|
||||
Ok(Value::String(content))
|
||||
}
|
||||
|
||||
/// 提取字符串内容（去除引号）
|
||||
fn extract_string_content(&self, node: Node, text: &str) -> Result<String> {
|
||||
let raw_text = &text[node.start_byte()..node.end_byte()];
|
||||
|
||||
// 移除前后的引号
|
||||
if raw_text.len() >= 2 && raw_text.starts_with('"') && raw_text.ends_with('"') {
|
||||
Ok(raw_text[1..raw_text.len()-1].to_string())
|
||||
} else {
|
||||
// 处理无引号的字符串
|
||||
Ok(raw_text.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// 提取数字值
|
||||
fn extract_number(&self, node: Node, text: &str) -> Result<Value> {
|
||||
let raw_text = &text[node.start_byte()..node.end_byte()];
|
||||
|
||||
// 尝试解析为整数
|
||||
if let Ok(int_val) = raw_text.parse::<i64>() {
|
||||
Ok(Value::Number(serde_json::Number::from(int_val)))
|
||||
}
|
||||
// 尝试解析为浮点数
|
||||
else if let Ok(float_val) = raw_text.parse::<f64>() {
|
||||
if let Some(num) = serde_json::Number::from_f64(float_val) {
|
||||
Ok(Value::Number(num))
|
||||
} else {
|
||||
Err(anyhow!("Invalid float number: {}", raw_text))
|
||||
}
|
||||
} else {
|
||||
Err(anyhow!("Invalid number: {}", raw_text))
|
||||
}
|
||||
}
|
||||
|
||||
/// 提取布尔值
|
||||
fn extract_boolean(&self, node: Node, text: &str) -> Result<Value> {
|
||||
let raw_text = &text[node.start_byte()..node.end_byte()];
|
||||
match raw_text {
|
||||
"true" => Ok(Value::Bool(true)),
|
||||
"false" => Ok(Value::Bool(false)),
|
||||
_ => Err(anyhow!("Invalid boolean: {}", raw_text)),
|
||||
}
|
||||
}
|
||||
|
||||
/// 从错误节点恢复数据
|
||||
fn recover_from_error(
|
||||
&self,
|
||||
node: Node,
|
||||
text: &str,
|
||||
recovery_strategies: &mut Vec<String>,
|
||||
) -> Result<Value> {
|
||||
let error_text = &text[node.start_byte()..node.end_byte()].trim();
|
||||
debug!("Attempting to recover from error: {}", error_text);
|
||||
|
||||
// 尝试不同的恢复策略
|
||||
for strategy in &self.config.recovery_strategies {
|
||||
match strategy {
|
||||
RecoveryStrategy::StandardJson => {
|
||||
if let Ok(result) = self.try_standard_json_parse(error_text) {
|
||||
recovery_strategies.push("StandardJson".to_string());
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
RecoveryStrategy::ManualFix => {
|
||||
if let Ok(result) = self.try_manual_fix(error_text) {
|
||||
recovery_strategies.push("ManualFix".to_string());
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
RecoveryStrategy::RegexExtract => {
|
||||
if let Ok(result) = self.try_regex_extract(error_text) {
|
||||
recovery_strategies.push("RegexExtract".to_string());
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
RecoveryStrategy::PartialParse => {
|
||||
if let Ok(result) = self.try_partial_parse(error_text) {
|
||||
recovery_strategies.push("PartialParse".to_string());
|
||||
return Ok(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 如果所有策略都失败，返回原始文本作为字符串
|
||||
warn!("All recovery strategies failed for: {}", error_text);
|
||||
Ok(Value::String(error_text.to_string()))
|
||||
}
|
||||
|
||||
/// 尝试标准JSON解析
|
||||
fn try_standard_json_parse(&self, text: &str) -> Result<Value> {
|
||||
serde_json::from_str(text).map_err(|e| anyhow!("Standard JSON parse failed: {}", e))
|
||||
}
|
||||
|
||||
/// 尝试手动修复常见的JSON错误
|
||||
fn try_manual_fix(&self, text: &str) -> Result<Value> {
|
||||
let mut fixed_text = text.to_string();
|
||||
|
||||
// 修复无引号的键
|
||||
if let Some(unquoted_regex) = self.regex_patterns.get("unquoted_key") {
|
||||
fixed_text = unquoted_regex.replace_all(&fixed_text, "\"$1\":").to_string();
|
||||
}
|
||||
|
||||
// 移除尾随逗号
|
||||
if let Some(trailing_regex) = self.regex_patterns.get("trailing_comma") {
|
||||
fixed_text = trailing_regex.replace_all(&fixed_text, |caps: ®ex::Captures| {
|
||||
let full_match = caps.get(0).unwrap().as_str();
|
||||
full_match[1..].to_string() // 移除逗号，保留括号
|
||||
}).to_string();
|
||||
}
|
||||
|
||||
// 修复单引号为双引号
|
||||
fixed_text = fixed_text.replace("'", "\"");
|
||||
|
||||
// 尝试解析修复后的文本
|
||||
serde_json::from_str(&fixed_text).map_err(|e| anyhow!("Manual fix failed: {}", e))
|
||||
}
|
||||
|
||||
/// 尝试正则表达式提取
|
||||
fn try_regex_extract(&self, text: &str) -> Result<Value> {
|
||||
// 尝试提取对象
|
||||
if let Some(object_regex) = self.regex_patterns.get("object") {
|
||||
if let Some(mat) = object_regex.find(text) {
|
||||
return self.try_standard_json_parse(mat.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
// 尝试提取数组
|
||||
if let Some(array_regex) = self.regex_patterns.get("array") {
|
||||
if let Some(mat) = array_regex.find(text) {
|
||||
return self.try_standard_json_parse(mat.as_str());
|
||||
}
|
||||
}
|
||||
|
||||
Err(anyhow!("No valid JSON pattern found"))
|
||||
}
|
||||
|
||||
/// 尝试部分解析
|
||||
fn try_partial_parse(&self, text: &str) -> Result<Value> {
|
||||
// 尝试解析文本的各个部分
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
|
||||
for i in 0..lines.len() {
|
||||
for j in (i+1)..=lines.len() {
|
||||
let partial_text = lines[i..j].join("\n");
|
||||
if let Ok(value) = serde_json::from_str::<Value>(&partial_text) {
|
||||
return Ok(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(anyhow!("Partial parse failed"))
|
||||
}
|
||||
|
||||
/// 恢复错误的对象成员
|
||||
fn recover_object_member(
|
||||
&self,
|
||||
node: Node,
|
||||
text: &str,
|
||||
recovery_strategies: &mut Vec<String>,
|
||||
) -> Result<Value> {
|
||||
let error_text = &text[node.start_byte()..node.end_byte()];
|
||||
|
||||
// 尝试解析为单个键值对
|
||||
if let Some(colon_pos) = error_text.find(':') {
|
||||
let key_part = error_text[..colon_pos].trim();
|
||||
let value_part = error_text[colon_pos+1..].trim();
|
||||
|
||||
// 清理键名
|
||||
let clean_key = key_part.trim_matches(|c| c == '"' || c == '\'' || c == ' ');
|
||||
|
||||
// 尝试解析值
|
||||
if let Ok(value) = self.try_standard_json_parse(value_part) {
|
||||
let mut object = Map::new();
|
||||
object.insert(clean_key.to_string(), value);
|
||||
recovery_strategies.push("ObjectMemberRecovery".to_string());
|
||||
return Ok(Value::Object(object));
|
||||
}
|
||||
}
|
||||
|
||||
Err(anyhow!("Failed to recover object member"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 带缓存的容错JSON解析器
|
||||
pub struct CachedTolerantJsonParser {
|
||||
parser: Arc<Mutex<TolerantJsonParser>>,
|
||||
cache: Arc<Mutex<HashMap<String, (Value, ParseStatistics)>>>,
|
||||
max_cache_size: usize,
|
||||
}
|
||||
|
||||
impl CachedTolerantJsonParser {
|
||||
/// 创建新的带缓存的解析器实例
|
||||
pub fn new(config: Option<ParserConfig>, max_cache_size: Option<usize>) -> Result<Self> {
|
||||
let parser = TolerantJsonParser::new(config)?;
|
||||
let max_cache_size = max_cache_size.unwrap_or(1000);
|
||||
|
||||
Ok(Self {
|
||||
parser: Arc::new(Mutex::new(parser)),
|
||||
cache: Arc::new(Mutex::new(HashMap::new())),
|
||||
max_cache_size,
|
||||
})
|
||||
}
|
||||
|
||||
/// 解析JSON文本，使用缓存机制
|
||||
pub fn parse(&self, text: &str) -> Result<(Value, ParseStatistics)> {
|
||||
// 计算文本哈希作为缓存键
|
||||
let cache_key = self.compute_cache_key(text);
|
||||
|
||||
// 检查缓存
|
||||
{
|
||||
let cache = self.cache.lock().unwrap();
|
||||
if let Some(cached_result) = cache.get(&cache_key) {
|
||||
debug!("Cache hit for JSON parsing");
|
||||
return Ok(cached_result.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// 缓存未命中，执行解析
|
||||
let result = {
|
||||
let mut parser = self.parser.lock().unwrap();
|
||||
parser.parse(text)?
|
||||
};
|
||||
|
||||
// 存储到缓存
|
||||
{
|
||||
let mut cache = self.cache.lock().unwrap();
|
||||
|
||||
// 如果缓存已满，移除最旧的条目
|
||||
if cache.len() >= self.max_cache_size {
|
||||
// 简单的LRU实现：移除第一个条目
|
||||
if let Some(first_key) = cache.keys().next().cloned() {
|
||||
cache.remove(&first_key);
|
||||
}
|
||||
}
|
||||
|
||||
cache.insert(cache_key, result.clone());
|
||||
}
|
||||
|
||||
debug!("Cached new JSON parsing result");
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// 计算缓存键
|
||||
fn compute_cache_key(&self, text: &str) -> String {
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
|
||||
let mut hasher = DefaultHasher::new();
|
||||
text.hash(&mut hasher);
|
||||
format!("{:x}", hasher.finish())
|
||||
}
|
||||
|
||||
/// 清空缓存
|
||||
pub fn clear_cache(&self) {
|
||||
let mut cache = self.cache.lock().unwrap();
|
||||
cache.clear();
|
||||
info!("JSON parser cache cleared");
|
||||
}
|
||||
|
||||
/// 获取缓存统计信息
|
||||
pub fn get_cache_stats(&self) -> (usize, usize) {
|
||||
let cache = self.cache.lock().unwrap();
|
||||
(cache.len(), self.max_cache_size)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
/// 创建测试用的解析器实例
|
||||
fn create_test_parser() -> TolerantJsonParser {
|
||||
TolerantJsonParser::new(None).expect("Failed to create test parser")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_standard_json_parsing() {
|
||||
let mut parser = create_test_parser();
|
||||
|
||||
// Test simple cases first
|
||||
let simple_cases = vec![
|
||||
(r#"true"#, json!(true)),
|
||||
(r#"false"#, json!(false)),
|
||||
(r#"null"#, json!(null)),
|
||||
(r#"42"#, json!(42)),
|
||||
(r#""hello world""#, json!("hello world")),
|
||||
];
|
||||
|
||||
for (input, expected) in simple_cases {
|
||||
let result = parser.parse(input);
|
||||
assert!(result.is_ok(), "Failed to parse: {}", input);
|
||||
|
||||
let (parsed_value, stats) = result.unwrap();
|
||||
println!("Input: {}, Parsed: {:?}, Expected: {:?}", input, parsed_value, expected);
|
||||
assert_eq!(parsed_value, expected, "Mismatch for input: {}", input);
|
||||
assert_eq!(stats.error_rate, 0.0, "Should have no errors for valid JSON");
|
||||
}
|
||||
|
||||
// Test more complex cases
|
||||
let complex_cases = vec![
|
||||
(r#"{"name": "test"}"#, json!({"name": "test"})),
|
||||
(r#"[1, 2, 3]"#, json!([1, 2, 3])),
|
||||
];
|
||||
|
||||
for (input, expected) in complex_cases {
|
||||
let result = parser.parse(input);
|
||||
assert!(result.is_ok(), "Failed to parse: {}", input);
|
||||
|
||||
let (parsed_value, _stats) = result.unwrap();
|
||||
println!("Input: {}, Parsed: {:?}, Expected: {:?}", input, parsed_value, expected);
|
||||
assert_eq!(parsed_value, expected, "Mismatch for input: {}", input);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_markdown_wrapped_json() {
|
||||
let mut parser = create_test_parser();
|
||||
|
||||
// Simple test case
|
||||
let input = r#"```json
|
||||
{"name": "test"}
|
||||
```"#;
|
||||
let expected = json!({"name": "test"});
|
||||
|
||||
let result = parser.parse(input);
|
||||
assert!(result.is_ok(), "Failed to parse markdown wrapped JSON: {}", input);
|
||||
|
||||
let (parsed_value, _) = result.unwrap();
|
||||
println!("Markdown Input: {}", input);
|
||||
println!("Parsed: {:?}", parsed_value);
|
||||
println!("Expected: {:?}", expected);
|
||||
assert_eq!(parsed_value, expected, "Mismatch for markdown input: {}", input);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_input() {
|
||||
let mut parser = create_test_parser();
|
||||
|
||||
let result = parser.parse("");
|
||||
assert!(result.is_ok(), "Should handle empty input");
|
||||
|
||||
let (parsed_value, _) = result.unwrap();
|
||||
assert_eq!(parsed_value, json!(null), "Empty input should parse to null");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cached_parser() {
|
||||
let cached_parser = CachedTolerantJsonParser::new(None, Some(10)).unwrap();
|
||||
|
||||
// Test with simple object first
|
||||
let test_json = r#"{"name": "test"}"#;
|
||||
let expected = json!({"name": "test"});
|
||||
|
||||
println!("Testing cached parser with: {}", test_json);
|
||||
|
||||
// First parse - should cache the result
|
||||
let result1 = cached_parser.parse(test_json);
|
||||
assert!(result1.is_ok(), "First parse failed");
|
||||
let (parsed1, _) = result1.unwrap();
|
||||
println!("First parse result: {:?}", parsed1);
|
||||
assert_eq!(parsed1, expected);
|
||||
|
||||
// Second parse - should use cache
|
||||
let result2 = cached_parser.parse(test_json);
|
||||
assert!(result2.is_ok(), "Second parse failed");
|
||||
let (parsed2, _) = result2.unwrap();
|
||||
println!("Second parse result: {:?}", parsed2);
|
||||
assert_eq!(parsed2, expected);
|
||||
|
||||
// Check cache stats
|
||||
let (cache_size, max_size) = cached_parser.get_cache_stats();
|
||||
assert_eq!(cache_size, 1);
|
||||
assert_eq!(max_size, 10);
|
||||
|
||||
// Clear cache
|
||||
cached_parser.clear_cache();
|
||||
let (cache_size_after_clear, _) = cached_parser.get_cache_stats();
|
||||
assert_eq!(cache_size_after_clear, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_recovery_strategies() {
|
||||
let mut parser = create_test_parser();
|
||||
|
||||
// Test unquoted keys - simplified
|
||||
let unquoted_json = r#"{name: "test"}"#;
|
||||
let result = parser.parse(unquoted_json);
|
||||
println!("Unquoted keys test input: {}", unquoted_json);
|
||||
assert!(result.is_ok(), "Unquoted keys parsing failed");
|
||||
let (parsed, stats) = result.unwrap();
|
||||
println!("Unquoted keys result: {:?}", parsed);
|
||||
println!("Recovery strategies used: {:?}", stats.recovery_strategies_used);
|
||||
assert_eq!(parsed, json!({"name": "test"}));
|
||||
|
||||
// Test trailing commas - simplified
|
||||
let trailing_comma_json = r#"{"name": "test",}"#;
|
||||
let result = parser.parse(trailing_comma_json);
|
||||
println!("Trailing comma test input: {}", trailing_comma_json);
|
||||
assert!(result.is_ok(), "Trailing comma parsing failed");
|
||||
let (parsed, stats) = result.unwrap();
|
||||
println!("Trailing comma result: {:?}", parsed);
|
||||
println!("Recovery strategies used: {:?}", stats.recovery_strategies_used);
|
||||
assert_eq!(parsed, json!({"name": "test"}));
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user