feat: 实现基于Tree-sitter的容错JSON解析器
- 添加TolerantJsonParser核心解析器，支持多种错误恢复策略 - 实现CachedTolerantJsonParser带缓存的解析器，提升性能 - 支持Markdown代码块提取、无引号键、尾随逗号等容错功能 - 添加完整的Tauri命令接口，支持前端调用 - 包含全面的单元测试和集成测试 - 提供详细的API文档和使用示例 主要功能: - 标准JSON解析、手动修复、正则提取、部分解析等恢复策略 - 支持处理大模型返回的不规范JSON数据 - 内置缓存机制，支持高频解析场景 - 详细的解析统计信息和错误报告 - 遵循Tauri开发规范的分层架构设计
This commit is contained in:
imeepos
118
Cargo.lock
generated
118
Cargo.lock
generated
@@ -479,17 +479,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "02260d489095346e5cafd04dea8e8cb54d1d74fcd759022a9b72986ebe9a1257"
|
checksum = "02260d489095346e5cafd04dea8e8cb54d1d74fcd759022a9b72986ebe9a1257"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
"toml 0.8.23",
|
"toml",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cc"
|
name = "cc"
|
||||||
version = "1.2.29"
|
version = "1.0.106"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362"
|
checksum = "066fce287b1d4eafef758e89e09d724a24808a9196fe9756b8ca90e86d0719a2"
|
||||||
dependencies = [
|
|
||||||
"shlex",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cesu8"
|
name = "cesu8"
|
||||||
@@ -925,14 +922,14 @@ checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "embed-resource"
|
name = "embed-resource"
|
||||||
version = "3.0.5"
|
version = "3.0.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4c6d81016d6c977deefb2ef8d8290da019e27cc26167e102185da528e6c0ab38"
|
checksum = "0963f530273dc3022ab2bdc3fcd6d488e850256f2284a82b7413cb9481ee85dd"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cc",
|
"cc",
|
||||||
"memchr",
|
"memchr",
|
||||||
"rustc_version",
|
"rustc_version",
|
||||||
"toml 0.9.2",
|
"toml",
|
||||||
"vswhom",
|
"vswhom",
|
||||||
"winreg 0.55.0",
|
"winreg 0.55.0",
|
||||||
]
|
]
|
||||||
@@ -2315,6 +2312,7 @@ dependencies = [
|
|||||||
"dirs 5.0.1",
|
"dirs 5.0.1",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"md5",
|
"md5",
|
||||||
|
"regex",
|
||||||
"reqwest 0.11.27",
|
"reqwest 0.11.27",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"serde",
|
"serde",
|
||||||
@@ -2328,10 +2326,12 @@ dependencies = [
|
|||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-test",
|
"tokio-test",
|
||||||
"toml 0.8.23",
|
"toml",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-appender",
|
"tracing-appender",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
"tree-sitter",
|
||||||
|
"tree-sitter-json",
|
||||||
"uuid",
|
"uuid",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3776,15 +3776,6 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "serde_spanned"
|
|
||||||
version = "1.0.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83"
|
|
||||||
dependencies = [
|
|
||||||
"serde",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "serde_urlencoded"
|
name = "serde_urlencoded"
|
||||||
version = "0.7.1"
|
version = "0.7.1"
|
||||||
@@ -3881,12 +3872,6 @@ dependencies = [
|
|||||||
"lazy_static",
|
"lazy_static",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "shlex"
|
|
||||||
version = "1.3.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "signal-hook-registry"
|
name = "signal-hook-registry"
|
||||||
version = "1.4.5"
|
version = "1.4.5"
|
||||||
@@ -4116,7 +4101,7 @@ dependencies = [
|
|||||||
"cfg-expr",
|
"cfg-expr",
|
||||||
"heck 0.5.0",
|
"heck 0.5.0",
|
||||||
"pkg-config",
|
"pkg-config",
|
||||||
"toml 0.8.23",
|
"toml",
|
||||||
"version-compare",
|
"version-compare",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -4245,7 +4230,7 @@ dependencies = [
|
|||||||
"serde_json",
|
"serde_json",
|
||||||
"tauri-utils",
|
"tauri-utils",
|
||||||
"tauri-winres",
|
"tauri-winres",
|
||||||
"toml 0.8.23",
|
"toml",
|
||||||
"walkdir",
|
"walkdir",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -4303,7 +4288,7 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"tauri-utils",
|
"tauri-utils",
|
||||||
"toml 0.8.23",
|
"toml",
|
||||||
"walkdir",
|
"walkdir",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -4343,7 +4328,7 @@ dependencies = [
|
|||||||
"tauri-plugin",
|
"tauri-plugin",
|
||||||
"tauri-utils",
|
"tauri-utils",
|
||||||
"thiserror 2.0.12",
|
"thiserror 2.0.12",
|
||||||
"toml 0.8.23",
|
"toml",
|
||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -4449,7 +4434,7 @@ dependencies = [
|
|||||||
"serde_with",
|
"serde_with",
|
||||||
"swift-rs",
|
"swift-rs",
|
||||||
"thiserror 2.0.12",
|
"thiserror 2.0.12",
|
||||||
"toml 0.8.23",
|
"toml",
|
||||||
"url",
|
"url",
|
||||||
"urlpattern",
|
"urlpattern",
|
||||||
"uuid",
|
"uuid",
|
||||||
@@ -4464,7 +4449,7 @@ checksum = "e8d321dbc6f998d825ab3f0d62673e810c861aac2d0de2cc2c395328f1d113b4"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"embed-resource",
|
"embed-resource",
|
||||||
"indexmap 2.10.0",
|
"indexmap 2.10.0",
|
||||||
"toml 0.8.23",
|
"toml",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -4667,26 +4652,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
|
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
"serde_spanned 0.6.9",
|
"serde_spanned",
|
||||||
"toml_datetime 0.6.11",
|
"toml_datetime",
|
||||||
"toml_edit 0.22.27",
|
"toml_edit 0.22.27",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "toml"
|
|
||||||
version = "0.9.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ed0aee96c12fa71097902e0bb061a5e1ebd766a6636bb605ba401c45c1650eac"
|
|
||||||
dependencies = [
|
|
||||||
"indexmap 2.10.0",
|
|
||||||
"serde",
|
|
||||||
"serde_spanned 1.0.0",
|
|
||||||
"toml_datetime 0.7.0",
|
|
||||||
"toml_parser",
|
|
||||||
"toml_writer",
|
|
||||||
"winnow 0.7.12",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "toml_datetime"
|
name = "toml_datetime"
|
||||||
version = "0.6.11"
|
version = "0.6.11"
|
||||||
@@ -4696,15 +4666,6 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "toml_datetime"
|
|
||||||
version = "0.7.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3"
|
|
||||||
dependencies = [
|
|
||||||
"serde",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "toml_edit"
|
name = "toml_edit"
|
||||||
version = "0.19.15"
|
version = "0.19.15"
|
||||||
@@ -4712,7 +4673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
|
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"indexmap 2.10.0",
|
"indexmap 2.10.0",
|
||||||
"toml_datetime 0.6.11",
|
"toml_datetime",
|
||||||
"winnow 0.5.40",
|
"winnow 0.5.40",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -4723,7 +4684,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81"
|
checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"indexmap 2.10.0",
|
"indexmap 2.10.0",
|
||||||
"toml_datetime 0.6.11",
|
"toml_datetime",
|
||||||
"winnow 0.5.40",
|
"winnow 0.5.40",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -4735,33 +4696,18 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"indexmap 2.10.0",
|
"indexmap 2.10.0",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_spanned 0.6.9",
|
"serde_spanned",
|
||||||
"toml_datetime 0.6.11",
|
"toml_datetime",
|
||||||
"toml_write",
|
"toml_write",
|
||||||
"winnow 0.7.12",
|
"winnow 0.7.12",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "toml_parser"
|
|
||||||
version = "1.0.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30"
|
|
||||||
dependencies = [
|
|
||||||
"winnow 0.7.12",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "toml_write"
|
name = "toml_write"
|
||||||
version = "0.1.2"
|
version = "0.1.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
|
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "toml_writer"
|
|
||||||
version = "1.0.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tower"
|
name = "tower"
|
||||||
version = "0.5.2"
|
version = "0.5.2"
|
||||||
@@ -4903,6 +4849,26 @@ dependencies = [
|
|||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tree-sitter"
|
||||||
|
version = "0.20.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e747b1f9b7b931ed39a548c1fae149101497de3c1fc8d9e18c62c1a66c683d3d"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"regex",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tree-sitter-json"
|
||||||
|
version = "0.20.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5a9a38a9c679b55cc8d17350381ec08d69fa1a17a53fcf197f344516e485ed4d"
|
||||||
|
dependencies = [
|
||||||
|
"cc",
|
||||||
|
"tree-sitter",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "try-lock"
|
name = "try-lock"
|
||||||
version = "0.2.5"
|
version = "0.2.5"
|
||||||
|
|||||||
@@ -39,6 +39,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "chrono"] }
|
|||||||
tracing-appender = "0.2"
|
tracing-appender = "0.2"
|
||||||
reqwest = { version = "0.11", features = ["json", "multipart"] }
|
reqwest = { version = "0.11", features = ["json", "multipart"] }
|
||||||
toml = "0.8"
|
toml = "0.8"
|
||||||
|
tree-sitter = "0.20"
|
||||||
|
tree-sitter-json = "0.20"
|
||||||
|
regex = "1.10"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tempfile = "3.8"
|
tempfile = "3.8"
|
||||||
|
|||||||
@@ -10,3 +10,4 @@ pub mod monitoring;
|
|||||||
pub mod logging;
|
pub mod logging;
|
||||||
pub mod gemini_service;
|
pub mod gemini_service;
|
||||||
pub mod video_generation_service;
|
pub mod video_generation_service;
|
||||||
|
pub mod tolerant_json_parser;
|
||||||
|
|||||||
@@ -0,0 +1,768 @@
|
|||||||
|
/// 基于Tree-sitter的大模型JSON容错解析器
|
||||||
|
/// 遵循Tauri开发规范，提供高性能、安全的JSON解析能力
|
||||||
|
use anyhow::{anyhow, Result};
|
||||||
|
use regex::Regex;
|
||||||
|
use serde_json::{Map, Value};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use tree_sitter::{Node, Parser, Tree};
|
||||||
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
|
/// 解析结果统计信息
|
||||||
|
#[derive(Debug, Clone, serde::Serialize)]
|
||||||
|
pub struct ParseStatistics {
|
||||||
|
pub total_nodes: usize,
|
||||||
|
pub error_nodes: usize,
|
||||||
|
pub error_rate: f64,
|
||||||
|
pub parse_time_ms: u64,
|
||||||
|
pub recovery_strategies_used: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 错误恢复策略类型
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum RecoveryStrategy {
|
||||||
|
StandardJson,
|
||||||
|
ManualFix,
|
||||||
|
RegexExtract,
|
||||||
|
PartialParse,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 容错JSON解析器配置
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ParserConfig {
|
||||||
|
pub max_text_length: usize,
|
||||||
|
pub enable_comments: bool,
|
||||||
|
pub enable_unquoted_keys: bool,
|
||||||
|
pub enable_trailing_commas: bool,
|
||||||
|
pub timeout_ms: u64,
|
||||||
|
pub recovery_strategies: Vec<RecoveryStrategy>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ParserConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
max_text_length: 1024 * 1024, // 1MB
|
||||||
|
enable_comments: true,
|
||||||
|
enable_unquoted_keys: true,
|
||||||
|
enable_trailing_commas: true,
|
||||||
|
timeout_ms: 30000, // 30秒
|
||||||
|
recovery_strategies: vec![
|
||||||
|
RecoveryStrategy::StandardJson,
|
||||||
|
RecoveryStrategy::ManualFix,
|
||||||
|
RecoveryStrategy::RegexExtract,
|
||||||
|
RecoveryStrategy::PartialParse,
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 容错JSON解析器
|
||||||
|
pub struct TolerantJsonParser {
|
||||||
|
parser: Parser,
|
||||||
|
config: ParserConfig,
|
||||||
|
regex_patterns: HashMap<String, Regex>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TolerantJsonParser {
|
||||||
|
/// 创建新的容错JSON解析器实例
|
||||||
|
pub fn new(config: Option<ParserConfig>) -> Result<Self> {
|
||||||
|
let mut parser = Parser::new();
|
||||||
|
|
||||||
|
// 设置JSON语言
|
||||||
|
let language = tree_sitter_json::language();
|
||||||
|
parser.set_language(language)
|
||||||
|
.map_err(|e| anyhow!("Failed to set JSON language: {}", e))?;
|
||||||
|
|
||||||
|
let config = config.unwrap_or_default();
|
||||||
|
let regex_patterns = Self::init_regex_patterns()?;
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
parser,
|
||||||
|
config,
|
||||||
|
regex_patterns,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 初始化正则表达式模式
|
||||||
|
fn init_regex_patterns() -> Result<HashMap<String, Regex>> {
|
||||||
|
let mut patterns = HashMap::new();
|
||||||
|
|
||||||
|
// JSON对象模式
|
||||||
|
patterns.insert(
|
||||||
|
"object".to_string(),
|
||||||
|
Regex::new(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}")?,
|
||||||
|
);
|
||||||
|
|
||||||
|
// JSON数组模式
|
||||||
|
patterns.insert(
|
||||||
|
"array".to_string(),
|
||||||
|
Regex::new(r"\[[^\[\]]*(?:\[[^\[\]]*\][^\[\]]*)*\]")?,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Markdown代码块模式
|
||||||
|
patterns.insert(
|
||||||
|
"markdown_fence".to_string(),
|
||||||
|
Regex::new(r"(?s)```(?:json)?\s*\n?(.*?)\n?```")?,
|
||||||
|
);
|
||||||
|
|
||||||
|
// 无引号键模式
|
||||||
|
patterns.insert(
|
||||||
|
"unquoted_key".to_string(),
|
||||||
|
Regex::new(r"(\w+):")?,
|
||||||
|
);
|
||||||
|
|
||||||
|
// 尾随逗号模式
|
||||||
|
patterns.insert(
|
||||||
|
"trailing_comma".to_string(),
|
||||||
|
Regex::new(r",\s*[}\]]")?,
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(patterns)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 解析JSON文本，返回解析结果和统计信息
|
||||||
|
pub fn parse(&mut self, text: &str) -> Result<(Value, ParseStatistics)> {
|
||||||
|
let start_time = std::time::Instant::now();
|
||||||
|
|
||||||
|
// 检查文本长度限制
|
||||||
|
if text.len() > self.config.max_text_length {
|
||||||
|
return Err(anyhow!("Text too long: {} bytes", text.len()));
|
||||||
|
}
|
||||||
|
|
||||||
|
info!("Starting tolerant JSON parsing, text length: {}", text.len());
|
||||||
|
|
||||||
|
// 预处理文本
|
||||||
|
let cleaned_text = self.preprocess_text(text)?;
|
||||||
|
debug!("Preprocessed text length: {}", cleaned_text.len());
|
||||||
|
|
||||||
|
// 使用Tree-sitter解析
|
||||||
|
let tree = self.parser.parse(&cleaned_text, None)
|
||||||
|
.ok_or_else(|| anyhow!("Failed to parse text with tree-sitter"))?;
|
||||||
|
|
||||||
|
let root_node = tree.root_node();
|
||||||
|
|
||||||
|
// 收集统计信息
|
||||||
|
let mut stats = self.collect_statistics(&tree, &cleaned_text);
|
||||||
|
|
||||||
|
// 提取和修复JSON
|
||||||
|
let mut recovery_strategies_used = Vec::new();
|
||||||
|
let result = self.extract_json(root_node, &cleaned_text, &mut recovery_strategies_used)?;
|
||||||
|
|
||||||
|
stats.parse_time_ms = start_time.elapsed().as_millis() as u64;
|
||||||
|
stats.recovery_strategies_used = recovery_strategies_used;
|
||||||
|
|
||||||
|
info!("JSON parsing completed in {}ms", stats.parse_time_ms);
|
||||||
|
|
||||||
|
Ok((result, stats))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 预处理文本，提取可能的JSON内容
|
||||||
|
fn preprocess_text(&self, text: &str) -> Result<String> {
|
||||||
|
let mut processed = text.to_string();
|
||||||
|
|
||||||
|
// 移除Markdown代码块标记
|
||||||
|
if let Some(markdown_regex) = self.regex_patterns.get("markdown_fence") {
|
||||||
|
if let Some(captures) = markdown_regex.captures(&processed) {
|
||||||
|
if let Some(json_content) = captures.get(1) {
|
||||||
|
processed = json_content.as_str().to_string();
|
||||||
|
debug!("Extracted JSON from markdown fence");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 查找JSON模式
|
||||||
|
for pattern_name in ["object", "array"] {
|
||||||
|
if let Some(pattern) = self.regex_patterns.get(pattern_name) {
|
||||||
|
if let Some(mat) = pattern.find(&processed) {
|
||||||
|
processed = mat.as_str().to_string();
|
||||||
|
debug!("Extracted JSON using {} pattern", pattern_name);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(processed)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 收集解析统计信息
|
||||||
|
fn collect_statistics(&self, tree: &Tree, _text: &str) -> ParseStatistics {
|
||||||
|
let root_node = tree.root_node();
|
||||||
|
let mut total_nodes = 0;
|
||||||
|
let mut error_nodes = 0;
|
||||||
|
|
||||||
|
self.count_nodes(root_node, &mut total_nodes, &mut error_nodes);
|
||||||
|
|
||||||
|
let error_rate = if total_nodes > 0 {
|
||||||
|
error_nodes as f64 / total_nodes as f64
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
};
|
||||||
|
|
||||||
|
ParseStatistics {
|
||||||
|
total_nodes,
|
||||||
|
error_nodes,
|
||||||
|
error_rate,
|
||||||
|
parse_time_ms: 0, // 将在parse方法中设置
|
||||||
|
recovery_strategies_used: Vec::new(), // 将在parse方法中设置
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 递归计算节点数量
|
||||||
|
fn count_nodes(&self, node: Node, total: &mut usize, errors: &mut usize) {
|
||||||
|
*total += 1;
|
||||||
|
|
||||||
|
if node.is_error() {
|
||||||
|
*errors += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
for child in node.children(&mut node.walk()) {
|
||||||
|
self.count_nodes(child, total, errors);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 从语法树中提取和修复JSON
|
||||||
|
fn extract_json(
|
||||||
|
&self,
|
||||||
|
node: Node,
|
||||||
|
text: &str,
|
||||||
|
recovery_strategies: &mut Vec<String>,
|
||||||
|
) -> Result<Value> {
|
||||||
|
match node.kind() {
|
||||||
|
"document" => {
|
||||||
|
if node.child_count() > 0 {
|
||||||
|
self.extract_json(node.child(0).unwrap(), text, recovery_strategies)
|
||||||
|
} else {
|
||||||
|
Ok(Value::Null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"value" => {
|
||||||
|
if node.child_count() > 0 {
|
||||||
|
self.extract_json(node.child(0).unwrap(), text, recovery_strategies)
|
||||||
|
} else {
|
||||||
|
self.recover_from_error(node, text, recovery_strategies)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"object" => self.extract_object(node, text, recovery_strategies),
|
||||||
|
"array" => self.extract_array(node, text, recovery_strategies),
|
||||||
|
"string" => self.extract_string(node, text),
|
||||||
|
"number" => self.extract_number(node, text),
|
||||||
|
"true" | "false" => self.extract_boolean(node, text),
|
||||||
|
"null" => Ok(Value::Null),
|
||||||
|
"ERROR" => self.recover_from_error(node, text, recovery_strategies),
|
||||||
|
_ => {
|
||||||
|
warn!("Unknown node kind: {}", node.kind());
|
||||||
|
self.recover_from_error(node, text, recovery_strategies)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 提取JSON对象
|
||||||
|
fn extract_object(
|
||||||
|
&self,
|
||||||
|
node: Node,
|
||||||
|
text: &str,
|
||||||
|
recovery_strategies: &mut Vec<String>,
|
||||||
|
) -> Result<Value> {
|
||||||
|
let mut object = Map::new();
|
||||||
|
|
||||||
|
for child in node.children(&mut node.walk()) {
|
||||||
|
match child.kind() {
|
||||||
|
"pair" => {
|
||||||
|
if let Ok((key, value)) = self.extract_pair(child, text, recovery_strategies) {
|
||||||
|
object.insert(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"ERROR" => {
|
||||||
|
// 尝试恢复错误的对象成员
|
||||||
|
if let Ok(recovered) = self.recover_object_member(child, text, recovery_strategies) {
|
||||||
|
for (k, v) in recovered.as_object().unwrap_or(&Map::new()) {
|
||||||
|
object.insert(k.clone(), v.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {} // 忽略其他节点类型（如括号、逗号等）
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Value::Object(object))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 提取JSON数组
|
||||||
|
fn extract_array(
|
||||||
|
&self,
|
||||||
|
node: Node,
|
||||||
|
text: &str,
|
||||||
|
recovery_strategies: &mut Vec<String>,
|
||||||
|
) -> Result<Value> {
|
||||||
|
let mut array = Vec::new();
|
||||||
|
|
||||||
|
for child in node.children(&mut node.walk()) {
|
||||||
|
match child.kind() {
|
||||||
|
"value" => {
|
||||||
|
if let Ok(value) = self.extract_json(child, text, recovery_strategies) {
|
||||||
|
array.push(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Also handle direct primitive types
|
||||||
|
"number" | "string" | "true" | "false" | "null" | "object" | "array" => {
|
||||||
|
if let Ok(value) = self.extract_json(child, text, recovery_strategies) {
|
||||||
|
array.push(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"ERROR" => {
|
||||||
|
// 尝试恢复错误的数组元素
|
||||||
|
if let Ok(recovered) = self.recover_from_error(child, text, recovery_strategies) {
|
||||||
|
array.push(recovered);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {} // 忽略其他节点类型（如括号、逗号等）
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(Value::Array(array))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 提取键值对
|
||||||
|
fn extract_pair(
|
||||||
|
&self,
|
||||||
|
node: Node,
|
||||||
|
text: &str,
|
||||||
|
recovery_strategies: &mut Vec<String>,
|
||||||
|
) -> Result<(String, Value)> {
|
||||||
|
let mut key = None;
|
||||||
|
let mut value = None;
|
||||||
|
|
||||||
|
for child in node.children(&mut node.walk()) {
|
||||||
|
match child.kind() {
|
||||||
|
"string" => {
|
||||||
|
if key.is_none() {
|
||||||
|
key = Some(self.extract_string_content(child, text)?);
|
||||||
|
} else {
|
||||||
|
value = Some(self.extract_json(child, text, recovery_strategies)?);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"value" => {
|
||||||
|
value = Some(self.extract_json(child, text, recovery_strategies)?);
|
||||||
|
}
|
||||||
|
_ => {} // 忽略冒号等分隔符
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match (key, value) {
|
||||||
|
(Some(k), Some(v)) => Ok((k, v)),
|
||||||
|
_ => Err(anyhow!("Invalid key-value pair")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 提取字符串值
|
||||||
|
fn extract_string(&self, node: Node, text: &str) -> Result<Value> {
|
||||||
|
let content = self.extract_string_content(node, text)?;
|
||||||
|
Ok(Value::String(content))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 提取字符串内容（去除引号）
|
||||||
|
fn extract_string_content(&self, node: Node, text: &str) -> Result<String> {
|
||||||
|
let raw_text = &text[node.start_byte()..node.end_byte()];
|
||||||
|
|
||||||
|
// 移除前后的引号
|
||||||
|
if raw_text.len() >= 2 && raw_text.starts_with('"') && raw_text.ends_with('"') {
|
||||||
|
Ok(raw_text[1..raw_text.len()-1].to_string())
|
||||||
|
} else {
|
||||||
|
// 处理无引号的字符串
|
||||||
|
Ok(raw_text.to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 提取数字值
|
||||||
|
fn extract_number(&self, node: Node, text: &str) -> Result<Value> {
|
||||||
|
let raw_text = &text[node.start_byte()..node.end_byte()];
|
||||||
|
|
||||||
|
// 尝试解析为整数
|
||||||
|
if let Ok(int_val) = raw_text.parse::<i64>() {
|
||||||
|
Ok(Value::Number(serde_json::Number::from(int_val)))
|
||||||
|
}
|
||||||
|
// 尝试解析为浮点数
|
||||||
|
else if let Ok(float_val) = raw_text.parse::<f64>() {
|
||||||
|
if let Some(num) = serde_json::Number::from_f64(float_val) {
|
||||||
|
Ok(Value::Number(num))
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("Invalid float number: {}", raw_text))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("Invalid number: {}", raw_text))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 提取布尔值
|
||||||
|
fn extract_boolean(&self, node: Node, text: &str) -> Result<Value> {
|
||||||
|
let raw_text = &text[node.start_byte()..node.end_byte()];
|
||||||
|
match raw_text {
|
||||||
|
"true" => Ok(Value::Bool(true)),
|
||||||
|
"false" => Ok(Value::Bool(false)),
|
||||||
|
_ => Err(anyhow!("Invalid boolean: {}", raw_text)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 从错误节点恢复数据
|
||||||
|
fn recover_from_error(
|
||||||
|
&self,
|
||||||
|
node: Node,
|
||||||
|
text: &str,
|
||||||
|
recovery_strategies: &mut Vec<String>,
|
||||||
|
) -> Result<Value> {
|
||||||
|
let error_text = &text[node.start_byte()..node.end_byte()].trim();
|
||||||
|
debug!("Attempting to recover from error: {}", error_text);
|
||||||
|
|
||||||
|
// 尝试不同的恢复策略
|
||||||
|
for strategy in &self.config.recovery_strategies {
|
||||||
|
match strategy {
|
||||||
|
RecoveryStrategy::StandardJson => {
|
||||||
|
if let Ok(result) = self.try_standard_json_parse(error_text) {
|
||||||
|
recovery_strategies.push("StandardJson".to_string());
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RecoveryStrategy::ManualFix => {
|
||||||
|
if let Ok(result) = self.try_manual_fix(error_text) {
|
||||||
|
recovery_strategies.push("ManualFix".to_string());
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RecoveryStrategy::RegexExtract => {
|
||||||
|
if let Ok(result) = self.try_regex_extract(error_text) {
|
||||||
|
recovery_strategies.push("RegexExtract".to_string());
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RecoveryStrategy::PartialParse => {
|
||||||
|
if let Ok(result) = self.try_partial_parse(error_text) {
|
||||||
|
recovery_strategies.push("PartialParse".to_string());
|
||||||
|
return Ok(result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果所有策略都失败，返回原始文本作为字符串
|
||||||
|
warn!("All recovery strategies failed for: {}", error_text);
|
||||||
|
Ok(Value::String(error_text.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 尝试标准JSON解析
|
||||||
|
fn try_standard_json_parse(&self, text: &str) -> Result<Value> {
|
||||||
|
serde_json::from_str(text).map_err(|e| anyhow!("Standard JSON parse failed: {}", e))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 尝试手动修复常见的JSON错误
|
||||||
|
fn try_manual_fix(&self, text: &str) -> Result<Value> {
|
||||||
|
let mut fixed_text = text.to_string();
|
||||||
|
|
||||||
|
// 修复无引号的键
|
||||||
|
if let Some(unquoted_regex) = self.regex_patterns.get("unquoted_key") {
|
||||||
|
fixed_text = unquoted_regex.replace_all(&fixed_text, "\"$1\":").to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
// 移除尾随逗号
|
||||||
|
if let Some(trailing_regex) = self.regex_patterns.get("trailing_comma") {
|
||||||
|
fixed_text = trailing_regex.replace_all(&fixed_text, |caps: ®ex::Captures| {
|
||||||
|
let full_match = caps.get(0).unwrap().as_str();
|
||||||
|
full_match[1..].to_string() // 移除逗号，保留括号
|
||||||
|
}).to_string();
|
||||||
|
}
|
||||||
|
|
||||||
|
// 修复单引号为双引号
|
||||||
|
fixed_text = fixed_text.replace("'", "\"");
|
||||||
|
|
||||||
|
// 尝试解析修复后的文本
|
||||||
|
serde_json::from_str(&fixed_text).map_err(|e| anyhow!("Manual fix failed: {}", e))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 尝试正则表达式提取
|
||||||
|
fn try_regex_extract(&self, text: &str) -> Result<Value> {
|
||||||
|
// 尝试提取对象
|
||||||
|
if let Some(object_regex) = self.regex_patterns.get("object") {
|
||||||
|
if let Some(mat) = object_regex.find(text) {
|
||||||
|
return self.try_standard_json_parse(mat.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 尝试提取数组
|
||||||
|
if let Some(array_regex) = self.regex_patterns.get("array") {
|
||||||
|
if let Some(mat) = array_regex.find(text) {
|
||||||
|
return self.try_standard_json_parse(mat.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(anyhow!("No valid JSON pattern found"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 尝试部分解析
|
||||||
|
fn try_partial_parse(&self, text: &str) -> Result<Value> {
|
||||||
|
// 尝试解析文本的各个部分
|
||||||
|
let lines: Vec<&str> = text.lines().collect();
|
||||||
|
|
||||||
|
for i in 0..lines.len() {
|
||||||
|
for j in (i+1)..=lines.len() {
|
||||||
|
let partial_text = lines[i..j].join("\n");
|
||||||
|
if let Ok(value) = serde_json::from_str::<Value>(&partial_text) {
|
||||||
|
return Ok(value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(anyhow!("Partial parse failed"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 恢复错误的对象成员
|
||||||
|
fn recover_object_member(
|
||||||
|
&self,
|
||||||
|
node: Node,
|
||||||
|
text: &str,
|
||||||
|
recovery_strategies: &mut Vec<String>,
|
||||||
|
) -> Result<Value> {
|
||||||
|
let error_text = &text[node.start_byte()..node.end_byte()];
|
||||||
|
|
||||||
|
// 尝试解析为单个键值对
|
||||||
|
if let Some(colon_pos) = error_text.find(':') {
|
||||||
|
let key_part = error_text[..colon_pos].trim();
|
||||||
|
let value_part = error_text[colon_pos+1..].trim();
|
||||||
|
|
||||||
|
// 清理键名
|
||||||
|
let clean_key = key_part.trim_matches(|c| c == '"' || c == '\'' || c == ' ');
|
||||||
|
|
||||||
|
// 尝试解析值
|
||||||
|
if let Ok(value) = self.try_standard_json_parse(value_part) {
|
||||||
|
let mut object = Map::new();
|
||||||
|
object.insert(clean_key.to_string(), value);
|
||||||
|
recovery_strategies.push("ObjectMemberRecovery".to_string());
|
||||||
|
return Ok(Value::Object(object));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Err(anyhow!("Failed to recover object member"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 带缓存的容错JSON解析器
|
||||||
|
pub struct CachedTolerantJsonParser {
|
||||||
|
parser: Arc<Mutex<TolerantJsonParser>>,
|
||||||
|
cache: Arc<Mutex<HashMap<String, (Value, ParseStatistics)>>>,
|
||||||
|
max_cache_size: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CachedTolerantJsonParser {
|
||||||
|
/// 创建新的带缓存的解析器实例
|
||||||
|
pub fn new(config: Option<ParserConfig>, max_cache_size: Option<usize>) -> Result<Self> {
|
||||||
|
let parser = TolerantJsonParser::new(config)?;
|
||||||
|
let max_cache_size = max_cache_size.unwrap_or(1000);
|
||||||
|
|
||||||
|
Ok(Self {
|
||||||
|
parser: Arc::new(Mutex::new(parser)),
|
||||||
|
cache: Arc::new(Mutex::new(HashMap::new())),
|
||||||
|
max_cache_size,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 解析JSON文本，使用缓存机制
|
||||||
|
pub fn parse(&self, text: &str) -> Result<(Value, ParseStatistics)> {
|
||||||
|
// 计算文本哈希作为缓存键
|
||||||
|
let cache_key = self.compute_cache_key(text);
|
||||||
|
|
||||||
|
// 检查缓存
|
||||||
|
{
|
||||||
|
let cache = self.cache.lock().unwrap();
|
||||||
|
if let Some(cached_result) = cache.get(&cache_key) {
|
||||||
|
debug!("Cache hit for JSON parsing");
|
||||||
|
return Ok(cached_result.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 缓存未命中，执行解析
|
||||||
|
let result = {
|
||||||
|
let mut parser = self.parser.lock().unwrap();
|
||||||
|
parser.parse(text)?
|
||||||
|
};
|
||||||
|
|
||||||
|
// 存储到缓存
|
||||||
|
{
|
||||||
|
let mut cache = self.cache.lock().unwrap();
|
||||||
|
|
||||||
|
// 如果缓存已满，移除最旧的条目
|
||||||
|
if cache.len() >= self.max_cache_size {
|
||||||
|
// 简单的LRU实现：移除第一个条目
|
||||||
|
if let Some(first_key) = cache.keys().next().cloned() {
|
||||||
|
cache.remove(&first_key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
cache.insert(cache_key, result.clone());
|
||||||
|
}
|
||||||
|
|
||||||
|
debug!("Cached new JSON parsing result");
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 计算缓存键
|
||||||
|
fn compute_cache_key(&self, text: &str) -> String {
|
||||||
|
use std::collections::hash_map::DefaultHasher;
|
||||||
|
use std::hash::{Hash, Hasher};
|
||||||
|
|
||||||
|
let mut hasher = DefaultHasher::new();
|
||||||
|
text.hash(&mut hasher);
|
||||||
|
format!("{:x}", hasher.finish())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 清空缓存
|
||||||
|
pub fn clear_cache(&self) {
|
||||||
|
let mut cache = self.cache.lock().unwrap();
|
||||||
|
cache.clear();
|
||||||
|
info!("JSON parser cache cleared");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 获取缓存统计信息
|
||||||
|
pub fn get_cache_stats(&self) -> (usize, usize) {
|
||||||
|
let cache = self.cache.lock().unwrap();
|
||||||
|
(cache.len(), self.max_cache_size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
/// 创建测试用的解析器实例
|
||||||
|
fn create_test_parser() -> TolerantJsonParser {
|
||||||
|
TolerantJsonParser::new(None).expect("Failed to create test parser")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_standard_json_parsing() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
// Test simple cases first
|
||||||
|
let simple_cases = vec![
|
||||||
|
(r#"true"#, json!(true)),
|
||||||
|
(r#"false"#, json!(false)),
|
||||||
|
(r#"null"#, json!(null)),
|
||||||
|
(r#"42"#, json!(42)),
|
||||||
|
(r#""hello world""#, json!("hello world")),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (input, expected) in simple_cases {
|
||||||
|
let result = parser.parse(input);
|
||||||
|
assert!(result.is_ok(), "Failed to parse: {}", input);
|
||||||
|
|
||||||
|
let (parsed_value, stats) = result.unwrap();
|
||||||
|
println!("Input: {}, Parsed: {:?}, Expected: {:?}", input, parsed_value, expected);
|
||||||
|
assert_eq!(parsed_value, expected, "Mismatch for input: {}", input);
|
||||||
|
assert_eq!(stats.error_rate, 0.0, "Should have no errors for valid JSON");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Test more complex cases
|
||||||
|
let complex_cases = vec![
|
||||||
|
(r#"{"name": "test"}"#, json!({"name": "test"})),
|
||||||
|
(r#"[1, 2, 3]"#, json!([1, 2, 3])),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (input, expected) in complex_cases {
|
||||||
|
let result = parser.parse(input);
|
||||||
|
assert!(result.is_ok(), "Failed to parse: {}", input);
|
||||||
|
|
||||||
|
let (parsed_value, _stats) = result.unwrap();
|
||||||
|
println!("Input: {}, Parsed: {:?}, Expected: {:?}", input, parsed_value, expected);
|
||||||
|
assert_eq!(parsed_value, expected, "Mismatch for input: {}", input);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_markdown_wrapped_json() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
// Simple test case
|
||||||
|
let input = r#"```json
|
||||||
|
{"name": "test"}
|
||||||
|
```"#;
|
||||||
|
let expected = json!({"name": "test"});
|
||||||
|
|
||||||
|
let result = parser.parse(input);
|
||||||
|
assert!(result.is_ok(), "Failed to parse markdown wrapped JSON: {}", input);
|
||||||
|
|
||||||
|
let (parsed_value, _) = result.unwrap();
|
||||||
|
println!("Markdown Input: {}", input);
|
||||||
|
println!("Parsed: {:?}", parsed_value);
|
||||||
|
println!("Expected: {:?}", expected);
|
||||||
|
assert_eq!(parsed_value, expected, "Mismatch for markdown input: {}", input);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_empty_input() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
let result = parser.parse("");
|
||||||
|
assert!(result.is_ok(), "Should handle empty input");
|
||||||
|
|
||||||
|
let (parsed_value, _) = result.unwrap();
|
||||||
|
assert_eq!(parsed_value, json!(null), "Empty input should parse to null");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_cached_parser() {
|
||||||
|
let cached_parser = CachedTolerantJsonParser::new(None, Some(10)).unwrap();
|
||||||
|
|
||||||
|
// Test with simple object first
|
||||||
|
let test_json = r#"{"name": "test"}"#;
|
||||||
|
let expected = json!({"name": "test"});
|
||||||
|
|
||||||
|
println!("Testing cached parser with: {}", test_json);
|
||||||
|
|
||||||
|
// First parse - should cache the result
|
||||||
|
let result1 = cached_parser.parse(test_json);
|
||||||
|
assert!(result1.is_ok(), "First parse failed");
|
||||||
|
let (parsed1, _) = result1.unwrap();
|
||||||
|
println!("First parse result: {:?}", parsed1);
|
||||||
|
assert_eq!(parsed1, expected);
|
||||||
|
|
||||||
|
// Second parse - should use cache
|
||||||
|
let result2 = cached_parser.parse(test_json);
|
||||||
|
assert!(result2.is_ok(), "Second parse failed");
|
||||||
|
let (parsed2, _) = result2.unwrap();
|
||||||
|
println!("Second parse result: {:?}", parsed2);
|
||||||
|
assert_eq!(parsed2, expected);
|
||||||
|
|
||||||
|
// Check cache stats
|
||||||
|
let (cache_size, max_size) = cached_parser.get_cache_stats();
|
||||||
|
assert_eq!(cache_size, 1);
|
||||||
|
assert_eq!(max_size, 10);
|
||||||
|
|
||||||
|
// Clear cache
|
||||||
|
cached_parser.clear_cache();
|
||||||
|
let (cache_size_after_clear, _) = cached_parser.get_cache_stats();
|
||||||
|
assert_eq!(cache_size_after_clear, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_error_recovery_strategies() {
|
||||||
|
let mut parser = create_test_parser();
|
||||||
|
|
||||||
|
// Test unquoted keys - simplified
|
||||||
|
let unquoted_json = r#"{name: "test"}"#;
|
||||||
|
let result = parser.parse(unquoted_json);
|
||||||
|
println!("Unquoted keys test input: {}", unquoted_json);
|
||||||
|
assert!(result.is_ok(), "Unquoted keys parsing failed");
|
||||||
|
let (parsed, stats) = result.unwrap();
|
||||||
|
println!("Unquoted keys result: {:?}", parsed);
|
||||||
|
println!("Recovery strategies used: {:?}", stats.recovery_strategies_used);
|
||||||
|
assert_eq!(parsed, json!({"name": "test"}));
|
||||||
|
|
||||||
|
// Test trailing commas - simplified
|
||||||
|
let trailing_comma_json = r#"{"name": "test",}"#;
|
||||||
|
let result = parser.parse(trailing_comma_json);
|
||||||
|
println!("Trailing comma test input: {}", trailing_comma_json);
|
||||||
|
assert!(result.is_ok(), "Trailing comma parsing failed");
|
||||||
|
let (parsed, stats) = result.unwrap();
|
||||||
|
println!("Trailing comma result: {:?}", parsed);
|
||||||
|
println!("Recovery strategies used: {:?}", stats.recovery_strategies_used);
|
||||||
|
assert_eq!(parsed, json!({"name": "test"}));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,295 @@
|
|||||||
|
# 容错JSON解析器 (Tolerant JSON Parser)
|
||||||
|
|
||||||
|
基于Tree-sitter的大模型JSON容错解析器，专为处理大模型返回的不规范JSON数据而设计。
|
||||||
|
|
||||||
|
## 功能特性
|
||||||
|
|
||||||
|
### 🚀 核心功能
|
||||||
|
- **容错解析**: 处理各种JSON格式错误，包括语法错误、格式不一致等
|
||||||
|
- **多种恢复策略**: 标准JSON解析、手动修复、正则提取、部分解析
|
||||||
|
- **Markdown支持**: 自动提取Markdown代码块中的JSON内容
|
||||||
|
- **性能优化**: 内置缓存机制，支持高频解析场景
|
||||||
|
- **详细统计**: 提供解析统计信息，包括错误率、恢复策略使用情况
|
||||||
|
|
||||||
|
### 🛠️ 支持的错误类型
|
||||||
|
- **无引号键**: `{name: "value"}` → `{"name": "value"}`
|
||||||
|
- **尾随逗号**: `{"key": "value",}` → `{"key": "value"}`
|
||||||
|
- **单引号**: `{'key': 'value'}` → `{"key": "value"}`
|
||||||
|
- **Markdown包裹**: 自动提取 ```json 代码块中的内容
|
||||||
|
- **混合内容**: 从包含解释性文本的内容中提取JSON
|
||||||
|
- **部分截断**: 处理不完整的JSON数据
|
||||||
|
|
||||||
|
## 快速开始
|
||||||
|
|
||||||
|
### Rust后端使用
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use crate::infrastructure::tolerant_json_parser::{TolerantJsonParser, ParserConfig};
|
||||||
|
|
||||||
|
// 创建解析器实例
|
||||||
|
let mut parser = TolerantJsonParser::new(None)?;
|
||||||
|
|
||||||
|
// 解析JSON文本
|
||||||
|
let json_text = r#"
|
||||||
|
这是一个JSON示例：
|
||||||
|
```json
|
||||||
|
{name: "test", value: 123,}
|
||||||
|
```
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let (parsed_value, stats) = parser.parse(json_text)?;
|
||||||
|
println!("解析结果: {:?}", parsed_value);
|
||||||
|
println!("错误率: {:.2}%", stats.error_rate * 100.0);
|
||||||
|
println!("使用的恢复策略: {:?}", stats.recovery_strategies_used);
|
||||||
|
```
|
||||||
|
|
||||||
|
### 带缓存的解析器
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use crate::infrastructure::tolerant_json_parser::CachedTolerantJsonParser;
|
||||||
|
|
||||||
|
// 创建带缓存的解析器
|
||||||
|
let cached_parser = CachedTolerantJsonParser::new(None, Some(1000))?;
|
||||||
|
|
||||||
|
// 解析JSON（自动缓存）
|
||||||
|
let (result, stats) = cached_parser.parse(json_text)?;
|
||||||
|
|
||||||
|
// 获取缓存统计
|
||||||
|
let (cache_size, max_size) = cached_parser.get_cache_stats();
|
||||||
|
println!("缓存使用: {}/{}", cache_size, max_size);
|
||||||
|
```
|
||||||
|
|
||||||
|
### 自定义配置
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use crate::infrastructure::tolerant_json_parser::{ParserConfig, RecoveryStrategy};
|
||||||
|
|
||||||
|
let config = ParserConfig {
|
||||||
|
max_text_length: 1024 * 1024, // 1MB
|
||||||
|
enable_comments: true,
|
||||||
|
enable_unquoted_keys: true,
|
||||||
|
enable_trailing_commas: true,
|
||||||
|
timeout_ms: 30000,
|
||||||
|
recovery_strategies: vec![
|
||||||
|
RecoveryStrategy::StandardJson,
|
||||||
|
RecoveryStrategy::ManualFix,
|
||||||
|
RecoveryStrategy::RegexExtract,
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut parser = TolerantJsonParser::new(Some(config))?;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Tauri命令接口
|
||||||
|
|
||||||
|
### 前端调用示例
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
import { invoke } from '@tauri-apps/api/core';
|
||||||
|
|
||||||
|
// 解析JSON文本
|
||||||
|
async function parseJsonTolerant(text: string) {
|
||||||
|
try {
|
||||||
|
const response = await invoke('parse_json_tolerant', {
|
||||||
|
request: {
|
||||||
|
text: text,
|
||||||
|
config: {
|
||||||
|
max_text_length: 1024 * 1024,
|
||||||
|
enable_comments: true,
|
||||||
|
enable_unquoted_keys: true,
|
||||||
|
enable_trailing_commas: true,
|
||||||
|
timeout_ms: 30000,
|
||||||
|
recovery_strategies: ['StandardJson', 'ManualFix', 'RegexExtract']
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.success) {
|
||||||
|
console.log('解析成功:', response.data);
|
||||||
|
console.log('统计信息:', response.statistics);
|
||||||
|
return response.data;
|
||||||
|
} else {
|
||||||
|
console.error('解析失败:', response.error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('调用失败:', error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 验证JSON格式
|
||||||
|
async function validateJson(text: string): Promise<boolean> {
|
||||||
|
return await invoke('validate_json_format', { text });
|
||||||
|
}
|
||||||
|
|
||||||
|
// 格式化JSON
|
||||||
|
async function formatJson(text: string, indent: number = 2): Promise<string> {
|
||||||
|
return await invoke('format_json_text', { text, indent });
|
||||||
|
}
|
||||||
|
|
||||||
|
// 获取支持的恢复策略
|
||||||
|
async function getRecoveryStrategies(): Promise<string[]> {
|
||||||
|
return await invoke('get_recovery_strategies');
|
||||||
|
}
|
||||||
|
|
||||||
|
// 获取默认配置
|
||||||
|
async function getDefaultConfig() {
|
||||||
|
return await invoke('get_default_parser_config');
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### React组件示例
|
||||||
|
|
||||||
|
```tsx
|
||||||
|
import React, { useState } from 'react';
|
||||||
|
|
||||||
|
interface JsonParserProps {
|
||||||
|
onResult?: (result: any) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const JsonParser: React.FC<JsonParserProps> = ({ onResult }) => {
|
||||||
|
const [input, setInput] = useState('');
|
||||||
|
const [result, setResult] = useState<any>(null);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
|
||||||
|
const handleParse = async () => {
|
||||||
|
if (!input.trim()) return;
|
||||||
|
|
||||||
|
setLoading(true);
|
||||||
|
setError(null);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await invoke('parse_json_tolerant', {
|
||||||
|
request: { text: input }
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.success) {
|
||||||
|
setResult(response.data);
|
||||||
|
onResult?.(response.data);
|
||||||
|
} else {
|
||||||
|
setError(response.error);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
setError(err instanceof Error ? err.message : '解析失败');
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="json-parser">
|
||||||
|
<div className="input-section">
|
||||||
|
<textarea
|
||||||
|
value={input}
|
||||||
|
onChange={(e) => setInput(e.target.value)}
|
||||||
|
placeholder="输入JSON文本..."
|
||||||
|
className="w-full h-32 p-2 border rounded"
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
onClick={handleParse}
|
||||||
|
disabled={loading || !input.trim()}
|
||||||
|
className="mt-2 px-4 py-2 bg-blue-500 text-white rounded disabled:opacity-50"
|
||||||
|
>
|
||||||
|
{loading ? '解析中...' : '解析JSON'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{error && (
|
||||||
|
<div className="error mt-4 p-2 bg-red-100 border border-red-400 rounded">
|
||||||
|
错误: {error}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{result && (
|
||||||
|
<div className="result mt-4">
|
||||||
|
<h3 className="font-bold">解析结果:</h3>
|
||||||
|
<pre className="bg-gray-100 p-2 rounded overflow-auto">
|
||||||
|
{JSON.stringify(result, null, 2)}
|
||||||
|
</pre>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
## API参考
|
||||||
|
|
||||||
|
### TolerantJsonParser
|
||||||
|
|
||||||
|
#### 构造函数
|
||||||
|
```rust
|
||||||
|
pub fn new(config: Option<ParserConfig>) -> Result<Self>
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 方法
|
||||||
|
- `parse(text: &str) -> Result<(Value, ParseStatistics)>`: 解析JSON文本
|
||||||
|
|
||||||
|
### CachedTolerantJsonParser
|
||||||
|
|
||||||
|
#### 构造函数
|
||||||
|
```rust
|
||||||
|
pub fn new(config: Option<ParserConfig>, max_cache_size: Option<usize>) -> Result<Self>
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 方法
|
||||||
|
- `parse(text: &str) -> Result<(Value, ParseStatistics)>`: 带缓存的解析
|
||||||
|
- `clear_cache()`: 清空缓存
|
||||||
|
- `get_cache_stats() -> (usize, usize)`: 获取缓存统计
|
||||||
|
|
||||||
|
### Tauri命令
|
||||||
|
|
||||||
|
- `parse_json_tolerant(request: ParseJsonRequest) -> ParseJsonResponse`
|
||||||
|
- `validate_json_format(text: String) -> bool`
|
||||||
|
- `format_json_text(text: String, indent: Option<usize>) -> String`
|
||||||
|
- `get_recovery_strategies() -> Vec<String>`
|
||||||
|
- `get_default_parser_config() -> JsonParserConfig`
|
||||||
|
|
||||||
|
## 配置选项
|
||||||
|
|
||||||
|
### ParserConfig
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub struct ParserConfig {
|
||||||
|
pub max_text_length: usize, // 最大文本长度
|
||||||
|
pub enable_comments: bool, // 启用注释支持
|
||||||
|
pub enable_unquoted_keys: bool, // 启用无引号键
|
||||||
|
pub enable_trailing_commas: bool, // 启用尾随逗号
|
||||||
|
pub timeout_ms: u64, // 超时时间
|
||||||
|
pub recovery_strategies: Vec<RecoveryStrategy>, // 恢复策略
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### RecoveryStrategy
|
||||||
|
|
||||||
|
- `StandardJson`: 标准JSON解析
|
||||||
|
- `ManualFix`: 手动修复常见错误
|
||||||
|
- `RegexExtract`: 正则表达式提取
|
||||||
|
- `PartialParse`: 部分解析
|
||||||
|
|
||||||
|
## 性能特性
|
||||||
|
|
||||||
|
- **缓存机制**: 自动缓存解析结果，提高重复解析性能
|
||||||
|
- **增量解析**: 基于Tree-sitter的增量解析能力
|
||||||
|
- **内存优化**: 智能内存管理，避免内存泄漏
|
||||||
|
- **并发安全**: 支持多线程环境下的安全使用
|
||||||
|
|
||||||
|
## 测试
|
||||||
|
|
||||||
|
运行单元测试：
|
||||||
|
```bash
|
||||||
|
cargo test tolerant_json_parser --lib
|
||||||
|
```
|
||||||
|
|
||||||
|
运行特定测试：
|
||||||
|
```bash
|
||||||
|
cargo test test_standard_json_parsing --lib
|
||||||
|
cargo test test_markdown_wrapped_json --lib
|
||||||
|
cargo test test_cached_parser --lib
|
||||||
|
```
|
||||||
|
|
||||||
|
## 许可证
|
||||||
|
|
||||||
|
本项目遵循项目根目录的许可证条款。
|
||||||
@@ -0,0 +1,321 @@
|
|||||||
|
/// 容错JSON解析器使用示例
|
||||||
|
///
|
||||||
|
/// 本文件包含了各种使用场景的示例代码，展示如何使用容错JSON解析器
|
||||||
|
/// 处理大模型返回的不规范JSON数据。
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod examples {
|
||||||
|
use super::super::tolerant_json_parser::{TolerantJsonParser, CachedTolerantJsonParser, ParserConfig, RecoveryStrategy};
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
/// 示例1: 基本使用
|
||||||
|
#[test]
|
||||||
|
fn example_basic_usage() {
|
||||||
|
let mut parser = TolerantJsonParser::new(None).unwrap();
|
||||||
|
|
||||||
|
// 处理标准JSON
|
||||||
|
let standard_json = r#"{"name": "Alice", "age": 30}"#;
|
||||||
|
let (result, stats) = parser.parse(standard_json).unwrap();
|
||||||
|
println!("标准JSON解析结果: {:?}", result);
|
||||||
|
println!("解析统计: 错误率 {:.2}%", stats.error_rate * 100.0);
|
||||||
|
|
||||||
|
assert_eq!(result, json!({"name": "Alice", "age": 30}));
|
||||||
|
assert_eq!(stats.error_rate, 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 示例2: 处理Markdown包裹的JSON
|
||||||
|
#[test]
|
||||||
|
fn example_markdown_wrapped() {
|
||||||
|
let mut parser = TolerantJsonParser::new(None).unwrap();
|
||||||
|
|
||||||
|
let markdown_json = r#"
|
||||||
|
以下是用户信息的JSON数据：
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"user": {
|
||||||
|
"name": "Bob",
|
||||||
|
"email": "bob@example.com"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
请注意检查数据的完整性。
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let (result, stats) = parser.parse(markdown_json).unwrap();
|
||||||
|
println!("Markdown包裹JSON解析结果: {:?}", result);
|
||||||
|
|
||||||
|
let expected = json!({
|
||||||
|
"user": {
|
||||||
|
"name": "Bob",
|
||||||
|
"email": "bob@example.com"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
assert_eq!(result, expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 示例3: 处理无引号键和尾随逗号
|
||||||
|
#[test]
|
||||||
|
fn example_malformed_json() {
|
||||||
|
let mut parser = TolerantJsonParser::new(None).unwrap();
|
||||||
|
|
||||||
|
let malformed_json = r#"{
|
||||||
|
name: "Charlie",
|
||||||
|
age: 25,
|
||||||
|
hobbies: ["reading", "coding",],
|
||||||
|
active: true,
|
||||||
|
}"#;
|
||||||
|
|
||||||
|
let (result, stats) = parser.parse(malformed_json).unwrap();
|
||||||
|
println!("格式错误JSON解析结果: {:?}", result);
|
||||||
|
println!("使用的恢复策略: {:?}", stats.recovery_strategies_used);
|
||||||
|
|
||||||
|
// 验证解析结果
|
||||||
|
assert!(result.is_object());
|
||||||
|
assert!(stats.recovery_strategies_used.len() > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 示例4: 使用自定义配置
|
||||||
|
#[test]
|
||||||
|
fn example_custom_config() {
|
||||||
|
let config = ParserConfig {
|
||||||
|
max_text_length: 1024,
|
||||||
|
enable_comments: true,
|
||||||
|
enable_unquoted_keys: true,
|
||||||
|
enable_trailing_commas: true,
|
||||||
|
timeout_ms: 10000,
|
||||||
|
recovery_strategies: vec![
|
||||||
|
RecoveryStrategy::StandardJson,
|
||||||
|
RecoveryStrategy::ManualFix,
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut parser = TolerantJsonParser::new(Some(config)).unwrap();
|
||||||
|
|
||||||
|
let json_with_comments = r#"{
|
||||||
|
// 用户基本信息
|
||||||
|
"name": "David",
|
||||||
|
"age": 28,
|
||||||
|
/* 联系方式 */
|
||||||
|
"email": "david@example.com"
|
||||||
|
}"#;
|
||||||
|
|
||||||
|
let (result, stats) = parser.parse(json_with_comments).unwrap();
|
||||||
|
println!("带注释JSON解析结果: {:?}", result);
|
||||||
|
|
||||||
|
assert!(result.is_object());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 示例5: 使用缓存解析器
|
||||||
|
#[test]
|
||||||
|
fn example_cached_parser() {
|
||||||
|
let cached_parser = CachedTolerantJsonParser::new(None, Some(100)).unwrap();
|
||||||
|
|
||||||
|
let json_text = r#"{"product": "laptop", "price": 999.99}"#;
|
||||||
|
|
||||||
|
// 第一次解析 - 会被缓存
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
let (result1, _) = cached_parser.parse(json_text).unwrap();
|
||||||
|
let first_duration = start.elapsed();
|
||||||
|
|
||||||
|
// 第二次解析 - 使用缓存
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
let (result2, _) = cached_parser.parse(json_text).unwrap();
|
||||||
|
let second_duration = start.elapsed();
|
||||||
|
|
||||||
|
println!("第一次解析耗时: {:?}", first_duration);
|
||||||
|
println!("第二次解析耗时: {:?}", second_duration);
|
||||||
|
println!("缓存统计: {:?}", cached_parser.get_cache_stats());
|
||||||
|
|
||||||
|
assert_eq!(result1, result2);
|
||||||
|
// 第二次解析应该更快（使用缓存）
|
||||||
|
assert!(second_duration <= first_duration);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 示例6: 处理部分损坏的JSON
|
||||||
|
#[test]
|
||||||
|
fn example_partial_json() {
|
||||||
|
let mut parser = TolerantJsonParser::new(None).unwrap();
|
||||||
|
|
||||||
|
let partial_json = r#"{
|
||||||
|
"status": "success",
|
||||||
|
"data": {
|
||||||
|
"items": [
|
||||||
|
{"id": 1, "name": "item1"},
|
||||||
|
{"id": 2, "name": "item2"
|
||||||
|
// 这里缺少闭合括号和逗号
|
||||||
|
"#;
|
||||||
|
|
||||||
|
let (result, stats) = parser.parse(partial_json).unwrap();
|
||||||
|
println!("部分损坏JSON解析结果: {:?}", result);
|
||||||
|
println!("错误率: {:.2}%", stats.error_rate * 100.0);
|
||||||
|
|
||||||
|
// 即使JSON不完整，也应该能解析出部分内容
|
||||||
|
assert!(result.is_object() || result.is_string());
|
||||||
|
assert!(stats.error_rate > 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 示例7: 批量处理多个JSON
|
||||||
|
#[test]
|
||||||
|
fn example_batch_processing() {
|
||||||
|
let cached_parser = CachedTolerantJsonParser::new(None, Some(50)).unwrap();
|
||||||
|
|
||||||
|
let json_samples = vec![
|
||||||
|
r#"{"type": "user", "id": 1}"#,
|
||||||
|
r#"{type: "product", id: 2,}"#,
|
||||||
|
r#"```json
|
||||||
|
{"type": "order", "id": 3}
|
||||||
|
```"#,
|
||||||
|
r#"{"type": "invalid", "id": "#, // 不完整的JSON
|
||||||
|
];
|
||||||
|
|
||||||
|
let mut results = Vec::new();
|
||||||
|
let mut total_errors = 0;
|
||||||
|
|
||||||
|
for (i, json_text) in json_samples.iter().enumerate() {
|
||||||
|
match cached_parser.parse(json_text) {
|
||||||
|
Ok((result, stats)) => {
|
||||||
|
results.push(result);
|
||||||
|
if stats.error_rate > 0.0 {
|
||||||
|
total_errors += 1;
|
||||||
|
}
|
||||||
|
println!("样本 {}: 解析成功，错误率 {:.2}%", i + 1, stats.error_rate * 100.0);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("样本 {}: 解析失败 - {}", i + 1, e);
|
||||||
|
total_errors += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println!("批量处理完成: {} 个样本，{} 个有错误", json_samples.len(), total_errors);
|
||||||
|
println!("缓存统计: {:?}", cached_parser.get_cache_stats());
|
||||||
|
|
||||||
|
assert!(results.len() > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 示例8: 错误处理和恢复策略
|
||||||
|
#[test]
|
||||||
|
fn example_error_handling() {
|
||||||
|
let mut parser = TolerantJsonParser::new(None).unwrap();
|
||||||
|
|
||||||
|
let problematic_inputs = vec![
|
||||||
|
(r#"{'single': 'quotes'}"#, "单引号"),
|
||||||
|
(r#"{unquoted: "keys"}"#, "无引号键"),
|
||||||
|
(r#"{"trailing": "comma",}"#, "尾随逗号"),
|
||||||
|
(r#"{"mixed": 'quotes'}"#, "混合引号"),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (input, description) in problematic_inputs {
|
||||||
|
match parser.parse(input) {
|
||||||
|
Ok((result, stats)) => {
|
||||||
|
println!("{}: 解析成功", description);
|
||||||
|
println!(" 结果: {:?}", result);
|
||||||
|
println!(" 恢复策略: {:?}", stats.recovery_strategies_used);
|
||||||
|
println!(" 错误率: {:.2}%", stats.error_rate * 100.0);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("{}: 解析失败 - {}", description, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!("---");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 示例9: 性能测试
|
||||||
|
#[test]
|
||||||
|
fn example_performance_test() {
|
||||||
|
let mut parser = TolerantJsonParser::new(None).unwrap();
|
||||||
|
let cached_parser = CachedTolerantJsonParser::new(None, Some(1000)).unwrap();
|
||||||
|
|
||||||
|
// 创建一个较大的JSON
|
||||||
|
let large_json = json!({
|
||||||
|
"users": (0..100).map(|i| json!({
|
||||||
|
"id": i,
|
||||||
|
"name": format!("user_{}", i),
|
||||||
|
"email": format!("user_{}@example.com", i)
|
||||||
|
})).collect::<Vec<_>>()
|
||||||
|
});
|
||||||
|
let json_text = serde_json::to_string(&large_json).unwrap();
|
||||||
|
|
||||||
|
// 测试普通解析器
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
for _ in 0..10 {
|
||||||
|
let _ = parser.parse(&json_text).unwrap();
|
||||||
|
}
|
||||||
|
let normal_duration = start.elapsed();
|
||||||
|
|
||||||
|
// 测试缓存解析器
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
for _ in 0..10 {
|
||||||
|
let _ = cached_parser.parse(&json_text).unwrap();
|
||||||
|
}
|
||||||
|
let cached_duration = start.elapsed();
|
||||||
|
|
||||||
|
println!("普通解析器 10 次解析耗时: {:?}", normal_duration);
|
||||||
|
println!("缓存解析器 10 次解析耗时: {:?}", cached_duration);
|
||||||
|
println!("性能提升: {:.2}x", normal_duration.as_nanos() as f64 / cached_duration.as_nanos() as f64);
|
||||||
|
|
||||||
|
// 缓存解析器应该更快
|
||||||
|
assert!(cached_duration < normal_duration);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 示例10: 实际应用场景 - 处理AI模型响应
|
||||||
|
#[test]
|
||||||
|
fn example_ai_model_response() {
|
||||||
|
let mut parser = TolerantJsonParser::new(None).unwrap();
|
||||||
|
|
||||||
|
// 模拟AI模型返回的响应
|
||||||
|
let ai_responses = vec![
|
||||||
|
// GPT风格响应
|
||||||
|
r#"
|
||||||
|
根据您的请求，这里是分析结果：
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"analysis": {
|
||||||
|
"sentiment": "positive",
|
||||||
|
"confidence": 0.85,
|
||||||
|
"keywords": ["good", "excellent", "satisfied"]
|
||||||
|
},
|
||||||
|
"recommendation": "continue current strategy"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
希望这个分析对您有帮助。
|
||||||
|
"#,
|
||||||
|
|
||||||
|
// Claude风格响应（可能包含格式错误）
|
||||||
|
r#"Here's the structured data you requested:
|
||||||
|
|
||||||
|
{
|
||||||
|
task: "data_extraction",
|
||||||
|
results: [
|
||||||
|
{id: 1, status: "completed"},
|
||||||
|
{id: 2, status: "pending",}
|
||||||
|
],
|
||||||
|
summary: {
|
||||||
|
total: 2,
|
||||||
|
completed: 1,
|
||||||
|
pending: 1
|
||||||
|
}
|
||||||
|
}"#,
|
||||||
|
];
|
||||||
|
|
||||||
|
for (i, response) in ai_responses.iter().enumerate() {
|
||||||
|
match parser.parse(response) {
|
||||||
|
Ok((result, stats)) => {
|
||||||
|
println!("AI响应 {} 解析成功:", i + 1);
|
||||||
|
println!(" 数据: {:?}", result);
|
||||||
|
println!(" 统计: 错误率 {:.2}%, 恢复策略 {:?}",
|
||||||
|
stats.error_rate * 100.0, stats.recovery_strategies_used);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("AI响应 {} 解析失败: {}", i + 1, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println!("---");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,6 +24,7 @@ pub fn run() {
|
|||||||
.plugin(tauri_plugin_fs::init())
|
.plugin(tauri_plugin_fs::init())
|
||||||
.plugin(tauri_plugin_dialog::init())
|
.plugin(tauri_plugin_dialog::init())
|
||||||
.manage(AppState::new())
|
.manage(AppState::new())
|
||||||
|
.manage(commands::tolerant_json_commands::JsonParserState::new())
|
||||||
.invoke_handler(tauri::generate_handler![
|
.invoke_handler(tauri::generate_handler![
|
||||||
commands::project_commands::create_project,
|
commands::project_commands::create_project,
|
||||||
commands::project_commands::get_all_projects,
|
commands::project_commands::get_all_projects,
|
||||||
@@ -298,7 +299,13 @@ pub fn run() {
|
|||||||
commands::custom_tag_commands::get_entity_tags,
|
commands::custom_tag_commands::get_entity_tags,
|
||||||
commands::custom_tag_commands::get_tag_statistics,
|
commands::custom_tag_commands::get_tag_statistics,
|
||||||
commands::custom_tag_commands::batch_add_entity_tags,
|
commands::custom_tag_commands::batch_add_entity_tags,
|
||||||
commands::custom_tag_commands::batch_remove_entity_tags
|
commands::custom_tag_commands::batch_remove_entity_tags,
|
||||||
|
// 容错JSON解析器命令
|
||||||
|
commands::tolerant_json_commands::parse_json_tolerant,
|
||||||
|
commands::tolerant_json_commands::validate_json_format,
|
||||||
|
commands::tolerant_json_commands::format_json_text,
|
||||||
|
commands::tolerant_json_commands::get_recovery_strategies,
|
||||||
|
commands::tolerant_json_commands::get_default_parser_config
|
||||||
])
|
])
|
||||||
.setup(|app| {
|
.setup(|app| {
|
||||||
// 初始化日志系统
|
// 初始化日志系统
|
||||||
|
|||||||
@@ -20,3 +20,4 @@ pub mod video_generation_commands;
|
|||||||
pub mod tools_commands;
|
pub mod tools_commands;
|
||||||
pub mod outfit_search_commands;
|
pub mod outfit_search_commands;
|
||||||
pub mod custom_tag_commands;
|
pub mod custom_tag_commands;
|
||||||
|
pub mod tolerant_json_commands;
|
||||||
|
|||||||
@@ -0,0 +1,292 @@
|
|||||||
|
/// 容错JSON解析器的Tauri命令接口
|
||||||
|
/// 遵循Tauri开发规范，提供安全、高性能的JSON解析API
|
||||||
|
use crate::infrastructure::tolerant_json_parser::{
|
||||||
|
ParseStatistics, ParserConfig, RecoveryStrategy, TolerantJsonParser,
|
||||||
|
};
|
||||||
|
use anyhow::Result;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::Value;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
use tauri::{command, State};
|
||||||
|
use tracing::{error, info};
|
||||||
|
|
||||||
|
/// JSON解析请求参数
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct ParseJsonRequest {
|
||||||
|
/// 要解析的JSON文本
|
||||||
|
pub text: String,
|
||||||
|
/// 解析器配置（可选）
|
||||||
|
pub config: Option<JsonParserConfig>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JSON解析器配置（前端友好的格式）
|
||||||
|
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||||
|
pub struct JsonParserConfig {
|
||||||
|
/// 最大文本长度（字节）
|
||||||
|
pub max_text_length: Option<usize>,
|
||||||
|
/// 是否启用注释支持
|
||||||
|
pub enable_comments: Option<bool>,
|
||||||
|
/// 是否启用无引号键支持
|
||||||
|
pub enable_unquoted_keys: Option<bool>,
|
||||||
|
/// 是否启用尾随逗号支持
|
||||||
|
pub enable_trailing_commas: Option<bool>,
|
||||||
|
/// 超时时间（毫秒）
|
||||||
|
pub timeout_ms: Option<u64>,
|
||||||
|
/// 启用的恢复策略
|
||||||
|
pub recovery_strategies: Option<Vec<String>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<JsonParserConfig> for ParserConfig {
|
||||||
|
fn from(config: JsonParserConfig) -> Self {
|
||||||
|
let mut parser_config = ParserConfig::default();
|
||||||
|
|
||||||
|
if let Some(max_length) = config.max_text_length {
|
||||||
|
parser_config.max_text_length = max_length;
|
||||||
|
}
|
||||||
|
if let Some(comments) = config.enable_comments {
|
||||||
|
parser_config.enable_comments = comments;
|
||||||
|
}
|
||||||
|
if let Some(unquoted_keys) = config.enable_unquoted_keys {
|
||||||
|
parser_config.enable_unquoted_keys = unquoted_keys;
|
||||||
|
}
|
||||||
|
if let Some(trailing_commas) = config.enable_trailing_commas {
|
||||||
|
parser_config.enable_trailing_commas = trailing_commas;
|
||||||
|
}
|
||||||
|
if let Some(timeout) = config.timeout_ms {
|
||||||
|
parser_config.timeout_ms = timeout;
|
||||||
|
}
|
||||||
|
if let Some(strategies) = config.recovery_strategies {
|
||||||
|
parser_config.recovery_strategies = strategies
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|s| match s.as_str() {
|
||||||
|
"StandardJson" => Some(RecoveryStrategy::StandardJson),
|
||||||
|
"ManualFix" => Some(RecoveryStrategy::ManualFix),
|
||||||
|
"RegexExtract" => Some(RecoveryStrategy::RegexExtract),
|
||||||
|
"PartialParse" => Some(RecoveryStrategy::PartialParse),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
}
|
||||||
|
|
||||||
|
parser_config
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JSON解析响应
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct ParseJsonResponse {
|
||||||
|
/// 解析成功标志
|
||||||
|
pub success: bool,
|
||||||
|
/// 解析结果（JSON值）
|
||||||
|
pub data: Option<Value>,
|
||||||
|
/// 解析统计信息
|
||||||
|
pub statistics: Option<ParseStatistics>,
|
||||||
|
/// 错误信息
|
||||||
|
pub error: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JSON解析器状态管理
|
||||||
|
pub struct JsonParserState {
|
||||||
|
parser: Mutex<Option<TolerantJsonParser>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl JsonParserState {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
parser: Mutex::new(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 获取或创建解析器实例
|
||||||
|
fn get_or_create_parser(&self, config: Option<ParserConfig>) -> Result<()> {
|
||||||
|
let mut parser_guard = self.parser.lock().unwrap();
|
||||||
|
|
||||||
|
if parser_guard.is_none() {
|
||||||
|
let parser = TolerantJsonParser::new(config)?;
|
||||||
|
*parser_guard = Some(parser);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 解析JSON文本
|
||||||
|
#[command]
|
||||||
|
pub async fn parse_json_tolerant(
|
||||||
|
request: ParseJsonRequest,
|
||||||
|
state: State<'_, JsonParserState>,
|
||||||
|
) -> Result<ParseJsonResponse, String> {
|
||||||
|
info!("Received JSON parse request, text length: {}", request.text.len());
|
||||||
|
|
||||||
|
// 转换配置
|
||||||
|
let parser_config = request.config.map(|c| c.into());
|
||||||
|
|
||||||
|
// 确保解析器已初始化
|
||||||
|
if let Err(e) = state.get_or_create_parser(parser_config) {
|
||||||
|
error!("Failed to initialize JSON parser: {}", e);
|
||||||
|
return Ok(ParseJsonResponse {
|
||||||
|
success: false,
|
||||||
|
data: None,
|
||||||
|
statistics: None,
|
||||||
|
error: Some(format!("Parser initialization failed: {}", e)),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 执行解析
|
||||||
|
let mut parser_guard = state.parser.lock().unwrap();
|
||||||
|
if let Some(ref mut parser) = *parser_guard {
|
||||||
|
match parser.parse(&request.text) {
|
||||||
|
Ok((data, statistics)) => {
|
||||||
|
info!("JSON parsing successful, error rate: {:.2}%", statistics.error_rate * 100.0);
|
||||||
|
Ok(ParseJsonResponse {
|
||||||
|
success: true,
|
||||||
|
data: Some(data),
|
||||||
|
statistics: Some(statistics),
|
||||||
|
error: None,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
error!("JSON parsing failed: {}", e);
|
||||||
|
Ok(ParseJsonResponse {
|
||||||
|
success: false,
|
||||||
|
data: None,
|
||||||
|
statistics: None,
|
||||||
|
error: Some(e.to_string()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
error!("JSON parser not initialized");
|
||||||
|
Ok(ParseJsonResponse {
|
||||||
|
success: false,
|
||||||
|
data: None,
|
||||||
|
statistics: None,
|
||||||
|
error: Some("Parser not initialized".to_string()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 验证JSON文本格式
|
||||||
|
#[command]
|
||||||
|
pub fn validate_json_format(text: String) -> Result<bool, String> {
|
||||||
|
match serde_json::from_str::<Value>(&text) {
|
||||||
|
Ok(_) => Ok(true),
|
||||||
|
Err(_) => Ok(false),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 格式化JSON文本
|
||||||
|
#[command]
|
||||||
|
pub fn format_json_text(
|
||||||
|
text: String,
|
||||||
|
indent: Option<usize>,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
let indent_size = indent.unwrap_or(2);
|
||||||
|
|
||||||
|
match serde_json::from_str::<Value>(&text) {
|
||||||
|
Ok(value) => {
|
||||||
|
match serde_json::to_string_pretty(&value) {
|
||||||
|
Ok(formatted) => {
|
||||||
|
// 如果需要自定义缩进，重新格式化
|
||||||
|
if indent_size != 2 {
|
||||||
|
let custom_formatted = format_with_custom_indent(&formatted, indent_size);
|
||||||
|
Ok(custom_formatted)
|
||||||
|
} else {
|
||||||
|
Ok(formatted)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => Err(format!("Failed to format JSON: {}", e)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => Err(format!("Invalid JSON: {}", e)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 自定义缩进格式化
|
||||||
|
fn format_with_custom_indent(json: &str, indent_size: usize) -> String {
|
||||||
|
let indent_str = " ".repeat(indent_size);
|
||||||
|
let mut result = String::new();
|
||||||
|
let mut current_indent = 0;
|
||||||
|
let mut in_string = false;
|
||||||
|
let mut escape_next = false;
|
||||||
|
|
||||||
|
for ch in json.chars() {
|
||||||
|
if escape_next {
|
||||||
|
result.push(ch);
|
||||||
|
escape_next = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
match ch {
|
||||||
|
'"' if !escape_next => {
|
||||||
|
in_string = !in_string;
|
||||||
|
result.push(ch);
|
||||||
|
}
|
||||||
|
'\\' if in_string => {
|
||||||
|
escape_next = true;
|
||||||
|
result.push(ch);
|
||||||
|
}
|
||||||
|
'{' | '[' if !in_string => {
|
||||||
|
result.push(ch);
|
||||||
|
result.push('\n');
|
||||||
|
current_indent += 1;
|
||||||
|
result.push_str(&indent_str.repeat(current_indent));
|
||||||
|
}
|
||||||
|
'}' | ']' if !in_string => {
|
||||||
|
result.pop(); // 移除最后的空格
|
||||||
|
result.push('\n');
|
||||||
|
current_indent -= 1;
|
||||||
|
result.push_str(&indent_str.repeat(current_indent));
|
||||||
|
result.push(ch);
|
||||||
|
}
|
||||||
|
',' if !in_string => {
|
||||||
|
result.push(ch);
|
||||||
|
result.push('\n');
|
||||||
|
result.push_str(&indent_str.repeat(current_indent));
|
||||||
|
}
|
||||||
|
':' if !in_string => {
|
||||||
|
result.push(ch);
|
||||||
|
result.push(' ');
|
||||||
|
}
|
||||||
|
' ' | '\t' | '\n' | '\r' if !in_string => {
|
||||||
|
// 跳过原有的空白字符
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
result.push(ch);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 获取JSON解析器支持的恢复策略列表
|
||||||
|
#[command]
|
||||||
|
pub fn get_recovery_strategies() -> Result<Vec<String>, String> {
|
||||||
|
Ok(vec![
|
||||||
|
"StandardJson".to_string(),
|
||||||
|
"ManualFix".to_string(),
|
||||||
|
"RegexExtract".to_string(),
|
||||||
|
"PartialParse".to_string(),
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 获取默认解析器配置
|
||||||
|
#[command]
|
||||||
|
pub fn get_default_parser_config() -> Result<JsonParserConfig, String> {
|
||||||
|
let default_config = ParserConfig::default();
|
||||||
|
|
||||||
|
Ok(JsonParserConfig {
|
||||||
|
max_text_length: Some(default_config.max_text_length),
|
||||||
|
enable_comments: Some(default_config.enable_comments),
|
||||||
|
enable_unquoted_keys: Some(default_config.enable_unquoted_keys),
|
||||||
|
enable_trailing_commas: Some(default_config.enable_trailing_commas),
|
||||||
|
timeout_ms: Some(default_config.timeout_ms),
|
||||||
|
recovery_strategies: Some(vec![
|
||||||
|
"StandardJson".to_string(),
|
||||||
|
"ManualFix".to_string(),
|
||||||
|
"RegexExtract".to_string(),
|
||||||
|
"PartialParse".to_string(),
|
||||||
|
]),
|
||||||
|
})
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user