fix: 优化markdown解析器

apps/desktop/src-tauri/src/infrastructure/markdown_parser.rs

@@ -229,31 +229,17 @@ impl MarkdownParser {
             return Err(anyhow!("Text too large: {} bytes", text.len()));
         }
 
-        // 使用pulldown-cmark解析
-        let parser = CmarkParser::new(text);
+        // 使用pulldown-cmark解析，带有偏移量信息
+        let parser = CmarkParser::new_with_broken_link_callback(
+            text,
+            pulldown_cmark::Options::all(),
+            None
+        );
         let mut events = Vec::new();
-        let mut current_offset = 0;
 
-        // 收集所有事件和位置信息
-        for event in parser {
-            events.push((event.clone(), current_offset));
-
-            // 根据事件类型正确计算字节偏移
-            match &event {
-                Event::Text(text) => {
-                    current_offset += text.len();
-                }
-                Event::Code(code) => {
-                    current_offset += code.len();
-                }
-                Event::SoftBreak | Event::HardBreak => {
-                    current_offset += 1; // 换行符通常是1字节
-                }
-                _ => {
-                    // 对于其他事件类型，不增加偏移量
-                    // 因为它们通常是结构性的，不对应实际的文本内容
-                }
-            }
+        // 收集所有事件，pulldown-cmark会提供正确的偏移量信息
+        for (event, range) in parser.into_offset_iter() {
+            events.push((event, range.start));
         }
 
         // 构建AST
@@ -285,18 +271,16 @@ impl MarkdownParser {
             attributes: HashMap::new(),
         };
 
-        let mut current_offset = 0;
-
-        for (event, _) in events {
+        for (event, byte_offset) in events {
             match event {
                 Event::Start(tag) => {
-                    let node = self.create_node_from_tag(tag, current_offset, source_text)?;
+                    let node = self.create_node_from_tag(tag, *byte_offset, source_text)?;
                     stack.push(node);
                 }
                 Event::End(_) => {
                     if let Some(mut node) = stack.pop() {
-                        // 更新结束位置
-                        node.range.end = self.calculate_position(source_text, current_offset);
+                        // 更新结束位置 - 使用当前字节偏移
+                        node.range.end = self.calculate_position_from_byte_offset(source_text, *byte_offset);
 
                         if let Some(parent) = stack.last_mut() {
                             parent.children.push(node);
@@ -310,8 +294,8 @@ impl MarkdownParser {
                         node_type: MarkdownNodeType::Text,
                         content: text.to_string(),
                         range: Range {
-                            start: self.calculate_position(source_text, current_offset),
-                            end: self.calculate_position(source_text, current_offset + text.len()),
+                            start: self.calculate_position_from_byte_offset(source_text, *byte_offset),
+                            end: self.calculate_position_from_byte_offset(source_text, *byte_offset + text.as_bytes().len()),
                         },
                         children: Vec::new(),
                         attributes: HashMap::new(),
@@ -328,8 +312,8 @@ impl MarkdownParser {
                         node_type: MarkdownNodeType::InlineCode,
                         content: code.to_string(),
                         range: Range {
-                            start: self.calculate_position(source_text, current_offset),
-                            end: self.calculate_position(source_text, current_offset + code.len()),
+                            start: self.calculate_position_from_byte_offset(source_text, *byte_offset),
+                            end: self.calculate_position_from_byte_offset(source_text, *byte_offset + code.as_bytes().len()),
                         },
                         children: Vec::new(),
                         attributes: HashMap::new(),
@@ -346,8 +330,8 @@ impl MarkdownParser {
                         node_type: MarkdownNodeType::LineBreak,
                         content: "\n".to_string(),
                         range: Range {
-                            start: self.calculate_position(source_text, current_offset),
-                            end: self.calculate_position(source_text, current_offset + 1),
+                            start: self.calculate_position_from_byte_offset(source_text, *byte_offset),
+                            end: self.calculate_position_from_byte_offset(source_text, *byte_offset + 1),
                         },
                         children: Vec::new(),
                         attributes: HashMap::new(),
@@ -370,7 +354,7 @@ impl MarkdownParser {
 
     /// 从pulldown-cmark标签创建节点
     fn create_node_from_tag(&self, tag: &Tag, offset: usize, source_text: &str) -> Result<MarkdownNode> {
-        let start_pos = self.calculate_position(source_text, offset);
+        let start_pos = self.calculate_position_from_byte_offset(source_text, offset);
 
         let (node_type, attributes) = match tag {
             Tag::Heading(level, _, _) => {
@@ -895,6 +879,188 @@ mod tests {
         assert!(parse_result.statistics.total_nodes > 1000);
     }
 
+    #[test]
+    fn test_byte_offset_calculation_ascii() {
+        let parser = create_test_parser();
+        let text = "Hello\nWorld";
+
+        // Test position at start
+        let pos = parser.calculate_position_from_byte_offset(text, 0);
+        assert_eq!(pos.line, 0);
+        assert_eq!(pos.column, 0);
+        assert_eq!(pos.offset, 0);
+        assert_eq!(pos.byte_offset, 0);
+
+        // Test position after "Hello"
+        let pos = parser.calculate_position_from_byte_offset(text, 5);
+        assert_eq!(pos.line, 0);
+        assert_eq!(pos.column, 5);
+        assert_eq!(pos.offset, 5);
+        assert_eq!(pos.byte_offset, 5);
+
+        // Test position after newline
+        let pos = parser.calculate_position_from_byte_offset(text, 6);
+        assert_eq!(pos.line, 1);
+        assert_eq!(pos.column, 0);
+        assert_eq!(pos.offset, 6);
+        assert_eq!(pos.byte_offset, 6);
+
+        // Test position at end
+        let pos = parser.calculate_position_from_byte_offset(text, text.len());
+        assert_eq!(pos.line, 1);
+        assert_eq!(pos.column, 5);
+        assert_eq!(pos.offset, 11);
+        assert_eq!(pos.byte_offset, 11);
+    }
+
+    #[test]
+    fn test_byte_offset_calculation_utf8() {
+        let parser = create_test_parser();
+        let text = "你好\n世界"; // UTF-8 characters: 你(3 bytes) 好(3 bytes) \n(1 byte) 世(3 bytes) 界(3 bytes)
+
+        // Test position at start
+        let pos = parser.calculate_position_from_byte_offset(text, 0);
+        assert_eq!(pos.line, 0);
+        assert_eq!(pos.column, 0);
+        assert_eq!(pos.offset, 0);
+        assert_eq!(pos.byte_offset, 0);
+
+        // Test position after first character "你" (3 bytes)
+        let pos = parser.calculate_position_from_byte_offset(text, 3);
+        assert_eq!(pos.line, 0);
+        assert_eq!(pos.column, 1);
+        assert_eq!(pos.offset, 1);
+        assert_eq!(pos.byte_offset, 3);
+
+        // Test position after "你好" (6 bytes)
+        let pos = parser.calculate_position_from_byte_offset(text, 6);
+        assert_eq!(pos.line, 0);
+        assert_eq!(pos.column, 2);
+        assert_eq!(pos.offset, 2);
+        assert_eq!(pos.byte_offset, 6);
+
+        // Test position after newline (7 bytes)
+        let pos = parser.calculate_position_from_byte_offset(text, 7);
+        assert_eq!(pos.line, 1);
+        assert_eq!(pos.column, 0);
+        assert_eq!(pos.offset, 3);
+        assert_eq!(pos.byte_offset, 7);
+
+        // Test position after "世" (10 bytes)
+        let pos = parser.calculate_position_from_byte_offset(text, 10);
+        assert_eq!(pos.line, 1);
+        assert_eq!(pos.column, 1);
+        assert_eq!(pos.offset, 4);
+        assert_eq!(pos.byte_offset, 10);
+    }
+
+    #[test]
+    fn test_char_offset_to_byte_offset_conversion() {
+        let parser = create_test_parser();
+        let text = "你好\n世界";
+
+        // Test char offset 0 -> byte offset 0
+        let pos = parser.calculate_position_from_char_offset(text, 0);
+        assert_eq!(pos.byte_offset, 0);
+
+        // Test char offset 1 -> byte offset 3 (after "你")
+        let pos = parser.calculate_position_from_char_offset(text, 1);
+        assert_eq!(pos.byte_offset, 3);
+
+        // Test char offset 2 -> byte offset 6 (after "你好")
+        let pos = parser.calculate_position_from_char_offset(text, 2);
+        assert_eq!(pos.byte_offset, 6);
+
+        // Test char offset 3 -> byte offset 7 (after newline)
+        let pos = parser.calculate_position_from_char_offset(text, 3);
+        assert_eq!(pos.byte_offset, 7);
+
+        // Test char offset 4 -> byte offset 10 (after "世")
+        let pos = parser.calculate_position_from_char_offset(text, 4);
+        assert_eq!(pos.byte_offset, 10);
+
+        // Test char offset 5 -> byte offset 13 (after "世界")
+        let pos = parser.calculate_position_from_char_offset(text, 5);
+        assert_eq!(pos.byte_offset, 13);
+    }
+
+    #[test]
+    fn test_markdown_parsing_with_utf8() {
+        let mut parser = create_test_parser();
+        let markdown = "# 中文标题\n\n这是一段**中文**内容。";
+
+        let result = parser.parse(markdown);
+        assert!(result.is_ok(), "Failed to parse UTF-8 markdown");
+
+        let parse_result = result.unwrap();
+        assert_eq!(parse_result.source_text, markdown);
+
+        // Verify that positions are calculated correctly
+        let root = &parse_result.root;
+        assert_eq!(root.range.start.byte_offset, 0);
+        assert_eq!(root.range.end.byte_offset, markdown.len());
+
+        // Check that child nodes have valid byte offsets
+        for child in &root.children {
+            assert!(child.range.start.byte_offset <= child.range.end.byte_offset);
+            assert!(child.range.end.byte_offset <= markdown.len());
+        }
+    }
+
+    #[test]
+    fn test_complex_markdown_byte_offsets() {
+        let mut parser = create_test_parser();
+        let markdown = "# 标题\n\n这是**粗体**和*斜体*文本。\n\n```rust\nfn main() {\n    println!(\"你好\");\n}\n```\n\n- 列表项1\n- 列表项2";
+
+        let result = parser.parse(markdown);
+        assert!(result.is_ok(), "Failed to parse complex UTF-8 markdown");
+
+        let parse_result = result.unwrap();
+
+        // 验证所有节点的字节偏移量都在有效范围内
+        fn validate_node_offsets(node: &MarkdownNode, source_len: usize) {
+            assert!(node.range.start.byte_offset <= node.range.end.byte_offset,
+                "Start offset should be <= end offset for node: {:?}", node.node_type);
+            assert!(node.range.end.byte_offset <= source_len,
+                "End offset should be <= source length for node: {:?}", node.node_type);
+
+            // 验证行列号与字节偏移的一致性
+            assert!(node.range.start.line <= node.range.end.line,
+                "Start line should be <= end line for node: {:?}", node.node_type);
+
+            if node.range.start.line == node.range.end.line {
+                assert!(node.range.start.column <= node.range.end.column,
+                    "Start column should be <= end column on same line for node: {:?}", node.node_type);
+            }
+
+            // 递归验证子节点
+            for child in &node.children {
+                validate_node_offsets(child, source_len);
+            }
+        }
+
+        validate_node_offsets(&parse_result.root, markdown.len());
+    }
+
+    #[test]
+    fn test_position_consistency() {
+        let parser = create_test_parser();
+        let text = "Hello 世界\nNew line";
+
+        // 测试字节偏移和字符偏移之间的一致性
+        for i in 0..text.chars().count() {
+            let pos_from_char = parser.calculate_position_from_char_offset(text, i);
+            let pos_from_byte = parser.calculate_position_from_byte_offset(text, pos_from_char.byte_offset);
+
+            assert_eq!(pos_from_char.line, pos_from_byte.line,
+                "Line mismatch at char offset {}", i);
+            assert_eq!(pos_from_char.column, pos_from_byte.column,
+                "Column mismatch at char offset {}", i);
+            assert_eq!(pos_from_char.offset, pos_from_byte.offset,
+                "Char offset mismatch at char offset {}", i);
+        }
+    }
+
     #[test]
     fn test_parser_config() {
         let config = MarkdownParserConfig {