001/* 002 * Copyright (c) 2023-2026, Agents-Flex (fuhai999@gmail.com). 003 * <p> 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * <p> 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * <p> 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package com.agentsflex.core.document.splitter; 017 018import com.agentsflex.core.document.Document; 019import com.agentsflex.core.document.DocumentSplitter; 020import com.agentsflex.core.document.id.DocumentIdGenerator; 021import com.agentsflex.core.util.StringUtil; 022 023import java.util.*; 024 025public class MarkdownHeaderSplitter implements DocumentSplitter { 026 027 /** 028 * 最大标题级别(inclusive),用于触发拆分。 029 * 例如:splitLevel = 2 表示在 # 和 ## 处拆分,### 及以下不作为新块起点。 030 */ 031 private int splitLevel; 032 033 /** 034 * 是否在每个 chunk 中保留父级标题路径(如 "Introduction > Background") 035 */ 036 private boolean includeParentHeaders = true; 037 038 public MarkdownHeaderSplitter() { 039 } 040 041 public MarkdownHeaderSplitter(int splitLevel) { 042 if (splitLevel < 1 || splitLevel > 6) { 043 throw new IllegalArgumentException("splitLevel must be between 1 and 6, got: " + splitLevel); 044 } 045 this.splitLevel = splitLevel; 046 } 047 048 public MarkdownHeaderSplitter(int splitLevel, boolean includeParentHeaders) { 049 this(splitLevel); 050 this.includeParentHeaders = includeParentHeaders; 051 } 052 053 public int getSplitLevel() { 054 return splitLevel; 055 } 056 057 public void setSplitLevel(int splitLevel) { 058 if (splitLevel < 1 || splitLevel > 6) { 059 throw new IllegalArgumentException("splitLevel must be between 1 and 6"); 060 } 061 this.splitLevel = splitLevel; 062 } 063 064 public boolean isIncludeParentHeaders() { 065 return includeParentHeaders; 066 } 067 068 public void setIncludeParentHeaders(boolean includeParentHeaders) { 069 this.includeParentHeaders = includeParentHeaders; 070 } 071 072 @Override 073 public List<Document> split(Document document, DocumentIdGenerator idGenerator) { 074 if (document == null || StringUtil.noText(document.getContent())) { 075 return Collections.emptyList(); 076 } 077 078 String content = document.getContent(); 079 String[] lines = content.split("\n"); 080 081 List<DocumentChunk> chunks = new ArrayList<>(); 082 Deque<HeaderInfo> headerStack = new ArrayDeque<>(); 083 084 StringBuilder currentContent = new StringBuilder(); 085 int currentStartLine = 0; 086 087 boolean inCodeBlock = false; 088 089 for (int i = 0; i < lines.length; i++) { 090 String line = lines[i]; 091 if (line == null) { 092 currentContent.append("\n"); 093 continue; 094 } 095 096 // 检测围栏代码块的开始或结束(支持 ``` 或 ~~~) 097 String trimmedLine = stripLeading(line); 098 if (trimmedLine.startsWith("```") || trimmedLine.startsWith("~~~")) { 099 inCodeBlock = !inCodeBlock; 100 currentContent.append(line).append("\n"); 101 continue; 102 } 103 104 if (!inCodeBlock) { 105 HeaderInfo header = parseHeader(line); 106 if (header != null && header.level <= splitLevel) { 107 // 触发新 chunk 108 if (currentContent.length() > 0 || !chunks.isEmpty()) { 109 flushChunk(chunks, currentContent.toString(), headerStack, currentStartLine, i - 1, document); 110 currentContent.setLength(0); 111 } 112 currentStartLine = i; 113 114 // 弹出栈中层级大于等于当前的标题 115 while (!headerStack.isEmpty() && headerStack.peek().level >= header.level) { 116 headerStack.pop(); 117 } 118 headerStack.push(header); 119 120 // 将标题行加入当前内容(保留结构) 121 currentContent.append(line).append("\n"); 122 continue; 123 } 124 } 125 126 // 普通文本行或代码块内行 127 currentContent.append(line).append("\n"); 128 } 129 130 // Flush remaining content 131 if (currentContent.length() > 0) { 132 flushChunk(chunks, currentContent.toString(), headerStack, currentStartLine, lines.length - 1, document); 133 } 134 135 // 构建结果 Document 列表 136 List<Document> result = new ArrayList<>(); 137 for (DocumentChunk chunk : chunks) { 138 Document doc = new Document(); 139 doc.setContent(chunk.content.trim()); 140 doc.addMetadata(document.getMetadataMap()); 141 142 if (includeParentHeaders && !chunk.headerPath.isEmpty()) { 143 doc.addMetadata("header_path", String.join(" > ", chunk.headerPath)); 144 } 145 doc.addMetadata("start_line", String.valueOf(chunk.startLine)); 146 doc.addMetadata("end_line", String.valueOf(chunk.endLine)); 147 148 if (idGenerator != null) { 149 doc.setId(idGenerator.generateId(doc)); 150 } 151 result.add(doc); 152 } 153 154 return result; 155 } 156 157 private void flushChunk(List<DocumentChunk> chunks, String content, 158 Deque<HeaderInfo> headerStack, int startLine, int endLine, Document sourceDoc) { 159 if (StringUtil.noText(content.trim())) { 160 return; 161 } 162 163 // 从根到当前构建标题路径 164 List<String> headerPath = new ArrayList<>(); 165 List<HeaderInfo> stackCopy = new ArrayList<>(headerStack); 166 Collections.reverse(stackCopy); 167 for (HeaderInfo h : stackCopy) { 168 headerPath.add(h.text); 169 } 170 171 chunks.add(new DocumentChunk(content, headerPath, startLine, endLine)); 172 } 173 174 /** 175 * 解析一行是否为合法的 ATX 标题(# Title) 176 * 177 * @param line 输入行 178 * @return HeaderInfo 或 null 179 */ 180 private HeaderInfo parseHeader(String line) { 181 if (line == null || line.isEmpty()) { 182 return null; 183 } 184 line = stripLeading(line); 185 if (!line.startsWith("#")) { 186 return null; 187 } 188 189 int level = 0; 190 int i = 0; 191 while (i < line.length() && line.charAt(i) == '#') { 192 level++; 193 i++; 194 } 195 196 if (level > 6) { 197 return null; // 非法标题 198 } 199 200 // 必须后跟空格或行结束(符合 CommonMark 规范) 201 if (i < line.length() && line.charAt(i) != ' ') { 202 return null; 203 } 204 205 String text = line.substring(i).trim(); 206 return new HeaderInfo(level, text); 207 } 208 209 private static String stripLeading(String s) { 210 if (s == null || s.isEmpty()) { 211 return s; 212 } 213 int i = 0; 214 while (i < s.length() && Character.isWhitespace(s.charAt(i))) { 215 i++; 216 } 217 return i == 0 ? s : s.substring(i); 218 } 219 220 // -- 内部辅助类 -- 221 222 private static class HeaderInfo { 223 final int level; 224 final String text; 225 226 HeaderInfo(int level, String text) { 227 this.level = level; 228 this.text = text; 229 } 230 } 231 232 private static class DocumentChunk { 233 final String content; 234 final List<String> headerPath; 235 final int startLine; 236 final int endLine; 237 238 DocumentChunk(String content, List<String> headerPath, int startLine, int endLine) { 239 this.content = content; 240 this.headerPath = headerPath; 241 this.startLine = startLine; 242 this.endLine = endLine; 243 } 244 } 245}