001/*
002 *  Copyright (c) 2023-2026, Agents-Flex (fuhai999@gmail.com).
003 *  <p>
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *  <p>
008 *  http://www.apache.org/licenses/LICENSE-2.0
009 *  <p>
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package com.agentsflex.core.document.splitter;
017
018import com.agentsflex.core.document.Document;
019import com.agentsflex.core.document.DocumentSplitter;
020import com.agentsflex.core.document.id.DocumentIdGenerator;
021import com.agentsflex.core.util.StringUtil;
022import com.knuddels.jtokkit.Encodings;
023import com.knuddels.jtokkit.api.Encoding;
024import com.knuddels.jtokkit.api.EncodingRegistry;
025import com.knuddels.jtokkit.api.EncodingType;
026import com.knuddels.jtokkit.api.IntArrayList;
027
028import java.util.ArrayList;
029import java.util.Collections;
030import java.util.List;
031
032public class SimpleTokenizeSplitter implements DocumentSplitter {
033    private EncodingRegistry registry = Encodings.newLazyEncodingRegistry();
034    private EncodingType encodingType = EncodingType.CL100K_BASE;
035    private int chunkSize;
036    private int overlapSize;
037
038    public SimpleTokenizeSplitter(int chunkSize) {
039        this.chunkSize = chunkSize;
040        if (this.chunkSize <= 0) {
041            throw new IllegalArgumentException("chunkSize must be greater than 0, chunkSize: " + this.chunkSize);
042        }
043    }
044
045    public SimpleTokenizeSplitter(int chunkSize, int overlapSize) {
046        this.chunkSize = chunkSize;
047        this.overlapSize = overlapSize;
048
049        if (this.chunkSize <= 0) {
050            throw new IllegalArgumentException("chunkSize must be greater than 0, chunkSize: " + this.chunkSize);
051        }
052        if (this.overlapSize >= this.chunkSize) {
053            throw new IllegalArgumentException("overlapSize must be less than chunkSize, overlapSize: " + this.overlapSize + ", chunkSize: " + this.chunkSize);
054        }
055    }
056
057    public int getChunkSize() {
058        return chunkSize;
059    }
060
061    public void setChunkSize(int chunkSize) {
062        this.chunkSize = chunkSize;
063    }
064
065    public int getOverlapSize() {
066        return overlapSize;
067    }
068
069    public void setOverlapSize(int overlapSize) {
070        this.overlapSize = overlapSize;
071    }
072
073    public EncodingRegistry getRegistry() {
074        return registry;
075    }
076
077    public void setRegistry(EncodingRegistry registry) {
078        this.registry = registry;
079    }
080
081    public EncodingType getEncodingType() {
082        return encodingType;
083    }
084
085    public void setEncodingType(EncodingType encodingType) {
086        this.encodingType = encodingType;
087    }
088
089    @Override
090    public List<Document> split(Document document, DocumentIdGenerator idGenerator) {
091        if (document == null || StringUtil.noText(document.getContent())) {
092            return Collections.emptyList();
093        }
094
095        String content = document.getContent();
096        Encoding encoding = this.registry.getEncoding(this.encodingType);
097
098        List<Integer> tokens = encoding.encode(content).boxed();
099
100
101        int index = 0, currentIndex = index;
102        int maxIndex = tokens.size();
103
104        List<Document> chunks = new ArrayList<>();
105        while (currentIndex < maxIndex) {
106            int endIndex = Math.min(currentIndex + chunkSize, maxIndex);
107            List<Integer> chunkTokens = tokens.subList(currentIndex, endIndex);
108
109            IntArrayList intArrayList = new IntArrayList();
110            for (Integer chunkToken : chunkTokens) {
111                intArrayList.add(chunkToken);
112            }
113            String chunkText = encoding.decode(intArrayList).trim();
114            if (chunkText.isEmpty()) {
115                continue;
116            }
117
118            //UTF-8 'Unicode replacement character' which in your case is 0xFFFD (65533 in Hex).
119            //fix 修复中文乱码的问题
120            boolean firstIsReplacement = chunkText.charAt(0) == 65533;
121            boolean lastIsReplacement = chunkText.charAt(chunkText.length() - 1) == 65533;
122
123            if (firstIsReplacement || lastIsReplacement) {
124                if (firstIsReplacement) currentIndex -= 1;
125                if (lastIsReplacement) endIndex += 1;
126
127                chunkTokens = tokens.subList(currentIndex, endIndex);
128                intArrayList = new IntArrayList();
129                for (Integer chunkToken : chunkTokens) {
130                    intArrayList.add(chunkToken);
131                }
132
133                chunkText = encoding.decode(intArrayList).trim();
134            }
135
136            currentIndex = currentIndex + chunkSize - overlapSize;
137
138            Document newDocument = new Document();
139            newDocument.addMetadata(document.getMetadataMap());
140            newDocument.setContent(chunkText);
141
142            //we should invoke setId after setContent
143            newDocument.setId(idGenerator == null ? null : idGenerator.generateId(newDocument));
144            chunks.add(newDocument);
145        }
146
147        return chunks;
148    }
149}