001/*
002 *  Copyright (c) 2023-2026, Agents-Flex (fuhai999@gmail.com).
003 *  <p>
004 *  Licensed under the Apache License, Version 2.0 (the "License");
005 *  you may not use this file except in compliance with the License.
006 *  You may obtain a copy of the License at
007 *  <p>
008 *  http://www.apache.org/licenses/LICENSE-2.0
009 *  <p>
010 *  Unless required by applicable law or agreed to in writing, software
011 *  distributed under the License is distributed on an "AS IS" BASIS,
012 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 *  See the License for the specific language governing permissions and
014 *  limitations under the License.
015 */
016package com.agentsflex.core.file2text.util;
017
018
019import com.ibm.icu.text.CharsetDetector;
020import com.ibm.icu.text.CharsetMatch;
021
022import java.io.*;
023import java.nio.charset.Charset;
024import java.nio.charset.StandardCharsets;
025
026/**
027 * 工业级编码检测工具类（支持流式读取大文件，自动识别编码并返回 Reader）
028 *
029 * <p>特性：
030 * <ul>
031 *     <li>支持 PushbackInputStream，避免依赖 mark/reset</li>
032 *     <li>支持 UTF-8, UTF-16LE/BE, UTF-32LE/BE BOM 检测</li>
033 *     <li>ICU4J 异常降级处理，默认回退 UTF-8</li>
034 *     <li>支持自定义置信度阈值</li>
035 *     <li>编码名称标准化</li>
036 *     <li>返回的 Reader 关闭时会级联关闭底层 InputStream</li>
037 * </ul>
038 *
039 * <p>使用示例：
040 * <pre>{@code
041 * try (Reader reader = EncodingDetectUtil.getAutoDetectReader(inputStream)) {
042 *     BufferedReader br = new BufferedReader(reader);
043 *     String line;
044 *     while ((line = br.readLine()) != null) {
045 *         System.out.println(line);
046 *     }
047 * }
048 * }</pre>
049 *
050 * <p>调用方必须关闭返回的 Reader，以释放底层 InputStream。
051 */
052public class EncodingDetectUtil {
053
054    private static final int DEFAULT_DETECT_BUFFER_SIZE = 1024 * 1024; // 1MB
055    private static final int DEFAULT_CONFIDENCE_THRESHOLD = 60;
056
057    private EncodingDetectUtil() {
058        // 工具类禁止实例化
059    }
060
061    // ==========================
062    // 对外 API
063    // ==========================
064
065
066    /**
067     * 自动读取 InputStream 为文本
068     *
069     * @param inputStream 待读取流
070     * @return 完整文本
071     * @throws IOException IO 异常
072     */
073    public static String readToString(InputStream inputStream) throws IOException {
074        StringBuilder sb = new StringBuilder();
075        try (BufferedReader br = new BufferedReader(getAutoDetectReader(inputStream))) {
076            String line;
077            while ((line = br.readLine()) != null) {
078                sb.append(line).append(System.lineSeparator());
079            }
080        }
081        return sb.toString();
082    }
083
084    /**
085     * 自动识别 InputStream 编码并返回 Reader（默认置信度阈值 60）
086     *
087     * @param inputStream 待检测流
088     * @return 自动识别编码的 Reader
089     * @throws IOException IO 异常
090     */
091    public static Reader getAutoDetectReader(InputStream inputStream) throws IOException {
092        return getAutoDetectReader(inputStream, DEFAULT_CONFIDENCE_THRESHOLD);
093    }
094
095    /**
096     * 自动识别 InputStream 编码并返回 Reader，可自定义置信度阈值
097     *
098     * @param inputStream         待检测流
099     * @param confidenceThreshold ICU4J 检测置信度阈值（0-100）
100     * @return 自动识别编码的 Reader
101     * @throws IOException IO 异常
102     */
103    public static Reader getAutoDetectReader(InputStream inputStream, int confidenceThreshold) throws IOException {
104        if (inputStream == null) {
105            throw new IllegalArgumentException("inputStream cannot be null");
106        }
107        if (confidenceThreshold < 0 || confidenceThreshold > 100) {
108            confidenceThreshold = DEFAULT_CONFIDENCE_THRESHOLD;
109        }
110
111        PushbackInputStream pbis = new PushbackInputStream(inputStream, DEFAULT_DETECT_BUFFER_SIZE);
112        byte[] detectBytes = new byte[DEFAULT_DETECT_BUFFER_SIZE];
113        int read = pbis.read(detectBytes);
114
115        String charsetName;
116        if (read <= 0) {
117            charsetName = StandardCharsets.UTF_8.name();
118        } else {
119            // 1. 检测 BOM
120            String bomCharset = detectBom(detectBytes, read);
121            if (bomCharset != null) {
122                charsetName = bomCharset;
123            } else {
124                // 2. ICU4J 自动检测
125                charsetName = detectCharsetICU(detectBytes, read, confidenceThreshold);
126            }
127        }
128
129        // 推回已读取字节，保证流式读取完整数据
130        if (read > 0) {
131            pbis.unread(detectBytes, 0, read);
132        }
133
134        return new InputStreamReader(pbis, Charset.forName(charsetName));
135    }
136
137    /**
138     * 自动检测编码（只返回编码名称）
139     *
140     * @param inputStream 待检测流
141     * @return 编码名称（标准 JVM 名称）
142     * @throws IOException IO 异常
143     */
144    public static String detectCharset(InputStream inputStream) throws IOException {
145        try (Reader r = getAutoDetectReader(inputStream)) {
146            return ((InputStreamReader) r).getEncoding();
147        }
148    }
149
150    // ==========================
151    // 内部方法
152    // ==========================
153
154    /**
155     * BOM 检测（含 UTF-8/16/32）
156     */
157    private static String detectBom(byte[] bytes, int length) {
158        if (length >= 3 && (bytes[0] & 0xFF) == 0xEF
159            && (bytes[1] & 0xFF) == 0xBB
160            && (bytes[2] & 0xFF) == 0xBF) {
161            return StandardCharsets.UTF_8.name();
162        }
163        if (length >= 2 && (bytes[0] & 0xFF) == 0xFF && (bytes[1] & 0xFF) == 0xFE) {
164            return "UTF-16LE";
165        }
166        if (length >= 2 && (bytes[0] & 0xFF) == 0xFE && (bytes[1] & 0xFF) == 0xFF) {
167            return "UTF-16BE";
168        }
169        if (length >= 4 && (bytes[0] & 0xFF) == 0xFF
170            && (bytes[1] & 0xFF) == 0xFE
171            && (bytes[2] & 0xFF) == 0x00
172            && (bytes[3] & 0xFF) == 0x00) {
173            return "UTF-32LE";
174        }
175        if (length >= 4 && (bytes[0] & 0xFF) == 0x00
176            && (bytes[1] & 0xFF) == 0x00
177            && (bytes[2] & 0xFF) == 0xFE
178            && (bytes[3] & 0xFF) == 0xFF) {
179            return "UTF-32BE";
180        }
181        return null;
182    }
183
184    /**
185     * 使用 ICU4J 检测编码，支持置信度阈值
186     */
187    private static String detectCharsetICU(byte[] bytes, int length, int confidenceThreshold) {
188        try {
189            CharsetDetector detector = new CharsetDetector();
190            detector.enableInputFilter(true); // 过滤 HTML/XML 标签提高准确性
191
192            if (length < bytes.length) {
193                // 只取前 length 字节
194                byte[] sample = new byte[length];
195                System.arraycopy(bytes, 0, sample, 0, length);
196                detector.setText(sample);
197            } else {
198                detector.setText(bytes);
199            }
200
201            CharsetMatch match = detector.detect();
202
203            if (match != null && match.getConfidence() >= confidenceThreshold) {
204                String name = match.getName();
205                // GBK 升级为 GB18030
206                if ("GBK".equalsIgnoreCase(name)) {
207                    return "GB18030";
208                }
209                return Charset.forName(name).name(); // 标准化
210            }
211        } catch (Exception e) {
212            // ICU4J 异常降级到 UTF-8
213        }
214        return StandardCharsets.UTF_8.name();
215    }
216}