001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.gzip;
019
020import org.jwat.common.Diagnosis;
021import org.jwat.common.DiagnosisType;
022import org.jwat.common.ISO8859_1;
023
024import java.io.BufferedOutputStream;
025import java.io.Closeable;
026import java.io.IOException;
027import java.io.InputStream;
028import java.io.OutputStream;
029import java.nio.ByteBuffer;
030import java.util.Date;
031import java.util.zip.CRC32;
032import java.util.zip.DataFormatException;
033import java.util.zip.Deflater;
034
035/**
036 * A writer for (multi-part) GZip files.
037 * As per the GZIP file format specification version 4.3.
038 * The class writes a GZip header. The compressed data and trailer is also
039 * written by this class but is instigated by calling the write method on the
040 * GZip entry itself.
041 *
042 * @author nicl
043 */
044public class GzipWriter implements Closeable {
045
046    /** Default input buffer size. */
047    public static final int DEFAULT_INPUT_BUFFER_SIZE = 8192;
048
049    /** Output stream for GZip (multi-part) file. */
050    protected OutputStream out;
051    /** Deflater used to compress GZip entries. */
052    protected Deflater def = new Deflater(Deflater.DEFAULT_COMPRESSION, true);
053    /** Checksum object used to calculate CRC16 and CRC32 values. */
054    protected CRC32 crc = new CRC32();
055    /** Input buffer used to feed the deflater. */
056    protected byte[] inputBytes;
057
058    /** Compression level to be used by deflater. */
059    protected int compressionLevel = Deflater.DEFAULT_COMPRESSION;
060
061    /** ISO-8859-1 validating de-/encoder. */
062    protected final ISO8859_1 iso8859_1 = new ISO8859_1();
063
064    /** Compliance status for records parsed up to now. */
065    protected boolean bIsCompliant = true;
066
067    /** Current GZip entry object. */
068    protected GzipEntry gzipEntry;
069
070    /** Buffer used to read header.  */
071    byte[] headerBytes = new byte[10];
072    /** Buffer used to read the XLEN value. */
073    byte[] xlenBytes = new byte[2];
074    /** Buffer used to read the FNAME data. */
075    byte[] fnameBytes = null;
076    /** Buffer used to read the FCOMMENT data. */
077    byte[] fcommentBytes = null;
078    /** Buffer used to read the CRC16 value. */
079    byte[] crc16Bytes = new byte[2];
080    /** Buffer used to read trailer. */
081    byte[] trailerBytes = new byte[8];
082
083    /**
084     * Construct a GZip writer with a default input buffer size of 1024.
085     * @param out output stream of GZip file
086     */
087    public GzipWriter(OutputStream out) {
088        if (out == null) {
089            throw new IllegalArgumentException("out is null!");
090        }
091        this.out = new BufferedOutputStream(out, DEFAULT_INPUT_BUFFER_SIZE);
092        inputBytes = new byte[DEFAULT_INPUT_BUFFER_SIZE];
093    }
094
095    /**
096     * Construct a GZip writer with the specified input buffer size.
097     * @param out output stream of GZip file
098     * @param buffer_size input buffer size to use
099     */
100    public GzipWriter(OutputStream out, int buffer_size) {
101        if (out == null) {
102            throw new IllegalArgumentException("out is null!");
103        }
104        if (buffer_size <= 0) {
105            throw new IllegalArgumentException("buffer_size is less or equals to zero!");
106        }
107        this.out = new BufferedOutputStream(out, buffer_size);
108        inputBytes = new byte[buffer_size];
109    }
110
111    /**
112     * Release resources associated with this writer.
113     * @throws IOException if an I/O error occurs while closing writer
114     */
115    public void close() throws IOException {
116        if (gzipEntry != null) {
117            gzipEntry.close();
118            gzipEntry = null;
119        }
120        if (out != null) {
121            out.flush();
122            out = null;
123        }
124        if (def != null) {
125            def.end();
126            def = null;
127        }
128    }
129
130    /**
131     * Returns a boolean indicating whether all entries written so far are compliant.
132     * @return a boolean indicating whether all entries written so far are compliant
133     */
134    public boolean isCompliant() {
135        return bIsCompliant;
136    }
137
138    /**
139     * Set compression level used by deflater. Only changed in deflater prior
140     * to writing an entry header.
141     * @param compressionLevel compression level
142     * @throws IllegalArgumentException If CompressionLevel is invalid
143     */
144    public void setCompressionLevel(int compressionLevel) {
145        if (compressionLevel == Deflater.DEFAULT_COMPRESSION
146                || compressionLevel == Deflater.NO_COMPRESSION
147                || (compressionLevel >= 1 && compressionLevel <= 9)) {
148            this.compressionLevel = compressionLevel;
149        }
150        else {
151            throw new IllegalArgumentException("Invalid compression level: " + compressionLevel);
152        }
153    }
154
155    /**
156     * Returns current compression level used by deflater.
157     * @return current compression level
158     */
159    public int getCompressionLevel() {
160        return compressionLevel;
161    }
162
163    /**
164     * Write a GZip entry header and prepare for compressing input data.
165     * @param entry GZip entry object
166     * @throws IOException if an I/O error occurs while writing header
167     */
168    public void writeEntryHeader(GzipEntry entry) throws IOException {
169        if (gzipEntry != null) {
170            gzipEntry.close();
171            gzipEntry = null;
172        }
173        if (entry == null) {
174            throw new IllegalArgumentException("entry is null!");
175        }
176        crc.reset();
177        def.reset();
178        def.setLevel(compressionLevel);
179        gzipEntry = entry;
180        /*
181         * Header.
182         */
183        entry.magic = GzipConstants.GZIP_MAGIC;
184        if (entry.date != null) {
185            entry.mtime = entry.date.getTime() / 1000;
186        } else if (entry.mtime != 0) {
187            entry.date = new Date(entry.mtime * 1000);
188        }
189        entry.xfl = 0;
190        if (compressionLevel == 1) {
191            entry.xfl |= GzipConstants.DEFLATE_XFL_FASTEST_COMPRESSION;
192        } else if (compressionLevel == 9) {
193            entry.xfl |= GzipConstants.DEFLATE_XFL_MAXIMUM_COMPRESSION;
194        }
195        entry.flg = 0;
196        if (!GzipConstants.osIdxStr.containsKey((int)entry.os)) {
197            entry.diagnostics.addWarning(
198                    new Diagnosis(
199                            DiagnosisType.UNKNOWN,
200                            "Operating System",
201                            Integer.toString(entry.os)
202                )
203            );
204        }
205        /*
206         * FTEXT.
207         */
208        if (entry.bFText) {
209            entry.flg |= GzipConstants.FLG_FTEXT;
210        }
211        /*
212         * FEXTRA.
213         */
214        if (entry.extraBytes == null) {
215            if (entry.extraData.size() > 0) {
216                int xlen = 0;
217                for (int i=0; i<entry.extraData.size(); ++i) {
218                    xlen += 4 + entry.extraData.get(i).data.length;
219                }
220                entry.extraBytes = new byte[xlen];
221                GzipExtraData extraData;
222                int idx = 0;
223                for (int i=0; i<entry.extraData.size(); ++i) {
224                    extraData = entry.extraData.get(i);
225                    entry.extraBytes[idx++] = extraData.si1;
226                    entry.extraBytes[idx++] = extraData.si2;
227                    entry.extraBytes[idx++] = (byte)(extraData.data.length & 255);
228                    entry.extraBytes[idx++] = (byte)((extraData.data.length >> 8) & 255);
229                    System.arraycopy(extraData.data, 0, entry.extraBytes, idx, extraData.data.length);
230                    idx += extraData.data.length;
231                }
232            }
233        } else {
234            int idx = 0;
235            boolean b = true;
236            int len;
237            while (b) {
238                if (idx <= entry.extraBytes.length - 4) {
239                    idx += 2;
240                    len = ((entry.extraBytes[idx + 1] & 255) << 8) | (entry.extraBytes[idx] & 255);
241                    idx += 2;
242                    if (idx + len <= entry.extraBytes.length) {
243                        idx += len;
244                    } else {
245                        b = false;
246                    }
247                } else {
248                    b = false;
249                }
250            }
251            if (idx != gzipEntry.extraBytes.length) {
252                gzipEntry.diagnostics.addError(
253                        new Diagnosis(
254                                DiagnosisType.INVALID_DATA,
255                                "FEXTRA",
256                                "Invalid structure",
257                                "Data truncated"
258                            )
259                        );
260            }
261        }
262        if (entry.extraBytes != null) {
263            entry.flg |= GzipConstants.FLG_FEXTRA;
264            entry.xlen = entry.extraBytes.length;
265            xlenBytes[0] = (byte)(entry.xlen & 255);
266            xlenBytes[1] = (byte)((entry.xlen >> 8) & 255);
267        }
268        /*
269         * FNAME.
270         */
271        if (entry.fname != null) {
272            entry.flg |= GzipConstants.FLG_FNAME;
273            if (!iso8859_1.encode(entry.fname, "")) {
274                entry.diagnostics.addWarning(
275                        new Diagnosis(
276                                DiagnosisType.INVALID_ENCODING,
277                                "FName",
278                                entry.fname,
279                                "ISO-8859-1"
280                            )
281                        );
282            }
283            entry.fname = iso8859_1.decoded;
284            fnameBytes = iso8859_1.encoded;
285        }
286        /*
287         * FCOMMENT.
288         */
289        if (entry.fcomment != null) {
290            entry.flg |= GzipConstants.FLG_FCOMMENT;
291            if (!iso8859_1.encode(entry.fcomment, "\n")) {
292                entry.diagnostics.addWarning(
293                        new Diagnosis(
294                                DiagnosisType.INVALID_ENCODING,
295                                "FComment",
296                                entry.fcomment,
297                                "ISO-8859-1"
298                            )
299                        );
300            }
301            entry.fcomment = iso8859_1.decoded;
302            fcommentBytes = iso8859_1.encoded;
303        }
304        /*
305         * FHCRC.
306         */
307        if (entry.bFhCrc) {
308            entry.flg |= GzipConstants.FLG_FHCRC;
309        }
310        /*
311         * Header.
312         */
313        headerBytes[0] = (byte)(entry.magic & 255);
314        headerBytes[1] = (byte)((entry.magic >> 8) & 255);
315        headerBytes[2] = (byte)entry.cm;
316        headerBytes[3] = (byte)entry.flg;
317        headerBytes[4] = (byte)(entry.mtime & 255);
318        headerBytes[5] = (byte)((entry.mtime >> 8) & 255);
319        headerBytes[6] = (byte)((entry.mtime >> 16) & 255);
320        headerBytes[7] = (byte)((entry.mtime >> 24) & 255);
321        headerBytes[8] = (byte)entry.xfl;
322        headerBytes[9] = (byte)entry.os;
323        out.write(headerBytes);
324        crc.update(headerBytes);
325        if ((entry.flg & GzipConstants.FLG_FEXTRA) == GzipConstants.FLG_FEXTRA) {
326            out.write(xlenBytes);
327            out.write(entry.extraBytes);
328            crc.update(xlenBytes);
329            crc.update(entry.extraBytes);
330        }
331        if ((entry.flg & GzipConstants.FLG_FNAME) == GzipConstants.FLG_FNAME) {
332            out.write(fnameBytes);
333            out.write(0);
334            crc.update(fnameBytes);
335            crc.update(0);
336        }
337        if ((entry.flg & GzipConstants.FLG_FCOMMENT) == GzipConstants.FLG_FCOMMENT) {
338            out.write(fcommentBytes);
339            out.write(0);
340            crc.update(fcommentBytes);
341            crc.update(0);
342        }
343        if ((entry.flg & GzipConstants.FLG_FHCRC) == GzipConstants.FLG_FHCRC) {
344            entry.comp_crc16 = ((int)crc.getValue()) & 0x0000ffff;
345            entry.crc16 = entry.comp_crc16;
346            crc16Bytes[0] = (byte)(entry.crc16 & 255);
347            crc16Bytes[1] = (byte)((entry.crc16 >> 8) & 255);
348            out.write(crc16Bytes);
349        }
350        /*
351         * Prepare Entry InputStream.
352         */
353        crc.reset();
354        entry.isize = 0;
355        entry.writer = this;
356        entry.bEof = false;
357        entry.out = new GzipEntryOutputStream(this, gzipEntry);
358        // Compliance
359        if (entry.diagnostics.hasErrors() || entry.diagnostics.hasWarnings()) {
360            entry.bIsCompliant = false;
361        } else {
362            entry.bIsCompliant = true;
363        }
364        bIsCompliant &= entry.bIsCompliant;
365    }
366
367    /**
368     * Computes the CRC32 and iSize values and then writes the GZip entry trailer.
369     * @param entry GZip entry object
370     * @throws IOException if an I/O error occurs while writing trailer
371     */
372    protected void writeTrailer(GzipEntry entry) throws IOException {
373        // Compliance
374        if (entry.diagnostics.hasErrors() || entry.diagnostics.hasWarnings()) {
375            entry.bIsCompliant = false;
376        } else {
377            entry.bIsCompliant = true;
378        }
379        bIsCompliant &= gzipEntry.bIsCompliant;
380        // Trailer
381        entry.uncompressed_size = def.getBytesRead();
382        entry.compressed_size = def.getBytesWritten();
383        entry.comp_crc32 = (int)(crc.getValue() & 0xffffffff);
384        entry.crc32 = entry.comp_crc32;
385        entry.comp_isize = (int)(def.getBytesRead() & 0xffffffff);
386        entry.isize = entry.comp_isize;
387        trailerBytes[0] = (byte)(entry.crc32 & 255);
388        trailerBytes[1] = (byte)((entry.crc32 >> 8) & 255);
389        trailerBytes[2] = (byte)((entry.crc32 >> 16) & 255);
390        trailerBytes[3] = (byte)((entry.crc32 >> 24) & 255);
391        trailerBytes[4] = (byte)(entry.isize & 255);
392        trailerBytes[5] = (byte)((entry.isize >> 8) & 255);
393        trailerBytes[6] = (byte)((entry.isize >> 16) & 255);
394        trailerBytes[7] = (byte)((entry.isize >> 24) & 255);
395        out.write(trailerBytes);
396        out.flush();
397    }
398
399    /**
400     * Read data from input stream and fill buffer with compressed data.
401     * @param in input stream with uncompressed data
402     * @param b compressed data buffer
403     * @param off offset in compressed data buffer
404     * @param len length of compressed data to read
405     * @return number of compressed bytes read
406     * @throws DataFormatException if an error occurs in deflater
407     * @throws IOException if an I/O error occurs while compressing
408     */
409    protected int readCompressed(InputStream in, byte[] b, int off, int len) throws DataFormatException, IOException {
410        int deflated = 0;
411        while ((deflated = def.deflate(b, off, len)) == 0) {
412            if (def.finished()) {
413                return -1;
414            } else if (def.needsInput()) {
415                int read = in.read(inputBytes, 0, inputBytes.length);
416                if (read != -1) {
417                    def.setInput(inputBytes, 0, read);
418                    crc.update(inputBytes, 0, read);
419                } else {
420                    def.finish();
421                }
422            } else {
423                throw new DataFormatException("Deflater malfunction!");
424            }
425        }
426        return deflated;
427    }
428
429    /**
430     * Read data from <code>ByteBuffer</code> and fill buffer with compressed
431     * data.
432     * @param bb <code>ByteBuffer</code> with uncompressed data
433     * @param b compressed data buffer
434     * @param off offset in compressed data buffer
435     * @param len length of compressed data to read
436     * @param bFinish boolean indicating that there is no more data to compress
437     * @return number of compressed bytes read
438     * @throws DataFormatException if an error occurs in deflater
439     * @throws IOException if an error occurs while compressing
440     */
441    protected int readCompressed(ByteBuffer bb, byte[] b, int off, int len, boolean bFinish) throws DataFormatException, IOException {
442        int deflated = 0;
443        while ((deflated = def.deflate(b, off, len)) == 0) {
444            if (def.finished()) {
445                return -1;
446            } else if (def.needsInput()) {
447                int write = bb.remaining();
448                if (write > 0) {
449                    if (write > inputBytes.length) {
450                        write = inputBytes.length;
451                    }
452                    bb.get(inputBytes, 0, write);
453                    def.setInput(inputBytes, 0, write);
454                    crc.update(inputBytes, 0, write);
455                } else {
456                    if (bFinish) {
457                        def.finish();
458                    } else {
459                        // Deflater needs more input.
460                        return 0;
461                    }
462                }
463            } else {
464                throw new DataFormatException("Deflater malfunction!");
465            }
466        }
467        return deflated;
468    }
469
470    /**
471     * <code>OutputStream</code> to GZip compress data in a controlled fashion.
472     *
473     * @author nicl
474     */
475    protected static class GzipEntryOutputStream extends OutputStream {
476
477        /** GZip reader used to inflate. */
478        GzipWriter writer;
479
480        /** Associated GZip entry. */
481        GzipEntry gzipEntry;
482
483        /** End of compressed file status. */
484        boolean bEof = false;
485
486        /** Small buffer used by the read() method. */
487        byte[] singleByteArray = new byte[1];
488
489        /** Buffer for feeding the deflater. */
490        ByteBuffer bb = ByteBuffer.allocate(DEFAULT_INPUT_BUFFER_SIZE);
491
492        /** Buffer used for compressed data. */
493        byte[] compressedBytes = new byte[DEFAULT_INPUT_BUFFER_SIZE];
494
495        /**
496         * Construct output stream bound to a specific writer and entry.
497         * @param writer GZip writer
498         * @param gzipEntry GZip entry
499         */
500        public GzipEntryOutputStream(GzipWriter writer,
501                                    GzipEntry gzipEntry) {
502            this.writer = writer;
503            this.gzipEntry = gzipEntry;
504        }
505
506        @Override
507        public void close() throws IOException {
508            if (!bEof) {
509                bEof = true;
510                try {
511                    int deflated = 0;
512                    while (deflated != -1) {
513                        bb.flip();
514                        deflated = writer.readCompressed(bb, compressedBytes, 0, compressedBytes.length, true);
515                        bb.compact();
516                        if (deflated > 0) {
517                            writer.out.write(compressedBytes, 0, deflated);
518                        }
519                    }
520                    writer.writeTrailer(gzipEntry);
521                } catch (DataFormatException e) {
522                    throw new IOException("Deflater malfunction!", e);
523                } finally {
524                    writer = null;
525                    gzipEntry = null;
526                    singleByteArray = null;
527                }
528            }
529        }
530
531        @Override
532        public void flush() throws IOException {
533            // Flush is performed in the <code>GzipWriter</code> close method.
534        }
535
536        @Override
537        public void write(int b) throws IOException {
538            singleByteArray[0] = (byte)b;
539            write(singleByteArray, 0, 1);
540        }
541
542        @Override
543        public void write(byte[] b) throws IOException {
544            write(b, 0, b.length);
545        }
546
547        @Override
548        public void write(byte[] b, int off, int len) throws IOException {
549            int pLen;
550            try {
551                while (len > 0) {
552                    if (bb.remaining() > 0) {
553                        pLen = Math.min(bb.remaining(), len);
554                        bb.put(b, off, pLen);
555                        off += pLen;
556                        len -= pLen;
557                    } else {
558                        bb.flip();
559                        int compressed = writer.readCompressed(bb, compressedBytes, 0, compressedBytes.length, false);
560                        bb.compact();
561                        if (compressed > 0) {
562                            writer.out.write(compressedBytes, 0, compressed);
563                        }
564                    }
565                }
566            } catch (DataFormatException e) {
567                throw new IOException("Deflater malfunction!", e);
568            }
569        }
570
571    }
572
573}