001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.gzip; 019 020import org.jwat.common.Diagnosis; 021import org.jwat.common.DiagnosisType; 022import org.jwat.common.ISO8859_1; 023 024import java.io.BufferedOutputStream; 025import java.io.Closeable; 026import java.io.IOException; 027import java.io.InputStream; 028import java.io.OutputStream; 029import java.nio.ByteBuffer; 030import java.util.Date; 031import java.util.zip.CRC32; 032import java.util.zip.DataFormatException; 033import java.util.zip.Deflater; 034 035/** 036 * A writer for (multi-part) GZip files. 037 * As per the GZIP file format specification version 4.3. 038 * The class writes a GZip header. The compressed data and trailer is also 039 * written by this class but is instigated by calling the write method on the 040 * GZip entry itself. 041 * 042 * @author nicl 043 */ 044public class GzipWriter implements Closeable { 045 046 /** Default input buffer size. */ 047 public static final int DEFAULT_INPUT_BUFFER_SIZE = 8192; 048 049 /** Output stream for GZip (multi-part) file. */ 050 protected OutputStream out; 051 /** Deflater used to compress GZip entries. */ 052 protected Deflater def = new Deflater(Deflater.DEFAULT_COMPRESSION, true); 053 /** Checksum object used to calculate CRC16 and CRC32 values. */ 054 protected CRC32 crc = new CRC32(); 055 /** Input buffer used to feed the deflater. */ 056 protected byte[] inputBytes; 057 058 /** Compression level to be used by deflater. */ 059 protected int compressionLevel = Deflater.DEFAULT_COMPRESSION; 060 061 /** ISO-8859-1 validating de-/encoder. */ 062 protected final ISO8859_1 iso8859_1 = new ISO8859_1(); 063 064 /** Compliance status for records parsed up to now. */ 065 protected boolean bIsCompliant = true; 066 067 /** Current GZip entry object. */ 068 protected GzipEntry gzipEntry; 069 070 /** Buffer used to read header. */ 071 byte[] headerBytes = new byte[10]; 072 /** Buffer used to read the XLEN value. */ 073 byte[] xlenBytes = new byte[2]; 074 /** Buffer used to read the FNAME data. */ 075 byte[] fnameBytes = null; 076 /** Buffer used to read the FCOMMENT data. */ 077 byte[] fcommentBytes = null; 078 /** Buffer used to read the CRC16 value. */ 079 byte[] crc16Bytes = new byte[2]; 080 /** Buffer used to read trailer. */ 081 byte[] trailerBytes = new byte[8]; 082 083 /** 084 * Construct a GZip writer with a default input buffer size of 1024. 085 * @param out output stream of GZip file 086 */ 087 public GzipWriter(OutputStream out) { 088 if (out == null) { 089 throw new IllegalArgumentException("out is null!"); 090 } 091 this.out = new BufferedOutputStream(out, DEFAULT_INPUT_BUFFER_SIZE); 092 inputBytes = new byte[DEFAULT_INPUT_BUFFER_SIZE]; 093 } 094 095 /** 096 * Construct a GZip writer with the specified input buffer size. 097 * @param out output stream of GZip file 098 * @param buffer_size input buffer size to use 099 */ 100 public GzipWriter(OutputStream out, int buffer_size) { 101 if (out == null) { 102 throw new IllegalArgumentException("out is null!"); 103 } 104 if (buffer_size <= 0) { 105 throw new IllegalArgumentException("buffer_size is less or equals to zero!"); 106 } 107 this.out = new BufferedOutputStream(out, buffer_size); 108 inputBytes = new byte[buffer_size]; 109 } 110 111 /** 112 * Release resources associated with this writer. 113 * @throws IOException if an I/O error occurs while closing writer 114 */ 115 public void close() throws IOException { 116 if (gzipEntry != null) { 117 gzipEntry.close(); 118 gzipEntry = null; 119 } 120 if (out != null) { 121 out.flush(); 122 out = null; 123 } 124 if (def != null) { 125 def.end(); 126 def = null; 127 } 128 } 129 130 /** 131 * Returns a boolean indicating whether all entries written so far are compliant. 132 * @return a boolean indicating whether all entries written so far are compliant 133 */ 134 public boolean isCompliant() { 135 return bIsCompliant; 136 } 137 138 /** 139 * Set compression level used by deflater. Only changed in deflater prior 140 * to writing an entry header. 141 * @param compressionLevel compression level 142 * @throws IllegalArgumentException If CompressionLevel is invalid 143 */ 144 public void setCompressionLevel(int compressionLevel) { 145 if (compressionLevel == Deflater.DEFAULT_COMPRESSION 146 || compressionLevel == Deflater.NO_COMPRESSION 147 || (compressionLevel >= 1 && compressionLevel <= 9)) { 148 this.compressionLevel = compressionLevel; 149 } 150 else { 151 throw new IllegalArgumentException("Invalid compression level: " + compressionLevel); 152 } 153 } 154 155 /** 156 * Returns current compression level used by deflater. 157 * @return current compression level 158 */ 159 public int getCompressionLevel() { 160 return compressionLevel; 161 } 162 163 /** 164 * Write a GZip entry header and prepare for compressing input data. 165 * @param entry GZip entry object 166 * @throws IOException if an I/O error occurs while writing header 167 */ 168 public void writeEntryHeader(GzipEntry entry) throws IOException { 169 if (gzipEntry != null) { 170 gzipEntry.close(); 171 gzipEntry = null; 172 } 173 if (entry == null) { 174 throw new IllegalArgumentException("entry is null!"); 175 } 176 crc.reset(); 177 def.reset(); 178 def.setLevel(compressionLevel); 179 gzipEntry = entry; 180 /* 181 * Header. 182 */ 183 entry.magic = GzipConstants.GZIP_MAGIC; 184 if (entry.date != null) { 185 entry.mtime = entry.date.getTime() / 1000; 186 } else if (entry.mtime != 0) { 187 entry.date = new Date(entry.mtime * 1000); 188 } 189 entry.xfl = 0; 190 if (compressionLevel == 1) { 191 entry.xfl |= GzipConstants.DEFLATE_XFL_FASTEST_COMPRESSION; 192 } else if (compressionLevel == 9) { 193 entry.xfl |= GzipConstants.DEFLATE_XFL_MAXIMUM_COMPRESSION; 194 } 195 entry.flg = 0; 196 if (!GzipConstants.osIdxStr.containsKey((int)entry.os)) { 197 entry.diagnostics.addWarning( 198 new Diagnosis( 199 DiagnosisType.UNKNOWN, 200 "Operating System", 201 Integer.toString(entry.os) 202 ) 203 ); 204 } 205 /* 206 * FTEXT. 207 */ 208 if (entry.bFText) { 209 entry.flg |= GzipConstants.FLG_FTEXT; 210 } 211 /* 212 * FEXTRA. 213 */ 214 if (entry.extraBytes == null) { 215 if (entry.extraData.size() > 0) { 216 int xlen = 0; 217 for (int i=0; i<entry.extraData.size(); ++i) { 218 xlen += 4 + entry.extraData.get(i).data.length; 219 } 220 entry.extraBytes = new byte[xlen]; 221 GzipExtraData extraData; 222 int idx = 0; 223 for (int i=0; i<entry.extraData.size(); ++i) { 224 extraData = entry.extraData.get(i); 225 entry.extraBytes[idx++] = extraData.si1; 226 entry.extraBytes[idx++] = extraData.si2; 227 entry.extraBytes[idx++] = (byte)(extraData.data.length & 255); 228 entry.extraBytes[idx++] = (byte)((extraData.data.length >> 8) & 255); 229 System.arraycopy(extraData.data, 0, entry.extraBytes, idx, extraData.data.length); 230 idx += extraData.data.length; 231 } 232 } 233 } else { 234 int idx = 0; 235 boolean b = true; 236 int len; 237 while (b) { 238 if (idx <= entry.extraBytes.length - 4) { 239 idx += 2; 240 len = ((entry.extraBytes[idx + 1] & 255) << 8) | (entry.extraBytes[idx] & 255); 241 idx += 2; 242 if (idx + len <= entry.extraBytes.length) { 243 idx += len; 244 } else { 245 b = false; 246 } 247 } else { 248 b = false; 249 } 250 } 251 if (idx != gzipEntry.extraBytes.length) { 252 gzipEntry.diagnostics.addError( 253 new Diagnosis( 254 DiagnosisType.INVALID_DATA, 255 "FEXTRA", 256 "Invalid structure", 257 "Data truncated" 258 ) 259 ); 260 } 261 } 262 if (entry.extraBytes != null) { 263 entry.flg |= GzipConstants.FLG_FEXTRA; 264 entry.xlen = entry.extraBytes.length; 265 xlenBytes[0] = (byte)(entry.xlen & 255); 266 xlenBytes[1] = (byte)((entry.xlen >> 8) & 255); 267 } 268 /* 269 * FNAME. 270 */ 271 if (entry.fname != null) { 272 entry.flg |= GzipConstants.FLG_FNAME; 273 if (!iso8859_1.encode(entry.fname, "")) { 274 entry.diagnostics.addWarning( 275 new Diagnosis( 276 DiagnosisType.INVALID_ENCODING, 277 "FName", 278 entry.fname, 279 "ISO-8859-1" 280 ) 281 ); 282 } 283 entry.fname = iso8859_1.decoded; 284 fnameBytes = iso8859_1.encoded; 285 } 286 /* 287 * FCOMMENT. 288 */ 289 if (entry.fcomment != null) { 290 entry.flg |= GzipConstants.FLG_FCOMMENT; 291 if (!iso8859_1.encode(entry.fcomment, "\n")) { 292 entry.diagnostics.addWarning( 293 new Diagnosis( 294 DiagnosisType.INVALID_ENCODING, 295 "FComment", 296 entry.fcomment, 297 "ISO-8859-1" 298 ) 299 ); 300 } 301 entry.fcomment = iso8859_1.decoded; 302 fcommentBytes = iso8859_1.encoded; 303 } 304 /* 305 * FHCRC. 306 */ 307 if (entry.bFhCrc) { 308 entry.flg |= GzipConstants.FLG_FHCRC; 309 } 310 /* 311 * Header. 312 */ 313 headerBytes[0] = (byte)(entry.magic & 255); 314 headerBytes[1] = (byte)((entry.magic >> 8) & 255); 315 headerBytes[2] = (byte)entry.cm; 316 headerBytes[3] = (byte)entry.flg; 317 headerBytes[4] = (byte)(entry.mtime & 255); 318 headerBytes[5] = (byte)((entry.mtime >> 8) & 255); 319 headerBytes[6] = (byte)((entry.mtime >> 16) & 255); 320 headerBytes[7] = (byte)((entry.mtime >> 24) & 255); 321 headerBytes[8] = (byte)entry.xfl; 322 headerBytes[9] = (byte)entry.os; 323 out.write(headerBytes); 324 crc.update(headerBytes); 325 if ((entry.flg & GzipConstants.FLG_FEXTRA) == GzipConstants.FLG_FEXTRA) { 326 out.write(xlenBytes); 327 out.write(entry.extraBytes); 328 crc.update(xlenBytes); 329 crc.update(entry.extraBytes); 330 } 331 if ((entry.flg & GzipConstants.FLG_FNAME) == GzipConstants.FLG_FNAME) { 332 out.write(fnameBytes); 333 out.write(0); 334 crc.update(fnameBytes); 335 crc.update(0); 336 } 337 if ((entry.flg & GzipConstants.FLG_FCOMMENT) == GzipConstants.FLG_FCOMMENT) { 338 out.write(fcommentBytes); 339 out.write(0); 340 crc.update(fcommentBytes); 341 crc.update(0); 342 } 343 if ((entry.flg & GzipConstants.FLG_FHCRC) == GzipConstants.FLG_FHCRC) { 344 entry.comp_crc16 = ((int)crc.getValue()) & 0x0000ffff; 345 entry.crc16 = entry.comp_crc16; 346 crc16Bytes[0] = (byte)(entry.crc16 & 255); 347 crc16Bytes[1] = (byte)((entry.crc16 >> 8) & 255); 348 out.write(crc16Bytes); 349 } 350 /* 351 * Prepare Entry InputStream. 352 */ 353 crc.reset(); 354 entry.isize = 0; 355 entry.writer = this; 356 entry.bEof = false; 357 entry.out = new GzipEntryOutputStream(this, gzipEntry); 358 // Compliance 359 if (entry.diagnostics.hasErrors() || entry.diagnostics.hasWarnings()) { 360 entry.bIsCompliant = false; 361 } else { 362 entry.bIsCompliant = true; 363 } 364 bIsCompliant &= entry.bIsCompliant; 365 } 366 367 /** 368 * Computes the CRC32 and iSize values and then writes the GZip entry trailer. 369 * @param entry GZip entry object 370 * @throws IOException if an I/O error occurs while writing trailer 371 */ 372 protected void writeTrailer(GzipEntry entry) throws IOException { 373 // Compliance 374 if (entry.diagnostics.hasErrors() || entry.diagnostics.hasWarnings()) { 375 entry.bIsCompliant = false; 376 } else { 377 entry.bIsCompliant = true; 378 } 379 bIsCompliant &= gzipEntry.bIsCompliant; 380 // Trailer 381 entry.uncompressed_size = def.getBytesRead(); 382 entry.compressed_size = def.getBytesWritten(); 383 entry.comp_crc32 = (int)(crc.getValue() & 0xffffffff); 384 entry.crc32 = entry.comp_crc32; 385 entry.comp_isize = (int)(def.getBytesRead() & 0xffffffff); 386 entry.isize = entry.comp_isize; 387 trailerBytes[0] = (byte)(entry.crc32 & 255); 388 trailerBytes[1] = (byte)((entry.crc32 >> 8) & 255); 389 trailerBytes[2] = (byte)((entry.crc32 >> 16) & 255); 390 trailerBytes[3] = (byte)((entry.crc32 >> 24) & 255); 391 trailerBytes[4] = (byte)(entry.isize & 255); 392 trailerBytes[5] = (byte)((entry.isize >> 8) & 255); 393 trailerBytes[6] = (byte)((entry.isize >> 16) & 255); 394 trailerBytes[7] = (byte)((entry.isize >> 24) & 255); 395 out.write(trailerBytes); 396 out.flush(); 397 } 398 399 /** 400 * Read data from input stream and fill buffer with compressed data. 401 * @param in input stream with uncompressed data 402 * @param b compressed data buffer 403 * @param off offset in compressed data buffer 404 * @param len length of compressed data to read 405 * @return number of compressed bytes read 406 * @throws DataFormatException if an error occurs in deflater 407 * @throws IOException if an I/O error occurs while compressing 408 */ 409 protected int readCompressed(InputStream in, byte[] b, int off, int len) throws DataFormatException, IOException { 410 int deflated = 0; 411 while ((deflated = def.deflate(b, off, len)) == 0) { 412 if (def.finished()) { 413 return -1; 414 } else if (def.needsInput()) { 415 int read = in.read(inputBytes, 0, inputBytes.length); 416 if (read != -1) { 417 def.setInput(inputBytes, 0, read); 418 crc.update(inputBytes, 0, read); 419 } else { 420 def.finish(); 421 } 422 } else { 423 throw new DataFormatException("Deflater malfunction!"); 424 } 425 } 426 return deflated; 427 } 428 429 /** 430 * Read data from <code>ByteBuffer</code> and fill buffer with compressed 431 * data. 432 * @param bb <code>ByteBuffer</code> with uncompressed data 433 * @param b compressed data buffer 434 * @param off offset in compressed data buffer 435 * @param len length of compressed data to read 436 * @param bFinish boolean indicating that there is no more data to compress 437 * @return number of compressed bytes read 438 * @throws DataFormatException if an error occurs in deflater 439 * @throws IOException if an error occurs while compressing 440 */ 441 protected int readCompressed(ByteBuffer bb, byte[] b, int off, int len, boolean bFinish) throws DataFormatException, IOException { 442 int deflated = 0; 443 while ((deflated = def.deflate(b, off, len)) == 0) { 444 if (def.finished()) { 445 return -1; 446 } else if (def.needsInput()) { 447 int write = bb.remaining(); 448 if (write > 0) { 449 if (write > inputBytes.length) { 450 write = inputBytes.length; 451 } 452 bb.get(inputBytes, 0, write); 453 def.setInput(inputBytes, 0, write); 454 crc.update(inputBytes, 0, write); 455 } else { 456 if (bFinish) { 457 def.finish(); 458 } else { 459 // Deflater needs more input. 460 return 0; 461 } 462 } 463 } else { 464 throw new DataFormatException("Deflater malfunction!"); 465 } 466 } 467 return deflated; 468 } 469 470 /** 471 * <code>OutputStream</code> to GZip compress data in a controlled fashion. 472 * 473 * @author nicl 474 */ 475 protected static class GzipEntryOutputStream extends OutputStream { 476 477 /** GZip reader used to inflate. */ 478 GzipWriter writer; 479 480 /** Associated GZip entry. */ 481 GzipEntry gzipEntry; 482 483 /** End of compressed file status. */ 484 boolean bEof = false; 485 486 /** Small buffer used by the read() method. */ 487 byte[] singleByteArray = new byte[1]; 488 489 /** Buffer for feeding the deflater. */ 490 ByteBuffer bb = ByteBuffer.allocate(DEFAULT_INPUT_BUFFER_SIZE); 491 492 /** Buffer used for compressed data. */ 493 byte[] compressedBytes = new byte[DEFAULT_INPUT_BUFFER_SIZE]; 494 495 /** 496 * Construct output stream bound to a specific writer and entry. 497 * @param writer GZip writer 498 * @param gzipEntry GZip entry 499 */ 500 public GzipEntryOutputStream(GzipWriter writer, 501 GzipEntry gzipEntry) { 502 this.writer = writer; 503 this.gzipEntry = gzipEntry; 504 } 505 506 @Override 507 public void close() throws IOException { 508 if (!bEof) { 509 bEof = true; 510 try { 511 int deflated = 0; 512 while (deflated != -1) { 513 bb.flip(); 514 deflated = writer.readCompressed(bb, compressedBytes, 0, compressedBytes.length, true); 515 bb.compact(); 516 if (deflated > 0) { 517 writer.out.write(compressedBytes, 0, deflated); 518 } 519 } 520 writer.writeTrailer(gzipEntry); 521 } catch (DataFormatException e) { 522 throw new IOException("Deflater malfunction!", e); 523 } finally { 524 writer = null; 525 gzipEntry = null; 526 singleByteArray = null; 527 } 528 } 529 } 530 531 @Override 532 public void flush() throws IOException { 533 // Flush is performed in the <code>GzipWriter</code> close method. 534 } 535 536 @Override 537 public void write(int b) throws IOException { 538 singleByteArray[0] = (byte)b; 539 write(singleByteArray, 0, 1); 540 } 541 542 @Override 543 public void write(byte[] b) throws IOException { 544 write(b, 0, b.length); 545 } 546 547 @Override 548 public void write(byte[] b, int off, int len) throws IOException { 549 int pLen; 550 try { 551 while (len > 0) { 552 if (bb.remaining() > 0) { 553 pLen = Math.min(bb.remaining(), len); 554 bb.put(b, off, pLen); 555 off += pLen; 556 len -= pLen; 557 } else { 558 bb.flip(); 559 int compressed = writer.readCompressed(bb, compressedBytes, 0, compressedBytes.length, false); 560 bb.compact(); 561 if (compressed > 0) { 562 writer.out.write(compressedBytes, 0, compressed); 563 } 564 } 565 } 566 } catch (DataFormatException e) { 567 throw new IOException("Deflater malfunction!", e); 568 } 569 } 570 571 } 572 573}