001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.arc; 019 020import java.io.ByteArrayOutputStream; 021import java.io.Closeable; 022import java.io.IOException; 023import java.io.InputStream; 024import java.io.OutputStream; 025 026import org.jwat.common.Diagnosis; 027import org.jwat.common.DiagnosisType; 028import org.jwat.common.Diagnostics; 029 030/** 031 * Base class for ARC writer implementations. 032 * 033 * Note: the content-length is synonymous to the archive-length. 034 * 035 * @author nicl 036 */ 037public abstract class ArcWriter implements Closeable { 038 039 /** State after writer has been constructed and before records have been written. */ 040 protected static final int S_INIT = 0; 041 042 /** State after header has been written. */ 043 protected static final int S_HEADER_WRITTEN = 1; 044 045 /** State after payload has been written. */ 046 protected static final int S_PAYLOAD_WRITTEN = 2; 047 048 /** State after record has been closed. */ 049 protected static final int S_RECORD_CLOSED = 3; 050 051 /* 052 * Settings. 053 */ 054 055 /** ARC field parser used. */ 056 protected ArcFieldParsers fieldParsers; 057 058 /** Buffer used by streamPayload() to copy from one stream to another. */ 059 protected byte[] stream_copy_buffer; 060 061 /** Configuration for throwing exception on content-length mismatch. 062 * (Default is true) */ 063 protected boolean bExceptionOnContentLengthMismatch; 064 065 /* 066 * State. 067 */ 068 069 /** Writer level errors and warnings or when writing byte headers. */ 070 public final Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>(); 071 072 /** Current state of writer. */ 073 protected int state = S_INIT; 074 075 /** Outputstream used to write ARC records. */ 076 protected OutputStream out; 077 078 protected ArcHeader header; 079 080 /** Content-Length from the ARC header. */ 081 protected Long headerContentLength; 082 083 /** Total bytes written for current record payload. */ 084 protected long payloadWrittenTotal; 085 086 /** 087 * Method used to initialize a readers internal state. 088 * Must be called by all constructors. 089 */ 090 protected void init() { 091 fieldParsers = new ArcFieldParsers(); 092 stream_copy_buffer = new byte[8192]; 093 bExceptionOnContentLengthMismatch = true; 094 } 095 096 /** 097 * Is this writer producing compressed output. 098 * @return boolean indicating whether compressed output is produced 099 */ 100 public abstract boolean isCompressed(); 101 102 /** 103 * Does this writer throw an exception if the content-length does not match 104 * the payload amount written. 105 * @return boolean indicating if an exception is thrown or not 106 */ 107 public boolean exceptionOnContentLengthMismatch() { 108 return bExceptionOnContentLengthMismatch; 109 } 110 111 /** 112 * Tell the writer what to do in case of mismatch between content-length 113 * and amount payload written. 114 * @param enabled boolean indicating exception throwing on/off 115 */ 116 public void setExceptionOnContentLengthMismatch(boolean enabled) { 117 bExceptionOnContentLengthMismatch = enabled; 118 } 119 120 /** 121 * Close ARC writer and free its resources. 122 * @throws IOException if an I/O exception occurs while closing the writer 123 */ 124 public abstract void close() throws IOException; 125 126 /** 127 * Close the ARC record in an implementation specific way. 128 * @throws IOException if an I/O exception occurs while closing the record 129 */ 130 public abstract void closeRecord() throws IOException; 131 132 /** 133 * Closes the ARC record by writing one newline and comparing the amount of 134 * payload data streamed with the content-length supplied with the header. 135 * @throws IOException if an I/O exception occurs while closing the record 136 */ 137 protected void closeRecord_impl() throws IOException { 138 Diagnosis diagnosis = null; 139 out.write(ArcConstants.endMark); 140 out.flush(); 141 if (headerContentLength == null) { 142 diagnosis = new Diagnosis( 143 DiagnosisType.ERROR_EXPECTED, 144 "'" + ArcConstants.FN_ARCHIVE_LENGTH + "' header", 145 "Mandatory!"); 146 } else { 147 if (headerContentLength != payloadWrittenTotal) { 148 diagnosis = new Diagnosis( 149 DiagnosisType.INVALID_EXPECTED, 150 "'" + ArcConstants.FN_ARCHIVE_LENGTH + "' header", 151 Long.toString(payloadWrittenTotal), 152 headerContentLength.toString()); 153 } 154 } 155 if (diagnosis != null) { 156 if (header != null) { 157 header.diagnostics.addError(diagnosis); 158 } else { 159 diagnostics.addError(diagnosis); 160 } 161 if (bExceptionOnContentLengthMismatch) { 162 throw new IllegalStateException("Payload size does not match content-length!"); 163 } 164 } 165 header = null; 166 headerContentLength = null; 167 } 168 169 /** 170 * Write a raw ARC header to the ARC output stream. Closes any previously 171 * written record that has not been closed prior to this call. 172 * Errors and warnings are reported on the writers diagnostics object. 173 * @param header_bytes raw ARC record to output 174 * @param contentLength the expected content-length to be written and validated 175 * The contentLength is synonymous to the Archive-Length. 176 * @throws IOException if an exception occurs while writing header data 177 */ 178 public void writeRawHeader(byte[] header_bytes, Long contentLength) throws IOException { 179 if (header_bytes == null) { 180 throw new IllegalArgumentException( 181 "The 'header_bytes' parameter is null!"); 182 } 183 if (contentLength != null && contentLength < 0) { 184 throw new IllegalArgumentException( 185 "The 'contentLength' parameter is negative!"); 186 } 187 if (state == S_HEADER_WRITTEN) { 188 throw new IllegalStateException("Headers written back to back!"); 189 } else if (state == S_PAYLOAD_WRITTEN) { 190 closeRecord_impl(); 191 } 192 out.write(header_bytes); 193 state = S_HEADER_WRITTEN; 194 header = null; 195 headerContentLength = contentLength; 196 payloadWrittenTotal = 0; 197 } 198 199 /** 200 * Write a ARC header to the ARC output stream. 201 * Errors and warnings are reported on the records diagnostics object. 202 * @param record ARC record to output 203 * @return byte array version of header as it was written 204 * @throws IOException if an I/O exception occurs while writing header data 205 */ 206 public abstract byte[] writeHeader(ArcRecordBase record) throws IOException; 207 208 /** 209 * Write an ARC header to the ARC output stream. 210 * The WARC header is not required to be valid. 211 * Errors and warnings are reported on the records diagnostics object. 212 * @param record ARC record to output 213 * @return byte array version of header as it was written 214 * @throws IOException if an I/O exception occurs while writing header data 215 */ 216 protected byte[] writeHeader_impl(ArcRecordBase record) throws IOException { 217 header = record.header; 218 headerContentLength = header.archiveLength; 219 if (headerContentLength == null && header.archiveLengthStr != null) { 220 headerContentLength = fieldParsers.parseLong(header.archiveLengthStr, ArcConstants.FN_ARCHIVE_LENGTH, false); 221 if (headerContentLength != null && headerContentLength < 0) { 222 diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_ARCHIVE_LENGTH + "' value", header.archiveLengthStr, "A non negative number")); 223 } 224 } 225 ByteArrayOutputStream outBuf = new ByteArrayOutputStream(); 226 /* 227 * URL 228 */ 229 String urlStr; 230 if (header.urlUri != null) { 231 urlStr = header.urlUri.toString(); 232 } else if (header.urlStr != null && header.urlStr.length() > 0) { 233 urlStr = header.urlStr; 234 } else { 235 urlStr = "-"; 236 } 237 outBuf.write(urlStr.getBytes()); 238 /* 239 * IP-Address 240 */ 241 String ipAddressStr; 242 if (header.inetAddress != null) { 243 ipAddressStr = header.inetAddress.getHostAddress(); 244 } else if (header.ipAddressStr != null && header.ipAddressStr.length() > 0) { 245 ipAddressStr = header.ipAddressStr; 246 } else { 247 ipAddressStr = "-"; 248 } 249 outBuf.write(" ".getBytes()); 250 outBuf.write(ipAddressStr.getBytes()); 251 /* 252 * Archive-Date 253 */ 254 String archiveDateStr; 255 if (header.archiveDate != null) { 256 archiveDateStr = ArcDateParser.getDateFormat().format(header.archiveDate); 257 } else if (header.archiveDateStr != null && header.archiveDateStr.length() > 0) { 258 archiveDateStr = header.archiveDateStr; 259 } else { 260 archiveDateStr = "-"; 261 } 262 outBuf.write(" ".getBytes()); 263 outBuf.write(archiveDateStr.getBytes()); 264 /* 265 * Content-Type 266 */ 267 String contentTypeStr; 268 if (header.contentType != null) { 269 contentTypeStr = header.contentType.toStringShort(); 270 } else if (header.contentTypeStr != null && header.contentTypeStr.length() > 0) { 271 contentTypeStr = header.contentTypeStr; 272 } else { 273 contentTypeStr = "-"; 274 } 275 outBuf.write(" ".getBytes()); 276 outBuf.write(contentTypeStr.getBytes()); 277 /* 278 * Version 2 fields. 279 */ 280 if (header.recordFieldVersion == 2) { 281 /* 282 * Result-Code 283 */ 284 String resultCodeStr; 285 if (header.resultCode != null) { 286 resultCodeStr = header.resultCode.toString(); 287 } else if (header.resultCodeStr != null && header.resultCodeStr.length() > 0) { 288 resultCodeStr = header.resultCodeStr; 289 } else { 290 resultCodeStr = "-"; 291 } 292 outBuf.write(" ".getBytes()); 293 outBuf.write(resultCodeStr.getBytes()); 294 /* 295 * Checksum 296 */ 297 String checksumStr; 298 if (header.checksumStr != null && header.checksumStr.length() > 0) { 299 checksumStr = header.checksumStr; 300 } else { 301 checksumStr = "-"; 302 } 303 outBuf.write(" ".getBytes()); 304 outBuf.write(checksumStr.getBytes()); 305 /* 306 * Location 307 */ 308 String locationStr; 309 if (header.locationStr != null && header.locationStr.length() > 0) { 310 locationStr = header.locationStr; 311 } else { 312 locationStr = "-"; 313 } 314 outBuf.write(" ".getBytes()); 315 outBuf.write(locationStr.getBytes()); 316 /* 317 * Offset 318 */ 319 String offsetStr; 320 if (header.offset != null) { 321 offsetStr = header.offset.toString(); 322 } else if (header.offsetStr != null && header.offsetStr.length() > 0) { 323 offsetStr = header.offsetStr; 324 } else { 325 offsetStr = "-"; 326 } 327 outBuf.write(" ".getBytes()); 328 outBuf.write(offsetStr.getBytes()); 329 /* 330 * Filename 331 */ 332 String filenameStr; 333 if (header.filenameStr != null && header.filenameStr.length() > 0) { 334 filenameStr = header.filenameStr; 335 } else { 336 filenameStr = "-"; 337 } 338 outBuf.write(" ".getBytes()); 339 outBuf.write(filenameStr.getBytes()); 340 } 341 /* 342 * Archive-Length 343 */ 344 String archiveLengthStr; 345 if (header.archiveLength != null) { 346 archiveLengthStr = header.archiveLength.toString(); 347 } else if (header.archiveLengthStr != null && header.archiveLengthStr.length() > 0) { 348 archiveLengthStr = header.archiveLengthStr; 349 } else { 350 archiveLengthStr = "-"; 351 } 352 outBuf.write(" ".getBytes()); 353 outBuf.write(archiveLengthStr.getBytes()); 354 outBuf.write("\n".getBytes()); 355 /* 356 * End Of Header 357 */ 358 byte[] headerBytes = outBuf.toByteArray(); 359 out.write(headerBytes); 360 state = S_HEADER_WRITTEN; 361 payloadWrittenTotal = 0; 362 return headerBytes; 363 } 364 365 /** 366 * Stream the content of an input stream to the payload content. 367 * @param in input stream containing payload data 368 * @return number of bytes written during method invocation 369 * @throws IOException if an I/O exception occurs while writing payload data 370 */ 371 public long streamPayload(InputStream in) throws IOException { 372 if (in == null) { 373 throw new IllegalArgumentException( 374 "The 'in' parameter is null!"); 375 } 376 if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) { 377 throw new IllegalStateException("Write a header before writing payload!"); 378 } 379 long written = 0; 380 int read = 0; 381 while (read != -1) { 382 out.write(stream_copy_buffer, 0, read); 383 written += read; 384 read = in.read(stream_copy_buffer); 385 } 386 state = S_PAYLOAD_WRITTEN; 387 payloadWrittenTotal += written; 388 return written; 389 } 390 391 /** 392 * Append the content of a byte array to the payload content. 393 * @param b byte array with data to be written 394 * @return number of bytes written during method invocation 395 * @throws IOException if an I/O exception occurs while writing payload data 396 */ 397 public long writePayload(byte[] b) throws IOException { 398 if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) { 399 throw new IllegalStateException("Write a header before writing payload!"); 400 } 401 out.write(b); 402 state = S_PAYLOAD_WRITTEN; 403 payloadWrittenTotal += b.length; 404 return b.length; 405 } 406 407 /** 408 * Append the partial content of a byte array to the payload content. 409 * @param b byte array with partial data to be written 410 * @param offset offset to data to be written 411 * @param len length of data to be written 412 * @return number of bytes written during method invocation 413 * @throws IOException if an I/O exception occurs while writing payload data 414 */ 415 public long writePayload(byte[] b, int offset, int len) throws IOException { 416 if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) { 417 throw new IllegalStateException("Write a header before writing payload!"); 418 } 419 out.write(b, offset, len); 420 state = S_PAYLOAD_WRITTEN; 421 payloadWrittenTotal += len; 422 return len; 423 } 424 425}