001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.io.Closeable; 021import java.io.IOException; 022import java.io.InputStream; 023import java.util.Arrays; 024import java.util.Collections; 025import java.util.List; 026 027import org.jwat.common.Base16; 028import org.jwat.common.Base32; 029import org.jwat.common.Base64; 030import org.jwat.common.ByteCountingPushBackInputStream; 031import org.jwat.common.Diagnosis; 032import org.jwat.common.DiagnosisType; 033import org.jwat.common.Diagnostics; 034import org.jwat.common.HeaderLine; 035import org.jwat.common.HttpHeader; 036import org.jwat.common.NewlineParser; 037import org.jwat.common.Payload; 038import org.jwat.common.PayloadOnClosedHandler; 039 040/** 041 * This class represents a parsed WARC record header block including 042 * possible validation and format warnings/errors encountered in the process. 043 * The payload of the WARC record is accessible through a wrapped payload 044 * object. 045 * 046 * @author nicl 047 */ 048public class WarcRecord implements PayloadOnClosedHandler, Closeable { 049 050 /** Reader instance used, required for file compliance. */ 051 protected WarcReader reader; 052 053 /** Input stream used to read this record. */ 054 protected ByteCountingPushBackInputStream in; 055 056 /** Is this record compliant ie. error free. */ 057 protected boolean bIsCompliant; 058 059 /** WARC record parsing start offset relative to the source WARC file input 060 * stream. Used to keep track of the uncompressed amount of bytes consumed. */ 061 protected long startOffset = -1; 062 063 /** Uncompressed bytes consumed while validating this record. */ 064 protected long consumed; 065 066 /** Validation errors and warnings. */ 067 public final Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>(); 068 069 /** Newline parser for counting/validating trailing newlines. */ 070 public NewlineParser nlp = new NewlineParser(); 071 072 /** Is Warc-Block-Digest valid. (Null is equal to not tested) */ 073 public Boolean isValidBlockDigest = null; 074 075 /** Is Warc-Payload-Digest valid. (Null is equal to not tested) */ 076 public Boolean isValidPayloadDigest = null; 077 078 /** Number of trailing newlines after record. */ 079 public int trailingNewlines; 080 081 /* 082 * Header-Fields. 083 */ 084 085 /** WARC header. */ 086 public WarcHeader header; 087 088 /* 089 * Payload 090 */ 091 092 /** Has payload been closed before. */ 093 protected boolean bPayloadClosed; 094 095 /** Has record been closed before. */ 096 protected boolean bClosed; 097 098 /** Payload object if any exists. */ 099 protected Payload payload; 100 101 /** HTTP header content parsed from payload. */ 102 protected HttpHeader httpHeader; 103 104 /** Computed block digest. */ 105 public WarcDigest computedBlockDigest; 106 107 /** Computed payload digest. */ 108 public WarcDigest computedPayloadDigest; 109 110 /** 111 * Non public constructor to allow unit testing. 112 */ 113 protected WarcRecord() { 114 } 115 116 /** 117 * Create a <code>WarcRecord</code> and prepare it for writing. 118 * @param writer writer which will be used to write the record 119 * @return a <code>WarcRecord</code> ready to be changed and then written 120 */ 121 public static WarcRecord createRecord(WarcWriter writer) { 122 WarcRecord record = new WarcRecord(); 123 record.header = WarcHeader.initHeader(writer, record.diagnostics); 124 writer.fieldParsers.diagnostics = record.diagnostics; 125 return record; 126 } 127 128 /** 129 * Given an <code>InputStream</code> it tries to read and validate a WARC 130 * header block. 131 * @param in <code>InputStream</code> containing WARC record data 132 * @param reader <code>WarcReader</code> used, with access to user defined 133 * options 134 * @return <code>WarcRecord</code> or <code>null</code> 135 * @throws IOException I/O exception in the process of reading record 136 */ 137 public static WarcRecord parseRecord(ByteCountingPushBackInputStream in, 138 WarcReader reader) throws IOException { 139 WarcRecord record = new WarcRecord(); 140 record.in = in; 141 record.reader = reader; 142 record.startOffset = in.getConsumed(); 143 // Initialize WarcHeader with required context. 144 record.header = WarcHeader.initHeader(reader, in.getConsumed(), record.diagnostics); 145 WarcHeader header = record.header; 146 // Initialize WarcFieldParser to report diagnoses here. 147 reader.fieldParsers.diagnostics = record.diagnostics; 148 if (header.parseHeader(in)) { 149 ++reader.records; 150 if (reader.wrpCallback != null) { 151 reader.wrpCallback.warcParsedRecordHeader(reader, record.startOffset, header); 152 } 153 /* 154 * Payload processing. 155 */ 156 if (header.contentLength != null && header.contentLength > 0) { 157 /* 158 * Payload. 159 */ 160 String digestAlgorithm = null; 161 if (reader.bBlockDigest) { 162 if (header.warcBlockDigest != null && header.warcBlockDigest.algorithm != null) { 163 // If a WARC block digest header is present in the 164 // record, use that algorithm. 165 digestAlgorithm = header.warcBlockDigest.algorithm; 166 } else { 167 // If no WARC block digest header is present, 168 // use the optional user specified algorithm. 169 // Can be null in which case nothing is computed. 170 digestAlgorithm = reader.blockDigestAlgorithm; 171 } 172 } 173 record.payload = Payload.processPayload(in, header.contentLength, 174 reader.payloadHeaderMaxSize, digestAlgorithm); 175 record.payload.setOnClosedHandler(record); 176 /* 177 * HttpHeader. 178 */ 179 if (header.contentType != null 180 && header.contentType.contentType.equals("application") 181 && header.contentType.mediaType.equals("http")) { 182 String value = header.contentType.getParameter("msgtype"); 183 // request 184 int httpHeaderType = 0; 185 if ("response".equalsIgnoreCase(value)) { 186 httpHeaderType = HttpHeader.HT_RESPONSE; 187 } else if ("request".equalsIgnoreCase(value)) { 188 httpHeaderType = HttpHeader.HT_REQUEST; 189 } 190 if (httpHeaderType != 0) { 191 digestAlgorithm = null; 192 if (reader.bPayloadDigest) { 193 if (header.warcPayloadDigest != null && header.warcPayloadDigest.algorithm != null) { 194 // If a WARC payload digest header is present in the 195 // record, use that algorithm. 196 digestAlgorithm = header.warcPayloadDigest.algorithm; 197 } else { 198 // If no WARC payload digest header is present, 199 // use the optional user specified algorithm. 200 // Can be null in which case nothing is computed. 201 digestAlgorithm = reader.payloadDigestAlgorithm; 202 } 203 } 204 // Try to read a valid HTTP request/response header from the payload. 205 record.httpHeader = HttpHeader.processPayload(httpHeaderType, 206 record.payload.getInputStream(), header.contentLength, 207 digestAlgorithm); 208 if (record.httpHeader != null) { 209 if (record.httpHeader.isValid()) { 210 record.payload.setPayloadHeaderWrapped(record.httpHeader); 211 } else { 212 record.diagnostics.addError( 213 new Diagnosis(DiagnosisType.ERROR, 214 "http header", 215 "Unable to parse http header!")); 216 } 217 } 218 } 219 } 220 } 221 // Preliminary compliance status, will be updated when the 222 // payload/record is closed. 223 if (record.diagnostics.hasErrors() || record.diagnostics.hasWarnings()) { 224 record.bIsCompliant = false; 225 } else { 226 record.bIsCompliant = true; 227 } 228 reader.bIsCompliant &= record.bIsCompliant; 229 } else { 230 // In case no record is found the errors/warnings in the record object are transfered to the Reader. 231 long excess = in.getConsumed() - record.startOffset; 232 reader.consumed += excess; 233 reader.diagnostics.addAll(record.diagnostics); 234 if (record.diagnostics.hasErrors() || record.diagnostics.hasWarnings()) { 235 reader.errors += record.diagnostics.getErrors().size(); 236 reader.warnings += record.diagnostics.getWarnings().size(); 237 reader.bIsCompliant = false; 238 } 239 // Require one or more records to be present. 240 if (reader.records == 0) { 241 reader.diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, "WARC file", "One or more records")); 242 ++reader.errors; 243 reader.bIsCompliant = false; 244 } 245 if (excess != 0) { 246 reader.diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA, "Trailing data", "Garbage data found at offset=" + record.startOffset + " - length=" + excess)); 247 } 248 // EOF 249 record = null; 250 } 251 return record; 252 } 253 254 /** 255 * Called when the payload object is closed and final steps in the 256 * validation process can be performed. 257 * @throws IOException I/O exception in final validation processing 258 */ 259 @Override 260 public void payloadClosed() throws IOException { 261 if (!bPayloadClosed) { 262 if (payload != null) { 263 // Check for truncated payload. 264 if (payload.getUnavailable() > 0) { 265 // Payload length mismatch - Payload truncated 266 addErrorDiagnosis(DiagnosisType.INVALID_DATA, "Payload length mismatch", "Payload truncated"); 267 } 268 /* 269 * Check block digest. 270 */ 271 byte[] digest = payload.getDigest(); 272 // Check for computed block digest. 273 if (digest != null) { 274 computedBlockDigest = new WarcDigest(); 275 computedBlockDigest.digestBytes = digest; 276 } 277 // Auto detect encoding used in WARC header. 278 if (header.warcBlockDigest != null && header.warcBlockDigest.digestString != null) { 279 isValidBlockDigest = processWarcDigest(header.warcBlockDigest, computedBlockDigest, "block"); 280 } 281 // Adjust information about computed block digest. 282 if (computedBlockDigest != null) { 283 processComputedDigest(computedBlockDigest, 284 reader.blockDigestAlgorithm, reader.blockDigestEncoding, "block"); 285 } 286 // Revisit payload digest refers to the original. Continuation payload digest in first record also refers to original. 287 if ((header.warcTypeIdx != null && header.warcTypeIdx != WarcConstants.RT_IDX_REVISIT && header.warcTypeIdx != WarcConstants.RT_IDX_CONTINUATION) && httpHeader != null && httpHeader.isValid()) { 288 /* 289 * Check payload digest. 290 */ 291 digest = httpHeader.getDigest(); 292 // Check for computed payload digest. 293 if (digest != null) { 294 computedPayloadDigest = new WarcDigest(); 295 computedPayloadDigest.digestBytes = digest; 296 } 297 // Auto detect encoding used in WARC header. 298 if (header.warcPayloadDigest != null && header.warcPayloadDigest.digestString != null ) { 299 isValidPayloadDigest = processWarcDigest(header.warcPayloadDigest, computedPayloadDigest, "payload"); 300 } 301 // Adjust information about computed payload digest. 302 if (computedPayloadDigest != null) { 303 processComputedDigest(computedPayloadDigest, 304 reader.payloadDigestAlgorithm, reader.payloadDigestEncoding, "payload"); 305 } 306 } 307 } 308 // Check for trailing newlines. 309 trailingNewlines = nlp.parseCRLFs(in, diagnostics); 310 if (trailingNewlines != WarcConstants.WARC_RECORD_TRAILING_NEWLINES) { 311 addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED, 312 "Trailing newlines", 313 Integer.toString(trailingNewlines), 314 Integer.toString(WarcConstants.WARC_RECORD_TRAILING_NEWLINES)); 315 } 316 // isCompliant status update. 317 if (diagnostics.hasErrors() || diagnostics.hasWarnings()) { 318 bIsCompliant = false; 319 reader.errors += diagnostics.getErrors().size(); 320 reader.warnings += diagnostics.getWarnings().size(); 321 } else { 322 bIsCompliant = true; 323 } 324 reader.bIsCompliant &= bIsCompliant; 325 // Updated consumed after payload has been consumed. 326 consumed = in.getConsumed() - startOffset; 327 // Don't not close payload again. 328 bPayloadClosed = true; 329 // Callback. 330 reader.recordClosed(); 331 } 332 } 333 334 /** 335 * Auto-detect encoding used in WARC digest header and compare it to the 336 * internal one, if it has been computed. 337 * @param warcDigest digest from WARC header 338 * @param computedDigest internally compute digest 339 * @param digestName used to identify the digest ("block" or "payload") 340 * @return WARC digest validity indication 341 */ 342 protected Boolean processWarcDigest(WarcDigest warcDigest, WarcDigest computedDigest, String digestName) { 343 byte[] digest; 344 Boolean isValidDigest = null; 345 int digestAlgorithmLength = WarcDigest.digestAlgorithmLength(warcDigest.algorithm); 346 digest = Base16.decodeToArray(warcDigest.digestString); 347 if (digest != null && digest.length == digestAlgorithmLength) { 348 warcDigest.digestBytes = digest; 349 warcDigest.encoding = "base16"; 350 } 351 if (warcDigest.digestBytes == null) { 352 digest = Base32.decodeToArray(warcDigest.digestString, true); 353 if (digest != null && digest.length == digestAlgorithmLength) { 354 warcDigest.digestBytes = digest; 355 warcDigest.encoding = "base32"; 356 } 357 if (warcDigest.digestBytes == null) { 358 digest = Base64.decodeToArray(warcDigest.digestString, true); 359 if (digest != null && digest.length == digestAlgorithmLength) { 360 warcDigest.digestBytes = digest; 361 warcDigest.encoding = "base64"; 362 } 363 } 364 } 365 if (warcDigest.encoding == null) { 366 // Encoding - Unrecognized block digest encoding scheme 367 addErrorDiagnosis(DiagnosisType.UNKNOWN, 368 "Record " + digestName + " digest encoding scheme", 369 warcDigest.digestString); 370 } 371 if (computedDigest != null) { 372 computedDigest.algorithm = warcDigest.algorithm; 373 computedDigest.encoding = warcDigest.encoding; 374 if (warcDigest.digestBytes != null) { 375 if (!Arrays.equals(computedDigest.digestBytes, warcDigest.digestBytes)) { 376 // Block digest - Computed block digest does not match 377 addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED, 378 "Incorrect " + digestName + " digest", 379 Base16.encodeArray(warcDigest.digestBytes), 380 Base16.encodeArray(computedDigest.digestBytes)); 381 isValidDigest = false; 382 } else { 383 isValidDigest = true; 384 } 385 } else { 386 isValidDigest = false; 387 } 388 } 389 return isValidDigest; 390 } 391 392 /** 393 * Adjust algorithm and encoding information about computed block digest. 394 * @param computedDigest internally compute digest 395 * @param digestAlgorithm default algorithm 396 * @param digestEncoding default encoding 397 * @param digestName used to identify the digest ("block" or "payload") 398 */ 399 protected void processComputedDigest(WarcDigest computedDigest, String digestAlgorithm, String digestEncoding, String digestName) { 400 if (computedDigest.algorithm == null) { 401 computedDigest.algorithm = digestAlgorithm; 402 } 403 if (computedDigest.encoding == null && digestEncoding != null) { 404 if ("base32".equals(digestEncoding)) { 405 computedDigest.encoding = "base32"; 406 } else if ("base64".equals(digestEncoding)) { 407 computedDigest.encoding = "base64"; 408 } else if ("base16".equals(digestEncoding)) { 409 computedDigest.encoding = "base16"; 410 } else { 411 // Encoding - Unknown block digest encoding scheme .. 412 addErrorDiagnosis(DiagnosisType.UNKNOWN, 413 "Default " + digestName + " digest encoding scheme", 414 digestEncoding); 415 } 416 } 417 if (computedDigest.encoding != null) { 418 if ("base32".equals(computedDigest.encoding)) { 419 computedDigest.digestString = Base32.encodeArray(computedDigest.digestBytes); 420 } else if ("base64".equals(computedDigest.encoding)) { 421 computedDigest.digestString = Base64.encodeArray(computedDigest.digestBytes); 422 } else if ("base16".equals(computedDigest.encoding)) { 423 computedDigest.digestString = Base16.encodeArray(computedDigest.digestBytes); 424 } 425 } 426 } 427 428 /** 429 * Check to see if the record has been closed. 430 * @return boolean indicating whether this record is closed or not 431 */ 432 public boolean isClosed() { 433 return bClosed; 434 } 435 436 /** 437 * Close resources associated with the WARC record. 438 * Mainly payload stream if any. 439 * @throws IOException if unable to close resources 440 */ 441 public void close() throws IOException { 442 if (!bClosed) { 443 // Ensure input stream is at the end of the record payload. 444 if (payload != null) { 445 payload.close(); 446 } 447 payloadClosed(); 448 reader = null; 449 in = null; 450 bClosed = true; 451 } 452 } 453 454 /** 455 * Returns a boolean indicating the ISO compliance status of this record. 456 * @return a boolean indicating the ISO compliance status of this record 457 */ 458 public boolean isCompliant() { 459 return bIsCompliant; 460 } 461 462 /** 463 * Get the record offset relative to the start of the WARC file 464 * <code>InputStream</code>. 465 * @return the record offset relative to the start of the WARC file 466 */ 467 public long getStartOffset() { 468 return header.startOffset; 469 } 470 471 /** 472 * Return number of uncompressed bytes consumed validating this record. 473 * @return number of uncompressed bytes consumed validating this record 474 */ 475 public long getConsumed() { 476 return consumed; 477 } 478 479 /** 480 * Get a <code>List</code> of all the non-standard WARC headers found 481 * during parsing. 482 * @return <code>List</code> of <code>HeaderLine</code> 483 */ 484 public List<HeaderLine> getHeaderList() { 485 return Collections.unmodifiableList(header.headerList); 486 } 487 488 /** 489 * Get a non-standard WARC header or null, if nothing is stored for this 490 * header name. 491 * @param field header name 492 * @return <code>HeaderLine</code> structure or null 493 */ 494 public HeaderLine getHeader(String field) { 495 if (field != null && field.length() > 0) { 496 return header.headerMap.get(field.toLowerCase()); 497 } else { 498 return null; 499 } 500 } 501 502 /** 503 * Specifies whether this record has a payload or not. 504 * @return true/false whether the ARC record has a payload 505 */ 506 public boolean hasPayload() { 507 return (payload != null); 508 } 509 510 /** 511 * Return Payload object. 512 * @return payload or <code>null</code> 513 */ 514 public Payload getPayload() { 515 return payload; 516 } 517 518 /** 519 * Payload content <code>InputStream</code> getter. 520 * @return Payload content <code>InputStream</code> 521 */ 522 public InputStream getPayloadContent() { 523 return (payload != null) ? payload.getInputStream() : null; 524 } 525 526 /** 527 * Returns the <code>HttpHeader</code> object if identified in the payload, 528 * or null. 529 * @return the <code>HttpHeader</code> object if identified or null 530 */ 531 public HttpHeader getHttpHeader() { 532 return httpHeader; 533 } 534 535 /** 536 * Add an error diagnosis of the given type on a specific entity with 537 * optional extra information. The information varies according to the 538 * diagnosis type. 539 * @param type diagnosis type 540 * @param entity entity examined 541 * @param information optional extra information 542 */ 543 protected void addErrorDiagnosis(DiagnosisType type, String entity, String... information) { 544 diagnostics.addError(new Diagnosis(type, entity, information)); 545 } 546 547}