001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.io.Closeable; 021import java.io.IOException; 022import java.io.InputStream; 023import java.util.Arrays; 024import java.util.Collections; 025import java.util.List; 026 027import org.jwat.common.Base16; 028import org.jwat.common.Base32; 029import org.jwat.common.Base64; 030import org.jwat.common.ByteCountingPushBackInputStream; 031import org.jwat.common.Diagnosis; 032import org.jwat.common.DiagnosisType; 033import org.jwat.common.Diagnostics; 034import org.jwat.common.HeaderLine; 035import org.jwat.common.HttpHeader; 036import org.jwat.common.NewlineParser; 037import org.jwat.common.Payload; 038import org.jwat.common.PayloadOnClosedHandler; 039 040/** 041 * This class represents a parsed WARC record header block including 042 * possible validation and format warnings/errors encountered in the process. 043 * The payload of the WARC record is accessible through a wrapped payload 044 * object. 045 * 046 * @author nicl 047 */ 048public class WarcRecord implements PayloadOnClosedHandler, Closeable { 049 050 /** Reader instance used, required for file compliance. */ 051 protected WarcReader reader; 052 053 /** Input stream used to read this record. */ 054 protected ByteCountingPushBackInputStream in; 055 056 /** Is this record compliant ie. error free. */ 057 protected boolean bIsCompliant; 058 059 /** WARC record parsing start offset relative to the source WARC file input 060 * stream. Used to keep track of the uncompressed amount of bytes consumed. */ 061 protected long startOffset = -1; 062 063 /** Uncompressed bytes consumed while validating this record. */ 064 protected long consumed; 065 066 /** Validation errors and warnings. */ 067 public final Diagnostics diagnostics = new Diagnostics(); 068 069 /** Newline parser for counting/validating trailing newlines. */ 070 public NewlineParser nlp = new NewlineParser(); 071 072 /** Is Warc-Block-Digest valid. (Null is equal to not tested) */ 073 public Boolean isValidBlockDigest = null; 074 075 /** Is Warc-Payload-Digest valid. (Null is equal to not tested) */ 076 public Boolean isValidPayloadDigest = null; 077 078 /** Number of trailing newlines after record. */ 079 public int trailingNewlines; 080 081 /* 082 * Header-Fields. 083 */ 084 085 /** WARC header. */ 086 public WarcHeader header; 087 088 /* 089 * Payload 090 */ 091 092 /** Has payload been closed before. */ 093 protected boolean bPayloadClosed; 094 095 /** Has record been closed before. */ 096 protected boolean bClosed; 097 098 /** Payload object if any exists. */ 099 protected Payload payload; 100 101 /** HTTP header content parsed from payload. */ 102 protected HttpHeader httpHeader; 103 104 /** Computed block digest. */ 105 public WarcDigest computedBlockDigest; 106 107 /** Computed payload digest. */ 108 public WarcDigest computedPayloadDigest; 109 110 /** 111 * Non public constructor to allow unit testing. 112 */ 113 protected WarcRecord() { 114 } 115 116 /** 117 * Create a <code>WarcRecord</code> and prepare it for writing. 118 * @param writer writer which will be used to write the record 119 * @return a <code>WarcRecord</code> ready to be changed and then written 120 */ 121 public static WarcRecord createRecord(WarcWriter writer) { 122 WarcRecord record = new WarcRecord(); 123 record.header = WarcHeader.initHeader(writer, record.diagnostics); 124 writer.fieldParsers.diagnostics = record.diagnostics; 125 return record; 126 } 127 128 /** 129 * Given an <code>InputStream</code> it tries to read and validate a WARC 130 * header block. 131 * @param in <code>InputStream</code> containing WARC record data 132 * @param reader <code>WarcReader</code> used, with access to user defined 133 * options 134 * @return <code>WarcRecord</code> or <code>null</code> 135 * @throws IOException I/O exception in the process of reading record 136 */ 137 public static WarcRecord parseRecord(ByteCountingPushBackInputStream in, 138 WarcReader reader) throws IOException { 139 WarcRecord record = new WarcRecord(); 140 record.in = in; 141 record.reader = reader; 142 record.startOffset = in.getConsumed(); 143 // Initialize WarcHeader with required context. 144 record.header = WarcHeader.initHeader(reader, in.getConsumed(), record.diagnostics); 145 WarcHeader header = record.header; 146 // Initialize WarcFieldParser to report diagnoses here. 147 reader.fieldParsers.diagnostics = record.diagnostics; 148 if (header.parseHeader(in)) { 149 ++reader.records; 150 if (reader.wrpCallback != null) { 151 reader.wrpCallback.warcParsedRecordHeader(reader, record.startOffset, header); 152 } 153 /* 154 * Payload processing. 155 */ 156 if (header.contentLength != null && header.contentLength > 0) { 157 /* 158 * Payload. 159 */ 160 String digestAlgorithm = null; 161 if (reader.bBlockDigest) { 162 if (header.warcBlockDigest != null && header.warcBlockDigest.algorithm != null) { 163 // If a WARC block digest header is present in the 164 // record, use that algorithm. 165 digestAlgorithm = header.warcBlockDigest.algorithm; 166 } else { 167 // If no WARC block digest header is present, 168 // use the optional user specified algorithm. 169 // Can be null in which case nothing is computed. 170 digestAlgorithm = reader.blockDigestAlgorithm; 171 } 172 } 173 record.payload = Payload.processPayload(in, header.contentLength, 174 reader.payloadHeaderMaxSize, digestAlgorithm); 175 record.payload.setOnClosedHandler(record); 176 /* 177 * HttpHeader. 178 */ 179 if (header.contentType != null 180 && header.contentType.contentType.equals("application") 181 && header.contentType.mediaType.equals("http")) { 182 String value = header.contentType.getParameter("msgtype"); 183 // request 184 int httpHeaderType = 0; 185 if ("response".equalsIgnoreCase(value)) { 186 httpHeaderType = HttpHeader.HT_RESPONSE; 187 } else if ("request".equalsIgnoreCase(value)) { 188 httpHeaderType = HttpHeader.HT_REQUEST; 189 } 190 if (httpHeaderType != 0) { 191 digestAlgorithm = null; 192 if (reader.bPayloadDigest) { 193 if (header.warcPayloadDigest != null && header.warcPayloadDigest.algorithm != null) { 194 // If a WARC payload digest header is present in the 195 // record, use that algorithm. 196 digestAlgorithm = header.warcPayloadDigest.algorithm; 197 } else { 198 // If no WARC payload digest header is present, 199 // use the optional user specified algorithm. 200 // Can be null in which case nothing is computed. 201 digestAlgorithm = reader.payloadDigestAlgorithm; 202 } 203 } 204 // Try to read a valid HTTP request/response header from the payload. 205 record.httpHeader = HttpHeader.processPayload(httpHeaderType, 206 record.payload.getInputStream(), header.contentLength, 207 digestAlgorithm); 208 if (record.httpHeader != null) { 209 if (record.httpHeader.isValid()) { 210 record.payload.setPayloadHeaderWrapped(record.httpHeader); 211 } else if (reader.bReportHttpHeaderError) { 212 record.diagnostics.addWarning( 213 DiagnosisType.ERROR, "http header", "Unable to parse http header!"); 214 } 215 } 216 } 217 } 218 } 219 // Preliminary compliance status, will be updated when the 220 // payload/record is closed. 221 if (record.diagnostics.hasErrors() || record.diagnostics.hasWarnings()) { 222 record.bIsCompliant = false; 223 } else { 224 record.bIsCompliant = true; 225 } 226 reader.bIsCompliant &= record.bIsCompliant; 227 } else { 228 // In case no record is found the errors/warnings in the record object are transfered to the Reader. 229 long excess = in.getConsumed() - record.startOffset; 230 reader.consumed += excess; 231 reader.diagnostics.addAll(record.diagnostics); 232 if (record.diagnostics.hasErrors() || record.diagnostics.hasWarnings()) { 233 reader.errors += record.diagnostics.getErrors().size(); 234 reader.warnings += record.diagnostics.getWarnings().size(); 235 reader.bIsCompliant = false; 236 } 237 // Require one or more records to be present. 238 if (reader.records == 0) { 239 reader.diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, "WARC file", "One or more records")); 240 ++reader.errors; 241 reader.bIsCompliant = false; 242 } 243 if (excess != 0) { 244 reader.diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA, "Trailing data", "Garbage data found at offset=" + record.startOffset + " - length=" + excess)); 245 } 246 // EOF 247 record = null; 248 } 249 return record; 250 } 251 252 /** 253 * Called when the payload object is closed and final steps in the 254 * validation process can be performed. 255 * @throws IOException I/O exception in final validation processing 256 */ 257 @Override 258 public void payloadClosed() throws IOException { 259 if (!bPayloadClosed) { 260 if (payload != null) { 261 // Check for truncated payload. 262 if (payload.getUnavailable() > 0) { 263 // Payload length mismatch - Payload truncated 264 addErrorDiagnosis(DiagnosisType.INVALID_DATA, "Payload length mismatch", "Payload truncated"); 265 } 266 /* 267 * Check block digest. 268 */ 269 byte[] digest = payload.getDigest(); 270 // Check for computed block digest. 271 if (digest != null) { 272 computedBlockDigest = new WarcDigest(); 273 computedBlockDigest.digestBytes = digest; 274 } 275 // Auto detect encoding used in WARC header. 276 if (header.warcBlockDigest != null && header.warcBlockDigest.digestString != null) { 277 isValidBlockDigest = processWarcDigest(header.warcBlockDigest, computedBlockDigest, "block"); 278 } 279 // Adjust information about computed block digest. 280 if (computedBlockDigest != null) { 281 processComputedDigest(computedBlockDigest, 282 reader.blockDigestAlgorithm, reader.blockDigestEncoding, "block"); 283 } 284 // Revisit payload digest refers to the original. Continuation payload digest in first record also refers to original. 285 if ((header.warcTypeIdx != null && header.warcTypeIdx != WarcConstants.RT_IDX_REVISIT && header.warcTypeIdx != WarcConstants.RT_IDX_CONTINUATION) && httpHeader != null && httpHeader.isValid()) { 286 /* 287 * Check payload digest. 288 */ 289 digest = httpHeader.getDigest(); 290 // Check for computed payload digest. 291 if (digest != null) { 292 computedPayloadDigest = new WarcDigest(); 293 computedPayloadDigest.digestBytes = digest; 294 } 295 // Auto detect encoding used in WARC header. 296 if (header.warcPayloadDigest != null && header.warcPayloadDigest.digestString != null ) { 297 isValidPayloadDigest = processWarcDigest(header.warcPayloadDigest, computedPayloadDigest, "payload"); 298 } 299 // Adjust information about computed payload digest. 300 if (computedPayloadDigest != null) { 301 processComputedDigest(computedPayloadDigest, 302 reader.payloadDigestAlgorithm, reader.payloadDigestEncoding, "payload"); 303 } 304 } 305 } 306 // Check for trailing newlines. 307 trailingNewlines = nlp.parseCRLFs(in, diagnostics); 308 if (trailingNewlines != WarcConstants.WARC_RECORD_TRAILING_NEWLINES) { 309 addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED, 310 "Trailing newlines", 311 Integer.toString(trailingNewlines), 312 Integer.toString(WarcConstants.WARC_RECORD_TRAILING_NEWLINES)); 313 } 314 // isCompliant status update. 315 if (diagnostics.hasErrors() || diagnostics.hasWarnings()) { 316 bIsCompliant = false; 317 reader.errors += diagnostics.getErrors().size(); 318 reader.warnings += diagnostics.getWarnings().size(); 319 } else { 320 bIsCompliant = true; 321 } 322 reader.bIsCompliant &= bIsCompliant; 323 // Updated consumed after payload has been consumed. 324 consumed = in.getConsumed() - startOffset; 325 // Don't not close payload again. 326 bPayloadClosed = true; 327 // Callback. 328 reader.recordClosed(); 329 } 330 } 331 332 /** 333 * Auto-detect encoding used in WARC digest header and compare it to the 334 * internal one, if it has been computed. 335 * @param warcDigest digest from WARC header 336 * @param computedDigest internally compute digest 337 * @param digestName used to identify the digest ("block" or "payload") 338 * @return WARC digest validity indication 339 */ 340 protected Boolean processWarcDigest(WarcDigest warcDigest, WarcDigest computedDigest, String digestName) { 341 byte[] digest; 342 Boolean isValidDigest = null; 343 int digestAlgorithmLength = WarcDigest.digestAlgorithmLength(warcDigest.algorithm); 344 digest = Base16.decodeToArray(warcDigest.digestString); 345 if (digest != null && digest.length == digestAlgorithmLength) { 346 warcDigest.digestBytes = digest; 347 warcDigest.encoding = "base16"; 348 } 349 if (warcDigest.digestBytes == null) { 350 digest = Base32.decodeToArray(warcDigest.digestString, true); 351 if (digest != null && digest.length == digestAlgorithmLength) { 352 warcDigest.digestBytes = digest; 353 warcDigest.encoding = "base32"; 354 } 355 if (warcDigest.digestBytes == null) { 356 digest = Base64.decodeToArray(warcDigest.digestString, true); 357 if (digest != null && digest.length == digestAlgorithmLength) { 358 warcDigest.digestBytes = digest; 359 warcDigest.encoding = "base64"; 360 } 361 } 362 } 363 if (warcDigest.encoding == null) { 364 // Encoding - Unrecognized block digest encoding scheme 365 addErrorDiagnosis(DiagnosisType.UNKNOWN, 366 "Record " + digestName + " digest encoding scheme", 367 warcDigest.digestString); 368 } 369 if (computedDigest != null) { 370 computedDigest.algorithm = warcDigest.algorithm; 371 computedDigest.encoding = warcDigest.encoding; 372 if (warcDigest.digestBytes != null) { 373 if (!Arrays.equals(computedDigest.digestBytes, warcDigest.digestBytes)) { 374 // Block digest - Computed block digest does not match 375 addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED, 376 "Incorrect " + digestName + " digest", 377 Base16.encodeArray(warcDigest.digestBytes), 378 Base16.encodeArray(computedDigest.digestBytes)); 379 isValidDigest = false; 380 } else { 381 isValidDigest = true; 382 } 383 } else { 384 isValidDigest = false; 385 } 386 } 387 return isValidDigest; 388 } 389 390 /** 391 * Adjust algorithm and encoding information about computed block digest. 392 * @param computedDigest internally compute digest 393 * @param digestAlgorithm default algorithm 394 * @param digestEncoding default encoding 395 * @param digestName used to identify the digest ("block" or "payload") 396 */ 397 protected void processComputedDigest(WarcDigest computedDigest, String digestAlgorithm, String digestEncoding, String digestName) { 398 if (computedDigest.algorithm == null) { 399 computedDigest.algorithm = digestAlgorithm; 400 } 401 if (computedDigest.encoding == null && digestEncoding != null) { 402 if ("base32".equals(digestEncoding)) { 403 computedDigest.encoding = "base32"; 404 } else if ("base64".equals(digestEncoding)) { 405 computedDigest.encoding = "base64"; 406 } else if ("base16".equals(digestEncoding)) { 407 computedDigest.encoding = "base16"; 408 } else { 409 // Encoding - Unknown block digest encoding scheme .. 410 addErrorDiagnosis(DiagnosisType.UNKNOWN, 411 "Default " + digestName + " digest encoding scheme", 412 digestEncoding); 413 } 414 } 415 if (computedDigest.encoding != null) { 416 if ("base32".equals(computedDigest.encoding)) { 417 computedDigest.digestString = Base32.encodeArray(computedDigest.digestBytes); 418 } else if ("base64".equals(computedDigest.encoding)) { 419 computedDigest.digestString = Base64.encodeArray(computedDigest.digestBytes); 420 } else if ("base16".equals(computedDigest.encoding)) { 421 computedDigest.digestString = Base16.encodeArray(computedDigest.digestBytes); 422 } 423 } 424 } 425 426 /** 427 * Check to see if the record has been closed. 428 * @return boolean indicating whether this record is closed or not 429 */ 430 public boolean isClosed() { 431 return bClosed; 432 } 433 434 /** 435 * Close resources associated with the WARC record. 436 * Mainly payload stream if any. 437 * @throws IOException if unable to close resources 438 */ 439 public void close() throws IOException { 440 if (!bClosed) { 441 // Ensure input stream is at the end of the record payload. 442 if (payload != null) { 443 payload.close(); 444 } 445 payloadClosed(); 446 reader = null; 447 in = null; 448 bClosed = true; 449 } 450 } 451 452 /** 453 * Returns a boolean indicating the ISO compliance status of this record. 454 * @return a boolean indicating the ISO compliance status of this record 455 */ 456 public boolean isCompliant() { 457 return bIsCompliant; 458 } 459 460 /** 461 * Get the record offset relative to the start of the WARC file 462 * <code>InputStream</code>. 463 * @return the record offset relative to the start of the WARC file 464 */ 465 public long getStartOffset() { 466 return header.startOffset; 467 } 468 469 /** 470 * Return number of uncompressed bytes consumed validating this record. 471 * @return number of uncompressed bytes consumed validating this record 472 */ 473 public long getConsumed() { 474 return consumed; 475 } 476 477 /** 478 * Get a <code>List</code> of all the non-standard WARC headers found 479 * during parsing. 480 * @return <code>List</code> of <code>HeaderLine</code> 481 */ 482 public List<HeaderLine> getHeaderList() { 483 return Collections.unmodifiableList(header.headerList); 484 } 485 486 /** 487 * Get a non-standard WARC header or null, if nothing is stored for this 488 * header name. 489 * @param field header name 490 * @return <code>HeaderLine</code> structure or null 491 */ 492 public HeaderLine getHeader(String field) { 493 if (field != null && field.length() > 0) { 494 return header.headerMap.get(field.toLowerCase()); 495 } else { 496 return null; 497 } 498 } 499 500 /** 501 * Specifies whether this record has a payload or not. 502 * @return true/false whether the ARC record has a payload 503 */ 504 public boolean hasPayload() { 505 return (payload != null); 506 } 507 508 /** 509 * Return Payload object. 510 * @return payload or <code>null</code> 511 */ 512 public Payload getPayload() { 513 return payload; 514 } 515 516 /** 517 * Payload content <code>InputStream</code> getter. 518 * @return Payload content <code>InputStream</code> 519 */ 520 public InputStream getPayloadContent() { 521 return (payload != null) ? payload.getInputStream() : null; 522 } 523 524 /** 525 * Returns the <code>HttpHeader</code> object if identified in the payload, 526 * or null. 527 * @return the <code>HttpHeader</code> object if identified or null 528 */ 529 public HttpHeader getHttpHeader() { 530 return httpHeader; 531 } 532 533 /** 534 * Add an error diagnosis of the given type on a specific entity with 535 * optional extra information. The information varies according to the 536 * diagnosis type. 537 * @param type diagnosis type 538 * @param entity entity examined 539 * @param information optional extra information 540 */ 541 protected void addErrorDiagnosis(DiagnosisType type, String entity, String... information) { 542 diagnostics.addError(new Diagnosis(type, entity, information)); 543 } 544 545}