001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.arc; 019 020import java.io.Closeable; 021import java.io.IOException; 022import java.io.InputStream; 023import java.net.InetAddress; 024import java.util.Date; 025 026import org.jwat.common.Base16; 027import org.jwat.common.Base32; 028import org.jwat.common.Base64; 029import org.jwat.common.ByteCountingPushBackInputStream; 030import org.jwat.common.ContentType; 031import org.jwat.common.Diagnosis; 032import org.jwat.common.DiagnosisType; 033import org.jwat.common.Diagnostics; 034import org.jwat.common.Digest; 035import org.jwat.common.HttpHeader; 036import org.jwat.common.NewlineParser; 037import org.jwat.common.Payload; 038import org.jwat.common.PayloadOnClosedHandler; 039import org.jwat.common.PayloadWithHeaderAbstract; 040import org.jwat.common.Uri; 041 042/** 043 * This abstract class represents the common base ARC data which is present in 044 * both a record or version block. The parser in this class is responsible for 045 * parsing the general record line and determining if the rest of the record 046 * should be processed as an ARC record or an ARC version block. 047 * This includes possible common validation and format warnings/errors 048 * encountered in the process. 049 * 050 * @author lbihanic, selghissassi, nicl 051 */ 052public abstract class ArcRecordBase implements PayloadOnClosedHandler, Closeable { 053 054 /** Version block record type. */ 055 public static final int RT_VERSION_BLOCK = 1; 056 057 /** Arc record record type. */ 058 public static final int RT_ARC_RECORD = 2; 059 060 /** Buffer size used in toString(). */ 061 public static final int TOSTRING_BUFFER_SIZE = 256; 062 063 /** Is this record compliant ie. error free. */ 064 protected boolean bIsCompliant; 065 066 /** Reader instance used, required for file compliance. */ 067 protected ArcReader reader; 068 069 /** Input stream used to read this record. */ 070 protected ByteCountingPushBackInputStream in; 071 072 /** ARC record version. */ 073 protected ArcVersion version; 074 075 /** ARC record parsing start offset relative to the source ARC file input 076 * stream. Used to keep track of the uncompressed amount of bytes consumed. */ 077 protected long startOffset = -1; 078 079 /** Uncompressed bytes consumed while validating this record. */ 080 protected long consumed; 081 082 /** Validation errors and warnings. */ 083 public Diagnostics<Diagnosis> diagnostics; 084 085 /** Newline parser for counting/validating trailing newlines. */ 086 public NewlineParser nlp = new NewlineParser(); 087 088 /** Record type, version block or arc record. */ 089 public int recordType; 090 091 /** Number of trailing newlines after record. */ 092 public int trailingNewLines; 093 094 /* 095 * Header-Fields. 096 */ 097 098 /** ARC record header. */ 099 public ArcHeader header; 100 101 /** ARC version header. */ 102 public ArcVersionHeader versionHeader; 103 104 /* 105 * Payload 106 */ 107 108 /** Has payload been closed before. */ 109 protected boolean bPayloadClosed; 110 111 /** Has record been closed flag. */ 112 protected boolean bClosed; 113 114 /** Payload object if any exists. */ 115 protected Payload payload; 116 117 /** Has all the payload data been processed while reading the record header. */ 118 protected boolean bHasPseudoEmptyPayload; 119 120 /** HTTP header content parsed from payload. */ 121 protected HttpHeader httpHeader; 122 123 /** Computed block digest. */ 124 public Digest computedBlockDigest; 125 126 /** Computed payload digest. */ 127 public Digest computedPayloadDigest; 128 129 /** This array is used to store metadata which should not be present. 130 * Only available for forensic purposes. */ 131 public byte[] excessiveMetadata; 132 133 /** 134 * Reads from the input stream and tries to parse and identify an 135 * <code>ArcRecord</code> or <code>ArcVersionBlock</code> record. 136 * @param in input stream with ARC record data 137 * @param reader ARC reader used with access to overall configuration and status 138 * @return <code>ArcRecord</code>, <code>ArcVersionBlock</code> or null 139 * @throws IOException I/O exception while parsing ARC record data 140 */ 141 public static ArcRecordBase parseRecord(ByteCountingPushBackInputStream in, ArcReader reader) throws IOException { 142 ArcRecordBase record = null; 143 long startOffset = in.getConsumed(); 144 // Initialize ArcHeader with required context. 145 Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>(); 146 ArcHeader header = ArcHeader.initHeader(reader, startOffset, diagnostics); 147 // Initialize ArcFieldParser to report diagnoses here. 148 reader.fieldParsers.diagnostics = diagnostics; 149 // Returns true if a record has been processed. 150 // And false if only garbage has been processed. 151 if (header.parseHeader(in)) { 152 if (reader.arpCallback != null) { 153 reader.arpCallback.arcParsedRecordHeader(reader, startOffset, header); 154 } 155 if (header.urlScheme != null && header.urlScheme.startsWith(ArcConstants.ARC_SCHEME)) { 156 record = ArcVersionBlock.parseVersionBlock(reader, diagnostics, header, reader.fieldParsers, in); 157 if (record != null) { 158 reader.versionHeader = record.versionHeader; 159 } 160 } 161 if (record == null) { 162 record = ArcRecord.parseArcRecord(reader, diagnostics, header, in); 163 if (record != null && reader.versionHeader != null) { 164 record.version = reader.versionHeader.version; 165 } 166 } 167 } 168 // Record identified above. 169 if (record != null) { 170 ++reader.records; 171 record.startOffset = startOffset; 172 // Check read and computed offset value only if we're reading 173 // a plain ARC file, not a GZipped ARC. 174 if ((header.offset != null) && (header.startOffset > 0L) 175 && (header.offset.longValue() != header.startOffset)) { 176 diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, 177 "'" + ArcConstants.FN_OFFSET + "' value", 178 header.offset.toString(), 179 Long.toString(header.startOffset))); 180 } 181 if (reader.records == 1) { 182 if (record.recordType == ArcRecordBase.RT_ARC_RECORD) { 183 diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, 184 ArcConstants.ARC_FILE, 185 "Expected a version block as the first record.")); 186 } 187 } else { 188 if (record.recordType == ArcRecordBase.RT_VERSION_BLOCK) { 189 diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, 190 ArcConstants.ARC_FILE, 191 "Expected an ARC record not version block.")); 192 } 193 } 194 if (reader.versionHeader != null && reader.versionHeader.blockDescVersion > 0 195 && record.header.recordFieldVersion != reader.versionHeader.blockDescVersion) { 196 diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, 197 "ARC record does not match the version block definition", 198 Integer.toString(record.header.recordFieldVersion), 199 Integer.toString(reader.versionHeader.blockDescVersion))); 200 } 201 // Preliminary compliance status, will be updated when the 202 // payload/record is closed. 203 if (diagnostics.hasErrors() || diagnostics.hasWarnings()) { 204 record.bIsCompliant = false; 205 } else { 206 record.bIsCompliant = true; 207 } 208 reader.bIsCompliant &= record.bIsCompliant; 209 } else { 210 // Transfer errors/warnings identified in the header parser to the reader since we are not returning a record. 211 long excess = in.getConsumed() - startOffset; 212 reader.consumed += excess; 213 reader.diagnostics.addAll(diagnostics); 214 if (diagnostics.hasErrors() || diagnostics.hasWarnings()) { 215 reader.errors += diagnostics.getErrors().size(); 216 reader.warnings += diagnostics.getWarnings().size(); 217 reader.bIsCompliant = false; 218 } 219 // Require one or more records to be present. 220 if (reader.records == 0) { 221 reader.diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, "ARC file", "One or more records")); 222 ++reader.errors; 223 reader.bIsCompliant = false; 224 } 225 if (excess != 0) { 226 reader.diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA, "Trailing data", "Garbage data found at offset=" + startOffset + " - length=" + excess)); 227 } 228 } 229 return record; 230 } 231 232 /** 233 * Called when the payload object is closed and final steps in the 234 * validation process can be performed. 235 * @throws IOException I/O exception in final validation processing 236 */ 237 @Override 238 public void payloadClosed() throws IOException { 239 if (!bPayloadClosed) { 240 if (payload != null) { 241 // Check for truncated payload. 242 if (payload.getUnavailable() > 0) { 243 // Payload length mismatch - Payload truncated 244 addErrorDiagnosis(DiagnosisType.INVALID_DATA, "Payload length mismatch", "Payload truncated"); 245 } 246 /* 247 * Check block digest. 248 */ 249 byte[] digest = payload.getDigest(); 250 if (digest != null) { 251 computedBlockDigest = new Digest(); 252 computedBlockDigest.digestBytes = digest; 253 computedBlockDigest.algorithm = reader.blockDigestAlgorithm; 254 if (reader.blockDigestEncoding != null) { 255 if ("base32".equals(reader.blockDigestEncoding)) { 256 computedBlockDigest.encoding = "base32"; 257 computedBlockDigest.digestString = Base32.encodeArray(computedBlockDigest.digestBytes); 258 } else if ("base64".equals(reader.blockDigestEncoding)) { 259 computedBlockDigest.encoding = "base64"; 260 computedBlockDigest.digestString = Base64.encodeArray(computedBlockDigest.digestBytes); 261 } else if ("base16".equals(reader.blockDigestEncoding)) { 262 computedBlockDigest.encoding = "base16"; 263 computedBlockDigest.digestString = Base16.encodeArray(computedBlockDigest.digestBytes); 264 } else { 265 // Encoding - Unknown block digest encoding scheme .. 266 addErrorDiagnosis(DiagnosisType.INVALID_DATA, 267 "Block digest encoding scheme", 268 reader.blockDigestEncoding); 269 } 270 } 271 } 272 PayloadWithHeaderAbstract payloadHeaderWrapped = payload.getPayloadHeaderWrapped(); 273 if (payloadHeaderWrapped != null && payloadHeaderWrapped.isValid()) { 274 /* 275 * Check payload digest. 276 */ 277 digest = payloadHeaderWrapped.getDigest(); 278 if (digest != null) { 279 computedPayloadDigest = new Digest(); 280 computedPayloadDigest.digestBytes = digest; 281 computedPayloadDigest.algorithm = reader.payloadDigestAlgorithm; 282 if (reader.payloadDigestEncoding != null) { 283 if ("base32".equals(reader.payloadDigestEncoding)) { 284 computedPayloadDigest.encoding = "base32"; 285 computedPayloadDigest.digestString = Base32.encodeArray(computedPayloadDigest.digestBytes); 286 } else if ("base64".equals(reader.payloadDigestEncoding)) { 287 computedPayloadDigest.encoding = "base64"; 288 computedPayloadDigest.digestString = Base64.encodeArray(computedPayloadDigest.digestBytes); 289 } else if ("base16".equals(reader.payloadDigestEncoding)) { 290 computedPayloadDigest.encoding = "base16"; 291 computedPayloadDigest.digestString = Base16.encodeArray(computedPayloadDigest.digestBytes); 292 } else { 293 // Encoding - Unknown payload digest encoding scheme .. 294 addErrorDiagnosis(DiagnosisType.INVALID_DATA, 295 "Payload digest encoding scheme", 296 reader.payloadDigestEncoding); 297 } 298 } 299 } 300 } 301 } 302 // Check for trailing newlines. 303 trailingNewLines = nlp.parseLFs(in, diagnostics); 304 if (reader.bStrict && trailingNewLines != ArcConstants.ARC_RECORD_TRAILING_NEWLINES) { 305 addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED, 306 "Trailing newlines", 307 Integer.toString(trailingNewLines), 308 Integer.toString(ArcConstants.ARC_RECORD_TRAILING_NEWLINES)); 309 } 310 // isCompliant status update. 311 if (diagnostics.hasErrors() || diagnostics.hasWarnings()) { 312 bIsCompliant = false; 313 reader.errors += diagnostics.getErrors().size(); 314 reader.warnings += diagnostics.getWarnings().size(); 315 } else { 316 bIsCompliant = true; 317 } 318 reader.bIsCompliant &= bIsCompliant; 319 // Updated consumed after payload has been consumed. 320 consumed = in.getConsumed() - startOffset; 321 // Don't not close payload again. 322 bPayloadClosed = true; 323 // Callback. 324 reader.recordClosed(); 325 } 326 } 327 328 /** 329 * Check to see if the record has been closed. 330 * @return boolean indicating whether this record is closed or not 331 */ 332 public boolean isClosed() { 333 return bClosed; 334 } 335 336 /** 337 * Close resources associated with the ARC record. 338 * Mainly payload stream if any. 339 * @throws IOException I/O exception close the payload resources 340 */ 341 public void close() throws IOException { 342 if (!bClosed) { 343 // Ensure input stream is at the end of the record payload. 344 if (payload != null) { 345 payload.close(); 346 } 347 payloadClosed(); 348 reader = null; 349 in = null; 350 bClosed = true; 351 } 352 } 353 354 /** 355 * Returns a boolean indicating the standard compliance of this record. 356 * @return a boolean indicating the standard compliance of this record 357 */ 358 public boolean isCompliant() { 359 return bIsCompliant; 360 } 361 362 /** 363 * Returns the parsing start offset of the record in the containing ARC. 364 * @return the parsing start offset of the record 365 */ 366 public long getStartOffset() { 367 return header.startOffset; 368 } 369 370 /** 371 * Return number of uncompressed bytes consumed validating this record. 372 * @return number of uncompressed bytes consumed validating this record 373 */ 374 public long getConsumed() { 375 return consumed; 376 } 377 378 /** 379 * Process the ARC record stream for possible payload data. 380 * @param in ARC record <code>InputStream</code> 381 * @param reader <code>ArcReader</code> used, with access to user defined 382 * options 383 * @throws IOException I/O exception in the parsing process 384 */ 385 protected abstract void processPayload(ByteCountingPushBackInputStream in, 386 ArcReader reader) throws IOException; 387 388 /** 389 * Specifies whether this record has a payload or not. 390 * @return true/false whether the ARC record has a payload 391 */ 392 public boolean hasPayload() { 393 return (payload != null); 394 } 395 396 /** 397 * Specifies whether this record has had all it's payload processed already. 398 * @return true/false whether this record's payload has been completely processed 399 */ 400 public boolean hasPseudoEmptyPayload() { 401 return bHasPseudoEmptyPayload; 402 } 403 404 /** 405 * Return Payload object. 406 * @return payload or <code>null</code> 407 */ 408 public Payload getPayload() { 409 return payload; 410 } 411 412 /** 413 * Payload content <code>InputStream</code> getter. 414 * @return Payload content <code>InputStream</code> 415 */ 416 public InputStream getPayloadContent() { 417 return (payload != null) ? payload.getInputStream() : null; 418 } 419 420 /** 421 * Returns the <code>HttpHeader</code> object if identified in the payload, 422 * or null. 423 * @return the <code>HttpHeader</code> object if identified or null 424 */ 425 public HttpHeader getHttpHeader() { 426 return httpHeader; 427 } 428 429 /** 430 * Parses the remaining input stream and validates that the characters 431 * encountered are equal to LF or CR. 432 * @param in <code>InputStream</code> to validate 433 * @return true/false based on whether the remaining input stream contains 434 * only LF and CR characters or not. 435 * @throws IOException I/O exception in parsing 436 */ 437 public boolean isValidStreamOfCRLF(InputStream in) throws IOException{ 438 if (in == null) { 439 throw new IllegalArgumentException("'in' is null!"); 440 } 441 boolean isValid = true; 442 int b; 443 while ((b = in.read()) != -1) { 444 if (b != '\n' && b != '\r') { 445 isValid = false; 446 break; 447 } 448 } 449 return isValid; 450 } 451 452 /** 453 * Add an error diagnosis of the given type on a specific entity with 454 * optional extra information. The information varies according to the 455 * diagnosis type. 456 * @param type diagnosis type 457 * @param entity entity examined 458 * @param information optional extra information 459 */ 460 protected void addErrorDiagnosis(DiagnosisType type, String entity, String... information) { 461 diagnostics.addError(new Diagnosis(type, entity, information)); 462 } 463 464 /** 465 * Add a warning diagnosis of the given type on a specific entity with 466 * optional extra information. The information varies according to the 467 * diagnosis type. 468 * @param type diagnosis type 469 * @param entity entity examined 470 * @param information optional extra information 471 */ 472 /* 473 protected void addWarningDiagnosis(DiagnosisType type, String entity, String... information) { 474 diagnostics.addWarning(new Diagnosis(type, entity, information)); 475 } 476 */ 477 478 /** 479 * Version getter. 480 * @return the version 481 */ 482 public ArcVersion getVersion() { 483 return version; 484 } 485 486 /** 487 * Raw URL getter. 488 * @return the raw URL 489 */ 490 public String getUrlStr() { 491 return header.urlStr; 492 } 493 494 /** 495 * URL getter. 496 * @return the URL 497 */ 498 public Uri getUrl() { 499 return header.urlUri; 500 } 501 502 /** 503 * Protocol getter. 504 * @return the protocol 505 */ 506 public String getScheme() { 507 return header.urlScheme; 508 } 509 510 /** 511 * IpAddress getter. 512 * @return the ipAddress 513 */ 514 public String getIpAddress() { 515 return header.ipAddressStr; 516 } 517 518 /** 519 * <code>inetAddress</code> getter. 520 * @return the InetAddress 521 */ 522 public InetAddress getInetAddress() { 523 return header.inetAddress; 524 } 525 526 /** 527 * Raw ArchiveDate getter. 528 * @return the rawArchiveDate 529 */ 530 public String getArchiveDateStr() { 531 return header.archiveDateStr; 532 } 533 534 /** 535 * ArchiveDate getter. 536 * @return the archiveDate 537 */ 538 public Date getArchiveDate() { 539 return header.archiveDate; 540 } 541 542 /** 543 * Raw Content-Type getter. 544 * @return the contentType 545 */ 546 public String getContentTypeStr() { 547 return header.contentTypeStr; 548 } 549 550 /** 551 * Content-Type getter. 552 * @return the contentType 553 */ 554 public ContentType getContentType() { 555 return header.contentType; 556 } 557 558 /** 559 * Raw Result-Code getter. 560 * @return the resultCode 561 */ 562 public String getResultCodeStr() { 563 return header.resultCodeStr; 564 } 565 566 /** 567 * Result-Code getter. 568 * @return the resultCode 569 */ 570 public Integer getResultCode() { 571 return header.resultCode; 572 } 573 574 /** 575 * Checksum getter. 576 * @return the checksum 577 */ 578 public String getChecksum() { 579 return header.checksumStr; 580 } 581 582 /** 583 * Location getter. 584 * @return the location 585 */ 586 public String getLocation() { 587 return header.locationStr; 588 } 589 590 /** 591 * Raw Offset getter. 592 * @return the offset 593 */ 594 public String getOffsetStr() { 595 return header.offsetStr; 596 } 597 598 /** 599 * Offset getter. 600 * @return the offset 601 */ 602 public Long getOffset() { 603 return header.offset; 604 } 605 606 /** 607 * FileName getter. 608 * @return the fileName 609 */ 610 public String getFileName() { 611 return header.filenameStr; 612 } 613 614 /** 615 * Raw Length getter. 616 * @return the length 617 */ 618 public String getArchiveLengthStr() { 619 return header.archiveLengthStr; 620 } 621 622 /** 623 * Length getter. 624 * @return the length 625 */ 626 public Long getArchiveLength() { 627 return header.archiveLength; 628 } 629 630 @Override 631 public String toString() { 632 StringBuilder sb = new StringBuilder(TOSTRING_BUFFER_SIZE); 633 header.toStringBuilder(sb); 634 return sb.toString(); 635 } 636 637}