001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.io.Closeable; 021import java.io.IOException; 022import java.io.InputStream; 023import java.util.Iterator; 024import java.util.NoSuchElementException; 025 026import org.jwat.common.Diagnostics; 027import org.jwat.common.HeaderLineReader; 028import org.jwat.common.UriProfile; 029 030/** 031 * Base class for WARC reader implementations. 032 * 033 * @author nicl 034 */ 035public abstract class WarcReader implements Closeable, Iterable<WarcRecord> { 036 037 /* 038 * Settings. 039 */ 040 041 /** WARC-Target-URI profile. */ 042 protected UriProfile warcTargetUriProfile; 043 044 /** URI profile. */ 045 protected UriProfile uriProfile; 046 047 /** Default block digest algorithm to use if none is present in the 048 * record. */ 049 protected String blockDigestAlgorithm; 050 051 /** Default encoding scheme used to encode block digest into a string, 052 * if none is detected from the record. */ 053 protected String blockDigestEncoding = "base32"; 054 055 /** Payload Digest enabled/disabled. */ 056 protected boolean bPayloadDigest = false; 057 058 /** Default payload digest algorithm to use if none is present in the 059 * record. */ 060 protected String payloadDigestAlgorithm; 061 062 /** Default encoding scheme used to encode payload digest into a string, 063 * if none is detected from the record. */ 064 protected String payloadDigestEncoding = "base32"; 065 066 /** Block Digest enabled/disabled. */ 067 protected boolean bBlockDigest = false; 068 069 /** Max size allowed for a record header. */ 070 protected int recordHeaderMaxSize; 071 072 /** Max size allowed for a payload header. */ 073 protected int payloadHeaderMaxSize; 074 075 /** Line reader used to read version lines. */ 076 protected HeaderLineReader lineReader; 077 078 /** Header line reader used to read the WARC headers. */ 079 protected HeaderLineReader headerLineReader; 080 081 /** WARC field parser used. */ 082 protected WarcFieldParsers fieldParsers; 083 084 /** Report HTTP header errors on reader diagnosis list. */ 085 protected boolean bReportHttpHeaderError = true; 086 087 /* 088 * State. 089 */ 090 091 /** Used for version based validation. */ 092 public final WarcValidation validation = new WarcValidation(); 093 094 /** Reader level errors and warnings or when no record is available. */ 095 public final Diagnostics diagnostics = new Diagnostics(); 096 097 /** Compliance status for records parsed up to now. */ 098 protected boolean bIsCompliant = true; 099 100 /** Number of bytes consumed by this reader. */ 101 protected long consumed = 0; 102 103 /** Records parsed. */ 104 protected int records = 0; 105 106 /** Aggregated number of errors encountered while parsing. */ 107 protected int errors = 0; 108 109 /** Aggregate number of warnings encountered while parsing. */ 110 protected int warnings = 0; 111 112 /** Current WARC record object. */ 113 protected WarcRecord currentRecord; 114 115 /** Exception thrown while using the iterator. */ 116 protected Exception iteratorExceptionThrown; 117 118 /** Callback used to monitor and modify internal data as it is being parsed. */ 119 protected WarcRecordParserCallback wrpCallback; 120 121 /** 122 * Method used to initialize a readers internal state. 123 * Must be called by all constructors. 124 */ 125 protected void init() { 126 warcTargetUriProfile = UriProfile.RFC3986; 127 uriProfile = UriProfile.RFC3986; 128 recordHeaderMaxSize = 8192; 129 payloadHeaderMaxSize = 32768; 130 lineReader = HeaderLineReader.getReader(); 131 lineReader.bNameValue = false; 132 lineReader.encoding = HeaderLineReader.ENC_US_ASCII; 133 headerLineReader = HeaderLineReader.getReader(); 134 headerLineReader.bNameValue = true; 135 headerLineReader.encoding = HeaderLineReader.ENC_UTF8; 136 headerLineReader.bLWS = true; 137 headerLineReader.bQuotedText = true; 138 headerLineReader.bEncodedWords = true; 139 fieldParsers = new WarcFieldParsers(); 140 } 141 142 /** 143 * Reset reader for reuse. 144 */ 145 public void reset() { 146 diagnostics.reset(); 147 bIsCompliant = true; 148 consumed = 0; 149 records = 0; 150 errors = 0; 151 warnings = 0; 152 currentRecord = null; 153 } 154 155 /** 156 * Returns a boolean indicating if all records parsed so far are compliant. 157 * @return a boolean indicating if all records parsed so far are compliant 158 */ 159 public boolean isCompliant() { 160 return bIsCompliant; 161 } 162 163 /** 164 * Is this reader assuming GZip compressed input. 165 * @return boolean indicating the assumption of GZip compressed input 166 */ 167 public abstract boolean isCompressed(); 168 169 /** 170 * Set the URI profile used to validate WARC-Target URIs. 171 * If null, the uriProfile is set to RCF3986. 172 * @param uriProfile URI profile to use 173 */ 174 public void setWarcTargetUriProfile(UriProfile uriProfile) { 175 if (uriProfile == null) { 176 uriProfile = UriProfile.RFC3986; 177 } 178 this.warcTargetUriProfile = uriProfile; 179 } 180 181 /** 182 * Get the URI profile used to validate WARC-Target URIs. 183 * @return the URI profile used to validate WARC-Target URIs 184 */ 185 public UriProfile getWarcTargetUriProfile() { 186 return warcTargetUriProfile; 187 } 188 189 /** 190 * Set the URI profile used to validate URIs. 191 * If null, the uriProfile is set to RCF3986. 192 * @param uriProfile URI profile to use 193 */ 194 public void setUriProfile(UriProfile uriProfile) { 195 if (uriProfile == null) { 196 uriProfile = UriProfile.RFC3986; 197 } 198 this.uriProfile = uriProfile; 199 } 200 201 /** 202 * Get the URI profile used to validate URIs. 203 * @return the URI profile used to validate URIs 204 */ 205 public UriProfile getUriProfile() { 206 return uriProfile; 207 } 208 209 /** 210 * Get the readers block digest on/off status. 211 * @return boolean indicating block digest on/off 212 */ 213 public boolean getBlockDigestEnabled() { 214 return bBlockDigest; 215 } 216 217 /** 218 * Set the readers block digest on/off status. Digest, however, 219 * will only be computed if either a Warc-Block-Digest header is 220 * present or an optional algorithm has been chosen. 221 * The Warc-Block-Digest always takes precedence. 222 * @param enabled boolean indicating block digest on/off 223 */ 224 public void setBlockDigestEnabled(boolean enabled) { 225 bBlockDigest = enabled; 226 } 227 228 /** 229 * Get the readers payload digest on/off status. 230 * @return boolean indicating payload digest on/off 231 */ 232 public boolean getPayloadDigestEnabled() { 233 return bPayloadDigest; 234 } 235 236 /** 237 * Set the readers payload digest on/off status. Digest, however, 238 * will only be computed if either a Warc-Payload-Digest header is 239 * present or an optional algorithm has been chosen. 240 * The Warc-Payload-Digest always takes precedence. 241 * @param enabled boolean indicating payload digest on/off 242 */ 243 public void setPayloadDigestEnabled(boolean enabled) { 244 bPayloadDigest = enabled; 245 } 246 247 /** 248 * Get the default block digest algorithm. 249 * @return default block digest algorithm 250 */ 251 public String getBlockDigestAlgorithm() { 252 return blockDigestAlgorithm; 253 } 254 255 /** 256 * Tries to set the default block digest algorithm and returns a boolean 257 * indicating whether the algorithm was accepted or not. This algorithm is 258 * only used in case no WARC payload digest header is present in the record. 259 * @param digestAlgorithm block digest algorithm 260 * (null means no default block digest algorithm is selected) 261 * @return boolean indicating the validity of the algorithm supplied 262 */ 263 public boolean setBlockDigestAlgorithm(String digestAlgorithm) { 264 if (digestAlgorithm == null || digestAlgorithm.length() == 0) { 265 blockDigestAlgorithm = null; 266 return true; 267 } 268 if (WarcDigest.digestAlgorithmLength(digestAlgorithm) > 0) { 269 blockDigestAlgorithm = digestAlgorithm; 270 return true; 271 } 272 return false; 273 } 274 275 /** 276 * Get the default payload digest algorithm. 277 * @return default payload digest algorithm 278 */ 279 public String getPayloadDigestAlgorithm() { 280 return payloadDigestAlgorithm; 281 } 282 283 /** 284 * Tries to set the default payload digest algorithm and returns a boolean 285 * indicating whether the algorithm was accepted or not. This algorithm is 286 * only used in case no WARC payload digest header is present in the record. 287 * @param digestAlgorithm payload digest algorithm 288 * (null means no default payload digest algorithm is selected) 289 * @return boolean indicating the validity of the algorithm supplied 290 */ 291 public boolean setPayloadDigestAlgorithm(String digestAlgorithm) { 292 if (digestAlgorithm == null || digestAlgorithm.length() == 0) { 293 payloadDigestAlgorithm = null; 294 return true; 295 } 296 if (WarcDigest.digestAlgorithmLength(digestAlgorithm) > 0) { 297 payloadDigestAlgorithm = digestAlgorithm; 298 return true; 299 } 300 return false; 301 } 302 303 /** 304 * Get the default block digest encoding scheme. 305 * @return default block digest encoding scheme 306 */ 307 public String getBlockDigestEncoding() { 308 return blockDigestEncoding; 309 } 310 311 /** 312 * Set the default block digest encoding scheme. This scheme is only 313 * used if none can be inferred from an existing block digest header. 314 * @param encodingScheme encoding scheme 315 * (null means default block digest is not encoded) 316 */ 317 public void setBlockDigestEncoding(String encodingScheme) { 318 if (encodingScheme != null && encodingScheme.length() > 0) { 319 blockDigestEncoding = encodingScheme.toLowerCase(); 320 } else { 321 blockDigestEncoding = null; 322 } 323 } 324 325 /** 326 * Get the default payload digest encoding scheme. 327 * @return default payload digest encoding scheme 328 */ 329 public String getPayloadDigestEncoding() { 330 return payloadDigestEncoding; 331 } 332 333 /** 334 * Set the default payload digest encoding scheme. This scheme is only 335 * used if none can be inferred from an existing payload digest header. 336 * @param encodingScheme encoding scheme 337 * (null means default payload digest is not encoded) 338 */ 339 public void setPayloadDigestEncoding(String encodingScheme) { 340 if (encodingScheme != null && encodingScheme.length() > 0) { 341 payloadDigestEncoding = encodingScheme.toLowerCase(); 342 } else { 343 payloadDigestEncoding = null; 344 } 345 } 346 347 /** 348 * Get the max size allowed for a record header. 349 * @return max size allowed for a record header 350 */ 351 public int getRecordHeaderMaxSize() { 352 return recordHeaderMaxSize; 353 } 354 355 /** 356 * Set the max size allowed for a record header. 357 * @param size max size allowed 358 */ 359 public void setRecordHeaderMaxSize(int size) { 360 recordHeaderMaxSize = size; 361 } 362 363 /** 364 * Get the max size allowed for a payload header. 365 * @return max size allowed for a payload header 366 */ 367 public int getPayloadHeaderMaxSize() { 368 return payloadHeaderMaxSize; 369 } 370 371 /** 372 * Set the max size allowed for a payload header. 373 * @param size max size allowed 374 */ 375 public void setPayloadHeaderMaxSize(int size) { 376 payloadHeaderMaxSize = size; 377 } 378 379 /** 380 * Get a boolean indicating whether HTTP header error reporting is enabled or disabled. 381 * @return boolean indicating whether HTTP header error reporting is enabled or disabled 382 */ 383 public boolean getReportHttpHeaderErrors() { 384 return bReportHttpHeaderError; 385 } 386 387 /** 388 * Enable or disable the readers HTTP header error being included in the readers diagnosis list. 389 * @param bReportHttpHeaderError Enable or disable HTTP header error reporting 390 */ 391 public void setReportHttpHeaderErrors(boolean bReportHttpHeaderError) { 392 this.bReportHttpHeaderError = bReportHttpHeaderError; 393 } 394 395 /** 396 * Close current record resource(s) and input stream(s). 397 */ 398 public abstract void close(); 399 400 /** 401 * Callback method called when the payload has been processed. 402 */ 403 protected abstract void recordClosed(); 404 405 /** 406 * Get the offset of the current WARC record or -1 if none have been read. 407 * @return offset of the current WARC record or -1 408 */ 409 public abstract long getStartOffset(); 410 411 /** 412 * Get the current offset in the WARC <code>InputStream</code>. 413 * @return offset in WARC <code>InputStream</code> 414 */ 415 public abstract long getOffset(); 416 417 /** 418 * Get number of bytes consumed by this reader. 419 * @return number of bytes consumed by this reader 420 */ 421 public abstract long getConsumed(); 422 423 /** 424 * Parses and gets the next record. 425 * This method is for linear access to records. 426 * @return the next record 427 * @throws IOException I/O exception in parsing process 428 */ 429 public abstract WarcRecord getNextRecord() throws IOException; 430 431 /** 432 * Parses and gets the next record from an <code>Inputstream</code>. 433 * This method is mainly for random access use since there are serious 434 * side-effects involved in using multiple <code>PushBackInputStream</code> 435 * instances. 436 * @param in <code>InputStream</code> used to read next record 437 * @param offset offset provided by caller 438 * @return the next record 439 * @throws IOException I/O exception in parsing process 440 */ 441 public abstract WarcRecord getNextRecordFrom(InputStream in, long offset) 442 throws IOException; 443 444 /** 445 * Parses and gets the next record from an <code>Inputstream</code> wrapped 446 * by a <code>BufferedInputStream</code>. 447 * This method is mainly for random access use since there are serious 448 * side-effects involved in using multiple <code>PushBackInputStream</code> 449 * instances. 450 * @param in <code>InputStream</code> used to read next record 451 * @param offset offset provided by caller 452 * @param buffer_size buffer size to use 453 * @return the next record 454 * @throws IOException I/O exception in parsing process 455 */ 456 public abstract WarcRecord getNextRecordFrom(InputStream in, long offset, 457 int buffer_size) throws IOException; 458 459 /** 460 * Gets an exception thrown in the iterator if any or null. 461 * @return exception thrown in the iterator if any or null 462 */ 463 public Exception getIteratorExceptionThrown() { 464 return iteratorExceptionThrown; 465 } 466 467 public WarcRecordParserCallback getWarcRecordParserCallback() { 468 return wrpCallback; 469 } 470 471 public void setWarcRecordParserCallback(WarcRecordParserCallback wrpCallback) { 472 this.wrpCallback = wrpCallback; 473 } 474 475 /** 476 * Returns an <code>Iterator</code> over the records as they are being 477 * parsed. Any exception thrown during parsing is accessible through the 478 * <code>getIteratorExceptionThrown</code> method. 479 * @return <code>Iterator</code> over the records 480 */ 481 public Iterator<WarcRecord> iterator() { 482 return new Iterator<WarcRecord>() { 483 484 /** Internal next record updated by either hasNext() or next(). */ 485 private WarcRecord next; 486 487 /** Entry returned by next(). */ 488 private WarcRecord current; 489 490 @Override 491 public boolean hasNext() { 492 if (next == null) { 493 iteratorExceptionThrown = null; 494 try { 495 next = getNextRecord(); 496 } catch (IOException e) { 497 iteratorExceptionThrown = e; 498 } 499 } 500 return (next != null); 501 } 502 503 @Override 504 public WarcRecord next() { 505 if (next == null) { 506 iteratorExceptionThrown = null; 507 try { 508 next = getNextRecord(); 509 } catch (IOException e) { 510 iteratorExceptionThrown = e; 511 } 512 } 513 if (next == null) { 514 throw new NoSuchElementException(); 515 } 516 current = next; 517 next = null; 518 return current; 519 } 520 521 @Override 522 public void remove() { 523 throw new UnsupportedOperationException(); 524 } 525 }; 526 } 527 528}