001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.io.Closeable; 021import java.io.IOException; 022import java.io.InputStream; 023import java.util.Iterator; 024import java.util.NoSuchElementException; 025 026import org.jwat.common.Diagnosis; 027import org.jwat.common.Diagnostics; 028import org.jwat.common.HeaderLineReader; 029import org.jwat.common.UriProfile; 030 031/** 032 * Base class for WARC reader implementations. 033 * 034 * @author nicl 035 */ 036public abstract class WarcReader implements Closeable, Iterable<WarcRecord> { 037 038 /* 039 * Settings. 040 */ 041 042 /** WARC-Target-URI profile. */ 043 protected UriProfile warcTargetUriProfile; 044 045 /** URI profile. */ 046 protected UriProfile uriProfile; 047 048 /** Default block digest algorithm to use if none is present in the 049 * record. */ 050 protected String blockDigestAlgorithm; 051 052 /** Default encoding scheme used to encode block digest into a string, 053 * if none is detected from the record. */ 054 protected String blockDigestEncoding = "base32"; 055 056 /** Payload Digest enabled/disabled. */ 057 protected boolean bPayloadDigest = false; 058 059 /** Default payload digest algorithm to use if none is present in the 060 * record. */ 061 protected String payloadDigestAlgorithm; 062 063 /** Default encoding scheme used to encode payload digest into a string, 064 * if none is detected from the record. */ 065 protected String payloadDigestEncoding = "base32"; 066 067 /** Block Digest enabled/disabled. */ 068 protected boolean bBlockDigest = false; 069 070 /** Max size allowed for a record header. */ 071 protected int recordHeaderMaxSize; 072 073 /** Max size allowed for a payload header. */ 074 protected int payloadHeaderMaxSize; 075 076 /** Line reader used to read version lines. */ 077 protected HeaderLineReader lineReader; 078 079 /** Header line reader used to read the WARC headers. */ 080 protected HeaderLineReader headerLineReader; 081 082 /** WARC field parser used. */ 083 protected WarcFieldParsers fieldParsers; 084 085 /* 086 * State. 087 */ 088 089 /** Reader level errors and warnings or when no record is available. */ 090 public final Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>(); 091 092 /** Compliance status for records parsed up to now. */ 093 protected boolean bIsCompliant = true; 094 095 /** Number of bytes consumed by this reader. */ 096 protected long consumed = 0; 097 098 /** Records parsed. */ 099 protected int records = 0; 100 101 /** Aggregated number of errors encountered while parsing. */ 102 protected int errors = 0; 103 104 /** Aggregate number of warnings encountered while parsing. */ 105 protected int warnings = 0; 106 107 /** Current WARC record object. */ 108 protected WarcRecord currentRecord; 109 110 /** Exception thrown while using the iterator. */ 111 protected Exception iteratorExceptionThrown; 112 113 /** Callback used to monitor and modify internal data as it is being parsed. */ 114 protected WarcRecordParserCallback wrpCallback; 115 116 /** 117 * Method used to initialize a readers internal state. 118 * Must be called by all constructors. 119 */ 120 protected void init() { 121 warcTargetUriProfile = UriProfile.RFC3986; 122 uriProfile = UriProfile.RFC3986; 123 recordHeaderMaxSize = 8192; 124 payloadHeaderMaxSize = 32768; 125 lineReader = HeaderLineReader.getReader(); 126 lineReader.bNameValue = false; 127 lineReader.encoding = HeaderLineReader.ENC_US_ASCII; 128 headerLineReader = HeaderLineReader.getReader(); 129 headerLineReader.bNameValue = true; 130 headerLineReader.encoding = HeaderLineReader.ENC_UTF8; 131 headerLineReader.bLWS = true; 132 headerLineReader.bQuotedText = true; 133 headerLineReader.bEncodedWords = true; 134 fieldParsers = new WarcFieldParsers(); 135 } 136 137 /** 138 * Reset reader for reuse. 139 */ 140 public void reset() { 141 diagnostics.reset(); 142 bIsCompliant = true; 143 consumed = 0; 144 records = 0; 145 errors = 0; 146 warnings = 0; 147 currentRecord = null; 148 } 149 150 /** 151 * Returns a boolean indicating if all records parsed so far are compliant. 152 * @return a boolean indicating if all records parsed so far are compliant 153 */ 154 public boolean isCompliant() { 155 return bIsCompliant; 156 } 157 158 /** 159 * Is this reader assuming GZip compressed input. 160 * @return boolean indicating the assumption of GZip compressed input 161 */ 162 public abstract boolean isCompressed(); 163 164 /** 165 * Set the URI profile used to validate WARC-Target URIs. 166 * If null, the uriProfile is set to RCF3986. 167 * @param uriProfile URI profile to use 168 */ 169 public void setWarcTargetUriProfile(UriProfile uriProfile) { 170 if (uriProfile == null) { 171 uriProfile = UriProfile.RFC3986; 172 } 173 this.warcTargetUriProfile = uriProfile; 174 } 175 176 /** 177 * Get the URI profile used to validate WARC-Target URIs. 178 * @return the URI profile used to validate WARC-Target URIs 179 */ 180 public UriProfile getWarcTargetUriProfile() { 181 return warcTargetUriProfile; 182 } 183 184 /** 185 * Set the URI profile used to validate URIs. 186 * If null, the uriProfile is set to RCF3986. 187 * @param uriProfile URI profile to use 188 */ 189 public void setUriProfile(UriProfile uriProfile) { 190 if (uriProfile == null) { 191 uriProfile = UriProfile.RFC3986; 192 } 193 this.uriProfile = uriProfile; 194 } 195 196 /** 197 * Get the URI profile used to validate URIs. 198 * @return the URI profile used to validate URIs 199 */ 200 public UriProfile getUriProfile() { 201 return uriProfile; 202 } 203 204 /** 205 * Get the readers block digest on/off status. 206 * @return boolean indicating block digest on/off 207 */ 208 public boolean getBlockDigestEnabled() { 209 return bBlockDigest; 210 } 211 212 /** 213 * Set the readers block digest on/off status. Digest, however, 214 * will only be computed if either a Warc-Block-Digest header is 215 * present or an optional algorithm has been chosen. 216 * The Warc-Block-Digest always takes precedence. 217 * @param enabled boolean indicating block digest on/off 218 */ 219 public void setBlockDigestEnabled(boolean enabled) { 220 bBlockDigest = enabled; 221 } 222 223 /** 224 * Get the readers payload digest on/off status. 225 * @return boolean indicating payload digest on/off 226 */ 227 public boolean getPayloadDigestEnabled() { 228 return bPayloadDigest; 229 } 230 231 /** 232 * Set the readers payload digest on/off status. Digest, however, 233 * will only be computed if either a Warc-Payload-Digest header is 234 * present or an optional algorithm has been chosen. 235 * The Warc-Payload-Digest always takes precedence. 236 * @param enabled boolean indicating payload digest on/off 237 */ 238 public void setPayloadDigestEnabled(boolean enabled) { 239 bPayloadDigest = enabled; 240 } 241 242 /** 243 * Get the default block digest algorithm. 244 * @return default block digest algorithm 245 */ 246 public String getBlockDigestAlgorithm() { 247 return blockDigestAlgorithm; 248 } 249 250 /** 251 * Tries to set the default block digest algorithm and returns a boolean 252 * indicating whether the algorithm was accepted or not. This algorithm is 253 * only used in case no WARC payload digest header is present in the record. 254 * @param digestAlgorithm block digest algorithm 255 * (null means no default block digest algorithm is selected) 256 * @return boolean indicating the validity of the algorithm supplied 257 */ 258 public boolean setBlockDigestAlgorithm(String digestAlgorithm) { 259 if (digestAlgorithm == null || digestAlgorithm.length() == 0) { 260 blockDigestAlgorithm = null; 261 return true; 262 } 263 if (WarcDigest.digestAlgorithmLength(digestAlgorithm) > 0) { 264 blockDigestAlgorithm = digestAlgorithm; 265 return true; 266 } 267 return false; 268 } 269 270 /** 271 * Get the default payload digest algorithm. 272 * @return default payload digest algorithm 273 */ 274 public String getPayloadDigestAlgorithm() { 275 return payloadDigestAlgorithm; 276 } 277 278 /** 279 * Tries to set the default payload digest algorithm and returns a boolean 280 * indicating whether the algorithm was accepted or not. This algorithm is 281 * only used in case no WARC payload digest header is present in the record. 282 * @param digestAlgorithm payload digest algorithm 283 * (null means no default payload digest algorithm is selected) 284 * @return boolean indicating the validity of the algorithm supplied 285 */ 286 public boolean setPayloadDigestAlgorithm(String digestAlgorithm) { 287 if (digestAlgorithm == null || digestAlgorithm.length() == 0) { 288 payloadDigestAlgorithm = null; 289 return true; 290 } 291 if (WarcDigest.digestAlgorithmLength(digestAlgorithm) > 0) { 292 payloadDigestAlgorithm = digestAlgorithm; 293 return true; 294 } 295 return false; 296 } 297 298 /** 299 * Get the default block digest encoding scheme. 300 * @return default block digest encoding scheme 301 */ 302 public String getBlockDigestEncoding() { 303 return blockDigestEncoding; 304 } 305 306 /** 307 * Set the default block digest encoding scheme. This scheme is only 308 * used if none can be inferred from an existing block digest header. 309 * @param encodingScheme encoding scheme 310 * (null means default block digest is not encoded) 311 */ 312 public void setBlockDigestEncoding(String encodingScheme) { 313 if (encodingScheme != null && encodingScheme.length() > 0) { 314 blockDigestEncoding = encodingScheme.toLowerCase(); 315 } else { 316 blockDigestEncoding = null; 317 } 318 } 319 320 /** 321 * Get the default payload digest encoding scheme. 322 * @return default payload digest encoding scheme 323 */ 324 public String getPayloadDigestEncoding() { 325 return payloadDigestEncoding; 326 } 327 328 /** 329 * Set the default payload digest encoding scheme. This scheme is only 330 * used if none can be inferred from an existing payload digest header. 331 * @param encodingScheme encoding scheme 332 * (null means default payload digest is not encoded) 333 */ 334 public void setPayloadDigestEncoding(String encodingScheme) { 335 if (encodingScheme != null && encodingScheme.length() > 0) { 336 payloadDigestEncoding = encodingScheme.toLowerCase(); 337 } else { 338 payloadDigestEncoding = null; 339 } 340 } 341 342 /** 343 * Get the max size allowed for a record header. 344 * @return max size allowed for a record header 345 */ 346 public int getRecordHeaderMaxSize() { 347 return recordHeaderMaxSize; 348 } 349 350 /** 351 * Set the max size allowed for a record header. 352 * @param size max size allowed 353 */ 354 public void setRecordHeaderMaxSize(int size) { 355 recordHeaderMaxSize = size; 356 } 357 358 /** 359 * Get the max size allowed for a payload header. 360 * @return max size allowed for a payload header 361 */ 362 public int getPayloadHeaderMaxSize() { 363 return payloadHeaderMaxSize; 364 } 365 366 /** 367 * Set the max size allowed for a payload header. 368 * @param size max size allowed 369 */ 370 public void setPayloadHeaderMaxSize(int size) { 371 payloadHeaderMaxSize = size; 372 } 373 374 /** 375 * Close current record resource(s) and input stream(s). 376 */ 377 public abstract void close(); 378 379 /** 380 * Callback method called when the payload has been processed. 381 */ 382 protected abstract void recordClosed(); 383 384 /** 385 * Get the offset of the current WARC record or -1 if none have been read. 386 * @return offset of the current WARC record or -1 387 */ 388 public abstract long getStartOffset(); 389 390 /** 391 * Get the current offset in the WARC <code>InputStream</code>. 392 * @return offset in WARC <code>InputStream</code> 393 */ 394 public abstract long getOffset(); 395 396 /** 397 * Get number of bytes consumed by this reader. 398 * @return number of bytes consumed by this reader 399 */ 400 public abstract long getConsumed(); 401 402 /** 403 * Parses and gets the next record. 404 * This method is for linear access to records. 405 * @return the next record 406 * @throws IOException I/O exception in parsing process 407 */ 408 public abstract WarcRecord getNextRecord() throws IOException; 409 410 /** 411 * Parses and gets the next record from an <code>Inputstream</code>. 412 * This method is mainly for random access use since there are serious 413 * side-effects involved in using multiple <code>PushBackInputStream</code> 414 * instances. 415 * @param in <code>InputStream</code> used to read next record 416 * @param offset offset provided by caller 417 * @return the next record 418 * @throws IOException I/O exception in parsing process 419 */ 420 public abstract WarcRecord getNextRecordFrom(InputStream in, long offset) 421 throws IOException; 422 423 /** 424 * Parses and gets the next record from an <code>Inputstream</code> wrapped 425 * by a <code>BufferedInputStream</code>. 426 * This method is mainly for random access use since there are serious 427 * side-effects involved in using multiple <code>PushBackInputStream</code> 428 * instances. 429 * @param in <code>InputStream</code> used to read next record 430 * @param offset offset provided by caller 431 * @param buffer_size buffer size to use 432 * @return the next record 433 * @throws IOException I/O exception in parsing process 434 */ 435 public abstract WarcRecord getNextRecordFrom(InputStream in, long offset, 436 int buffer_size) throws IOException; 437 438 /** 439 * Gets an exception thrown in the iterator if any or null. 440 * @return exception thrown in the iterator if any or null 441 */ 442 public Exception getIteratorExceptionThrown() { 443 return iteratorExceptionThrown; 444 } 445 446 public WarcRecordParserCallback getWarcRecordParserCallback() { 447 return wrpCallback; 448 } 449 450 public void setWarcRecordParserCallback(WarcRecordParserCallback wrpCallback) { 451 this.wrpCallback = wrpCallback; 452 } 453 454 /** 455 * Returns an <code>Iterator</code> over the records as they are being 456 * parsed. Any exception thrown during parsing is accessible through the 457 * <code>getIteratorExceptionThrown</code> method. 458 * @return <code>Iterator</code> over the records 459 */ 460 public Iterator<WarcRecord> iterator() { 461 return new Iterator<WarcRecord>() { 462 463 /** Internal next record updated by either hasNext() or next(). */ 464 private WarcRecord next; 465 466 /** Entry returned by next(). */ 467 private WarcRecord current; 468 469 @Override 470 public boolean hasNext() { 471 if (next == null) { 472 iteratorExceptionThrown = null; 473 try { 474 next = getNextRecord(); 475 } catch (IOException e) { 476 iteratorExceptionThrown = e; 477 } 478 } 479 return (next != null); 480 } 481 482 @Override 483 public WarcRecord next() { 484 if (next == null) { 485 iteratorExceptionThrown = null; 486 try { 487 next = getNextRecord(); 488 } catch (IOException e) { 489 iteratorExceptionThrown = e; 490 } 491 } 492 if (next == null) { 493 throw new NoSuchElementException(); 494 } 495 current = next; 496 next = null; 497 return current; 498 } 499 500 @Override 501 public void remove() { 502 throw new UnsupportedOperationException(); 503 } 504 }; 505 } 506 507}