001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.arc; 019 020import java.io.Closeable; 021import java.io.IOException; 022import java.io.InputStream; 023import java.util.Iterator; 024import java.util.NoSuchElementException; 025 026import org.jwat.common.Diagnosis; 027import org.jwat.common.Diagnostics; 028import org.jwat.common.Digest; 029import org.jwat.common.HeaderLineReader; 030import org.jwat.common.UriProfile; 031 032/** 033 * ARC Reader base class. 034 * 035 * @author nicl 036 */ 037public abstract class ArcReader implements Closeable, Iterable<ArcRecordBase> { 038 039 /* 040 * Settings. 041 */ 042 043 /** Strict mode enabled or disabled. 044 * Non strict mode allows newlines when there should be no xml metadata. 045 * Non strict mode allows a varied number of newlines between records. */ 046 protected boolean bStrict = false; 047 048 /** URL URI profile. */ 049 protected UriProfile uriProfile; 050 051 /** Block Digest enabled/disabled. */ 052 protected boolean bBlockDigest = false; 053 054 /** Optional block digest algorithm to use. */ 055 protected String blockDigestAlgorithm; 056 057 /** Encoding scheme used to encode block digest into a string. */ 058 protected String blockDigestEncoding = "base32"; 059 060 /** Payload Digest enabled/disabled. */ 061 protected boolean bPayloadDigest = false; 062 063 /** Optional payload digest algorithm to use. */ 064 protected String payloadDigestAlgorithm; 065 066 /** Encoding scheme used to encode payload digest into a string. */ 067 protected String payloadDigestEncoding = "base32"; 068 069 /** Max size allowed for a record header. */ 070 protected int recordHeaderMaxSize; 071 072 /** Max size allowed for a payload header. */ 073 protected int payloadHeaderMaxSize; 074 075 /** ARC field parser used. */ 076 protected ArcFieldParsers fieldParsers; 077 078 /** Line reader used to read header lines. */ 079 protected HeaderLineReader lineReader; 080 081 /* 082 * State. 083 */ 084 085 /** Reader level errors and warnings or when no record is available. */ 086 public final Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>(); 087 088 /** Compliance status for records parsed up to now. */ 089 protected boolean bIsCompliant = true; 090 091 /** Number of bytes consumed by this reader. */ 092 protected long consumed = 0; 093 094 /** Records parsed. */ 095 protected int records = 0; 096 097 /** Aggregated number of errors encountered while parsing. */ 098 protected int errors = 0; 099 100 /** Aggregate number of warnings encountered while parsing. */ 101 protected int warnings = 0; 102 103 /** Version header from version block. */ 104 public ArcVersionHeader versionHeader; 105 106 /** Current record of either kind. */ 107 protected ArcRecordBase currentRecord = null; 108 109 /** Exception thrown while using the iterator. */ 110 protected Exception iteratorExceptionThrown; 111 112 /** Callback used to monitor and modify internal data as it is being parsed. */ 113 protected ArcRecordParserCallback arpCallback; 114 115 /** 116 * Method used to initialize a readers internal state. 117 */ 118 protected void init() { 119 uriProfile = UriProfile.RFC3986; 120 recordHeaderMaxSize = 8192; 121 payloadHeaderMaxSize = 32768; 122 lineReader = HeaderLineReader.getReader(); 123 lineReader.bNameValue = false; 124 lineReader.encoding = HeaderLineReader.ENC_US_ASCII; 125 fieldParsers = new ArcFieldParsers(); 126 } 127 128 /** 129 * Reset reader for reuse. 130 */ 131 public void reset() { 132 diagnostics.reset(); 133 bIsCompliant = true; 134 consumed = 0; 135 records = 0; 136 errors = 0; 137 warnings = 0; 138 versionHeader = null; 139 currentRecord = null; 140 } 141 142 /** 143 * Returns a boolean indicating whether the reader has only parsed 144 * compliant records up to now. 145 * @return a boolean indicating all compliant records parsed to far 146 */ 147 public boolean isCompliant() { 148 return bIsCompliant; 149 } 150 151 /** 152 * Is this reader assuming GZip compressed input. 153 * @return boolean indicating the assumption of GZip compressed input 154 */ 155 public abstract boolean isCompressed(); 156 157 /** 158 * Set the readers strict mode on/off. 159 * @param bStrict strict mode on/off 160 */ 161 public void setStrict(boolean bStrict) { 162 this.bStrict = bStrict; 163 } 164 165 /** 166 * Get the readers strict mode setting. 167 * @return readers strict mode setting 168 */ 169 public boolean isStrict() { 170 return bStrict; 171 } 172 173 /** 174 * Set the URI profile used to validate URL URIs. 175 * If null, the uriProfile is set to RCF3986. 176 * @param uriProfile URI profile to use 177 */ 178 public void setUriProfile(UriProfile uriProfile) { 179 if (uriProfile == null) { 180 uriProfile = UriProfile.RFC3986; 181 } 182 this.uriProfile = uriProfile; 183 } 184 185 /** 186 * Get the URI profile used to validate URL URIs. 187 * @return the URI profile used to validate URL URIs 188 */ 189 public UriProfile getUriProfile() { 190 return uriProfile; 191 } 192 193 /** 194 * Get the readers block digest on/off status. 195 * @return boolean indicating block digest on/off 196 */ 197 public boolean getBlockDigestEnabled() { 198 return bBlockDigest; 199 } 200 201 /** 202 * Set the readers block digest on/off status. Digest, however, 203 * will only be computed if an algorithm has also been chosen. 204 * @param enabled boolean indicating block digest on/off 205 */ 206 public void setBlockDigestEnabled(boolean enabled) { 207 bBlockDigest = enabled; 208 } 209 210 /** 211 * Get the readers payload digest on/off status. 212 * @return boolean indicating payload digest on/off 213 */ 214 public boolean getPayloadDigestEnabled() { 215 return bPayloadDigest; 216 } 217 218 /** 219 * Set the readers payload digest on/off status. Digest, however, 220 * will only be computed if an algorithm has also been chosen. 221 * @param enabled boolean indicating payload digest on/off 222 */ 223 public void setPayloadDigestEnabled(boolean enabled) { 224 bPayloadDigest = enabled; 225 } 226 227 /** 228 * Get the optional block digest algorithm. 229 * @return optional block digest algorithm 230 */ 231 public String getBlockDigestAlgorithm() { 232 return blockDigestAlgorithm; 233 } 234 235 /** 236 * Tries to set the optional block digest algorithm and returns a boolean 237 * indicating whether the algorithm was accepted or not. 238 * @param digestAlgorithm block digest algorithm 239 * (null means no optional block digest algorithm is selected) 240 * @return boolean indicating the validity of the algorithm supplied 241 */ 242 public boolean setBlockDigestAlgorithm(String digestAlgorithm) { 243 if (digestAlgorithm == null || digestAlgorithm.length() == 0) { 244 blockDigestAlgorithm = null; 245 return true; 246 } 247 if (Digest.digestAlgorithmLength(digestAlgorithm) > 0) { 248 blockDigestAlgorithm = digestAlgorithm; 249 return true; 250 } 251 return false; 252 } 253 254 /** 255 * Get the optional payload digest algorithm. 256 * @return optional payload digest algorithm 257 */ 258 public String getPayloadDigestAlgorithm() { 259 return payloadDigestAlgorithm; 260 } 261 262 /** 263 * Tries to set the optional payload digest algorithm and returns a boolean 264 * indicating whether the algorithm was accepted or not. 265 * @param digestAlgorithm payload digest algorithm 266 * (null means no optional payload digest algorithm is selected) 267 * @return boolean indicating the validity of the algorithm supplied 268 */ 269 public boolean setPayloadDigestAlgorithm(String digestAlgorithm) { 270 if (digestAlgorithm == null || digestAlgorithm.length() == 0) { 271 payloadDigestAlgorithm = null; 272 return true; 273 } 274 if (Digest.digestAlgorithmLength(digestAlgorithm) > 0) { 275 payloadDigestAlgorithm = digestAlgorithm; 276 return true; 277 } 278 return false; 279 } 280 281 /** 282 * Get the optional block digest encoding scheme. 283 * @return optional block digest encoding scheme 284 */ 285 public String getBlockDigestEncoding() { 286 return blockDigestEncoding; 287 } 288 289 /** 290 * Set the optional block digest encoding scheme. 291 * @param encodingScheme encoding scheme 292 * (null means optional block digest is not encoded) 293 */ 294 public void setBlockDigestEncoding(String encodingScheme) { 295 if (encodingScheme != null && encodingScheme.length() > 0) { 296 blockDigestEncoding = encodingScheme.toLowerCase(); 297 } else { 298 blockDigestEncoding = null; 299 } 300 } 301 302 /** 303 * Get the optional payload digest encoding scheme. 304 * @return optional payload digest encoding scheme 305 */ 306 public String getPayloadDigestEncoding() { 307 return payloadDigestEncoding; 308 } 309 310 /** 311 * Set the optional payload digest encoding scheme. 312 * @param encodingScheme encoding scheme 313 * (null means optional payload digest is not encoded) 314 */ 315 public void setPayloadDigestEncoding(String encodingScheme) { 316 if (encodingScheme != null && encodingScheme.length() > 0) { 317 payloadDigestEncoding = encodingScheme.toLowerCase(); 318 } else { 319 payloadDigestEncoding = null; 320 } 321 } 322 323 /** 324 * Get the max size allowed for a record header. 325 * @return max size allowed for a record header 326 */ 327 public int getRecordHeaderMaxSize() { 328 return recordHeaderMaxSize; 329 } 330 331 /** 332 * Set the max size allowed for a record header. 333 * @param size max size allowed 334 */ 335 public void setRecordHeaderMaxSize(int size) { 336 recordHeaderMaxSize = size; 337 } 338 339 /** 340 * Get the max size allowed for a payload header. 341 * @return max size allowed for a payload header 342 */ 343 public int getPayloadHeaderMaxSize() { 344 return payloadHeaderMaxSize; 345 } 346 347 /** 348 * Set the max size allowed for a payload header. 349 * @param size max size allowed 350 */ 351 public void setPayloadHeaderMaxSize(int size) { 352 payloadHeaderMaxSize = size; 353 } 354 355 /** 356 * Close current record resource(s) and input stream(s). 357 */ 358 public abstract void close(); 359 360 /** 361 * Callback method called when the payload has been processed. 362 */ 363 protected abstract void recordClosed(); 364 365 /** 366 * Get the offset of the current ARC record or -1 if none have been read. 367 * @return offset of the current ARC record or -1 368 */ 369 public abstract long getStartOffset(); 370 371 /** 372 * Get the current offset in the ARC <code>InputStream</code>. 373 * @return offset in ARC <code>InputStream</code> 374 * @see ArcRecordBase#getOffset() 375 */ 376 public abstract long getOffset(); 377 378 /** 379 * Get number of bytes consumed by this reader. 380 * @return number of bytes consumed by this reader 381 */ 382 public abstract long getConsumed(); 383 384 /** 385 * Parses and gets the next ARC record. 386 * @return the next ARC record 387 * @throws IOException I/O exception in reading process 388 */ 389 public abstract ArcRecordBase getNextRecord() throws IOException; 390 391 /** 392 * Parses and gets the next ARC record. 393 * @param in ARC record <code>InputStream</code> 394 * @param offset offset provided by caller 395 * @return the next ARC record 396 * @throws IOException I/O exception in reading process 397 */ 398 public abstract ArcRecordBase getNextRecordFrom(InputStream in, long offset) 399 throws IOException; 400 401 /** 402 * Parses and gets the next ARC record. 403 * @param in ARC record <code>InputStream</code> 404 * @param buffer_size size of buffer used to wrap <code>InputStream</code> 405 * @param offset offset provided by caller 406 * @return the next ARC record 407 * @throws IOException I/O exception in reading process 408 */ 409 public abstract ArcRecordBase getNextRecordFrom(InputStream in, 410 long offset, int buffer_size) throws IOException; 411 412 /** 413 * Gets an exception thrown in the iterator if any or null. 414 * @return exception thrown in the iterator if any or null 415 */ 416 public Exception getIteratorExceptionThrown() { 417 return iteratorExceptionThrown; 418 } 419 420 public ArcRecordParserCallback getArcRecordParserCallback() { 421 return arpCallback; 422 } 423 424 public void setArcRecordParserCallback(ArcRecordParserCallback arpCallback) { 425 this.arpCallback = arpCallback; 426 } 427 428 /** 429 * Returns an <code>Iterator</code> over the records as they are being 430 * parsed. Any exception thrown during parsing is accessible through the 431 * <code>getIteratorExceptionThrown</code> method. 432 * @return <code>Iterator</code> over the <code>WARC</code> records 433 */ 434 public Iterator<ArcRecordBase> iterator() { 435 return new Iterator<ArcRecordBase>() { 436 437 private ArcRecordBase next; 438 439 private ArcRecordBase current; 440 441 @Override 442 public boolean hasNext() { 443 if (next == null) { 444 iteratorExceptionThrown = null; 445 try { 446 next = getNextRecord(); 447 } catch (IOException e) { 448 iteratorExceptionThrown = e; 449 } 450 } 451 return (next != null); 452 } 453 454 @Override 455 public ArcRecordBase next() { 456 if (next == null) { 457 iteratorExceptionThrown = null; 458 try { 459 next = getNextRecord(); 460 } catch (IOException e) { 461 iteratorExceptionThrown = e; 462 } 463 } 464 if (next == null) { 465 throw new NoSuchElementException(); 466 } 467 current = next; 468 next = null; 469 return current; 470 } 471 472 @Override 473 public void remove() { 474 throw new UnsupportedOperationException(); 475 } 476 }; 477 } 478 479}