public abstract class ArcReader extends java.lang.Object implements java.io.Closeable, java.lang.Iterable<ArcRecordBase>
| Modifier and Type | Field and Description |
|---|---|
protected ArcRecordParserCallback |
arpCallback
Callback used to monitor and modify internal data as it is being parsed.
|
protected boolean |
bBlockDigest
Block Digest enabled/disabled.
|
protected boolean |
bIsCompliant
Compliance status for records parsed up to now.
|
protected java.lang.String |
blockDigestAlgorithm
Optional block digest algorithm to use.
|
protected java.lang.String |
blockDigestEncoding
Encoding scheme used to encode block digest into a string.
|
protected boolean |
bPayloadDigest
Payload Digest enabled/disabled.
|
protected boolean |
bStrict
Strict mode enabled or disabled.
|
protected long |
consumed
Number of bytes consumed by this reader.
|
protected ArcRecordBase |
currentRecord
Current record of either kind.
|
Diagnostics<Diagnosis> |
diagnostics
Reader level errors and warnings or when no record is available.
|
protected int |
errors
Aggregated number of errors encountered while parsing.
|
protected ArcFieldParsers |
fieldParsers
ARC field parser used.
|
protected java.lang.Exception |
iteratorExceptionThrown
Exception thrown while using the iterator.
|
protected HeaderLineReader |
lineReader
Line reader used to read header lines.
|
protected java.lang.String |
payloadDigestAlgorithm
Optional payload digest algorithm to use.
|
protected java.lang.String |
payloadDigestEncoding
Encoding scheme used to encode payload digest into a string.
|
protected int |
payloadHeaderMaxSize
Max size allowed for a payload header.
|
protected int |
recordHeaderMaxSize
Max size allowed for a record header.
|
protected int |
records
Records parsed.
|
protected UriProfile |
uriProfile
URL URI profile.
|
ArcVersionHeader |
versionHeader
Version header from version block.
|
protected int |
warnings
Aggregate number of warnings encountered while parsing.
|
| Constructor and Description |
|---|
ArcReader() |
| Modifier and Type | Method and Description |
|---|---|
abstract void |
close()
Close current record resource(s) and input stream(s).
|
ArcRecordParserCallback |
getArcRecordParserCallback() |
java.lang.String |
getBlockDigestAlgorithm()
Get the optional block digest algorithm.
|
boolean |
getBlockDigestEnabled()
Get the readers block digest on/off status.
|
java.lang.String |
getBlockDigestEncoding()
Get the optional block digest encoding scheme.
|
abstract long |
getConsumed()
Get number of bytes consumed by this reader.
|
java.lang.Exception |
getIteratorExceptionThrown()
Gets an exception thrown in the iterator if any or null.
|
abstract ArcRecordBase |
getNextRecord()
Parses and gets the next ARC record.
|
abstract ArcRecordBase |
getNextRecordFrom(java.io.InputStream in,
long offset)
Parses and gets the next ARC record.
|
abstract ArcRecordBase |
getNextRecordFrom(java.io.InputStream in,
long offset,
int buffer_size)
Parses and gets the next ARC record.
|
abstract long |
getOffset()
Get the current offset in the ARC
InputStream. |
java.lang.String |
getPayloadDigestAlgorithm()
Get the optional payload digest algorithm.
|
boolean |
getPayloadDigestEnabled()
Get the readers payload digest on/off status.
|
java.lang.String |
getPayloadDigestEncoding()
Get the optional payload digest encoding scheme.
|
int |
getPayloadHeaderMaxSize()
Get the max size allowed for a payload header.
|
int |
getRecordHeaderMaxSize()
Get the max size allowed for a record header.
|
abstract long |
getStartOffset()
Get the offset of the current ARC record or -1 if none have been read.
|
UriProfile |
getUriProfile()
Get the URI profile used to validate URL URIs.
|
protected void |
init()
Method used to initialize a readers internal state.
|
boolean |
isCompliant()
Returns a boolean indicating whether the reader has only parsed
compliant records up to now.
|
abstract boolean |
isCompressed()
Is this reader assuming GZip compressed input.
|
boolean |
isStrict()
Get the readers strict mode setting.
|
java.util.Iterator<ArcRecordBase> |
iterator()
Returns an
Iterator over the records as they are being
parsed. |
protected abstract void |
recordClosed()
Callback method called when the payload has been processed.
|
void |
reset()
Reset reader for reuse.
|
void |
setArcRecordParserCallback(ArcRecordParserCallback arpCallback) |
boolean |
setBlockDigestAlgorithm(java.lang.String digestAlgorithm)
Tries to set the optional block digest algorithm and returns a boolean
indicating whether the algorithm was accepted or not.
|
void |
setBlockDigestEnabled(boolean enabled)
Set the readers block digest on/off status.
|
void |
setBlockDigestEncoding(java.lang.String encodingScheme)
Set the optional block digest encoding scheme.
|
boolean |
setPayloadDigestAlgorithm(java.lang.String digestAlgorithm)
Tries to set the optional payload digest algorithm and returns a boolean
indicating whether the algorithm was accepted or not.
|
void |
setPayloadDigestEnabled(boolean enabled)
Set the readers payload digest on/off status.
|
void |
setPayloadDigestEncoding(java.lang.String encodingScheme)
Set the optional payload digest encoding scheme.
|
void |
setPayloadHeaderMaxSize(int size)
Set the max size allowed for a payload header.
|
void |
setRecordHeaderMaxSize(int size)
Set the max size allowed for a record header.
|
void |
setStrict(boolean bStrict)
Set the readers strict mode on/off.
|
void |
setUriProfile(UriProfile uriProfile)
Set the URI profile used to validate URL URIs.
|
protected boolean bStrict
protected UriProfile uriProfile
protected boolean bBlockDigest
protected java.lang.String blockDigestAlgorithm
protected java.lang.String blockDigestEncoding
protected boolean bPayloadDigest
protected java.lang.String payloadDigestAlgorithm
protected java.lang.String payloadDigestEncoding
protected int recordHeaderMaxSize
protected int payloadHeaderMaxSize
protected ArcFieldParsers fieldParsers
protected HeaderLineReader lineReader
public final Diagnostics<Diagnosis> diagnostics
protected boolean bIsCompliant
protected long consumed
protected int records
protected int errors
protected int warnings
public ArcVersionHeader versionHeader
protected ArcRecordBase currentRecord
protected java.lang.Exception iteratorExceptionThrown
protected ArcRecordParserCallback arpCallback
public ArcReader()
protected void init()
public void reset()
public boolean isCompliant()
public abstract boolean isCompressed()
public void setStrict(boolean bStrict)
bStrict - strict mode on/offpublic boolean isStrict()
public void setUriProfile(UriProfile uriProfile)
uriProfile - URI profile to usepublic UriProfile getUriProfile()
public boolean getBlockDigestEnabled()
public void setBlockDigestEnabled(boolean enabled)
enabled - boolean indicating block digest on/offpublic boolean getPayloadDigestEnabled()
public void setPayloadDigestEnabled(boolean enabled)
enabled - boolean indicating payload digest on/offpublic java.lang.String getBlockDigestAlgorithm()
public boolean setBlockDigestAlgorithm(java.lang.String digestAlgorithm)
digestAlgorithm - block digest algorithm
(null means no optional block digest algorithm is selected)public java.lang.String getPayloadDigestAlgorithm()
public boolean setPayloadDigestAlgorithm(java.lang.String digestAlgorithm)
digestAlgorithm - payload digest algorithm
(null means no optional payload digest algorithm is selected)public java.lang.String getBlockDigestEncoding()
public void setBlockDigestEncoding(java.lang.String encodingScheme)
encodingScheme - encoding scheme
(null means optional block digest is not encoded)public java.lang.String getPayloadDigestEncoding()
public void setPayloadDigestEncoding(java.lang.String encodingScheme)
encodingScheme - encoding scheme
(null means optional payload digest is not encoded)public int getRecordHeaderMaxSize()
public void setRecordHeaderMaxSize(int size)
size - max size allowedpublic int getPayloadHeaderMaxSize()
public void setPayloadHeaderMaxSize(int size)
size - max size allowedpublic abstract void close()
close in interface java.io.Closeableclose in interface java.lang.AutoCloseableprotected abstract void recordClosed()
public abstract long getStartOffset()
public abstract long getOffset()
InputStream.InputStreamArcRecordBase.getOffset()public abstract long getConsumed()
public abstract ArcRecordBase getNextRecord() throws java.io.IOException
java.io.IOException - I/O exception in reading processpublic abstract ArcRecordBase getNextRecordFrom(java.io.InputStream in, long offset) throws java.io.IOException
in - ARC record InputStreamoffset - offset provided by callerjava.io.IOException - I/O exception in reading processpublic abstract ArcRecordBase getNextRecordFrom(java.io.InputStream in, long offset, int buffer_size) throws java.io.IOException
in - ARC record InputStreambuffer_size - size of buffer used to wrap InputStreamoffset - offset provided by callerjava.io.IOException - I/O exception in reading processpublic java.lang.Exception getIteratorExceptionThrown()
public ArcRecordParserCallback getArcRecordParserCallback()
public void setArcRecordParserCallback(ArcRecordParserCallback arpCallback)
public java.util.Iterator<ArcRecordBase> iterator()
Iterator over the records as they are being
parsed. Any exception thrown during parsing is accessible through the
getIteratorExceptionThrown method.iterator in interface java.lang.Iterable<ArcRecordBase>Iterator over the WARC recordsCopyright © 2011–2022. All rights reserved.