001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.arc;
019
020import java.io.Closeable;
021import java.io.IOException;
022import java.io.InputStream;
023import java.util.Iterator;
024import java.util.NoSuchElementException;
025
026import org.jwat.common.Diagnosis;
027import org.jwat.common.Diagnostics;
028import org.jwat.common.Digest;
029import org.jwat.common.HeaderLineReader;
030import org.jwat.common.UriProfile;
031
032/**
033 * ARC Reader base class.
034 *
035 * @author nicl
036 */
037public abstract class ArcReader implements Closeable, Iterable<ArcRecordBase> {
038
039    /*
040     * Settings.
041     */
042
043    /** Strict mode enabled or disabled.
044     *  Non strict mode allows newlines when there should be no xml metadata.
045     *  Non strict mode allows a varied number of newlines between records. */
046    protected boolean bStrict = false;
047
048    /** URL URI profile. */
049    protected UriProfile uriProfile;
050
051    /** Block Digest enabled/disabled. */
052    protected boolean bBlockDigest = false;
053
054    /** Optional block digest algorithm to use. */
055    protected String blockDigestAlgorithm;
056
057    /** Encoding scheme used to encode block digest into a string. */
058    protected String blockDigestEncoding = "base32";
059
060    /** Payload Digest enabled/disabled. */
061    protected boolean bPayloadDigest = false;
062
063    /** Optional payload digest algorithm to use. */
064    protected String payloadDigestAlgorithm;
065
066    /** Encoding scheme used to encode payload digest into a string. */
067    protected String payloadDigestEncoding = "base32";
068
069    /** Max size allowed for a record header. */
070    protected int recordHeaderMaxSize;
071
072    /** Max size allowed for a payload header. */
073    protected int payloadHeaderMaxSize;
074
075    /** ARC field parser used. */
076    protected ArcFieldParsers fieldParsers;
077
078    /** Line reader used to read header lines. */
079    protected HeaderLineReader lineReader;
080
081    /*
082     * State.
083     */
084
085    /** Reader level errors and warnings or when no record is available. */
086    public final Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>();
087
088    /** Compliance status for records parsed up to now. */
089    protected boolean bIsCompliant = true;
090
091    /** Number of bytes consumed by this reader. */
092    protected long consumed = 0;
093
094    /** Records parsed. */
095    protected int records = 0;
096
097    /** Aggregated number of errors encountered while parsing. */
098    protected int errors = 0;
099
100    /** Aggregate number of warnings encountered while parsing. */
101    protected int warnings = 0;
102
103    /** Version header from version block. */
104    public ArcVersionHeader versionHeader;
105
106    /** Current record of either kind. */
107    protected ArcRecordBase currentRecord = null;
108
109    /** Exception thrown while using the iterator. */
110    protected Exception iteratorExceptionThrown;
111
112    /** Callback used to monitor and modify internal data as it is being parsed. */
113    protected ArcRecordParserCallback arpCallback;
114
115    /**
116     * Method used to initialize a readers internal state.
117     */
118    protected void init() {
119        uriProfile = UriProfile.RFC3986;
120        recordHeaderMaxSize = 8192;
121        payloadHeaderMaxSize = 32768;
122        lineReader = HeaderLineReader.getReader();
123        lineReader.bNameValue = false;
124        lineReader.encoding = HeaderLineReader.ENC_US_ASCII;
125        fieldParsers = new ArcFieldParsers();
126    }
127
128    /**
129     * Reset reader for reuse.
130     */
131    public void reset() {
132        diagnostics.reset();
133        bIsCompliant = true;
134        consumed = 0;
135        records = 0;
136        errors = 0;
137        warnings = 0;
138        versionHeader = null;
139        currentRecord = null;
140    }
141
142    /**
143     * Returns a boolean indicating whether the reader has only parsed
144     * compliant records up to now.
145     * @return a boolean indicating all compliant records parsed to far
146     */
147    public boolean isCompliant() {
148        return bIsCompliant;
149    }
150
151    /**
152     * Is this reader assuming GZip compressed input.
153     * @return boolean indicating the assumption of GZip compressed input
154     */
155    public abstract boolean isCompressed();
156
157    /**
158     * Set the readers strict mode on/off.
159     * @param bStrict strict mode on/off
160     */
161    public void setStrict(boolean bStrict) {
162        this.bStrict = bStrict;
163    }
164
165    /**
166     * Get the readers strict mode setting.
167     * @return readers strict mode setting
168     */
169    public boolean isStrict() {
170        return bStrict;
171    }
172
173    /**
174     * Set the URI profile used to validate URL URIs.
175     * If null, the uriProfile is set to RCF3986.
176     * @param uriProfile URI profile to use
177     */
178    public void setUriProfile(UriProfile uriProfile) {
179        if (uriProfile == null) {
180            uriProfile = UriProfile.RFC3986;
181        }
182        this.uriProfile = uriProfile;
183    }
184
185    /**
186     * Get the URI profile used to validate URL URIs.
187     * @return the URI profile used to validate URL URIs
188     */
189    public UriProfile getUriProfile() {
190        return uriProfile;
191    }
192
193    /**
194     * Get the readers block digest on/off status.
195     * @return boolean indicating block digest on/off
196     */
197    public boolean getBlockDigestEnabled() {
198        return bBlockDigest;
199    }
200
201    /**
202     * Set the readers block digest on/off status. Digest, however,
203     * will only be computed if an algorithm has also been chosen.
204     * @param enabled boolean indicating block digest on/off
205     */
206    public void setBlockDigestEnabled(boolean enabled) {
207        bBlockDigest = enabled;
208    }
209
210    /**
211     * Get the readers payload digest on/off status.
212     * @return boolean indicating payload digest on/off
213     */
214    public boolean getPayloadDigestEnabled() {
215        return bPayloadDigest;
216    }
217
218    /**
219     * Set the readers payload digest on/off status. Digest, however,
220     * will only be computed if an algorithm has also been chosen.
221     * @param enabled boolean indicating payload digest on/off
222     */
223    public void setPayloadDigestEnabled(boolean enabled) {
224        bPayloadDigest = enabled;
225    }
226
227    /**
228     * Get the optional block digest algorithm.
229     * @return optional block digest algorithm
230     */
231    public String getBlockDigestAlgorithm() {
232        return blockDigestAlgorithm;
233    }
234
235    /**
236     * Tries to set the optional block digest algorithm and returns a boolean
237     * indicating whether the algorithm was accepted or not.
238     * @param digestAlgorithm block digest algorithm
239     * (null means no optional block digest algorithm is selected)
240     * @return boolean indicating the validity of the algorithm supplied
241     */
242    public boolean setBlockDigestAlgorithm(String digestAlgorithm) {
243        if (digestAlgorithm == null || digestAlgorithm.length() == 0) {
244            blockDigestAlgorithm = null;
245            return true;
246        }
247        if (Digest.digestAlgorithmLength(digestAlgorithm) > 0) {
248            blockDigestAlgorithm = digestAlgorithm;
249            return true;
250        }
251        return false;
252    }
253
254    /**
255     * Get the optional payload digest algorithm.
256     * @return optional payload digest algorithm
257     */
258    public String getPayloadDigestAlgorithm() {
259        return payloadDigestAlgorithm;
260    }
261
262    /**
263     * Tries to set the optional payload digest algorithm and returns a boolean
264     * indicating whether the algorithm was accepted or not.
265     * @param digestAlgorithm payload digest algorithm
266     * (null means no optional payload digest algorithm is selected)
267     * @return boolean indicating the validity of the algorithm supplied
268     */
269    public boolean setPayloadDigestAlgorithm(String digestAlgorithm) {
270        if (digestAlgorithm == null || digestAlgorithm.length() == 0) {
271            payloadDigestAlgorithm = null;
272            return true;
273        }
274        if (Digest.digestAlgorithmLength(digestAlgorithm) > 0) {
275            payloadDigestAlgorithm = digestAlgorithm;
276            return true;
277        }
278        return false;
279    }
280
281    /**
282     * Get the optional block digest encoding scheme.
283     * @return optional block digest encoding scheme
284     */
285    public String getBlockDigestEncoding() {
286        return blockDigestEncoding;
287    }
288
289    /**
290     * Set the optional block digest encoding scheme.
291     * @param encodingScheme encoding scheme
292     * (null means optional block digest is not encoded)
293     */
294    public void setBlockDigestEncoding(String encodingScheme) {
295        if (encodingScheme != null && encodingScheme.length() > 0) {
296            blockDigestEncoding = encodingScheme.toLowerCase();
297        } else {
298            blockDigestEncoding = null;
299        }
300    }
301
302    /**
303     * Get the optional payload digest encoding scheme.
304     * @return optional payload digest encoding scheme
305     */
306    public String getPayloadDigestEncoding() {
307        return payloadDigestEncoding;
308    }
309
310    /**
311     * Set the optional payload digest encoding scheme.
312     * @param encodingScheme encoding scheme
313     * (null means optional payload digest is not encoded)
314     */
315    public void setPayloadDigestEncoding(String encodingScheme) {
316        if (encodingScheme != null && encodingScheme.length() > 0) {
317            payloadDigestEncoding = encodingScheme.toLowerCase();
318        } else {
319            payloadDigestEncoding = null;
320        }
321    }
322
323    /**
324     * Get the max size allowed for a record header.
325     * @return max size allowed for a record header
326     */
327    public int getRecordHeaderMaxSize() {
328        return recordHeaderMaxSize;
329    }
330
331    /**
332     * Set the max size allowed for a record header.
333     * @param size max size allowed
334     */
335    public void setRecordHeaderMaxSize(int size) {
336        recordHeaderMaxSize = size;
337    }
338
339    /**
340     * Get the max size allowed for a payload header.
341     * @return max size allowed for a payload header
342     */
343    public int getPayloadHeaderMaxSize() {
344        return payloadHeaderMaxSize;
345    }
346
347    /**
348     * Set the max size allowed for a payload header.
349     * @param size max size allowed
350     */
351    public void setPayloadHeaderMaxSize(int size) {
352        payloadHeaderMaxSize = size;
353    }
354
355    /**
356     * Close current record resource(s) and input stream(s).
357     */
358    public abstract void close();
359
360    /**
361     * Callback method called when the payload has been processed.
362     */
363    protected abstract void recordClosed();
364
365    /**
366     * Get the offset of the current ARC record or -1 if none have been read.
367     * @return offset of the current ARC record or -1
368     */
369    public abstract long getStartOffset();
370
371    /**
372     * Get the current offset in the ARC <code>InputStream</code>.
373     * @return offset in ARC <code>InputStream</code>
374     * @see ArcRecordBase#getOffset()
375     */
376    public abstract long getOffset();
377
378    /**
379     * Get number of bytes consumed by this reader.
380     * @return number of bytes consumed by this reader
381     */
382    public abstract long getConsumed();
383
384    /**
385     * Parses and gets the next ARC record.
386     * @return the next ARC record
387     * @throws IOException I/O exception in reading process
388     */
389    public abstract ArcRecordBase getNextRecord() throws IOException;
390
391    /**
392     * Parses and gets the next ARC record.
393     * @param in ARC record <code>InputStream</code>
394     * @param offset offset provided by caller
395     * @return the next ARC record
396     * @throws IOException I/O exception in reading process
397     */
398    public abstract ArcRecordBase getNextRecordFrom(InputStream in, long offset)
399            throws IOException;
400
401    /**
402     * Parses and gets the next ARC record.
403     * @param in ARC record <code>InputStream</code>
404     * @param buffer_size size of buffer used to wrap <code>InputStream</code>
405     * @param offset offset provided by caller
406     * @return the next ARC record
407     * @throws IOException I/O exception in reading process
408     */
409    public abstract ArcRecordBase getNextRecordFrom(InputStream in,
410            long offset, int buffer_size) throws IOException;
411
412    /**
413     * Gets an exception thrown in the iterator if any or null.
414     * @return exception thrown in the iterator if any or null
415     */
416    public Exception getIteratorExceptionThrown() {
417        return iteratorExceptionThrown;
418    }
419
420    public ArcRecordParserCallback getArcRecordParserCallback() {
421        return arpCallback;
422    }
423
424    public void setArcRecordParserCallback(ArcRecordParserCallback arpCallback) {
425        this.arpCallback = arpCallback;
426    }
427
428    /**
429     * Returns an <code>Iterator</code> over the records as they are being
430     * parsed. Any exception thrown during parsing is accessible through the
431     * <code>getIteratorExceptionThrown</code> method.
432     * @return <code>Iterator</code> over the <code>WARC</code> records
433     */
434    public Iterator<ArcRecordBase> iterator() {
435        return new Iterator<ArcRecordBase>() {
436
437            private ArcRecordBase next;
438
439            private ArcRecordBase current;
440
441            @Override
442            public boolean hasNext() {
443                if (next == null) {
444                    iteratorExceptionThrown = null;
445                    try {
446                        next = getNextRecord();
447                    } catch (IOException e) {
448                        iteratorExceptionThrown = e;
449                    }
450                }
451                return (next != null);
452            }
453
454            @Override
455            public ArcRecordBase next() {
456                if (next == null) {
457                    iteratorExceptionThrown = null;
458                    try {
459                        next = getNextRecord();
460                    } catch (IOException e) {
461                        iteratorExceptionThrown = e;
462                    }
463                }
464                if (next == null) {
465                    throw new NoSuchElementException();
466                }
467                current = next;
468                next = null;
469                return current;
470            }
471
472            @Override
473            public void remove() {
474                throw new UnsupportedOperationException();
475            }
476        };
477    }
478
479}