001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.warc;
019
020import java.io.Closeable;
021import java.io.IOException;
022import java.io.InputStream;
023import java.util.Iterator;
024import java.util.NoSuchElementException;
025
026import org.jwat.common.Diagnosis;
027import org.jwat.common.Diagnostics;
028import org.jwat.common.HeaderLineReader;
029import org.jwat.common.UriProfile;
030
031/**
032 * Base class for WARC reader implementations.
033 *
034 * @author nicl
035 */
036public abstract class WarcReader implements Closeable, Iterable<WarcRecord> {
037
038    /*
039     * Settings.
040     */
041
042    /** WARC-Target-URI profile. */
043    protected UriProfile warcTargetUriProfile;
044
045    /** URI profile. */
046    protected UriProfile uriProfile;
047
048    /** Default block digest algorithm to use if none is present in the
049     *  record. */
050    protected String blockDigestAlgorithm;
051
052    /** Default encoding scheme used to encode block digest into a string,
053     *  if none is detected from the record. */
054    protected String blockDigestEncoding = "base32";
055
056    /** Payload Digest enabled/disabled. */
057    protected boolean bPayloadDigest = false;
058
059    /** Default payload digest algorithm to use if none is present in the
060     *  record. */
061    protected String payloadDigestAlgorithm;
062
063    /** Default encoding scheme used to encode payload digest into a string,
064     *  if none is detected from the record. */
065    protected String payloadDigestEncoding = "base32";
066
067    /** Block Digest enabled/disabled. */
068    protected boolean bBlockDigest = false;
069
070    /** Max size allowed for a record header. */
071    protected int recordHeaderMaxSize;
072
073    /** Max size allowed for a payload header. */
074    protected int payloadHeaderMaxSize;
075
076    /** Line reader used to read version lines. */
077    protected HeaderLineReader lineReader;
078
079    /** Header line reader used to read the WARC headers. */
080    protected HeaderLineReader headerLineReader;
081
082    /** WARC field parser used. */
083    protected WarcFieldParsers fieldParsers;
084
085    /*
086     * State.
087     */
088
089    /** Reader level errors and warnings or when no record is available. */
090    public final Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>();
091
092    /** Compliance status for records parsed up to now. */
093    protected boolean bIsCompliant = true;
094
095    /** Number of bytes consumed by this reader. */
096    protected long consumed = 0;
097
098    /** Records parsed. */
099    protected int records = 0;
100
101    /** Aggregated number of errors encountered while parsing. */
102    protected int errors = 0;
103
104    /** Aggregate number of warnings encountered while parsing. */
105    protected int warnings = 0;
106
107    /** Current WARC record object. */
108    protected WarcRecord currentRecord;
109
110    /** Exception thrown while using the iterator. */
111    protected Exception iteratorExceptionThrown;
112
113    /** Callback used to monitor and modify internal data as it is being parsed. */
114    protected WarcRecordParserCallback wrpCallback;
115
116    /**
117     * Method used to initialize a readers internal state.
118     * Must be called by all constructors.
119     */
120    protected void init() {
121        warcTargetUriProfile = UriProfile.RFC3986;
122        uriProfile = UriProfile.RFC3986;
123        recordHeaderMaxSize = 8192;
124        payloadHeaderMaxSize = 32768;
125        lineReader = HeaderLineReader.getReader();
126        lineReader.bNameValue = false;
127        lineReader.encoding = HeaderLineReader.ENC_US_ASCII;
128        headerLineReader = HeaderLineReader.getReader();
129        headerLineReader.bNameValue = true;
130        headerLineReader.encoding = HeaderLineReader.ENC_UTF8;
131        headerLineReader.bLWS = true;
132        headerLineReader.bQuotedText = true;
133        headerLineReader.bEncodedWords = true;
134        fieldParsers = new WarcFieldParsers();
135    }
136
137    /**
138     * Reset reader for reuse.
139     */
140    public void reset() {
141        diagnostics.reset();
142        bIsCompliant = true;
143        consumed = 0;
144        records = 0;
145        errors = 0;
146        warnings = 0;
147        currentRecord = null;
148    }
149
150    /**
151     * Returns a boolean indicating if all records parsed so far are compliant.
152     * @return a boolean indicating if all records parsed so far are compliant
153     */
154    public boolean isCompliant() {
155        return bIsCompliant;
156    }
157
158    /**
159     * Is this reader assuming GZip compressed input.
160     * @return boolean indicating the assumption of GZip compressed input
161     */
162    public abstract boolean isCompressed();
163
164    /**
165     * Set the URI profile used to validate WARC-Target URIs.
166     * If null, the uriProfile is set to RCF3986.
167     * @param uriProfile URI profile to use
168     */
169    public void setWarcTargetUriProfile(UriProfile uriProfile) {
170        if (uriProfile == null) {
171            uriProfile = UriProfile.RFC3986;
172        }
173        this.warcTargetUriProfile = uriProfile;
174    }
175
176    /**
177     * Get the URI profile used to validate WARC-Target URIs.
178     * @return the URI profile used to validate WARC-Target URIs
179     */
180    public UriProfile getWarcTargetUriProfile() {
181        return warcTargetUriProfile;
182    }
183
184    /**
185     * Set the URI profile used to validate URIs.
186     * If null, the uriProfile is set to RCF3986.
187     * @param uriProfile URI profile to use
188     */
189    public void setUriProfile(UriProfile uriProfile) {
190        if (uriProfile == null) {
191            uriProfile = UriProfile.RFC3986;
192        }
193        this.uriProfile = uriProfile;
194    }
195
196    /**
197     * Get the URI profile used to validate URIs.
198     * @return the URI profile used to validate URIs
199     */
200    public UriProfile getUriProfile() {
201        return uriProfile;
202    }
203
204    /**
205     * Get the readers block digest on/off status.
206     * @return boolean indicating block digest on/off
207     */
208    public boolean getBlockDigestEnabled() {
209        return bBlockDigest;
210    }
211
212    /**
213     * Set the readers block digest on/off status. Digest, however,
214     * will only be computed if either a Warc-Block-Digest header is
215     * present or an optional algorithm has been chosen.
216     * The Warc-Block-Digest always takes precedence.
217     * @param enabled boolean indicating block digest on/off
218     */
219    public void setBlockDigestEnabled(boolean enabled) {
220        bBlockDigest = enabled;
221    }
222
223    /**
224     * Get the readers payload digest on/off status.
225     * @return boolean indicating payload digest on/off
226     */
227    public boolean getPayloadDigestEnabled() {
228        return bPayloadDigest;
229    }
230
231    /**
232     * Set the readers payload digest on/off status. Digest, however,
233     * will only be computed if either a Warc-Payload-Digest header is
234     * present or an optional algorithm has been chosen.
235     * The Warc-Payload-Digest always takes precedence.
236     * @param enabled boolean indicating payload digest on/off
237     */
238    public void setPayloadDigestEnabled(boolean enabled) {
239        bPayloadDigest = enabled;
240    }
241
242    /**
243     * Get the default block digest algorithm.
244     * @return default block digest algorithm
245     */
246    public String getBlockDigestAlgorithm() {
247        return blockDigestAlgorithm;
248    }
249
250    /**
251     * Tries to set the default block digest algorithm and returns a boolean
252     * indicating whether the algorithm was accepted or not. This algorithm is
253     * only used in case no WARC payload digest header is present in the record.
254     * @param digestAlgorithm block digest algorithm
255     * (null means no default block digest algorithm is selected)
256     * @return boolean indicating the validity of the algorithm supplied
257     */
258    public boolean setBlockDigestAlgorithm(String digestAlgorithm) {
259        if (digestAlgorithm == null || digestAlgorithm.length() == 0) {
260            blockDigestAlgorithm = null;
261            return true;
262        }
263        if (WarcDigest.digestAlgorithmLength(digestAlgorithm) > 0) {
264            blockDigestAlgorithm = digestAlgorithm;
265            return true;
266        }
267        return false;
268    }
269
270    /**
271     * Get the default payload digest algorithm.
272     * @return default payload digest algorithm
273     */
274    public String getPayloadDigestAlgorithm() {
275        return payloadDigestAlgorithm;
276    }
277
278    /**
279     * Tries to set the default payload digest algorithm and returns a boolean
280     * indicating whether the algorithm was accepted or not. This algorithm is
281     * only used in case no WARC payload digest header is present in the record.
282     * @param digestAlgorithm payload digest algorithm
283     * (null means no default payload digest algorithm is selected)
284     * @return boolean indicating the validity of the algorithm supplied
285     */
286    public boolean setPayloadDigestAlgorithm(String digestAlgorithm) {
287        if (digestAlgorithm == null || digestAlgorithm.length() == 0) {
288            payloadDigestAlgorithm = null;
289            return true;
290        }
291        if (WarcDigest.digestAlgorithmLength(digestAlgorithm) > 0) {
292            payloadDigestAlgorithm = digestAlgorithm;
293            return true;
294        }
295        return false;
296    }
297
298    /**
299     * Get the default block digest encoding scheme.
300     * @return default block digest encoding scheme
301     */
302    public String getBlockDigestEncoding() {
303        return blockDigestEncoding;
304    }
305
306    /**
307     * Set the default block digest encoding scheme. This scheme is only
308     * used if none can be inferred from an existing block digest header.
309     * @param encodingScheme encoding scheme
310     * (null means default block digest is not encoded)
311     */
312    public void setBlockDigestEncoding(String encodingScheme) {
313        if (encodingScheme != null && encodingScheme.length() > 0) {
314            blockDigestEncoding = encodingScheme.toLowerCase();
315        } else {
316            blockDigestEncoding = null;
317        }
318    }
319
320    /**
321     * Get the default payload digest encoding scheme.
322     * @return default payload digest encoding scheme
323     */
324    public String getPayloadDigestEncoding() {
325        return payloadDigestEncoding;
326    }
327
328    /**
329     * Set the default payload digest encoding scheme. This scheme is only
330     * used if none can be inferred from an existing payload digest header.
331     * @param encodingScheme encoding scheme
332     * (null means default payload digest is not encoded)
333     */
334    public void setPayloadDigestEncoding(String encodingScheme) {
335        if (encodingScheme != null && encodingScheme.length() > 0) {
336            payloadDigestEncoding = encodingScheme.toLowerCase();
337        } else {
338            payloadDigestEncoding = null;
339        }
340    }
341
342    /**
343     * Get the max size allowed for a record header.
344     * @return max size allowed for a record header
345     */
346    public int getRecordHeaderMaxSize() {
347        return recordHeaderMaxSize;
348    }
349
350    /**
351     * Set the max size allowed for a record header.
352     * @param size max size allowed
353     */
354    public void setRecordHeaderMaxSize(int size) {
355        recordHeaderMaxSize = size;
356    }
357
358    /**
359     * Get the max size allowed for a payload header.
360     * @return max size allowed for a payload header
361     */
362    public int getPayloadHeaderMaxSize() {
363        return payloadHeaderMaxSize;
364    }
365
366    /**
367     * Set the max size allowed for a payload header.
368     * @param size max size allowed
369     */
370    public void setPayloadHeaderMaxSize(int size) {
371        payloadHeaderMaxSize = size;
372    }
373
374    /**
375     * Close current record resource(s) and input stream(s).
376     */
377    public abstract void close();
378
379    /**
380     * Callback method called when the payload has been processed.
381     */
382    protected abstract void recordClosed();
383
384    /**
385     * Get the offset of the current WARC record or -1 if none have been read.
386     * @return offset of the current WARC record or -1
387     */
388    public abstract long getStartOffset();
389
390    /**
391     * Get the current offset in the WARC <code>InputStream</code>.
392     * @return offset in WARC <code>InputStream</code>
393     */
394    public abstract long getOffset();
395
396    /**
397     * Get number of bytes consumed by this reader.
398     * @return number of bytes consumed by this reader
399     */
400   public abstract long getConsumed();
401
402   /**
403     * Parses and gets the next record.
404     * This method is for linear access to records.
405     * @return the next record
406     * @throws IOException I/O exception in parsing process
407     */
408    public abstract WarcRecord getNextRecord() throws IOException;
409
410    /**
411     * Parses and gets the next record from an <code>Inputstream</code>.
412     * This method is mainly for random access use since there are serious
413     * side-effects involved in using multiple <code>PushBackInputStream</code>
414     * instances.
415     * @param in <code>InputStream</code> used to read next record
416     * @param offset offset provided by caller
417     * @return the next record
418     * @throws IOException I/O exception in parsing process
419     */
420    public abstract WarcRecord getNextRecordFrom(InputStream in, long offset)
421                                                        throws IOException;
422
423    /**
424     * Parses and gets the next record from an <code>Inputstream</code> wrapped
425     * by a <code>BufferedInputStream</code>.
426     * This method is mainly for random access use since there are serious
427     * side-effects involved in using multiple <code>PushBackInputStream</code>
428     * instances.
429     * @param in <code>InputStream</code> used to read next record
430     * @param offset offset provided by caller
431     * @param buffer_size buffer size to use
432     * @return the next record
433     * @throws IOException I/O exception in parsing process
434     */
435    public abstract WarcRecord getNextRecordFrom(InputStream in, long offset,
436                                        int buffer_size) throws IOException;
437
438    /**
439     * Gets an exception thrown in the iterator if any or null.
440     * @return exception thrown in the iterator if any or null
441     */
442    public Exception getIteratorExceptionThrown() {
443        return iteratorExceptionThrown;
444    }
445
446    public WarcRecordParserCallback getWarcRecordParserCallback() {
447        return wrpCallback;
448    }
449
450    public void setWarcRecordParserCallback(WarcRecordParserCallback wrpCallback) {
451        this.wrpCallback = wrpCallback;
452    }
453
454    /**
455     * Returns an <code>Iterator</code> over the records as they are being
456     * parsed. Any exception thrown during parsing is accessible through the
457     * <code>getIteratorExceptionThrown</code> method.
458     * @return <code>Iterator</code> over the records
459     */
460    public Iterator<WarcRecord> iterator() {
461        return new Iterator<WarcRecord>() {
462
463            /** Internal next record updated by either hasNext() or next(). */
464            private WarcRecord next;
465
466            /** Entry returned by next(). */
467            private WarcRecord current;
468
469            @Override
470            public boolean hasNext() {
471                if (next == null) {
472                    iteratorExceptionThrown = null;
473                    try {
474                        next = getNextRecord();
475                    } catch (IOException e) {
476                        iteratorExceptionThrown = e;
477                    }
478                }
479                return (next != null);
480            }
481
482            @Override
483            public WarcRecord next() {
484                if (next == null) {
485                    iteratorExceptionThrown = null;
486                    try {
487                        next = getNextRecord();
488                    } catch (IOException e) {
489                        iteratorExceptionThrown = e;
490                    }
491                }
492                if (next == null) {
493                    throw new NoSuchElementException();
494                }
495                current = next;
496                next = null;
497                return current;
498            }
499
500            @Override
501            public void remove() {
502                throw new UnsupportedOperationException();
503            }
504        };
505    }
506
507}