001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.warc;
019
020import java.io.Closeable;
021import java.io.IOException;
022import java.io.InputStream;
023import java.util.Iterator;
024import java.util.NoSuchElementException;
025
026import org.jwat.common.Diagnostics;
027import org.jwat.common.HeaderLineReader;
028import org.jwat.common.UriProfile;
029
030/**
031 * Base class for WARC reader implementations.
032 *
033 * @author nicl
034 */
035public abstract class WarcReader implements Closeable, Iterable<WarcRecord> {
036
037    /*
038     * Settings.
039     */
040
041    /** WARC-Target-URI profile. */
042    protected UriProfile warcTargetUriProfile;
043
044    /** URI profile. */
045    protected UriProfile uriProfile;
046
047    /** Default block digest algorithm to use if none is present in the
048     *  record. */
049    protected String blockDigestAlgorithm;
050
051    /** Default encoding scheme used to encode block digest into a string,
052     *  if none is detected from the record. */
053    protected String blockDigestEncoding = "base32";
054
055    /** Payload Digest enabled/disabled. */
056    protected boolean bPayloadDigest = false;
057
058    /** Default payload digest algorithm to use if none is present in the
059     *  record. */
060    protected String payloadDigestAlgorithm;
061
062    /** Default encoding scheme used to encode payload digest into a string,
063     *  if none is detected from the record. */
064    protected String payloadDigestEncoding = "base32";
065
066    /** Block Digest enabled/disabled. */
067    protected boolean bBlockDigest = false;
068
069    /** Max size allowed for a record header. */
070    protected int recordHeaderMaxSize;
071
072    /** Max size allowed for a payload header. */
073    protected int payloadHeaderMaxSize;
074
075    /** Line reader used to read version lines. */
076    protected HeaderLineReader lineReader;
077
078    /** Header line reader used to read the WARC headers. */
079    protected HeaderLineReader headerLineReader;
080
081    /** WARC field parser used. */
082    protected WarcFieldParsers fieldParsers;
083
084    /** Report HTTP header errors on reader diagnosis list. */
085    protected boolean bReportHttpHeaderError = true;
086
087    /*
088     * State.
089     */
090
091    /** Used for version based validation. */
092    public final WarcValidation validation = new WarcValidation();
093
094    /** Reader level errors and warnings or when no record is available. */
095    public final Diagnostics diagnostics = new Diagnostics();
096
097    /** Compliance status for records parsed up to now. */
098    protected boolean bIsCompliant = true;
099
100    /** Number of bytes consumed by this reader. */
101    protected long consumed = 0;
102
103    /** Records parsed. */
104    protected int records = 0;
105
106    /** Aggregated number of errors encountered while parsing. */
107    protected int errors = 0;
108
109    /** Aggregate number of warnings encountered while parsing. */
110    protected int warnings = 0;
111
112    /** Current WARC record object. */
113    protected WarcRecord currentRecord;
114
115    /** Exception thrown while using the iterator. */
116    protected Exception iteratorExceptionThrown;
117
118    /** Callback used to monitor and modify internal data as it is being parsed. */
119    protected WarcRecordParserCallback wrpCallback;
120
121    /**
122     * Method used to initialize a readers internal state.
123     * Must be called by all constructors.
124     */
125    protected void init() {
126        warcTargetUriProfile = UriProfile.RFC3986;
127        uriProfile = UriProfile.RFC3986;
128        recordHeaderMaxSize = 8192;
129        payloadHeaderMaxSize = 32768;
130        lineReader = HeaderLineReader.getReader();
131        lineReader.bNameValue = false;
132        lineReader.encoding = HeaderLineReader.ENC_US_ASCII;
133        headerLineReader = HeaderLineReader.getReader();
134        headerLineReader.bNameValue = true;
135        headerLineReader.encoding = HeaderLineReader.ENC_UTF8;
136        headerLineReader.bLWS = true;
137        headerLineReader.bQuotedText = true;
138        headerLineReader.bEncodedWords = true;
139        fieldParsers = new WarcFieldParsers();
140    }
141
142    /**
143     * Reset reader for reuse.
144     */
145    public void reset() {
146        diagnostics.reset();
147        bIsCompliant = true;
148        consumed = 0;
149        records = 0;
150        errors = 0;
151        warnings = 0;
152        currentRecord = null;
153    }
154
155    /**
156     * Returns a boolean indicating if all records parsed so far are compliant.
157     * @return a boolean indicating if all records parsed so far are compliant
158     */
159    public boolean isCompliant() {
160        return bIsCompliant;
161    }
162
163    /**
164     * Is this reader assuming GZip compressed input.
165     * @return boolean indicating the assumption of GZip compressed input
166     */
167    public abstract boolean isCompressed();
168
169    /**
170     * Set the URI profile used to validate WARC-Target URIs.
171     * If null, the uriProfile is set to RCF3986.
172     * @param uriProfile URI profile to use
173     */
174    public void setWarcTargetUriProfile(UriProfile uriProfile) {
175        if (uriProfile == null) {
176            uriProfile = UriProfile.RFC3986;
177        }
178        this.warcTargetUriProfile = uriProfile;
179    }
180
181    /**
182     * Get the URI profile used to validate WARC-Target URIs.
183     * @return the URI profile used to validate WARC-Target URIs
184     */
185    public UriProfile getWarcTargetUriProfile() {
186        return warcTargetUriProfile;
187    }
188
189    /**
190     * Set the URI profile used to validate URIs.
191     * If null, the uriProfile is set to RCF3986.
192     * @param uriProfile URI profile to use
193     */
194    public void setUriProfile(UriProfile uriProfile) {
195        if (uriProfile == null) {
196            uriProfile = UriProfile.RFC3986;
197        }
198        this.uriProfile = uriProfile;
199    }
200
201    /**
202     * Get the URI profile used to validate URIs.
203     * @return the URI profile used to validate URIs
204     */
205    public UriProfile getUriProfile() {
206        return uriProfile;
207    }
208
209    /**
210     * Get the readers block digest on/off status.
211     * @return boolean indicating block digest on/off
212     */
213    public boolean getBlockDigestEnabled() {
214        return bBlockDigest;
215    }
216
217    /**
218     * Set the readers block digest on/off status. Digest, however,
219     * will only be computed if either a Warc-Block-Digest header is
220     * present or an optional algorithm has been chosen.
221     * The Warc-Block-Digest always takes precedence.
222     * @param enabled boolean indicating block digest on/off
223     */
224    public void setBlockDigestEnabled(boolean enabled) {
225        bBlockDigest = enabled;
226    }
227
228    /**
229     * Get the readers payload digest on/off status.
230     * @return boolean indicating payload digest on/off
231     */
232    public boolean getPayloadDigestEnabled() {
233        return bPayloadDigest;
234    }
235
236    /**
237     * Set the readers payload digest on/off status. Digest, however,
238     * will only be computed if either a Warc-Payload-Digest header is
239     * present or an optional algorithm has been chosen.
240     * The Warc-Payload-Digest always takes precedence.
241     * @param enabled boolean indicating payload digest on/off
242     */
243    public void setPayloadDigestEnabled(boolean enabled) {
244        bPayloadDigest = enabled;
245    }
246
247    /**
248     * Get the default block digest algorithm.
249     * @return default block digest algorithm
250     */
251    public String getBlockDigestAlgorithm() {
252        return blockDigestAlgorithm;
253    }
254
255    /**
256     * Tries to set the default block digest algorithm and returns a boolean
257     * indicating whether the algorithm was accepted or not. This algorithm is
258     * only used in case no WARC payload digest header is present in the record.
259     * @param digestAlgorithm block digest algorithm
260     * (null means no default block digest algorithm is selected)
261     * @return boolean indicating the validity of the algorithm supplied
262     */
263    public boolean setBlockDigestAlgorithm(String digestAlgorithm) {
264        if (digestAlgorithm == null || digestAlgorithm.length() == 0) {
265            blockDigestAlgorithm = null;
266            return true;
267        }
268        if (WarcDigest.digestAlgorithmLength(digestAlgorithm) > 0) {
269            blockDigestAlgorithm = digestAlgorithm;
270            return true;
271        }
272        return false;
273    }
274
275    /**
276     * Get the default payload digest algorithm.
277     * @return default payload digest algorithm
278     */
279    public String getPayloadDigestAlgorithm() {
280        return payloadDigestAlgorithm;
281    }
282
283    /**
284     * Tries to set the default payload digest algorithm and returns a boolean
285     * indicating whether the algorithm was accepted or not. This algorithm is
286     * only used in case no WARC payload digest header is present in the record.
287     * @param digestAlgorithm payload digest algorithm
288     * (null means no default payload digest algorithm is selected)
289     * @return boolean indicating the validity of the algorithm supplied
290     */
291    public boolean setPayloadDigestAlgorithm(String digestAlgorithm) {
292        if (digestAlgorithm == null || digestAlgorithm.length() == 0) {
293            payloadDigestAlgorithm = null;
294            return true;
295        }
296        if (WarcDigest.digestAlgorithmLength(digestAlgorithm) > 0) {
297            payloadDigestAlgorithm = digestAlgorithm;
298            return true;
299        }
300        return false;
301    }
302
303    /**
304     * Get the default block digest encoding scheme.
305     * @return default block digest encoding scheme
306     */
307    public String getBlockDigestEncoding() {
308        return blockDigestEncoding;
309    }
310
311    /**
312     * Set the default block digest encoding scheme. This scheme is only
313     * used if none can be inferred from an existing block digest header.
314     * @param encodingScheme encoding scheme
315     * (null means default block digest is not encoded)
316     */
317    public void setBlockDigestEncoding(String encodingScheme) {
318        if (encodingScheme != null && encodingScheme.length() > 0) {
319            blockDigestEncoding = encodingScheme.toLowerCase();
320        } else {
321            blockDigestEncoding = null;
322        }
323    }
324
325    /**
326     * Get the default payload digest encoding scheme.
327     * @return default payload digest encoding scheme
328     */
329    public String getPayloadDigestEncoding() {
330        return payloadDigestEncoding;
331    }
332
333    /**
334     * Set the default payload digest encoding scheme. This scheme is only
335     * used if none can be inferred from an existing payload digest header.
336     * @param encodingScheme encoding scheme
337     * (null means default payload digest is not encoded)
338     */
339    public void setPayloadDigestEncoding(String encodingScheme) {
340        if (encodingScheme != null && encodingScheme.length() > 0) {
341            payloadDigestEncoding = encodingScheme.toLowerCase();
342        } else {
343            payloadDigestEncoding = null;
344        }
345    }
346
347    /**
348     * Get the max size allowed for a record header.
349     * @return max size allowed for a record header
350     */
351    public int getRecordHeaderMaxSize() {
352        return recordHeaderMaxSize;
353    }
354
355    /**
356     * Set the max size allowed for a record header.
357     * @param size max size allowed
358     */
359    public void setRecordHeaderMaxSize(int size) {
360        recordHeaderMaxSize = size;
361    }
362
363    /**
364     * Get the max size allowed for a payload header.
365     * @return max size allowed for a payload header
366     */
367    public int getPayloadHeaderMaxSize() {
368        return payloadHeaderMaxSize;
369    }
370
371    /**
372     * Set the max size allowed for a payload header.
373     * @param size max size allowed
374     */
375    public void setPayloadHeaderMaxSize(int size) {
376        payloadHeaderMaxSize = size;
377    }
378
379    /**
380     * Get a boolean indicating whether HTTP header error reporting is enabled or disabled.
381     * @return boolean indicating whether HTTP header error reporting is enabled or disabled
382     */
383    public boolean getReportHttpHeaderErrors() {
384        return bReportHttpHeaderError;
385    }
386
387    /**
388     * Enable or disable the readers HTTP header error being included in the readers diagnosis list.
389     * @param bReportHttpHeaderError Enable or disable HTTP header error reporting
390     */
391    public void setReportHttpHeaderErrors(boolean bReportHttpHeaderError) {
392        this.bReportHttpHeaderError = bReportHttpHeaderError;
393    }
394
395    /**
396     * Close current record resource(s) and input stream(s).
397     */
398    public abstract void close();
399
400    /**
401     * Callback method called when the payload has been processed.
402     */
403    protected abstract void recordClosed();
404
405    /**
406     * Get the offset of the current WARC record or -1 if none have been read.
407     * @return offset of the current WARC record or -1
408     */
409    public abstract long getStartOffset();
410
411    /**
412     * Get the current offset in the WARC <code>InputStream</code>.
413     * @return offset in WARC <code>InputStream</code>
414     */
415    public abstract long getOffset();
416
417    /**
418     * Get number of bytes consumed by this reader.
419     * @return number of bytes consumed by this reader
420     */
421   public abstract long getConsumed();
422
423   /**
424     * Parses and gets the next record.
425     * This method is for linear access to records.
426     * @return the next record
427     * @throws IOException I/O exception in parsing process
428     */
429    public abstract WarcRecord getNextRecord() throws IOException;
430
431    /**
432     * Parses and gets the next record from an <code>Inputstream</code>.
433     * This method is mainly for random access use since there are serious
434     * side-effects involved in using multiple <code>PushBackInputStream</code>
435     * instances.
436     * @param in <code>InputStream</code> used to read next record
437     * @param offset offset provided by caller
438     * @return the next record
439     * @throws IOException I/O exception in parsing process
440     */
441    public abstract WarcRecord getNextRecordFrom(InputStream in, long offset)
442                                                        throws IOException;
443
444    /**
445     * Parses and gets the next record from an <code>Inputstream</code> wrapped
446     * by a <code>BufferedInputStream</code>.
447     * This method is mainly for random access use since there are serious
448     * side-effects involved in using multiple <code>PushBackInputStream</code>
449     * instances.
450     * @param in <code>InputStream</code> used to read next record
451     * @param offset offset provided by caller
452     * @param buffer_size buffer size to use
453     * @return the next record
454     * @throws IOException I/O exception in parsing process
455     */
456    public abstract WarcRecord getNextRecordFrom(InputStream in, long offset,
457                                        int buffer_size) throws IOException;
458
459    /**
460     * Gets an exception thrown in the iterator if any or null.
461     * @return exception thrown in the iterator if any or null
462     */
463    public Exception getIteratorExceptionThrown() {
464        return iteratorExceptionThrown;
465    }
466
467    public WarcRecordParserCallback getWarcRecordParserCallback() {
468        return wrpCallback;
469    }
470
471    public void setWarcRecordParserCallback(WarcRecordParserCallback wrpCallback) {
472        this.wrpCallback = wrpCallback;
473    }
474
475    /**
476     * Returns an <code>Iterator</code> over the records as they are being
477     * parsed. Any exception thrown during parsing is accessible through the
478     * <code>getIteratorExceptionThrown</code> method.
479     * @return <code>Iterator</code> over the records
480     */
481    public Iterator<WarcRecord> iterator() {
482        return new Iterator<WarcRecord>() {
483
484            /** Internal next record updated by either hasNext() or next(). */
485            private WarcRecord next;
486
487            /** Entry returned by next(). */
488            private WarcRecord current;
489
490            @Override
491            public boolean hasNext() {
492                if (next == null) {
493                    iteratorExceptionThrown = null;
494                    try {
495                        next = getNextRecord();
496                    } catch (IOException e) {
497                        iteratorExceptionThrown = e;
498                    }
499                }
500                return (next != null);
501            }
502
503            @Override
504            public WarcRecord next() {
505                if (next == null) {
506                    iteratorExceptionThrown = null;
507                    try {
508                        next = getNextRecord();
509                    } catch (IOException e) {
510                        iteratorExceptionThrown = e;
511                    }
512                }
513                if (next == null) {
514                    throw new NoSuchElementException();
515                }
516                current = next;
517                next = null;
518                return current;
519            }
520
521            @Override
522            public void remove() {
523                throw new UnsupportedOperationException();
524            }
525        };
526    }
527
528}