001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.warc;
019
020import java.io.Closeable;
021import java.io.IOException;
022import java.io.InputStream;
023import java.util.Arrays;
024import java.util.Collections;
025import java.util.List;
026
027import org.jwat.common.Base16;
028import org.jwat.common.Base32;
029import org.jwat.common.Base64;
030import org.jwat.common.ByteCountingPushBackInputStream;
031import org.jwat.common.Diagnosis;
032import org.jwat.common.DiagnosisType;
033import org.jwat.common.Diagnostics;
034import org.jwat.common.HeaderLine;
035import org.jwat.common.HttpHeader;
036import org.jwat.common.NewlineParser;
037import org.jwat.common.Payload;
038import org.jwat.common.PayloadOnClosedHandler;
039
040/**
041 * This class represents a parsed WARC record header block including
042 * possible validation and format warnings/errors encountered in the process.
043 * The payload of the WARC record is accessible through a wrapped payload
044 * object.
045 *
046 * @author nicl
047 */
048public class WarcRecord implements PayloadOnClosedHandler, Closeable {
049
050    /** Reader instance used, required for file compliance. */
051    protected WarcReader reader;
052
053    /** Input stream used to read this record. */
054    protected ByteCountingPushBackInputStream in;
055
056    /** Is this record compliant ie. error free. */
057    protected boolean bIsCompliant;
058
059    /** WARC record parsing start offset relative to the source WARC file input
060     *  stream. Used to keep track of the uncompressed amount of bytes consumed. */
061    protected long startOffset = -1;
062
063    /** Uncompressed bytes consumed while validating this record. */
064    protected long consumed;
065
066    /** Validation errors and warnings. */
067    public final Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>();
068
069    /** Newline parser for counting/validating trailing newlines. */
070    public NewlineParser nlp = new NewlineParser();
071
072    /** Is Warc-Block-Digest valid. (Null is equal to not tested) */
073    public Boolean isValidBlockDigest = null;
074
075    /** Is Warc-Payload-Digest valid. (Null is equal to not tested) */
076    public Boolean isValidPayloadDigest = null;
077
078    /** Number of trailing newlines after record. */
079    public int trailingNewlines;
080
081    /*
082     * Header-Fields.
083     */
084
085    /** WARC header. */
086    public WarcHeader header;
087
088    /*
089     * Payload
090     */
091
092    /** Has payload been closed before. */
093    protected boolean bPayloadClosed;
094
095    /** Has record been closed before. */
096    protected boolean bClosed;
097
098    /** Payload object if any exists. */
099    protected Payload payload;
100
101    /** HTTP header content parsed from payload. */
102    protected HttpHeader httpHeader;
103
104    /** Computed block digest. */
105    public WarcDigest computedBlockDigest;
106
107    /** Computed payload digest. */
108    public WarcDigest computedPayloadDigest;
109
110    /**
111     * Non public constructor to allow unit testing.
112     */
113    protected WarcRecord() {
114    }
115
116    /**
117     * Create a <code>WarcRecord</code> and prepare it for writing.
118     * @param writer writer which will be used to write the record
119     * @return a <code>WarcRecord</code> ready to be changed and then written
120     */
121    public static WarcRecord createRecord(WarcWriter writer) {
122        WarcRecord record = new WarcRecord();
123        record.header = WarcHeader.initHeader(writer, record.diagnostics);
124        writer.fieldParsers.diagnostics = record.diagnostics;
125        return record;
126    }
127
128    /**
129     * Given an <code>InputStream</code> it tries to read and validate a WARC
130     * header block.
131     * @param in <code>InputStream</code> containing WARC record data
132     * @param reader <code>WarcReader</code> used, with access to user defined
133     * options
134     * @return <code>WarcRecord</code> or <code>null</code>
135     * @throws IOException I/O exception in the process of reading record
136     */
137    public static WarcRecord parseRecord(ByteCountingPushBackInputStream in,
138                                    WarcReader reader) throws IOException {
139        WarcRecord record = new WarcRecord();
140        record.in = in;
141        record.reader = reader;
142        record.startOffset = in.getConsumed();
143        // Initialize WarcHeader with required context.
144        record.header = WarcHeader.initHeader(reader, in.getConsumed(), record.diagnostics);
145        WarcHeader header = record.header;
146        // Initialize WarcFieldParser to report diagnoses here.
147        reader.fieldParsers.diagnostics = record.diagnostics;
148        if (header.parseHeader(in)) {
149            ++reader.records;
150            if (reader.wrpCallback != null) {
151                reader.wrpCallback.warcParsedRecordHeader(reader, record.startOffset, header);
152            }
153            /*
154             * Payload processing.
155             */
156            if (header.contentLength != null && header.contentLength > 0) {
157                /*
158                 * Payload.
159                 */
160                String digestAlgorithm = null;
161                if (reader.bBlockDigest) {
162                    if (header.warcBlockDigest != null && header.warcBlockDigest.algorithm != null) {
163                        // If a WARC block digest header is present in the
164                        // record, use that algorithm.
165                        digestAlgorithm = header.warcBlockDigest.algorithm;
166                    } else {
167                        // If no WARC block digest header is present,
168                        // use the optional user specified algorithm.
169                        // Can be null in which case nothing is computed.
170                        digestAlgorithm = reader.blockDigestAlgorithm;
171                    }
172                }
173                record.payload = Payload.processPayload(in, header.contentLength,
174                                         reader.payloadHeaderMaxSize, digestAlgorithm);
175                record.payload.setOnClosedHandler(record);
176                /*
177                 * HttpHeader.
178                 */
179                if (header.contentType != null
180                        && header.contentType.contentType.equals("application")
181                        && header.contentType.mediaType.equals("http")) {
182                    String value = header.contentType.getParameter("msgtype");
183                    // request
184                    int httpHeaderType = 0;
185                    if ("response".equalsIgnoreCase(value)) {
186                        httpHeaderType = HttpHeader.HT_RESPONSE;
187                    } else if ("request".equalsIgnoreCase(value)) {
188                        httpHeaderType = HttpHeader.HT_REQUEST;
189                    }
190                    if (httpHeaderType != 0) {
191                        digestAlgorithm = null;
192                        if (reader.bPayloadDigest) {
193                            if (header.warcPayloadDigest != null && header.warcPayloadDigest.algorithm != null) {
194                                // If a WARC payload digest header is present in the
195                                // record, use that algorithm.
196                                digestAlgorithm = header.warcPayloadDigest.algorithm;
197                            } else {
198                                // If no WARC payload digest header is present,
199                                // use the optional user specified algorithm.
200                                // Can be null in which case nothing is computed.
201                                digestAlgorithm = reader.payloadDigestAlgorithm;
202                            }
203                        }
204                        // Try to read a valid HTTP request/response header from the payload.
205                        record.httpHeader = HttpHeader.processPayload(httpHeaderType,
206                                record.payload.getInputStream(), header.contentLength,
207                                digestAlgorithm);
208                        if (record.httpHeader != null) {
209                            if (record.httpHeader.isValid()) {
210                                record.payload.setPayloadHeaderWrapped(record.httpHeader);
211                            } else {
212                                record.diagnostics.addError(
213                                        new Diagnosis(DiagnosisType.ERROR,
214                                                "http header",
215                                                "Unable to parse http header!"));
216                            }
217                        }
218                    }
219                }
220            }
221            // Preliminary compliance status, will be updated when the
222            // payload/record is closed.
223            if (record.diagnostics.hasErrors() || record.diagnostics.hasWarnings()) {
224                record.bIsCompliant = false;
225            } else {
226                record.bIsCompliant = true;
227            }
228            reader.bIsCompliant &= record.bIsCompliant;
229        } else {
230            // In case no record is found the errors/warnings in the record object are transfered to the Reader.
231                long excess = in.getConsumed() - record.startOffset;
232            reader.consumed += excess;
233            reader.diagnostics.addAll(record.diagnostics);
234            if (record.diagnostics.hasErrors() || record.diagnostics.hasWarnings()) {
235                reader.errors += record.diagnostics.getErrors().size();
236                reader.warnings += record.diagnostics.getWarnings().size();
237                reader.bIsCompliant = false;
238            }
239            // Require one or more records to be present.
240            if (reader.records == 0) {
241                reader.diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, "WARC file", "One or more records"));
242                ++reader.errors;
243                reader.bIsCompliant = false;
244            }
245                if (excess != 0) {
246                        reader.diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA, "Trailing data", "Garbage data found at offset=" + record.startOffset + " - length=" + excess));
247                }
248            // EOF
249            record = null;
250        }
251        return record;
252    }
253
254    /**
255     * Called when the payload object is closed and final steps in the
256     * validation process can be performed.
257     * @throws IOException I/O exception in final validation processing
258     */
259    @Override
260    public void payloadClosed() throws IOException {
261        if (!bPayloadClosed) {
262            if (payload != null) {
263                // Check for truncated payload.
264                if (payload.getUnavailable() > 0) {
265                    // Payload length mismatch - Payload truncated
266                    addErrorDiagnosis(DiagnosisType.INVALID_DATA, "Payload length mismatch", "Payload truncated");
267                }
268                /*
269                 * Check block digest.
270                 */
271                byte[] digest = payload.getDigest();
272                // Check for computed block digest.
273                if (digest != null) {
274                    computedBlockDigest = new WarcDigest();
275                    computedBlockDigest.digestBytes = digest;
276                }
277                // Auto detect encoding used in WARC header.
278                if (header.warcBlockDigest != null && header.warcBlockDigest.digestString != null) {
279                    isValidBlockDigest = processWarcDigest(header.warcBlockDigest, computedBlockDigest, "block");
280                }
281                // Adjust information about computed block digest.
282                if (computedBlockDigest != null) {
283                    processComputedDigest(computedBlockDigest,
284                            reader.blockDigestAlgorithm, reader.blockDigestEncoding, "block");
285                }
286                // Revisit payload digest refers to the original. Continuation payload digest in first record also refers to original.
287                if ((header.warcTypeIdx != null && header.warcTypeIdx != WarcConstants.RT_IDX_REVISIT && header.warcTypeIdx != WarcConstants.RT_IDX_CONTINUATION) && httpHeader != null && httpHeader.isValid()) {
288                    /*
289                     * Check payload digest.
290                     */
291                    digest = httpHeader.getDigest();
292                    // Check for computed payload digest.
293                    if (digest != null) {
294                        computedPayloadDigest = new WarcDigest();
295                        computedPayloadDigest.digestBytes = digest;
296                    }
297                    // Auto detect encoding used in WARC header.
298                    if (header.warcPayloadDigest != null && header.warcPayloadDigest.digestString != null ) {
299                        isValidPayloadDigest = processWarcDigest(header.warcPayloadDigest, computedPayloadDigest, "payload");
300                    }
301                    // Adjust information about computed payload digest.
302                    if (computedPayloadDigest != null) {
303                        processComputedDigest(computedPayloadDigest,
304                                reader.payloadDigestAlgorithm, reader.payloadDigestEncoding, "payload");
305                    }
306                }
307            }
308            // Check for trailing newlines.
309            trailingNewlines = nlp.parseCRLFs(in, diagnostics);
310            if (trailingNewlines != WarcConstants.WARC_RECORD_TRAILING_NEWLINES) {
311                addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED,
312                        "Trailing newlines",
313                        Integer.toString(trailingNewlines),
314                        Integer.toString(WarcConstants.WARC_RECORD_TRAILING_NEWLINES));
315            }
316            // isCompliant status update.
317            if (diagnostics.hasErrors() || diagnostics.hasWarnings()) {
318                bIsCompliant = false;
319                reader.errors += diagnostics.getErrors().size();
320                reader.warnings += diagnostics.getWarnings().size();
321            } else {
322                bIsCompliant = true;
323            }
324            reader.bIsCompliant &= bIsCompliant;
325            // Updated consumed after payload has been consumed.
326            consumed = in.getConsumed() - startOffset;
327            // Don't not close payload again.
328            bPayloadClosed = true;
329            // Callback.
330            reader.recordClosed();
331        }
332    }
333
334    /**
335     * Auto-detect encoding used in WARC digest header and compare it to the
336     * internal one, if it has been computed.
337     * @param warcDigest digest from WARC header
338     * @param computedDigest internally compute digest
339     * @param digestName used to identify the digest ("block" or "payload")
340     * @return WARC digest validity indication
341     */
342    protected Boolean processWarcDigest(WarcDigest warcDigest, WarcDigest computedDigest, String digestName) {
343        byte[] digest;
344        Boolean isValidDigest = null;
345        int digestAlgorithmLength = WarcDigest.digestAlgorithmLength(warcDigest.algorithm);
346        digest = Base16.decodeToArray(warcDigest.digestString);
347        if (digest != null && digest.length == digestAlgorithmLength) {
348            warcDigest.digestBytes = digest;
349            warcDigest.encoding = "base16";
350        }
351        if (warcDigest.digestBytes == null) {
352            digest = Base32.decodeToArray(warcDigest.digestString, true);
353            if (digest != null && digest.length == digestAlgorithmLength) {
354                warcDigest.digestBytes = digest;
355                warcDigest.encoding = "base32";
356            }
357            if (warcDigest.digestBytes == null) {
358                digest = Base64.decodeToArray(warcDigest.digestString, true);
359                if (digest != null && digest.length == digestAlgorithmLength) {
360                    warcDigest.digestBytes = digest;
361                    warcDigest.encoding = "base64";
362                }
363            }
364        }
365        if (warcDigest.encoding == null) {
366            // Encoding - Unrecognized block digest encoding scheme
367            addErrorDiagnosis(DiagnosisType.UNKNOWN,
368                    "Record " + digestName + " digest encoding scheme",
369                    warcDigest.digestString);
370        }
371        if (computedDigest != null) {
372            computedDigest.algorithm = warcDigest.algorithm;
373            computedDigest.encoding = warcDigest.encoding;
374            if (warcDigest.digestBytes != null) {
375                if (!Arrays.equals(computedDigest.digestBytes, warcDigest.digestBytes)) {
376                    // Block digest - Computed block digest does not match
377                    addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED,
378                            "Incorrect " + digestName + " digest",
379                            Base16.encodeArray(warcDigest.digestBytes),
380                            Base16.encodeArray(computedDigest.digestBytes));
381                    isValidDigest = false;
382                } else {
383                    isValidDigest = true;
384                }
385            } else {
386                isValidDigest = false;
387            }
388        }
389        return isValidDigest;
390    }
391
392    /**
393     * Adjust algorithm and encoding information about computed block digest.
394     * @param computedDigest internally compute digest
395     * @param digestAlgorithm default algorithm
396     * @param digestEncoding default encoding
397     * @param digestName used to identify the digest ("block" or "payload")
398     */
399    protected void processComputedDigest(WarcDigest computedDigest, String digestAlgorithm, String digestEncoding, String digestName) {
400        if (computedDigest.algorithm == null) {
401            computedDigest.algorithm = digestAlgorithm;
402        }
403        if (computedDigest.encoding == null && digestEncoding != null) {
404            if ("base32".equals(digestEncoding)) {
405                computedDigest.encoding = "base32";
406            } else if ("base64".equals(digestEncoding)) {
407                computedDigest.encoding = "base64";
408            } else if ("base16".equals(digestEncoding)) {
409                computedDigest.encoding = "base16";
410            } else {
411                // Encoding - Unknown block digest encoding scheme ..
412                addErrorDiagnosis(DiagnosisType.UNKNOWN,
413                        "Default " + digestName + " digest encoding scheme",
414                        digestEncoding);
415            }
416        }
417        if (computedDigest.encoding != null) {
418            if ("base32".equals(computedDigest.encoding)) {
419                computedDigest.digestString = Base32.encodeArray(computedDigest.digestBytes);
420            } else if ("base64".equals(computedDigest.encoding)) {
421                computedDigest.digestString = Base64.encodeArray(computedDigest.digestBytes);
422            } else if ("base16".equals(computedDigest.encoding)) {
423                computedDigest.digestString = Base16.encodeArray(computedDigest.digestBytes);
424            }
425        }
426    }
427
428    /**
429     * Check to see if the record has been closed.
430     * @return boolean indicating whether this record is closed or not
431     */
432    public boolean isClosed() {
433        return bClosed;
434    }
435
436    /**
437     * Close resources associated with the WARC record.
438     * Mainly payload stream if any.
439     * @throws IOException if unable to close resources
440     */
441    public void close() throws IOException {
442        if (!bClosed) {
443            // Ensure input stream is at the end of the record payload.
444            if (payload != null) {
445                payload.close();
446            }
447            payloadClosed();
448            reader = null;
449            in = null;
450            bClosed = true;
451        }
452    }
453
454    /**
455     * Returns a boolean indicating the ISO compliance status of this record.
456     * @return a boolean indicating the ISO compliance status of this record
457     */
458    public boolean isCompliant() {
459        return bIsCompliant;
460    }
461
462    /**
463     * Get the record offset relative to the start of the WARC file
464     * <code>InputStream</code>.
465     * @return the record offset relative to the start of the WARC file
466     */
467    public long getStartOffset() {
468        return header.startOffset;
469    }
470
471    /**
472     * Return number of uncompressed bytes consumed validating this record.
473     * @return number of uncompressed bytes consumed validating this record
474     */
475    public long getConsumed() {
476        return consumed;
477    }
478
479    /**
480     * Get a <code>List</code> of all the non-standard WARC headers found
481     * during parsing.
482     * @return <code>List</code> of <code>HeaderLine</code>
483     */
484    public List<HeaderLine> getHeaderList() {
485        return Collections.unmodifiableList(header.headerList);
486    }
487
488    /**
489     * Get a non-standard WARC header or null, if nothing is stored for this
490     * header name.
491     * @param field header name
492     * @return <code>HeaderLine</code> structure or null
493     */
494    public HeaderLine getHeader(String field) {
495        if (field != null && field.length() > 0) {
496            return header.headerMap.get(field.toLowerCase());
497        } else {
498            return null;
499        }
500    }
501
502    /**
503     * Specifies whether this record has a payload or not.
504     * @return true/false whether the ARC record has a payload
505     */
506    public boolean hasPayload() {
507        return (payload != null);
508    }
509
510    /**
511     * Return Payload object.
512     * @return payload or <code>null</code>
513     */
514    public Payload getPayload() {
515        return payload;
516    }
517
518    /**
519     * Payload content <code>InputStream</code> getter.
520     * @return Payload content <code>InputStream</code>
521     */
522    public InputStream getPayloadContent() {
523        return (payload != null) ? payload.getInputStream() : null;
524    }
525
526    /**
527     * Returns the <code>HttpHeader</code> object if identified in the payload,
528     * or null.
529     * @return the <code>HttpHeader</code> object if identified or null
530     */
531    public HttpHeader getHttpHeader() {
532        return httpHeader;
533    }
534
535    /**
536     * Add an error diagnosis of the given type on a specific entity with
537     * optional extra information. The information varies according to the
538     * diagnosis type.
539     * @param type diagnosis type
540     * @param entity entity examined
541     * @param information optional extra information
542     */
543    protected void addErrorDiagnosis(DiagnosisType type, String entity, String... information) {
544        diagnostics.addError(new Diagnosis(type, entity, information));
545    }
546
547}