001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.warc;
019
020import java.io.Closeable;
021import java.io.IOException;
022import java.io.InputStream;
023import java.util.Arrays;
024import java.util.Collections;
025import java.util.List;
026
027import org.jwat.common.Base16;
028import org.jwat.common.Base32;
029import org.jwat.common.Base64;
030import org.jwat.common.ByteCountingPushBackInputStream;
031import org.jwat.common.Diagnosis;
032import org.jwat.common.DiagnosisType;
033import org.jwat.common.Diagnostics;
034import org.jwat.common.HeaderLine;
035import org.jwat.common.HttpHeader;
036import org.jwat.common.NewlineParser;
037import org.jwat.common.Payload;
038import org.jwat.common.PayloadOnClosedHandler;
039
040/**
041 * This class represents a parsed WARC record header block including
042 * possible validation and format warnings/errors encountered in the process.
043 * The payload of the WARC record is accessible through a wrapped payload
044 * object.
045 *
046 * @author nicl
047 */
048public class WarcRecord implements PayloadOnClosedHandler, Closeable {
049
050    /** Reader instance used, required for file compliance. */
051    protected WarcReader reader;
052
053    /** Input stream used to read this record. */
054    protected ByteCountingPushBackInputStream in;
055
056    /** Is this record compliant ie. error free. */
057    protected boolean bIsCompliant;
058
059    /** WARC record parsing start offset relative to the source WARC file input
060     *  stream. Used to keep track of the uncompressed amount of bytes consumed. */
061    protected long startOffset = -1;
062
063    /** Uncompressed bytes consumed while validating this record. */
064    protected long consumed;
065
066    /** Validation errors and warnings. */
067    public final Diagnostics diagnostics = new Diagnostics();
068
069    /** Newline parser for counting/validating trailing newlines. */
070    public NewlineParser nlp = new NewlineParser();
071
072    /** Is Warc-Block-Digest valid. (Null is equal to not tested) */
073    public Boolean isValidBlockDigest = null;
074
075    /** Is Warc-Payload-Digest valid. (Null is equal to not tested) */
076    public Boolean isValidPayloadDigest = null;
077
078    /** Number of trailing newlines after record. */
079    public int trailingNewlines;
080
081    /*
082     * Header-Fields.
083     */
084
085    /** WARC header. */
086    public WarcHeader header;
087
088    /*
089     * Payload
090     */
091
092    /** Has payload been closed before. */
093    protected boolean bPayloadClosed;
094
095    /** Has record been closed before. */
096    protected boolean bClosed;
097
098    /** Payload object if any exists. */
099    protected Payload payload;
100
101    /** HTTP header content parsed from payload. */
102    protected HttpHeader httpHeader;
103
104    /** Computed block digest. */
105    public WarcDigest computedBlockDigest;
106
107    /** Computed payload digest. */
108    public WarcDigest computedPayloadDigest;
109
110    /**
111     * Non public constructor to allow unit testing.
112     */
113    protected WarcRecord() {
114    }
115
116    /**
117     * Create a <code>WarcRecord</code> and prepare it for writing.
118     * @param writer writer which will be used to write the record
119     * @return a <code>WarcRecord</code> ready to be changed and then written
120     */
121    public static WarcRecord createRecord(WarcWriter writer) {
122        WarcRecord record = new WarcRecord();
123        record.header = WarcHeader.initHeader(writer, record.diagnostics);
124        writer.fieldParsers.diagnostics = record.diagnostics;
125        return record;
126    }
127
128    /**
129     * Given an <code>InputStream</code> it tries to read and validate a WARC
130     * header block.
131     * @param in <code>InputStream</code> containing WARC record data
132     * @param reader <code>WarcReader</code> used, with access to user defined
133     * options
134     * @return <code>WarcRecord</code> or <code>null</code>
135     * @throws IOException I/O exception in the process of reading record
136     */
137    public static WarcRecord parseRecord(ByteCountingPushBackInputStream in,
138                                    WarcReader reader) throws IOException {
139        WarcRecord record = new WarcRecord();
140        record.in = in;
141        record.reader = reader;
142        record.startOffset = in.getConsumed();
143        // Initialize WarcHeader with required context.
144        record.header = WarcHeader.initHeader(reader, in.getConsumed(), record.diagnostics);
145        WarcHeader header = record.header;
146        // Initialize WarcFieldParser to report diagnoses here.
147        reader.fieldParsers.diagnostics = record.diagnostics;
148        if (header.parseHeader(in)) {
149            ++reader.records;
150            if (reader.wrpCallback != null) {
151                reader.wrpCallback.warcParsedRecordHeader(reader, record.startOffset, header);
152            }
153            /*
154             * Payload processing.
155             */
156            if (header.contentLength != null && header.contentLength > 0) {
157                /*
158                 * Payload.
159                 */
160                String digestAlgorithm = null;
161                if (reader.bBlockDigest) {
162                    if (header.warcBlockDigest != null && header.warcBlockDigest.algorithm != null) {
163                        // If a WARC block digest header is present in the
164                        // record, use that algorithm.
165                        digestAlgorithm = header.warcBlockDigest.algorithm;
166                    } else {
167                        // If no WARC block digest header is present,
168                        // use the optional user specified algorithm.
169                        // Can be null in which case nothing is computed.
170                        digestAlgorithm = reader.blockDigestAlgorithm;
171                    }
172                }
173                record.payload = Payload.processPayload(in, header.contentLength,
174                                         reader.payloadHeaderMaxSize, digestAlgorithm);
175                record.payload.setOnClosedHandler(record);
176                /*
177                 * HttpHeader.
178                 */
179                if (header.contentType != null
180                        && header.contentType.contentType.equals("application")
181                        && header.contentType.mediaType.equals("http")) {
182                    String value = header.contentType.getParameter("msgtype");
183                    // request
184                    int httpHeaderType = 0;
185                    if ("response".equalsIgnoreCase(value)) {
186                        httpHeaderType = HttpHeader.HT_RESPONSE;
187                    } else if ("request".equalsIgnoreCase(value)) {
188                        httpHeaderType = HttpHeader.HT_REQUEST;
189                    }
190                    if (httpHeaderType != 0) {
191                        digestAlgorithm = null;
192                        if (reader.bPayloadDigest) {
193                            if (header.warcPayloadDigest != null && header.warcPayloadDigest.algorithm != null) {
194                                // If a WARC payload digest header is present in the
195                                // record, use that algorithm.
196                                digestAlgorithm = header.warcPayloadDigest.algorithm;
197                            } else {
198                                // If no WARC payload digest header is present,
199                                // use the optional user specified algorithm.
200                                // Can be null in which case nothing is computed.
201                                digestAlgorithm = reader.payloadDigestAlgorithm;
202                            }
203                        }
204                        // Try to read a valid HTTP request/response header from the payload.
205                        record.httpHeader = HttpHeader.processPayload(httpHeaderType,
206                                record.payload.getInputStream(), header.contentLength,
207                                digestAlgorithm);
208                        if (record.httpHeader != null) {
209                            if (record.httpHeader.isValid()) {
210                                record.payload.setPayloadHeaderWrapped(record.httpHeader);
211                            } else if (reader.bReportHttpHeaderError) {
212                                record.diagnostics.addWarning(
213                                        DiagnosisType.ERROR, "http header", "Unable to parse http header!");
214                            }
215                        }
216                    }
217                }
218            }
219            // Preliminary compliance status, will be updated when the
220            // payload/record is closed.
221            if (record.diagnostics.hasErrors() || record.diagnostics.hasWarnings()) {
222                record.bIsCompliant = false;
223            } else {
224                record.bIsCompliant = true;
225            }
226            reader.bIsCompliant &= record.bIsCompliant;
227        } else {
228            // In case no record is found the errors/warnings in the record object are transfered to the Reader.
229            long excess = in.getConsumed() - record.startOffset;
230            reader.consumed += excess;
231            reader.diagnostics.addAll(record.diagnostics);
232            if (record.diagnostics.hasErrors() || record.diagnostics.hasWarnings()) {
233                reader.errors += record.diagnostics.getErrors().size();
234                reader.warnings += record.diagnostics.getWarnings().size();
235                reader.bIsCompliant = false;
236            }
237            // Require one or more records to be present.
238            if (reader.records == 0) {
239                reader.diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, "WARC file", "One or more records"));
240                ++reader.errors;
241                reader.bIsCompliant = false;
242            }
243            if (excess != 0) {
244                reader.diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA, "Trailing data", "Garbage data found at offset=" + record.startOffset + " - length=" + excess));
245            }
246            // EOF
247            record = null;
248        }
249        return record;
250    }
251
252    /**
253     * Called when the payload object is closed and final steps in the
254     * validation process can be performed.
255     * @throws IOException I/O exception in final validation processing
256     */
257    @Override
258    public void payloadClosed() throws IOException {
259        if (!bPayloadClosed) {
260            if (payload != null) {
261                // Check for truncated payload.
262                if (payload.getUnavailable() > 0) {
263                    // Payload length mismatch - Payload truncated
264                    addErrorDiagnosis(DiagnosisType.INVALID_DATA, "Payload length mismatch", "Payload truncated");
265                }
266                /*
267                 * Check block digest.
268                 */
269                byte[] digest = payload.getDigest();
270                // Check for computed block digest.
271                if (digest != null) {
272                    computedBlockDigest = new WarcDigest();
273                    computedBlockDigest.digestBytes = digest;
274                }
275                // Auto detect encoding used in WARC header.
276                if (header.warcBlockDigest != null && header.warcBlockDigest.digestString != null) {
277                    isValidBlockDigest = processWarcDigest(header.warcBlockDigest, computedBlockDigest, "block");
278                }
279                // Adjust information about computed block digest.
280                if (computedBlockDigest != null) {
281                    processComputedDigest(computedBlockDigest,
282                            reader.blockDigestAlgorithm, reader.blockDigestEncoding, "block");
283                }
284                // Revisit payload digest refers to the original. Continuation payload digest in first record also refers to original.
285                if ((header.warcTypeIdx != null && header.warcTypeIdx != WarcConstants.RT_IDX_REVISIT && header.warcTypeIdx != WarcConstants.RT_IDX_CONTINUATION) && httpHeader != null && httpHeader.isValid()) {
286                    /*
287                     * Check payload digest.
288                     */
289                    digest = httpHeader.getDigest();
290                    // Check for computed payload digest.
291                    if (digest != null) {
292                        computedPayloadDigest = new WarcDigest();
293                        computedPayloadDigest.digestBytes = digest;
294                    }
295                    // Auto detect encoding used in WARC header.
296                    if (header.warcPayloadDigest != null && header.warcPayloadDigest.digestString != null ) {
297                        isValidPayloadDigest = processWarcDigest(header.warcPayloadDigest, computedPayloadDigest, "payload");
298                    }
299                    // Adjust information about computed payload digest.
300                    if (computedPayloadDigest != null) {
301                        processComputedDigest(computedPayloadDigest,
302                                reader.payloadDigestAlgorithm, reader.payloadDigestEncoding, "payload");
303                    }
304                }
305            }
306            // Check for trailing newlines.
307            trailingNewlines = nlp.parseCRLFs(in, diagnostics);
308            if (trailingNewlines != WarcConstants.WARC_RECORD_TRAILING_NEWLINES) {
309                addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED,
310                        "Trailing newlines",
311                        Integer.toString(trailingNewlines),
312                        Integer.toString(WarcConstants.WARC_RECORD_TRAILING_NEWLINES));
313            }
314            // isCompliant status update.
315            if (diagnostics.hasErrors() || diagnostics.hasWarnings()) {
316                bIsCompliant = false;
317                reader.errors += diagnostics.getErrors().size();
318                reader.warnings += diagnostics.getWarnings().size();
319            } else {
320                bIsCompliant = true;
321            }
322            reader.bIsCompliant &= bIsCompliant;
323            // Updated consumed after payload has been consumed.
324            consumed = in.getConsumed() - startOffset;
325            // Don't not close payload again.
326            bPayloadClosed = true;
327            // Callback.
328            reader.recordClosed();
329        }
330    }
331
332    /**
333     * Auto-detect encoding used in WARC digest header and compare it to the
334     * internal one, if it has been computed.
335     * @param warcDigest digest from WARC header
336     * @param computedDigest internally compute digest
337     * @param digestName used to identify the digest ("block" or "payload")
338     * @return WARC digest validity indication
339     */
340    protected Boolean processWarcDigest(WarcDigest warcDigest, WarcDigest computedDigest, String digestName) {
341        byte[] digest;
342        Boolean isValidDigest = null;
343        int digestAlgorithmLength = WarcDigest.digestAlgorithmLength(warcDigest.algorithm);
344        digest = Base16.decodeToArray(warcDigest.digestString);
345        if (digest != null && digest.length == digestAlgorithmLength) {
346            warcDigest.digestBytes = digest;
347            warcDigest.encoding = "base16";
348        }
349        if (warcDigest.digestBytes == null) {
350            digest = Base32.decodeToArray(warcDigest.digestString, true);
351            if (digest != null && digest.length == digestAlgorithmLength) {
352                warcDigest.digestBytes = digest;
353                warcDigest.encoding = "base32";
354            }
355            if (warcDigest.digestBytes == null) {
356                digest = Base64.decodeToArray(warcDigest.digestString, true);
357                if (digest != null && digest.length == digestAlgorithmLength) {
358                    warcDigest.digestBytes = digest;
359                    warcDigest.encoding = "base64";
360                }
361            }
362        }
363        if (warcDigest.encoding == null) {
364            // Encoding - Unrecognized block digest encoding scheme
365            addErrorDiagnosis(DiagnosisType.UNKNOWN,
366                    "Record " + digestName + " digest encoding scheme",
367                    warcDigest.digestString);
368        }
369        if (computedDigest != null) {
370            computedDigest.algorithm = warcDigest.algorithm;
371            computedDigest.encoding = warcDigest.encoding;
372            if (warcDigest.digestBytes != null) {
373                if (!Arrays.equals(computedDigest.digestBytes, warcDigest.digestBytes)) {
374                    // Block digest - Computed block digest does not match
375                    addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED,
376                            "Incorrect " + digestName + " digest",
377                            Base16.encodeArray(warcDigest.digestBytes),
378                            Base16.encodeArray(computedDigest.digestBytes));
379                    isValidDigest = false;
380                } else {
381                    isValidDigest = true;
382                }
383            } else {
384                isValidDigest = false;
385            }
386        }
387        return isValidDigest;
388    }
389
390    /**
391     * Adjust algorithm and encoding information about computed block digest.
392     * @param computedDigest internally compute digest
393     * @param digestAlgorithm default algorithm
394     * @param digestEncoding default encoding
395     * @param digestName used to identify the digest ("block" or "payload")
396     */
397    protected void processComputedDigest(WarcDigest computedDigest, String digestAlgorithm, String digestEncoding, String digestName) {
398        if (computedDigest.algorithm == null) {
399            computedDigest.algorithm = digestAlgorithm;
400        }
401        if (computedDigest.encoding == null && digestEncoding != null) {
402            if ("base32".equals(digestEncoding)) {
403                computedDigest.encoding = "base32";
404            } else if ("base64".equals(digestEncoding)) {
405                computedDigest.encoding = "base64";
406            } else if ("base16".equals(digestEncoding)) {
407                computedDigest.encoding = "base16";
408            } else {
409                // Encoding - Unknown block digest encoding scheme ..
410                addErrorDiagnosis(DiagnosisType.UNKNOWN,
411                        "Default " + digestName + " digest encoding scheme",
412                        digestEncoding);
413            }
414        }
415        if (computedDigest.encoding != null) {
416            if ("base32".equals(computedDigest.encoding)) {
417                computedDigest.digestString = Base32.encodeArray(computedDigest.digestBytes);
418            } else if ("base64".equals(computedDigest.encoding)) {
419                computedDigest.digestString = Base64.encodeArray(computedDigest.digestBytes);
420            } else if ("base16".equals(computedDigest.encoding)) {
421                computedDigest.digestString = Base16.encodeArray(computedDigest.digestBytes);
422            }
423        }
424    }
425
426    /**
427     * Check to see if the record has been closed.
428     * @return boolean indicating whether this record is closed or not
429     */
430    public boolean isClosed() {
431        return bClosed;
432    }
433
434    /**
435     * Close resources associated with the WARC record.
436     * Mainly payload stream if any.
437     * @throws IOException if unable to close resources
438     */
439    public void close() throws IOException {
440        if (!bClosed) {
441            // Ensure input stream is at the end of the record payload.
442            if (payload != null) {
443                payload.close();
444            }
445            payloadClosed();
446            reader = null;
447            in = null;
448            bClosed = true;
449        }
450    }
451
452    /**
453     * Returns a boolean indicating the ISO compliance status of this record.
454     * @return a boolean indicating the ISO compliance status of this record
455     */
456    public boolean isCompliant() {
457        return bIsCompliant;
458    }
459
460    /**
461     * Get the record offset relative to the start of the WARC file
462     * <code>InputStream</code>.
463     * @return the record offset relative to the start of the WARC file
464     */
465    public long getStartOffset() {
466        return header.startOffset;
467    }
468
469    /**
470     * Return number of uncompressed bytes consumed validating this record.
471     * @return number of uncompressed bytes consumed validating this record
472     */
473    public long getConsumed() {
474        return consumed;
475    }
476
477    /**
478     * Get a <code>List</code> of all the non-standard WARC headers found
479     * during parsing.
480     * @return <code>List</code> of <code>HeaderLine</code>
481     */
482    public List<HeaderLine> getHeaderList() {
483        return Collections.unmodifiableList(header.headerList);
484    }
485
486    /**
487     * Get a non-standard WARC header or null, if nothing is stored for this
488     * header name.
489     * @param field header name
490     * @return <code>HeaderLine</code> structure or null
491     */
492    public HeaderLine getHeader(String field) {
493        if (field != null && field.length() > 0) {
494            return header.headerMap.get(field.toLowerCase());
495        } else {
496            return null;
497        }
498    }
499
500    /**
501     * Specifies whether this record has a payload or not.
502     * @return true/false whether the ARC record has a payload
503     */
504    public boolean hasPayload() {
505        return (payload != null);
506    }
507
508    /**
509     * Return Payload object.
510     * @return payload or <code>null</code>
511     */
512    public Payload getPayload() {
513        return payload;
514    }
515
516    /**
517     * Payload content <code>InputStream</code> getter.
518     * @return Payload content <code>InputStream</code>
519     */
520    public InputStream getPayloadContent() {
521        return (payload != null) ? payload.getInputStream() : null;
522    }
523
524    /**
525     * Returns the <code>HttpHeader</code> object if identified in the payload,
526     * or null.
527     * @return the <code>HttpHeader</code> object if identified or null
528     */
529    public HttpHeader getHttpHeader() {
530        return httpHeader;
531    }
532
533    /**
534     * Add an error diagnosis of the given type on a specific entity with
535     * optional extra information. The information varies according to the
536     * diagnosis type.
537     * @param type diagnosis type
538     * @param entity entity examined
539     * @param information optional extra information
540     */
541    protected void addErrorDiagnosis(DiagnosisType type, String entity, String... information) {
542        diagnostics.addError(new Diagnosis(type, entity, information));
543    }
544
545}