001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.arc;
019
020import java.io.Closeable;
021import java.io.IOException;
022import java.io.InputStream;
023import java.net.InetAddress;
024import java.util.Date;
025
026import org.jwat.common.Base16;
027import org.jwat.common.Base32;
028import org.jwat.common.Base64;
029import org.jwat.common.ByteCountingPushBackInputStream;
030import org.jwat.common.ContentType;
031import org.jwat.common.Diagnosis;
032import org.jwat.common.DiagnosisType;
033import org.jwat.common.Diagnostics;
034import org.jwat.common.Digest;
035import org.jwat.common.HttpHeader;
036import org.jwat.common.NewlineParser;
037import org.jwat.common.Payload;
038import org.jwat.common.PayloadOnClosedHandler;
039import org.jwat.common.PayloadWithHeaderAbstract;
040import org.jwat.common.Uri;
041
042/**
043 * This abstract class represents the common base ARC data which is present in
044 * both a record or version block. The parser in this class is responsible for
045 * parsing the general record line and determining if the rest of the record
046 * should be processed as an ARC record or an ARC version block.
047 * This includes possible common validation and format warnings/errors
048 * encountered in the process.
049 *
050 * @author lbihanic, selghissassi, nicl
051 */
052public abstract class ArcRecordBase implements PayloadOnClosedHandler, Closeable {
053
054    /** Version block record type. */
055    public static final int RT_VERSION_BLOCK = 1;
056
057    /** Arc record record type. */
058    public static final int RT_ARC_RECORD = 2;
059
060    /** Buffer size used in toString(). */
061    public static final int TOSTRING_BUFFER_SIZE = 256;
062
063    /** Is this record compliant ie. error free. */
064    protected boolean bIsCompliant;
065
066    /** Reader instance used, required for file compliance. */
067    protected ArcReader reader;
068
069    /** Input stream used to read this record. */
070    protected ByteCountingPushBackInputStream in;
071
072    /** ARC record version. */
073    protected ArcVersion version;
074
075    /** ARC record parsing start offset relative to the source ARC file input
076     *  stream. Used to keep track of the uncompressed amount of bytes consumed. */
077    protected long startOffset = -1;
078
079    /** Uncompressed bytes consumed while validating this record. */
080    protected long consumed;
081
082    /** Validation errors and warnings. */
083    public Diagnostics<Diagnosis> diagnostics;
084
085    /** Newline parser for counting/validating trailing newlines. */
086    public NewlineParser nlp = new NewlineParser();
087
088    /** Record type, version block or arc record. */
089    public int recordType;
090
091    /** Number of trailing newlines after record. */
092    public int trailingNewLines;
093
094    /*
095     * Header-Fields.
096     */
097
098    /** ARC record header. */
099    public ArcHeader header;
100
101    /** ARC version header. */
102    public ArcVersionHeader versionHeader;
103
104    /*
105     * Payload
106     */
107
108    /** Has payload been closed before. */
109    protected boolean bPayloadClosed;
110
111    /** Has record been closed flag. */
112    protected boolean bClosed;
113
114    /** Payload object if any exists. */
115    protected Payload payload;
116
117    /** Has all the payload data been processed while reading the record header. */
118    protected boolean bHasPseudoEmptyPayload;
119
120    /** HTTP header content parsed from payload. */
121    protected HttpHeader httpHeader;
122
123    /** Computed block digest. */
124    public Digest computedBlockDigest;
125
126    /** Computed payload digest. */
127    public Digest computedPayloadDigest;
128
129    /** This array is used to store metadata which should not be present.
130     *  Only available for forensic purposes. */
131    public byte[] excessiveMetadata;
132
133    /**
134     * Reads from the input stream and tries to parse and identify an
135     * <code>ArcRecord</code> or <code>ArcVersionBlock</code> record.
136     * @param in input stream with ARC record data
137     * @param reader ARC reader used with access to overall configuration and status
138     * @return <code>ArcRecord</code>, <code>ArcVersionBlock</code> or null
139     * @throws IOException I/O exception while parsing ARC record data
140     */
141    public static ArcRecordBase parseRecord(ByteCountingPushBackInputStream in, ArcReader reader) throws IOException {
142        ArcRecordBase record = null;
143        long startOffset = in.getConsumed();
144        // Initialize ArcHeader with required context.
145        Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>();
146        ArcHeader header = ArcHeader.initHeader(reader, startOffset, diagnostics);
147        // Initialize ArcFieldParser to report diagnoses here.
148        reader.fieldParsers.diagnostics = diagnostics;
149        // Returns true if a record has been processed.
150        // And false if only garbage has been processed.
151        if (header.parseHeader(in)) {
152            if (reader.arpCallback != null) {
153                reader.arpCallback.arcParsedRecordHeader(reader, startOffset, header);
154            }
155            if (header.urlScheme != null && header.urlScheme.startsWith(ArcConstants.ARC_SCHEME)) {
156                record = ArcVersionBlock.parseVersionBlock(reader, diagnostics, header, reader.fieldParsers, in);
157                if (record != null) {
158                    reader.versionHeader = record.versionHeader;
159                }
160            }
161            if (record == null) {
162                record = ArcRecord.parseArcRecord(reader, diagnostics, header, in);
163                if (record != null && reader.versionHeader != null) {
164                    record.version = reader.versionHeader.version;
165                }
166            }
167        }
168        // Record identified above.
169        if (record != null) {
170            ++reader.records;
171            record.startOffset = startOffset;
172            // Check read and computed offset value only if we're reading
173            // a plain ARC file, not a GZipped ARC.
174            if ((header.offset != null) && (header.startOffset > 0L)
175                                && (header.offset.longValue() != header.startOffset)) {
176                diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED,
177                        "'" + ArcConstants.FN_OFFSET + "' value",
178                        header.offset.toString(),
179                        Long.toString(header.startOffset)));
180            }
181            if (reader.records == 1) {
182                if (record.recordType == ArcRecordBase.RT_ARC_RECORD) {
183                    diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED,
184                            ArcConstants.ARC_FILE,
185                            "Expected a version block as the first record."));
186                }
187            } else {
188                if (record.recordType == ArcRecordBase.RT_VERSION_BLOCK) {
189                    diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED,
190                            ArcConstants.ARC_FILE,
191                            "Expected an ARC record not version block."));
192                }
193            }
194            if (reader.versionHeader != null && reader.versionHeader.blockDescVersion > 0
195                    && record.header.recordFieldVersion != reader.versionHeader.blockDescVersion) {
196                diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED,
197                        "ARC record does not match the version block definition",
198                        Integer.toString(record.header.recordFieldVersion),
199                        Integer.toString(reader.versionHeader.blockDescVersion)));
200            }
201            // Preliminary compliance status, will be updated when the
202            // payload/record is closed.
203            if (diagnostics.hasErrors() || diagnostics.hasWarnings()) {
204                record.bIsCompliant = false;
205            } else {
206                record.bIsCompliant = true;
207            }
208            reader.bIsCompliant &= record.bIsCompliant;
209        } else {
210            // Transfer errors/warnings identified in the header parser to the reader since we are not returning a record.
211                long excess = in.getConsumed() - startOffset;
212            reader.consumed += excess;
213            reader.diagnostics.addAll(diagnostics);
214            if (diagnostics.hasErrors() || diagnostics.hasWarnings()) {
215                reader.errors += diagnostics.getErrors().size();
216                reader.warnings += diagnostics.getWarnings().size();
217                reader.bIsCompliant = false;
218            }
219            // Require one or more records to be present.
220            if (reader.records == 0) {
221                reader.diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, "ARC file", "One or more records"));
222                ++reader.errors;
223                reader.bIsCompliant = false;
224            }
225                if (excess != 0) {
226                        reader.diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA, "Trailing data", "Garbage data found at offset=" + startOffset + " - length=" + excess));
227                }
228        }
229        return record;
230    }
231
232    /**
233     * Called when the payload object is closed and final steps in the
234     * validation process can be performed.
235     * @throws IOException I/O exception in final validation processing
236     */
237    @Override
238    public void payloadClosed() throws IOException {
239        if (!bPayloadClosed) {
240            if (payload != null) {
241                // Check for truncated payload.
242                if (payload.getUnavailable() > 0) {
243                    // Payload length mismatch - Payload truncated
244                    addErrorDiagnosis(DiagnosisType.INVALID_DATA, "Payload length mismatch", "Payload truncated");
245                }
246                /*
247                 * Check block digest.
248                 */
249                byte[] digest = payload.getDigest();
250                if (digest != null) {
251                    computedBlockDigest = new Digest();
252                    computedBlockDigest.digestBytes = digest;
253                    computedBlockDigest.algorithm = reader.blockDigestAlgorithm;
254                    if (reader.blockDigestEncoding != null) {
255                        if ("base32".equals(reader.blockDigestEncoding)) {
256                            computedBlockDigest.encoding = "base32";
257                            computedBlockDigest.digestString = Base32.encodeArray(computedBlockDigest.digestBytes);
258                        } else if ("base64".equals(reader.blockDigestEncoding)) {
259                            computedBlockDigest.encoding = "base64";
260                            computedBlockDigest.digestString = Base64.encodeArray(computedBlockDigest.digestBytes);
261                        } else if ("base16".equals(reader.blockDigestEncoding)) {
262                            computedBlockDigest.encoding = "base16";
263                            computedBlockDigest.digestString = Base16.encodeArray(computedBlockDigest.digestBytes);
264                        } else {
265                            // Encoding - Unknown block digest encoding scheme ..
266                            addErrorDiagnosis(DiagnosisType.INVALID_DATA,
267                                    "Block digest encoding scheme",
268                                    reader.blockDigestEncoding);
269                        }
270                    }
271                }
272                PayloadWithHeaderAbstract payloadHeaderWrapped = payload.getPayloadHeaderWrapped();
273                if (payloadHeaderWrapped != null && payloadHeaderWrapped.isValid()) {
274                    /*
275                     * Check payload digest.
276                     */
277                    digest = payloadHeaderWrapped.getDigest();
278                    if (digest != null) {
279                        computedPayloadDigest = new Digest();
280                        computedPayloadDigest.digestBytes = digest;
281                        computedPayloadDigest.algorithm = reader.payloadDigestAlgorithm;
282                        if (reader.payloadDigestEncoding != null) {
283                            if ("base32".equals(reader.payloadDigestEncoding)) {
284                                computedPayloadDigest.encoding = "base32";
285                                computedPayloadDigest.digestString = Base32.encodeArray(computedPayloadDigest.digestBytes);
286                            } else if ("base64".equals(reader.payloadDigestEncoding)) {
287                                computedPayloadDigest.encoding = "base64";
288                                computedPayloadDigest.digestString = Base64.encodeArray(computedPayloadDigest.digestBytes);
289                            } else if ("base16".equals(reader.payloadDigestEncoding)) {
290                                computedPayloadDigest.encoding = "base16";
291                                computedPayloadDigest.digestString = Base16.encodeArray(computedPayloadDigest.digestBytes);
292                            } else {
293                                // Encoding - Unknown payload digest encoding scheme ..
294                                addErrorDiagnosis(DiagnosisType.INVALID_DATA,
295                                        "Payload digest encoding scheme",
296                                        reader.payloadDigestEncoding);
297                            }
298                        }
299                    }
300                }
301            }
302            // Check for trailing newlines.
303            trailingNewLines = nlp.parseLFs(in, diagnostics);
304            if (reader.bStrict && trailingNewLines != ArcConstants.ARC_RECORD_TRAILING_NEWLINES) {
305                addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED,
306                        "Trailing newlines",
307                        Integer.toString(trailingNewLines),
308                        Integer.toString(ArcConstants.ARC_RECORD_TRAILING_NEWLINES));
309            }
310            // isCompliant status update.
311            if (diagnostics.hasErrors() || diagnostics.hasWarnings()) {
312                bIsCompliant = false;
313                reader.errors += diagnostics.getErrors().size();
314                reader.warnings += diagnostics.getWarnings().size();
315            } else {
316                bIsCompliant = true;
317            }
318            reader.bIsCompliant &= bIsCompliant;
319            // Updated consumed after payload has been consumed.
320            consumed = in.getConsumed() - startOffset;
321            // Don't not close payload again.
322            bPayloadClosed = true;
323            // Callback.
324            reader.recordClosed();
325        }
326    }
327
328    /**
329     * Check to see if the record has been closed.
330     * @return boolean indicating whether this record is closed or not
331     */
332    public boolean isClosed() {
333        return bClosed;
334    }
335
336    /**
337     * Close resources associated with the ARC record.
338     * Mainly payload stream if any.
339     * @throws IOException I/O exception close the payload resources
340     */
341    public void close() throws IOException {
342        if (!bClosed) {
343            // Ensure input stream is at the end of the record payload.
344            if (payload != null) {
345                payload.close();
346            }
347            payloadClosed();
348            reader = null;
349            in = null;
350            bClosed = true;
351        }
352    }
353
354    /**
355     * Returns a boolean indicating the standard compliance of this record.
356     * @return a boolean indicating the standard compliance of this record
357     */
358    public boolean isCompliant() {
359        return bIsCompliant;
360    }
361
362    /**
363     * Returns the parsing start offset of the record in the containing ARC.
364     * @return the parsing start offset of the record
365     */
366    public long getStartOffset() {
367        return header.startOffset;
368    }
369
370    /**
371     * Return number of uncompressed bytes consumed validating this record.
372     * @return number of uncompressed bytes consumed validating this record
373     */
374    public long getConsumed() {
375        return consumed;
376    }
377
378    /**
379     * Process the ARC record stream for possible payload data.
380     * @param in ARC record <code>InputStream</code>
381     * @param reader <code>ArcReader</code> used, with access to user defined
382     * options
383     * @throws IOException I/O exception in the parsing process
384     */
385    protected abstract void processPayload(ByteCountingPushBackInputStream in,
386                                        ArcReader reader) throws IOException;
387
388    /**
389     * Specifies whether this record has a payload or not.
390     * @return true/false whether the ARC record has a payload
391     */
392    public boolean hasPayload() {
393        return (payload != null);
394    }
395
396    /**
397     * Specifies whether this record has had all it's payload processed already.
398     * @return true/false whether this record's payload has been completely processed
399     */
400    public boolean hasPseudoEmptyPayload() {
401        return bHasPseudoEmptyPayload;
402    }
403
404    /**
405     * Return Payload object.
406     * @return payload or <code>null</code>
407     */
408    public Payload getPayload() {
409        return payload;
410    }
411
412    /**
413     * Payload content <code>InputStream</code> getter.
414     * @return Payload content <code>InputStream</code>
415     */
416    public InputStream getPayloadContent() {
417        return (payload != null) ? payload.getInputStream() : null;
418    }
419
420    /**
421     * Returns the <code>HttpHeader</code> object if identified in the payload,
422     * or null.
423     * @return the <code>HttpHeader</code> object if identified or null
424     */
425    public HttpHeader getHttpHeader() {
426        return httpHeader;
427    }
428
429    /**
430     * Parses the remaining input stream and validates that the characters
431     * encountered are equal to LF or CR.
432     * @param in <code>InputStream</code> to validate
433     * @return true/false based on whether the remaining input stream contains
434     * only LF and CR characters or not.
435     * @throws IOException I/O exception in parsing
436     */
437    public boolean isValidStreamOfCRLF(InputStream in) throws IOException{
438        if (in == null) {
439            throw new IllegalArgumentException("'in' is null!");
440        }
441        boolean isValid = true;
442        int b;
443        while ((b = in.read()) != -1) {
444            if (b != '\n' && b != '\r') {
445                isValid = false;
446                break;
447            }
448        }
449        return isValid;
450    }
451
452    /**
453     * Add an error diagnosis of the given type on a specific entity with
454     * optional extra information. The information varies according to the
455     * diagnosis type.
456     * @param type diagnosis type
457     * @param entity entity examined
458     * @param information optional extra information
459     */
460    protected void addErrorDiagnosis(DiagnosisType type, String entity, String... information) {
461        diagnostics.addError(new Diagnosis(type, entity, information));
462    }
463
464    /**
465     * Add a warning diagnosis of the given type on a specific entity with
466     * optional extra information. The information varies according to the
467     * diagnosis type.
468     * @param type diagnosis type
469     * @param entity entity examined
470     * @param information optional extra information
471     */
472    /*
473    protected void addWarningDiagnosis(DiagnosisType type, String entity, String... information) {
474        diagnostics.addWarning(new Diagnosis(type, entity, information));
475    }
476    */
477
478    /**
479     * Version getter.
480     * @return the version
481     */
482    public ArcVersion getVersion() {
483        return version;
484    }
485
486    /**
487     * Raw URL getter.
488     * @return the raw URL
489     */
490    public String getUrlStr() {
491        return header.urlStr;
492    }
493
494    /**
495     * URL getter.
496     * @return the URL
497     */
498    public Uri getUrl() {
499        return header.urlUri;
500    }
501
502    /**
503     * Protocol getter.
504     * @return the protocol
505     */
506    public String getScheme() {
507        return header.urlScheme;
508    }
509
510    /**
511     * IpAddress getter.
512     * @return the ipAddress
513     */
514    public String getIpAddress() {
515        return header.ipAddressStr;
516    }
517
518    /**
519     * <code>inetAddress</code> getter.
520     * @return the InetAddress
521     */
522    public InetAddress getInetAddress() {
523        return header.inetAddress;
524    }
525
526    /**
527     * Raw ArchiveDate getter.
528     * @return the rawArchiveDate
529     */
530    public String getArchiveDateStr() {
531        return header.archiveDateStr;
532    }
533
534    /**
535     * ArchiveDate getter.
536     * @return the archiveDate
537     */
538    public Date getArchiveDate() {
539        return header.archiveDate;
540    }
541
542    /**
543     * Raw Content-Type getter.
544     * @return the contentType
545     */
546    public String getContentTypeStr() {
547        return header.contentTypeStr;
548    }
549
550    /**
551     * Content-Type getter.
552     * @return the contentType
553     */
554    public ContentType getContentType() {
555        return header.contentType;
556    }
557
558    /**
559     * Raw Result-Code getter.
560     * @return the resultCode
561     */
562    public String getResultCodeStr() {
563        return header.resultCodeStr;
564    }
565
566    /**
567     * Result-Code getter.
568     * @return the resultCode
569     */
570    public Integer getResultCode() {
571        return header.resultCode;
572    }
573
574    /**
575     * Checksum getter.
576     * @return the checksum
577     */
578    public String getChecksum() {
579        return header.checksumStr;
580    }
581
582    /**
583     * Location getter.
584     * @return the location
585     */
586    public String getLocation() {
587        return header.locationStr;
588    }
589
590    /**
591     * Raw Offset getter.
592     * @return the offset
593     */
594    public String getOffsetStr() {
595        return header.offsetStr;
596    }
597
598    /**
599     * Offset getter.
600     * @return the offset
601     */
602    public Long getOffset() {
603        return header.offset;
604    }
605
606    /**
607     * FileName getter.
608     * @return the fileName
609     */
610    public String getFileName() {
611        return header.filenameStr;
612    }
613
614    /**
615     * Raw Length getter.
616     * @return the length
617     */
618    public String getArchiveLengthStr() {
619        return header.archiveLengthStr;
620    }
621
622    /**
623     * Length getter.
624     * @return the length
625     */
626    public Long getArchiveLength() {
627        return header.archiveLength;
628    }
629
630    @Override
631    public String toString() {
632        StringBuilder sb = new StringBuilder(TOSTRING_BUFFER_SIZE);
633        header.toStringBuilder(sb);
634        return sb.toString();
635    }
636
637}