001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.arc;
019
020import java.io.ByteArrayInputStream;
021import java.io.ByteArrayOutputStream;
022import java.io.IOException;
023import java.io.InputStream;
024
025import org.jwat.common.ByteCountingPushBackInputStream;
026import org.jwat.common.Diagnosis;
027import org.jwat.common.DiagnosisType;
028import org.jwat.common.Diagnostics;
029import org.jwat.common.Payload;
030
031/**
032 * This class represents an ARC version block and header including possible
033 * validation and format warnings/errors encountered in the process.
034 * This class also contains the specific ARC version block payload parser.
035 * Any metadata present in the ARC version block is accessible
036 * through the payload object.
037 *
038 * @author lbihanic, selghissassi, nicl
039 */
040public class ArcVersionBlock extends ArcRecordBase {
041
042    /** Buffer size used in toString(). */
043    public static final int TOSTRING_BUFFER_SIZE = 256;
044
045    /**
046     * Protected constructor to force instantiation of version block
047     * from stream.
048     */
049    protected ArcVersionBlock() {
050    }
051
052    /**
053     * Create and initialize a new <code>ArcVersionBlock</code> for writing.
054     * @param writer writer which shall be used
055     * @return an <code>ArcVersionBlock</code> prepared for writing
056     */
057    public static ArcVersionBlock createRecord(ArcWriter writer) {
058        ArcVersionBlock vb = new ArcVersionBlock();
059        vb.trailingNewLines = 1;
060        vb.diagnostics = new Diagnostics<Diagnosis>();
061        vb.header = ArcHeader.initHeader(writer, vb.diagnostics);
062        writer.fieldParsers.diagnostics = vb.diagnostics;
063        return vb;
064    }
065
066    /**
067     * Creates a new <code>VersionBlock</code> based on the supplied header and
068     * the version block in the payload, if present.
069     * @param reader <code>ArcReader</code> used, with access to user defined
070     * options
071     * @param diagnostics diagnostics used to report errors and/or warnings
072     * @param header record header that has already been processed
073     * @param fieldParsers parser used to read and validate fields
074     * @param in <code>InputStream</code> used to read version block
075     * @return an <code>ArcVersionBlock</code>
076     * @throws IOException if an I/O exception occurs in the process of reading
077     * version block
078     */
079    public static ArcVersionBlock parseVersionBlock(ArcReader reader,
080            Diagnostics<Diagnosis> diagnostics,
081            ArcHeader header, ArcFieldParsers fieldParsers,
082            ByteCountingPushBackInputStream in) throws IOException {
083        ArcVersionBlock vb = new ArcVersionBlock();
084        vb.recordType = RT_VERSION_BLOCK;
085        vb.reader = reader;
086        vb.diagnostics = diagnostics;
087        vb.header = header;
088        vb.in = in;
089        vb.processPayload(in, reader);
090        vb.consumed = in.getConsumed() - vb.header.startOffset;
091        return vb;
092    }
093
094    /**
095     * Validates the version block content type.
096     * Errors and/or warnings are reported on the diagnostics object.
097     */
098    protected void validateContentType() {
099        if (header.contentType == null) {
100            // Version block content-type is required.
101            diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED,
102                    "'" + ArcConstants.FN_CONTENT_TYPE + "' value",
103                    ArcConstants.CONTENT_TYPE_FORMAT));
104        } else if (!ArcConstants.VERSION_BLOCK_CONTENT_TYPE.equals(
105                header.contentType.contentType) ||
106                !ArcConstants.VERSION_BLOCK_MEDIA_TYPE.equals(header.contentType.mediaType)) {
107            // Version block content-type should be equal to "text/plain"
108            diagnostics.addWarning(new Diagnosis(DiagnosisType.INVALID_EXPECTED,
109                    "'" + ArcConstants.FN_CONTENT_TYPE + "' value",
110                    header.contentTypeStr,
111                    ArcConstants.CONTENT_TYPE_TEXT_PLAIN));
112        }
113    }
114
115    /**
116     * An ARC version block payload must include a version line and a
117     * block description line and in v1.1 the rest of the payload consists
118     * of XML formatted metadata related to the harvesters configuration.
119     * @param in input stream containing the payload
120     * @param reader <code>ArcReader</code> used, with access to user defined
121     * options
122     * @throws IOException if an I/O exception occurs while reading the payload
123     */
124    @Override
125    protected void processPayload(ByteCountingPushBackInputStream in,
126                                        ArcReader reader) throws IOException {
127        payload = null;
128        validateContentType();
129        if (header.archiveLength != null && header.archiveLength > 0L) {
130            String digestAlgorithm = null;
131            if (reader.bBlockDigest) {
132                digestAlgorithm = reader.blockDigestAlgorithm;
133            }
134            payload = Payload.processPayload(in, header.archiveLength.longValue(),
135                    reader.payloadHeaderMaxSize, digestAlgorithm);
136            payload.setOnClosedHandler(this);
137            // ArcVersionHeader.
138            digestAlgorithm = null;
139            if (reader.bPayloadDigest) {
140                digestAlgorithm = reader.payloadDigestAlgorithm;
141            }
142            // Try to read a valid ARC version block from the payload.
143            versionHeader = ArcVersionHeader.processPayload(
144                    payload.getInputStream(), header.archiveLength.longValue(),
145                    digestAlgorithm, reader.fieldParsers, diagnostics);
146            if (versionHeader != null) {
147                version = versionHeader.version;
148                if (versionHeader.isValid()) {
149                    payload.setPayloadHeaderWrapped(versionHeader);
150                } else {
151                    diagnostics.addError(
152                            new Diagnosis(DiagnosisType.ERROR,
153                                    ArcConstants.ARC_VERSION_BLOCK,
154                                    "Version block is not valid!"));
155                }
156            }
157        } else {
158            diagnostics.addError(
159                    new Diagnosis(DiagnosisType.INVALID,
160                            ArcConstants.ARC_FILE,
161                            "VersionBlock length missing!"));
162        }
163        if (versionHeader != null && versionHeader.isValid()) {
164            if (ArcVersion.VERSION_1_1.equals(version)) {
165                if ((versionHeader.getRemaining() == 0)) {
166                    bHasPseudoEmptyPayload = true;
167                    diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED,
168                            ArcConstants.ARC_FILE,
169                            "Expected metadata payload not found in the version block"));
170                }
171            } else {
172                if (versionHeader.getRemaining() == 0) {
173                    bHasPseudoEmptyPayload = true;
174                } else {
175                    if (!reader.bStrict) {
176                        // I'm going on a limb here that IA's ARC writer will
177                        // not write in excess of 4GB useless newlines.
178                        if (versionHeader.getRemaining() > ArcConstants.ARC_VB_MAX_TRAILING_NEWLINES) {
179                            throw new IOException(
180                                    "This amount of useless data in the version block is unacceptable!");
181                        }
182                        ByteArrayOutputStream out_payload = new ByteArrayOutputStream(
183                                (int)versionHeader.getRemaining());
184                        InputStream in_payload = versionHeader.getPayloadInputStream();
185                        int read;
186                        byte[] tmpBuf = new byte[1024];
187                        while ((read = in_payload.read(tmpBuf)) != -1) {
188                            out_payload.write(tmpBuf, 0, read);
189                        }
190                        in_payload.close();
191                        out_payload.close();
192                        excessiveMetadata = out_payload.toByteArray();
193                        ByteArrayInputStream in_newlines = new ByteArrayInputStream(excessiveMetadata);
194                        if (!isValidStreamOfCRLF(in_newlines)) {
195                            diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA,
196                                    "version block metadata payload",
197                                    "Metadata payload must not be present in this version"));
198                        } else {
199                            bHasPseudoEmptyPayload = true;
200                        }
201                        in_newlines.close();
202                    } else {
203                        diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA,
204                                "version block metadata payload",
205                                "Metadata payload must not be present in this version"));
206                    }
207                }
208            }
209        }
210    }
211
212    @Override
213    public String toString() {
214        StringBuilder builder = new StringBuilder(TOSTRING_BUFFER_SIZE);
215        builder.append("\nVersionBlock : [\n");
216        builder.append(super.toString());
217        builder.append("]\n");
218        if (versionHeader != null) {
219            builder.append(versionHeader.toString());
220        }
221        return builder.toString();
222    }
223
224}