001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.warc;
019
020import java.net.InetAddress;
021import java.util.List;
022
023import org.jwat.common.ContentType;
024import org.jwat.common.DiagnosisType;
025import org.jwat.common.Diagnostics;
026import org.jwat.common.Uri;
027
028/**
029 * WARC validator abstract base class. Has some common fields and methods.
030 *
031 * @author nicl
032 */
033public abstract class WarcValidatorBase extends WarcConstants {
034
035    /** Version denoted as an integer array. */
036    public int[] usedOnVersion;
037
038    /** A (Warc-Types x Warc-Header-Fields) matrix used for policy validation. */
039    protected int[][] field_policy;
040
041    /**
042     * Internal constructor.
043     */
044    protected WarcValidatorBase() {
045    }
046
047    /**
048     * Set the version this validator is used on. Differs from the internal version
049     * if it is used on WARC version that are not yet supported.
050     * @param arr integer array denoting a version
051     */
052    public void setVersions(int [] arr) {
053        if (arr != null) {
054            usedOnVersion = new int[arr.length];
055            System.arraycopy(arr, 0, usedOnVersion, 0, arr.length);
056        }
057    }
058
059    /**
060     * The following section initializes the policy matrix used to check the
061     * usage of each known warc header line against each known warc record
062     * type.
063     * The ISO standard was used to build the data in the matrix.
064     */
065    public abstract void setWarcHeaderPolicyMatrix();
066
067    /**
068     * Validate the WARC header relative to the WARC-Type and according to the
069     * WARC ISO standard.
070     * @param header warc header to validate
071     */
072    protected void checkFields(WarcHeader header) {
073        Diagnostics diagnostics = header.diagnostics;
074
075        String entityStr;
076        String infoStr;
077        Integer warcTypeIdx = header.warcTypeIdx;
078        String warcTypeStr = header.warcTypeStr;
079        Integer warcProfileIdx = header.warcProfileIdx;
080        Uri warcProfileUri = header.warcProfileUri;
081        String warcProfileStr = header.warcProfileStr;
082        Uri warcRecordIdUri = header.warcRecordIdUri;
083        String warcRecordIdStr = header.warcRecordIdStr;
084        WarcDate warcDate = header.warcDate;
085        String warcDateStr = header.warcDateStr;
086        Long contentLength = header.contentLength;
087        String contentLengthStr = header.contentLengthStr;
088        ContentType contentType = header.contentType;
089        String contentTypeStr = header.contentTypeStr;
090        Integer warcSegmentNumber = header.warcSegmentNumber;
091        String warcSegmentNumberStr = header.warcSegmentNumberStr;
092
093        boolean bMandatoryMissing = false;
094
095        /*
096         * Unknown Warc-Type and/or Warc-Profile.
097         */
098
099        if (warcTypeIdx != null && warcTypeIdx == RT_IDX_UNKNOWN) {
100            // Warning: Unknown Warc-Type.
101            entityStr = "'" + FN_WARC_TYPE + "' value";
102            diagnostics.addWarning(DiagnosisType.UNKNOWN, entityStr, warcTypeStr);
103        }
104
105        if (warcProfileIdx != null) {
106            if (warcProfileIdx == WARC_PROFILE_IDX_UNKNOWN) {
107                // Warning: Unknown Warc-Profile.
108                entityStr = "'" + FN_WARC_PROFILE + "' value";
109                diagnostics.addWarning(DiagnosisType.UNKNOWN, entityStr, warcProfileStr);
110            }
111            else {
112                switch (warcProfileIdx) {
113                case WARC10_PROFILE_IDX_IDENTICAL_PAYLOAD_DIGEST:
114                case WARC10_PROFILE_IDX_SERVER_NOT_MODIFIED:
115                    if (header.major != 1 || header.minor != 0) {
116                        // Warning: Unknown Warc-Profile.
117                        entityStr = "'" + FN_WARC_PROFILE + "' value";
118                        infoStr = "WARC/" + header.major + "." + header.minor + " profile";
119                        diagnostics.addWarning(DiagnosisType.ERROR_EXPECTED, entityStr, infoStr);
120                    }
121                    break;
122                case WARC11_PROFILE_IDX_IDENTICAL_PAYLOAD_DIGEST:
123                case WARC11_PROFILE_IDX_SERVER_NOT_MODIFIED:
124                    if (header.major != 1 || header.minor != 1) {
125                        // Warning: Unknown Warc-Profile.
126                        entityStr = "'" + FN_WARC_PROFILE + "' value";
127                        infoStr = "WARC/" + header.major + "." + header.minor + " profile";
128                        diagnostics.addWarning(DiagnosisType.ERROR_EXPECTED, entityStr, infoStr);
129                    }
130                    break;
131                default:
132                    break;
133                }
134            }
135        }
136
137        /*
138         * Mandatory fields.
139         */
140
141        // TODO Required yes, but is it always invalid.
142        if (warcTypeIdx == null) {
143            // Mandatory valid Warc-Type missing.
144            entityStr = "'" + FN_WARC_TYPE + "' header";
145            diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, warcTypeStr);
146            bMandatoryMissing = true;
147        }
148        if (warcRecordIdUri == null) {
149            // Mandatory valid Warc-Record-Id missing.
150            entityStr = "'" + FN_WARC_RECORD_ID + "' header";
151            diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, warcRecordIdStr);
152            bMandatoryMissing = true;
153        }
154        if (warcDate == null) {
155            // Mandatory valid Warc-Date missing.
156            entityStr = "'" + FN_WARC_DATE + "' header";
157            diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, warcDateStr);
158            bMandatoryMissing = true;
159        }
160        if (contentLength == null) {
161            // Mandatory valid Content-Length missing.
162            entityStr = "'" + FN_CONTENT_LENGTH + "' header";
163            diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, contentLengthStr);
164            bMandatoryMissing = true;
165        }
166
167        if (warcDate != null) {
168            if (header.major == 1 && header.minor == 0 && warcDate.precision != WarcDate.P_SECOND) {
169                diagnostics.addError(DiagnosisType.ERROR_EXPECTED, "'" + FN_WARC_DATE + "' value", warcDateStr, WarcConstants.WARC_DATE_FORMAT);
170            }
171        }
172
173        header.bMandatoryMissing = bMandatoryMissing;
174
175        /*
176         * Content-Type should be present if Content-Length > 0.
177         * Except for continuation records.
178         */
179
180        if (contentLength != null && contentLength.longValue() > 0L &&
181                        (contentTypeStr == null || contentTypeStr.length() == 0)) {
182            if (warcTypeIdx == null || warcTypeIdx != WarcConstants.RT_IDX_CONTINUATION) {
183                entityStr = "'" + FN_CONTENT_TYPE + "' header";
184                diagnostics.addWarning(DiagnosisType.RECOMMENDED_MISSING, entityStr);
185            }
186        }
187
188        /*
189         * WARC record type dependent policies.
190         */
191
192        if (warcTypeIdx != null) {
193            /*
194             * Warcinfo record should have "application/warc-fields" content-type.
195             */
196
197            if (warcTypeIdx == WarcConstants.RT_IDX_WARCINFO) {
198                if (contentType != null &&
199                        (!contentType.contentType.equals("application")
200                        || !contentType.mediaType.equals("warc-fields"))) {
201                    entityStr = "'" + FN_CONTENT_TYPE + "' value";
202                    diagnostics.addWarning(DiagnosisType.RECOMMENDED, entityStr, WarcConstants.CT_APP_WARC_FIELDS, contentTypeStr);
203                }
204            }
205
206            if (warcTypeIdx == WarcConstants.RT_IDX_RESPONSE) {
207                if (warcSegmentNumber != null && warcSegmentNumber != 1) {
208                    entityStr = "'" + FN_WARC_SEGMENT_NUMBER + "' value";
209                    diagnostics.addError(DiagnosisType.INVALID_EXPECTED, entityStr, warcSegmentNumber.toString(), "1");
210                }
211            }
212
213            if (warcTypeIdx == WarcConstants.RT_IDX_CONTINUATION) {
214                if (warcSegmentNumber != null && warcSegmentNumber < 2) {
215                    entityStr = "'" + FN_WARC_SEGMENT_NUMBER + "' value";
216                    diagnostics.addError(DiagnosisType.INVALID_EXPECTED, entityStr, warcSegmentNumber.toString(), ">1");
217                }
218            }
219
220            /*
221             * Check the policies for each field.
222             */
223
224            InetAddress warcInetAddress = header.warcInetAddress;
225            String warcIpAddress = header.warcIpAddress;
226            List<WarcConcurrentTo> warcConcurrentToList = header.warcConcurrentToList;
227            Uri warcRefersToUri = header.warcRefersToUri;
228            String warcRefersToStr = header.warcRefersToStr;
229            Uri warcTargetUriUri = header.warcTargetUriUri;
230            String warcTargetUriStr = header.warcTargetUriStr;
231            Integer warcTruncatedIdx = header.warcTruncatedIdx;
232            String warcTruncatedStr = header.warcTruncatedStr;
233            Uri warcWarcinfoIdUri = header.warcWarcinfoIdUri;
234            String warcWarcinfoIdStr = header.warcWarcinfoIdStr;
235            WarcDigest warcBlockDigest = header.warcBlockDigest;
236            String warcBlockDigestStr = header.warcBlockDigestStr;
237            WarcDigest warcPayloadDigest = header.warcPayloadDigest;
238            String warcPayloadDigestStr = header.warcPayloadDigestStr;
239            String warcFilename = header.warcFilename;
240            ContentType warcIdentifiedPayloadType = header.warcIdentifiedPayloadType;
241            String warcIdentifiedPayloadTypeStr = header.warcIdentifiedPayloadTypeStr;
242            Uri warcSegmentOriginIdUrl = header.warcSegmentOriginIdUrl;
243            String warcSegmentOriginIdStr = header.warcSegmentOriginIdStr;
244            Long warcSegmentTotalLength = header.warcSegmentTotalLength;
245            String warcSegmentTotalLengthStr = header.warcSegmentTotalLengthStr;
246            Uri warcRefersToTargetUriUri = header.warcRefersToTargetUriUri;
247            String warcRefersToTargetUriStr = header.warcRefersToTargetUriStr;
248            WarcDate warcRefersToDate = header.warcRefersToDate;
249            String warcRefersToDateStr = header.warcRefersToDateStr;
250
251            WarcConcurrentTo warcConcurrentTo;
252            if (warcTypeIdx  > 0) {
253                checkFieldPolicy(warcTypeIdx, FN_IDX_CONTENT_TYPE, contentType, contentTypeStr, diagnostics);
254                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_IP_ADDRESS, warcInetAddress, warcIpAddress, diagnostics);
255                for (int i=0; i<warcConcurrentToList.size(); ++i) {
256                    warcConcurrentTo = warcConcurrentToList.get(0);
257                    checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_CONCURRENT_TO, warcConcurrentTo.warcConcurrentToUri, warcConcurrentTo.warcConcurrentToStr, diagnostics);
258                }
259                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_REFERS_TO, warcRefersToUri, warcRefersToStr, diagnostics);
260                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_TARGET_URI, warcTargetUriUri, warcTargetUriStr, diagnostics);
261                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_TRUNCATED, warcTruncatedIdx, warcTruncatedStr, diagnostics);
262                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_WARCINFO_ID, warcWarcinfoIdUri, warcWarcinfoIdStr, diagnostics);
263                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_BLOCK_DIGEST, warcBlockDigest, warcBlockDigestStr, diagnostics);
264                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_PAYLOAD_DIGEST, warcPayloadDigest, warcPayloadDigestStr, diagnostics);
265                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_FILENAME, warcFilename, warcFilename, diagnostics);
266                // Could also use warcProfileIdx for really strict.
267                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_PROFILE, warcProfileUri, warcProfileStr, diagnostics);
268                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE, warcIdentifiedPayloadType, warcIdentifiedPayloadTypeStr, diagnostics);
269                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_SEGMENT_NUMBER, warcSegmentNumber, warcSegmentNumberStr, diagnostics);
270                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_SEGMENT_ORIGIN_ID, warcSegmentOriginIdUrl, warcSegmentOriginIdStr, diagnostics);
271                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_SEGMENT_TOTAL_LENGTH, warcSegmentTotalLength, warcSegmentTotalLengthStr, diagnostics);
272                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_REFERS_TO_TARGET_URI, warcRefersToTargetUriUri, warcRefersToTargetUriStr, diagnostics);
273                checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_REFERS_TO_DATE, warcRefersToDate, warcRefersToDateStr, diagnostics);
274            }
275        }
276    }
277
278    /**
279     * Given a WARC record type and a WARC field looks up the policy in a
280     * matrix build from the WARC ISO standard.
281     * @param recordType WARC record type id
282     * @param fieldType WARC field type id
283     * @param fieldObj WARC field
284     * @param valueStr WARC raw field value
285     * @param diagnostics diagnostics object used to record warnings/errors
286     */
287    public void checkFieldPolicy(int recordType, int fieldType, Object fieldObj, String valueStr, Diagnostics diagnostics) {
288        int policy = field_policy[recordType][fieldType];
289        String entityStr;
290        switch (policy) {
291        case POLICY_MANDATORY:
292            if (fieldObj == null) {
293                entityStr = "'" + FN_IDX_STRINGS[fieldType] + "' value";
294                diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, valueStr);
295            }
296            break;
297        case POLICY_SHALL:
298            if (fieldObj == null) {
299                entityStr = "'" + FN_IDX_STRINGS[fieldType] + "' value";
300                diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, valueStr);
301            }
302            break;
303        case POLICY_SHALL_NOT:
304            if (fieldObj != null) {
305                entityStr = "'" + FN_IDX_STRINGS[fieldType] + "' value";
306                diagnostics.addError(DiagnosisType.UNDESIRED_DATA, entityStr, valueStr);
307            }
308            break;
309        case POLICY_MAY_NOT:
310            if (fieldObj != null) {
311                entityStr = "'" + FN_IDX_STRINGS[fieldType] + "' value";
312                diagnostics.addWarning(DiagnosisType.UNDESIRED_DATA, entityStr, valueStr);
313            }
314            break;
315        case POLICY_MAY:
316        case POLICY_IGNORE:
317        default:
318            break;
319        }
320    }
321
322}