001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.net.InetAddress; 021import java.util.List; 022 023import org.jwat.common.ContentType; 024import org.jwat.common.DiagnosisType; 025import org.jwat.common.Diagnostics; 026import org.jwat.common.Uri; 027 028/** 029 * WARC validator abstract base class. Has some common fields and methods. 030 * 031 * @author nicl 032 */ 033public abstract class WarcValidatorBase extends WarcConstants { 034 035 /** Version denoted as an integer array. */ 036 public int[] usedOnVersion; 037 038 /** A (Warc-Types x Warc-Header-Fields) matrix used for policy validation. */ 039 protected int[][] field_policy; 040 041 /** 042 * Internal constructor. 043 */ 044 protected WarcValidatorBase() { 045 } 046 047 /** 048 * Set the version this validator is used on. Differs from the internal version 049 * if it is used on WARC version that are not yet supported. 050 * @param arr integer array denoting a version 051 */ 052 public void setVersions(int [] arr) { 053 if (arr != null) { 054 usedOnVersion = new int[arr.length]; 055 System.arraycopy(arr, 0, usedOnVersion, 0, arr.length); 056 } 057 } 058 059 /** 060 * The following section initializes the policy matrix used to check the 061 * usage of each known warc header line against each known warc record 062 * type. 063 * The ISO standard was used to build the data in the matrix. 064 */ 065 public abstract void setWarcHeaderPolicyMatrix(); 066 067 /** 068 * Validate the WARC header relative to the WARC-Type and according to the 069 * WARC ISO standard. 070 * @param header warc header to validate 071 */ 072 protected void checkFields(WarcHeader header) { 073 Diagnostics diagnostics = header.diagnostics; 074 075 String entityStr; 076 String infoStr; 077 Integer warcTypeIdx = header.warcTypeIdx; 078 String warcTypeStr = header.warcTypeStr; 079 Integer warcProfileIdx = header.warcProfileIdx; 080 Uri warcProfileUri = header.warcProfileUri; 081 String warcProfileStr = header.warcProfileStr; 082 Uri warcRecordIdUri = header.warcRecordIdUri; 083 String warcRecordIdStr = header.warcRecordIdStr; 084 WarcDate warcDate = header.warcDate; 085 String warcDateStr = header.warcDateStr; 086 Long contentLength = header.contentLength; 087 String contentLengthStr = header.contentLengthStr; 088 ContentType contentType = header.contentType; 089 String contentTypeStr = header.contentTypeStr; 090 Integer warcSegmentNumber = header.warcSegmentNumber; 091 String warcSegmentNumberStr = header.warcSegmentNumberStr; 092 093 boolean bMandatoryMissing = false; 094 095 /* 096 * Unknown Warc-Type and/or Warc-Profile. 097 */ 098 099 if (warcTypeIdx != null && warcTypeIdx == RT_IDX_UNKNOWN) { 100 // Warning: Unknown Warc-Type. 101 entityStr = "'" + FN_WARC_TYPE + "' value"; 102 diagnostics.addWarning(DiagnosisType.UNKNOWN, entityStr, warcTypeStr); 103 } 104 105 if (warcProfileIdx != null) { 106 if (warcProfileIdx == WARC_PROFILE_IDX_UNKNOWN) { 107 // Warning: Unknown Warc-Profile. 108 entityStr = "'" + FN_WARC_PROFILE + "' value"; 109 diagnostics.addWarning(DiagnosisType.UNKNOWN, entityStr, warcProfileStr); 110 } 111 else { 112 switch (warcProfileIdx) { 113 case WARC10_PROFILE_IDX_IDENTICAL_PAYLOAD_DIGEST: 114 case WARC10_PROFILE_IDX_SERVER_NOT_MODIFIED: 115 if (header.major != 1 || header.minor != 0) { 116 // Warning: Unknown Warc-Profile. 117 entityStr = "'" + FN_WARC_PROFILE + "' value"; 118 infoStr = "WARC/" + header.major + "." + header.minor + " profile"; 119 diagnostics.addWarning(DiagnosisType.ERROR_EXPECTED, entityStr, infoStr); 120 } 121 break; 122 case WARC11_PROFILE_IDX_IDENTICAL_PAYLOAD_DIGEST: 123 case WARC11_PROFILE_IDX_SERVER_NOT_MODIFIED: 124 if (header.major != 1 || header.minor != 1) { 125 // Warning: Unknown Warc-Profile. 126 entityStr = "'" + FN_WARC_PROFILE + "' value"; 127 infoStr = "WARC/" + header.major + "." + header.minor + " profile"; 128 diagnostics.addWarning(DiagnosisType.ERROR_EXPECTED, entityStr, infoStr); 129 } 130 break; 131 default: 132 break; 133 } 134 } 135 } 136 137 /* 138 * Mandatory fields. 139 */ 140 141 // TODO Required yes, but is it always invalid. 142 if (warcTypeIdx == null) { 143 // Mandatory valid Warc-Type missing. 144 entityStr = "'" + FN_WARC_TYPE + "' header"; 145 diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, warcTypeStr); 146 bMandatoryMissing = true; 147 } 148 if (warcRecordIdUri == null) { 149 // Mandatory valid Warc-Record-Id missing. 150 entityStr = "'" + FN_WARC_RECORD_ID + "' header"; 151 diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, warcRecordIdStr); 152 bMandatoryMissing = true; 153 } 154 if (warcDate == null) { 155 // Mandatory valid Warc-Date missing. 156 entityStr = "'" + FN_WARC_DATE + "' header"; 157 diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, warcDateStr); 158 bMandatoryMissing = true; 159 } 160 if (contentLength == null) { 161 // Mandatory valid Content-Length missing. 162 entityStr = "'" + FN_CONTENT_LENGTH + "' header"; 163 diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, contentLengthStr); 164 bMandatoryMissing = true; 165 } 166 167 if (warcDate != null) { 168 if (header.major == 1 && header.minor == 0 && warcDate.precision != WarcDate.P_SECOND) { 169 diagnostics.addError(DiagnosisType.ERROR_EXPECTED, "'" + FN_WARC_DATE + "' value", warcDateStr, WarcConstants.WARC_DATE_FORMAT); 170 } 171 } 172 173 header.bMandatoryMissing = bMandatoryMissing; 174 175 /* 176 * Content-Type should be present if Content-Length > 0. 177 * Except for continuation records. 178 */ 179 180 if (contentLength != null && contentLength.longValue() > 0L && 181 (contentTypeStr == null || contentTypeStr.length() == 0)) { 182 if (warcTypeIdx == null || warcTypeIdx != WarcConstants.RT_IDX_CONTINUATION) { 183 entityStr = "'" + FN_CONTENT_TYPE + "' header"; 184 diagnostics.addWarning(DiagnosisType.RECOMMENDED_MISSING, entityStr); 185 } 186 } 187 188 /* 189 * WARC record type dependent policies. 190 */ 191 192 if (warcTypeIdx != null) { 193 /* 194 * Warcinfo record should have "application/warc-fields" content-type. 195 */ 196 197 if (warcTypeIdx == WarcConstants.RT_IDX_WARCINFO) { 198 if (contentType != null && 199 (!contentType.contentType.equals("application") 200 || !contentType.mediaType.equals("warc-fields"))) { 201 entityStr = "'" + FN_CONTENT_TYPE + "' value"; 202 diagnostics.addWarning(DiagnosisType.RECOMMENDED, entityStr, WarcConstants.CT_APP_WARC_FIELDS, contentTypeStr); 203 } 204 } 205 206 if (warcTypeIdx == WarcConstants.RT_IDX_RESPONSE) { 207 if (warcSegmentNumber != null && warcSegmentNumber != 1) { 208 entityStr = "'" + FN_WARC_SEGMENT_NUMBER + "' value"; 209 diagnostics.addError(DiagnosisType.INVALID_EXPECTED, entityStr, warcSegmentNumber.toString(), "1"); 210 } 211 } 212 213 if (warcTypeIdx == WarcConstants.RT_IDX_CONTINUATION) { 214 if (warcSegmentNumber != null && warcSegmentNumber < 2) { 215 entityStr = "'" + FN_WARC_SEGMENT_NUMBER + "' value"; 216 diagnostics.addError(DiagnosisType.INVALID_EXPECTED, entityStr, warcSegmentNumber.toString(), ">1"); 217 } 218 } 219 220 /* 221 * Check the policies for each field. 222 */ 223 224 InetAddress warcInetAddress = header.warcInetAddress; 225 String warcIpAddress = header.warcIpAddress; 226 List<WarcConcurrentTo> warcConcurrentToList = header.warcConcurrentToList; 227 Uri warcRefersToUri = header.warcRefersToUri; 228 String warcRefersToStr = header.warcRefersToStr; 229 Uri warcTargetUriUri = header.warcTargetUriUri; 230 String warcTargetUriStr = header.warcTargetUriStr; 231 Integer warcTruncatedIdx = header.warcTruncatedIdx; 232 String warcTruncatedStr = header.warcTruncatedStr; 233 Uri warcWarcinfoIdUri = header.warcWarcinfoIdUri; 234 String warcWarcinfoIdStr = header.warcWarcinfoIdStr; 235 WarcDigest warcBlockDigest = header.warcBlockDigest; 236 String warcBlockDigestStr = header.warcBlockDigestStr; 237 WarcDigest warcPayloadDigest = header.warcPayloadDigest; 238 String warcPayloadDigestStr = header.warcPayloadDigestStr; 239 String warcFilename = header.warcFilename; 240 ContentType warcIdentifiedPayloadType = header.warcIdentifiedPayloadType; 241 String warcIdentifiedPayloadTypeStr = header.warcIdentifiedPayloadTypeStr; 242 Uri warcSegmentOriginIdUrl = header.warcSegmentOriginIdUrl; 243 String warcSegmentOriginIdStr = header.warcSegmentOriginIdStr; 244 Long warcSegmentTotalLength = header.warcSegmentTotalLength; 245 String warcSegmentTotalLengthStr = header.warcSegmentTotalLengthStr; 246 Uri warcRefersToTargetUriUri = header.warcRefersToTargetUriUri; 247 String warcRefersToTargetUriStr = header.warcRefersToTargetUriStr; 248 WarcDate warcRefersToDate = header.warcRefersToDate; 249 String warcRefersToDateStr = header.warcRefersToDateStr; 250 251 WarcConcurrentTo warcConcurrentTo; 252 if (warcTypeIdx > 0) { 253 checkFieldPolicy(warcTypeIdx, FN_IDX_CONTENT_TYPE, contentType, contentTypeStr, diagnostics); 254 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_IP_ADDRESS, warcInetAddress, warcIpAddress, diagnostics); 255 for (int i=0; i<warcConcurrentToList.size(); ++i) { 256 warcConcurrentTo = warcConcurrentToList.get(0); 257 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_CONCURRENT_TO, warcConcurrentTo.warcConcurrentToUri, warcConcurrentTo.warcConcurrentToStr, diagnostics); 258 } 259 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_REFERS_TO, warcRefersToUri, warcRefersToStr, diagnostics); 260 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_TARGET_URI, warcTargetUriUri, warcTargetUriStr, diagnostics); 261 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_TRUNCATED, warcTruncatedIdx, warcTruncatedStr, diagnostics); 262 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_WARCINFO_ID, warcWarcinfoIdUri, warcWarcinfoIdStr, diagnostics); 263 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_BLOCK_DIGEST, warcBlockDigest, warcBlockDigestStr, diagnostics); 264 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_PAYLOAD_DIGEST, warcPayloadDigest, warcPayloadDigestStr, diagnostics); 265 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_FILENAME, warcFilename, warcFilename, diagnostics); 266 // Could also use warcProfileIdx for really strict. 267 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_PROFILE, warcProfileUri, warcProfileStr, diagnostics); 268 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE, warcIdentifiedPayloadType, warcIdentifiedPayloadTypeStr, diagnostics); 269 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_SEGMENT_NUMBER, warcSegmentNumber, warcSegmentNumberStr, diagnostics); 270 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_SEGMENT_ORIGIN_ID, warcSegmentOriginIdUrl, warcSegmentOriginIdStr, diagnostics); 271 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_SEGMENT_TOTAL_LENGTH, warcSegmentTotalLength, warcSegmentTotalLengthStr, diagnostics); 272 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_REFERS_TO_TARGET_URI, warcRefersToTargetUriUri, warcRefersToTargetUriStr, diagnostics); 273 checkFieldPolicy(warcTypeIdx, FN_IDX_WARC_REFERS_TO_DATE, warcRefersToDate, warcRefersToDateStr, diagnostics); 274 } 275 } 276 } 277 278 /** 279 * Given a WARC record type and a WARC field looks up the policy in a 280 * matrix build from the WARC ISO standard. 281 * @param recordType WARC record type id 282 * @param fieldType WARC field type id 283 * @param fieldObj WARC field 284 * @param valueStr WARC raw field value 285 * @param diagnostics diagnostics object used to record warnings/errors 286 */ 287 public void checkFieldPolicy(int recordType, int fieldType, Object fieldObj, String valueStr, Diagnostics diagnostics) { 288 int policy = field_policy[recordType][fieldType]; 289 String entityStr; 290 switch (policy) { 291 case POLICY_MANDATORY: 292 if (fieldObj == null) { 293 entityStr = "'" + FN_IDX_STRINGS[fieldType] + "' value"; 294 diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, valueStr); 295 } 296 break; 297 case POLICY_SHALL: 298 if (fieldObj == null) { 299 entityStr = "'" + FN_IDX_STRINGS[fieldType] + "' value"; 300 diagnostics.addError(DiagnosisType.REQUIRED_INVALID, entityStr, valueStr); 301 } 302 break; 303 case POLICY_SHALL_NOT: 304 if (fieldObj != null) { 305 entityStr = "'" + FN_IDX_STRINGS[fieldType] + "' value"; 306 diagnostics.addError(DiagnosisType.UNDESIRED_DATA, entityStr, valueStr); 307 } 308 break; 309 case POLICY_MAY_NOT: 310 if (fieldObj != null) { 311 entityStr = "'" + FN_IDX_STRINGS[fieldType] + "' value"; 312 diagnostics.addWarning(DiagnosisType.UNDESIRED_DATA, entityStr, valueStr); 313 } 314 break; 315 case POLICY_MAY: 316 case POLICY_IGNORE: 317 default: 318 break; 319 } 320 } 321 322}