001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.io.ByteArrayOutputStream; 021import java.io.IOException; 022import java.net.InetAddress; 023import java.util.Collections; 024import java.util.Date; 025import java.util.HashMap; 026import java.util.LinkedList; 027import java.util.List; 028import java.util.Map; 029 030import org.jwat.common.ByteCountingPushBackInputStream; 031import org.jwat.common.ContentType; 032import org.jwat.common.Diagnosis; 033import org.jwat.common.DiagnosisType; 034import org.jwat.common.Diagnostics; 035import org.jwat.common.HeaderLine; 036import org.jwat.common.HeaderLineReader; 037import org.jwat.common.MaxLengthRecordingInputStream; 038import org.jwat.common.Uri; 039import org.jwat.common.UriProfile; 040 041/** 042 * Central class for working with WARC headers. This class includes support for 043 * reading and writing WARC headers. Methods are also available to validate 044 * individual headers and a WARC header as a whole. 045 * 046 * @author nicl 047 */ 048public class WarcHeader { 049 050 /** An URI with encapsulating <> characters. */ 051 public static final boolean URI_LTGT = true; 052 053 /** An URI without encapsulating <> characters. */ 054 public static final boolean URI_NAKED = false; 055 056 /** Associated WarcReader context. 057 * Must be set prior to calling the various methods. */ 058 protected WarcReader reader; 059 060 /** Diagnostics used to report diagnoses. 061 * Must be set prior to calling the various methods. */ 062 protected Diagnostics<Diagnosis> diagnostics; 063 064 /** WARC-Target-URI profile. */ 065 protected UriProfile warcTargetUriProfile; 066 067 /** URI profile. */ 068 protected UriProfile uriProfile; 069 070 /** WARC field parser used. 071 * Must be set prior to calling the various methods. */ 072 protected WarcFieldParsers fieldParsers; 073 074 /** Max size allowed for a record header. */ 075 protected int recordHeaderMaxSize = -1; 076 077 /** Line reader used to read version lines. */ 078 protected HeaderLineReader lineReader; 079 080 /** Header line reader used to read the WARC headers. */ 081 protected HeaderLineReader headerLineReader; 082 083 /** WARC record starting offset relative to the source WARC file input 084 * stream. The offset is correct for both compressed and uncompressed streams. */ 085 protected long startOffset = -1; 086 087 /* 088 * Version related fields. 089 */ 090 091 /** Was "WARC/" identified while looking for the version string. */ 092 public boolean bMagicIdentified; 093 /** Did the version string include between 2 and 4 substrings delimited by ".". */ 094 public boolean bVersionParsed; 095 /** Is the version format valid. */ 096 public boolean bValidVersionFormat; 097 /** Is the version recognized. (0.17, 0.18 or 1.0) */ 098 public boolean bValidVersion; 099 100 /** Raw version string. */ 101 public String versionStr; 102 /** Array based on the version string split by the "." delimiter and converted to integers. */ 103 public int[] versionArr; 104 105 /** Major version number from WARC header. */ 106 public int major = -1; 107 /** Minor version number from WARC header. */ 108 public int minor = -1; 109 110 /* 111 * WARC header fields. 112 */ 113 114 /** Array used for duplicate header detection. */ 115 protected boolean[] seen = new boolean[WarcConstants.FN_INDEX_OF_LAST]; 116 117 /** Is the header missing one of the mandatory headers. */ 118 public boolean bMandatoryMissing; 119 120 /** WARC-Type field string value. */ 121 public String warcTypeStr; 122 /** WARC-Type converted to an integer id, if identified. */ 123 public Integer warcTypeIdx; 124 125 /** WARC-Filename field string value. 126 * (warcinfo record type only) */ 127 public String warcFilename; 128 129 /** WARC-Record-Id field string value. */ 130 public String warcRecordIdStr; 131 /** WARC-Record-Id converted to an <code>Uri</code> object, if valid. */ 132 public Uri warcRecordIdUri; 133 134 /** WARC-Date field string value. */ 135 public String warcDateStr; 136 /** WARC-Date converted to a <code>Date</code> object, if valid. */ 137 public Date warcDate; 138 139 /** Content-Length field string value. */ 140 public String contentLengthStr; 141 /** Content-Length converted to a <code>Long</code> object, if valid. */ 142 public Long contentLength; 143 144 /** Content-Type field string value. */ 145 public String contentTypeStr; 146 /** Content-Type converted to a <code>ContentType</code> object, if valid. */ 147 public ContentType contentType; 148 149 /** WARC-Truncated field string value. */ 150 public String warcTruncatedStr; 151 /** WARC-Truncated converted to an integer id, if valid. */ 152 public Integer warcTruncatedIdx; 153 154 /** WARC-IP-Address field string value. */ 155 public String warcIpAddress; 156 /** WARC-IP-Address converted to an <code>InetAddress</code> object, if valid. */ 157 public InetAddress warcInetAddress; 158 159 /** List of WARC-Concurrent-To field string values and converted <code>URI</code> objects, if valid. */ 160 public List<WarcConcurrentTo> warcConcurrentToList = new LinkedList<WarcConcurrentTo>(); 161 162 /** WARC-Refers-To field string value. */ 163 public String warcRefersToStr; 164 /** WARC-Refers-To converted to an <code>Uri</code> object, if valid. */ 165 public Uri warcRefersToUri; 166 167 /** WARC_Target-URI field string value. */ 168 public String warcTargetUriStr; 169 /** WARC-TargetURI converted to an <code>Uri</code> object, if valid. */ 170 public Uri warcTargetUriUri; 171 172 /** WARC-Warcinfo-Id field string value. */ 173 public String warcWarcinfoIdStr; 174 /** WARC-Warcinfo-Id converted to an <code>Uri</code> object, if valid. */ 175 public Uri warcWarcinfoIdUri; 176 177 /** WARC-Block-Digest field string value. */ 178 public String warcBlockDigestStr; 179 /** WARC-Block-Digest converted to a <code>WarcDigest</code> object, if valid. */ 180 public WarcDigest warcBlockDigest; 181 182 /** WARC-Payload-Digest field string value. */ 183 public String warcPayloadDigestStr; 184 /** WARC-Payload-Digest converted to a <code>WarcDigest</code> object, if valid. */ 185 public WarcDigest warcPayloadDigest; 186 187 /** WARC-Identified-Payload-Type field string value. */ 188 public String warcIdentifiedPayloadTypeStr; 189 /** WARC-Identified-Payload-Type converted to a <code>ContentType</code> object, if valid. */ 190 public ContentType warcIdentifiedPayloadType; 191 192 /** WARC-Profile field string value. 193 * (revisit record only) */ 194 public String warcProfileStr; 195 /** WARC-Profile field converted to an <code>Uri</code> object, if valid. 196 * (revisit record only) */ 197 public Uri warcProfileUri; 198 /** WARC-Profile converted to an integer id, if valid. 199 * (revisit record only) */ 200 public Integer warcProfileIdx; 201 202 /** WARC-Segment-Number field string value. */ 203 public String warcSegmentNumberStr; 204 /** WARC-Segment-Number converted to an <code>Integer</code> object, if valid. */ 205 public Integer warcSegmentNumber; 206 207 /** WARC-Segment-Origin-Id field string value. 208 * (continuation record only) */ 209 public String warcSegmentOriginIdStr; 210 /** WARC-Segment-Origin-Id converted to an <code>Uri</code> object, if valid. 211 * (continuation record only) */ 212 public Uri warcSegmentOriginIdUrl; 213 214 /** WARC-Segment-Total-Length field string value. 215 * (continuation record only) */ 216 public String warcSegmentTotalLengthStr; 217 /** WARC-Segment-Total-Length converted to a <code>Long</code> object, if valid. 218 * (continuation record only) */ 219 public Long warcSegmentTotalLength; 220 221 // see https://docs.google.com/document/d/1QyQBA7Ykgxie75V8Jziz_O7hbhwf7PF6_u9O6w6zgp0/edit 222 /** WARC-Refers-To-Target-URI field string value. */ 223 public String warcRefersToTargetUriStr; 224 /** WARC-Refers-To-Target-URI converted to an <code>Uri</code> object, if valid. */ 225 public Uri warcRefersToTargetUriUri; 226 /** WARC-Refers-To-Date */ 227 public String warcRefersToDateStr; 228 /** WARC-Date converted to a <code>Date</code> object, if valid. */ 229 public Date warcRefersToDate; 230 231 /* 232 * WARC header fields collections. 233 */ 234 235 /** Raw WARC header output stream. */ 236 protected ByteArrayOutputStream headerBytesOut = new ByteArrayOutputStream(); 237 238 /** Raw WARC header byte array. */ 239 public byte[] headerBytes; 240 241 /** List of parsed header fields. */ 242 protected List<HeaderLine> headerList = new LinkedList<HeaderLine>(); 243 244 /** Map of parsed header fields. */ 245 protected Map<String, HeaderLine> headerMap = new HashMap<String, HeaderLine>(); 246 247 /** 248 * Non public constructor to allow unit testing. 249 */ 250 protected WarcHeader() { 251 } 252 253 /** 254 * Create a new <code>WarcHeader</code> for recreating a header object. 255 * @param fieldParsers parsers used for the individual types 256 * @param uriProfile uri profile used to validate urls 257 * @param diagnostics diagnostics object used by parser 258 * @return a <code>WarcHeader</code> prepared for reanimation 259 */ 260 public static WarcHeader initHeader(int recordHeaderMaxSize, HeaderLineReader lineReader, HeaderLineReader headerLineReader, WarcFieldParsers fieldParsers, UriProfile uriProfile, Diagnostics<Diagnosis> diagnostics) { 261 WarcHeader header = new WarcHeader(); 262 header.warcTargetUriProfile = uriProfile; 263 header.uriProfile = uriProfile; 264 header.fieldParsers = fieldParsers; 265 header.diagnostics = diagnostics; 266 header.recordHeaderMaxSize = recordHeaderMaxSize; 267 header.lineReader = lineReader; 268 header.headerLineReader = headerLineReader; 269 return header; 270 } 271 272 /** 273 * Create and initialize a new <code>WarcHeader</code> for writing. 274 * @param writer writer which shall be used 275 * @param diagnostics diagnostics object used by writer 276 * @return a <code>WarcHeader</code> prepared for writing 277 */ 278 public static WarcHeader initHeader(WarcWriter writer, Diagnostics<Diagnosis> diagnostics) { 279 WarcHeader header = new WarcHeader(); 280 // Set default version to "1.0". 281 header.major = 1; 282 header.minor = 0; 283 header.warcTargetUriProfile = writer.warcTargetUriProfile; 284 header.uriProfile = writer.uriProfile; 285 header.fieldParsers = writer.fieldParsers; 286 header.diagnostics = diagnostics; 287 return header; 288 } 289 290 /** 291 * Create and initialize a new <code>WarcHeader</code> for reading. 292 * @param reader reader which shall be used 293 * @param startOffset start offset of header 294 * @param diagnostics diagnostics object used by reader 295 * @return a <code>WarcHeader</code> prepared for reading 296 */ 297 public static WarcHeader initHeader(WarcReader reader, long startOffset, Diagnostics<Diagnosis> diagnostics) { 298 WarcHeader header = new WarcHeader(); 299 header.reader = reader; 300 header.warcTargetUriProfile = reader.warcTargetUriProfile; 301 header.uriProfile = reader.uriProfile; 302 header.fieldParsers = reader.fieldParsers; 303 header.diagnostics = diagnostics; 304 header.recordHeaderMaxSize = reader.recordHeaderMaxSize; 305 header.lineReader = reader.lineReader; 306 header.headerLineReader = reader.headerLineReader; 307 // This is only relevant for uncompressed sequentially read records 308 header.startOffset = startOffset; 309 return header; 310 } 311 312 /** 313 * Add an error diagnosis of the given type on a specific entity with 314 * optional extra information. The information varies according to the 315 * diagnosis type. 316 * @param type diagnosis type 317 * @param entity entity examined 318 * @param information optional extra information 319 */ 320 protected void addErrorDiagnosis(DiagnosisType type, String entity, String... information) { 321 diagnostics.addError(new Diagnosis(type, entity, information)); 322 } 323 324 /** 325 * Add a warning diagnosis of the given type on a specific entity with 326 * optional extra information. The information varies according to the 327 * diagnosis type. 328 * @param type diagnosis type 329 * @param entity entity examined 330 * @param information optional extra information 331 */ 332 protected void addWarningDiagnosis(DiagnosisType type, String entity, String... information) { 333 diagnostics.addWarning(new Diagnosis(type, entity, information)); 334 } 335 336 /** 337 * Returns the starting offset of the record in the containing WARC. 338 * @return the starting offset of the record 339 */ 340 public long getStartOffset() { 341 return startOffset; 342 } 343 344 /** 345 * Try to parse a WARC header and return a boolean indicating the success or 346 * failure of this. 347 * @param in input stream with WARC data 348 * @return boolean indicating whether a header was parsed or not 349 * @throws IOException if an I/O exception occurs while parsing for a header 350 */ 351 public boolean parseHeader(ByteCountingPushBackInputStream in) throws IOException { 352 if (parseVersion(in)) { 353 // debug 354 //System.out.println(wr.bMagicIdentified); 355 //System.out.println(wr.bVersionParsed); 356 //System.out.println(wr.major + "." + wr.minor); 357 if (bVersionParsed && versionArr.length == 2) { 358 switch (major) { 359 case 1: 360 if (minor == 0) { 361 bValidVersion = true; 362 } 363 break; 364 case 0: 365 switch (minor) { 366 case 17: 367 case 18: 368 bValidVersion = true; 369 break; 370 } 371 break; 372 default: 373 break; 374 } 375 if (!bValidVersion) { 376 diagnostics.addError( 377 new Diagnosis(DiagnosisType.UNKNOWN, 378 "Magic version number", versionStr)); 379 } 380 } else { 381 diagnostics.addError( 382 new Diagnosis(DiagnosisType.INVALID_DATA, 383 "Magic Version string", versionStr)); 384 } 385 386 MaxLengthRecordingInputStream mrin = new MaxLengthRecordingInputStream(in, recordHeaderMaxSize); 387 ByteCountingPushBackInputStream pbin = new ByteCountingPushBackInputStream(mrin, recordHeaderMaxSize); 388 389 parseHeaders(pbin); 390 pbin.close(); 391 392 checkFields(); 393 394 headerBytes = headerBytesOut.toByteArray(); 395 } 396 return bMagicIdentified; 397 } 398 399 /** 400 * Looks forward in the input stream for a valid WARC version line. 401 * @param in data input stream 402 * @return true, if magic WARC header found 403 * @throws IOException if an error occurs while reading version data 404 */ 405 public boolean parseVersion(ByteCountingPushBackInputStream in) throws IOException { 406 bMagicIdentified = false; 407 bVersionParsed = false; 408 boolean bInvalidDataBeforeVersion = false; 409 boolean bEmptyLinesBeforeVersion = false; 410 HeaderLine line; 411 String tmpStr; 412 boolean bSeekMagic = true; 413 // Loop until when have found something that looks like a version line. 414 while (bSeekMagic) { 415 // This is only relevant for uncompressed sequentially read records 416 startOffset = in.getConsumed(); 417 line = lineReader.readLine(in); 418 if (!lineReader.bEof) { 419 switch (line.type) { 420 case HeaderLine.HLT_LINE: 421 tmpStr = line.line; 422 // debug 423 //System.out.println(tmpStr); 424 if (tmpStr.length() > 0) { 425 if (tmpStr.toUpperCase().startsWith(WarcConstants.WARC_MAGIC_HEADER)) { 426 bMagicIdentified = true; 427 versionStr = tmpStr.substring(WarcConstants.WARC_MAGIC_HEADER.length()); 428 String[] tmpArr = versionStr.split("\\.", -1); // Not optimal 429 if (tmpArr.length >= 2 && tmpArr.length <= 4) { 430 bVersionParsed = true; 431 bValidVersionFormat = true; 432 versionArr = new int[tmpArr.length]; 433 for (int i=0; i<tmpArr.length; ++i) { 434 try { 435 versionArr[i] = Integer.parseInt(tmpArr[i]); 436 } catch (NumberFormatException e) { 437 versionArr[i] = -1; 438 bValidVersionFormat = false; 439 } 440 } 441 major = versionArr[0]; 442 minor = versionArr[1]; 443 } 444 headerBytesOut.write(line.raw); 445 bSeekMagic = false; 446 } else { 447 // Invalid data aka Gibberish. 448 bInvalidDataBeforeVersion = true; 449 } 450 } else { 451 // Empty line. 452 bEmptyLinesBeforeVersion = true; 453 454 } 455 break; 456 case HeaderLine.HLT_HEADERLINE: 457 // Invalid data - header or binary. 458 bInvalidDataBeforeVersion = true; 459 break; 460 } 461 } else { 462 // EOF. 463 bSeekMagic = false; 464 } 465 } 466 if (bInvalidDataBeforeVersion) { 467 addErrorDiagnosis(DiagnosisType.INVALID, "Data before WARC version"); 468 } 469 if (bEmptyLinesBeforeVersion) { 470 addErrorDiagnosis(DiagnosisType.INVALID, "Empty lines before WARC version"); 471 } 472 return bMagicIdentified; 473 } 474 475 /** 476 * Reads WARC header lines one line at a time until an empty line is 477 * encountered. 478 * @param in header input stream 479 * @throws IOException if an error occurs while reading the WARC header 480 */ 481 public void parseHeaders(ByteCountingPushBackInputStream in) throws IOException { 482 HeaderLine headerLine; 483 boolean bLoop = true; 484 while (bLoop) { 485 headerLine = headerLineReader.readLine(in); 486 if ((headerLine.bfErrors & HeaderLineReader.E_BIT_INVALID_CHARSET) != 0) { 487 addErrorDiagnosis(DiagnosisType.INVALID_ENCODING, "Invalid encoding in header line", headerLine.value, "UNKNOWN"); 488 } 489 if (!headerLineReader.bEof) { 490 headerBytesOut.write(headerLine.raw); 491 switch (headerLine.type) { 492 case HeaderLine.HLT_HEADERLINE: 493 if (headerLine.name != null && headerLine.name.length() > 0) { 494 // debug 495 //System.out.println(headerLine.name); 496 //System.out.println(headerLine.value); 497 addHeader(headerLine); 498 } else { 499 // Empty field name. 500 addWarningDiagnosis(DiagnosisType.EMPTY, "Header line"); 501 } 502 break; 503 case HeaderLine.HLT_LINE: 504 if (headerLine.line.length() == 0) { 505 // Empty line. 506 bLoop = false; 507 } else { 508 // Unknown header line. 509 addWarningDiagnosis(DiagnosisType.UNKNOWN, "Header line", headerLine.line); 510 } 511 break; 512 default: 513 throw new IllegalStateException("Invalid HeaderLine output!"); 514 } 515 } else { 516 // EOF. 517 bLoop = false; 518 } 519 } 520 } 521 522 /** 523 * Identify a (WARC) header name, validate the value and set the header. 524 * @param headerLine the headerLine 525 */ 526 public void addHeader(HeaderLine headerLine) { 527 String fieldName = headerLine.name; 528 String fieldValue = headerLine.value; 529 WarcConcurrentTo warcConcurrentTo; 530 Integer fn_idx = WarcConstants.fieldNameIdxMap.get(fieldName.toLowerCase()); 531 if (fn_idx != null) { 532 // WARC field name defined in WARC specification. 533 if (!seen[fn_idx] || WarcConstants.fieldNamesRepeatableLookup[fn_idx]) { 534 seen[fn_idx] = true; 535 switch (fn_idx.intValue()) { 536 case WarcConstants.FN_IDX_WARC_TYPE: 537 warcTypeStr = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_TYPE); 538 if (warcTypeStr != null) { 539 warcTypeIdx = WarcConstants.recordTypeIdxMap.get(warcTypeStr.toLowerCase()); 540 } 541 if (warcTypeIdx == null && warcTypeStr != null && warcTypeStr.length() > 0) { 542 warcTypeIdx = WarcConstants.RT_IDX_UNKNOWN; 543 } 544 break; 545 case WarcConstants.FN_IDX_WARC_RECORD_ID: 546 warcRecordIdStr = fieldValue; 547 warcRecordIdUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_RECORD_ID); 548 break; 549 case WarcConstants.FN_IDX_WARC_DATE: 550 warcDateStr = fieldValue; 551 warcDate = fieldParsers.parseDate(fieldValue, WarcConstants.FN_WARC_DATE); 552 break; 553 case WarcConstants.FN_IDX_CONTENT_LENGTH: 554 contentLengthStr = fieldValue; 555 contentLength = fieldParsers.parseLong(fieldValue, WarcConstants.FN_CONTENT_LENGTH); 556 break; 557 case WarcConstants.FN_IDX_CONTENT_TYPE: 558 contentTypeStr = fieldValue; 559 contentType = fieldParsers.parseContentType(fieldValue, WarcConstants.FN_CONTENT_TYPE); 560 break; 561 case WarcConstants.FN_IDX_WARC_CONCURRENT_TO: 562 Uri tmpUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_CONCURRENT_TO); 563 if (fieldValue != null && fieldValue.trim().length() > 0) { 564 warcConcurrentTo = new WarcConcurrentTo(); 565 warcConcurrentTo.warcConcurrentToStr = fieldValue; 566 warcConcurrentTo.warcConcurrentToUri = tmpUri; 567 warcConcurrentToList.add(warcConcurrentTo); 568 } 569 break; 570 case WarcConstants.FN_IDX_WARC_BLOCK_DIGEST: 571 warcBlockDigestStr = fieldValue; 572 warcBlockDigest = fieldParsers.parseDigest(fieldValue, WarcConstants.FN_WARC_BLOCK_DIGEST); 573 break; 574 case WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST: 575 warcPayloadDigestStr = fieldValue; 576 warcPayloadDigest = fieldParsers.parseDigest(fieldValue, WarcConstants.FN_WARC_PAYLOAD_DIGEST); 577 break; 578 case WarcConstants.FN_IDX_WARC_IP_ADDRESS: 579 warcIpAddress = fieldValue; 580 warcInetAddress = fieldParsers.parseIpAddress(fieldValue, WarcConstants.FN_WARC_IP_ADDRESS); 581 break; 582 case WarcConstants.FN_IDX_WARC_REFERS_TO: 583 warcRefersToStr = fieldValue; 584 warcRefersToUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_REFERS_TO); 585 break; 586 case WarcConstants.FN_IDX_WARC_TARGET_URI: 587 warcTargetUriStr = fieldValue; 588 warcTargetUriUri = fieldParsers.parseUri(fieldValue, URI_NAKED, warcTargetUriProfile, WarcConstants.FN_WARC_TARGET_URI); 589 break; 590 case WarcConstants.FN_IDX_WARC_TRUNCATED: 591 warcTruncatedStr = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_TRUNCATED); 592 if (warcTruncatedStr != null) { 593 warcTruncatedIdx = WarcConstants.truncatedTypeIdxMap.get(warcTruncatedStr.toLowerCase()); 594 } 595 if (warcTruncatedIdx == null && warcTruncatedStr != null && warcTruncatedStr.length() > 0) { 596 warcTruncatedIdx = WarcConstants.TT_IDX_FUTURE_REASON; 597 } 598 break; 599 case WarcConstants.FN_IDX_WARC_WARCINFO_ID: 600 warcWarcinfoIdStr = fieldValue; 601 warcWarcinfoIdUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_WARCINFO_ID); 602 break; 603 case WarcConstants.FN_IDX_WARC_FILENAME: 604 warcFilename = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_FILENAME); 605 break; 606 case WarcConstants.FN_IDX_WARC_PROFILE: 607 warcProfileStr = fieldValue; 608 warcProfileUri = fieldParsers.parseUri(fieldValue, URI_NAKED, uriProfile, WarcConstants.FN_WARC_PROFILE); 609 if (warcProfileStr != null) { 610 warcProfileIdx = WarcConstants.profileIdxMap.get(warcProfileStr.toLowerCase()); 611 } 612 if (warcProfileIdx == null && warcProfileStr != null && warcProfileStr.length() > 0) { 613 warcProfileIdx = WarcConstants.PROFILE_IDX_UNKNOWN; 614 } 615 break; 616 case WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE: 617 warcIdentifiedPayloadTypeStr = fieldValue; 618 warcIdentifiedPayloadType = fieldParsers.parseContentType(fieldValue, WarcConstants.FN_WARC_IDENTIFIED_PAYLOAD_TYPE); 619 break; 620 case WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID: 621 warcSegmentOriginIdStr = fieldValue; 622 warcSegmentOriginIdUrl = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_SEGMENT_ORIGIN_ID); 623 break; 624 case WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER: 625 warcSegmentNumberStr = fieldValue; 626 warcSegmentNumber = fieldParsers.parseInteger(fieldValue, WarcConstants.FN_WARC_SEGMENT_NUMBER); 627 break; 628 case WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH: 629 warcSegmentTotalLengthStr = fieldValue; 630 warcSegmentTotalLength = fieldParsers.parseLong(fieldValue, WarcConstants.FN_WARC_SEGMENT_TOTAL_LENGTH); 631 break; 632 case WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI: 633 warcRefersToTargetUriStr = fieldValue; 634 warcRefersToTargetUriUri = fieldParsers.parseUri(fieldValue, URI_NAKED, uriProfile, WarcConstants.FN_WARC_REFERS_TO_TARGET_URI); 635 break; 636 case WarcConstants.FN_IDX_WARC_REFERS_TO_DATE: 637 warcRefersToDateStr = fieldValue; 638 warcRefersToDate = fieldParsers.parseDate(fieldValue, WarcConstants.FN_WARC_REFERS_TO_DATE); 639 break; 640 } 641 } else { 642 // Duplicate field. 643 addErrorDiagnosis(DiagnosisType.DUPLICATE, "'" + fieldName + "' header", fieldValue); 644 } 645 } 646 HeaderLine tmpLine = headerMap.get(fieldName.toLowerCase()); 647 if (tmpLine == null) { 648 headerMap.put(fieldName.toLowerCase(), headerLine); 649 } else { 650 tmpLine.lines.add(headerLine); 651 } 652 headerList.add(headerLine); 653 } 654 655 /** 656 * Get a <code>List</code> of all the headers found during parsing. 657 * @return <code>List</code> of <code>HeaderLine</code> 658 */ 659 public List<HeaderLine> getHeaderList() { 660 return Collections.unmodifiableList(headerList); 661 } 662 663 /** 664 * Get a header line structure or null, if no header line structure is 665 * stored with the given header name. 666 * @param field header name 667 * @return <code>HeaderLine</code> structure or null 668 */ 669 public HeaderLine getHeader(String field) { 670 if (field != null && field.length() > 0) { 671 return headerMap.get(field.toLowerCase()); 672 } else { 673 return null; 674 } 675 } 676 677 /** 678 * Add a String header using the supplied string and return a 679 * <code>HeaderLine</code> object corresponding to how the header would be 680 * read. 681 * @param fieldName name of field to add 682 * @param fieldValue field value string 683 * @return <code>HeaderLine</code> object corresponding to what would have been read 684 */ 685 public HeaderLine addHeader(String fieldName, String fieldValue) { 686 HeaderLine headerLine = new HeaderLine(); 687 headerLine.name = fieldName; 688 headerLine.value = fieldValue; 689 addHeader(headerLine); 690 return headerLine; 691 } 692 693 /** 694 * Add an Integer header using the supplied string and object values and return 695 * a <code>HeaderLine</code> object corresponding to how the header would be read. 696 * If both string and object values are not null they are used as is. 697 * If the string value is null and the object is not null, 698 * the object's toString method is called. 699 * If the object is null and the string is not null, the string is parsed 700 * and validated resulting in an object, if valid. 701 * @param fieldName name of field to add 702 * @param integerFieldValue <code>Integer</code> field value object 703 * @param fieldValueStr Integer field value string 704 * @return <code>HeaderLine</code> object corresponding to what would have been read 705 */ 706 public HeaderLine addHeader(String fieldName, Integer integerFieldValue, String fieldValueStr) { 707 if (integerFieldValue == null && fieldValueStr != null) { 708 integerFieldValue = fieldParsers.parseInteger(fieldValueStr, fieldName); 709 } else if (fieldValueStr == null && integerFieldValue != null) { 710 fieldValueStr = integerFieldValue.toString(); 711 } 712 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_INTEGER, 713 integerFieldValue, null, null, null, null, null, null); 714 } 715 716 /** 717 * Add a Long header using the supplied string and object values and return 718 * a <code>HeaderLine</code> object corresponding to how the header would be read. 719 * If both string and object values are not null they are used as is. 720 * If the string value is null and the object is not null, 721 * the object's toString method is called. 722 * If the object is null and the string is not null, the string is parsed 723 * and validated resulting in an object, if valid. 724 * @param fieldName name of field to add 725 * @param longFieldValue <code>Long</code> field value object 726 * @param fieldValueStr Long field value string 727 * @return <code>HeaderLine</code> object corresponding to what would have been read 728 */ 729 public HeaderLine addHeader(String fieldName, Long longFieldValue, String fieldValueStr) { 730 if (longFieldValue == null && fieldValueStr != null) { 731 longFieldValue = fieldParsers.parseLong(fieldValueStr, fieldName); 732 } else if (fieldValueStr == null && longFieldValue != null) { 733 fieldValueStr = longFieldValue.toString(); 734 } 735 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_LONG, 736 null, longFieldValue, null, null, null, null, null); 737 } 738 739 /** 740 * Add an Digest header using the supplied string and object values and return 741 * a <code>HeaderLine</code> object corresponding to how the header would be read. 742 * If both string and object values are not null they are used as is. 743 * If the string value is null and the object is not null, 744 * the object's toString method is called. 745 * If the object is null and the string is not null, the string is parsed 746 * and validated resulting in an object, if valid. 747 * @param fieldName name of field to add 748 * @param digestFieldValue <code>Digest</code> field value object 749 * @param fieldValueStr Digest field value string 750 * @return <code>HeaderLine</code> object corresponding to what would have been read 751 */ 752 public HeaderLine addHeader(String fieldName, WarcDigest digestFieldValue, String fieldValueStr) { 753 if (digestFieldValue == null && fieldValueStr != null) { 754 digestFieldValue = fieldParsers.parseDigest(fieldValueStr, fieldName); 755 } else if (fieldValueStr == null && digestFieldValue != null) { 756 fieldValueStr = digestFieldValue.toString(); 757 } 758 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_DIGEST, 759 null, null, digestFieldValue, null, null, null, null); 760 } 761 762 /** 763 * Add an Content-Type header using the supplied string and object values and return 764 * a <code>HeaderLine</code> object corresponding to how the header would be read. 765 * If both string and object values are not null they are used as is. 766 * If the string value is null and the object is not null, 767 * the object's toString method is called. 768 * If the object is null and the string is not null, the string is parsed 769 * and validated resulting in an object, if valid. 770 * @param fieldName name of field to add 771 * @param contentTypeFieldValue <code>ContentType</code> field value object 772 * @param fieldValueStr Content-Type field value string 773 * @return <code>HeaderLine</code> object corresponding to what would have been read 774 */ 775 public HeaderLine addHeader(String fieldName, ContentType contentTypeFieldValue, String fieldValueStr) { 776 if (contentTypeFieldValue == null && fieldValueStr != null) { 777 contentTypeFieldValue = fieldParsers.parseContentType(fieldValueStr, fieldName); 778 } else if (fieldValueStr == null && contentTypeFieldValue != null) { 779 fieldValueStr = contentTypeFieldValue.toString(); 780 } 781 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_CONTENTTYPE, 782 null, null, null, contentTypeFieldValue, null, null, null); 783 } 784 785 /** 786 * Add an Date header using the supplied string and object values and return 787 * a <code>HeaderLine</code> object corresponding to how the header would be read. 788 * If both string and object values are not null they are used as is. 789 * If the string value is null and the object is not null, 790 * the object's toString method is called. 791 * If the object is null and the string is not null, the string is parsed 792 * and validated resulting in an object, if valid. 793 * @param fieldName name of field to add 794 * @param dateFieldValue <code>Date</code> field value object 795 * @param fieldValueStr Date field value string 796 * @return <code>HeaderLine</code> object corresponding to what would have been read 797 */ 798 public HeaderLine addHeader(String fieldName, Date dateFieldValue, String fieldValueStr) { 799 if (dateFieldValue == null && fieldValueStr != null) { 800 dateFieldValue = fieldParsers.parseDate(fieldValueStr, fieldName); 801 } else if (fieldValueStr == null && dateFieldValue != null) { 802 fieldValueStr = WarcDateParser.getDateFormat().format(dateFieldValue); 803 } 804 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_DATE, 805 null, null, null, null, dateFieldValue, null, null); 806 } 807 808 /** 809 * Add an InetAddress header using the supplied string and object values and return 810 * a <code>HeaderLine</code> object corresponding to how the header would be read. 811 * If both string and object values are not null they are used as is. 812 * If the string value is null and the object is not null, 813 * the object's toString method is called. 814 * If the object is null and the string is not null, the string is parsed 815 * and validated resulting in an object, if valid. 816 * @param fieldName name of field to add 817 * @param inetAddrFieldValue <code>InetAddress</code> field value object 818 * @param fieldValueStr IP-Address field value string 819 * @return <code>HeaderLine</code> object corresponding to what would have been read 820 */ 821 public HeaderLine addHeader(String fieldName, InetAddress inetAddrFieldValue, String fieldValueStr) { 822 if (inetAddrFieldValue == null && fieldValueStr != null) { 823 inetAddrFieldValue = fieldParsers.parseIpAddress(fieldValueStr, fieldName); 824 } else if (fieldValueStr == null && inetAddrFieldValue != null) { 825 fieldValueStr = inetAddrFieldValue.getHostAddress(); 826 } 827 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_INETADDRESS, 828 null, null, null, null, null, inetAddrFieldValue, null); 829 } 830 831 /** 832 * Add an URI header using the supplied string and object values and return 833 * a <code>HeaderLine</code> object corresponding to how the header would be read. 834 * If both string and object values are not null they are used as is. 835 * If the string value is null and the object is not null, 836 * the object's toString method is called. 837 * If the object is null and the string is not null, the string is parsed 838 * and validated resulting in an object, if valid. 839 * @param fieldName name of field to add 840 * @param uriFieldValue <code>URI</code> field value object 841 * @param fieldValueStr URI field value string 842 * @return <code>HeaderLine</code> object corresponding to what would have been read 843 */ 844 public HeaderLine addHeader(String fieldName, Uri uriFieldValue, String fieldValueStr) { 845 if (uriFieldValue == null && fieldValueStr != null) { 846 if (WarcConstants.FN_WARC_TARGET_URI.equalsIgnoreCase(fieldName)) { 847 uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, warcTargetUriProfile, fieldName); 848 } else if (WarcConstants.FN_WARC_PROFILE.equalsIgnoreCase(fieldName)) { 849 uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, uriProfile, fieldName); 850 } else if (WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.equalsIgnoreCase(fieldName)) { 851 uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, warcTargetUriProfile, fieldName); 852 } else { 853 uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_LTGT, uriProfile, fieldName); 854 } 855 } else if (fieldValueStr == null && uriFieldValue != null) { 856 if (WarcConstants.FN_WARC_TARGET_URI.equalsIgnoreCase(fieldName) 857 || WarcConstants.FN_WARC_PROFILE.equalsIgnoreCase(fieldName) 858 || WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.equalsIgnoreCase(fieldName)) { 859 fieldValueStr = uriFieldValue.toString(); 860 } else { 861 fieldValueStr = "<" + uriFieldValue.toString() + ">"; 862 } 863 } 864 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_URI, 865 null, null, null, null, null, null, uriFieldValue); 866 } 867 868 /** 869 * Add a header with the supplied field name, data type and value and 870 * return a <code>HeaderLine</code> corresponding to how the header will 871 * be read. The data type is validated against the field data type. 872 * The values used are the field value string and the parameter 873 * corresponding to the data type. 874 * @param fieldName header field name 875 * @param fieldValueStr field value in string form 876 * @param dt data type of the field value string when converted to an object 877 * @param integerFieldValue <code>Integer</code> object field value 878 * @param longFieldValue <code>Long</code> object field value 879 * @param digestFieldValue <code>Digest</code> object field value 880 * @param contentTypeFieldValue <code>ContentType</code> object field value 881 * @param dateFieldValue <code>Date</code> object field value 882 * @param inetAddrFieldValue <code>InetAddress</code> object field value 883 * @param uriFieldValue <code>URI</code> object field value 884 * @return <code>HeaderLine</code> object corresponding to what would have been read 885 */ 886 public HeaderLine addHeader(String fieldName, String fieldValueStr, int dt, 887 Integer integerFieldValue, Long longFieldValue, 888 WarcDigest digestFieldValue, ContentType contentTypeFieldValue, 889 Date dateFieldValue, InetAddress inetAddrFieldValue, 890 Uri uriFieldValue) { 891 Integer fn_idx = WarcConstants.fieldNameIdxMap.get(fieldName.toLowerCase()); 892 if (fn_idx != null) { 893 // Implicit cast from integer to long, if needed. 894 if (WarcConstants.FN_IDX_DT[fn_idx] == WarcConstants.FDT_LONG 895 && dt == WarcConstants.FDT_INTEGER) { 896 longFieldValue = (long)integerFieldValue; 897 dt = WarcConstants.FDT_LONG; 898 } 899 if (dt == WarcConstants.FN_IDX_DT[fn_idx]) { 900 // WARC field name defined in WARC specification. 901 if (seen[fn_idx] && !WarcConstants.fieldNamesRepeatableLookup[fn_idx]) { 902 // Duplicate field. 903 addErrorDiagnosis(DiagnosisType.DUPLICATE, 904 "'" + fieldName + "' header", 905 fieldValueStr); 906 } 907 seen[fn_idx] = true; 908 switch (fn_idx.intValue()) { 909 /* 910 * Integer. 911 */ 912 case WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER: 913 warcSegmentNumberStr = fieldValueStr; 914 warcSegmentNumber = integerFieldValue; 915 break; 916 /* 917 * Long. 918 */ 919 case WarcConstants.FN_IDX_CONTENT_LENGTH: 920 contentLengthStr = fieldValueStr; 921 contentLength = longFieldValue; 922 break; 923 case WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH: 924 warcSegmentTotalLengthStr = fieldValueStr; 925 warcSegmentTotalLength = longFieldValue; 926 break; 927 /* 928 * Digest. 929 */ 930 case WarcConstants.FN_IDX_WARC_BLOCK_DIGEST: 931 warcBlockDigestStr = fieldValueStr; 932 warcBlockDigest = digestFieldValue; 933 break; 934 case WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST: 935 warcPayloadDigestStr = fieldValueStr; 936 warcPayloadDigest = digestFieldValue; 937 break; 938 /* 939 * ContentType. 940 */ 941 case WarcConstants.FN_IDX_CONTENT_TYPE: 942 contentTypeStr = fieldValueStr; 943 contentType = contentTypeFieldValue; 944 break; 945 case WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE: 946 warcIdentifiedPayloadTypeStr = fieldValueStr; 947 warcIdentifiedPayloadType = contentTypeFieldValue; 948 break; 949 /* 950 * Date. 951 */ 952 case WarcConstants.FN_IDX_WARC_DATE: 953 warcDateStr = fieldValueStr; 954 warcDate = dateFieldValue; 955 break; 956 case WarcConstants.FN_IDX_WARC_REFERS_TO_DATE: 957 warcRefersToDateStr = fieldValueStr; 958 warcRefersToDate = dateFieldValue; 959 break; 960 /* 961 * InetAddress. 962 */ 963 case WarcConstants.FN_IDX_WARC_IP_ADDRESS: 964 warcIpAddress = fieldValueStr; 965 warcInetAddress = inetAddrFieldValue; 966 break; 967 /* 968 * URI. 969 */ 970 case WarcConstants.FN_IDX_WARC_RECORD_ID: 971 warcRecordIdStr = fieldValueStr; 972 warcRecordIdUri = uriFieldValue; 973 break; 974 case WarcConstants.FN_IDX_WARC_CONCURRENT_TO: 975 if (fieldValueStr != null || uriFieldValue != null) { 976 WarcConcurrentTo warcConcurrentTo = new WarcConcurrentTo(); 977 warcConcurrentTo.warcConcurrentToStr = fieldValueStr; 978 warcConcurrentTo.warcConcurrentToUri = uriFieldValue; 979 warcConcurrentToList.add(warcConcurrentTo); 980 } 981 break; 982 case WarcConstants.FN_IDX_WARC_REFERS_TO: 983 warcRefersToStr = fieldValueStr; 984 warcRefersToUri = uriFieldValue; 985 break; 986 case WarcConstants.FN_IDX_WARC_TARGET_URI: 987 warcTargetUriStr = fieldValueStr; 988 warcTargetUriUri = uriFieldValue; 989 break; 990 case WarcConstants.FN_IDX_WARC_WARCINFO_ID: 991 warcWarcinfoIdStr = fieldValueStr; 992 warcWarcinfoIdUri = uriFieldValue; 993 break; 994 case WarcConstants.FN_IDX_WARC_PROFILE: 995 warcProfileStr = fieldValueStr; 996 warcProfileUri = uriFieldValue; 997 if (warcProfileStr != null) { 998 warcProfileIdx = WarcConstants.profileIdxMap.get(warcProfileStr.toLowerCase()); 999 } 1000 if (warcProfileIdx == null && warcProfileStr != null && warcProfileStr.length() > 0) { 1001 warcProfileIdx = WarcConstants.PROFILE_IDX_UNKNOWN; 1002 } 1003 break; 1004 case WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID: 1005 warcSegmentOriginIdStr = fieldValueStr; 1006 warcSegmentOriginIdUrl = uriFieldValue; 1007 break; 1008 case WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI: 1009 warcRefersToTargetUriStr = fieldValueStr; 1010 warcRefersToTargetUriUri = uriFieldValue; 1011 break; 1012 default: 1013 break; 1014 } 1015 } else { 1016 // Invalid datatype for field. 1017 addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED, 1018 "Invalid datatype for '" + fieldName + "' header", 1019 WarcConstants.FDT_IDX_STRINGS[WarcConstants.FN_IDX_DT[fn_idx]], 1020 WarcConstants.FDT_IDX_STRINGS[dt]); 1021 // Consider throwing exception at some point. 1022 } 1023 } 1024 HeaderLine headerLine = new HeaderLine(); 1025 headerLine.name = fieldName; 1026 headerLine.value = fieldValueStr; 1027 HeaderLine tmpLine = headerMap.get(fieldName.toLowerCase()); 1028 if (tmpLine == null) { 1029 headerMap.put(fieldName.toLowerCase(), headerLine); 1030 } else { 1031 tmpLine.lines.add(headerLine); 1032 } 1033 return headerLine; 1034 } 1035 1036 /** 1037 * Validate the WARC header relative to the WARC-Type and according to the 1038 * WARC ISO standard. 1039 */ 1040 protected void checkFields() { 1041 bMandatoryMissing = false; 1042 1043 /* 1044 * Unknown Warc-Type and/or Warc-Profile. 1045 */ 1046 1047 if (warcTypeIdx != null && warcTypeIdx == WarcConstants.RT_IDX_UNKNOWN) { 1048 // Warning: Unknown Warc-Type. 1049 addWarningDiagnosis(DiagnosisType.UNKNOWN, "'" + WarcConstants.FN_WARC_TYPE + "' value", warcTypeStr); 1050 } 1051 1052 if (warcProfileIdx != null && warcProfileIdx == WarcConstants.PROFILE_IDX_UNKNOWN) { 1053 // Warning: Unknown Warc-Profile. 1054 addWarningDiagnosis(DiagnosisType.UNKNOWN, "'" + WarcConstants.FN_WARC_PROFILE + "' value", warcProfileStr); 1055 } 1056 1057 /* 1058 * Mandatory fields. 1059 */ 1060 1061 // TODO Required yes, but is it always invalid. 1062 if (warcTypeIdx == null) { 1063 // Mandatory valid Warc-Type missing. 1064 addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, "'" + WarcConstants.FN_WARC_TYPE + "' header", warcTypeStr); 1065 bMandatoryMissing = true; 1066 } 1067 if (warcRecordIdUri == null) { 1068 // Mandatory valid Warc-Record-Id missing. 1069 addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, "'" + WarcConstants.FN_WARC_RECORD_ID + "' header", warcRecordIdStr); 1070 bMandatoryMissing = true; 1071 } 1072 if (warcDate == null) { 1073 // Mandatory valid Warc-Date missing. 1074 addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, "'" + WarcConstants.FN_WARC_DATE + "' header", warcDateStr); 1075 bMandatoryMissing = true; 1076 } 1077 if (contentLength == null) { 1078 // Mandatory valid Content-Length missing. 1079 addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, "'" + WarcConstants.FN_CONTENT_LENGTH + "' header", contentLengthStr); 1080 bMandatoryMissing = true; 1081 } 1082 1083 /* 1084 * Content-Type should be present if Content-Length > 0. 1085 * Except for continuation records. 1086 */ 1087 1088 if (contentLength != null && contentLength.longValue() > 0L && 1089 (contentTypeStr == null || contentTypeStr.length() == 0)) { 1090 if (warcTypeIdx == null || warcTypeIdx != WarcConstants.RT_IDX_CONTINUATION) { 1091 addWarningDiagnosis(DiagnosisType.RECOMMENDED_MISSING, 1092 "'" + WarcConstants.FN_CONTENT_TYPE + "' header"); 1093 } 1094 } 1095 1096 /* 1097 * WARC record type dependent policies. 1098 */ 1099 1100 if (warcTypeIdx != null) { 1101 /* 1102 * Warcinfo record should have "application/warc-fields" content-type. 1103 */ 1104 1105 if (warcTypeIdx == WarcConstants.RT_IDX_WARCINFO) { 1106 if (contentType != null && 1107 (!contentType.contentType.equals("application") 1108 || !contentType.mediaType.equals("warc-fields"))) { 1109 addWarningDiagnosis(DiagnosisType.RECOMMENDED, 1110 "'" + WarcConstants.FN_CONTENT_TYPE + "' value", 1111 WarcConstants.CT_APP_WARC_FIELDS, 1112 contentTypeStr); 1113 } 1114 } 1115 1116 if (warcTypeIdx == WarcConstants.RT_IDX_RESPONSE) { 1117 if (warcSegmentNumber != null && warcSegmentNumber != 1) { 1118 addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED, 1119 "'" + WarcConstants.FN_WARC_SEGMENT_NUMBER + "' value", 1120 warcSegmentNumber.toString(), 1121 "1"); 1122 } 1123 } 1124 1125 if (warcTypeIdx == WarcConstants.RT_IDX_CONTINUATION) { 1126 if (warcSegmentNumber != null && warcSegmentNumber < 2) { 1127 addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED, 1128 "'" + WarcConstants.FN_WARC_SEGMENT_NUMBER + "' value", 1129 warcSegmentNumber.toString(), 1130 ">1"); 1131 } 1132 } 1133 1134 /* 1135 * Check the policies for each field. 1136 */ 1137 1138 WarcConcurrentTo warcConcurrentTo; 1139 if (warcTypeIdx > 0) { 1140 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_CONTENT_TYPE, contentType, contentTypeStr); 1141 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_IP_ADDRESS, warcInetAddress, warcIpAddress); 1142 for (int i=0; i<warcConcurrentToList.size(); ++i) { 1143 warcConcurrentTo = warcConcurrentToList.get(0); 1144 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_CONCURRENT_TO, warcConcurrentTo.warcConcurrentToUri, warcConcurrentTo.warcConcurrentToStr); 1145 } 1146 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_REFERS_TO, warcRefersToUri, warcRefersToStr); 1147 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_TARGET_URI, warcTargetUriUri, warcTargetUriStr); 1148 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_TRUNCATED, warcTruncatedIdx, warcTruncatedStr); 1149 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_WARCINFO_ID, warcWarcinfoIdUri, warcWarcinfoIdStr); 1150 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_BLOCK_DIGEST, warcBlockDigest, warcBlockDigestStr); 1151 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST, warcPayloadDigest, warcPayloadDigestStr); 1152 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_FILENAME, warcFilename, warcFilename); 1153 // Could also use warcProfileIdx for really strict. 1154 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_PROFILE, warcProfileUri, warcProfileStr); 1155 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE, warcIdentifiedPayloadType, warcIdentifiedPayloadTypeStr); 1156 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER, warcSegmentNumber, warcSegmentNumberStr); 1157 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID, warcSegmentOriginIdUrl, warcSegmentOriginIdStr); 1158 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH, warcSegmentTotalLength, warcSegmentTotalLengthStr); 1159 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI, warcRefersToTargetUriUri, warcRefersToTargetUriStr); 1160 checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_REFERS_TO_DATE, warcRefersToDate, warcRefersToDateStr); 1161 } 1162 } 1163 } 1164 1165 /** 1166 * Given a WARC record type and a WARC field looks up the policy in a 1167 * matrix build from the WARC ISO standard. 1168 * @param recordType WARC record type id 1169 * @param fieldType WARC field type id 1170 * @param fieldObj WARC field 1171 * @param valueStr WARC raw field value 1172 */ 1173 protected void checkFieldPolicy(int recordType, int fieldType, Object fieldObj, String valueStr) { 1174 int policy = WarcConstants.field_policy[recordType][fieldType]; 1175 switch (policy) { 1176 case WarcConstants.POLICY_MANDATORY: 1177 if (fieldObj == null) { 1178 addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, 1179 "'" + WarcConstants.FN_IDX_STRINGS[fieldType] + "' value", 1180 valueStr); 1181 } 1182 break; 1183 case WarcConstants.POLICY_SHALL: 1184 if (fieldObj == null) { 1185 addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, 1186 "'" + WarcConstants.FN_IDX_STRINGS[fieldType] + "' value", 1187 valueStr); 1188 } 1189 break; 1190 case WarcConstants.POLICY_SHALL_NOT: 1191 if (fieldObj != null) { 1192 addErrorDiagnosis(DiagnosisType.UNDESIRED_DATA, 1193 "'" + WarcConstants.FN_IDX_STRINGS[fieldType] + "' value", 1194 valueStr); 1195 } 1196 break; 1197 case WarcConstants.POLICY_MAY_NOT: 1198 if (fieldObj != null) { 1199 addWarningDiagnosis(DiagnosisType.UNDESIRED_DATA, 1200 "'" + WarcConstants.FN_IDX_STRINGS[fieldType] + "' value", 1201 valueStr); 1202 } 1203 break; 1204 case WarcConstants.POLICY_MAY: 1205 case WarcConstants.POLICY_IGNORE: 1206 default: 1207 break; 1208 } 1209 } 1210 1211}