001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.io.ByteArrayOutputStream; 021import java.io.IOException; 022import java.net.InetAddress; 023import java.util.Collections; 024import java.util.HashMap; 025import java.util.LinkedList; 026import java.util.List; 027import java.util.Map; 028 029import org.jwat.common.ByteCountingPushBackInputStream; 030import org.jwat.common.ContentType; 031import org.jwat.common.Diagnosis; 032import org.jwat.common.DiagnosisType; 033import org.jwat.common.Diagnostics; 034import org.jwat.common.HeaderLine; 035import org.jwat.common.HeaderLineReader; 036import org.jwat.common.MaxLengthRecordingInputStream; 037import org.jwat.common.Uri; 038import org.jwat.common.UriProfile; 039 040/** 041 * Central class for working with WARC headers. This class includes support for 042 * reading and writing WARC headers. Methods are also available to validate 043 * individual headers and a WARC header as a whole. 044 * 045 * @author nicl 046 */ 047public class WarcHeader { 048 049 /** An URI with encapsulating <> characters. */ 050 public static final boolean URI_LTGT = true; 051 052 /** An URI without encapsulating <> characters. */ 053 public static final boolean URI_NAKED = false; 054 055 /** Associated WarcReader context. 056 * Must be set prior to calling the various methods. */ 057 protected WarcReader reader; 058 059 /** Version based WARC validation. */ 060 protected WarcValidatorBase warcValidator; 061 062 /** Diagnostics used to report diagnoses. 063 * Must be set prior to calling the various methods. */ 064 protected Diagnostics diagnostics; 065 066 /** WARC-Target-URI profile. */ 067 protected UriProfile warcTargetUriProfile; 068 069 /** URI profile. */ 070 protected UriProfile uriProfile; 071 072 /** WARC field parser used. 073 * Must be set prior to calling the various methods. */ 074 protected WarcFieldParsers fieldParsers; 075 076 /** Max size allowed for a record header. */ 077 protected int recordHeaderMaxSize = -1; 078 079 /** Line reader used to read version lines. */ 080 protected HeaderLineReader lineReader; 081 082 /** Header line reader used to read the WARC headers. */ 083 protected HeaderLineReader headerLineReader; 084 085 /** WARC record starting offset relative to the source WARC file input 086 * stream. The offset is correct for both compressed and uncompressed streams. */ 087 protected long startOffset = -1; 088 089 /* 090 * Version related fields. 091 */ 092 093 /** Was "WARC/" identified while looking for the version string. */ 094 public boolean bMagicIdentified; 095 /** Did the version string include between 2 and 4 substrings delimited by ".". */ 096 public boolean bVersionParsed; 097 /** Is the version format valid. */ 098 public boolean bValidVersionFormat; 099 /** Is the version recognized. (0.17, 0.18 or 1.0) */ 100 public boolean bValidVersion; 101 102 /** Raw version string. */ 103 public String versionStr; 104 /** Array based on the version string split by the "." delimiter and converted to integers. */ 105 public int[] versionArr; 106 107 /** Major version number from WARC header. */ 108 public int major = -1; 109 /** Minor version number from WARC header. */ 110 public int minor = -1; 111 112 /* 113 * WARC header fields. 114 */ 115 116 /** Array used for duplicate header detection. */ 117 protected boolean[] seen = new boolean[WarcConstants.FN_INDEX_OF_LAST]; 118 119 /** Is the header missing one of the mandatory headers. */ 120 public boolean bMandatoryMissing; 121 122 /** WARC-Type field string value. */ 123 public String warcTypeStr; 124 /** WARC-Type converted to an integer id, if identified. */ 125 public Integer warcTypeIdx; 126 127 /** WARC-Filename field string value. 128 * (warcinfo record type only) */ 129 public String warcFilename; 130 131 /** WARC-Record-Id field string value. */ 132 public String warcRecordIdStr; 133 /** WARC-Record-Id converted to an <code>Uri</code> object, if valid. */ 134 public Uri warcRecordIdUri; 135 136 /** WARC-Date field string value. */ 137 public String warcDateStr; 138 /** WARC-Date converted to a <code>Date</code> object, if valid. */ 139 public WarcDate warcDate; 140 141 /** Content-Length field string value. */ 142 public String contentLengthStr; 143 /** Content-Length converted to a <code>Long</code> object, if valid. */ 144 public Long contentLength; 145 146 /** Content-Type field string value. */ 147 public String contentTypeStr; 148 /** Content-Type converted to a <code>ContentType</code> object, if valid. */ 149 public ContentType contentType; 150 151 /** WARC-Truncated field string value. */ 152 public String warcTruncatedStr; 153 /** WARC-Truncated converted to an integer id, if valid. */ 154 public Integer warcTruncatedIdx; 155 156 /** WARC-IP-Address field string value. */ 157 public String warcIpAddress; 158 /** WARC-IP-Address converted to an <code>InetAddress</code> object, if valid. */ 159 public InetAddress warcInetAddress; 160 161 /** List of WARC-Concurrent-To field string values and converted <code>URI</code> objects, if valid. */ 162 public List<WarcConcurrentTo> warcConcurrentToList = new LinkedList<WarcConcurrentTo>(); 163 164 /** WARC-Refers-To field string value. */ 165 public String warcRefersToStr; 166 /** WARC-Refers-To converted to an <code>Uri</code> object, if valid. */ 167 public Uri warcRefersToUri; 168 169 /** WARC_Target-URI field string value. */ 170 public String warcTargetUriStr; 171 /** WARC-TargetURI converted to an <code>Uri</code> object, if valid. */ 172 public Uri warcTargetUriUri; 173 174 /** WARC-Warcinfo-Id field string value. */ 175 public String warcWarcinfoIdStr; 176 /** WARC-Warcinfo-Id converted to an <code>Uri</code> object, if valid. */ 177 public Uri warcWarcinfoIdUri; 178 179 /** WARC-Block-Digest field string value. */ 180 public String warcBlockDigestStr; 181 /** WARC-Block-Digest converted to a <code>WarcDigest</code> object, if valid. */ 182 public WarcDigest warcBlockDigest; 183 184 /** WARC-Payload-Digest field string value. */ 185 public String warcPayloadDigestStr; 186 /** WARC-Payload-Digest converted to a <code>WarcDigest</code> object, if valid. */ 187 public WarcDigest warcPayloadDigest; 188 189 /** WARC-Identified-Payload-Type field string value. */ 190 public String warcIdentifiedPayloadTypeStr; 191 /** WARC-Identified-Payload-Type converted to a <code>ContentType</code> object, if valid. */ 192 public ContentType warcIdentifiedPayloadType; 193 194 /** WARC-Profile field string value. 195 * (revisit record only) */ 196 public String warcProfileStr; 197 /** WARC-Profile field converted to an <code>Uri</code> object, if valid. 198 * (revisit record only) */ 199 public Uri warcProfileUri; 200 /** WARC-Profile converted to an integer id, if valid. 201 * (revisit record only) */ 202 public Integer warcProfileIdx; 203 204 /** WARC-Segment-Number field string value. */ 205 public String warcSegmentNumberStr; 206 /** WARC-Segment-Number converted to an <code>Integer</code> object, if valid. */ 207 public Integer warcSegmentNumber; 208 209 /** WARC-Segment-Origin-Id field string value. 210 * (continuation record only) */ 211 public String warcSegmentOriginIdStr; 212 /** WARC-Segment-Origin-Id converted to an <code>Uri</code> object, if valid. 213 * (continuation record only) */ 214 public Uri warcSegmentOriginIdUrl; 215 216 /** WARC-Segment-Total-Length field string value. 217 * (continuation record only) */ 218 public String warcSegmentTotalLengthStr; 219 /** WARC-Segment-Total-Length converted to a <code>Long</code> object, if valid. 220 * (continuation record only) */ 221 public Long warcSegmentTotalLength; 222 223 // see https://docs.google.com/document/d/1QyQBA7Ykgxie75V8Jziz_O7hbhwf7PF6_u9O6w6zgp0/edit 224 /** WARC-Refers-To-Target-URI field string value. */ 225 public String warcRefersToTargetUriStr; 226 /** WARC-Refers-To-Target-URI converted to an <code>Uri</code> object, if valid. */ 227 public Uri warcRefersToTargetUriUri; 228 /** WARC-Refers-To-Date */ 229 public String warcRefersToDateStr; 230 /** WARC-Date converted to a <code>Date</code> object, if valid. */ 231 public WarcDate warcRefersToDate; 232 233 /* 234 * WARC header fields collections. 235 */ 236 237 /** Raw WARC header output stream. */ 238 protected ByteArrayOutputStream headerBytesOut = new ByteArrayOutputStream(); 239 240 /** Raw WARC header byte array. */ 241 public byte[] headerBytes; 242 243 /** List of parsed header fields. */ 244 protected List<HeaderLine> headerList = new LinkedList<HeaderLine>(); 245 246 /** Map of parsed header fields. */ 247 protected Map<String, HeaderLine> headerMap = new HashMap<String, HeaderLine>(); 248 249 /** 250 * Non public constructor to allow unit testing. 251 */ 252 protected WarcHeader() { 253 } 254 255 /** 256 * Create a new <code>WarcHeader</code> for recreating a header object. 257 * @param recordHeaderMaxSize max size allowed for a record header 258 * @param lineReader line reader used to read version lines 259 * @param headerLineReader header line reader used to read the WARC headers 260 * @param fieldParsers parsers used for the individual types 261 * @param uriProfile uri profile used to validate urls 262 * @param diagnostics diagnostics object used by parser 263 * @return a <code>WarcHeader</code> prepared for reanimation 264 */ 265 public static WarcHeader initHeader(int recordHeaderMaxSize, HeaderLineReader lineReader, HeaderLineReader headerLineReader, WarcFieldParsers fieldParsers, UriProfile uriProfile, Diagnostics diagnostics) { 266 WarcHeader header = new WarcHeader(); 267 header.warcTargetUriProfile = uriProfile; 268 header.uriProfile = uriProfile; 269 header.fieldParsers = fieldParsers; 270 header.diagnostics = diagnostics; 271 header.recordHeaderMaxSize = recordHeaderMaxSize; 272 header.lineReader = lineReader; 273 header.headerLineReader = headerLineReader; 274 return header; 275 } 276 277 /** 278 * Create and initialize a new <code>WarcHeader</code> for writing. 279 * @param writer writer which shall be used 280 * @param diagnostics diagnostics object used by writer 281 * @return a <code>WarcHeader</code> prepared for writing 282 */ 283 public static WarcHeader initHeader(WarcWriter writer, Diagnostics diagnostics) { 284 WarcHeader header = new WarcHeader(); 285 // Set default version to "1.0". 286 header.major = 1; 287 header.minor = 0; 288 header.warcTargetUriProfile = writer.warcTargetUriProfile; 289 header.uriProfile = writer.uriProfile; 290 header.fieldParsers = writer.fieldParsers; 291 header.diagnostics = diagnostics; 292 return header; 293 } 294 295 /** 296 * Create and initialize a new <code>WarcHeader</code> for reading. 297 * @param reader reader which shall be used 298 * @param startOffset start offset of header 299 * @param diagnostics diagnostics object used by reader 300 * @return a <code>WarcHeader</code> prepared for reading 301 */ 302 public static WarcHeader initHeader(WarcReader reader, long startOffset, Diagnostics diagnostics) { 303 WarcHeader header = new WarcHeader(); 304 header.reader = reader; 305 header.warcTargetUriProfile = reader.warcTargetUriProfile; 306 header.uriProfile = reader.uriProfile; 307 header.fieldParsers = reader.fieldParsers; 308 header.diagnostics = diagnostics; 309 header.recordHeaderMaxSize = reader.recordHeaderMaxSize; 310 header.lineReader = reader.lineReader; 311 header.headerLineReader = reader.headerLineReader; 312 // This is only relevant for uncompressed sequentially read records 313 header.startOffset = startOffset; 314 return header; 315 } 316 317 /** 318 * Add a warning diagnosis of the given type on a specific entity with 319 * optional extra information. The information varies according to the 320 * diagnosis type. 321 * @param type diagnosis type 322 * @param entity entity examined 323 * @param information optional extra information 324 */ 325 protected void addWarningDiagnosis(DiagnosisType type, String entity, String... information) { 326 diagnostics.addWarning(new Diagnosis(type, entity, information)); 327 } 328 329 /** 330 * Returns the starting offset of the record in the containing WARC. 331 * @return the starting offset of the record 332 */ 333 public long getStartOffset() { 334 return startOffset; 335 } 336 337 /** 338 * Try to parse a WARC header and return a boolean indicating the success or 339 * failure of this. 340 * @param in input stream with WARC data 341 * @return boolean indicating whether a header was parsed or not 342 * @throws IOException if an I/O exception occurs while parsing for a header 343 */ 344 public boolean parseHeader(ByteCountingPushBackInputStream in) throws IOException { 345 if (parseVersion(in)) { 346 // debug 347 //System.out.println(wr.bMagicIdentified); 348 //System.out.println(wr.bVersionParsed); 349 //System.out.println(wr.major + "." + wr.minor); 350 warcValidator = reader.validation.getValidatorFor(this, diagnostics); 351 352 MaxLengthRecordingInputStream mrin = new MaxLengthRecordingInputStream(in, recordHeaderMaxSize); 353 ByteCountingPushBackInputStream pbin = new ByteCountingPushBackInputStream(mrin, recordHeaderMaxSize); 354 355 parseHeaders(pbin); 356 pbin.close(); 357 358 warcValidator.checkFields(this); 359 360 headerBytes = headerBytesOut.toByteArray(); 361 } 362 return bMagicIdentified; 363 } 364 365 /** 366 * Looks forward in the input stream for a valid WARC version line. 367 * @param in data input stream 368 * @return true, if magic WARC header found 369 * @throws IOException if an error occurs while reading version data 370 */ 371 public boolean parseVersion(ByteCountingPushBackInputStream in) throws IOException { 372 bMagicIdentified = false; 373 bVersionParsed = false; 374 boolean bInvalidDataBeforeVersion = false; 375 boolean bEmptyLinesBeforeVersion = false; 376 HeaderLine line; 377 String tmpStr; 378 boolean bSeekMagic = true; 379 // Loop until when have found something that looks like a version line. 380 while (bSeekMagic) { 381 // This is only relevant for uncompressed sequentially read records 382 startOffset = in.getConsumed(); 383 line = lineReader.readLine(in); 384 if (!lineReader.bEof) { 385 switch (line.type) { 386 case HeaderLine.HLT_LINE: 387 tmpStr = line.line; 388 // debug 389 //System.out.println(tmpStr); 390 if (tmpStr.length() > 0) { 391 if (tmpStr.toUpperCase().startsWith(WarcConstants.WARC_MAGIC_HEADER)) { 392 bMagicIdentified = true; 393 versionStr = tmpStr.substring(WarcConstants.WARC_MAGIC_HEADER.length()); 394 String[] tmpArr = versionStr.split("\\.", -1); // Not optimal 395 if (tmpArr.length >= 2 && tmpArr.length <= 4) { 396 bVersionParsed = true; 397 bValidVersionFormat = true; 398 versionArr = new int[tmpArr.length]; 399 for (int i=0; i<tmpArr.length; ++i) { 400 try { 401 versionArr[i] = Integer.parseInt(tmpArr[i]); 402 } catch (NumberFormatException e) { 403 versionArr[i] = -1; 404 bValidVersionFormat = false; 405 } 406 } 407 major = versionArr[0]; 408 minor = versionArr[1]; 409 } 410 headerBytesOut.write(line.raw); 411 bSeekMagic = false; 412 } else { 413 // Invalid data aka Gibberish. 414 bInvalidDataBeforeVersion = true; 415 } 416 } else { 417 // Empty line. 418 bEmptyLinesBeforeVersion = true; 419 420 } 421 break; 422 case HeaderLine.HLT_HEADERLINE: 423 // Invalid data - header or binary. 424 bInvalidDataBeforeVersion = true; 425 break; 426 } 427 } else { 428 // EOF. 429 bSeekMagic = false; 430 } 431 } 432 if (bInvalidDataBeforeVersion) { 433 diagnostics.addError(DiagnosisType.INVALID, "Data before WARC version"); 434 } 435 if (bEmptyLinesBeforeVersion) { 436 diagnostics.addError(DiagnosisType.INVALID, "Empty lines before WARC version"); 437 } 438 return bMagicIdentified; 439 } 440 441 /** 442 * Reads WARC header lines one line at a time until an empty line is 443 * encountered. 444 * @param in header input stream 445 * @throws IOException if an error occurs while reading the WARC header 446 */ 447 public void parseHeaders(ByteCountingPushBackInputStream in) throws IOException { 448 HeaderLine headerLine; 449 boolean bLoop = true; 450 while (bLoop) { 451 headerLine = headerLineReader.readLine(in); 452 if ((headerLine.bfErrors & HeaderLineReader.E_BIT_INVALID_CHARSET) != 0) { 453 diagnostics.addError(DiagnosisType.INVALID_ENCODING, "Invalid encoding in header line", headerLine.value, "UNKNOWN"); 454 } 455 if (!headerLineReader.bEof) { 456 headerBytesOut.write(headerLine.raw); 457 switch (headerLine.type) { 458 case HeaderLine.HLT_HEADERLINE: 459 if (headerLine.name != null && headerLine.name.length() > 0) { 460 // debug 461 //System.out.println(headerLine.name); 462 //System.out.println(headerLine.value); 463 addHeader(headerLine); 464 } else { 465 // Empty field name. 466 addWarningDiagnosis(DiagnosisType.EMPTY, "Header line"); 467 } 468 break; 469 case HeaderLine.HLT_LINE: 470 if (headerLine.line.length() == 0) { 471 // Empty line. 472 bLoop = false; 473 } else { 474 // Unknown header line. 475 addWarningDiagnosis(DiagnosisType.UNKNOWN, "Header line", headerLine.line); 476 } 477 break; 478 default: 479 throw new IllegalStateException("Invalid HeaderLine output!"); 480 } 481 } else { 482 // EOF. 483 bLoop = false; 484 } 485 } 486 } 487 488 /** 489 * Identify a (WARC) header name, validate the value and set the header. 490 * @param headerLine the headerLine 491 */ 492 public void addHeader(HeaderLine headerLine) { 493 String fieldName = headerLine.name; 494 String fieldValue = headerLine.value; 495 WarcConcurrentTo warcConcurrentTo; 496 Integer fn_idx = WarcConstants.fieldNameIdxMap.get(fieldName.toLowerCase()); 497 if (fn_idx != null) { 498 // WARC field name defined in WARC specification. 499 if (!seen[fn_idx] || WarcConstants.fieldNamesRepeatableLookup[fn_idx]) { 500 seen[fn_idx] = true; 501 switch (fn_idx.intValue()) { 502 case WarcConstants.FN_IDX_WARC_TYPE: 503 warcTypeStr = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_TYPE); 504 if (warcTypeStr != null) { 505 warcTypeIdx = WarcConstants.recordTypeIdxMap.get(warcTypeStr.toLowerCase()); 506 } 507 if (warcTypeIdx == null && warcTypeStr != null && warcTypeStr.length() > 0) { 508 warcTypeIdx = WarcConstants.RT_IDX_UNKNOWN; 509 } 510 break; 511 case WarcConstants.FN_IDX_WARC_RECORD_ID: 512 warcRecordIdStr = fieldValue; 513 warcRecordIdUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_RECORD_ID); 514 break; 515 case WarcConstants.FN_IDX_WARC_DATE: 516 warcDateStr = fieldValue; 517 //warcDate = fieldParsers.parseDate(fieldValue, WarcConstants.FN_WARC_DATE); 518 warcDate = fieldParsers.parseWarcDate(fieldValue, WarcConstants.FN_WARC_DATE); 519 break; 520 case WarcConstants.FN_IDX_CONTENT_LENGTH: 521 contentLengthStr = fieldValue; 522 contentLength = fieldParsers.parseLong(fieldValue, WarcConstants.FN_CONTENT_LENGTH); 523 break; 524 case WarcConstants.FN_IDX_CONTENT_TYPE: 525 contentTypeStr = fieldValue; 526 contentType = fieldParsers.parseContentType(fieldValue, WarcConstants.FN_CONTENT_TYPE); 527 break; 528 case WarcConstants.FN_IDX_WARC_CONCURRENT_TO: 529 Uri tmpUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_CONCURRENT_TO); 530 if (fieldValue != null && fieldValue.trim().length() > 0) { 531 warcConcurrentTo = new WarcConcurrentTo(); 532 warcConcurrentTo.warcConcurrentToStr = fieldValue; 533 warcConcurrentTo.warcConcurrentToUri = tmpUri; 534 warcConcurrentToList.add(warcConcurrentTo); 535 } 536 break; 537 case WarcConstants.FN_IDX_WARC_BLOCK_DIGEST: 538 warcBlockDigestStr = fieldValue; 539 warcBlockDigest = fieldParsers.parseDigest(fieldValue, WarcConstants.FN_WARC_BLOCK_DIGEST); 540 break; 541 case WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST: 542 warcPayloadDigestStr = fieldValue; 543 warcPayloadDigest = fieldParsers.parseDigest(fieldValue, WarcConstants.FN_WARC_PAYLOAD_DIGEST); 544 break; 545 case WarcConstants.FN_IDX_WARC_IP_ADDRESS: 546 warcIpAddress = fieldValue; 547 warcInetAddress = fieldParsers.parseIpAddress(fieldValue, WarcConstants.FN_WARC_IP_ADDRESS); 548 break; 549 case WarcConstants.FN_IDX_WARC_REFERS_TO: 550 warcRefersToStr = fieldValue; 551 warcRefersToUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_REFERS_TO); 552 break; 553 case WarcConstants.FN_IDX_WARC_TARGET_URI: 554 warcTargetUriStr = fieldValue; 555 warcTargetUriUri = fieldParsers.parseUri(fieldValue, URI_NAKED, warcTargetUriProfile, WarcConstants.FN_WARC_TARGET_URI); 556 break; 557 case WarcConstants.FN_IDX_WARC_TRUNCATED: 558 warcTruncatedStr = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_TRUNCATED); 559 if (warcTruncatedStr != null) { 560 warcTruncatedIdx = WarcConstants.truncatedTypeIdxMap.get(warcTruncatedStr.toLowerCase()); 561 } 562 if (warcTruncatedIdx == null && warcTruncatedStr != null && warcTruncatedStr.length() > 0) { 563 warcTruncatedIdx = WarcConstants.TT_IDX_FUTURE_REASON; 564 } 565 break; 566 case WarcConstants.FN_IDX_WARC_WARCINFO_ID: 567 warcWarcinfoIdStr = fieldValue; 568 warcWarcinfoIdUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_WARCINFO_ID); 569 break; 570 case WarcConstants.FN_IDX_WARC_FILENAME: 571 warcFilename = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_FILENAME); 572 break; 573 case WarcConstants.FN_IDX_WARC_PROFILE: 574 warcProfileStr = fieldValue; 575 warcProfileUri = fieldParsers.parseUri(fieldValue, URI_NAKED, uriProfile, WarcConstants.FN_WARC_PROFILE); 576 if (warcProfileStr != null) { 577 warcProfileIdx = WarcConstants.profileIdxMap.get(warcProfileStr.toLowerCase()); 578 } 579 if (warcProfileIdx == null && warcProfileStr != null && warcProfileStr.length() > 0) { 580 warcProfileIdx = WarcConstants.WARC_PROFILE_IDX_UNKNOWN; 581 } 582 break; 583 case WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE: 584 warcIdentifiedPayloadTypeStr = fieldValue; 585 warcIdentifiedPayloadType = fieldParsers.parseContentType(fieldValue, WarcConstants.FN_WARC_IDENTIFIED_PAYLOAD_TYPE); 586 break; 587 case WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID: 588 warcSegmentOriginIdStr = fieldValue; 589 warcSegmentOriginIdUrl = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_SEGMENT_ORIGIN_ID); 590 break; 591 case WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER: 592 warcSegmentNumberStr = fieldValue; 593 warcSegmentNumber = fieldParsers.parseInteger(fieldValue, WarcConstants.FN_WARC_SEGMENT_NUMBER); 594 break; 595 case WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH: 596 warcSegmentTotalLengthStr = fieldValue; 597 warcSegmentTotalLength = fieldParsers.parseLong(fieldValue, WarcConstants.FN_WARC_SEGMENT_TOTAL_LENGTH); 598 break; 599 case WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI: 600 warcRefersToTargetUriStr = fieldValue; 601 warcRefersToTargetUriUri = fieldParsers.parseUri(fieldValue, URI_NAKED, warcTargetUriProfile, WarcConstants.FN_WARC_REFERS_TO_TARGET_URI); 602 break; 603 case WarcConstants.FN_IDX_WARC_REFERS_TO_DATE: 604 warcRefersToDateStr = fieldValue; 605 //warcRefersToDate = fieldParsers.parseDate(fieldValue, WarcConstants.FN_WARC_REFERS_TO_DATE); 606 warcRefersToDate = fieldParsers.parseWarcDate(fieldValue, WarcConstants.FN_WARC_REFERS_TO_DATE); 607 break; 608 } 609 } else { 610 // Duplicate field. 611 diagnostics.addError(DiagnosisType.DUPLICATE, "'" + fieldName + "' header", fieldValue); 612 } 613 } 614 HeaderLine tmpLine = headerMap.get(fieldName.toLowerCase()); 615 if (tmpLine == null) { 616 headerMap.put(fieldName.toLowerCase(), headerLine); 617 } else { 618 tmpLine.lines.add(headerLine); 619 } 620 headerList.add(headerLine); 621 } 622 623 /** 624 * Get a <code>List</code> of all the headers found during parsing. 625 * @return <code>List</code> of <code>HeaderLine</code> 626 */ 627 public List<HeaderLine> getHeaderList() { 628 return Collections.unmodifiableList(headerList); 629 } 630 631 /** 632 * Get a header line structure or null, if no header line structure is 633 * stored with the given header name. 634 * @param field header name 635 * @return <code>HeaderLine</code> structure or null 636 */ 637 public HeaderLine getHeader(String field) { 638 if (field != null && field.length() > 0) { 639 return headerMap.get(field.toLowerCase()); 640 } else { 641 return null; 642 } 643 } 644 645 /** 646 * Add a String header using the supplied string and return a 647 * <code>HeaderLine</code> object corresponding to how the header would be 648 * read. 649 * @param fieldName name of field to add 650 * @param fieldValue field value string 651 * @return <code>HeaderLine</code> object corresponding to what would have been read 652 */ 653 public HeaderLine addHeader(String fieldName, String fieldValue) { 654 HeaderLine headerLine = new HeaderLine(); 655 headerLine.name = fieldName; 656 headerLine.value = fieldValue; 657 addHeader(headerLine); 658 return headerLine; 659 } 660 661 /** 662 * Add an Integer header using the supplied string and object values and return 663 * a <code>HeaderLine</code> object corresponding to how the header would be read. 664 * If both string and object values are not null they are used as is. 665 * If the string value is null and the object is not null, 666 * the object's toString method is called. 667 * If the object is null and the string is not null, the string is parsed 668 * and validated resulting in an object, if valid. 669 * @param fieldName name of field to add 670 * @param integerFieldValue <code>Integer</code> field value object 671 * @param fieldValueStr Integer field value string 672 * @return <code>HeaderLine</code> object corresponding to what would have been read 673 */ 674 public HeaderLine addHeader(String fieldName, Integer integerFieldValue, String fieldValueStr) { 675 if (integerFieldValue == null && fieldValueStr != null) { 676 integerFieldValue = fieldParsers.parseInteger(fieldValueStr, fieldName); 677 } else if (fieldValueStr == null && integerFieldValue != null) { 678 fieldValueStr = integerFieldValue.toString(); 679 } 680 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_INTEGER, 681 integerFieldValue, null, null, null, null, null, null); 682 } 683 684 /** 685 * Add a Long header using the supplied string and object values and return 686 * a <code>HeaderLine</code> object corresponding to how the header would be read. 687 * If both string and object values are not null they are used as is. 688 * If the string value is null and the object is not null, 689 * the object's toString method is called. 690 * If the object is null and the string is not null, the string is parsed 691 * and validated resulting in an object, if valid. 692 * @param fieldName name of field to add 693 * @param longFieldValue <code>Long</code> field value object 694 * @param fieldValueStr Long field value string 695 * @return <code>HeaderLine</code> object corresponding to what would have been read 696 */ 697 public HeaderLine addHeader(String fieldName, Long longFieldValue, String fieldValueStr) { 698 if (longFieldValue == null && fieldValueStr != null) { 699 longFieldValue = fieldParsers.parseLong(fieldValueStr, fieldName); 700 } else if (fieldValueStr == null && longFieldValue != null) { 701 fieldValueStr = longFieldValue.toString(); 702 } 703 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_LONG, 704 null, longFieldValue, null, null, null, null, null); 705 } 706 707 /** 708 * Add an Digest header using the supplied string and object values and return 709 * a <code>HeaderLine</code> object corresponding to how the header would be read. 710 * If both string and object values are not null they are used as is. 711 * If the string value is null and the object is not null, 712 * the object's toString method is called. 713 * If the object is null and the string is not null, the string is parsed 714 * and validated resulting in an object, if valid. 715 * @param fieldName name of field to add 716 * @param digestFieldValue <code>Digest</code> field value object 717 * @param fieldValueStr Digest field value string 718 * @return <code>HeaderLine</code> object corresponding to what would have been read 719 */ 720 public HeaderLine addHeader(String fieldName, WarcDigest digestFieldValue, String fieldValueStr) { 721 if (digestFieldValue == null && fieldValueStr != null) { 722 digestFieldValue = fieldParsers.parseDigest(fieldValueStr, fieldName); 723 } else if (fieldValueStr == null && digestFieldValue != null) { 724 fieldValueStr = digestFieldValue.toString(); 725 } 726 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_DIGEST, 727 null, null, digestFieldValue, null, null, null, null); 728 } 729 730 /** 731 * Add an Content-Type header using the supplied string and object values and return 732 * a <code>HeaderLine</code> object corresponding to how the header would be read. 733 * If both string and object values are not null they are used as is. 734 * If the string value is null and the object is not null, 735 * the object's toString method is called. 736 * If the object is null and the string is not null, the string is parsed 737 * and validated resulting in an object, if valid. 738 * @param fieldName name of field to add 739 * @param contentTypeFieldValue <code>ContentType</code> field value object 740 * @param fieldValueStr Content-Type field value string 741 * @return <code>HeaderLine</code> object corresponding to what would have been read 742 */ 743 public HeaderLine addHeader(String fieldName, ContentType contentTypeFieldValue, String fieldValueStr) { 744 if (contentTypeFieldValue == null && fieldValueStr != null) { 745 contentTypeFieldValue = fieldParsers.parseContentType(fieldValueStr, fieldName); 746 } else if (fieldValueStr == null && contentTypeFieldValue != null) { 747 fieldValueStr = contentTypeFieldValue.toString(); 748 } 749 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_CONTENTTYPE, 750 null, null, null, contentTypeFieldValue, null, null, null); 751 } 752 753 /** 754 * Add an Date header using the supplied string and object values and return 755 * a <code>HeaderLine</code> object corresponding to how the header would be read. 756 * If both string and object values are not null they are used as is. 757 * If the string value is null and the object is not null, 758 * the object's toString method is called. 759 * If the object is null and the string is not null, the string is parsed 760 * and validated resulting in an object, if valid. 761 * @param fieldName name of field to add 762 * @param dateFieldValue <code>Date</code> field value object 763 * @param fieldValueStr Date field value string 764 * @return <code>HeaderLine</code> object corresponding to what would have been read 765 */ 766 public HeaderLine addHeader(String fieldName, WarcDate dateFieldValue, String fieldValueStr) { 767 if (dateFieldValue == null && fieldValueStr != null) { 768 // FIXME Check check check. 769 //dateFieldValue = fieldParsers.parseDate(fieldValueStr, fieldName); 770 dateFieldValue = fieldParsers.parseWarcDate(fieldValueStr, fieldName); 771 } else if (fieldValueStr == null && dateFieldValue != null) { 772 // FIXME Check check check. 773 //fieldValueStr = WarcDateParser.getDateFormat().format(dateFieldValue); 774 // FIXME Tricky since precision is WARC version dependent! 775 fieldValueStr = dateFieldValue.toString(); 776 } 777 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_DATE, 778 null, null, null, null, dateFieldValue, null, null); 779 } 780 781 /** 782 * Add an InetAddress header using the supplied string and object values and return 783 * a <code>HeaderLine</code> object corresponding to how the header would be read. 784 * If both string and object values are not null they are used as is. 785 * If the string value is null and the object is not null, 786 * the object's toString method is called. 787 * If the object is null and the string is not null, the string is parsed 788 * and validated resulting in an object, if valid. 789 * @param fieldName name of field to add 790 * @param inetAddrFieldValue <code>InetAddress</code> field value object 791 * @param fieldValueStr IP-Address field value string 792 * @return <code>HeaderLine</code> object corresponding to what would have been read 793 */ 794 public HeaderLine addHeader(String fieldName, InetAddress inetAddrFieldValue, String fieldValueStr) { 795 if (inetAddrFieldValue == null && fieldValueStr != null) { 796 inetAddrFieldValue = fieldParsers.parseIpAddress(fieldValueStr, fieldName); 797 } else if (fieldValueStr == null && inetAddrFieldValue != null) { 798 fieldValueStr = inetAddrFieldValue.getHostAddress(); 799 } 800 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_INETADDRESS, 801 null, null, null, null, null, inetAddrFieldValue, null); 802 } 803 804 /** 805 * Add an URI header using the supplied string and object values and return 806 * a <code>HeaderLine</code> object corresponding to how the header would be read. 807 * If both string and object values are not null they are used as is. 808 * If the string value is null and the object is not null, 809 * the object's toString method is called. 810 * If the object is null and the string is not null, the string is parsed 811 * and validated resulting in an object, if valid. 812 * @param fieldName name of field to add 813 * @param uriFieldValue <code>URI</code> field value object 814 * @param fieldValueStr URI field value string 815 * @return <code>HeaderLine</code> object corresponding to what would have been read 816 */ 817 public HeaderLine addHeader(String fieldName, Uri uriFieldValue, String fieldValueStr) { 818 if (uriFieldValue == null && fieldValueStr != null) { 819 if (WarcConstants.FN_WARC_TARGET_URI.equalsIgnoreCase(fieldName)) { 820 uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, warcTargetUriProfile, fieldName); 821 } else if (WarcConstants.FN_WARC_PROFILE.equalsIgnoreCase(fieldName)) { 822 uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, uriProfile, fieldName); 823 } else if (WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.equalsIgnoreCase(fieldName)) { 824 uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, warcTargetUriProfile, fieldName); 825 } else { 826 uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_LTGT, uriProfile, fieldName); 827 } 828 } else if (fieldValueStr == null && uriFieldValue != null) { 829 if (WarcConstants.FN_WARC_TARGET_URI.equalsIgnoreCase(fieldName) 830 || WarcConstants.FN_WARC_PROFILE.equalsIgnoreCase(fieldName) 831 || WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.equalsIgnoreCase(fieldName)) { 832 fieldValueStr = uriFieldValue.toString(); 833 } else { 834 fieldValueStr = "<" + uriFieldValue.toString() + ">"; 835 } 836 } 837 return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_URI, 838 null, null, null, null, null, null, uriFieldValue); 839 } 840 841 /** 842 * Add a header with the supplied field name, data type and value and 843 * return a <code>HeaderLine</code> corresponding to how the header will 844 * be read. The data type is validated against the field data type. 845 * The values used are the field value string and the parameter 846 * corresponding to the data type. 847 * @param fieldName header field name 848 * @param fieldValueStr field value in string form 849 * @param dt data type of the field value string when converted to an object 850 * @param integerFieldValue <code>Integer</code> object field value 851 * @param longFieldValue <code>Long</code> object field value 852 * @param digestFieldValue <code>Digest</code> object field value 853 * @param contentTypeFieldValue <code>ContentType</code> object field value 854 * @param dateFieldValue <code>Date</code> object field value 855 * @param inetAddrFieldValue <code>InetAddress</code> object field value 856 * @param uriFieldValue <code>URI</code> object field value 857 * @return <code>HeaderLine</code> object corresponding to what would have been read 858 */ 859 public HeaderLine addHeader(String fieldName, String fieldValueStr, int dt, 860 Integer integerFieldValue, Long longFieldValue, 861 WarcDigest digestFieldValue, ContentType contentTypeFieldValue, 862 WarcDate dateFieldValue, InetAddress inetAddrFieldValue, 863 Uri uriFieldValue) { 864 Integer fn_idx = WarcConstants.fieldNameIdxMap.get(fieldName.toLowerCase()); 865 if (fn_idx != null) { 866 // Implicit cast from integer to long, if needed. 867 if (WarcConstants.FN_IDX_DT[fn_idx] == WarcConstants.FDT_LONG && dt == WarcConstants.FDT_INTEGER) { 868 longFieldValue = (long)integerFieldValue; 869 dt = WarcConstants.FDT_LONG; 870 } 871 if (dt == WarcConstants.FN_IDX_DT[fn_idx]) { 872 // WARC field name defined in WARC specification. 873 if (seen[fn_idx] && !WarcConstants.fieldNamesRepeatableLookup[fn_idx]) { 874 // Duplicate field. 875 diagnostics.addError(DiagnosisType.DUPLICATE, "'" + fieldName + "' header", fieldValueStr); 876 } 877 seen[fn_idx] = true; 878 switch (fn_idx.intValue()) { 879 /* 880 * Integer. 881 */ 882 case WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER: 883 warcSegmentNumberStr = fieldValueStr; 884 warcSegmentNumber = integerFieldValue; 885 break; 886 /* 887 * Long. 888 */ 889 case WarcConstants.FN_IDX_CONTENT_LENGTH: 890 contentLengthStr = fieldValueStr; 891 contentLength = longFieldValue; 892 break; 893 case WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH: 894 warcSegmentTotalLengthStr = fieldValueStr; 895 warcSegmentTotalLength = longFieldValue; 896 break; 897 /* 898 * Digest. 899 */ 900 case WarcConstants.FN_IDX_WARC_BLOCK_DIGEST: 901 warcBlockDigestStr = fieldValueStr; 902 warcBlockDigest = digestFieldValue; 903 break; 904 case WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST: 905 warcPayloadDigestStr = fieldValueStr; 906 warcPayloadDigest = digestFieldValue; 907 break; 908 /* 909 * ContentType. 910 */ 911 case WarcConstants.FN_IDX_CONTENT_TYPE: 912 contentTypeStr = fieldValueStr; 913 contentType = contentTypeFieldValue; 914 break; 915 case WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE: 916 warcIdentifiedPayloadTypeStr = fieldValueStr; 917 warcIdentifiedPayloadType = contentTypeFieldValue; 918 break; 919 /* 920 * Date. 921 */ 922 case WarcConstants.FN_IDX_WARC_DATE: 923 warcDateStr = fieldValueStr; 924 // FIXME Maybe change parameter type if it does not break too many things. 925 //warcDate = WarcDate.getWarcDate(dateFieldValue); 926 warcDate = dateFieldValue; 927 break; 928 case WarcConstants.FN_IDX_WARC_REFERS_TO_DATE: 929 warcRefersToDateStr = fieldValueStr; 930 // FIXME Maybe change parameter type if it does not break too many things. 931 //warcRefersToDate = WarcDate.getWarcDate(dateFieldValue); 932 warcRefersToDate = dateFieldValue; 933 break; 934 /* 935 * InetAddress. 936 */ 937 case WarcConstants.FN_IDX_WARC_IP_ADDRESS: 938 warcIpAddress = fieldValueStr; 939 warcInetAddress = inetAddrFieldValue; 940 break; 941 /* 942 * URI. 943 */ 944 case WarcConstants.FN_IDX_WARC_RECORD_ID: 945 warcRecordIdStr = fieldValueStr; 946 warcRecordIdUri = uriFieldValue; 947 break; 948 case WarcConstants.FN_IDX_WARC_CONCURRENT_TO: 949 if (fieldValueStr != null || uriFieldValue != null) { 950 WarcConcurrentTo warcConcurrentTo = new WarcConcurrentTo(); 951 warcConcurrentTo.warcConcurrentToStr = fieldValueStr; 952 warcConcurrentTo.warcConcurrentToUri = uriFieldValue; 953 warcConcurrentToList.add(warcConcurrentTo); 954 } 955 break; 956 case WarcConstants.FN_IDX_WARC_REFERS_TO: 957 warcRefersToStr = fieldValueStr; 958 warcRefersToUri = uriFieldValue; 959 break; 960 case WarcConstants.FN_IDX_WARC_TARGET_URI: 961 warcTargetUriStr = fieldValueStr; 962 warcTargetUriUri = uriFieldValue; 963 break; 964 case WarcConstants.FN_IDX_WARC_WARCINFO_ID: 965 warcWarcinfoIdStr = fieldValueStr; 966 warcWarcinfoIdUri = uriFieldValue; 967 break; 968 case WarcConstants.FN_IDX_WARC_PROFILE: 969 warcProfileStr = fieldValueStr; 970 warcProfileUri = uriFieldValue; 971 if (warcProfileStr != null) { 972 warcProfileIdx = WarcConstants.profileIdxMap.get(warcProfileStr.toLowerCase()); 973 } 974 if (warcProfileIdx == null && warcProfileStr != null && warcProfileStr.length() > 0) { 975 warcProfileIdx = WarcConstants.WARC_PROFILE_IDX_UNKNOWN; 976 } 977 break; 978 case WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID: 979 warcSegmentOriginIdStr = fieldValueStr; 980 warcSegmentOriginIdUrl = uriFieldValue; 981 break; 982 case WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI: 983 warcRefersToTargetUriStr = fieldValueStr; 984 warcRefersToTargetUriUri = uriFieldValue; 985 break; 986 default: 987 break; 988 } 989 } else { 990 // Invalid datatype for field. 991 diagnostics.addError(DiagnosisType.INVALID_EXPECTED, 992 "Invalid datatype for '" + fieldName + "' header", 993 WarcConstants.FDT_IDX_STRINGS[WarcConstants.FN_IDX_DT[fn_idx]], 994 WarcConstants.FDT_IDX_STRINGS[dt]); 995 // Consider throwing exception at some point. 996 } 997 } 998 HeaderLine headerLine = new HeaderLine(); 999 headerLine.name = fieldName; 1000 headerLine.value = fieldValueStr; 1001 HeaderLine tmpLine = headerMap.get(fieldName.toLowerCase()); 1002 if (tmpLine == null) { 1003 headerMap.put(fieldName.toLowerCase(), headerLine); 1004 } else { 1005 tmpLine.lines.add(headerLine); 1006 } 1007 return headerLine; 1008 } 1009 1010}