001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.io.ByteArrayOutputStream; 021import java.io.Closeable; 022import java.io.IOException; 023import java.io.InputStream; 024import java.io.OutputStream; 025 026import org.jwat.common.Diagnosis; 027import org.jwat.common.DiagnosisType; 028import org.jwat.common.Diagnostics; 029import org.jwat.common.UriProfile; 030 031/** 032 * Base class for WARC writer implementations. 033 * 034 * @author nicl 035 */ 036public abstract class WarcWriter implements Closeable { 037 038 /** State after writer has been constructed and before records have been written. */ 039 protected static final int S_INIT = 0; 040 041 /** State after header has been written. */ 042 protected static final int S_HEADER_WRITTEN = 1; 043 044 /** State after payload has been written. */ 045 protected static final int S_PAYLOAD_WRITTEN = 2; 046 047 /** State after record has been closed. */ 048 protected static final int S_RECORD_CLOSED = 3; 049 050 /* 051 * Settings. 052 */ 053 054 /** WARC-Target-URI profile. */ 055 protected UriProfile warcTargetUriProfile; 056 057 /** URI profile. */ 058 protected UriProfile uriProfile; 059 060 /** Block Digesting enabled/disabled. */ 061 //protected boolean bDigestBlock = false; 062 063 /** WARC field parser used. */ 064 protected WarcFieldParsers fieldParsers; 065 066 /** Buffer used by streamPayload() to copy from one stream to another. */ 067 protected byte[] stream_copy_buffer; 068 069 /** Configuration for throwing exception on content-length mismatch. 070 * (Default is true) */ 071 protected boolean bExceptionOnContentLengthMismatch; 072 073 /* 074 * State. 075 */ 076 077 /** Writer level errors and warnings or when writing byte headers. */ 078 public final Diagnostics diagnostics = new Diagnostics(); 079 080 /** Current state of writer. */ 081 protected int state = S_INIT; 082 083 /** Outputstream used to write WARC records. */ 084 protected OutputStream out; 085 086 /** Current WARC header written. */ 087 protected WarcHeader header; 088 089 /** Content-Length from the WARC header. */ 090 protected Long headerContentLength; 091 092 /** Total bytes written for current record payload. */ 093 protected long payloadWrittenTotal; 094 095 /** 096 * Method used to initialize a readers internal state. 097 * Must be called by all constructors. 098 */ 099 protected void init() { 100 warcTargetUriProfile = UriProfile.RFC3986; 101 uriProfile = UriProfile.RFC3986; 102 fieldParsers = new WarcFieldParsers(); 103 stream_copy_buffer = new byte[8192]; 104 bExceptionOnContentLengthMismatch = true; 105 } 106 107 /** 108 * Is this writer producing compressed output. 109 * @return boolean indicating whether compressed output is produced 110 */ 111 public abstract boolean isCompressed(); 112 113 /** 114 * Set the URI profile used to validate WARC-Target URIs. 115 * If null, the uriProfile is set to RCF3986. 116 * @param uriProfile URI profile to use 117 */ 118 public void setWarcTargetUriProfile(UriProfile uriProfile) { 119 if (uriProfile == null) { 120 uriProfile = UriProfile.RFC3986; 121 } 122 this.warcTargetUriProfile = uriProfile; 123 } 124 125 /** 126 * Get the URI profile used to validate WARC-Target URIs. 127 * @return the URI profile used to validate WARC-Target URIs 128 */ 129 public UriProfile getWarcTargetUriProfile() { 130 return warcTargetUriProfile; 131 } 132 133 /** 134 * Set the URI profile used to validate URIs. 135 * If null, the uriProfile is set to RCF3986. 136 * @param uriProfile URI profile to use 137 */ 138 public void setUriProfile(UriProfile uriProfile) { 139 if (uriProfile == null) { 140 uriProfile = UriProfile.RFC3986; 141 } 142 this.uriProfile = uriProfile; 143 } 144 145 /** 146 * Get the URI profile used to validate URIs. 147 * @return the URI profile used to validate URIs 148 */ 149 public UriProfile getUriProfile() { 150 return uriProfile; 151 } 152 153 /** 154 * Does this writer throw an exception if the content-length does not match 155 * the payload amount written. 156 * @return boolean indicating if an exception is thrown or not 157 */ 158 public boolean exceptionOnContentLengthMismatch() { 159 return bExceptionOnContentLengthMismatch; 160 } 161 162 /** 163 * Tell the writer what to do in case of mismatch between content-length 164 * and amount payload written. 165 * @param enabled boolean indicating exception throwing on/off 166 */ 167 public void setExceptionOnContentLengthMismatch(boolean enabled) { 168 bExceptionOnContentLengthMismatch = enabled; 169 } 170 171 /** 172 * Is this writer set to block digest payload. 173 * @return boolean indicating payload block digesting 174 */ 175 /* 176 public boolean digestBlock() { 177 return bDigestBlock; 178 } 179 */ 180 181 /** 182 * Set the writers payload block digest mode 183 * @param enabled boolean indicating digest on/off 184 */ 185 /* 186 public void setDigestBlock(boolean enabled) { 187 bDigestBlock = enabled; 188 } 189 */ 190 191 /** 192 * Close WARC writer and free its resources. 193 * @throws IOException if an I/O exception occurs while closing the writer 194 */ 195 public abstract void close() throws IOException; 196 197 /** 198 * Close the WARC record in an implementation specific way. 199 * @throws IOException if an I/O exception occurs while closing the record 200 */ 201 public abstract void closeRecord() throws IOException; 202 203 /** 204 * Closes the WARC record by writing two newlines and comparing the amount of 205 * payload data streamed with the content-length supplied with the header. 206 * @throws IOException if an I/O exception occurs while closing the record 207 */ 208 protected void closeRecord_impl() throws IOException { 209 Diagnosis diagnosis = null; 210 out.write(WarcConstants.endMark); 211 out.flush(); 212 if (headerContentLength == null) { 213 diagnosis = new Diagnosis( 214 DiagnosisType.ERROR_EXPECTED, 215 "'" + WarcConstants.FN_CONTENT_LENGTH + "' header", 216 "Mandatory!"); 217 } else { 218 if (headerContentLength != payloadWrittenTotal) { 219 diagnosis = new Diagnosis( 220 DiagnosisType.INVALID_EXPECTED, 221 "'" + WarcConstants.FN_CONTENT_LENGTH + "' header", 222 Long.toString(payloadWrittenTotal), 223 headerContentLength.toString()); 224 } 225 } 226 if (diagnosis != null) { 227 if (header != null) { 228 header.diagnostics.addError(diagnosis); 229 } else { 230 diagnostics.addError(diagnosis); 231 } 232 if (bExceptionOnContentLengthMismatch) { 233 throw new IllegalStateException("Payload size does not match content-length!"); 234 } 235 } 236 header = null; 237 headerContentLength = null; 238 } 239 240 /** 241 * Write a raw WARC header to the WARC output stream. Closes any previously 242 * written record that has not been closed prior to this call. 243 * Errors and warnings are reported on the writers diagnostics object. 244 * @param header_bytes raw WARC header to output 245 * @param contentLength the expected content-length to be written and validated 246 * @throws IOException if an I/O exception occurs while writing header data 247 */ 248 public void writeRawHeader(byte[] header_bytes, Long contentLength) throws IOException { 249 if (header_bytes == null) { 250 throw new IllegalArgumentException( 251 "The 'header_bytes' parameter is null!"); 252 } 253 if (contentLength != null && contentLength < 0) { 254 throw new IllegalArgumentException( 255 "The 'contentLength' parameter is negative!"); 256 } 257 if (state == S_HEADER_WRITTEN) { 258 throw new IllegalStateException("Headers written back to back!"); 259 } else if (state == S_PAYLOAD_WRITTEN) { 260 closeRecord_impl(); 261 } 262 out.write(header_bytes); 263 state = S_HEADER_WRITTEN; 264 header = null; 265 headerContentLength = contentLength; 266 payloadWrittenTotal = 0; 267 } 268 269 /** 270 * Write a WARC header to the WARC output stream. 271 * Errors and warnings are reported on the records diagnostics object. 272 * @param record WARC record to output 273 * @return byte array version of header as it was written 274 * @throws IOException if an I/O exception occurs while writing header data 275 */ 276 public abstract byte[] writeHeader(WarcRecord record) throws IOException; 277 278 /** 279 * Write a WARC header to the WARC output stream. 280 * The WARC header is not required to be valid. 281 * Errors and warnings are reported on the records diagnostics object. 282 * @param record WARC record to output 283 * @return byte array version of header as it was written 284 * @throws IOException if an I/O exception occurs while writing header data 285 */ 286 protected byte[] writeHeader_impl(WarcRecord record) throws IOException { 287 header = record.header; 288 headerContentLength = header.contentLength; 289 if (headerContentLength == null && header.contentLengthStr != null) { 290 try { 291 headerContentLength = Long.parseLong(header.contentLengthStr); 292 } catch (NumberFormatException e) { 293 // TODO Add warning... 294 } 295 } 296 ByteArrayOutputStream outBuf = new ByteArrayOutputStream(); 297 /* 298 * Version Line 299 */ 300 byte[] magicVersion = (WarcConstants.WARC_MAGIC_HEADER + header.major + "." + header.minor + "\r\n").getBytes(); 301 outBuf.write(magicVersion); 302 /* 303 * Warc-Type 304 */ 305 String warcTypeStr = null; 306 if (header.warcTypeIdx != null) { 307 if (header.warcTypeIdx > 0 308 && header.warcTypeIdx < WarcConstants.RT_IDX_STRINGS.length) { 309 warcTypeStr = WarcConstants.RT_IDX_STRINGS[header.warcTypeIdx]; 310 } else { 311 // Warning... 312 } 313 } 314 if (warcTypeStr == null) { 315 warcTypeStr = header.warcTypeStr; 316 } 317 if (warcTypeStr != null) { 318 outBuf.write(WarcConstants.FN_WARC_TYPE.getBytes()); 319 outBuf.write(": ".getBytes()); 320 outBuf.write(warcTypeStr.getBytes()); 321 outBuf.write("\r\n".getBytes()); 322 } 323 /* 324 * Warc-Record-Id 325 */ 326 String warcRecordIdStr = null; 327 if (header.warcRecordIdUri != null) { 328 warcRecordIdStr = header.warcRecordIdUri.toString(); 329 } else if (header.warcRecordIdStr != null) { 330 warcRecordIdStr = header.warcRecordIdStr; 331 // Warning... 332 } 333 if (warcRecordIdStr != null) { 334 outBuf.write(WarcConstants.FN_WARC_RECORD_ID.getBytes()); 335 outBuf.write(": <".getBytes()); 336 outBuf.write(warcRecordIdStr.getBytes()); 337 outBuf.write(">\r\n".getBytes()); 338 } 339 /* 340 * Warc-Date 341 */ 342 String warcDateStr = null; 343 if (header.warcDate != null) { 344 // FIXME Check.. 345 //warcDateStr = WarcDateParser.getDateFormat().format(header.warcDate); 346 if (header.major == 1 && header.minor == 0 && header.warcDate.precision != WarcDate.P_SECOND) { 347 header.warcDate.adjustPrecisionTo(WarcDate.P_SECOND); 348 } 349 warcDateStr = header.warcDate.toString(); 350 } else if (header.warcDateStr != null) { 351 warcDateStr = header.warcDateStr; 352 // Warning... 353 } 354 if (warcDateStr != null) { 355 outBuf.write(WarcConstants.FN_WARC_DATE.getBytes()); 356 outBuf.write(": ".getBytes()); 357 outBuf.write(warcDateStr.getBytes()); 358 outBuf.write("\r\n".getBytes()); 359 } 360 /* 361 * Content-Length 362 */ 363 String contentLengthStr = null; 364 if (header.contentLength != null) { 365 contentLengthStr = header.contentLength.toString(); 366 } else if (header.contentLengthStr != null) { 367 contentLengthStr = header.contentLengthStr; 368 // Warning... 369 } 370 if (contentLengthStr != null) { 371 outBuf.write(WarcConstants.FN_CONTENT_LENGTH.getBytes()); 372 outBuf.write(": ".getBytes()); 373 outBuf.write(contentLengthStr.getBytes()); 374 outBuf.write("\r\n".getBytes()); 375 } 376 /* 377 * Content-Type 378 */ 379 String contentTypeStr = null; 380 if (header.contentType != null) { 381 contentTypeStr = header.contentType.toString(); 382 } else if (header.contentTypeStr != null) { 383 contentTypeStr = header.contentTypeStr; 384 // Warning... 385 } 386 if (contentTypeStr != null) { 387 outBuf.write(WarcConstants.FN_CONTENT_TYPE.getBytes()); 388 outBuf.write(": ".getBytes()); 389 outBuf.write(contentTypeStr.getBytes()); 390 outBuf.write("\r\n".getBytes()); 391 } 392 /* 393 * Warc-Concurrent-To 394 */ 395 WarcConcurrentTo warcConcurrentTo; 396 String warcConcurrentToStr; 397 if (header.warcConcurrentToList != null) { 398 for (int i=0; i<header.warcConcurrentToList.size(); ++i) { 399 warcConcurrentTo = header.warcConcurrentToList.get(i); 400 warcConcurrentToStr = null; 401 if (warcConcurrentTo.warcConcurrentToUri != null) { 402 warcConcurrentToStr = warcConcurrentTo.warcConcurrentToUri.toString(); 403 } else if (warcConcurrentTo.warcConcurrentToStr != null) { 404 warcConcurrentToStr = warcConcurrentTo.warcConcurrentToStr; 405 // Warning... 406 } 407 if (warcConcurrentToStr != null) { 408 outBuf.write(WarcConstants.FN_WARC_CONCURRENT_TO.getBytes()); 409 outBuf.write(": <".getBytes()); 410 outBuf.write(warcConcurrentToStr.getBytes()); 411 outBuf.write(">\r\n".getBytes()); 412 } 413 } 414 } 415 /* 416 * Warc-Block-Digest 417 */ 418 String warcBlockDigestStr = null; 419 if (header.warcBlockDigest != null) { 420 warcBlockDigestStr = header.warcBlockDigest.toString(); 421 } else if (header.warcBlockDigestStr != null) { 422 warcBlockDigestStr = header.warcBlockDigestStr; 423 // Warning... 424 } 425 if (warcBlockDigestStr != null) { 426 outBuf.write(WarcConstants.FN_WARC_BLOCK_DIGEST.getBytes()); 427 outBuf.write(": ".getBytes()); 428 outBuf.write(warcBlockDigestStr.getBytes()); 429 outBuf.write("\r\n".getBytes()); 430 } 431 /* 432 * Warc-Payload-Digest 433 */ 434 String warcPayloadDigestStr = null; 435 if (header.warcPayloadDigest != null) { 436 warcPayloadDigestStr = header.warcPayloadDigest.toString(); 437 } else if (header.warcPayloadDigestStr != null) { 438 warcPayloadDigestStr = header.warcPayloadDigestStr; 439 // Warning... 440 } 441 if (warcPayloadDigestStr != null) { 442 outBuf.write(WarcConstants.FN_WARC_PAYLOAD_DIGEST.getBytes()); 443 outBuf.write(": ".getBytes()); 444 outBuf.write(warcPayloadDigestStr.getBytes()); 445 outBuf.write("\r\n".getBytes()); 446 } 447 /* 448 * Warc-Ip-Address 449 */ 450 String warcIpAddress = null; 451 if (header.warcInetAddress != null) { 452 warcIpAddress = header.warcInetAddress.getHostAddress(); 453 } else if (header.warcIpAddress != null) { 454 warcIpAddress = header.warcIpAddress; 455 // Warning... 456 } 457 if (warcIpAddress != null) { 458 outBuf.write(WarcConstants.FN_WARC_IP_ADDRESS.getBytes()); 459 outBuf.write(": ".getBytes()); 460 outBuf.write(warcIpAddress.getBytes()); 461 outBuf.write("\r\n".getBytes()); 462 } 463 /* 464 * Warc-Refers-To 465 */ 466 String warcRefersToUriStr = null; 467 if (header.warcRefersToUri != null) { 468 warcRefersToUriStr = header.warcRefersToUri.toString(); 469 } else if (header.warcRefersToStr != null) { 470 warcRefersToUriStr = header.warcRefersToStr; 471 // Warning... 472 } 473 if (warcRefersToUriStr != null) { 474 outBuf.write(WarcConstants.FN_WARC_REFERS_TO.getBytes()); 475 outBuf.write(": <".getBytes()); 476 outBuf.write(warcRefersToUriStr.getBytes()); 477 outBuf.write(">\r\n".getBytes()); 478 } 479 /* 480 * Warc-Target-Uri 481 */ 482 String warcTargetUriStr = null; 483 if (header.warcTargetUriUri != null) { 484 warcTargetUriStr = header.warcTargetUriUri.toString(); 485 } else if (header.warcTargetUriStr != null) { 486 warcTargetUriStr = header.warcTargetUriStr; 487 // Warning... 488 } 489 if (warcTargetUriStr != null) { 490 outBuf.write(WarcConstants.FN_WARC_TARGET_URI.getBytes()); 491 outBuf.write(": ".getBytes()); 492 outBuf.write(warcTargetUriStr.getBytes()); 493 outBuf.write("\r\n".getBytes()); 494 } 495 /* 496 * Warc-Truncated 497 */ 498 String warcTruncatedStr = null; 499 if (header.warcTruncatedIdx != null) { 500 if (header.warcTruncatedIdx > 0 501 && header.warcTruncatedIdx < WarcConstants.TT_IDX_STRINGS.length) { 502 warcTruncatedStr = WarcConstants.TT_IDX_STRINGS[header.warcTruncatedIdx]; 503 } else { 504 // Warning... 505 } 506 } 507 if (warcTruncatedStr == null) { 508 warcTruncatedStr = header.warcTruncatedStr; 509 } 510 if (warcTruncatedStr != null) { 511 outBuf.write(WarcConstants.FN_WARC_TRUNCATED.getBytes()); 512 outBuf.write(": ".getBytes()); 513 outBuf.write(warcTruncatedStr.getBytes()); 514 outBuf.write("\r\n".getBytes()); 515 } 516 /* 517 * Warc-Warcinfo-Id 518 */ 519 String warcWarcInfoIdStr = null; 520 if (header.warcWarcinfoIdUri != null) { 521 warcWarcInfoIdStr = header.warcWarcinfoIdUri.toString(); 522 } else if (header.warcWarcinfoIdStr != null) { 523 warcWarcInfoIdStr = header.warcWarcinfoIdStr; 524 // Warning... 525 } 526 if (warcWarcInfoIdStr != null) { 527 outBuf.write(WarcConstants.FN_WARC_WARCINFO_ID.getBytes()); 528 outBuf.write(": <".getBytes()); 529 outBuf.write(warcWarcInfoIdStr.getBytes()); 530 outBuf.write(">\r\n".getBytes()); 531 } 532 /* 533 * Warc-Filename 534 */ 535 if (header.warcFilename != null) { 536 outBuf.write(WarcConstants.FN_WARC_FILENAME.getBytes()); 537 outBuf.write(": ".getBytes()); 538 outBuf.write(header.warcFilename.getBytes()); 539 outBuf.write("\r\n".getBytes()); 540 } 541 /* 542 * Warc-Profile 543 */ 544 String warcProfileStr = null; 545 if (header.warcProfileUri != null) { 546 warcProfileStr = header.warcProfileUri.toString(); 547 } else if (header.warcProfileIdx != null) { 548 if (header.warcProfileIdx > 0 549 && header.warcProfileIdx < WarcConstants.P_IDX_STRINGS.length) { 550 warcProfileStr = WarcConstants.P_IDX_STRINGS[header.warcProfileIdx]; 551 } else { 552 // Warning... 553 } 554 } 555 if (warcProfileStr == null) { 556 warcProfileStr = header.warcProfileStr; 557 // Warning... 558 } 559 if (warcProfileStr != null) { 560 outBuf.write(WarcConstants.FN_WARC_PROFILE.getBytes()); 561 outBuf.write(": ".getBytes()); 562 outBuf.write(warcProfileStr.getBytes()); 563 outBuf.write("\r\n".getBytes()); 564 } 565 /* 566 * Warc-Identified-Payload-Type 567 */ 568 String warcIdentifiedPayloadTypeStr = null; 569 if (header.warcIdentifiedPayloadType != null) { 570 warcIdentifiedPayloadTypeStr = header.warcIdentifiedPayloadType.toString(); 571 } else if (header.warcIdentifiedPayloadTypeStr != null) { 572 warcIdentifiedPayloadTypeStr = header.warcIdentifiedPayloadTypeStr; 573 // Warning... 574 } 575 if (warcIdentifiedPayloadTypeStr != null) { 576 outBuf.write(WarcConstants.FN_WARC_IDENTIFIED_PAYLOAD_TYPE.getBytes()); 577 outBuf.write(": ".getBytes()); 578 outBuf.write(warcIdentifiedPayloadTypeStr.getBytes()); 579 outBuf.write("\r\n".getBytes()); 580 } 581 /* 582 * Warc-Segment-Number 583 */ 584 String warcSegmentNumberStr = null; 585 if (header.warcSegmentNumber != null) { 586 warcSegmentNumberStr = header.warcSegmentNumber.toString(); 587 } else if (header.warcSegmentNumberStr != null) { 588 warcSegmentNumberStr = header.warcSegmentNumberStr; 589 // Warning... 590 } 591 if (warcSegmentNumberStr != null) { 592 outBuf.write(WarcConstants.FN_WARC_SEGMENT_NUMBER.getBytes()); 593 outBuf.write(": ".getBytes()); 594 outBuf.write(warcSegmentNumberStr.getBytes()); 595 outBuf.write("\r\n".getBytes()); 596 } 597 /* 598 * Warc-Segment-Origin-Id 599 */ 600 String warcSegmentOriginIdStr = null; 601 if (header.warcSegmentOriginIdUrl != null) { 602 warcSegmentOriginIdStr = header.warcSegmentOriginIdUrl.toString(); 603 } else if (header.warcSegmentOriginIdStr != null) { 604 warcSegmentOriginIdStr = header.warcSegmentOriginIdStr; 605 // Warning... 606 } 607 if (warcSegmentOriginIdStr != null) { 608 outBuf.write(WarcConstants.FN_WARC_SEGMENT_ORIGIN_ID.getBytes()); 609 outBuf.write(": <".getBytes()); 610 outBuf.write(warcSegmentOriginIdStr.getBytes()); 611 outBuf.write(">\r\n".getBytes()); 612 } 613 /* 614 * Warc-Segment-Total-Length 615 */ 616 String warcSegmentTotalLengthStr = null; 617 if (header.warcSegmentTotalLength != null) { 618 warcSegmentTotalLengthStr = header.warcSegmentTotalLength.toString(); 619 } else if (header.warcSegmentTotalLengthStr != null) { 620 warcSegmentTotalLengthStr = header.warcSegmentTotalLengthStr; 621 // Warning... 622 } 623 if (warcSegmentTotalLengthStr != null) { 624 outBuf.write(WarcConstants.FN_WARC_SEGMENT_TOTAL_LENGTH.getBytes()); 625 outBuf.write(": ".getBytes()); 626 outBuf.write(warcSegmentTotalLengthStr.getBytes()); 627 outBuf.write("\r\n".getBytes()); 628 } 629 /* 630 * WARC-Refers-To-Target-URI 631 */ 632 String warcRefersToTargetUriStr = null; 633 if (header.warcRefersToTargetUriUri != null) { 634 warcRefersToTargetUriStr = header.warcRefersToTargetUriUri.toString(); 635 } else if (header.warcRefersToTargetUriStr != null) { 636 warcRefersToTargetUriStr = header.warcRefersToTargetUriStr; 637 } 638 if (warcRefersToTargetUriStr != null) { 639 outBuf.write(WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.getBytes()); 640 outBuf.write(": ".getBytes()); 641 outBuf.write(warcRefersToTargetUriStr.getBytes()); 642 outBuf.write("\r\n".getBytes()); 643 } 644 /* 645 * Warc-Refers-To-Date 646 */ 647 String warcRefersToDateStr = null; 648 if (header.warcRefersToDate != null) { 649 //warcRefersToDateStr = WarcDateParser.getDateFormat().format(header.warcRefersToDate); 650 // FIXME Check.. 651 if (header.major == 1 && header.minor == 0 && header.warcRefersToDate.precision != WarcDate.P_SECOND) { 652 header.warcRefersToDate.adjustPrecisionTo(WarcDate.P_SECOND); 653 } 654 warcRefersToDateStr = header.warcRefersToDate.toString(); 655 } else if (header.warcRefersToDateStr != null) { 656 warcRefersToDateStr = header.warcRefersToDateStr; 657 // Warning... 658 } 659 if (warcRefersToDateStr != null) { 660 outBuf.write(WarcConstants.FN_WARC_REFERS_TO_DATE.getBytes()); 661 outBuf.write(": ".getBytes()); 662 outBuf.write(warcRefersToDateStr.getBytes()); 663 outBuf.write("\r\n".getBytes()); 664 } 665 /* 666 * End Of Header 667 */ 668 outBuf.write("\r\n".getBytes()); 669 byte[] headerBytes = outBuf.toByteArray(); 670 out.write(headerBytes); 671 state = S_HEADER_WRITTEN; 672 payloadWrittenTotal = 0; 673 return headerBytes; 674 } 675 676 /** 677 * Stream the content of an input stream to the payload content. 678 * @param in input stream containing payload data 679 * @return number of bytes written during method invocation 680 * @throws IOException if an I/O exception occurs while writing payload data 681 */ 682 public long streamPayload(InputStream in) throws IOException { 683 if (in == null) { 684 throw new IllegalArgumentException( 685 "The 'in' parameter is null!"); 686 } 687 if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) { 688 throw new IllegalStateException("Write a header before writing payload!"); 689 } 690 long written = 0; 691 int read = 0; 692 while (read != -1) { 693 out.write(stream_copy_buffer, 0, read); 694 written += read; 695 read = in.read(stream_copy_buffer); 696 } 697 state = S_PAYLOAD_WRITTEN; 698 payloadWrittenTotal += written; 699 return written; 700 } 701 702 /** 703 * Append the content of a byte array to the payload content. 704 * @param b byte array with data to be written 705 * @return number of bytes written during method invocation 706 * @throws IOException if an I/O exception occurs while writing payload data 707 */ 708 public long writePayload(byte[] b) throws IOException { 709 if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) { 710 throw new IllegalStateException("Write a header before writing payload!"); 711 } 712 out.write(b); 713 state = S_PAYLOAD_WRITTEN; 714 payloadWrittenTotal += b.length; 715 return b.length; 716 } 717 718 /** 719 * Append the partial content of a byte array to the payload content. 720 * @param b byte array with partial data to be written 721 * @param offset offset to data to be written 722 * @param len length of data to be written 723 * @return number of bytes written during method invocation 724 * @throws IOException if an I/O exception occurs while writing payload data 725 */ 726 public long writePayload(byte[] b, int offset, int len) throws IOException { 727 if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) { 728 throw new IllegalStateException("Write a header before writing payload!"); 729 } 730 out.write(b, offset, len); 731 state = S_PAYLOAD_WRITTEN; 732 payloadWrittenTotal += len; 733 return len; 734 } 735 736}