001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.io.ByteArrayOutputStream; 021import java.io.Closeable; 022import java.io.IOException; 023import java.io.InputStream; 024import java.io.OutputStream; 025 026import org.jwat.common.Diagnosis; 027import org.jwat.common.DiagnosisType; 028import org.jwat.common.Diagnostics; 029import org.jwat.common.UriProfile; 030 031/** 032 * Base class for WARC writer implementations. 033 * 034 * @author nicl 035 */ 036public abstract class WarcWriter implements Closeable { 037 038 /** State after writer has been constructed and before records have been written. */ 039 protected static final int S_INIT = 0; 040 041 /** State after header has been written. */ 042 protected static final int S_HEADER_WRITTEN = 1; 043 044 /** State after payload has been written. */ 045 protected static final int S_PAYLOAD_WRITTEN = 2; 046 047 /** State after record has been closed. */ 048 protected static final int S_RECORD_CLOSED = 3; 049 050 /* 051 * Settings. 052 */ 053 054 /** WARC-Target-URI profile. */ 055 protected UriProfile warcTargetUriProfile; 056 057 /** URI profile. */ 058 protected UriProfile uriProfile; 059 060 /** Block Digesting enabled/disabled. */ 061 //protected boolean bDigestBlock = false; 062 063 /** WARC field parser used. */ 064 protected WarcFieldParsers fieldParsers; 065 066 /** Buffer used by streamPayload() to copy from one stream to another. */ 067 protected byte[] stream_copy_buffer; 068 069 /** Configuration for throwing exception on content-length mismatch. 070 * (Default is true) */ 071 protected boolean bExceptionOnContentLengthMismatch; 072 073 /* 074 * State. 075 */ 076 077 /** Writer level errors and warnings or when writing byte headers. */ 078 public final Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>(); 079 080 /** Current state of writer. */ 081 protected int state = S_INIT; 082 083 /** Outputstream used to write WARC records. */ 084 protected OutputStream out; 085 086 /** Current WARC header written. */ 087 protected WarcHeader header; 088 089 /** Content-Length from the WARC header. */ 090 protected Long headerContentLength; 091 092 /** Total bytes written for current record payload. */ 093 protected long payloadWrittenTotal; 094 095 /** 096 * Method used to initialize a readers internal state. 097 * Must be called by all constructors. 098 */ 099 protected void init() { 100 warcTargetUriProfile = UriProfile.RFC3986; 101 uriProfile = UriProfile.RFC3986; 102 fieldParsers = new WarcFieldParsers(); 103 stream_copy_buffer = new byte[8192]; 104 bExceptionOnContentLengthMismatch = true; 105 } 106 107 /** 108 * Is this writer producing compressed output. 109 * @return boolean indicating whether compressed output is produced 110 */ 111 public abstract boolean isCompressed(); 112 113 /** 114 * Set the URI profile used to validate WARC-Target URIs. 115 * If null, the uriProfile is set to RCF3986. 116 * @param uriProfile URI profile to use 117 */ 118 public void setWarcTargetUriProfile(UriProfile uriProfile) { 119 if (uriProfile == null) { 120 uriProfile = UriProfile.RFC3986; 121 } 122 this.warcTargetUriProfile = uriProfile; 123 } 124 125 /** 126 * Get the URI profile used to validate WARC-Target URIs. 127 * @return the URI profile used to validate WARC-Target URIs 128 */ 129 public UriProfile getWarcTargetUriProfile() { 130 return warcTargetUriProfile; 131 } 132 133 /** 134 * Set the URI profile used to validate URIs. 135 * If null, the uriProfile is set to RCF3986. 136 * @param uriProfile URI profile to use 137 */ 138 public void setUriProfile(UriProfile uriProfile) { 139 if (uriProfile == null) { 140 uriProfile = UriProfile.RFC3986; 141 } 142 this.uriProfile = uriProfile; 143 } 144 145 /** 146 * Get the URI profile used to validate URIs. 147 * @return the URI profile used to validate URIs 148 */ 149 public UriProfile getUriProfile() { 150 return uriProfile; 151 } 152 153 /** 154 * Does this writer throw an exception if the content-length does not match 155 * the payload amount written. 156 * @return boolean indicating if an exception is thrown or not 157 */ 158 public boolean exceptionOnContentLengthMismatch() { 159 return bExceptionOnContentLengthMismatch; 160 } 161 162 /** 163 * Tell the writer what to do in case of mismatch between content-length 164 * and amount payload written. 165 * @param enabled boolean indicating exception throwing on/off 166 */ 167 public void setExceptionOnContentLengthMismatch(boolean enabled) { 168 bExceptionOnContentLengthMismatch = enabled; 169 } 170 171 /** 172 * Is this writer set to block digest payload. 173 * @return boolean indicating payload block digesting 174 */ 175 /* 176 public boolean digestBlock() { 177 return bDigestBlock; 178 } 179 */ 180 181 /** 182 * Set the writers payload block digest mode 183 * @param enabled boolean indicating digest on/off 184 */ 185 /* 186 public void setDigestBlock(boolean enabled) { 187 bDigestBlock = enabled; 188 } 189 */ 190 191 /** 192 * Close WARC writer and free its resources. 193 * @throws IOException if an I/O exception occurs while closing the writer 194 */ 195 public abstract void close() throws IOException; 196 197 /** 198 * Close the WARC record in an implementation specific way. 199 * @throws IOException if an I/O exception occurs while closing the record 200 */ 201 public abstract void closeRecord() throws IOException; 202 203 /** 204 * Closes the WARC record by writing two newlines and comparing the amount of 205 * payload data streamed with the content-length supplied with the header. 206 * @throws IOException if an I/O exception occurs while closing the record 207 */ 208 protected void closeRecord_impl() throws IOException { 209 Diagnosis diagnosis = null; 210 out.write(WarcConstants.endMark); 211 out.flush(); 212 if (headerContentLength == null) { 213 diagnosis = new Diagnosis( 214 DiagnosisType.ERROR_EXPECTED, 215 "'" + WarcConstants.FN_CONTENT_LENGTH + "' header", 216 "Mandatory!"); 217 } else { 218 if (headerContentLength != payloadWrittenTotal) { 219 diagnosis = new Diagnosis( 220 DiagnosisType.INVALID_EXPECTED, 221 "'" + WarcConstants.FN_CONTENT_LENGTH + "' header", 222 Long.toString(payloadWrittenTotal), 223 headerContentLength.toString()); 224 } 225 } 226 if (diagnosis != null) { 227 if (header != null) { 228 header.diagnostics.addError(diagnosis); 229 } else { 230 diagnostics.addError(diagnosis); 231 } 232 if (bExceptionOnContentLengthMismatch) { 233 throw new IllegalStateException("Payload size does not match content-length!"); 234 } 235 } 236 header = null; 237 headerContentLength = null; 238 } 239 240 /** 241 * Write a raw WARC header to the WARC output stream. Closes any previously 242 * written record that has not been closed prior to this call. 243 * Errors and warnings are reported on the writers diagnostics object. 244 * @param header_bytes raw WARC header to output 245 * @param contentLength the expected content-length to be written and validated 246 * @throws IOException if an I/O exception occurs while writing header data 247 */ 248 public void writeRawHeader(byte[] header_bytes, Long contentLength) throws IOException { 249 if (header_bytes == null) { 250 throw new IllegalArgumentException( 251 "The 'header_bytes' parameter is null!"); 252 } 253 if (contentLength != null && contentLength < 0) { 254 throw new IllegalArgumentException( 255 "The 'contentLength' parameter is negative!"); 256 } 257 if (state == S_HEADER_WRITTEN) { 258 throw new IllegalStateException("Headers written back to back!"); 259 } else if (state == S_PAYLOAD_WRITTEN) { 260 closeRecord_impl(); 261 } 262 out.write(header_bytes); 263 state = S_HEADER_WRITTEN; 264 header = null; 265 headerContentLength = contentLength; 266 payloadWrittenTotal = 0; 267 } 268 269 /** 270 * Write a WARC header to the WARC output stream. 271 * Errors and warnings are reported on the records diagnostics object. 272 * @param record WARC record to output 273 * @return byte array version of header as it was written 274 * @throws IOException if an I/O exception occurs while writing header data 275 */ 276 public abstract byte[] writeHeader(WarcRecord record) throws IOException; 277 278 /** 279 * Write a WARC header to the WARC output stream. 280 * The WARC header is not required to be valid. 281 * Errors and warnings are reported on the records diagnostics object. 282 * @param record WARC record to output 283 * @return byte array version of header as it was written 284 * @throws IOException if an I/O exception occurs while writing header data 285 */ 286 protected byte[] writeHeader_impl(WarcRecord record) throws IOException { 287 header = record.header; 288 headerContentLength = header.contentLength; 289 if (headerContentLength == null && header.contentLengthStr != null) { 290 try { 291 headerContentLength = Long.parseLong(header.contentLengthStr); 292 } catch (NumberFormatException e) { 293 // TODO Add warning... 294 } 295 } 296 ByteArrayOutputStream outBuf = new ByteArrayOutputStream(); 297 /* 298 * Version Line 299 */ 300 byte[] magicVersion = (WarcConstants.WARC_MAGIC_HEADER + header.major + "." + header.minor + "\r\n").getBytes(); 301 outBuf.write(magicVersion); 302 /* 303 * Warc-Type 304 */ 305 String warcTypeStr = null; 306 if (header.warcTypeIdx != null) { 307 if (header.warcTypeIdx > 0 308 && header.warcTypeIdx < WarcConstants.RT_IDX_STRINGS.length) { 309 warcTypeStr = WarcConstants.RT_IDX_STRINGS[header.warcTypeIdx]; 310 } else { 311 // Warning... 312 } 313 } 314 if (warcTypeStr == null) { 315 warcTypeStr = header.warcTypeStr; 316 } 317 if (warcTypeStr != null) { 318 outBuf.write(WarcConstants.FN_WARC_TYPE.getBytes()); 319 outBuf.write(": ".getBytes()); 320 outBuf.write(warcTypeStr.getBytes()); 321 outBuf.write("\r\n".getBytes()); 322 } 323 /* 324 * Warc-Record-Id 325 */ 326 String warcRecordIdStr = null; 327 if (header.warcRecordIdUri != null) { 328 warcRecordIdStr = header.warcRecordIdUri.toString(); 329 } else if (header.warcRecordIdStr != null) { 330 warcRecordIdStr = header.warcRecordIdStr; 331 // Warning... 332 } 333 if (warcRecordIdStr != null) { 334 outBuf.write(WarcConstants.FN_WARC_RECORD_ID.getBytes()); 335 outBuf.write(": <".getBytes()); 336 outBuf.write(warcRecordIdStr.getBytes()); 337 outBuf.write(">\r\n".getBytes()); 338 } 339 /* 340 * Warc-Date 341 */ 342 String warcDateStr = null; 343 if (header.warcDate != null) { 344 warcDateStr = WarcDateParser.getDateFormat().format(header.warcDate); 345 } else if (header.warcDateStr != null) { 346 warcDateStr = header.warcDateStr; 347 // Warning... 348 } 349 if (warcDateStr != null) { 350 outBuf.write(WarcConstants.FN_WARC_DATE.getBytes()); 351 outBuf.write(": ".getBytes()); 352 outBuf.write(warcDateStr.getBytes()); 353 outBuf.write("\r\n".getBytes()); 354 } 355 /* 356 * Content-Length 357 */ 358 String contentLengthStr = null; 359 if (header.contentLength != null) { 360 contentLengthStr = header.contentLength.toString(); 361 } else if (header.contentLengthStr != null) { 362 contentLengthStr = header.contentLengthStr; 363 // Warning... 364 } 365 if (contentLengthStr != null) { 366 outBuf.write(WarcConstants.FN_CONTENT_LENGTH.getBytes()); 367 outBuf.write(": ".getBytes()); 368 outBuf.write(contentLengthStr.getBytes()); 369 outBuf.write("\r\n".getBytes()); 370 } 371 /* 372 * Content-Type 373 */ 374 String contentTypeStr = null; 375 if (header.contentType != null) { 376 contentTypeStr = header.contentType.toString(); 377 } else if (header.contentTypeStr != null) { 378 contentTypeStr = header.contentTypeStr; 379 // Warning... 380 } 381 if (contentTypeStr != null) { 382 outBuf.write(WarcConstants.FN_CONTENT_TYPE.getBytes()); 383 outBuf.write(": ".getBytes()); 384 outBuf.write(contentTypeStr.getBytes()); 385 outBuf.write("\r\n".getBytes()); 386 } 387 /* 388 * Warc-Concurrent-To 389 */ 390 WarcConcurrentTo warcConcurrentTo; 391 String warcConcurrentToStr; 392 if (header.warcConcurrentToList != null) { 393 for (int i=0; i<header.warcConcurrentToList.size(); ++i) { 394 warcConcurrentTo = header.warcConcurrentToList.get(i); 395 warcConcurrentToStr = null; 396 if (warcConcurrentTo.warcConcurrentToUri != null) { 397 warcConcurrentToStr = warcConcurrentTo.warcConcurrentToUri.toString(); 398 } else if (warcConcurrentTo.warcConcurrentToStr != null) { 399 warcConcurrentToStr = warcConcurrentTo.warcConcurrentToStr; 400 // Warning... 401 } 402 if (warcConcurrentToStr != null) { 403 outBuf.write(WarcConstants.FN_WARC_CONCURRENT_TO.getBytes()); 404 outBuf.write(": <".getBytes()); 405 outBuf.write(warcConcurrentToStr.getBytes()); 406 outBuf.write(">\r\n".getBytes()); 407 } 408 } 409 } 410 /* 411 * Warc-Block-Digest 412 */ 413 String warcBlockDigestStr = null; 414 if (header.warcBlockDigest != null) { 415 warcBlockDigestStr = header.warcBlockDigest.toString(); 416 } else if (header.warcBlockDigestStr != null) { 417 warcBlockDigestStr = header.warcBlockDigestStr; 418 // Warning... 419 } 420 if (warcBlockDigestStr != null) { 421 outBuf.write(WarcConstants.FN_WARC_BLOCK_DIGEST.getBytes()); 422 outBuf.write(": ".getBytes()); 423 outBuf.write(warcBlockDigestStr.getBytes()); 424 outBuf.write("\r\n".getBytes()); 425 } 426 /* 427 * Warc-Payload-Digest 428 */ 429 String warcPayloadDigestStr = null; 430 if (header.warcPayloadDigest != null) { 431 warcPayloadDigestStr = header.warcPayloadDigest.toString(); 432 } else if (header.warcPayloadDigestStr != null) { 433 warcPayloadDigestStr = header.warcPayloadDigestStr; 434 // Warning... 435 } 436 if (warcPayloadDigestStr != null) { 437 outBuf.write(WarcConstants.FN_WARC_PAYLOAD_DIGEST.getBytes()); 438 outBuf.write(": ".getBytes()); 439 outBuf.write(warcPayloadDigestStr.getBytes()); 440 outBuf.write("\r\n".getBytes()); 441 } 442 /* 443 * Warc-Ip-Address 444 */ 445 String warcIpAddress = null; 446 if (header.warcInetAddress != null) { 447 warcIpAddress = header.warcInetAddress.getHostAddress(); 448 } else if (header.warcIpAddress != null) { 449 warcIpAddress = header.warcIpAddress; 450 // Warning... 451 } 452 if (warcIpAddress != null) { 453 outBuf.write(WarcConstants.FN_WARC_IP_ADDRESS.getBytes()); 454 outBuf.write(": ".getBytes()); 455 outBuf.write(warcIpAddress.getBytes()); 456 outBuf.write("\r\n".getBytes()); 457 } 458 /* 459 * Warc-Refers-To 460 */ 461 String warcRefersToUriStr = null; 462 if (header.warcRefersToUri != null) { 463 warcRefersToUriStr = header.warcRefersToUri.toString(); 464 } else if (header.warcRefersToStr != null) { 465 warcRefersToUriStr = header.warcRefersToStr; 466 // Warning... 467 } 468 if (warcRefersToUriStr != null) { 469 outBuf.write(WarcConstants.FN_WARC_REFERS_TO.getBytes()); 470 outBuf.write(": <".getBytes()); 471 outBuf.write(warcRefersToUriStr.getBytes()); 472 outBuf.write(">\r\n".getBytes()); 473 } 474 /* 475 * Warc-Target-Uri 476 */ 477 String warcTargetUriStr = null; 478 if (header.warcTargetUriUri != null) { 479 warcTargetUriStr = header.warcTargetUriUri.toString(); 480 } else if (header.warcTargetUriStr != null) { 481 warcTargetUriStr = header.warcTargetUriStr; 482 // Warning... 483 } 484 if (warcTargetUriStr != null) { 485 outBuf.write(WarcConstants.FN_WARC_TARGET_URI.getBytes()); 486 outBuf.write(": ".getBytes()); 487 outBuf.write(warcTargetUriStr.getBytes()); 488 outBuf.write("\r\n".getBytes()); 489 } 490 /* 491 * Warc-Truncated 492 */ 493 String warcTruncatedStr = null; 494 if (header.warcTruncatedIdx != null) { 495 if (header.warcTruncatedIdx > 0 496 && header.warcTruncatedIdx < WarcConstants.TT_IDX_STRINGS.length) { 497 warcTruncatedStr = WarcConstants.TT_IDX_STRINGS[header.warcTruncatedIdx]; 498 } else { 499 // Warning... 500 } 501 } 502 if (warcTruncatedStr == null) { 503 warcTruncatedStr = header.warcTruncatedStr; 504 } 505 if (warcTruncatedStr != null) { 506 outBuf.write(WarcConstants.FN_WARC_TRUNCATED.getBytes()); 507 outBuf.write(": ".getBytes()); 508 outBuf.write(warcTruncatedStr.getBytes()); 509 outBuf.write("\r\n".getBytes()); 510 } 511 /* 512 * Warc-Warcinfo-Id 513 */ 514 String warcWarcInfoIdStr = null; 515 if (header.warcWarcinfoIdUri != null) { 516 warcWarcInfoIdStr = header.warcWarcinfoIdUri.toString(); 517 } else if (header.warcWarcinfoIdStr != null) { 518 warcWarcInfoIdStr = header.warcWarcinfoIdStr; 519 // Warning... 520 } 521 if (warcWarcInfoIdStr != null) { 522 outBuf.write(WarcConstants.FN_WARC_WARCINFO_ID.getBytes()); 523 outBuf.write(": <".getBytes()); 524 outBuf.write(warcWarcInfoIdStr.getBytes()); 525 outBuf.write(">\r\n".getBytes()); 526 } 527 /* 528 * Warc-Filename 529 */ 530 if (header.warcFilename != null) { 531 outBuf.write(WarcConstants.FN_WARC_FILENAME.getBytes()); 532 outBuf.write(": ".getBytes()); 533 outBuf.write(header.warcFilename.getBytes()); 534 outBuf.write("\r\n".getBytes()); 535 } 536 /* 537 * Warc-Profile 538 */ 539 String warcProfileStr = null; 540 if (header.warcProfileUri != null) { 541 warcProfileStr = header.warcProfileUri.toString(); 542 } else if (header.warcProfileIdx != null) { 543 if (header.warcProfileIdx > 0 544 && header.warcProfileIdx < WarcConstants.P_IDX_STRINGS.length) { 545 warcProfileStr = WarcConstants.P_IDX_STRINGS[header.warcProfileIdx]; 546 } else { 547 // Warning... 548 } 549 } 550 if (warcProfileStr == null) { 551 warcProfileStr = header.warcProfileStr; 552 // Warning... 553 } 554 if (warcProfileStr != null) { 555 outBuf.write(WarcConstants.FN_WARC_PROFILE.getBytes()); 556 outBuf.write(": ".getBytes()); 557 outBuf.write(warcProfileStr.getBytes()); 558 outBuf.write("\r\n".getBytes()); 559 } 560 /* 561 * Warc-Identified-Payload-Type 562 */ 563 String warcIdentifiedPayloadTypeStr = null; 564 if (header.warcIdentifiedPayloadType != null) { 565 warcIdentifiedPayloadTypeStr = header.warcIdentifiedPayloadType.toString(); 566 } else if (header.warcIdentifiedPayloadTypeStr != null) { 567 warcIdentifiedPayloadTypeStr = header.warcIdentifiedPayloadTypeStr; 568 // Warning... 569 } 570 if (warcIdentifiedPayloadTypeStr != null) { 571 outBuf.write(WarcConstants.FN_WARC_IDENTIFIED_PAYLOAD_TYPE.getBytes()); 572 outBuf.write(": ".getBytes()); 573 outBuf.write(warcIdentifiedPayloadTypeStr.getBytes()); 574 outBuf.write("\r\n".getBytes()); 575 } 576 /* 577 * Warc-Segment-Number 578 */ 579 String warcSegmentNumberStr = null; 580 if (header.warcSegmentNumber != null) { 581 warcSegmentNumberStr = header.warcSegmentNumber.toString(); 582 } else if (header.warcSegmentNumberStr != null) { 583 warcSegmentNumberStr = header.warcSegmentNumberStr; 584 // Warning... 585 } 586 if (warcSegmentNumberStr != null) { 587 outBuf.write(WarcConstants.FN_WARC_SEGMENT_NUMBER.getBytes()); 588 outBuf.write(": ".getBytes()); 589 outBuf.write(warcSegmentNumberStr.getBytes()); 590 outBuf.write("\r\n".getBytes()); 591 } 592 /* 593 * Warc-Segment-Origin-Id 594 */ 595 String warcSegmentOriginIdStr = null; 596 if (header.warcSegmentOriginIdUrl != null) { 597 warcSegmentOriginIdStr = header.warcSegmentOriginIdUrl.toString(); 598 } else if (header.warcSegmentOriginIdStr != null) { 599 warcSegmentOriginIdStr = header.warcSegmentOriginIdStr; 600 // Warning... 601 } 602 if (warcSegmentOriginIdStr != null) { 603 outBuf.write(WarcConstants.FN_WARC_SEGMENT_ORIGIN_ID.getBytes()); 604 outBuf.write(": <".getBytes()); 605 outBuf.write(warcSegmentOriginIdStr.getBytes()); 606 outBuf.write(">\r\n".getBytes()); 607 } 608 /* 609 * Warc-Segment-Total-Length 610 */ 611 String warcSegmentTotalLengthStr = null; 612 if (header.warcSegmentTotalLength != null) { 613 warcSegmentTotalLengthStr = header.warcSegmentTotalLength.toString(); 614 } else if (header.warcSegmentTotalLengthStr != null) { 615 warcSegmentTotalLengthStr = header.warcSegmentTotalLengthStr; 616 // Warning... 617 } 618 if (warcSegmentTotalLengthStr != null) { 619 outBuf.write(WarcConstants.FN_WARC_SEGMENT_TOTAL_LENGTH.getBytes()); 620 outBuf.write(": ".getBytes()); 621 outBuf.write(warcSegmentTotalLengthStr.getBytes()); 622 outBuf.write("\r\n".getBytes()); 623 } 624 /* 625 * WARC-Refers-To-Target-URI 626 */ 627 String warcRefersToTargetUriStr = null; 628 if (header.warcRefersToTargetUriUri != null) { 629 warcRefersToTargetUriStr = header.warcRefersToTargetUriUri.toString(); 630 } else if (header.warcRefersToTargetUriStr != null) { 631 warcRefersToTargetUriStr = header.warcRefersToTargetUriStr; 632 } 633 if (warcRefersToTargetUriStr != null) { 634 outBuf.write(WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.getBytes()); 635 outBuf.write(": ".getBytes()); 636 outBuf.write(warcRefersToTargetUriStr.getBytes()); 637 outBuf.write("\r\n".getBytes()); 638 } 639 /* 640 * Warc-Refers-To-Date 641 */ 642 String warcRefersToDateStr = null; 643 if (header.warcRefersToDate != null) { 644 warcRefersToDateStr = WarcDateParser.getDateFormat().format(header.warcRefersToDate); 645 } else if (header.warcRefersToDateStr != null) { 646 warcRefersToDateStr = header.warcRefersToDateStr; 647 // Warning... 648 } 649 if (warcRefersToDateStr != null) { 650 outBuf.write(WarcConstants.FN_WARC_REFERS_TO_DATE.getBytes()); 651 outBuf.write(": ".getBytes()); 652 outBuf.write(warcRefersToDateStr.getBytes()); 653 outBuf.write("\r\n".getBytes()); 654 } 655 /* 656 * End Of Header 657 */ 658 outBuf.write("\r\n".getBytes()); 659 byte[] headerBytes = outBuf.toByteArray(); 660 out.write(headerBytes); 661 state = S_HEADER_WRITTEN; 662 payloadWrittenTotal = 0; 663 return headerBytes; 664 } 665 666 /** 667 * Stream the content of an input stream to the payload content. 668 * @param in input stream containing payload data 669 * @return number of bytes written during method invocation 670 * @throws IOException if an I/O exception occurs while writing payload data 671 */ 672 public long streamPayload(InputStream in) throws IOException { 673 if (in == null) { 674 throw new IllegalArgumentException( 675 "The 'in' parameter is null!"); 676 } 677 if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) { 678 throw new IllegalStateException("Write a header before writing payload!"); 679 } 680 long written = 0; 681 int read = 0; 682 while (read != -1) { 683 out.write(stream_copy_buffer, 0, read); 684 written += read; 685 read = in.read(stream_copy_buffer); 686 } 687 state = S_PAYLOAD_WRITTEN; 688 payloadWrittenTotal += written; 689 return written; 690 } 691 692 /** 693 * Append the content of a byte array to the payload content. 694 * @param b byte array with data to be written 695 * @return number of bytes written during method invocation 696 * @throws IOException if an I/O exception occurs while writing payload data 697 */ 698 public long writePayload(byte[] b) throws IOException { 699 if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) { 700 throw new IllegalStateException("Write a header before writing payload!"); 701 } 702 out.write(b); 703 state = S_PAYLOAD_WRITTEN; 704 payloadWrittenTotal += b.length; 705 return b.length; 706 } 707 708 /** 709 * Append the partial content of a byte array to the payload content. 710 * @param b byte array with partial data to be written 711 * @param offset offset to data to be written 712 * @param len length of data to be written 713 * @return number of bytes written during method invocation 714 * @throws IOException if an I/O exception occurs while writing payload data 715 */ 716 public long writePayload(byte[] b, int offset, int len) throws IOException { 717 if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) { 718 throw new IllegalStateException("Write a header before writing payload!"); 719 } 720 out.write(b, offset, len); 721 state = S_PAYLOAD_WRITTEN; 722 payloadWrittenTotal += len; 723 return len; 724 } 725 726}