001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.warc;
019
020import java.io.ByteArrayOutputStream;
021import java.io.Closeable;
022import java.io.IOException;
023import java.io.InputStream;
024import java.io.OutputStream;
025
026import org.jwat.common.Diagnosis;
027import org.jwat.common.DiagnosisType;
028import org.jwat.common.Diagnostics;
029import org.jwat.common.UriProfile;
030
031/**
032 * Base class for WARC writer implementations.
033 *
034 * @author nicl
035 */
036public abstract class WarcWriter implements Closeable {
037
038    /** State after writer has been constructed and before records have been written. */
039    protected static final int S_INIT = 0;
040
041    /** State after header has been written. */
042    protected static final int S_HEADER_WRITTEN = 1;
043
044    /** State after payload has been written. */
045    protected static final int S_PAYLOAD_WRITTEN = 2;
046
047    /** State after record has been closed. */
048    protected static final int S_RECORD_CLOSED = 3;
049
050    /*
051     * Settings.
052     */
053
054    /** WARC-Target-URI profile. */
055    protected UriProfile warcTargetUriProfile;
056
057    /** URI profile. */
058    protected UriProfile uriProfile;
059
060    /** Block Digesting enabled/disabled. */
061    //protected boolean bDigestBlock = false;
062
063    /** WARC field parser used. */
064    protected WarcFieldParsers fieldParsers;
065
066    /** Buffer used by streamPayload() to copy from one stream to another. */
067    protected byte[] stream_copy_buffer;
068
069    /** Configuration for throwing exception on content-length mismatch.
070     *  (Default is true) */
071    protected boolean bExceptionOnContentLengthMismatch;
072
073    /*
074     * State.
075     */
076
077    /** Writer level errors and warnings or when writing byte headers. */
078    public final Diagnostics diagnostics = new Diagnostics();
079
080    /** Current state of writer. */
081    protected int state = S_INIT;
082
083    /** Outputstream used to write WARC records. */
084    protected OutputStream out;
085
086    /** Current WARC header written. */
087    protected WarcHeader header;
088
089    /** Content-Length from the WARC header. */
090    protected Long headerContentLength;
091
092    /** Total bytes written for current record payload. */
093    protected long payloadWrittenTotal;
094
095    /**
096     * Method used to initialize a readers internal state.
097     * Must be called by all constructors.
098     */
099    protected void init() {
100        warcTargetUriProfile = UriProfile.RFC3986;
101        uriProfile = UriProfile.RFC3986;
102        fieldParsers = new WarcFieldParsers();
103        stream_copy_buffer = new byte[8192];
104        bExceptionOnContentLengthMismatch = true;
105    }
106
107    /**
108     * Is this writer producing compressed output.
109     * @return boolean indicating whether compressed output is produced
110     */
111    public abstract boolean isCompressed();
112
113    /**
114     * Set the URI profile used to validate WARC-Target URIs.
115     * If null, the uriProfile is set to RCF3986.
116     * @param uriProfile URI profile to use
117     */
118    public void setWarcTargetUriProfile(UriProfile uriProfile) {
119        if (uriProfile == null) {
120            uriProfile = UriProfile.RFC3986;
121        }
122        this.warcTargetUriProfile = uriProfile;
123    }
124
125    /**
126     * Get the URI profile used to validate WARC-Target URIs.
127     * @return the URI profile used to validate WARC-Target URIs
128     */
129    public UriProfile getWarcTargetUriProfile() {
130        return warcTargetUriProfile;
131    }
132
133    /**
134     * Set the URI profile used to validate URIs.
135     * If null, the uriProfile is set to RCF3986.
136     * @param uriProfile URI profile to use
137     */
138    public void setUriProfile(UriProfile uriProfile) {
139        if (uriProfile == null) {
140            uriProfile = UriProfile.RFC3986;
141        }
142        this.uriProfile = uriProfile;
143    }
144
145    /**
146     * Get the URI profile used to validate URIs.
147     * @return the URI profile used to validate URIs
148     */
149    public UriProfile getUriProfile() {
150        return uriProfile;
151    }
152
153    /**
154     * Does this writer throw an exception if the content-length does not match
155     * the payload amount written.
156     * @return boolean indicating if an exception is thrown or not
157     */
158    public boolean exceptionOnContentLengthMismatch() {
159        return bExceptionOnContentLengthMismatch;
160    }
161
162    /**
163     * Tell the writer what to do in case of mismatch between content-length
164     * and amount payload written.
165     * @param enabled boolean indicating exception throwing on/off
166     */
167    public void setExceptionOnContentLengthMismatch(boolean enabled) {
168        bExceptionOnContentLengthMismatch = enabled;
169    }
170
171    /**
172     * Is this writer set to block digest payload.
173     * @return boolean indicating payload block digesting
174     */
175    /*
176    public boolean digestBlock() {
177        return bDigestBlock;
178    }
179    */
180
181    /**
182     * Set the writers payload block digest mode
183     * @param enabled boolean indicating digest on/off
184     */
185    /*
186    public void setDigestBlock(boolean enabled) {
187        bDigestBlock = enabled;
188    }
189    */
190
191    /**
192     * Close WARC writer and free its resources.
193     * @throws IOException if an I/O exception occurs while closing the writer
194     */
195    public abstract void close() throws IOException;
196
197    /**
198     * Close the WARC record in an implementation specific way.
199     * @throws IOException if an I/O exception occurs while closing the record
200     */
201    public abstract void closeRecord() throws IOException;
202
203    /**
204     * Closes the WARC record by writing two newlines and comparing the amount of
205     * payload data streamed with the content-length supplied with the header.
206     * @throws IOException if an I/O exception occurs while closing the record
207     */
208    protected void closeRecord_impl() throws IOException {
209        Diagnosis diagnosis = null;
210        out.write(WarcConstants.endMark);
211        out.flush();
212        if (headerContentLength == null) {
213            diagnosis = new Diagnosis(
214                    DiagnosisType.ERROR_EXPECTED,
215                    "'" + WarcConstants.FN_CONTENT_LENGTH + "' header",
216                    "Mandatory!");
217        } else {
218            if (headerContentLength != payloadWrittenTotal) {
219                diagnosis = new Diagnosis(
220                        DiagnosisType.INVALID_EXPECTED,
221                        "'" + WarcConstants.FN_CONTENT_LENGTH + "' header",
222                        Long.toString(payloadWrittenTotal),
223                        headerContentLength.toString());
224            }
225        }
226        if (diagnosis != null) {
227            if (header != null) {
228                header.diagnostics.addError(diagnosis);
229            } else {
230                diagnostics.addError(diagnosis);
231            }
232            if (bExceptionOnContentLengthMismatch) {
233                throw new IllegalStateException("Payload size does not match content-length!");
234            }
235        }
236        header = null;
237        headerContentLength = null;
238    }
239
240    /**
241     * Write a raw WARC header to the WARC output stream. Closes any previously
242     * written record that has not been closed prior to this call.
243     * Errors and warnings are reported on the writers diagnostics object.
244     * @param header_bytes raw WARC header to output
245     * @param contentLength the expected content-length to be written and validated
246     * @throws IOException if an I/O exception occurs while writing header data
247     */
248    public void writeRawHeader(byte[] header_bytes, Long contentLength) throws IOException {
249        if (header_bytes == null) {
250            throw new IllegalArgumentException(
251                    "The 'header_bytes' parameter is null!");
252        }
253        if (contentLength != null && contentLength < 0) {
254            throw new IllegalArgumentException(
255                    "The 'contentLength' parameter is negative!");
256        }
257        if (state == S_HEADER_WRITTEN) {
258            throw new IllegalStateException("Headers written back to back!");
259        } else if (state == S_PAYLOAD_WRITTEN) {
260            closeRecord_impl();
261        }
262        out.write(header_bytes);
263        state = S_HEADER_WRITTEN;
264        header = null;
265        headerContentLength = contentLength;
266        payloadWrittenTotal = 0;
267    }
268
269    /**
270     * Write a WARC header to the WARC output stream.
271     * Errors and warnings are reported on the records diagnostics object.
272     * @param record WARC record to output
273     * @return byte array version of header as it was written
274     * @throws IOException if an I/O exception occurs while writing header data
275     */
276    public abstract byte[] writeHeader(WarcRecord record) throws IOException;
277
278    /**
279     * Write a WARC header to the WARC output stream.
280     * The WARC header is not required to be valid.
281     * Errors and warnings are reported on the records diagnostics object.
282     * @param record WARC record to output
283     * @return byte array version of header as it was written
284     * @throws IOException if an I/O exception occurs while writing header data
285     */
286    protected byte[] writeHeader_impl(WarcRecord record) throws IOException {
287        header = record.header;
288        headerContentLength = header.contentLength;
289        if (headerContentLength == null && header.contentLengthStr != null) {
290            try {
291                headerContentLength = Long.parseLong(header.contentLengthStr);
292            } catch (NumberFormatException e) {
293                // TODO Add warning...
294            }
295        }
296        ByteArrayOutputStream outBuf = new ByteArrayOutputStream();
297        /*
298         * Version Line
299         */
300        byte[] magicVersion = (WarcConstants.WARC_MAGIC_HEADER + header.major + "." + header.minor + "\r\n").getBytes();
301        outBuf.write(magicVersion);
302        /*
303         * Warc-Type
304         */
305        String warcTypeStr = null;
306        if (header.warcTypeIdx != null) {
307            if (header.warcTypeIdx > 0
308                && header.warcTypeIdx < WarcConstants.RT_IDX_STRINGS.length) {
309                warcTypeStr = WarcConstants.RT_IDX_STRINGS[header.warcTypeIdx];
310            } else {
311                // Warning...
312            }
313        }
314        if (warcTypeStr == null) {
315            warcTypeStr = header.warcTypeStr;
316        }
317        if (warcTypeStr != null) {
318            outBuf.write(WarcConstants.FN_WARC_TYPE.getBytes());
319            outBuf.write(": ".getBytes());
320            outBuf.write(warcTypeStr.getBytes());
321            outBuf.write("\r\n".getBytes());
322        }
323        /*
324         * Warc-Record-Id
325         */
326        String warcRecordIdStr = null;
327        if (header.warcRecordIdUri != null) {
328            warcRecordIdStr = header.warcRecordIdUri.toString();
329        } else if (header.warcRecordIdStr != null) {
330            warcRecordIdStr = header.warcRecordIdStr;
331            // Warning...
332        }
333        if (warcRecordIdStr != null) {
334            outBuf.write(WarcConstants.FN_WARC_RECORD_ID.getBytes());
335            outBuf.write(": <".getBytes());
336            outBuf.write(warcRecordIdStr.getBytes());
337            outBuf.write(">\r\n".getBytes());
338        }
339        /*
340         * Warc-Date
341         */
342        String warcDateStr = null;
343        if (header.warcDate != null) {
344            // FIXME Check..
345            //warcDateStr = WarcDateParser.getDateFormat().format(header.warcDate);
346            if (header.major == 1 && header.minor == 0 && header.warcDate.precision != WarcDate.P_SECOND) {
347                header.warcDate.adjustPrecisionTo(WarcDate.P_SECOND);
348            }
349            warcDateStr = header.warcDate.toString();
350        } else if (header.warcDateStr != null) {
351            warcDateStr = header.warcDateStr;
352            // Warning...
353        }
354        if (warcDateStr != null) {
355            outBuf.write(WarcConstants.FN_WARC_DATE.getBytes());
356            outBuf.write(": ".getBytes());
357            outBuf.write(warcDateStr.getBytes());
358            outBuf.write("\r\n".getBytes());
359        }
360        /*
361         * Content-Length
362         */
363        String contentLengthStr = null;
364        if (header.contentLength != null) {
365            contentLengthStr = header.contentLength.toString();
366        } else if (header.contentLengthStr != null) {
367            contentLengthStr = header.contentLengthStr;
368            // Warning...
369        }
370        if (contentLengthStr != null) {
371            outBuf.write(WarcConstants.FN_CONTENT_LENGTH.getBytes());
372            outBuf.write(": ".getBytes());
373            outBuf.write(contentLengthStr.getBytes());
374            outBuf.write("\r\n".getBytes());
375        }
376        /*
377         * Content-Type
378         */
379        String contentTypeStr = null;
380        if (header.contentType != null) {
381            contentTypeStr = header.contentType.toString();
382        } else if (header.contentTypeStr != null) {
383            contentTypeStr = header.contentTypeStr;
384            // Warning...
385        }
386        if (contentTypeStr != null) {
387            outBuf.write(WarcConstants.FN_CONTENT_TYPE.getBytes());
388            outBuf.write(": ".getBytes());
389            outBuf.write(contentTypeStr.getBytes());
390            outBuf.write("\r\n".getBytes());
391        }
392        /*
393         * Warc-Concurrent-To
394         */
395        WarcConcurrentTo warcConcurrentTo;
396        String warcConcurrentToStr;
397        if (header.warcConcurrentToList != null) {
398            for (int i=0; i<header.warcConcurrentToList.size(); ++i) {
399                warcConcurrentTo = header.warcConcurrentToList.get(i);
400                warcConcurrentToStr = null;
401                if (warcConcurrentTo.warcConcurrentToUri != null) {
402                    warcConcurrentToStr = warcConcurrentTo.warcConcurrentToUri.toString();
403                } else if (warcConcurrentTo.warcConcurrentToStr != null) {
404                    warcConcurrentToStr = warcConcurrentTo.warcConcurrentToStr;
405                    // Warning...
406                }
407                if (warcConcurrentToStr != null) {
408                    outBuf.write(WarcConstants.FN_WARC_CONCURRENT_TO.getBytes());
409                    outBuf.write(": <".getBytes());
410                    outBuf.write(warcConcurrentToStr.getBytes());
411                    outBuf.write(">\r\n".getBytes());
412                }
413            }
414        }
415        /*
416         * Warc-Block-Digest
417         */
418        String warcBlockDigestStr = null;
419        if (header.warcBlockDigest != null) {
420            warcBlockDigestStr = header.warcBlockDigest.toString();
421        } else if (header.warcBlockDigestStr != null) {
422            warcBlockDigestStr = header.warcBlockDigestStr;
423            // Warning...
424        }
425        if (warcBlockDigestStr != null) {
426            outBuf.write(WarcConstants.FN_WARC_BLOCK_DIGEST.getBytes());
427            outBuf.write(": ".getBytes());
428            outBuf.write(warcBlockDigestStr.getBytes());
429            outBuf.write("\r\n".getBytes());
430        }
431        /*
432         * Warc-Payload-Digest
433         */
434        String warcPayloadDigestStr = null;
435        if (header.warcPayloadDigest != null) {
436            warcPayloadDigestStr = header.warcPayloadDigest.toString();
437        } else if (header.warcPayloadDigestStr != null) {
438            warcPayloadDigestStr = header.warcPayloadDigestStr;
439            // Warning...
440        }
441        if (warcPayloadDigestStr != null) {
442            outBuf.write(WarcConstants.FN_WARC_PAYLOAD_DIGEST.getBytes());
443            outBuf.write(": ".getBytes());
444            outBuf.write(warcPayloadDigestStr.getBytes());
445            outBuf.write("\r\n".getBytes());
446        }
447        /*
448         * Warc-Ip-Address
449         */
450        String warcIpAddress = null;
451        if (header.warcInetAddress != null) {
452            warcIpAddress = header.warcInetAddress.getHostAddress();
453        } else if (header.warcIpAddress != null) {
454            warcIpAddress = header.warcIpAddress;
455            // Warning...
456        }
457        if (warcIpAddress != null) {
458            outBuf.write(WarcConstants.FN_WARC_IP_ADDRESS.getBytes());
459            outBuf.write(": ".getBytes());
460            outBuf.write(warcIpAddress.getBytes());
461            outBuf.write("\r\n".getBytes());
462        }
463        /*
464         * Warc-Refers-To
465         */
466        String warcRefersToUriStr = null;
467        if (header.warcRefersToUri != null) {
468            warcRefersToUriStr = header.warcRefersToUri.toString();
469        } else if (header.warcRefersToStr != null) {
470            warcRefersToUriStr = header.warcRefersToStr;
471            // Warning...
472        }
473        if (warcRefersToUriStr != null) {
474            outBuf.write(WarcConstants.FN_WARC_REFERS_TO.getBytes());
475            outBuf.write(": <".getBytes());
476            outBuf.write(warcRefersToUriStr.getBytes());
477            outBuf.write(">\r\n".getBytes());
478        }
479        /*
480         * Warc-Target-Uri
481         */
482        String warcTargetUriStr = null;
483        if (header.warcTargetUriUri != null) {
484            warcTargetUriStr = header.warcTargetUriUri.toString();
485        } else if (header.warcTargetUriStr != null) {
486            warcTargetUriStr = header.warcTargetUriStr;
487            // Warning...
488        }
489        if (warcTargetUriStr != null) {
490            outBuf.write(WarcConstants.FN_WARC_TARGET_URI.getBytes());
491            outBuf.write(": ".getBytes());
492            outBuf.write(warcTargetUriStr.getBytes());
493            outBuf.write("\r\n".getBytes());
494        }
495        /*
496         * Warc-Truncated
497         */
498        String warcTruncatedStr = null;
499        if (header.warcTruncatedIdx != null) {
500            if (header.warcTruncatedIdx > 0
501                    && header.warcTruncatedIdx < WarcConstants.TT_IDX_STRINGS.length) {
502                warcTruncatedStr = WarcConstants.TT_IDX_STRINGS[header.warcTruncatedIdx];
503            } else {
504                // Warning...
505            }
506        }
507        if (warcTruncatedStr == null) {
508            warcTruncatedStr = header.warcTruncatedStr;
509        }
510        if (warcTruncatedStr != null) {
511            outBuf.write(WarcConstants.FN_WARC_TRUNCATED.getBytes());
512            outBuf.write(": ".getBytes());
513            outBuf.write(warcTruncatedStr.getBytes());
514            outBuf.write("\r\n".getBytes());
515        }
516        /*
517         * Warc-Warcinfo-Id
518         */
519        String warcWarcInfoIdStr = null;
520        if (header.warcWarcinfoIdUri != null) {
521            warcWarcInfoIdStr = header.warcWarcinfoIdUri.toString();
522        } else if (header.warcWarcinfoIdStr != null) {
523            warcWarcInfoIdStr = header.warcWarcinfoIdStr;
524            // Warning...
525        }
526        if (warcWarcInfoIdStr != null) {
527            outBuf.write(WarcConstants.FN_WARC_WARCINFO_ID.getBytes());
528            outBuf.write(": <".getBytes());
529            outBuf.write(warcWarcInfoIdStr.getBytes());
530            outBuf.write(">\r\n".getBytes());
531        }
532        /*
533         * Warc-Filename
534         */
535        if (header.warcFilename != null) {
536            outBuf.write(WarcConstants.FN_WARC_FILENAME.getBytes());
537            outBuf.write(": ".getBytes());
538            outBuf.write(header.warcFilename.getBytes());
539            outBuf.write("\r\n".getBytes());
540        }
541        /*
542         * Warc-Profile
543         */
544        String warcProfileStr = null;
545        if (header.warcProfileUri != null) {
546            warcProfileStr = header.warcProfileUri.toString();
547        } else if (header.warcProfileIdx != null) {
548            if (header.warcProfileIdx > 0
549                    && header.warcProfileIdx < WarcConstants.P_IDX_STRINGS.length) {
550                warcProfileStr = WarcConstants.P_IDX_STRINGS[header.warcProfileIdx];
551            } else {
552                // Warning...
553            }
554        }
555        if (warcProfileStr == null) {
556            warcProfileStr = header.warcProfileStr;
557            // Warning...
558        }
559        if (warcProfileStr != null) {
560            outBuf.write(WarcConstants.FN_WARC_PROFILE.getBytes());
561            outBuf.write(": ".getBytes());
562            outBuf.write(warcProfileStr.getBytes());
563            outBuf.write("\r\n".getBytes());
564        }
565        /*
566         * Warc-Identified-Payload-Type
567         */
568        String warcIdentifiedPayloadTypeStr = null;
569        if (header.warcIdentifiedPayloadType != null) {
570            warcIdentifiedPayloadTypeStr = header.warcIdentifiedPayloadType.toString();
571        } else if (header.warcIdentifiedPayloadTypeStr != null) {
572            warcIdentifiedPayloadTypeStr = header.warcIdentifiedPayloadTypeStr;
573            // Warning...
574        }
575        if (warcIdentifiedPayloadTypeStr != null) {
576            outBuf.write(WarcConstants.FN_WARC_IDENTIFIED_PAYLOAD_TYPE.getBytes());
577            outBuf.write(": ".getBytes());
578            outBuf.write(warcIdentifiedPayloadTypeStr.getBytes());
579            outBuf.write("\r\n".getBytes());
580        }
581        /*
582         * Warc-Segment-Number
583         */
584        String warcSegmentNumberStr = null;
585        if (header.warcSegmentNumber != null) {
586            warcSegmentNumberStr = header.warcSegmentNumber.toString();
587        } else if (header.warcSegmentNumberStr != null) {
588            warcSegmentNumberStr = header.warcSegmentNumberStr;
589            // Warning...
590        }
591        if (warcSegmentNumberStr != null) {
592            outBuf.write(WarcConstants.FN_WARC_SEGMENT_NUMBER.getBytes());
593            outBuf.write(": ".getBytes());
594            outBuf.write(warcSegmentNumberStr.getBytes());
595            outBuf.write("\r\n".getBytes());
596        }
597        /*
598         * Warc-Segment-Origin-Id
599         */
600        String warcSegmentOriginIdStr = null;
601        if (header.warcSegmentOriginIdUrl != null) {
602            warcSegmentOriginIdStr = header.warcSegmentOriginIdUrl.toString();
603        } else if (header.warcSegmentOriginIdStr != null) {
604            warcSegmentOriginIdStr = header.warcSegmentOriginIdStr;
605            // Warning...
606        }
607        if (warcSegmentOriginIdStr != null) {
608            outBuf.write(WarcConstants.FN_WARC_SEGMENT_ORIGIN_ID.getBytes());
609            outBuf.write(": <".getBytes());
610            outBuf.write(warcSegmentOriginIdStr.getBytes());
611            outBuf.write(">\r\n".getBytes());
612        }
613        /*
614         * Warc-Segment-Total-Length
615         */
616        String warcSegmentTotalLengthStr = null;
617        if (header.warcSegmentTotalLength != null) {
618            warcSegmentTotalLengthStr = header.warcSegmentTotalLength.toString();
619        } else if (header.warcSegmentTotalLengthStr != null) {
620            warcSegmentTotalLengthStr = header.warcSegmentTotalLengthStr;
621            // Warning...
622        }
623        if (warcSegmentTotalLengthStr != null) {
624            outBuf.write(WarcConstants.FN_WARC_SEGMENT_TOTAL_LENGTH.getBytes());
625            outBuf.write(": ".getBytes());
626            outBuf.write(warcSegmentTotalLengthStr.getBytes());
627            outBuf.write("\r\n".getBytes());
628        }
629        /*
630         * WARC-Refers-To-Target-URI
631         */
632        String warcRefersToTargetUriStr = null;
633        if (header.warcRefersToTargetUriUri != null) {
634            warcRefersToTargetUriStr = header.warcRefersToTargetUriUri.toString();
635        } else if (header.warcRefersToTargetUriStr != null) {
636            warcRefersToTargetUriStr = header.warcRefersToTargetUriStr;
637        }
638        if (warcRefersToTargetUriStr != null) {
639            outBuf.write(WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.getBytes());
640            outBuf.write(": ".getBytes());
641            outBuf.write(warcRefersToTargetUriStr.getBytes());
642            outBuf.write("\r\n".getBytes());
643        }
644        /*
645         * Warc-Refers-To-Date
646         */
647        String warcRefersToDateStr = null;
648        if (header.warcRefersToDate != null) {
649            //warcRefersToDateStr = WarcDateParser.getDateFormat().format(header.warcRefersToDate);
650            // FIXME Check..
651            if (header.major == 1 && header.minor == 0 && header.warcRefersToDate.precision != WarcDate.P_SECOND) {
652                header.warcRefersToDate.adjustPrecisionTo(WarcDate.P_SECOND);
653            }
654            warcRefersToDateStr = header.warcRefersToDate.toString();
655        } else if (header.warcRefersToDateStr != null) {
656            warcRefersToDateStr = header.warcRefersToDateStr;
657            // Warning...
658        }
659        if (warcRefersToDateStr != null) {
660            outBuf.write(WarcConstants.FN_WARC_REFERS_TO_DATE.getBytes());
661            outBuf.write(": ".getBytes());
662            outBuf.write(warcRefersToDateStr.getBytes());
663            outBuf.write("\r\n".getBytes());
664        }
665        /*
666         * End Of Header
667         */
668        outBuf.write("\r\n".getBytes());
669        byte[] headerBytes = outBuf.toByteArray();
670        out.write(headerBytes);
671        state = S_HEADER_WRITTEN;
672        payloadWrittenTotal = 0;
673        return headerBytes;
674    }
675
676    /**
677     * Stream the content of an input stream to the payload content.
678     * @param in input stream containing payload data
679     * @return number of bytes written during method invocation
680     * @throws IOException if an I/O exception occurs while writing payload data
681     */
682    public long streamPayload(InputStream in) throws IOException {
683        if (in == null) {
684            throw new IllegalArgumentException(
685                    "The 'in' parameter is null!");
686        }
687        if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) {
688            throw new IllegalStateException("Write a header before writing payload!");
689        }
690        long written = 0;
691        int read = 0;
692        while (read != -1) {
693            out.write(stream_copy_buffer, 0, read);
694            written += read;
695            read = in.read(stream_copy_buffer);
696        }
697        state = S_PAYLOAD_WRITTEN;
698        payloadWrittenTotal += written;
699        return written;
700    }
701
702    /**
703     * Append the content of a byte array to the payload content.
704     * @param b byte array with data to be written
705     * @return number of bytes written during method invocation
706     * @throws IOException if an I/O exception occurs while writing payload data
707     */
708    public long writePayload(byte[] b) throws IOException {
709        if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) {
710            throw new IllegalStateException("Write a header before writing payload!");
711        }
712        out.write(b);
713        state = S_PAYLOAD_WRITTEN;
714        payloadWrittenTotal += b.length;
715        return b.length;
716    }
717
718    /**
719     * Append the partial content of a byte array to the payload content.
720     * @param b byte array with partial data to be written
721     * @param offset offset to data to be written
722     * @param len length of data to be written
723     * @return number of bytes written during method invocation
724     * @throws IOException if an I/O exception occurs while writing payload data
725     */
726    public long writePayload(byte[] b, int offset, int len) throws IOException {
727        if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) {
728            throw new IllegalStateException("Write a header before writing payload!");
729        }
730        out.write(b, offset, len);
731        state = S_PAYLOAD_WRITTEN;
732        payloadWrittenTotal += len;
733        return len;
734    }
735
736}