001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.arc;
019
020import java.io.ByteArrayOutputStream;
021import java.io.Closeable;
022import java.io.IOException;
023import java.io.InputStream;
024import java.io.OutputStream;
025
026import org.jwat.common.Diagnosis;
027import org.jwat.common.DiagnosisType;
028import org.jwat.common.Diagnostics;
029
030/**
031 * Base class for ARC writer implementations.
032 *
033 * Note: the content-length is synonymous to the archive-length.
034 *
035 * @author nicl
036 */
037public abstract class ArcWriter implements Closeable {
038
039    /** State after writer has been constructed and before records have been written. */
040    protected static final int S_INIT = 0;
041
042    /** State after header has been written. */
043    protected static final int S_HEADER_WRITTEN = 1;
044
045    /** State after payload has been written. */
046    protected static final int S_PAYLOAD_WRITTEN = 2;
047
048    /** State after record has been closed. */
049    protected static final int S_RECORD_CLOSED = 3;
050
051    /*
052     * Settings.
053     */
054
055    /** ARC field parser used. */
056    protected ArcFieldParsers fieldParsers;
057
058    /** Buffer used by streamPayload() to copy from one stream to another. */
059    protected byte[] stream_copy_buffer;
060
061    /** Configuration for throwing exception on content-length mismatch.
062     *  (Default is true) */
063    protected boolean bExceptionOnContentLengthMismatch;
064
065    /*
066     * State.
067     */
068
069    /** Writer level errors and warnings or when writing byte headers. */
070    public final Diagnostics<Diagnosis> diagnostics = new Diagnostics<Diagnosis>();
071
072    /** Current state of writer. */
073    protected int state = S_INIT;
074
075    /** Outputstream used to write ARC records. */
076    protected OutputStream out;
077
078    protected ArcHeader header;
079
080    /** Content-Length from the ARC header. */
081    protected Long headerContentLength;
082
083    /** Total bytes written for current record payload. */
084    protected long payloadWrittenTotal;
085
086    /**
087     * Method used to initialize a readers internal state.
088     * Must be called by all constructors.
089     */
090    protected void init() {
091        fieldParsers = new ArcFieldParsers();
092        stream_copy_buffer = new byte[8192];
093        bExceptionOnContentLengthMismatch = true;
094    }
095
096    /**
097     * Is this writer producing compressed output.
098     * @return boolean indicating whether compressed output is produced
099     */
100    public abstract boolean isCompressed();
101
102    /**
103     * Does this writer throw an exception if the content-length does not match
104     * the payload amount written.
105     * @return boolean indicating if an exception is thrown or not
106     */
107    public boolean exceptionOnContentLengthMismatch() {
108        return bExceptionOnContentLengthMismatch;
109    }
110
111    /**
112     * Tell the writer what to do in case of mismatch between content-length
113     * and amount payload written.
114     * @param enabled boolean indicating exception throwing on/off
115     */
116    public void setExceptionOnContentLengthMismatch(boolean enabled) {
117        bExceptionOnContentLengthMismatch = enabled;
118    }
119
120    /**
121     * Close ARC writer and free its resources.
122     * @throws IOException if an I/O exception occurs while closing the writer
123     */
124    public abstract void close() throws IOException;
125
126    /**
127     * Close the ARC record in an implementation specific way.
128     * @throws IOException if an I/O exception occurs while closing the record
129     */
130    public abstract void closeRecord() throws IOException;
131
132    /**
133     * Closes the ARC record by writing one newline and comparing the amount of
134     * payload data streamed with the content-length supplied with the header.
135     * @throws IOException if an I/O exception occurs while closing the record
136     */
137    protected void closeRecord_impl() throws IOException {
138        Diagnosis diagnosis = null;
139        out.write(ArcConstants.endMark);
140        out.flush();
141        if (headerContentLength == null) {
142            diagnosis = new Diagnosis(
143                    DiagnosisType.ERROR_EXPECTED,
144                    "'" + ArcConstants.FN_ARCHIVE_LENGTH + "' header",
145                    "Mandatory!");
146        } else {
147            if (headerContentLength != payloadWrittenTotal) {
148                diagnosis = new Diagnosis(
149                        DiagnosisType.INVALID_EXPECTED,
150                        "'" + ArcConstants.FN_ARCHIVE_LENGTH + "' header",
151                        Long.toString(payloadWrittenTotal),
152                        headerContentLength.toString());
153            }
154        }
155        if (diagnosis != null) {
156            if (header != null) {
157                header.diagnostics.addError(diagnosis);
158            } else {
159                diagnostics.addError(diagnosis);
160            }
161            if (bExceptionOnContentLengthMismatch) {
162                throw new IllegalStateException("Payload size does not match content-length!");
163            }
164        }
165        header = null;
166        headerContentLength = null;
167    }
168
169    /**
170     * Write a raw ARC header to the ARC output stream. Closes any previously
171     * written record that has not been closed prior to this call.
172     * Errors and warnings are reported on the writers diagnostics object.
173     * @param header_bytes raw ARC record to output
174     * @param contentLength the expected content-length to be written and validated
175     * The contentLength is synonymous to the Archive-Length.
176     * @throws IOException if an exception occurs while writing header data
177     */
178    public void writeRawHeader(byte[] header_bytes, Long contentLength) throws IOException {
179        if (header_bytes == null) {
180            throw new IllegalArgumentException(
181                    "The 'header_bytes' parameter is null!");
182        }
183        if (contentLength != null && contentLength < 0) {
184            throw new IllegalArgumentException(
185                    "The 'contentLength' parameter is negative!");
186        }
187        if (state == S_HEADER_WRITTEN) {
188            throw new IllegalStateException("Headers written back to back!");
189        } else if (state == S_PAYLOAD_WRITTEN) {
190            closeRecord_impl();
191        }
192        out.write(header_bytes);
193        state = S_HEADER_WRITTEN;
194        header = null;
195        headerContentLength = contentLength;
196        payloadWrittenTotal = 0;
197    }
198
199    /**
200     * Write a ARC header to the ARC output stream.
201     * Errors and warnings are reported on the records diagnostics object.
202     * @param record ARC record to output
203     * @return byte array version of header as it was written
204     * @throws IOException if an I/O exception occurs while writing header data
205     */
206    public abstract byte[] writeHeader(ArcRecordBase record) throws IOException;
207
208    /**
209     * Write an ARC header to the ARC output stream.
210     * The WARC header is not required to be valid.
211     * Errors and warnings are reported on the records diagnostics object.
212     * @param record ARC record to output
213     * @return byte array version of header as it was written
214     * @throws IOException if an I/O exception occurs while writing header data
215     */
216    protected byte[] writeHeader_impl(ArcRecordBase record) throws IOException {
217        header = record.header;
218        headerContentLength = header.archiveLength;
219        if (headerContentLength == null && header.archiveLengthStr != null) {
220            headerContentLength = fieldParsers.parseLong(header.archiveLengthStr, ArcConstants.FN_ARCHIVE_LENGTH, false);
221            if (headerContentLength != null && headerContentLength < 0) {
222                diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_ARCHIVE_LENGTH + "' value", header.archiveLengthStr, "A non negative number"));
223            }
224        }
225        ByteArrayOutputStream outBuf = new ByteArrayOutputStream();
226        /*
227         * URL
228         */
229        String urlStr;
230        if (header.urlUri != null) {
231            urlStr = header.urlUri.toString();
232        } else if (header.urlStr != null && header.urlStr.length() > 0) {
233            urlStr = header.urlStr;
234        } else {
235            urlStr = "-";
236        }
237        outBuf.write(urlStr.getBytes());
238        /*
239         * IP-Address
240         */
241        String ipAddressStr;
242        if (header.inetAddress != null) {
243            ipAddressStr = header.inetAddress.getHostAddress();
244        } else if (header.ipAddressStr != null && header.ipAddressStr.length() > 0) {
245            ipAddressStr = header.ipAddressStr;
246        } else {
247            ipAddressStr = "-";
248        }
249        outBuf.write(" ".getBytes());
250        outBuf.write(ipAddressStr.getBytes());
251        /*
252         * Archive-Date
253         */
254        String archiveDateStr;
255        if (header.archiveDate != null) {
256            archiveDateStr = ArcDateParser.getDateFormat().format(header.archiveDate);
257        } else if (header.archiveDateStr != null && header.archiveDateStr.length() > 0) {
258            archiveDateStr = header.archiveDateStr;
259        } else {
260            archiveDateStr = "-";
261        }
262        outBuf.write(" ".getBytes());
263        outBuf.write(archiveDateStr.getBytes());
264        /*
265         * Content-Type
266         */
267        String contentTypeStr;
268        if (header.contentType != null) {
269            contentTypeStr = header.contentType.toStringShort();
270        } else if (header.contentTypeStr != null && header.contentTypeStr.length() > 0) {
271            contentTypeStr = header.contentTypeStr;
272        } else {
273            contentTypeStr = "-";
274        }
275        outBuf.write(" ".getBytes());
276        outBuf.write(contentTypeStr.getBytes());
277        /*
278         * Version 2 fields.
279         */
280        if (header.recordFieldVersion == 2) {
281            /*
282             * Result-Code
283             */
284            String resultCodeStr;
285            if (header.resultCode != null) {
286                resultCodeStr = header.resultCode.toString();
287            } else if (header.resultCodeStr != null && header.resultCodeStr.length() > 0) {
288                resultCodeStr = header.resultCodeStr;
289            } else {
290                resultCodeStr = "-";
291            }
292            outBuf.write(" ".getBytes());
293            outBuf.write(resultCodeStr.getBytes());
294            /*
295             * Checksum
296             */
297            String checksumStr;
298            if (header.checksumStr != null && header.checksumStr.length() > 0) {
299                checksumStr = header.checksumStr;
300            } else {
301                checksumStr = "-";
302            }
303            outBuf.write(" ".getBytes());
304            outBuf.write(checksumStr.getBytes());
305            /*
306             * Location
307             */
308            String locationStr;
309            if (header.locationStr != null && header.locationStr.length() > 0) {
310                locationStr = header.locationStr;
311            } else {
312                locationStr = "-";
313            }
314            outBuf.write(" ".getBytes());
315            outBuf.write(locationStr.getBytes());
316            /*
317             * Offset
318             */
319            String offsetStr;
320            if (header.offset != null) {
321                offsetStr = header.offset.toString();
322            } else if (header.offsetStr != null && header.offsetStr.length() > 0) {
323                offsetStr = header.offsetStr;
324            } else {
325                offsetStr = "-";
326            }
327            outBuf.write(" ".getBytes());
328            outBuf.write(offsetStr.getBytes());
329            /*
330             * Filename
331             */
332            String filenameStr;
333            if (header.filenameStr != null && header.filenameStr.length() > 0) {
334                filenameStr = header.filenameStr;
335            } else {
336                filenameStr = "-";
337            }
338            outBuf.write(" ".getBytes());
339            outBuf.write(filenameStr.getBytes());
340        }
341        /*
342         * Archive-Length
343         */
344        String archiveLengthStr;
345        if (header.archiveLength != null) {
346            archiveLengthStr = header.archiveLength.toString();
347        } else if (header.archiveLengthStr != null && header.archiveLengthStr.length() > 0) {
348            archiveLengthStr = header.archiveLengthStr;
349        } else {
350            archiveLengthStr = "-";
351        }
352        outBuf.write(" ".getBytes());
353        outBuf.write(archiveLengthStr.getBytes());
354        outBuf.write("\n".getBytes());
355        /*
356         * End Of Header
357         */
358        byte[] headerBytes = outBuf.toByteArray();
359        out.write(headerBytes);
360        state = S_HEADER_WRITTEN;
361        payloadWrittenTotal = 0;
362        return headerBytes;
363    }
364
365    /**
366     * Stream the content of an input stream to the payload content.
367     * @param in input stream containing payload data
368     * @return number of bytes written during method invocation
369     * @throws IOException if an I/O exception occurs while writing payload data
370     */
371   public long streamPayload(InputStream in) throws IOException {
372       if (in == null) {
373           throw new IllegalArgumentException(
374                   "The 'in' parameter is null!");
375       }
376       if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) {
377           throw new IllegalStateException("Write a header before writing payload!");
378       }
379       long written = 0;
380       int read = 0;
381       while (read != -1) {
382           out.write(stream_copy_buffer, 0, read);
383           written += read;
384           read = in.read(stream_copy_buffer);
385       }
386       state = S_PAYLOAD_WRITTEN;
387       payloadWrittenTotal += written;
388       return written;
389   }
390
391   /**
392    * Append the content of a byte array to the payload content.
393    * @param b byte array with data to be written
394    * @return number of bytes written during method invocation
395    * @throws IOException if an I/O exception occurs while writing payload data
396    */
397   public long writePayload(byte[] b) throws IOException {
398       if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) {
399           throw new IllegalStateException("Write a header before writing payload!");
400       }
401       out.write(b);
402       state = S_PAYLOAD_WRITTEN;
403       payloadWrittenTotal += b.length;
404       return b.length;
405   }
406
407   /**
408    * Append the partial content of a byte array to the payload content.
409    * @param b byte array with partial data to be written
410    * @param offset offset to data to be written
411    * @param len length of data to be written
412    * @return number of bytes written during method invocation
413    * @throws IOException if an I/O exception occurs while writing payload data
414    */
415   public long writePayload(byte[] b, int offset, int len) throws IOException {
416       if (state != S_HEADER_WRITTEN && state != S_PAYLOAD_WRITTEN) {
417           throw new IllegalStateException("Write a header before writing payload!");
418       }
419       out.write(b, offset, len);
420       state = S_PAYLOAD_WRITTEN;
421       payloadWrittenTotal += len;
422       return len;
423   }
424
425}