001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.warc;
019
020import java.io.File;
021import java.io.IOException;
022import java.io.RandomAccessFile;
023import java.util.UUID;
024
025import org.jwat.common.RandomAccessFileOutputStream;
026import org.jwat.common.Uri;
027
028/**
029 * Simple WARC file writer wrapping some of the trivial code related to writing records.
030 * Handles automatic max file size closing and renaming of old file and opening of new file.
031 * The preferred workflow is to class nextWriter() and close(). Using open() does not expose a clean internal state.
032 *
033 * @author nicl
034 */
035public class WarcFileWriter {
036
037    /** Suffix used for open files. */
038    public static final String ACTIVE_SUFFIX = ".open";
039
040    /** Overall WARC file writer configuration. */
041    protected WarcFileWriterConfig warcFileConfig;
042
043    /** WARC file naming Configuration. */
044    protected WarcFileNaming warcFileNaming;
045
046    /** Current sequence number. */
047    protected int sequenceNr = -1;
048
049    /** Current WARC file. */
050    protected File writerFile;
051
052    /** Current random access file. */
053    protected RandomAccessFile writer_raf;
054
055    /** Current random access output stream. */
056    protected RandomAccessFileOutputStream writer_rafout;
057
058    /** Current WARC writer. */
059    public WarcWriter writer;
060
061    /** Generated WARC-Info-Record-ID for the current file. */
062    public Uri warcinfoRecordId;
063
064    /**
065     * Constructor for internal and unit test use.
066     */
067    protected WarcFileWriter() {
068    }
069
070    /**
071     * Returns a configured WARC file writer.
072     * @param warcFileNaming WARC file naming configuration
073     * @param warcFileConfig overall WARC writer configuration
074     * @return WARC file writer instance using the supplied configuration
075     */
076    public static WarcFileWriter getWarcWriterInstance(WarcFileNaming warcFileNaming, WarcFileWriterConfig warcFileConfig) {
077        if (warcFileNaming == null) {
078            throw new IllegalArgumentException("'warcFileNaming' is null!");
079        }
080        if (warcFileConfig == null) {
081            throw new IllegalArgumentException("'warcFileConfig' is null!");
082        }
083        WarcFileWriter wfw = new WarcFileWriter();
084        wfw.warcFileNaming = warcFileNaming;
085        wfw.warcFileConfig = warcFileConfig;
086        return wfw;
087    }
088
089    /**
090     * Returns the current sequence number.
091     * @return the current sequence number
092     */
093    public int getSequenceNr() {
094        return sequenceNr;
095    }
096
097    /**
098     * Returns the current WARC file object.
099     * @return current EARC file object
100     */
101    public File getFile() {
102        return writerFile;
103    }
104
105    /**
106     * Returns the current WARC writer object.
107     * @return current WARC writer object
108     */
109    public WarcWriter getWriter() {
110        return writer;
111    }
112
113    /**
114     * Open new file with active prefix and prepare for writing.
115     * @throws IOException if an I/O exception occurs while opening file
116     */
117    public void open() throws IOException {
118        if (writer == null) {
119            ++sequenceNr;
120            String finishedFilename = warcFileNaming.getFilename(sequenceNr, warcFileConfig.bCompression);
121            String activeFilename = finishedFilename + ACTIVE_SUFFIX;
122            File finishedFile = new File(warcFileConfig.targetDir, finishedFilename);
123            writerFile = new File(warcFileConfig.targetDir, activeFilename);
124            if (writerFile.exists()) {
125                if (warcFileConfig.bOverwrite) {
126                    writerFile.delete();
127                } else {
128                    throw new IOException("'" + writerFile + "' already exists, will not overwrite");
129                }
130            }
131            if (finishedFile.exists()) {
132                if (warcFileConfig.bOverwrite) {
133                    finishedFile.delete();
134                } else {
135                    throw new IOException("'" + finishedFile + "' already exists, will not overwrite");
136                }
137            }
138            writer_raf = new RandomAccessFile(writerFile, "rw");
139            writer_raf.seek(0L);
140            writer_raf.setLength(0L);
141            writer_rafout = new RandomAccessFileOutputStream(writer_raf);
142            writer = WarcWriterFactory.getWriter(writer_rafout, 8192, warcFileConfig.bCompression);
143        }
144    }
145
146    /**
147     * Checks to see whether a new file needs to be created. Depending on the configuration this also checks if the max file size has been reached and closes/renames the old file and opens a new one.
148     * @return boolean indicating whether new writer/file was created
149     * @throws Exception if an exception occurs
150     */
151    public boolean nextWriter() throws Exception {
152        boolean bNewWriter = false;
153        if (writer_raf == null) {
154            bNewWriter = true;
155        } else if (warcFileNaming.supportMultipleFiles() && writer_raf.length() > warcFileConfig.maxFileSize) {
156            close();
157            bNewWriter = true;
158        }
159        if (bNewWriter) {
160            open();
161            warcinfoRecordId = new Uri("urn:uuid:" + UUID.randomUUID());
162        }
163        return bNewWriter;
164    }
165
166    /**
167     * Close writer and release all resources.
168     * @throws IOException in an I/O exception occurs while closing resources
169     */
170    public void close() throws IOException {
171        if (writer != null) {
172            writer.close();
173            writer = null;
174        }
175        if (writer_rafout != null) {
176            writer_rafout.close();
177            writer_rafout = null;
178        }
179        if (writer_raf != null) {
180            writer_raf.close();
181            writer_raf = null;
182        }
183        warcinfoRecordId = null;
184        if (writerFile != null && writerFile.getName().endsWith(ACTIVE_SUFFIX)) {
185            String finishedName = writerFile.getName().substring(0, writerFile.getName().length() - ACTIVE_SUFFIX.length());
186            File finishedFile = new File(writerFile.getParent(), finishedName);
187            if (finishedFile.exists()) {
188                throw new IOException("Unable to rename '" + writerFile + "' to '" + finishedFile + "' - destination file already exists");
189            }
190            boolean success = writerFile.renameTo(finishedFile);
191            if (!success) {
192                throw new IOException("Unable to rename '" + writerFile + "' to '" + finishedFile + "' - unknown problem");
193            }
194        }
195        writerFile = null;
196    }
197
198}