001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.io.File; 021import java.io.IOException; 022import java.io.RandomAccessFile; 023import java.util.UUID; 024 025import org.jwat.common.RandomAccessFileOutputStream; 026import org.jwat.common.Uri; 027 028/** 029 * Simple WARC file writer wrapping some of the trivial code related to writing records. 030 * Handles automatic max file size closing and renaming of old file and opening of new file. 031 * The preferred workflow is to class nextWriter() and close(). Using open() does not expose a clean internal state. 032 * 033 * @author nicl 034 */ 035public class WarcFileWriter { 036 037 /** Suffix used for open files. */ 038 public static final String ACTIVE_SUFFIX = ".open"; 039 040 /** Overall WARC file writer configuration. */ 041 protected WarcFileWriterConfig warcFileConfig; 042 043 /** WARC file naming Configuration. */ 044 protected WarcFileNaming warcFileNaming; 045 046 /** Current sequence number. */ 047 protected int sequenceNr = -1; 048 049 /** Current WARC file. */ 050 protected File writerFile; 051 052 /** Current random access file. */ 053 protected RandomAccessFile writer_raf; 054 055 /** Current random access output stream. */ 056 protected RandomAccessFileOutputStream writer_rafout; 057 058 /** Current WARC writer. */ 059 public WarcWriter writer; 060 061 /** Generated WARC-Info-Record-ID for the current file. */ 062 public Uri warcinfoRecordId; 063 064 /** 065 * Constructor for internal and unit test use. 066 */ 067 protected WarcFileWriter() { 068 } 069 070 /** 071 * Returns a configured WARC file writer. 072 * @param warcFileNaming WARC file naming configuration 073 * @param warcFileConfig overall WARC writer configuration 074 * @return WARC file writer instance using the supplied configuration 075 */ 076 public static WarcFileWriter getWarcWriterInstance(WarcFileNaming warcFileNaming, WarcFileWriterConfig warcFileConfig) { 077 if (warcFileNaming == null) { 078 throw new IllegalArgumentException("'warcFileNaming' is null!"); 079 } 080 if (warcFileConfig == null) { 081 throw new IllegalArgumentException("'warcFileConfig' is null!"); 082 } 083 WarcFileWriter wfw = new WarcFileWriter(); 084 wfw.warcFileNaming = warcFileNaming; 085 wfw.warcFileConfig = warcFileConfig; 086 return wfw; 087 } 088 089 /** 090 * Returns the current sequence number. 091 * @return the current sequence number 092 */ 093 public int getSequenceNr() { 094 return sequenceNr; 095 } 096 097 /** 098 * Returns the current WARC file object. 099 * @return current EARC file object 100 */ 101 public File getFile() { 102 return writerFile; 103 } 104 105 /** 106 * Returns the current WARC writer object. 107 * @return current WARC writer object 108 */ 109 public WarcWriter getWriter() { 110 return writer; 111 } 112 113 /** 114 * Open new file with active prefix and prepare for writing. 115 * @throws IOException if an I/O exception occurs while opening file 116 */ 117 public void open() throws IOException { 118 if (writer == null) { 119 ++sequenceNr; 120 String finishedFilename = warcFileNaming.getFilename(sequenceNr, warcFileConfig.bCompression); 121 String activeFilename = finishedFilename + ACTIVE_SUFFIX; 122 File finishedFile = new File(warcFileConfig.targetDir, finishedFilename); 123 writerFile = new File(warcFileConfig.targetDir, activeFilename); 124 if (writerFile.exists()) { 125 if (warcFileConfig.bOverwrite) { 126 writerFile.delete(); 127 } else { 128 throw new IOException("'" + writerFile + "' already exists, will not overwrite"); 129 } 130 } 131 if (finishedFile.exists()) { 132 if (warcFileConfig.bOverwrite) { 133 finishedFile.delete(); 134 } else { 135 throw new IOException("'" + finishedFile + "' already exists, will not overwrite"); 136 } 137 } 138 writer_raf = new RandomAccessFile(writerFile, "rw"); 139 writer_raf.seek(0L); 140 writer_raf.setLength(0L); 141 writer_rafout = new RandomAccessFileOutputStream(writer_raf); 142 writer = WarcWriterFactory.getWriter(writer_rafout, 8192, warcFileConfig.bCompression); 143 } 144 } 145 146 /** 147 * Checks to see whether a new file needs to be created. Depending on the configuration this also checks if the max file size has been reached and closes/renames the old file and opens a new one. 148 * @return boolean indicating whether new writer/file was created 149 * @throws Exception if an exception occurs 150 */ 151 public boolean nextWriter() throws Exception { 152 boolean bNewWriter = false; 153 if (writer_raf == null) { 154 bNewWriter = true; 155 } else if (warcFileNaming.supportMultipleFiles() && writer_raf.length() > warcFileConfig.maxFileSize) { 156 close(); 157 bNewWriter = true; 158 } 159 if (bNewWriter) { 160 open(); 161 warcinfoRecordId = new Uri("urn:uuid:" + UUID.randomUUID()); 162 } 163 return bNewWriter; 164 } 165 166 /** 167 * Close writer and release all resources. 168 * @throws IOException in an I/O exception occurs while closing resources 169 */ 170 public void close() throws IOException { 171 if (writer != null) { 172 writer.close(); 173 writer = null; 174 } 175 if (writer_rafout != null) { 176 writer_rafout.close(); 177 writer_rafout = null; 178 } 179 if (writer_raf != null) { 180 writer_raf.close(); 181 writer_raf = null; 182 } 183 warcinfoRecordId = null; 184 if (writerFile != null && writerFile.getName().endsWith(ACTIVE_SUFFIX)) { 185 String finishedName = writerFile.getName().substring(0, writerFile.getName().length() - ACTIVE_SUFFIX.length()); 186 File finishedFile = new File(writerFile.getParent(), finishedName); 187 if (finishedFile.exists()) { 188 throw new IOException("Unable to rename '" + writerFile + "' to '" + finishedFile + "' - destination file already exists"); 189 } 190 boolean success = writerFile.renameTo(finishedFile); 191 if (!success) { 192 throw new IOException("Unable to rename '" + writerFile + "' to '" + finishedFile + "' - unknown problem"); 193 } 194 } 195 writerFile = null; 196 } 197 198}