001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.arc; 019 020import java.io.BufferedInputStream; 021import java.io.IOException; 022import java.io.InputStream; 023 024import org.jwat.common.ByteCountingPushBackInputStream; 025import org.jwat.gzip.GzipEntry; 026import org.jwat.gzip.GzipReader; 027 028/** 029 * ARC Reader implementation for reading GZip compressed files. 030 * 031 * @author nicl 032 */ 033public class ArcReaderCompressed extends ArcReader { 034 035 /** Buffer size used by <code>PushbackInputStream</code>. */ 036 public static final int PUSHBACK_BUFFER_SIZE = 32; 037 038 /** ARC file <code>GzipInputStream</code>. */ 039 protected GzipReader reader; 040 041 /** Buffer size, if any, to use on GZip entry <code>InputStream</code>. */ 042 protected int bufferSize; 043 044 /** GZip reader used for the current record, if random access methods used. */ 045 protected GzipReader currentReader; 046 047 /** GZip entry for the current record, if random access methods used. */ 048 protected GzipEntry currentEntry; 049 050 /** 051 * This constructor is used to get random access to records. 052 * The records are then accessed using the getNextRecordFrom methods 053 * using a supplied input stream for each record. 054 */ 055 public ArcReaderCompressed() { 056 init(); 057 } 058 059 /** 060 * Construct reader using the supplied input stream. 061 * This method is primarily for sequential access to records. 062 * @param reader GZip reader 063 */ 064 public ArcReaderCompressed(GzipReader reader) { 065 if (reader == null) { 066 throw new IllegalArgumentException("'reader' is null"); 067 } 068 this.reader = reader; 069 init(); 070 } 071 072 /** 073 * Construct object using supplied <code>GzipInputStream</code>. 074 * This method is primarily for sequential access to records. 075 * @param reader GZip reader 076 * @param buffer_size buffer size used on entries 077 */ 078 public ArcReaderCompressed(GzipReader reader, int buffer_size) { 079 if (reader == null) { 080 throw new IllegalArgumentException("'reader' is null"); 081 } 082 if (buffer_size <= 0) { 083 throw new IllegalArgumentException( 084 "The 'buffer_size' is less than or equal to zero: " 085 + buffer_size); 086 } 087 this.reader = reader; 088 this.bufferSize = buffer_size; 089 init(); 090 } 091 092 @Override 093 public boolean isCompressed() { 094 return true; 095 } 096 097 @Override 098 public void close() { 099 if (currentRecord != null) { 100 try { 101 currentRecord.close(); 102 } catch (IOException e) { /* ignore */ } 103 currentRecord = null; 104 } 105 if (reader != null) { 106 startOffset = reader.getStartOffset(); 107 consumed = reader.getOffset(); 108 try { 109 reader.close(); 110 } catch (IOException e) { /* ignore */ } 111 reader = null; 112 } 113 } 114 115 @Override 116 protected void recordClosed() { 117 if (currentEntry != null) { 118 try { 119 currentEntry.close(); 120 consumed += currentEntry.consumed; 121 } catch (IOException e) { /* ignore */ } 122 currentEntry = null; 123 } else { 124 throw new IllegalStateException("'currentEntry' is null, this should never happen!"); 125 } 126 } 127 128 /** Cached start offset used after the reader is closed. */ 129 protected long startOffset = -1; 130 131 /** 132 * Get the offset of the current ARC record from the GZip entry or -1 if 133 * no records have been read yet. 134 * @return offset of the current ARC record from the GZip entry or -1 135 */ 136 @Override 137 public long getStartOffset() { 138 if (reader != null) { 139 return reader.getStartOffset(); 140 } else { 141 return startOffset; 142 } 143 } 144 145 /** 146 * Get the current offset in the ARC <code>GzipReader</code>. 147 * @return offset in ARC <code>InputStream</code> 148 */ 149 @Override 150 public long getOffset() { 151 if (reader != null) { 152 return reader.getOffset(); 153 } else { 154 return consumed; 155 } 156 } 157 158 /** Get number of bytes consumed by the ARC <code>GzipReader</code>. 159 * @return number of bytes consumed by the ARC <code>GzipReader</code> 160 */ 161 @Override 162 public long getConsumed() { 163 if (reader != null) { 164 return reader.getOffset(); 165 } else { 166 return consumed; 167 } 168 } 169 170 @Override 171 public ArcRecordBase getNextRecord() throws IOException { 172 if (currentRecord != null) { 173 currentRecord.close(); 174 } 175 if (reader == null) { 176 throw new IllegalStateException( 177 "This reader has been initialized with an incompatible constructor, 'reader' is null"); 178 } 179 currentRecord = null; 180 currentReader = reader; 181 currentEntry = reader.getNextEntry(); 182 if (currentEntry != null) { 183 ByteCountingPushBackInputStream pbin; 184 if (bufferSize > 0) { 185 pbin = new ByteCountingPushBackInputStream( 186 new BufferedInputStream( 187 currentEntry.getInputStream(), 188 bufferSize), 189 PUSHBACK_BUFFER_SIZE); 190 } else { 191 pbin = new ByteCountingPushBackInputStream( 192 currentEntry.getInputStream(), PUSHBACK_BUFFER_SIZE); 193 } 194 currentRecord = ArcRecordBase.parseRecord(pbin, this); 195 } 196 if (currentRecord != null) { 197 startOffset = currentEntry.getStartOffset(); 198 currentRecord.header.startOffset = currentEntry.getStartOffset(); 199 } 200 return currentRecord; 201 } 202 203 @Override 204 public ArcRecordBase getNextRecordFrom(InputStream rin, long offset) 205 throws IOException { 206 if (currentRecord != null) { 207 currentRecord.close(); 208 } 209 if (reader != null) { 210 throw new IllegalStateException( 211 "This reader has been initialized with an incompatible constructor, 'reader' is not null"); 212 } 213 if (rin == null) { 214 throw new IllegalArgumentException( 215 "The inputstream 'rin' is null"); 216 } 217 if (offset < -1) { 218 throw new IllegalArgumentException( 219 "The 'offset' is less than -1: " + offset); 220 } 221 currentRecord = null; 222 currentReader = new GzipReader(rin); 223 currentEntry = currentReader.getNextEntry(); 224 if (currentEntry != null) { 225 ByteCountingPushBackInputStream pbin = 226 new ByteCountingPushBackInputStream( 227 currentEntry.getInputStream(), PUSHBACK_BUFFER_SIZE); 228 currentRecord = ArcRecordBase.parseRecord(pbin, this); 229 } 230 if (currentRecord != null) { 231 startOffset = offset; 232 currentRecord.header.startOffset = offset; 233 } 234 return currentRecord; 235 } 236 237 @Override 238 public ArcRecordBase getNextRecordFrom(InputStream rin, long offset, 239 int buffer_size) throws IOException { 240 if (currentRecord != null) { 241 currentRecord.close(); 242 } 243 if (reader != null) { 244 throw new IllegalStateException( 245 "This reader has been initialized with an incompatible constructor, 'reader' is not null"); 246 } 247 if (rin == null) { 248 throw new IllegalArgumentException( 249 "The inputstream 'rin' is null"); 250 } 251 if (offset < -1) { 252 throw new IllegalArgumentException( 253 "The 'offset' is less than -1: " + offset); 254 } 255 if (buffer_size <= 0) { 256 throw new IllegalArgumentException( 257 "The 'buffer_size' is less than or equal to zero: " 258 + buffer_size); 259 } 260 currentRecord = null; 261 currentReader = new GzipReader(rin); 262 currentEntry = currentReader.getNextEntry(); 263 if (currentEntry != null) { 264 ByteCountingPushBackInputStream pbin = 265 new ByteCountingPushBackInputStream( 266 new BufferedInputStream( 267 currentEntry.getInputStream(), 268 buffer_size), 269 PUSHBACK_BUFFER_SIZE); 270 currentRecord = ArcRecordBase.parseRecord(pbin, this); 271 } 272 if (currentRecord != null) { 273 startOffset = offset; 274 currentRecord.header.startOffset = offset; 275 } 276 return currentRecord; 277 } 278 279}