001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.arc; 019 020import java.io.ByteArrayInputStream; 021import java.io.ByteArrayOutputStream; 022import java.io.IOException; 023import java.io.InputStream; 024 025import org.jwat.common.ByteCountingPushBackInputStream; 026import org.jwat.common.Diagnosis; 027import org.jwat.common.DiagnosisType; 028import org.jwat.common.Diagnostics; 029import org.jwat.common.Payload; 030 031/** 032 * This class represents an ARC version block and header including possible 033 * validation and format warnings/errors encountered in the process. 034 * This class also contains the specific ARC version block payload parser. 035 * Any metadata present in the ARC version block is accessible 036 * through the payload object. 037 * 038 * @author lbihanic, selghissassi, nicl 039 */ 040public class ArcVersionBlock extends ArcRecordBase { 041 042 /** Buffer size used in toString(). */ 043 public static final int TOSTRING_BUFFER_SIZE = 256; 044 045 /** 046 * Protected constructor to force instantiation of version block 047 * from stream. 048 */ 049 protected ArcVersionBlock() { 050 } 051 052 /** 053 * Create and initialize a new <code>ArcVersionBlock</code> for writing. 054 * @param writer writer which shall be used 055 * @return an <code>ArcVersionBlock</code> prepared for writing 056 */ 057 public static ArcVersionBlock createRecord(ArcWriter writer) { 058 ArcVersionBlock vb = new ArcVersionBlock(); 059 vb.trailingNewLines = 1; 060 vb.diagnostics = new Diagnostics<Diagnosis>(); 061 vb.header = ArcHeader.initHeader(writer, vb.diagnostics); 062 writer.fieldParsers.diagnostics = vb.diagnostics; 063 return vb; 064 } 065 066 /** 067 * Creates a new <code>VersionBlock</code> based on the supplied header and 068 * the version block in the payload, if present. 069 * @param reader <code>ArcReader</code> used, with access to user defined 070 * options 071 * @param diagnostics diagnostics used to report errors and/or warnings 072 * @param header record header that has already been processed 073 * @param fieldParsers parser used to read and validate fields 074 * @param in <code>InputStream</code> used to read version block 075 * @return an <code>ArcVersionBlock</code> 076 * @throws IOException if an I/O exception occurs in the process of reading 077 * version block 078 */ 079 public static ArcVersionBlock parseVersionBlock(ArcReader reader, 080 Diagnostics<Diagnosis> diagnostics, 081 ArcHeader header, ArcFieldParsers fieldParsers, 082 ByteCountingPushBackInputStream in) throws IOException { 083 ArcVersionBlock vb = new ArcVersionBlock(); 084 vb.recordType = RT_VERSION_BLOCK; 085 vb.reader = reader; 086 vb.diagnostics = diagnostics; 087 vb.header = header; 088 vb.in = in; 089 vb.processPayload(in, reader); 090 vb.consumed = in.getConsumed() - vb.header.startOffset; 091 return vb; 092 } 093 094 /** 095 * Validates the version block content type. 096 * Errors and/or warnings are reported on the diagnostics object. 097 */ 098 protected void validateContentType() { 099 if (header.contentType == null) { 100 // Version block content-type is required. 101 diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, 102 "'" + ArcConstants.FN_CONTENT_TYPE + "' value", 103 ArcConstants.CONTENT_TYPE_FORMAT)); 104 } else if (!ArcConstants.VERSION_BLOCK_CONTENT_TYPE.equals( 105 header.contentType.contentType) || 106 !ArcConstants.VERSION_BLOCK_MEDIA_TYPE.equals(header.contentType.mediaType)) { 107 // Version block content-type should be equal to "text/plain" 108 diagnostics.addWarning(new Diagnosis(DiagnosisType.INVALID_EXPECTED, 109 "'" + ArcConstants.FN_CONTENT_TYPE + "' value", 110 header.contentTypeStr, 111 ArcConstants.CONTENT_TYPE_TEXT_PLAIN)); 112 } 113 } 114 115 /** 116 * An ARC version block payload must include a version line and a 117 * block description line and in v1.1 the rest of the payload consists 118 * of XML formatted metadata related to the harvesters configuration. 119 * @param in input stream containing the payload 120 * @param reader <code>ArcReader</code> used, with access to user defined 121 * options 122 * @throws IOException if an I/O exception occurs while reading the payload 123 */ 124 @Override 125 protected void processPayload(ByteCountingPushBackInputStream in, 126 ArcReader reader) throws IOException { 127 payload = null; 128 validateContentType(); 129 if (header.archiveLength != null && header.archiveLength > 0L) { 130 String digestAlgorithm = null; 131 if (reader.bBlockDigest) { 132 digestAlgorithm = reader.blockDigestAlgorithm; 133 } 134 payload = Payload.processPayload(in, header.archiveLength.longValue(), 135 reader.payloadHeaderMaxSize, digestAlgorithm); 136 payload.setOnClosedHandler(this); 137 // ArcVersionHeader. 138 digestAlgorithm = null; 139 if (reader.bPayloadDigest) { 140 digestAlgorithm = reader.payloadDigestAlgorithm; 141 } 142 // Try to read a valid ARC version block from the payload. 143 versionHeader = ArcVersionHeader.processPayload( 144 payload.getInputStream(), header.archiveLength.longValue(), 145 digestAlgorithm, reader.fieldParsers, diagnostics); 146 if (versionHeader != null) { 147 version = versionHeader.version; 148 if (versionHeader.isValid()) { 149 payload.setPayloadHeaderWrapped(versionHeader); 150 } else { 151 diagnostics.addError( 152 new Diagnosis(DiagnosisType.ERROR, 153 ArcConstants.ARC_VERSION_BLOCK, 154 "Version block is not valid!")); 155 } 156 } 157 } else { 158 diagnostics.addError( 159 new Diagnosis(DiagnosisType.INVALID, 160 ArcConstants.ARC_FILE, 161 "VersionBlock length missing!")); 162 } 163 if (versionHeader != null && versionHeader.isValid()) { 164 if (ArcVersion.VERSION_1_1.equals(version)) { 165 if ((versionHeader.getRemaining() == 0)) { 166 bHasPseudoEmptyPayload = true; 167 diagnostics.addError(new Diagnosis(DiagnosisType.ERROR_EXPECTED, 168 ArcConstants.ARC_FILE, 169 "Expected metadata payload not found in the version block")); 170 } 171 } else { 172 if (versionHeader.getRemaining() == 0) { 173 bHasPseudoEmptyPayload = true; 174 } else { 175 if (!reader.bStrict) { 176 // I'm going on a limb here that IA's ARC writer will 177 // not write in excess of 4GB useless newlines. 178 if (versionHeader.getRemaining() > ArcConstants.ARC_VB_MAX_TRAILING_NEWLINES) { 179 throw new IOException( 180 "This amount of useless data in the version block is unacceptable!"); 181 } 182 ByteArrayOutputStream out_payload = new ByteArrayOutputStream( 183 (int)versionHeader.getRemaining()); 184 InputStream in_payload = versionHeader.getPayloadInputStream(); 185 int read; 186 byte[] tmpBuf = new byte[1024]; 187 while ((read = in_payload.read(tmpBuf)) != -1) { 188 out_payload.write(tmpBuf, 0, read); 189 } 190 in_payload.close(); 191 out_payload.close(); 192 excessiveMetadata = out_payload.toByteArray(); 193 ByteArrayInputStream in_newlines = new ByteArrayInputStream(excessiveMetadata); 194 if (!isValidStreamOfCRLF(in_newlines)) { 195 diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA, 196 "version block metadata payload", 197 "Metadata payload must not be present in this version")); 198 } else { 199 bHasPseudoEmptyPayload = true; 200 } 201 in_newlines.close(); 202 } else { 203 diagnostics.addError(new Diagnosis(DiagnosisType.UNDESIRED_DATA, 204 "version block metadata payload", 205 "Metadata payload must not be present in this version")); 206 } 207 } 208 } 209 } 210 } 211 212 @Override 213 public String toString() { 214 StringBuilder builder = new StringBuilder(TOSTRING_BUFFER_SIZE); 215 builder.append("\nVersionBlock : [\n"); 216 builder.append(super.toString()); 217 builder.append("]\n"); 218 if (versionHeader != null) { 219 builder.append(versionHeader.toString()); 220 } 221 return builder.toString(); 222 } 223 224}