001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.arc; 019 020import java.io.IOException; 021import java.net.InetAddress; 022import java.util.Date; 023 024import org.jwat.common.ByteCountingPushBackInputStream; 025import org.jwat.common.ContentType; 026import org.jwat.common.Diagnosis; 027import org.jwat.common.DiagnosisType; 028import org.jwat.common.Diagnostics; 029import org.jwat.common.HeaderLine; 030import org.jwat.common.Scheme; 031import org.jwat.common.Uri; 032import org.jwat.common.UriProfile; 033 034/** 035 * Class for parsing and validating the common ARC record header present in 036 * both ARC records and ARC version blocks. 037 * 038 * @author nicl 039 */ 040public class ArcHeader { 041 042 /** Associated WarcReader context. 043 * Must be set prior to calling the various methods. */ 044 protected ArcReader reader; 045 046 /** Diagnostics used to report diagnoses. 047 * Must be set prior to calling the various methods. */ 048 protected Diagnostics<Diagnosis> diagnostics; 049 050 /** URL URI profile. */ 051 protected UriProfile uriProfile; 052 053 /** ARC field parser used. 054 * Must be set prior to calling the various methods. */ 055 protected ArcFieldParsers fieldParsers; 056 057 /** ARC record starting offset relative to the source ARC file input 058 * stream. The offset is correct for both compressed and uncompressed streams. */ 059 protected long startOffset = -1; 060 061 /* 062 * Version related fields. 063 */ 064 065 /** Which version of the record fields was parsed, 1.x or 2.0. */ 066 public int recordFieldVersion; 067 068 /* 069 * ARC header fields. 070 */ 071 072 /** Do the record fields comply in number with the one dictated by its 073 * version. */ 074 protected boolean hasCompliantFields = false; 075 076 /** ARC record URL field string value. */ 077 public String urlStr; 078 /** ARC record URL validated and converted to an <code>URI</code> object. */ 079 public Uri urlUri; 080 /** URI Scheme (lowercase). (filedesc, http, https, dns, etc.) */ 081 public String urlScheme; 082 083 /** ARC record IP-Address field string value. */ 084 public String ipAddressStr; 085 /** IP-Address validated and converted to an <code>InetAddress</code> object. */ 086 public InetAddress inetAddress; 087 088 /** ARC record archive-date field string value. */ 089 public String archiveDateStr; 090 /** Archive date validated and converted to a <code>Date</code> object. */ 091 public Date archiveDate; 092 093 /** ARC record content-type field string value. */ 094 public String contentTypeStr; 095 /** Content-Type wrapper object with optional parameters. */ 096 public ContentType contentType; 097 098 /** ARC record result-code field string value. */ 099 public String resultCodeStr; 100 /** Result-code validated and converted into an <code>Integer</code> object. */ 101 public Integer resultCode; 102 103 /** ARC record checksum field string value. */ 104 public String checksumStr; 105 106 /** ARC record location field string value. */ 107 public String locationStr; 108 109 /** ARC record offset field string value. */ 110 public String offsetStr; 111 /** Offset validated and converted into a <code>Long</code> object. */ 112 public Long offset; 113 114 /** ARC record filename field string value. */ 115 public String filenameStr; 116 117 /** ARC record archive-length field string value. */ 118 public String archiveLengthStr; 119 /** Archive-length validated and converted into a <code>Long</code> object. */ 120 public Long archiveLength; 121 122 /** Raw ARC header line byte array. */ 123 public byte[] headerBytes; 124 125 /** 126 * Non public constructor to allow unit testing. 127 */ 128 protected ArcHeader() { 129 } 130 131 /** 132 * Create a new <code>ArcHeader</code> for recreating a header object. 133 * @param uriProfile uri profile used to validate urls 134 * @param fieldParsers parsers used for the individual types 135 * @param diagnostics diagnostics object used by parser 136 * @return a <code>ArcHeader</code> prepared for writing 137 */ 138 public static ArcHeader initHeader(ArcFieldParsers fieldParsers, UriProfile uriProfile, Diagnostics<Diagnosis> diagnostics) { 139 ArcHeader header = new ArcHeader(); 140 header.fieldParsers = fieldParsers; 141 header.uriProfile = uriProfile; 142 header.fieldParsers = fieldParsers; 143 header.diagnostics = diagnostics; 144 return header; 145 } 146 147 /** 148 * Create and initialize a new <code>ArcHeader</code> for writing. 149 * @param writer writer which shall be used 150 * @param diagnostics diagnostics object used by writer 151 * @return a <code>ArcHeader</code> prepared for writing 152 */ 153 public static ArcHeader initHeader(ArcWriter writer, Diagnostics<Diagnosis> diagnostics) { 154 ArcHeader header = new ArcHeader(); 155 header.fieldParsers = writer.fieldParsers; 156 header.diagnostics = diagnostics; 157 return header; 158 } 159 160 /** 161 * Create and initialize a new <code>ArcHeader</code> for reading. 162 * @param reader reader which shall be used 163 * @param startOffset start offset of header 164 * @param diagnostics diagnostics object used by reader 165 * @return a <code>ArcHeader</code> prepared for reading 166 */ 167 public static ArcHeader initHeader(ArcReader reader, long startOffset, Diagnostics<Diagnosis> diagnostics) { 168 ArcHeader header = new ArcHeader(); 169 header.reader = reader; 170 header.uriProfile = reader.uriProfile; 171 header.fieldParsers = reader.fieldParsers; 172 header.diagnostics = diagnostics; 173 // This is only relevant for uncompressed sequentially read records 174 header.startOffset = startOffset; 175 return header; 176 } 177 178 /** 179 * Tries to read lines from the input stream to determine if they could 180 * be possible ARC record header lines. Returns true if a line was 181 * encountered with delimited fields matching either a v1.x or v2.0 182 * description block. False is no ARC record header candidate was found 183 * in the remaining input stream. 184 * @param in input stream supposedly containing ARC records 185 * @return true if an ARC record header was parsed and validated 186 * @throws IOException if an I/O exception occurs while trying to read an 187 * ARC record header 188 */ 189 public boolean parseHeader(ByteCountingPushBackInputStream in) throws IOException { 190 boolean bHeaderParsed = false; 191 boolean bInvalidDataBeforeVersion = false; 192 boolean bEmptyLinesBeforeVersion = false; 193 boolean bSeekRecord = true; 194 while (bSeekRecord) { 195 startOffset = in.getConsumed(); 196 HeaderLine recordLine = HeaderLine.readLine(in); 197 if (recordLine != null) { 198 if (recordLine.line.length() > 0) { 199 String[] fields = recordLine.line.split(" ", -1); 200 if (fields.length == ArcConstants.VERSION_1_BLOCK_NUMBER_FIELDS 201 || fields.length == ArcConstants.VERSION_2_BLOCK_NUMBER_FIELDS) { 202 // debug 203 //System.out.println(recordLine); 204 parseHeaders(fields); 205 bHeaderParsed = true; 206 headerBytes = recordLine.raw; 207 bSeekRecord = false; 208 } else { 209 bInvalidDataBeforeVersion = true; 210 } 211 } else { 212 bEmptyLinesBeforeVersion = true; 213 } 214 } 215 else { 216 bSeekRecord = false; 217 } 218 } 219 if (bInvalidDataBeforeVersion) { 220 diagnostics.addError(new Diagnosis(DiagnosisType.INVALID, "Data before ARC record")); 221 } 222 if (bEmptyLinesBeforeVersion) { 223 diagnostics.addError(new Diagnosis(DiagnosisType.INVALID, "Empty lines before ARC record")); 224 } 225 return bHeaderParsed; 226 } 227 228 /** 229 * Examines an array of strings to determine if it could be an ARC record 230 * header. If the number of fields is not equal to that of either v1.x or 231 * v2.0 nothing is parsed. 232 * @param fields array of possible ARC record header fields 233 */ 234 public void parseHeaders(String[] fields) { 235 if (fields.length == ArcConstants.VERSION_1_BLOCK_FIELDS.length 236 || fields.length == ArcConstants.VERSION_2_BLOCK_FIELDS.length) { 237 recordFieldVersion = 1; 238 /* 239 * Version 1. 240 */ 241 urlStr = fields[ArcConstants.FN_IDX_URL]; 242 if ("-".equals(urlStr)) { 243 urlStr = null; 244 } 245 urlUri = fieldParsers.parseUri(urlStr, uriProfile, ArcConstants.FN_URL, false); 246 if (urlUri != null) { 247 urlScheme = urlUri.getScheme(); 248 } else if (urlStr != null) { 249 // TODO unit test 250 urlScheme = Scheme.getScheme(urlStr); 251 } 252 if (urlScheme != null) { 253 urlScheme = urlScheme.toLowerCase(); 254 } 255 256 ipAddressStr = fields[ArcConstants.FN_IDX_IP_ADDRESS]; 257 if ("-".equals(ipAddressStr)) { 258 ipAddressStr = null; 259 } 260 inetAddress = fieldParsers.parseIpAddress(ipAddressStr, ArcConstants.FN_IP_ADDRESS, false); 261 262 archiveDateStr = fields[ArcConstants.FN_IDX_ARCHIVE_DATE]; 263 if ("-".equals(archiveDateStr)) { 264 archiveDateStr = null; 265 } 266 archiveDate = fieldParsers.parseDate(archiveDateStr, ArcConstants.FN_ARCHIVE_DATE, false); 267 268 contentTypeStr = fields[ArcConstants.FN_IDX_CONTENT_TYPE]; 269 if ("-".equals(contentTypeStr)) { 270 contentTypeStr = null; 271 } 272 if (!ArcConstants.CONTENT_TYPE_NO_TYPE.equalsIgnoreCase(contentTypeStr)) { 273 contentType = fieldParsers.parseContentType(contentTypeStr, ArcConstants.FN_CONTENT_TYPE, false); 274 } 275 276 if (fields.length == ArcConstants.VERSION_2_BLOCK_FIELDS.length) { 277 recordFieldVersion = 2; 278 /* 279 * Version 2. 280 */ 281 resultCodeStr = fields[ArcConstants.FN_IDX_RESULT_CODE]; 282 if ("-".equals(resultCodeStr)) { 283 resultCodeStr = null; 284 } 285 resultCode = fieldParsers.parseInteger( 286 resultCodeStr, ArcConstants.FN_RESULT_CODE, false); 287 if (resultCode != null && (resultCode < 100 || resultCode > 999)) { 288 diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_RESULT_CODE + "' value", resultCodeStr, "A number between 100 and 999")); 289 } 290 291 checksumStr = fields[ArcConstants.FN_IDX_CHECKSUM]; 292 if ("-".equals(checksumStr)) { 293 checksumStr = null; 294 } 295 checksumStr = fieldParsers.parseString(checksumStr, ArcConstants.FN_CHECKSUM, true); 296 297 locationStr = fields[ArcConstants.FN_IDX_LOCATION]; 298 if ("-".equals(locationStr)) { 299 locationStr = null; 300 } 301 locationStr = fieldParsers.parseString(locationStr, ArcConstants.FN_LOCATION, true); 302 303 offsetStr = fields[ArcConstants.FN_IDX_OFFSET]; 304 if ("-".equals(offsetStr)) { 305 offsetStr = null; 306 } 307 offset = fieldParsers.parseLong( 308 offsetStr, ArcConstants.FN_OFFSET, false); 309 if (offset != null && offset < 0) { 310 diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_OFFSET + "' value", offsetStr, "A non negative number")); 311 } 312 313 filenameStr = fields[ArcConstants.FN_IDX_FILENAME]; 314 if ("-".equals(filenameStr)) { 315 filenameStr = null; 316 } 317 filenameStr = fieldParsers.parseString(filenameStr, ArcConstants.FN_FILENAME, false); 318 } 319 archiveLengthStr = fields[fields.length - 1]; 320 if ("-".equals(archiveLengthStr)) { 321 archiveLengthStr = null; 322 } 323 archiveLength = fieldParsers.parseLong(archiveLengthStr, ArcConstants.FN_ARCHIVE_LENGTH, false); 324 if (archiveLength != null && archiveLength < 0) { 325 diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_ARCHIVE_LENGTH + "' value", archiveLengthStr, "A non negative number")); 326 } 327 } 328 } 329 330 /** 331 * Returns the starting offset of the record in the containing ARC. 332 * @return the starting offset of the record 333 */ 334 public long getStartOffset() { 335 return startOffset; 336 } 337 338 /** 339 * Add object information to <code>StringBuilder</code>. 340 * @param sb <code>StringBuilder</code> where to add information 341 */ 342 public void toStringBuilder(StringBuilder sb) { 343 if (urlStr != null) { 344 sb.append("url: ").append(urlStr); 345 } 346 if (ipAddressStr != null) { 347 sb.append(", ipAddress: ").append(ipAddressStr); 348 } 349 if (archiveDateStr != null) { 350 sb.append(", archiveDate: ").append(archiveDateStr); 351 } 352 if (contentTypeStr != null) { 353 sb.append(", contentType: ").append(contentTypeStr); 354 } 355 if (resultCodeStr != null) { 356 sb.append(", resultCode: ").append(resultCodeStr); 357 } 358 if (checksumStr != null) { 359 sb.append(", checksum: ").append(checksumStr); 360 } 361 if (locationStr != null) { 362 sb.append(", location: ").append(locationStr); 363 } 364 if (offsetStr != null) { 365 sb.append(", offset: ").append(offsetStr); 366 } 367 if (filenameStr != null) { 368 sb.append(", fileName: ").append(filenameStr); 369 } 370 if (archiveLengthStr != null) { 371 sb.append(", length: ").append(archiveLengthStr); 372 } 373 } 374 375 @Override 376 public String toString() { 377 StringBuilder sb = new StringBuilder(); 378 toStringBuilder(sb); 379 return sb.toString(); 380 } 381 382}