001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.arc;
019
020import java.io.IOException;
021import java.net.InetAddress;
022import java.util.Date;
023
024import org.jwat.common.ByteCountingPushBackInputStream;
025import org.jwat.common.ContentType;
026import org.jwat.common.Diagnosis;
027import org.jwat.common.DiagnosisType;
028import org.jwat.common.Diagnostics;
029import org.jwat.common.HeaderLine;
030import org.jwat.common.Scheme;
031import org.jwat.common.Uri;
032import org.jwat.common.UriProfile;
033
034/**
035 * Class for parsing and validating the common ARC record header present in
036 * both ARC records and ARC version blocks.
037 *
038 * @author nicl
039 */
040public class ArcHeader {
041
042    /** Associated WarcReader context.
043     * Must be set prior to calling the various methods. */
044    protected ArcReader reader;
045
046    /** Diagnostics used to report diagnoses.
047     * Must be set prior to calling the various methods. */
048    protected Diagnostics<Diagnosis> diagnostics;
049
050    /** URL URI profile. */
051    protected UriProfile uriProfile;
052
053    /** ARC field parser used.
054     * Must be set prior to calling the various methods. */
055    protected ArcFieldParsers fieldParsers;
056
057    /** ARC record starting offset relative to the source ARC file input
058     *  stream. The offset is correct for both compressed and uncompressed streams. */
059    protected long startOffset = -1;
060
061    /*
062     * Version related fields.
063     */
064
065    /** Which version of the record fields was parsed, 1.x or 2.0. */
066    public int recordFieldVersion;
067
068    /*
069     * ARC header fields.
070     */
071
072    /** Do the record fields comply in number with the one dictated by its
073     *  version. */
074    protected boolean hasCompliantFields = false;
075
076    /** ARC record URL field string value. */
077    public String urlStr;
078    /** ARC record URL validated and converted to an <code>URI</code> object. */
079    public Uri urlUri;
080    /** URI Scheme (lowercase). (filedesc, http, https, dns, etc.) */
081    public String urlScheme;
082
083    /** ARC record IP-Address field string value. */
084    public String ipAddressStr;
085    /** IP-Address validated and converted to an <code>InetAddress</code> object. */
086    public InetAddress inetAddress;
087
088    /** ARC record archive-date field string value. */
089    public String archiveDateStr;
090    /** Archive date validated and converted to a <code>Date</code> object. */
091    public Date archiveDate;
092
093    /** ARC record content-type field string value. */
094    public String contentTypeStr;
095    /** Content-Type wrapper object with optional parameters. */
096    public ContentType contentType;
097
098    /** ARC record result-code field string value. */
099    public String resultCodeStr;
100    /** Result-code validated and converted into an <code>Integer</code> object. */
101    public Integer resultCode;
102
103    /** ARC record checksum field string value. */
104    public String checksumStr;
105
106    /** ARC record location field string value. */
107    public String locationStr;
108
109    /** ARC record offset field string value. */
110    public String offsetStr;
111    /** Offset validated and converted into a <code>Long</code> object. */
112    public Long offset;
113
114    /** ARC record filename field string value. */
115    public String filenameStr;
116
117    /** ARC record archive-length field string value. */
118    public String archiveLengthStr;
119    /** Archive-length validated and converted into a <code>Long</code> object. */
120    public Long archiveLength;
121
122    /** Raw ARC header line byte array. */
123    public byte[] headerBytes;
124
125    /**
126     * Non public constructor to allow unit testing.
127     */
128    protected ArcHeader() {
129    }
130
131    /**
132     * Create a new <code>ArcHeader</code> for recreating a header object.
133     * @param uriProfile uri profile used to validate urls
134     * @param fieldParsers parsers used for the individual types
135     * @param diagnostics diagnostics object used by parser
136     * @return a <code>ArcHeader</code> prepared for writing
137     */
138    public static ArcHeader initHeader(ArcFieldParsers fieldParsers, UriProfile uriProfile, Diagnostics<Diagnosis> diagnostics) {
139        ArcHeader header = new ArcHeader();
140        header.fieldParsers = fieldParsers;
141        header.uriProfile = uriProfile;
142        header.fieldParsers = fieldParsers;
143        header.diagnostics = diagnostics;
144        return header;
145    }
146
147    /**
148     * Create and initialize a new <code>ArcHeader</code> for writing.
149     * @param writer writer which shall be used
150     * @param diagnostics diagnostics object used by writer
151     * @return a <code>ArcHeader</code> prepared for writing
152     */
153    public static ArcHeader initHeader(ArcWriter writer, Diagnostics<Diagnosis> diagnostics) {
154        ArcHeader header = new ArcHeader();
155        header.fieldParsers = writer.fieldParsers;
156        header.diagnostics = diagnostics;
157        return header;
158    }
159
160    /**
161     * Create and initialize a new <code>ArcHeader</code> for reading.
162     * @param reader reader which shall be used
163     * @param startOffset start offset of header
164     * @param diagnostics diagnostics object used by reader
165     * @return a <code>ArcHeader</code> prepared for reading
166     */
167    public static ArcHeader initHeader(ArcReader reader, long startOffset, Diagnostics<Diagnosis> diagnostics) {
168        ArcHeader header = new ArcHeader();
169        header.reader = reader;
170        header.uriProfile = reader.uriProfile;
171        header.fieldParsers = reader.fieldParsers;
172        header.diagnostics = diagnostics;
173        // This is only relevant for uncompressed sequentially read records
174        header.startOffset = startOffset;
175        return header;
176    }
177
178    /**
179     * Tries to read lines from the input stream to determine if they could
180     * be possible ARC record header lines. Returns true if a line was
181     * encountered with delimited fields matching either a v1.x or v2.0
182     * description block. False is no ARC record header candidate was found
183     * in the remaining input stream.
184     * @param in input stream supposedly containing ARC records
185     * @return true if an ARC record header was parsed and validated
186     * @throws IOException if an I/O exception occurs while trying to read an
187     * ARC record header
188     */
189    public boolean parseHeader(ByteCountingPushBackInputStream in) throws IOException {
190        boolean bHeaderParsed = false;
191        boolean bInvalidDataBeforeVersion = false;
192        boolean bEmptyLinesBeforeVersion = false;
193        boolean bSeekRecord = true;
194        while (bSeekRecord) {
195            startOffset = in.getConsumed();
196            HeaderLine recordLine = HeaderLine.readLine(in);
197            if (recordLine != null) {
198                if (recordLine.line.length() > 0) {
199                    String[] fields = recordLine.line.split(" ", -1);
200                    if (fields.length == ArcConstants.VERSION_1_BLOCK_NUMBER_FIELDS
201                            || fields.length == ArcConstants.VERSION_2_BLOCK_NUMBER_FIELDS) {
202                        // debug
203                        //System.out.println(recordLine);
204                        parseHeaders(fields);
205                        bHeaderParsed = true;
206                        headerBytes = recordLine.raw;
207                        bSeekRecord = false;
208                    } else {
209                        bInvalidDataBeforeVersion = true;
210                    }
211                } else {
212                    bEmptyLinesBeforeVersion = true;
213                }
214            }
215            else {
216                bSeekRecord = false;
217            }
218        }
219        if (bInvalidDataBeforeVersion) {
220            diagnostics.addError(new Diagnosis(DiagnosisType.INVALID, "Data before ARC record"));
221        }
222        if (bEmptyLinesBeforeVersion) {
223            diagnostics.addError(new Diagnosis(DiagnosisType.INVALID, "Empty lines before ARC record"));
224        }
225        return bHeaderParsed;
226    }
227
228    /**
229     * Examines an array of strings to determine if it could be an ARC record
230     * header. If the number of fields is not equal to that of either v1.x or
231     * v2.0 nothing is parsed.
232     * @param fields array of possible ARC record header fields
233     */
234    public void parseHeaders(String[] fields) {
235        if (fields.length == ArcConstants.VERSION_1_BLOCK_FIELDS.length
236                || fields.length == ArcConstants.VERSION_2_BLOCK_FIELDS.length) {
237            recordFieldVersion = 1;
238            /*
239             * Version 1.
240             */
241            urlStr = fields[ArcConstants.FN_IDX_URL];
242            if ("-".equals(urlStr)) {
243                urlStr = null;
244            }
245            urlUri = fieldParsers.parseUri(urlStr, uriProfile, ArcConstants.FN_URL, false);
246            if (urlUri != null) {
247                urlScheme = urlUri.getScheme();
248            } else if (urlStr != null) {
249                // TODO unit test
250                urlScheme = Scheme.getScheme(urlStr);
251            }
252            if (urlScheme != null) {
253                urlScheme = urlScheme.toLowerCase();
254            }
255
256            ipAddressStr = fields[ArcConstants.FN_IDX_IP_ADDRESS];
257            if ("-".equals(ipAddressStr)) {
258                ipAddressStr = null;
259            }
260            inetAddress = fieldParsers.parseIpAddress(ipAddressStr, ArcConstants.FN_IP_ADDRESS, false);
261
262            archiveDateStr = fields[ArcConstants.FN_IDX_ARCHIVE_DATE];
263            if ("-".equals(archiveDateStr)) {
264                archiveDateStr = null;
265            }
266            archiveDate = fieldParsers.parseDate(archiveDateStr, ArcConstants.FN_ARCHIVE_DATE, false);
267
268            contentTypeStr = fields[ArcConstants.FN_IDX_CONTENT_TYPE];
269            if ("-".equals(contentTypeStr)) {
270                contentTypeStr = null;
271            }
272            if (!ArcConstants.CONTENT_TYPE_NO_TYPE.equalsIgnoreCase(contentTypeStr)) {
273                contentType = fieldParsers.parseContentType(contentTypeStr, ArcConstants.FN_CONTENT_TYPE, false);
274            }
275
276            if (fields.length == ArcConstants.VERSION_2_BLOCK_FIELDS.length) {
277                recordFieldVersion = 2;
278                /*
279                 *  Version 2.
280                 */
281                resultCodeStr = fields[ArcConstants.FN_IDX_RESULT_CODE];
282                if ("-".equals(resultCodeStr)) {
283                    resultCodeStr = null;
284                }
285                resultCode = fieldParsers.parseInteger(
286                        resultCodeStr, ArcConstants.FN_RESULT_CODE, false);
287                if (resultCode != null && (resultCode < 100 || resultCode > 999)) {
288                    diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_RESULT_CODE + "' value", resultCodeStr, "A number between 100 and 999"));
289                }
290
291                checksumStr = fields[ArcConstants.FN_IDX_CHECKSUM];
292                if ("-".equals(checksumStr)) {
293                    checksumStr = null;
294                }
295                checksumStr = fieldParsers.parseString(checksumStr, ArcConstants.FN_CHECKSUM, true);
296
297                locationStr = fields[ArcConstants.FN_IDX_LOCATION];
298                if ("-".equals(locationStr)) {
299                    locationStr = null;
300                }
301                locationStr = fieldParsers.parseString(locationStr, ArcConstants.FN_LOCATION, true);
302
303                offsetStr = fields[ArcConstants.FN_IDX_OFFSET];
304                if ("-".equals(offsetStr)) {
305                    offsetStr = null;
306                }
307                offset = fieldParsers.parseLong(
308                        offsetStr, ArcConstants.FN_OFFSET, false);
309                if (offset != null && offset < 0) {
310                    diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_OFFSET + "' value", offsetStr, "A non negative number"));
311                }
312
313                filenameStr = fields[ArcConstants.FN_IDX_FILENAME];
314                if ("-".equals(filenameStr)) {
315                    filenameStr = null;
316                }
317                filenameStr = fieldParsers.parseString(filenameStr, ArcConstants.FN_FILENAME, false);
318            }
319            archiveLengthStr = fields[fields.length - 1];
320            if ("-".equals(archiveLengthStr)) {
321                archiveLengthStr = null;
322            }
323            archiveLength = fieldParsers.parseLong(archiveLengthStr, ArcConstants.FN_ARCHIVE_LENGTH, false);
324            if (archiveLength != null && archiveLength < 0) {
325                diagnostics.addError(new Diagnosis(DiagnosisType.INVALID_EXPECTED, "'" + ArcConstants.FN_ARCHIVE_LENGTH + "' value", archiveLengthStr, "A non negative number"));
326            }
327        }
328    }
329
330    /**
331     * Returns the starting offset of the record in the containing ARC.
332     * @return the starting offset of the record
333     */
334    public long getStartOffset() {
335        return startOffset;
336    }
337
338    /**
339     * Add object information to <code>StringBuilder</code>.
340     * @param sb <code>StringBuilder</code> where to add information
341     */
342    public void toStringBuilder(StringBuilder sb) {
343        if (urlStr != null) {
344            sb.append("url: ").append(urlStr);
345        }
346        if (ipAddressStr != null) {
347            sb.append(", ipAddress: ").append(ipAddressStr);
348        }
349        if (archiveDateStr != null) {
350            sb.append(", archiveDate: ").append(archiveDateStr);
351        }
352        if (contentTypeStr != null) {
353            sb.append(", contentType: ").append(contentTypeStr);
354        }
355        if (resultCodeStr != null) {
356            sb.append(", resultCode: ").append(resultCodeStr);
357        }
358        if (checksumStr != null) {
359            sb.append(", checksum: ").append(checksumStr);
360        }
361        if (locationStr != null) {
362            sb.append(", location: ").append(locationStr);
363        }
364        if (offsetStr != null) {
365            sb.append(", offset: ").append(offsetStr);
366        }
367        if (filenameStr != null) {
368            sb.append(", fileName: ").append(filenameStr);
369        }
370        if (archiveLengthStr != null) {
371            sb.append(", length: ").append(archiveLengthStr);
372        }
373    }
374
375    @Override
376    public String toString() {
377        StringBuilder sb = new StringBuilder();
378        toStringBuilder(sb);
379        return sb.toString();
380    }
381
382}