001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.warc;
019
020import java.io.ByteArrayOutputStream;
021import java.io.IOException;
022import java.net.InetAddress;
023import java.util.Collections;
024import java.util.HashMap;
025import java.util.LinkedList;
026import java.util.List;
027import java.util.Map;
028
029import org.jwat.common.ByteCountingPushBackInputStream;
030import org.jwat.common.ContentType;
031import org.jwat.common.Diagnosis;
032import org.jwat.common.DiagnosisType;
033import org.jwat.common.Diagnostics;
034import org.jwat.common.HeaderLine;
035import org.jwat.common.HeaderLineReader;
036import org.jwat.common.MaxLengthRecordingInputStream;
037import org.jwat.common.Uri;
038import org.jwat.common.UriProfile;
039
040/**
041 * Central class for working with WARC headers. This class includes support for
042 * reading and writing WARC headers. Methods are also available to validate
043 * individual headers and a WARC header as a whole.
044 *
045 * @author nicl
046 */
047public class WarcHeader {
048
049    /** An URI with encapsulating <> characters. */
050    public static final boolean URI_LTGT = true;
051
052    /** An URI without encapsulating <> characters. */
053    public static final boolean URI_NAKED = false;
054
055    /** Associated WarcReader context.
056     *  Must be set prior to calling the various methods. */
057    protected WarcReader reader;
058
059    /** Version based WARC validation. */
060    protected WarcValidatorBase warcValidator;
061
062    /** Diagnostics used to report diagnoses.
063     *  Must be set prior to calling the various methods. */
064    protected Diagnostics diagnostics;
065
066    /** WARC-Target-URI profile. */
067    protected UriProfile warcTargetUriProfile;
068
069    /** URI profile. */
070    protected UriProfile uriProfile;
071
072    /** WARC field parser used.
073     *  Must be set prior to calling the various methods. */
074    protected WarcFieldParsers fieldParsers;
075
076    /** Max size allowed for a record header. */
077    protected int recordHeaderMaxSize = -1;
078
079    /** Line reader used to read version lines. */
080    protected HeaderLineReader lineReader;
081
082    /** Header line reader used to read the WARC headers. */
083    protected HeaderLineReader headerLineReader;
084
085    /** WARC record starting offset relative to the source WARC file input
086     *  stream. The offset is correct for both compressed and uncompressed streams. */
087    protected long startOffset = -1;
088
089    /*
090     * Version related fields.
091     */
092
093    /** Was "WARC/" identified while looking for the version string. */
094    public boolean bMagicIdentified;
095    /** Did the version string include between 2 and 4 substrings delimited by ".". */
096    public boolean bVersionParsed;
097    /** Is the version format valid. */
098    public boolean bValidVersionFormat;
099    /** Is the version recognized. (0.17, 0.18 or 1.0) */
100    public boolean bValidVersion;
101
102    /** Raw version string. */
103    public String versionStr;
104    /** Array based on the version string split by the "." delimiter and converted to integers. */
105    public int[] versionArr;
106
107    /** Major version number from WARC header. */
108    public int major = -1;
109    /** Minor version number from WARC header. */
110    public int minor = -1;
111
112    /*
113     * WARC header fields.
114     */
115
116    /** Array used for duplicate header detection. */
117    protected boolean[] seen = new boolean[WarcConstants.FN_INDEX_OF_LAST];
118
119    /** Is the header missing one of the mandatory headers. */
120    public boolean bMandatoryMissing;
121
122    /** WARC-Type field string value. */
123    public String warcTypeStr;
124    /** WARC-Type converted to an integer id, if identified. */
125    public Integer warcTypeIdx;
126
127    /** WARC-Filename field string value.
128     *  (warcinfo record type only) */
129    public String warcFilename;
130
131    /** WARC-Record-Id field string value. */
132    public String warcRecordIdStr;
133    /** WARC-Record-Id converted to an <code>Uri</code> object, if valid. */
134    public Uri warcRecordIdUri;
135
136    /** WARC-Date field string value. */
137    public String warcDateStr;
138    /** WARC-Date converted to a <code>Date</code> object, if valid. */
139    public WarcDate warcDate;
140
141    /** Content-Length field string value. */
142    public String contentLengthStr;
143    /** Content-Length converted to a <code>Long</code> object, if valid. */
144    public Long contentLength;
145
146    /** Content-Type field string value. */
147    public String contentTypeStr;
148    /** Content-Type converted to a <code>ContentType</code> object, if valid. */
149    public ContentType contentType;
150
151    /** WARC-Truncated field string value. */
152    public String warcTruncatedStr;
153    /** WARC-Truncated converted to an integer id, if valid. */
154    public Integer warcTruncatedIdx;
155
156    /** WARC-IP-Address field string value. */
157    public String warcIpAddress;
158    /** WARC-IP-Address converted to an <code>InetAddress</code> object, if valid. */
159    public InetAddress warcInetAddress;
160
161    /** List of WARC-Concurrent-To field string values and converted <code>URI</code> objects,  if valid. */
162    public List<WarcConcurrentTo> warcConcurrentToList = new LinkedList<WarcConcurrentTo>();
163
164    /** WARC-Refers-To field string value. */
165    public String warcRefersToStr;
166    /** WARC-Refers-To converted to an <code>Uri</code> object, if valid. */
167    public Uri warcRefersToUri;
168
169    /** WARC_Target-URI field string value. */
170    public String warcTargetUriStr;
171    /** WARC-TargetURI converted to an <code>Uri</code> object, if valid. */
172    public Uri warcTargetUriUri;
173
174    /** WARC-Warcinfo-Id field string value. */
175    public String warcWarcinfoIdStr;
176    /** WARC-Warcinfo-Id converted to an <code>Uri</code> object, if valid. */
177    public Uri warcWarcinfoIdUri;
178
179    /** WARC-Block-Digest field string value. */
180    public String warcBlockDigestStr;
181    /** WARC-Block-Digest converted to a <code>WarcDigest</code> object, if valid. */
182    public WarcDigest warcBlockDigest;
183
184    /** WARC-Payload-Digest field string value. */
185    public String warcPayloadDigestStr;
186    /** WARC-Payload-Digest converted to a <code>WarcDigest</code> object, if valid. */
187    public WarcDigest warcPayloadDigest;
188
189    /** WARC-Identified-Payload-Type field string value. */
190    public String warcIdentifiedPayloadTypeStr;
191    /** WARC-Identified-Payload-Type converted to a <code>ContentType</code> object, if valid. */
192    public ContentType warcIdentifiedPayloadType;
193
194    /** WARC-Profile field string value.
195     *  (revisit record only) */
196    public String warcProfileStr;
197    /** WARC-Profile field converted to an <code>Uri</code> object, if valid.
198     *  (revisit record only) */
199    public Uri warcProfileUri;
200    /** WARC-Profile converted to an integer id, if valid.
201     *  (revisit record only) */
202    public Integer warcProfileIdx;
203
204    /** WARC-Segment-Number field string value. */
205    public String warcSegmentNumberStr;
206    /** WARC-Segment-Number converted to an <code>Integer</code> object, if valid. */
207    public Integer warcSegmentNumber;
208
209    /** WARC-Segment-Origin-Id field string value.
210     *  (continuation record only) */
211    public String warcSegmentOriginIdStr;
212    /** WARC-Segment-Origin-Id converted to an <code>Uri</code> object, if valid.
213     *  (continuation record only) */
214    public Uri warcSegmentOriginIdUrl;
215
216    /** WARC-Segment-Total-Length field string value.
217     *  (continuation record only) */
218    public String warcSegmentTotalLengthStr;
219    /** WARC-Segment-Total-Length converted to a <code>Long</code> object, if valid.
220     *  (continuation record only) */
221    public Long warcSegmentTotalLength;
222
223    // see https://docs.google.com/document/d/1QyQBA7Ykgxie75V8Jziz_O7hbhwf7PF6_u9O6w6zgp0/edit
224    /** WARC-Refers-To-Target-URI field string value. */
225    public String warcRefersToTargetUriStr;
226    /** WARC-Refers-To-Target-URI converted to an <code>Uri</code> object, if valid. */
227    public Uri warcRefersToTargetUriUri;
228    /** WARC-Refers-To-Date */
229    public String warcRefersToDateStr;
230    /** WARC-Date converted to a <code>Date</code> object, if valid. */
231    public WarcDate warcRefersToDate;
232
233    /*
234     * WARC header fields collections.
235     */
236
237    /** Raw WARC header output stream. */
238    protected ByteArrayOutputStream headerBytesOut = new ByteArrayOutputStream();
239
240    /** Raw WARC header byte array. */
241    public byte[] headerBytes;
242
243    /** List of parsed header fields. */
244    protected List<HeaderLine> headerList = new LinkedList<HeaderLine>();
245
246    /** Map of parsed header fields. */
247    protected Map<String, HeaderLine> headerMap = new HashMap<String, HeaderLine>();
248
249    /**
250     * Non public constructor to allow unit testing.
251     */
252    protected WarcHeader() {
253    }
254
255    /**
256     * Create a new <code>WarcHeader</code> for recreating a header object.
257     * @param recordHeaderMaxSize max size allowed for a record header
258     * @param lineReader line reader used to read version lines
259     * @param headerLineReader header line reader used to read the WARC headers
260     * @param fieldParsers parsers used for the individual types
261     * @param uriProfile uri profile used to validate urls
262     * @param diagnostics diagnostics object used by parser
263     * @return a <code>WarcHeader</code> prepared for reanimation
264     */
265    public static WarcHeader initHeader(int recordHeaderMaxSize, HeaderLineReader lineReader, HeaderLineReader headerLineReader, WarcFieldParsers fieldParsers, UriProfile uriProfile, Diagnostics diagnostics) {
266        WarcHeader header = new WarcHeader();
267        header.warcTargetUriProfile = uriProfile;
268        header.uriProfile = uriProfile;
269        header.fieldParsers = fieldParsers;
270        header.diagnostics = diagnostics;
271        header.recordHeaderMaxSize = recordHeaderMaxSize;
272        header.lineReader = lineReader;
273        header.headerLineReader = headerLineReader;
274        return header;
275    }
276
277    /**
278     * Create and initialize a new <code>WarcHeader</code> for writing.
279     * @param writer writer which shall be used
280     * @param diagnostics diagnostics object used by writer
281     * @return a <code>WarcHeader</code> prepared for writing
282     */
283    public static WarcHeader initHeader(WarcWriter writer, Diagnostics diagnostics) {
284        WarcHeader header = new WarcHeader();
285        // Set default version to "1.0".
286        header.major = 1;
287        header.minor = 0;
288        header.warcTargetUriProfile = writer.warcTargetUriProfile;
289        header.uriProfile = writer.uriProfile;
290        header.fieldParsers = writer.fieldParsers;
291        header.diagnostics = diagnostics;
292        return header;
293    }
294
295    /**
296     * Create and initialize a new <code>WarcHeader</code> for reading.
297     * @param reader reader which shall be used
298     * @param startOffset start offset of header
299     * @param diagnostics diagnostics object used by reader
300     * @return a <code>WarcHeader</code> prepared for reading
301     */
302    public static WarcHeader initHeader(WarcReader reader, long startOffset, Diagnostics diagnostics) {
303        WarcHeader header = new WarcHeader();
304        header.reader = reader;
305        header.warcTargetUriProfile = reader.warcTargetUriProfile;
306        header.uriProfile = reader.uriProfile;
307        header.fieldParsers = reader.fieldParsers;
308        header.diagnostics = diagnostics;
309        header.recordHeaderMaxSize = reader.recordHeaderMaxSize;
310        header.lineReader = reader.lineReader;
311        header.headerLineReader = reader.headerLineReader;
312        // This is only relevant for uncompressed sequentially read records
313        header.startOffset = startOffset;
314        return header;
315    }
316
317    /**
318     * Add a warning diagnosis of the given type on a specific entity with
319     * optional extra information. The information varies according to the
320     * diagnosis type.
321     * @param type diagnosis type
322     * @param entity entity examined
323     * @param information optional extra information
324     */
325    protected void addWarningDiagnosis(DiagnosisType type, String entity, String... information) {
326        diagnostics.addWarning(new Diagnosis(type, entity, information));
327    }
328
329    /**
330     * Returns the starting offset of the record in the containing WARC.
331     * @return the starting offset of the record
332     */
333    public long getStartOffset() {
334        return startOffset;
335    }
336
337    /**
338     * Try to parse a WARC header and return a boolean indicating the success or
339     * failure of this.
340     * @param in input stream with WARC data
341     * @return boolean indicating whether a header was parsed or not
342     * @throws IOException if an I/O exception occurs while parsing for a header
343     */
344    public boolean parseHeader(ByteCountingPushBackInputStream in) throws IOException {
345        if (parseVersion(in)) {
346            // debug
347            //System.out.println(wr.bMagicIdentified);
348            //System.out.println(wr.bVersionParsed);
349            //System.out.println(wr.major + "." + wr.minor);
350            warcValidator = reader.validation.getValidatorFor(this, diagnostics);
351
352            MaxLengthRecordingInputStream mrin = new MaxLengthRecordingInputStream(in, recordHeaderMaxSize);
353            ByteCountingPushBackInputStream pbin = new ByteCountingPushBackInputStream(mrin, recordHeaderMaxSize);
354
355            parseHeaders(pbin);
356            pbin.close();
357
358            warcValidator.checkFields(this);
359
360            headerBytes = headerBytesOut.toByteArray();
361        }
362        return bMagicIdentified;
363    }
364
365    /**
366     * Looks forward in the input stream for a valid WARC version line.
367     * @param in data input stream
368     * @return true, if magic WARC header found
369     * @throws IOException if an error occurs while reading version data
370     */
371    public boolean parseVersion(ByteCountingPushBackInputStream in) throws IOException {
372        bMagicIdentified = false;
373        bVersionParsed = false;
374        boolean bInvalidDataBeforeVersion = false;
375        boolean bEmptyLinesBeforeVersion = false;
376        HeaderLine line;
377        String tmpStr;
378        boolean bSeekMagic = true;
379        // Loop until when have found something that looks like a version line.
380        while (bSeekMagic) {
381            // This is only relevant for uncompressed sequentially read records
382            startOffset = in.getConsumed();
383            line = lineReader.readLine(in);
384            if (!lineReader.bEof) {
385                switch (line.type) {
386                case HeaderLine.HLT_LINE:
387                    tmpStr = line.line;
388                    // debug
389                    //System.out.println(tmpStr);
390                    if (tmpStr.length() > 0) {
391                        if (tmpStr.toUpperCase().startsWith(WarcConstants.WARC_MAGIC_HEADER)) {
392                            bMagicIdentified = true;
393                            versionStr = tmpStr.substring(WarcConstants.WARC_MAGIC_HEADER.length());
394                            String[] tmpArr = versionStr.split("\\.", -1);        // Not optimal
395                            if (tmpArr.length >= 2 && tmpArr.length <= 4) {
396                                bVersionParsed = true;
397                                bValidVersionFormat = true;
398                                versionArr = new int[tmpArr.length];
399                                for (int i=0; i<tmpArr.length; ++i) {
400                                    try {
401                                        versionArr[i] = Integer.parseInt(tmpArr[i]);
402                                    } catch (NumberFormatException e) {
403                                        versionArr[i] = -1;
404                                        bValidVersionFormat = false;
405                                    }
406                                }
407                                major = versionArr[0];
408                                minor = versionArr[1];
409                            }
410                            headerBytesOut.write(line.raw);
411                            bSeekMagic = false;
412                        } else {
413                            // Invalid data aka Gibberish.
414                            bInvalidDataBeforeVersion = true;
415                        }
416                    } else {
417                        // Empty line.
418                        bEmptyLinesBeforeVersion = true;
419
420                    }
421                    break;
422                case HeaderLine.HLT_HEADERLINE:
423                    // Invalid data - header or binary.
424                    bInvalidDataBeforeVersion = true;
425                    break;
426                }
427            } else {
428                // EOF.
429                bSeekMagic = false;
430            }
431        }
432        if (bInvalidDataBeforeVersion) {
433            diagnostics.addError(DiagnosisType.INVALID, "Data before WARC version");
434        }
435        if (bEmptyLinesBeforeVersion) {
436            diagnostics.addError(DiagnosisType.INVALID, "Empty lines before WARC version");
437        }
438        return bMagicIdentified;
439    }
440
441    /**
442     * Reads WARC header lines one line at a time until an empty line is
443     * encountered.
444     * @param in header input stream
445     * @throws IOException if an error occurs while reading the WARC header
446     */
447    public void parseHeaders(ByteCountingPushBackInputStream in) throws IOException {
448        HeaderLine headerLine;
449        boolean bLoop = true;
450        while (bLoop) {
451            headerLine = headerLineReader.readLine(in);
452            if ((headerLine.bfErrors & HeaderLineReader.E_BIT_INVALID_CHARSET) != 0) {
453                diagnostics.addError(DiagnosisType.INVALID_ENCODING, "Invalid encoding in header line", headerLine.value, "UNKNOWN");
454            }
455            if (!headerLineReader.bEof) {
456                headerBytesOut.write(headerLine.raw);
457                switch (headerLine.type) {
458                case HeaderLine.HLT_HEADERLINE:
459                    if (headerLine.name != null && headerLine.name.length() > 0) {
460                        // debug
461                        //System.out.println(headerLine.name);
462                        //System.out.println(headerLine.value);
463                        addHeader(headerLine);
464                    } else {
465                        // Empty field name.
466                        addWarningDiagnosis(DiagnosisType.EMPTY, "Header line");
467                    }
468                    break;
469                case HeaderLine.HLT_LINE:
470                    if (headerLine.line.length() == 0) {
471                        // Empty line.
472                        bLoop = false;
473                    } else {
474                        // Unknown header line.
475                        addWarningDiagnosis(DiagnosisType.UNKNOWN, "Header line", headerLine.line);
476                    }
477                    break;
478                default:
479                    throw new IllegalStateException("Invalid HeaderLine output!");
480                }
481            } else {
482                // EOF.
483                bLoop = false;
484            }
485        }
486    }
487
488    /**
489     * Identify a (WARC) header name, validate the value and set the header.
490     * @param headerLine the headerLine
491     */
492    public void addHeader(HeaderLine headerLine) {
493        String fieldName = headerLine.name;
494        String fieldValue = headerLine.value;
495        WarcConcurrentTo warcConcurrentTo;
496        Integer fn_idx = WarcConstants.fieldNameIdxMap.get(fieldName.toLowerCase());
497        if (fn_idx != null) {
498            // WARC field name defined in WARC specification.
499            if (!seen[fn_idx] || WarcConstants.fieldNamesRepeatableLookup[fn_idx]) {
500                seen[fn_idx] = true;
501                switch (fn_idx.intValue()) {
502                case WarcConstants.FN_IDX_WARC_TYPE:
503                    warcTypeStr = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_TYPE);
504                    if (warcTypeStr != null) {
505                        warcTypeIdx = WarcConstants.recordTypeIdxMap.get(warcTypeStr.toLowerCase());
506                    }
507                    if (warcTypeIdx == null && warcTypeStr != null && warcTypeStr.length() > 0) {
508                        warcTypeIdx = WarcConstants.RT_IDX_UNKNOWN;
509                    }
510                    break;
511                case WarcConstants.FN_IDX_WARC_RECORD_ID:
512                    warcRecordIdStr = fieldValue;
513                    warcRecordIdUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_RECORD_ID);
514                    break;
515                case WarcConstants.FN_IDX_WARC_DATE:
516                    warcDateStr = fieldValue;
517                    //warcDate = fieldParsers.parseDate(fieldValue, WarcConstants.FN_WARC_DATE);
518                    warcDate = fieldParsers.parseWarcDate(fieldValue, WarcConstants.FN_WARC_DATE);
519                    break;
520                case WarcConstants.FN_IDX_CONTENT_LENGTH:
521                    contentLengthStr = fieldValue;
522                    contentLength = fieldParsers.parseLong(fieldValue, WarcConstants.FN_CONTENT_LENGTH);
523                    break;
524                case WarcConstants.FN_IDX_CONTENT_TYPE:
525                    contentTypeStr = fieldValue;
526                    contentType = fieldParsers.parseContentType(fieldValue, WarcConstants.FN_CONTENT_TYPE);
527                    break;
528                case WarcConstants.FN_IDX_WARC_CONCURRENT_TO:
529                    Uri tmpUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_CONCURRENT_TO);
530                    if (fieldValue != null && fieldValue.trim().length() > 0) {
531                        warcConcurrentTo = new WarcConcurrentTo();
532                        warcConcurrentTo.warcConcurrentToStr = fieldValue;
533                        warcConcurrentTo.warcConcurrentToUri = tmpUri;
534                        warcConcurrentToList.add(warcConcurrentTo);
535                    }
536                    break;
537                case WarcConstants.FN_IDX_WARC_BLOCK_DIGEST:
538                    warcBlockDigestStr = fieldValue;
539                    warcBlockDigest = fieldParsers.parseDigest(fieldValue, WarcConstants.FN_WARC_BLOCK_DIGEST);
540                    break;
541                case WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST:
542                    warcPayloadDigestStr = fieldValue;
543                    warcPayloadDigest = fieldParsers.parseDigest(fieldValue, WarcConstants.FN_WARC_PAYLOAD_DIGEST);
544                    break;
545                case WarcConstants.FN_IDX_WARC_IP_ADDRESS:
546                    warcIpAddress = fieldValue;
547                    warcInetAddress = fieldParsers.parseIpAddress(fieldValue, WarcConstants.FN_WARC_IP_ADDRESS);
548                    break;
549                case WarcConstants.FN_IDX_WARC_REFERS_TO:
550                    warcRefersToStr = fieldValue;
551                    warcRefersToUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_REFERS_TO);
552                    break;
553                case WarcConstants.FN_IDX_WARC_TARGET_URI:
554                    warcTargetUriStr = fieldValue;
555                    warcTargetUriUri = fieldParsers.parseUri(fieldValue, URI_NAKED, warcTargetUriProfile, WarcConstants.FN_WARC_TARGET_URI);
556                    break;
557                case WarcConstants.FN_IDX_WARC_TRUNCATED:
558                    warcTruncatedStr = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_TRUNCATED);
559                    if (warcTruncatedStr != null) {
560                        warcTruncatedIdx = WarcConstants.truncatedTypeIdxMap.get(warcTruncatedStr.toLowerCase());
561                    }
562                    if (warcTruncatedIdx == null && warcTruncatedStr != null && warcTruncatedStr.length() > 0) {
563                        warcTruncatedIdx = WarcConstants.TT_IDX_FUTURE_REASON;
564                    }
565                    break;
566                case WarcConstants.FN_IDX_WARC_WARCINFO_ID:
567                    warcWarcinfoIdStr = fieldValue;
568                    warcWarcinfoIdUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_WARCINFO_ID);
569                    break;
570                case WarcConstants.FN_IDX_WARC_FILENAME:
571                    warcFilename = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_FILENAME);
572                    break;
573                case WarcConstants.FN_IDX_WARC_PROFILE:
574                    warcProfileStr = fieldValue;
575                    warcProfileUri = fieldParsers.parseUri(fieldValue, URI_NAKED, uriProfile, WarcConstants.FN_WARC_PROFILE);
576                    if (warcProfileStr != null) {
577                        warcProfileIdx = WarcConstants.profileIdxMap.get(warcProfileStr.toLowerCase());
578                    }
579                    if (warcProfileIdx == null && warcProfileStr != null && warcProfileStr.length() > 0) {
580                        warcProfileIdx = WarcConstants.WARC_PROFILE_IDX_UNKNOWN;
581                    }
582                    break;
583                case WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE:
584                    warcIdentifiedPayloadTypeStr = fieldValue;
585                    warcIdentifiedPayloadType = fieldParsers.parseContentType(fieldValue, WarcConstants.FN_WARC_IDENTIFIED_PAYLOAD_TYPE);
586                    break;
587                case WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID:
588                    warcSegmentOriginIdStr = fieldValue;
589                    warcSegmentOriginIdUrl = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_SEGMENT_ORIGIN_ID);
590                    break;
591                case WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER:
592                    warcSegmentNumberStr = fieldValue;
593                    warcSegmentNumber = fieldParsers.parseInteger(fieldValue, WarcConstants.FN_WARC_SEGMENT_NUMBER);
594                    break;
595                case WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH:
596                    warcSegmentTotalLengthStr = fieldValue;
597                    warcSegmentTotalLength = fieldParsers.parseLong(fieldValue, WarcConstants.FN_WARC_SEGMENT_TOTAL_LENGTH);
598                    break;
599                case WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI:
600                    warcRefersToTargetUriStr = fieldValue;
601                    warcRefersToTargetUriUri = fieldParsers.parseUri(fieldValue, URI_NAKED, warcTargetUriProfile, WarcConstants.FN_WARC_REFERS_TO_TARGET_URI);
602                    break;
603                case WarcConstants.FN_IDX_WARC_REFERS_TO_DATE:
604                    warcRefersToDateStr = fieldValue;
605                    //warcRefersToDate = fieldParsers.parseDate(fieldValue, WarcConstants.FN_WARC_REFERS_TO_DATE);
606                    warcRefersToDate = fieldParsers.parseWarcDate(fieldValue, WarcConstants.FN_WARC_REFERS_TO_DATE);
607                    break;
608                }
609            } else {
610                // Duplicate field.
611                diagnostics.addError(DiagnosisType.DUPLICATE, "'" + fieldName + "' header", fieldValue);
612            }
613        }
614        HeaderLine tmpLine = headerMap.get(fieldName.toLowerCase());
615        if (tmpLine == null) {
616            headerMap.put(fieldName.toLowerCase(), headerLine);
617        } else {
618            tmpLine.lines.add(headerLine);
619        }
620        headerList.add(headerLine);
621    }
622
623    /**
624     * Get a <code>List</code> of all the headers found during parsing.
625     * @return <code>List</code> of <code>HeaderLine</code>
626     */
627    public List<HeaderLine> getHeaderList() {
628        return Collections.unmodifiableList(headerList);
629    }
630
631    /**
632     * Get a header line structure or null, if no header line structure is
633     * stored with the given header name.
634     * @param field header name
635     * @return <code>HeaderLine</code> structure or null
636     */
637    public HeaderLine getHeader(String field) {
638        if (field != null && field.length() > 0) {
639            return headerMap.get(field.toLowerCase());
640        } else {
641            return null;
642        }
643    }
644
645    /**
646     * Add a String header using the supplied string and return a
647     * <code>HeaderLine</code> object corresponding to how the header would be
648     * read.
649     * @param fieldName name of field to add
650     * @param fieldValue field value string
651     * @return <code>HeaderLine</code> object corresponding to what would have been read
652     */
653    public HeaderLine addHeader(String fieldName, String fieldValue) {
654        HeaderLine headerLine = new HeaderLine();
655        headerLine.name = fieldName;
656        headerLine.value = fieldValue;
657        addHeader(headerLine);
658        return headerLine;
659    }
660
661    /**
662     * Add an Integer header using the supplied string and object values and return
663     * a <code>HeaderLine</code> object corresponding to how the header would be read.
664     * If both string and object values are not null they are used as is.
665     * If the string value is null and the object is not null,
666     * the object's toString method is called.
667     * If the object is null and the string is not null, the string is parsed
668     * and validated resulting in an object, if valid.
669     * @param fieldName name of field to add
670     * @param integerFieldValue <code>Integer</code> field value object
671     * @param fieldValueStr Integer field value string
672     * @return <code>HeaderLine</code> object corresponding to what would have been read
673     */
674    public HeaderLine addHeader(String fieldName, Integer integerFieldValue, String fieldValueStr) {
675        if (integerFieldValue == null && fieldValueStr != null) {
676            integerFieldValue = fieldParsers.parseInteger(fieldValueStr, fieldName);
677        } else if (fieldValueStr == null && integerFieldValue != null) {
678            fieldValueStr = integerFieldValue.toString();
679        }
680        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_INTEGER,
681                integerFieldValue, null, null, null, null, null, null);
682    }
683
684    /**
685     * Add a Long header using the supplied string and object values and return
686     * a <code>HeaderLine</code> object corresponding to how the header would be read.
687     * If both string and object values are not null they are used as is.
688     * If the string value is null and the object is not null,
689     * the object's toString method is called.
690     * If the object is null and the string is not null, the string is parsed
691     * and validated resulting in an object, if valid.
692     * @param fieldName name of field to add
693     * @param longFieldValue <code>Long</code> field value object
694     * @param fieldValueStr Long field value string
695     * @return <code>HeaderLine</code> object corresponding to what would have been read
696     */
697    public HeaderLine addHeader(String fieldName, Long longFieldValue, String fieldValueStr) {
698        if (longFieldValue == null && fieldValueStr != null) {
699            longFieldValue = fieldParsers.parseLong(fieldValueStr, fieldName);
700        } else if (fieldValueStr == null && longFieldValue != null) {
701            fieldValueStr = longFieldValue.toString();
702        }
703        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_LONG,
704                null, longFieldValue, null, null, null, null, null);
705    }
706
707    /**
708     * Add an Digest header using the supplied string and object values and return
709     * a <code>HeaderLine</code> object corresponding to how the header would be read.
710     * If both string and object values are not null they are used as is.
711     * If the string value is null and the object is not null,
712     * the object's toString method is called.
713     * If the object is null and the string is not null, the string is parsed
714     * and validated resulting in an object, if valid.
715     * @param fieldName name of field to add
716     * @param digestFieldValue <code>Digest</code> field value object
717     * @param fieldValueStr Digest field value string
718     * @return <code>HeaderLine</code> object corresponding to what would have been read
719     */
720    public HeaderLine addHeader(String fieldName, WarcDigest digestFieldValue, String fieldValueStr) {
721        if (digestFieldValue == null && fieldValueStr != null) {
722            digestFieldValue = fieldParsers.parseDigest(fieldValueStr, fieldName);
723        } else if (fieldValueStr == null && digestFieldValue != null) {
724            fieldValueStr = digestFieldValue.toString();
725        }
726        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_DIGEST,
727                null, null, digestFieldValue, null, null, null, null);
728    }
729
730    /**
731     * Add an Content-Type header using the supplied string and object values and return
732     * a <code>HeaderLine</code> object corresponding to how the header would be read.
733     * If both string and object values are not null they are used as is.
734     * If the string value is null and the object is not null,
735     * the object's toString method is called.
736     * If the object is null and the string is not null, the string is parsed
737     * and validated resulting in an object, if valid.
738     * @param fieldName name of field to add
739     * @param contentTypeFieldValue <code>ContentType</code> field value object
740     * @param fieldValueStr Content-Type field value string
741     * @return <code>HeaderLine</code> object corresponding to what would have been read
742     */
743    public HeaderLine addHeader(String fieldName, ContentType contentTypeFieldValue, String fieldValueStr) {
744        if (contentTypeFieldValue == null && fieldValueStr != null) {
745            contentTypeFieldValue = fieldParsers.parseContentType(fieldValueStr, fieldName);
746        } else if (fieldValueStr == null && contentTypeFieldValue != null) {
747            fieldValueStr = contentTypeFieldValue.toString();
748        }
749        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_CONTENTTYPE,
750                null, null, null, contentTypeFieldValue, null, null, null);
751    }
752
753    /**
754     * Add an Date header using the supplied string and object values and return
755     * a <code>HeaderLine</code> object corresponding to how the header would be read.
756     * If both string and object values are not null they are used as is.
757     * If the string value is null and the object is not null,
758     * the object's toString method is called.
759     * If the object is null and the string is not null, the string is parsed
760     * and validated resulting in an object, if valid.
761     * @param fieldName name of field to add
762     * @param dateFieldValue <code>Date</code> field value object
763     * @param fieldValueStr Date field value string
764     * @return <code>HeaderLine</code> object corresponding to what would have been read
765     */
766    public HeaderLine addHeader(String fieldName, WarcDate dateFieldValue, String fieldValueStr) {
767        if (dateFieldValue == null && fieldValueStr != null) {
768            // FIXME Check check check.
769            //dateFieldValue = fieldParsers.parseDate(fieldValueStr, fieldName);
770            dateFieldValue = fieldParsers.parseWarcDate(fieldValueStr, fieldName);
771        } else if (fieldValueStr == null && dateFieldValue != null) {
772            // FIXME Check check check.
773            //fieldValueStr = WarcDateParser.getDateFormat().format(dateFieldValue);
774            // FIXME Tricky since precision is WARC version dependent!
775            fieldValueStr = dateFieldValue.toString();
776        }
777        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_DATE,
778                null, null, null, null, dateFieldValue, null, null);
779    }
780
781    /**
782     * Add an InetAddress header using the supplied string and object values and return
783     * a <code>HeaderLine</code> object corresponding to how the header would be read.
784     * If both string and object values are not null they are used as is.
785     * If the string value is null and the object is not null,
786     * the object's toString method is called.
787     * If the object is null and the string is not null, the string is parsed
788     * and validated resulting in an object, if valid.
789     * @param fieldName name of field to add
790     * @param inetAddrFieldValue <code>InetAddress</code> field value object
791     * @param fieldValueStr IP-Address field value string
792     * @return <code>HeaderLine</code> object corresponding to what would have been read
793     */
794    public HeaderLine addHeader(String fieldName, InetAddress inetAddrFieldValue, String fieldValueStr) {
795        if (inetAddrFieldValue == null && fieldValueStr != null) {
796            inetAddrFieldValue = fieldParsers.parseIpAddress(fieldValueStr, fieldName);
797        } else if (fieldValueStr == null && inetAddrFieldValue != null) {
798            fieldValueStr = inetAddrFieldValue.getHostAddress();
799        }
800        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_INETADDRESS,
801                null, null, null, null, null, inetAddrFieldValue, null);
802    }
803
804    /**
805     * Add an URI header using the supplied string and object values and return
806     * a <code>HeaderLine</code> object corresponding to how the header would be read.
807     * If both string and object values are not null they are used as is.
808     * If the string value is null and the object is not null,
809     * the object's toString method is called.
810     * If the object is null and the string is not null, the string is parsed
811     * and validated resulting in an object, if valid.
812     * @param fieldName name of field to add
813     * @param uriFieldValue <code>URI</code> field value object
814     * @param fieldValueStr URI field value string
815     * @return <code>HeaderLine</code> object corresponding to what would have been read
816     */
817    public HeaderLine addHeader(String fieldName, Uri uriFieldValue, String fieldValueStr) {
818        if (uriFieldValue == null && fieldValueStr != null) {
819            if (WarcConstants.FN_WARC_TARGET_URI.equalsIgnoreCase(fieldName)) {
820                uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, warcTargetUriProfile, fieldName);
821            } else if (WarcConstants.FN_WARC_PROFILE.equalsIgnoreCase(fieldName)) {
822                uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, uriProfile, fieldName);
823            } else if (WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.equalsIgnoreCase(fieldName)) {
824                uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, warcTargetUriProfile, fieldName);
825            } else {
826                uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_LTGT, uriProfile, fieldName);
827            }
828        } else if (fieldValueStr == null && uriFieldValue != null) {
829            if (WarcConstants.FN_WARC_TARGET_URI.equalsIgnoreCase(fieldName)
830                    || WarcConstants.FN_WARC_PROFILE.equalsIgnoreCase(fieldName)
831                    || WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.equalsIgnoreCase(fieldName)) {
832                fieldValueStr = uriFieldValue.toString();
833            } else {
834                fieldValueStr = "<" + uriFieldValue.toString() + ">";
835            }
836        }
837        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_URI,
838                null, null, null, null, null, null, uriFieldValue);
839    }
840
841    /**
842     * Add a header with the supplied field name, data type and value and
843     * return a <code>HeaderLine</code> corresponding to how the header will
844     * be read. The data type is validated against the field data type.
845     * The values used are the field value string and the parameter
846     * corresponding to the data type.
847     * @param fieldName header field name
848     * @param fieldValueStr field value in string form
849     * @param dt data type of the field value string when converted to an object
850     * @param integerFieldValue <code>Integer</code> object field value
851     * @param longFieldValue <code>Long</code> object field value
852     * @param digestFieldValue <code>Digest</code> object field value
853     * @param contentTypeFieldValue <code>ContentType</code> object field value
854     * @param dateFieldValue <code>Date</code> object field value
855     * @param inetAddrFieldValue <code>InetAddress</code> object field value
856     * @param uriFieldValue <code>URI</code> object field value
857     * @return <code>HeaderLine</code> object corresponding to what would have been read
858     */
859    public HeaderLine addHeader(String fieldName, String fieldValueStr, int dt,
860            Integer integerFieldValue, Long longFieldValue,
861            WarcDigest digestFieldValue, ContentType contentTypeFieldValue,
862            WarcDate dateFieldValue, InetAddress inetAddrFieldValue,
863            Uri uriFieldValue) {
864        Integer fn_idx = WarcConstants.fieldNameIdxMap.get(fieldName.toLowerCase());
865        if (fn_idx != null) {
866            // Implicit cast from integer to long, if needed.
867            if (WarcConstants.FN_IDX_DT[fn_idx] == WarcConstants.FDT_LONG && dt == WarcConstants.FDT_INTEGER) {
868                longFieldValue = (long)integerFieldValue;
869                dt = WarcConstants.FDT_LONG;
870            }
871            if (dt == WarcConstants.FN_IDX_DT[fn_idx]) {
872                // WARC field name defined in WARC specification.
873                if (seen[fn_idx] && !WarcConstants.fieldNamesRepeatableLookup[fn_idx]) {
874                    // Duplicate field.
875                    diagnostics.addError(DiagnosisType.DUPLICATE, "'" + fieldName + "' header", fieldValueStr);
876                }
877                seen[fn_idx] = true;
878                switch (fn_idx.intValue()) {
879                /*
880                 * Integer.
881                 */
882                case WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER:
883                    warcSegmentNumberStr = fieldValueStr;
884                    warcSegmentNumber = integerFieldValue;
885                    break;
886                /*
887                 * Long.
888                 */
889                case WarcConstants.FN_IDX_CONTENT_LENGTH:
890                    contentLengthStr = fieldValueStr;
891                    contentLength = longFieldValue;
892                    break;
893                case WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH:
894                    warcSegmentTotalLengthStr = fieldValueStr;
895                    warcSegmentTotalLength = longFieldValue;
896                    break;
897                /*
898                 * Digest.
899                 */
900                case WarcConstants.FN_IDX_WARC_BLOCK_DIGEST:
901                    warcBlockDigestStr = fieldValueStr;
902                    warcBlockDigest = digestFieldValue;
903                    break;
904                case WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST:
905                    warcPayloadDigestStr = fieldValueStr;
906                    warcPayloadDigest = digestFieldValue;
907                    break;
908                /*
909                 * ContentType.
910                 */
911                case WarcConstants.FN_IDX_CONTENT_TYPE:
912                    contentTypeStr = fieldValueStr;
913                    contentType = contentTypeFieldValue;
914                    break;
915                case WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE:
916                    warcIdentifiedPayloadTypeStr = fieldValueStr;
917                    warcIdentifiedPayloadType = contentTypeFieldValue;
918                    break;
919                /*
920                 * Date.
921                 */
922                case WarcConstants.FN_IDX_WARC_DATE:
923                    warcDateStr = fieldValueStr;
924                    // FIXME Maybe change parameter type if it does not break too many things.
925                    //warcDate = WarcDate.getWarcDate(dateFieldValue);
926                    warcDate = dateFieldValue;
927                    break;
928                case WarcConstants.FN_IDX_WARC_REFERS_TO_DATE:
929                    warcRefersToDateStr = fieldValueStr;
930                    // FIXME Maybe change parameter type if it does not break too many things.
931                    //warcRefersToDate = WarcDate.getWarcDate(dateFieldValue);
932                    warcRefersToDate = dateFieldValue;
933                    break;
934                /*
935                 * InetAddress.
936                 */
937                case WarcConstants.FN_IDX_WARC_IP_ADDRESS:
938                    warcIpAddress = fieldValueStr;
939                    warcInetAddress = inetAddrFieldValue;
940                    break;
941                /*
942                 * URI.
943                 */
944                case WarcConstants.FN_IDX_WARC_RECORD_ID:
945                    warcRecordIdStr = fieldValueStr;
946                    warcRecordIdUri = uriFieldValue;
947                    break;
948                case WarcConstants.FN_IDX_WARC_CONCURRENT_TO:
949                    if (fieldValueStr != null || uriFieldValue != null) {
950                        WarcConcurrentTo warcConcurrentTo = new WarcConcurrentTo();
951                        warcConcurrentTo.warcConcurrentToStr = fieldValueStr;
952                        warcConcurrentTo.warcConcurrentToUri = uriFieldValue;
953                        warcConcurrentToList.add(warcConcurrentTo);
954                    }
955                    break;
956                case WarcConstants.FN_IDX_WARC_REFERS_TO:
957                    warcRefersToStr = fieldValueStr;
958                    warcRefersToUri = uriFieldValue;
959                    break;
960                case WarcConstants.FN_IDX_WARC_TARGET_URI:
961                    warcTargetUriStr = fieldValueStr;
962                    warcTargetUriUri = uriFieldValue;
963                    break;
964                case WarcConstants.FN_IDX_WARC_WARCINFO_ID:
965                    warcWarcinfoIdStr = fieldValueStr;
966                    warcWarcinfoIdUri = uriFieldValue;
967                    break;
968                case WarcConstants.FN_IDX_WARC_PROFILE:
969                    warcProfileStr = fieldValueStr;
970                    warcProfileUri = uriFieldValue;
971                    if (warcProfileStr != null) {
972                        warcProfileIdx = WarcConstants.profileIdxMap.get(warcProfileStr.toLowerCase());
973                    }
974                    if (warcProfileIdx == null && warcProfileStr != null && warcProfileStr.length() > 0) {
975                        warcProfileIdx = WarcConstants.WARC_PROFILE_IDX_UNKNOWN;
976                    }
977                    break;
978                case WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID:
979                    warcSegmentOriginIdStr = fieldValueStr;
980                    warcSegmentOriginIdUrl = uriFieldValue;
981                    break;
982                case WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI:
983                    warcRefersToTargetUriStr = fieldValueStr;
984                    warcRefersToTargetUriUri = uriFieldValue;
985                    break;
986                default:
987                    break;
988                }
989            } else {
990                // Invalid datatype for field.
991                diagnostics.addError(DiagnosisType.INVALID_EXPECTED,
992                        "Invalid datatype for '" + fieldName + "' header",
993                        WarcConstants.FDT_IDX_STRINGS[WarcConstants.FN_IDX_DT[fn_idx]],
994                        WarcConstants.FDT_IDX_STRINGS[dt]);
995                // Consider throwing exception at some point.
996            }
997        }
998        HeaderLine headerLine = new HeaderLine();
999        headerLine.name = fieldName;
1000        headerLine.value = fieldValueStr;
1001        HeaderLine tmpLine = headerMap.get(fieldName.toLowerCase());
1002        if (tmpLine == null) {
1003            headerMap.put(fieldName.toLowerCase(), headerLine);
1004        } else {
1005            tmpLine.lines.add(headerLine);
1006        }
1007        return headerLine;
1008    }
1009
1010}