001/**
002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC
003 * and GZip files. (http://jwat.org/)
004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/)
005 *
006 * Licensed under the Apache License, Version 2.0 (the "License");
007 * you may not use this file except in compliance with the License.
008 * You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.jwat.warc;
019
020import java.io.ByteArrayOutputStream;
021import java.io.IOException;
022import java.net.InetAddress;
023import java.util.Collections;
024import java.util.Date;
025import java.util.HashMap;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029
030import org.jwat.common.ByteCountingPushBackInputStream;
031import org.jwat.common.ContentType;
032import org.jwat.common.Diagnosis;
033import org.jwat.common.DiagnosisType;
034import org.jwat.common.Diagnostics;
035import org.jwat.common.HeaderLine;
036import org.jwat.common.HeaderLineReader;
037import org.jwat.common.MaxLengthRecordingInputStream;
038import org.jwat.common.Uri;
039import org.jwat.common.UriProfile;
040
041/**
042 * Central class for working with WARC headers. This class includes support for
043 * reading and writing WARC headers. Methods are also available to validate
044 * individual headers and a WARC header as a whole.
045 *
046 * @author nicl
047 */
048public class WarcHeader {
049
050    /** An URI with encapsulating <> characters. */
051    public static final boolean URI_LTGT = true;
052
053    /** An URI without encapsulating <> characters. */
054    public static final boolean URI_NAKED = false;
055
056    /** Associated WarcReader context.
057     *  Must be set prior to calling the various methods. */
058    protected WarcReader reader;
059
060    /** Diagnostics used to report diagnoses.
061     *  Must be set prior to calling the various methods. */
062    protected Diagnostics<Diagnosis> diagnostics;
063
064    /** WARC-Target-URI profile. */
065    protected UriProfile warcTargetUriProfile;
066
067    /** URI profile. */
068    protected UriProfile uriProfile;
069
070    /** WARC field parser used.
071     *  Must be set prior to calling the various methods. */
072    protected WarcFieldParsers fieldParsers;
073
074    /** Max size allowed for a record header. */
075    protected int recordHeaderMaxSize = -1;
076
077    /** Line reader used to read version lines. */
078    protected HeaderLineReader lineReader;
079
080    /** Header line reader used to read the WARC headers. */
081    protected HeaderLineReader headerLineReader;
082
083    /** WARC record starting offset relative to the source WARC file input
084     *  stream. The offset is correct for both compressed and uncompressed streams. */
085    protected long startOffset = -1;
086
087    /*
088     * Version related fields.
089     */
090
091    /** Was "WARC/" identified while looking for the version string. */
092    public boolean bMagicIdentified;
093    /** Did the version string include between 2 and 4 substrings delimited by ".". */
094    public boolean bVersionParsed;
095    /** Is the version format valid. */
096    public boolean bValidVersionFormat;
097    /** Is the version recognized. (0.17, 0.18 or 1.0) */
098    public boolean bValidVersion;
099
100    /** Raw version string. */
101    public String versionStr;
102    /** Array based on the version string split by the "." delimiter and converted to integers. */
103    public int[] versionArr;
104
105    /** Major version number from WARC header. */
106    public int major = -1;
107    /** Minor version number from WARC header. */
108    public int minor = -1;
109
110    /*
111     * WARC header fields.
112     */
113
114    /** Array used for duplicate header detection. */
115    protected boolean[] seen = new boolean[WarcConstants.FN_INDEX_OF_LAST];
116
117    /** Is the header missing one of the mandatory headers. */
118    public boolean bMandatoryMissing;
119
120    /** WARC-Type field string value. */
121    public String warcTypeStr;
122    /** WARC-Type converted to an integer id, if identified. */
123    public Integer warcTypeIdx;
124
125    /** WARC-Filename field string value.
126     *  (warcinfo record type only) */
127    public String warcFilename;
128
129    /** WARC-Record-Id field string value. */
130    public String warcRecordIdStr;
131    /** WARC-Record-Id converted to an <code>Uri</code> object, if valid. */
132    public Uri warcRecordIdUri;
133
134    /** WARC-Date field string value. */
135    public String warcDateStr;
136    /** WARC-Date converted to a <code>Date</code> object, if valid. */
137    public Date warcDate;
138
139    /** Content-Length field string value. */
140    public String contentLengthStr;
141    /** Content-Length converted to a <code>Long</code> object, if valid. */
142    public Long contentLength;
143
144    /** Content-Type field string value. */
145    public String contentTypeStr;
146    /** Content-Type converted to a <code>ContentType</code> object, if valid. */
147    public ContentType contentType;
148
149    /** WARC-Truncated field string value. */
150    public String warcTruncatedStr;
151    /** WARC-Truncated converted to an integer id, if valid. */
152    public Integer warcTruncatedIdx;
153
154    /** WARC-IP-Address field string value. */
155    public String warcIpAddress;
156    /** WARC-IP-Address converted to an <code>InetAddress</code> object, if valid. */
157    public InetAddress warcInetAddress;
158
159    /** List of WARC-Concurrent-To field string values and converted <code>URI</code> objects,  if valid. */
160    public List<WarcConcurrentTo> warcConcurrentToList = new LinkedList<WarcConcurrentTo>();
161
162    /** WARC-Refers-To field string value. */
163    public String warcRefersToStr;
164    /** WARC-Refers-To converted to an <code>Uri</code> object, if valid. */
165    public Uri warcRefersToUri;
166
167    /** WARC_Target-URI field string value. */
168    public String warcTargetUriStr;
169    /** WARC-TargetURI converted to an <code>Uri</code> object, if valid. */
170    public Uri warcTargetUriUri;
171
172    /** WARC-Warcinfo-Id field string value. */
173    public String warcWarcinfoIdStr;
174    /** WARC-Warcinfo-Id converted to an <code>Uri</code> object, if valid. */
175    public Uri warcWarcinfoIdUri;
176
177    /** WARC-Block-Digest field string value. */
178    public String warcBlockDigestStr;
179    /** WARC-Block-Digest converted to a <code>WarcDigest</code> object, if valid. */
180    public WarcDigest warcBlockDigest;
181
182    /** WARC-Payload-Digest field string value. */
183    public String warcPayloadDigestStr;
184    /** WARC-Payload-Digest converted to a <code>WarcDigest</code> object, if valid. */
185    public WarcDigest warcPayloadDigest;
186
187    /** WARC-Identified-Payload-Type field string value. */
188    public String warcIdentifiedPayloadTypeStr;
189    /** WARC-Identified-Payload-Type converted to a <code>ContentType</code> object, if valid. */
190    public ContentType warcIdentifiedPayloadType;
191
192    /** WARC-Profile field string value.
193     *  (revisit record only) */
194    public String warcProfileStr;
195    /** WARC-Profile field converted to an <code>Uri</code> object, if valid.
196     *  (revisit record only) */
197    public Uri warcProfileUri;
198    /** WARC-Profile converted to an integer id, if valid.
199     *  (revisit record only) */
200    public Integer warcProfileIdx;
201
202    /** WARC-Segment-Number field string value. */
203    public String warcSegmentNumberStr;
204    /** WARC-Segment-Number converted to an <code>Integer</code> object, if valid. */
205    public Integer warcSegmentNumber;
206
207    /** WARC-Segment-Origin-Id field string value.
208     *  (continuation record only) */
209    public String warcSegmentOriginIdStr;
210    /** WARC-Segment-Origin-Id converted to an <code>Uri</code> object, if valid.
211     *  (continuation record only) */
212    public Uri warcSegmentOriginIdUrl;
213
214    /** WARC-Segment-Total-Length field string value.
215     *  (continuation record only) */
216    public String warcSegmentTotalLengthStr;
217    /** WARC-Segment-Total-Length converted to a <code>Long</code> object, if valid.
218     *  (continuation record only) */
219    public Long warcSegmentTotalLength;
220
221    // see https://docs.google.com/document/d/1QyQBA7Ykgxie75V8Jziz_O7hbhwf7PF6_u9O6w6zgp0/edit
222    /** WARC-Refers-To-Target-URI field string value. */
223    public String warcRefersToTargetUriStr;
224    /** WARC-Refers-To-Target-URI converted to an <code>Uri</code> object, if valid. */
225    public Uri warcRefersToTargetUriUri;
226    /** WARC-Refers-To-Date */
227    public String warcRefersToDateStr;
228    /** WARC-Date converted to a <code>Date</code> object, if valid. */
229    public Date warcRefersToDate;
230
231    /*
232     * WARC header fields collections.
233     */
234
235    /** Raw WARC header output stream. */
236    protected ByteArrayOutputStream headerBytesOut = new ByteArrayOutputStream();
237
238    /** Raw WARC header byte array. */
239    public byte[] headerBytes;
240
241    /** List of parsed header fields. */
242    protected List<HeaderLine> headerList = new LinkedList<HeaderLine>();
243
244    /** Map of parsed header fields. */
245    protected Map<String, HeaderLine> headerMap = new HashMap<String, HeaderLine>();
246
247    /**
248     * Non public constructor to allow unit testing.
249     */
250    protected WarcHeader() {
251    }
252
253    /**
254     * Create a new <code>WarcHeader</code> for recreating a header object.
255     * @param fieldParsers parsers used for the individual types
256     * @param uriProfile uri profile used to validate urls
257     * @param diagnostics diagnostics object used by parser
258     * @return a <code>WarcHeader</code> prepared for reanimation
259     */
260    public static WarcHeader initHeader(int recordHeaderMaxSize, HeaderLineReader lineReader, HeaderLineReader headerLineReader, WarcFieldParsers fieldParsers, UriProfile uriProfile, Diagnostics<Diagnosis> diagnostics) {
261        WarcHeader header = new WarcHeader();
262        header.warcTargetUriProfile = uriProfile;
263        header.uriProfile = uriProfile;
264        header.fieldParsers = fieldParsers;
265        header.diagnostics = diagnostics;
266        header.recordHeaderMaxSize = recordHeaderMaxSize;
267        header.lineReader = lineReader;
268        header.headerLineReader = headerLineReader;
269        return header;
270    }
271
272    /**
273     * Create and initialize a new <code>WarcHeader</code> for writing.
274     * @param writer writer which shall be used
275     * @param diagnostics diagnostics object used by writer
276     * @return a <code>WarcHeader</code> prepared for writing
277     */
278    public static WarcHeader initHeader(WarcWriter writer, Diagnostics<Diagnosis> diagnostics) {
279        WarcHeader header = new WarcHeader();
280        // Set default version to "1.0".
281        header.major = 1;
282        header.minor = 0;
283        header.warcTargetUriProfile = writer.warcTargetUriProfile;
284        header.uriProfile = writer.uriProfile;
285        header.fieldParsers = writer.fieldParsers;
286        header.diagnostics = diagnostics;
287        return header;
288    }
289
290    /**
291     * Create and initialize a new <code>WarcHeader</code> for reading.
292     * @param reader reader which shall be used
293     * @param startOffset start offset of header
294     * @param diagnostics diagnostics object used by reader
295     * @return a <code>WarcHeader</code> prepared for reading
296     */
297    public static WarcHeader initHeader(WarcReader reader, long startOffset, Diagnostics<Diagnosis> diagnostics) {
298        WarcHeader header = new WarcHeader();
299        header.reader = reader;
300        header.warcTargetUriProfile = reader.warcTargetUriProfile;
301        header.uriProfile = reader.uriProfile;
302        header.fieldParsers = reader.fieldParsers;
303        header.diagnostics = diagnostics;
304        header.recordHeaderMaxSize = reader.recordHeaderMaxSize;
305        header.lineReader = reader.lineReader;
306        header.headerLineReader = reader.headerLineReader;
307        // This is only relevant for uncompressed sequentially read records
308        header.startOffset = startOffset;
309        return header;
310    }
311
312    /**
313     * Add an error diagnosis of the given type on a specific entity with
314     * optional extra information. The information varies according to the
315     * diagnosis type.
316     * @param type diagnosis type
317     * @param entity entity examined
318     * @param information optional extra information
319     */
320    protected void addErrorDiagnosis(DiagnosisType type, String entity, String... information) {
321        diagnostics.addError(new Diagnosis(type, entity, information));
322    }
323
324    /**
325     * Add a warning diagnosis of the given type on a specific entity with
326     * optional extra information. The information varies according to the
327     * diagnosis type.
328     * @param type diagnosis type
329     * @param entity entity examined
330     * @param information optional extra information
331     */
332    protected void addWarningDiagnosis(DiagnosisType type, String entity, String... information) {
333        diagnostics.addWarning(new Diagnosis(type, entity, information));
334    }
335
336    /**
337     * Returns the starting offset of the record in the containing WARC.
338     * @return the starting offset of the record
339     */
340    public long getStartOffset() {
341        return startOffset;
342    }
343
344    /**
345     * Try to parse a WARC header and return a boolean indicating the success or
346     * failure of this.
347     * @param in input stream with WARC data
348     * @return boolean indicating whether a header was parsed or not
349     * @throws IOException if an I/O exception occurs while parsing for a header
350     */
351    public boolean parseHeader(ByteCountingPushBackInputStream in) throws IOException {
352        if (parseVersion(in)) {
353            // debug
354            //System.out.println(wr.bMagicIdentified);
355            //System.out.println(wr.bVersionParsed);
356            //System.out.println(wr.major + "." + wr.minor);
357            if (bVersionParsed && versionArr.length == 2) {
358                switch (major) {
359                case 1:
360                    if (minor == 0) {
361                        bValidVersion = true;
362                    }
363                    break;
364                case 0:
365                    switch (minor) {
366                    case 17:
367                    case 18:
368                        bValidVersion = true;
369                        break;
370                    }
371                    break;
372                default:
373                    break;
374                }
375                if (!bValidVersion) {
376                    diagnostics.addError(
377                            new Diagnosis(DiagnosisType.UNKNOWN,
378                                    "Magic version number", versionStr));
379                }
380            } else {
381                diagnostics.addError(
382                        new Diagnosis(DiagnosisType.INVALID_DATA,
383                                "Magic Version string", versionStr));
384            }
385
386            MaxLengthRecordingInputStream mrin = new MaxLengthRecordingInputStream(in, recordHeaderMaxSize);
387            ByteCountingPushBackInputStream pbin = new ByteCountingPushBackInputStream(mrin, recordHeaderMaxSize);
388
389            parseHeaders(pbin);
390            pbin.close();
391
392            checkFields();
393
394            headerBytes = headerBytesOut.toByteArray();
395        }
396        return bMagicIdentified;
397    }
398
399    /**
400     * Looks forward in the input stream for a valid WARC version line.
401     * @param in data input stream
402     * @return true, if magic WARC header found
403     * @throws IOException if an error occurs while reading version data
404     */
405    public boolean parseVersion(ByteCountingPushBackInputStream in) throws IOException {
406        bMagicIdentified = false;
407        bVersionParsed = false;
408        boolean bInvalidDataBeforeVersion = false;
409        boolean bEmptyLinesBeforeVersion = false;
410        HeaderLine line;
411        String tmpStr;
412        boolean bSeekMagic = true;
413        // Loop until when have found something that looks like a version line.
414        while (bSeekMagic) {
415            // This is only relevant for uncompressed sequentially read records
416            startOffset = in.getConsumed();
417            line = lineReader.readLine(in);
418            if (!lineReader.bEof) {
419                switch (line.type) {
420                case HeaderLine.HLT_LINE:
421                    tmpStr = line.line;
422                    // debug
423                    //System.out.println(tmpStr);
424                    if (tmpStr.length() > 0) {
425                        if (tmpStr.toUpperCase().startsWith(WarcConstants.WARC_MAGIC_HEADER)) {
426                            bMagicIdentified = true;
427                            versionStr = tmpStr.substring(WarcConstants.WARC_MAGIC_HEADER.length());
428                            String[] tmpArr = versionStr.split("\\.", -1);        // Not optimal
429                            if (tmpArr.length >= 2 && tmpArr.length <= 4) {
430                                bVersionParsed = true;
431                                bValidVersionFormat = true;
432                                versionArr = new int[tmpArr.length];
433                                for (int i=0; i<tmpArr.length; ++i) {
434                                    try {
435                                        versionArr[i] = Integer.parseInt(tmpArr[i]);
436                                    } catch (NumberFormatException e) {
437                                        versionArr[i] = -1;
438                                        bValidVersionFormat = false;
439                                    }
440                                }
441                                major = versionArr[0];
442                                minor = versionArr[1];
443                            }
444                            headerBytesOut.write(line.raw);
445                            bSeekMagic = false;
446                        } else {
447                            // Invalid data aka Gibberish.
448                            bInvalidDataBeforeVersion = true;
449                        }
450                    } else {
451                        // Empty line.
452                        bEmptyLinesBeforeVersion = true;
453
454                    }
455                    break;
456                case HeaderLine.HLT_HEADERLINE:
457                    // Invalid data - header or binary.
458                    bInvalidDataBeforeVersion = true;
459                    break;
460                }
461            } else {
462                // EOF.
463                bSeekMagic = false;
464            }
465        }
466        if (bInvalidDataBeforeVersion) {
467            addErrorDiagnosis(DiagnosisType.INVALID, "Data before WARC version");
468        }
469        if (bEmptyLinesBeforeVersion) {
470            addErrorDiagnosis(DiagnosisType.INVALID, "Empty lines before WARC version");
471        }
472        return bMagicIdentified;
473    }
474
475    /**
476     * Reads WARC header lines one line at a time until an empty line is
477     * encountered.
478     * @param in header input stream
479     * @throws IOException if an error occurs while reading the WARC header
480     */
481    public void parseHeaders(ByteCountingPushBackInputStream in) throws IOException {
482        HeaderLine headerLine;
483        boolean bLoop = true;
484        while (bLoop) {
485            headerLine = headerLineReader.readLine(in);
486            if ((headerLine.bfErrors & HeaderLineReader.E_BIT_INVALID_CHARSET) != 0) {
487                addErrorDiagnosis(DiagnosisType.INVALID_ENCODING, "Invalid encoding in header line", headerLine.value, "UNKNOWN");
488            }
489            if (!headerLineReader.bEof) {
490                headerBytesOut.write(headerLine.raw);
491                switch (headerLine.type) {
492                case HeaderLine.HLT_HEADERLINE:
493                    if (headerLine.name != null && headerLine.name.length() > 0) {
494                        // debug
495                        //System.out.println(headerLine.name);
496                        //System.out.println(headerLine.value);
497                        addHeader(headerLine);
498                    } else {
499                        // Empty field name.
500                        addWarningDiagnosis(DiagnosisType.EMPTY, "Header line");
501                    }
502                    break;
503                case HeaderLine.HLT_LINE:
504                    if (headerLine.line.length() == 0) {
505                        // Empty line.
506                        bLoop = false;
507                    } else {
508                        // Unknown header line.
509                        addWarningDiagnosis(DiagnosisType.UNKNOWN, "Header line", headerLine.line);
510                    }
511                    break;
512                default:
513                    throw new IllegalStateException("Invalid HeaderLine output!");
514                }
515            } else {
516                // EOF.
517                bLoop = false;
518            }
519        }
520    }
521
522    /**
523     * Identify a (WARC) header name, validate the value and set the header.
524     * @param headerLine the headerLine
525     */
526    public void addHeader(HeaderLine headerLine) {
527        String fieldName = headerLine.name;
528        String fieldValue = headerLine.value;
529        WarcConcurrentTo warcConcurrentTo;
530        Integer fn_idx = WarcConstants.fieldNameIdxMap.get(fieldName.toLowerCase());
531        if (fn_idx != null) {
532            // WARC field name defined in WARC specification.
533            if (!seen[fn_idx] || WarcConstants.fieldNamesRepeatableLookup[fn_idx]) {
534                seen[fn_idx] = true;
535                switch (fn_idx.intValue()) {
536                case WarcConstants.FN_IDX_WARC_TYPE:
537                    warcTypeStr = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_TYPE);
538                    if (warcTypeStr != null) {
539                        warcTypeIdx = WarcConstants.recordTypeIdxMap.get(warcTypeStr.toLowerCase());
540                    }
541                    if (warcTypeIdx == null && warcTypeStr != null && warcTypeStr.length() > 0) {
542                        warcTypeIdx = WarcConstants.RT_IDX_UNKNOWN;
543                    }
544                    break;
545                case WarcConstants.FN_IDX_WARC_RECORD_ID:
546                    warcRecordIdStr = fieldValue;
547                    warcRecordIdUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_RECORD_ID);
548                    break;
549                case WarcConstants.FN_IDX_WARC_DATE:
550                    warcDateStr = fieldValue;
551                    warcDate = fieldParsers.parseDate(fieldValue, WarcConstants.FN_WARC_DATE);
552                    break;
553                case WarcConstants.FN_IDX_CONTENT_LENGTH:
554                    contentLengthStr = fieldValue;
555                    contentLength = fieldParsers.parseLong(fieldValue, WarcConstants.FN_CONTENT_LENGTH);
556                    break;
557                case WarcConstants.FN_IDX_CONTENT_TYPE:
558                    contentTypeStr = fieldValue;
559                    contentType = fieldParsers.parseContentType(fieldValue, WarcConstants.FN_CONTENT_TYPE);
560                    break;
561                case WarcConstants.FN_IDX_WARC_CONCURRENT_TO:
562                    Uri tmpUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_CONCURRENT_TO);
563                    if (fieldValue != null && fieldValue.trim().length() > 0) {
564                        warcConcurrentTo = new WarcConcurrentTo();
565                        warcConcurrentTo.warcConcurrentToStr = fieldValue;
566                        warcConcurrentTo.warcConcurrentToUri = tmpUri;
567                        warcConcurrentToList.add(warcConcurrentTo);
568                    }
569                    break;
570                case WarcConstants.FN_IDX_WARC_BLOCK_DIGEST:
571                    warcBlockDigestStr = fieldValue;
572                    warcBlockDigest = fieldParsers.parseDigest(fieldValue, WarcConstants.FN_WARC_BLOCK_DIGEST);
573                    break;
574                case WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST:
575                    warcPayloadDigestStr = fieldValue;
576                    warcPayloadDigest = fieldParsers.parseDigest(fieldValue, WarcConstants.FN_WARC_PAYLOAD_DIGEST);
577                    break;
578                case WarcConstants.FN_IDX_WARC_IP_ADDRESS:
579                    warcIpAddress = fieldValue;
580                    warcInetAddress = fieldParsers.parseIpAddress(fieldValue, WarcConstants.FN_WARC_IP_ADDRESS);
581                    break;
582                case WarcConstants.FN_IDX_WARC_REFERS_TO:
583                    warcRefersToStr = fieldValue;
584                    warcRefersToUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_REFERS_TO);
585                    break;
586                case WarcConstants.FN_IDX_WARC_TARGET_URI:
587                    warcTargetUriStr = fieldValue;
588                    warcTargetUriUri = fieldParsers.parseUri(fieldValue, URI_NAKED, warcTargetUriProfile, WarcConstants.FN_WARC_TARGET_URI);
589                    break;
590                case WarcConstants.FN_IDX_WARC_TRUNCATED:
591                    warcTruncatedStr = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_TRUNCATED);
592                    if (warcTruncatedStr != null) {
593                        warcTruncatedIdx = WarcConstants.truncatedTypeIdxMap.get(warcTruncatedStr.toLowerCase());
594                    }
595                    if (warcTruncatedIdx == null && warcTruncatedStr != null && warcTruncatedStr.length() > 0) {
596                        warcTruncatedIdx = WarcConstants.TT_IDX_FUTURE_REASON;
597                    }
598                    break;
599                case WarcConstants.FN_IDX_WARC_WARCINFO_ID:
600                    warcWarcinfoIdStr = fieldValue;
601                    warcWarcinfoIdUri = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_WARCINFO_ID);
602                    break;
603                case WarcConstants.FN_IDX_WARC_FILENAME:
604                    warcFilename = fieldParsers.parseString(fieldValue, WarcConstants.FN_WARC_FILENAME);
605                    break;
606                case WarcConstants.FN_IDX_WARC_PROFILE:
607                    warcProfileStr = fieldValue;
608                    warcProfileUri = fieldParsers.parseUri(fieldValue, URI_NAKED, uriProfile, WarcConstants.FN_WARC_PROFILE);
609                    if (warcProfileStr != null) {
610                        warcProfileIdx = WarcConstants.profileIdxMap.get(warcProfileStr.toLowerCase());
611                    }
612                    if (warcProfileIdx == null && warcProfileStr != null && warcProfileStr.length() > 0) {
613                        warcProfileIdx = WarcConstants.PROFILE_IDX_UNKNOWN;
614                    }
615                    break;
616                case WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE:
617                    warcIdentifiedPayloadTypeStr = fieldValue;
618                    warcIdentifiedPayloadType = fieldParsers.parseContentType(fieldValue, WarcConstants.FN_WARC_IDENTIFIED_PAYLOAD_TYPE);
619                    break;
620                case WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID:
621                    warcSegmentOriginIdStr = fieldValue;
622                    warcSegmentOriginIdUrl = fieldParsers.parseUri(fieldValue, URI_LTGT, uriProfile, WarcConstants.FN_WARC_SEGMENT_ORIGIN_ID);
623                    break;
624                case WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER:
625                    warcSegmentNumberStr = fieldValue;
626                    warcSegmentNumber = fieldParsers.parseInteger(fieldValue, WarcConstants.FN_WARC_SEGMENT_NUMBER);
627                    break;
628                case WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH:
629                    warcSegmentTotalLengthStr = fieldValue;
630                    warcSegmentTotalLength = fieldParsers.parseLong(fieldValue, WarcConstants.FN_WARC_SEGMENT_TOTAL_LENGTH);
631                    break;
632                case WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI:
633                    warcRefersToTargetUriStr = fieldValue;
634                    warcRefersToTargetUriUri = fieldParsers.parseUri(fieldValue, URI_NAKED, uriProfile, WarcConstants.FN_WARC_REFERS_TO_TARGET_URI);
635                    break;
636                case WarcConstants.FN_IDX_WARC_REFERS_TO_DATE:
637                    warcRefersToDateStr = fieldValue;
638                    warcRefersToDate = fieldParsers.parseDate(fieldValue, WarcConstants.FN_WARC_REFERS_TO_DATE);
639                    break;
640                }
641            } else {
642                // Duplicate field.
643                addErrorDiagnosis(DiagnosisType.DUPLICATE, "'" + fieldName + "' header", fieldValue);
644            }
645        }
646        HeaderLine tmpLine = headerMap.get(fieldName.toLowerCase());
647        if (tmpLine == null) {
648            headerMap.put(fieldName.toLowerCase(), headerLine);
649        } else {
650            tmpLine.lines.add(headerLine);
651        }
652        headerList.add(headerLine);
653    }
654
655    /**
656     * Get a <code>List</code> of all the headers found during parsing.
657     * @return <code>List</code> of <code>HeaderLine</code>
658     */
659    public List<HeaderLine> getHeaderList() {
660        return Collections.unmodifiableList(headerList);
661    }
662
663    /**
664     * Get a header line structure or null, if no header line structure is
665     * stored with the given header name.
666     * @param field header name
667     * @return <code>HeaderLine</code> structure or null
668     */
669    public HeaderLine getHeader(String field) {
670        if (field != null && field.length() > 0) {
671            return headerMap.get(field.toLowerCase());
672        } else {
673            return null;
674        }
675    }
676
677    /**
678     * Add a String header using the supplied string and return a
679     * <code>HeaderLine</code> object corresponding to how the header would be
680     * read.
681     * @param fieldName name of field to add
682     * @param fieldValue field value string
683     * @return <code>HeaderLine</code> object corresponding to what would have been read
684     */
685    public HeaderLine addHeader(String fieldName, String fieldValue) {
686        HeaderLine headerLine = new HeaderLine();
687        headerLine.name = fieldName;
688        headerLine.value = fieldValue;
689        addHeader(headerLine);
690        return headerLine;
691    }
692
693    /**
694     * Add an Integer header using the supplied string and object values and return
695     * a <code>HeaderLine</code> object corresponding to how the header would be read.
696     * If both string and object values are not null they are used as is.
697     * If the string value is null and the object is not null,
698     * the object's toString method is called.
699     * If the object is null and the string is not null, the string is parsed
700     * and validated resulting in an object, if valid.
701     * @param fieldName name of field to add
702     * @param integerFieldValue <code>Integer</code> field value object
703     * @param fieldValueStr Integer field value string
704     * @return <code>HeaderLine</code> object corresponding to what would have been read
705     */
706    public HeaderLine addHeader(String fieldName, Integer integerFieldValue, String fieldValueStr) {
707        if (integerFieldValue == null && fieldValueStr != null) {
708            integerFieldValue = fieldParsers.parseInteger(fieldValueStr, fieldName);
709        } else if (fieldValueStr == null && integerFieldValue != null) {
710            fieldValueStr = integerFieldValue.toString();
711        }
712        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_INTEGER,
713                integerFieldValue, null, null, null, null, null, null);
714    }
715
716    /**
717     * Add a Long header using the supplied string and object values and return
718     * a <code>HeaderLine</code> object corresponding to how the header would be read.
719     * If both string and object values are not null they are used as is.
720     * If the string value is null and the object is not null,
721     * the object's toString method is called.
722     * If the object is null and the string is not null, the string is parsed
723     * and validated resulting in an object, if valid.
724     * @param fieldName name of field to add
725     * @param longFieldValue <code>Long</code> field value object
726     * @param fieldValueStr Long field value string
727     * @return <code>HeaderLine</code> object corresponding to what would have been read
728     */
729    public HeaderLine addHeader(String fieldName, Long longFieldValue, String fieldValueStr) {
730        if (longFieldValue == null && fieldValueStr != null) {
731            longFieldValue = fieldParsers.parseLong(fieldValueStr, fieldName);
732        } else if (fieldValueStr == null && longFieldValue != null) {
733            fieldValueStr = longFieldValue.toString();
734        }
735        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_LONG,
736                null, longFieldValue, null, null, null, null, null);
737    }
738
739    /**
740     * Add an Digest header using the supplied string and object values and return
741     * a <code>HeaderLine</code> object corresponding to how the header would be read.
742     * If both string and object values are not null they are used as is.
743     * If the string value is null and the object is not null,
744     * the object's toString method is called.
745     * If the object is null and the string is not null, the string is parsed
746     * and validated resulting in an object, if valid.
747     * @param fieldName name of field to add
748     * @param digestFieldValue <code>Digest</code> field value object
749     * @param fieldValueStr Digest field value string
750     * @return <code>HeaderLine</code> object corresponding to what would have been read
751     */
752    public HeaderLine addHeader(String fieldName, WarcDigest digestFieldValue, String fieldValueStr) {
753        if (digestFieldValue == null && fieldValueStr != null) {
754            digestFieldValue = fieldParsers.parseDigest(fieldValueStr, fieldName);
755        } else if (fieldValueStr == null && digestFieldValue != null) {
756            fieldValueStr = digestFieldValue.toString();
757        }
758        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_DIGEST,
759                null, null, digestFieldValue, null, null, null, null);
760    }
761
762    /**
763     * Add an Content-Type header using the supplied string and object values and return
764     * a <code>HeaderLine</code> object corresponding to how the header would be read.
765     * If both string and object values are not null they are used as is.
766     * If the string value is null and the object is not null,
767     * the object's toString method is called.
768     * If the object is null and the string is not null, the string is parsed
769     * and validated resulting in an object, if valid.
770     * @param fieldName name of field to add
771     * @param contentTypeFieldValue <code>ContentType</code> field value object
772     * @param fieldValueStr Content-Type field value string
773     * @return <code>HeaderLine</code> object corresponding to what would have been read
774     */
775    public HeaderLine addHeader(String fieldName, ContentType contentTypeFieldValue, String fieldValueStr) {
776        if (contentTypeFieldValue == null && fieldValueStr != null) {
777            contentTypeFieldValue = fieldParsers.parseContentType(fieldValueStr, fieldName);
778        } else if (fieldValueStr == null && contentTypeFieldValue != null) {
779            fieldValueStr = contentTypeFieldValue.toString();
780        }
781        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_CONTENTTYPE,
782                null, null, null, contentTypeFieldValue, null, null, null);
783    }
784
785    /**
786     * Add an Date header using the supplied string and object values and return
787     * a <code>HeaderLine</code> object corresponding to how the header would be read.
788     * If both string and object values are not null they are used as is.
789     * If the string value is null and the object is not null,
790     * the object's toString method is called.
791     * If the object is null and the string is not null, the string is parsed
792     * and validated resulting in an object, if valid.
793     * @param fieldName name of field to add
794     * @param dateFieldValue <code>Date</code> field value object
795     * @param fieldValueStr Date field value string
796     * @return <code>HeaderLine</code> object corresponding to what would have been read
797     */
798    public HeaderLine addHeader(String fieldName, Date dateFieldValue, String fieldValueStr) {
799        if (dateFieldValue == null && fieldValueStr != null) {
800            dateFieldValue = fieldParsers.parseDate(fieldValueStr, fieldName);
801        } else if (fieldValueStr == null && dateFieldValue != null) {
802            fieldValueStr = WarcDateParser.getDateFormat().format(dateFieldValue);
803        }
804        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_DATE,
805                null, null, null, null, dateFieldValue, null, null);
806    }
807
808    /**
809     * Add an InetAddress header using the supplied string and object values and return
810     * a <code>HeaderLine</code> object corresponding to how the header would be read.
811     * If both string and object values are not null they are used as is.
812     * If the string value is null and the object is not null,
813     * the object's toString method is called.
814     * If the object is null and the string is not null, the string is parsed
815     * and validated resulting in an object, if valid.
816     * @param fieldName name of field to add
817     * @param inetAddrFieldValue <code>InetAddress</code> field value object
818     * @param fieldValueStr IP-Address field value string
819     * @return <code>HeaderLine</code> object corresponding to what would have been read
820     */
821    public HeaderLine addHeader(String fieldName, InetAddress inetAddrFieldValue, String fieldValueStr) {
822        if (inetAddrFieldValue == null && fieldValueStr != null) {
823            inetAddrFieldValue = fieldParsers.parseIpAddress(fieldValueStr, fieldName);
824        } else if (fieldValueStr == null && inetAddrFieldValue != null) {
825            fieldValueStr = inetAddrFieldValue.getHostAddress();
826        }
827        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_INETADDRESS,
828                null, null, null, null, null, inetAddrFieldValue, null);
829    }
830
831    /**
832     * Add an URI header using the supplied string and object values and return
833     * a <code>HeaderLine</code> object corresponding to how the header would be read.
834     * If both string and object values are not null they are used as is.
835     * If the string value is null and the object is not null,
836     * the object's toString method is called.
837     * If the object is null and the string is not null, the string is parsed
838     * and validated resulting in an object, if valid.
839     * @param fieldName name of field to add
840     * @param uriFieldValue <code>URI</code> field value object
841     * @param fieldValueStr URI field value string
842     * @return <code>HeaderLine</code> object corresponding to what would have been read
843     */
844    public HeaderLine addHeader(String fieldName, Uri uriFieldValue, String fieldValueStr) {
845        if (uriFieldValue == null && fieldValueStr != null) {
846            if (WarcConstants.FN_WARC_TARGET_URI.equalsIgnoreCase(fieldName)) {
847                uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, warcTargetUriProfile, fieldName);
848            } else if (WarcConstants.FN_WARC_PROFILE.equalsIgnoreCase(fieldName)) {
849                uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, uriProfile, fieldName);
850            } else if (WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.equalsIgnoreCase(fieldName)) {
851                uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_NAKED, warcTargetUriProfile, fieldName);
852            } else {
853                uriFieldValue = fieldParsers.parseUri(fieldValueStr, URI_LTGT, uriProfile, fieldName);
854            }
855        } else if (fieldValueStr == null && uriFieldValue != null) {
856            if (WarcConstants.FN_WARC_TARGET_URI.equalsIgnoreCase(fieldName)
857                    || WarcConstants.FN_WARC_PROFILE.equalsIgnoreCase(fieldName)
858                    || WarcConstants.FN_WARC_REFERS_TO_TARGET_URI.equalsIgnoreCase(fieldName)) {
859                fieldValueStr = uriFieldValue.toString();
860            } else {
861                fieldValueStr = "<" + uriFieldValue.toString() + ">";
862            }
863        }
864        return addHeader(fieldName, fieldValueStr, WarcConstants.FDT_URI,
865                null, null, null, null, null, null, uriFieldValue);
866    }
867
868    /**
869     * Add a header with the supplied field name, data type and value and
870     * return a <code>HeaderLine</code> corresponding to how the header will
871     * be read. The data type is validated against the field data type.
872     * The values used are the field value string and the parameter
873     * corresponding to the data type.
874     * @param fieldName header field name
875     * @param fieldValueStr field value in string form
876     * @param dt data type of the field value string when converted to an object
877     * @param integerFieldValue <code>Integer</code> object field value
878     * @param longFieldValue <code>Long</code> object field value
879     * @param digestFieldValue <code>Digest</code> object field value
880     * @param contentTypeFieldValue <code>ContentType</code> object field value
881     * @param dateFieldValue <code>Date</code> object field value
882     * @param inetAddrFieldValue <code>InetAddress</code> object field value
883     * @param uriFieldValue <code>URI</code> object field value
884     * @return <code>HeaderLine</code> object corresponding to what would have been read
885     */
886    public HeaderLine addHeader(String fieldName, String fieldValueStr, int dt,
887            Integer integerFieldValue, Long longFieldValue,
888            WarcDigest digestFieldValue, ContentType contentTypeFieldValue,
889            Date dateFieldValue, InetAddress inetAddrFieldValue,
890            Uri uriFieldValue) {
891        Integer fn_idx = WarcConstants.fieldNameIdxMap.get(fieldName.toLowerCase());
892        if (fn_idx != null) {
893            // Implicit cast from integer to long, if needed.
894            if (WarcConstants.FN_IDX_DT[fn_idx] == WarcConstants.FDT_LONG
895                    && dt == WarcConstants.FDT_INTEGER) {
896                longFieldValue = (long)integerFieldValue;
897                dt = WarcConstants.FDT_LONG;
898            }
899            if (dt == WarcConstants.FN_IDX_DT[fn_idx]) {
900                // WARC field name defined in WARC specification.
901                if (seen[fn_idx] && !WarcConstants.fieldNamesRepeatableLookup[fn_idx]) {
902                    // Duplicate field.
903                    addErrorDiagnosis(DiagnosisType.DUPLICATE,
904                            "'" + fieldName + "' header",
905                            fieldValueStr);
906                }
907                seen[fn_idx] = true;
908                switch (fn_idx.intValue()) {
909                /*
910                 * Integer.
911                 */
912                case WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER:
913                    warcSegmentNumberStr = fieldValueStr;
914                    warcSegmentNumber = integerFieldValue;
915                    break;
916                /*
917                 * Long.
918                 */
919                case WarcConstants.FN_IDX_CONTENT_LENGTH:
920                    contentLengthStr = fieldValueStr;
921                    contentLength = longFieldValue;
922                    break;
923                case WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH:
924                    warcSegmentTotalLengthStr = fieldValueStr;
925                    warcSegmentTotalLength = longFieldValue;
926                    break;
927                /*
928                 * Digest.
929                 */
930                case WarcConstants.FN_IDX_WARC_BLOCK_DIGEST:
931                    warcBlockDigestStr = fieldValueStr;
932                    warcBlockDigest = digestFieldValue;
933                    break;
934                case WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST:
935                    warcPayloadDigestStr = fieldValueStr;
936                    warcPayloadDigest = digestFieldValue;
937                    break;
938                /*
939                 * ContentType.
940                 */
941                case WarcConstants.FN_IDX_CONTENT_TYPE:
942                    contentTypeStr = fieldValueStr;
943                    contentType = contentTypeFieldValue;
944                    break;
945                case WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE:
946                    warcIdentifiedPayloadTypeStr = fieldValueStr;
947                    warcIdentifiedPayloadType = contentTypeFieldValue;
948                    break;
949                /*
950                 * Date.
951                 */
952                case WarcConstants.FN_IDX_WARC_DATE:
953                    warcDateStr = fieldValueStr;
954                    warcDate = dateFieldValue;
955                    break;
956                case WarcConstants.FN_IDX_WARC_REFERS_TO_DATE:
957                    warcRefersToDateStr = fieldValueStr;
958                    warcRefersToDate = dateFieldValue;
959                    break;
960                /*
961                 * InetAddress.
962                 */
963                case WarcConstants.FN_IDX_WARC_IP_ADDRESS:
964                    warcIpAddress = fieldValueStr;
965                    warcInetAddress = inetAddrFieldValue;
966                    break;
967                /*
968                 * URI.
969                 */
970                case WarcConstants.FN_IDX_WARC_RECORD_ID:
971                    warcRecordIdStr = fieldValueStr;
972                    warcRecordIdUri = uriFieldValue;
973                    break;
974                case WarcConstants.FN_IDX_WARC_CONCURRENT_TO:
975                    if (fieldValueStr != null || uriFieldValue != null) {
976                        WarcConcurrentTo warcConcurrentTo = new WarcConcurrentTo();
977                        warcConcurrentTo.warcConcurrentToStr = fieldValueStr;
978                        warcConcurrentTo.warcConcurrentToUri = uriFieldValue;
979                        warcConcurrentToList.add(warcConcurrentTo);
980                    }
981                    break;
982                case WarcConstants.FN_IDX_WARC_REFERS_TO:
983                    warcRefersToStr = fieldValueStr;
984                    warcRefersToUri = uriFieldValue;
985                    break;
986                case WarcConstants.FN_IDX_WARC_TARGET_URI:
987                    warcTargetUriStr = fieldValueStr;
988                    warcTargetUriUri = uriFieldValue;
989                    break;
990                case WarcConstants.FN_IDX_WARC_WARCINFO_ID:
991                    warcWarcinfoIdStr = fieldValueStr;
992                    warcWarcinfoIdUri = uriFieldValue;
993                    break;
994                case WarcConstants.FN_IDX_WARC_PROFILE:
995                    warcProfileStr = fieldValueStr;
996                    warcProfileUri = uriFieldValue;
997                    if (warcProfileStr != null) {
998                        warcProfileIdx = WarcConstants.profileIdxMap.get(warcProfileStr.toLowerCase());
999                    }
1000                    if (warcProfileIdx == null && warcProfileStr != null && warcProfileStr.length() > 0) {
1001                        warcProfileIdx = WarcConstants.PROFILE_IDX_UNKNOWN;
1002                    }
1003                    break;
1004                case WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID:
1005                    warcSegmentOriginIdStr = fieldValueStr;
1006                    warcSegmentOriginIdUrl = uriFieldValue;
1007                    break;
1008                case WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI:
1009                    warcRefersToTargetUriStr = fieldValueStr;
1010                    warcRefersToTargetUriUri = uriFieldValue;
1011                    break;
1012                default:
1013                    break;
1014                }
1015            } else {
1016                // Invalid datatype for field.
1017                addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED,
1018                        "Invalid datatype for '" + fieldName + "' header",
1019                        WarcConstants.FDT_IDX_STRINGS[WarcConstants.FN_IDX_DT[fn_idx]],
1020                        WarcConstants.FDT_IDX_STRINGS[dt]);
1021                // Consider throwing exception at some point.
1022            }
1023        }
1024        HeaderLine headerLine = new HeaderLine();
1025        headerLine.name = fieldName;
1026        headerLine.value = fieldValueStr;
1027        HeaderLine tmpLine = headerMap.get(fieldName.toLowerCase());
1028        if (tmpLine == null) {
1029            headerMap.put(fieldName.toLowerCase(), headerLine);
1030        } else {
1031            tmpLine.lines.add(headerLine);
1032        }
1033        return headerLine;
1034    }
1035
1036    /**
1037     * Validate the WARC header relative to the WARC-Type and according to the
1038     * WARC ISO standard.
1039     */
1040    protected void checkFields() {
1041        bMandatoryMissing = false;
1042
1043        /*
1044         * Unknown Warc-Type and/or Warc-Profile.
1045         */
1046
1047        if (warcTypeIdx != null && warcTypeIdx == WarcConstants.RT_IDX_UNKNOWN) {
1048            // Warning: Unknown Warc-Type.
1049            addWarningDiagnosis(DiagnosisType.UNKNOWN, "'" + WarcConstants.FN_WARC_TYPE + "' value", warcTypeStr);
1050        }
1051
1052        if (warcProfileIdx != null && warcProfileIdx == WarcConstants.PROFILE_IDX_UNKNOWN) {
1053            // Warning: Unknown Warc-Profile.
1054            addWarningDiagnosis(DiagnosisType.UNKNOWN, "'" + WarcConstants.FN_WARC_PROFILE + "' value", warcProfileStr);
1055        }
1056
1057        /*
1058         * Mandatory fields.
1059         */
1060
1061        // TODO Required yes, but is it always invalid.
1062        if (warcTypeIdx == null) {
1063            // Mandatory valid Warc-Type missing.
1064            addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, "'" + WarcConstants.FN_WARC_TYPE + "' header", warcTypeStr);
1065            bMandatoryMissing = true;
1066        }
1067        if (warcRecordIdUri == null) {
1068            // Mandatory valid Warc-Record-Id missing.
1069            addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, "'" + WarcConstants.FN_WARC_RECORD_ID + "' header", warcRecordIdStr);
1070            bMandatoryMissing = true;
1071        }
1072        if (warcDate == null) {
1073            // Mandatory valid Warc-Date missing.
1074            addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, "'" + WarcConstants.FN_WARC_DATE + "' header", warcDateStr);
1075            bMandatoryMissing = true;
1076        }
1077        if (contentLength == null) {
1078            // Mandatory valid Content-Length missing.
1079            addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID, "'" + WarcConstants.FN_CONTENT_LENGTH + "' header", contentLengthStr);
1080            bMandatoryMissing = true;
1081        }
1082
1083        /*
1084         * Content-Type should be present if Content-Length > 0.
1085         * Except for continuation records.
1086         */
1087
1088        if (contentLength != null && contentLength.longValue() > 0L &&
1089                        (contentTypeStr == null || contentTypeStr.length() == 0)) {
1090            if (warcTypeIdx == null || warcTypeIdx != WarcConstants.RT_IDX_CONTINUATION) {
1091                addWarningDiagnosis(DiagnosisType.RECOMMENDED_MISSING,
1092                        "'" + WarcConstants.FN_CONTENT_TYPE + "' header");
1093            }
1094        }
1095
1096        /*
1097         * WARC record type dependent policies.
1098         */
1099
1100        if (warcTypeIdx != null) {
1101            /*
1102             * Warcinfo record should have "application/warc-fields" content-type.
1103             */
1104
1105            if (warcTypeIdx == WarcConstants.RT_IDX_WARCINFO) {
1106                if (contentType != null &&
1107                        (!contentType.contentType.equals("application")
1108                        || !contentType.mediaType.equals("warc-fields"))) {
1109                    addWarningDiagnosis(DiagnosisType.RECOMMENDED,
1110                            "'" + WarcConstants.FN_CONTENT_TYPE + "' value",
1111                            WarcConstants.CT_APP_WARC_FIELDS,
1112                            contentTypeStr);
1113                }
1114            }
1115
1116            if (warcTypeIdx == WarcConstants.RT_IDX_RESPONSE) {
1117                if (warcSegmentNumber != null && warcSegmentNumber != 1) {
1118                    addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED,
1119                            "'" + WarcConstants.FN_WARC_SEGMENT_NUMBER + "' value",
1120                            warcSegmentNumber.toString(),
1121                            "1");
1122                }
1123            }
1124
1125            if (warcTypeIdx == WarcConstants.RT_IDX_CONTINUATION) {
1126                if (warcSegmentNumber != null && warcSegmentNumber < 2) {
1127                    addErrorDiagnosis(DiagnosisType.INVALID_EXPECTED,
1128                            "'" + WarcConstants.FN_WARC_SEGMENT_NUMBER + "' value",
1129                            warcSegmentNumber.toString(),
1130                            ">1");
1131                }
1132            }
1133
1134            /*
1135             * Check the policies for each field.
1136             */
1137
1138            WarcConcurrentTo warcConcurrentTo;
1139            if (warcTypeIdx  > 0) {
1140                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_CONTENT_TYPE, contentType, contentTypeStr);
1141                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_IP_ADDRESS, warcInetAddress, warcIpAddress);
1142                for (int i=0; i<warcConcurrentToList.size(); ++i) {
1143                    warcConcurrentTo = warcConcurrentToList.get(0);
1144                    checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_CONCURRENT_TO, warcConcurrentTo.warcConcurrentToUri, warcConcurrentTo.warcConcurrentToStr);
1145                }
1146                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_REFERS_TO, warcRefersToUri, warcRefersToStr);
1147                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_TARGET_URI, warcTargetUriUri, warcTargetUriStr);
1148                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_TRUNCATED, warcTruncatedIdx, warcTruncatedStr);
1149                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_WARCINFO_ID, warcWarcinfoIdUri, warcWarcinfoIdStr);
1150                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_BLOCK_DIGEST, warcBlockDigest, warcBlockDigestStr);
1151                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_PAYLOAD_DIGEST, warcPayloadDigest, warcPayloadDigestStr);
1152                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_FILENAME, warcFilename, warcFilename);
1153                // Could also use warcProfileIdx for really strict.
1154                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_PROFILE, warcProfileUri, warcProfileStr);
1155                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE, warcIdentifiedPayloadType, warcIdentifiedPayloadTypeStr);
1156                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_SEGMENT_NUMBER, warcSegmentNumber, warcSegmentNumberStr);
1157                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_SEGMENT_ORIGIN_ID, warcSegmentOriginIdUrl, warcSegmentOriginIdStr);
1158                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_SEGMENT_TOTAL_LENGTH, warcSegmentTotalLength, warcSegmentTotalLengthStr);
1159                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_REFERS_TO_TARGET_URI, warcRefersToTargetUriUri, warcRefersToTargetUriStr);
1160                checkFieldPolicy(warcTypeIdx, WarcConstants.FN_IDX_WARC_REFERS_TO_DATE, warcRefersToDate, warcRefersToDateStr);
1161            }
1162        }
1163    }
1164
1165    /**
1166     * Given a WARC record type and a WARC field looks up the policy in a
1167     * matrix build from the WARC ISO standard.
1168     * @param recordType WARC record type id
1169     * @param fieldType WARC field type id
1170     * @param fieldObj WARC field
1171     * @param valueStr WARC raw field value
1172     */
1173    protected void checkFieldPolicy(int recordType, int fieldType, Object fieldObj, String valueStr) {
1174        int policy = WarcConstants.field_policy[recordType][fieldType];
1175        switch (policy) {
1176        case WarcConstants.POLICY_MANDATORY:
1177            if (fieldObj == null) {
1178                addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID,
1179                        "'" + WarcConstants.FN_IDX_STRINGS[fieldType] + "' value",
1180                        valueStr);
1181            }
1182            break;
1183        case WarcConstants.POLICY_SHALL:
1184            if (fieldObj == null) {
1185                addErrorDiagnosis(DiagnosisType.REQUIRED_INVALID,
1186                        "'" + WarcConstants.FN_IDX_STRINGS[fieldType] + "' value",
1187                        valueStr);
1188            }
1189            break;
1190        case WarcConstants.POLICY_SHALL_NOT:
1191            if (fieldObj != null) {
1192                addErrorDiagnosis(DiagnosisType.UNDESIRED_DATA,
1193                        "'" + WarcConstants.FN_IDX_STRINGS[fieldType] + "' value",
1194                        valueStr);
1195            }
1196            break;
1197        case WarcConstants.POLICY_MAY_NOT:
1198            if (fieldObj != null) {
1199                addWarningDiagnosis(DiagnosisType.UNDESIRED_DATA,
1200                        "'" + WarcConstants.FN_IDX_STRINGS[fieldType] + "' value",
1201                        valueStr);
1202            }
1203            break;
1204        case WarcConstants.POLICY_MAY:
1205        case WarcConstants.POLICY_IGNORE:
1206        default:
1207            break;
1208        }
1209    }
1210
1211}