001/** 002 * Java Web Archive Toolkit - Software to read and validate ARC, WARC 003 * and GZip files. (http://jwat.org/) 004 * Copyright 2011-2012 Netarkivet.dk (http://netarkivet.dk/) 005 * 006 * Licensed under the Apache License, Version 2.0 (the "License"); 007 * you may not use this file except in compliance with the License. 008 * You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018package org.jwat.warc; 019 020import java.util.HashMap; 021import java.util.Map; 022 023/** 024 * Class containing all relevant WARC constants and structures. 025 * Including but not limited to field names and mime-types. 026 * Also includes statically initialized structures for validation. 027 * 028 * @author nicl 029 */ 030public class WarcConstants { 031 032 /** 033 * This utility class does not require instantiation. 034 */ 035 protected WarcConstants() { 036 } 037 038 /** 039 * A WARC header block starts with this string including trailing version 040 * information. 041 * */ 042 public static final String WARC_MAGIC_HEADER = "WARC/"; 043 044 /** End mark used after each record consisting of two newlines. */ 045 protected static byte[] endMark = "\r\n\r\n".getBytes(); 046 047 /** WARC date format string as specified by the WARC ISO standard. */ 048 public static final String WARC_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; 049 050 /** WARC digest format string as specified by the WARC ISO standard. */ 051 public static final String WARC_DIGEST_FORMAT = "<digest-algorithm>:<digest-encoded>"; 052 053 /** Content-type format string as specified in RFC2616.*/ 054 public static final String CONTENT_TYPE_FORMAT = "<type>/<sub-type>(; <argument>=<value>)*"; 055 056 /* 057 * WARC content-types (MIME). 058 */ 059 060 /** WARC mime type. */ 061 public static final String WARC_MIME_TYPE = "application/warc"; 062 063 /** Suggested content-type/media-type for metadata records and others. */ 064 public static final String CT_APP_WARC_FIELDS = "application/warc-fields"; 065 066 /** Suggested content-type for metadata records and others. */ 067 public static final String CONTENT_TYPE_METADATA = "application"; 068 069 /** Suggested media-type for metadata records and others. */ 070 public static final String MEDIA_TYPE_METADATA = "warc-fields"; 071 072 /* 073 * Voodoo magic constants. 074 */ 075 076 /** Trailing newlines after each record as per the WARC ISO standard. */ 077 public static final int WARC_RECORD_TRAILING_NEWLINES = 2; 078 079 /** Number of WARC fields. */ 080 public static final int FN_NUMBER = 21; 081 082 /** Index of last WARC field (zero-indexed). */ 083 public static final int FN_INDEX_OF_LAST = FN_NUMBER+1; 084 085 /** Number of WARC record types. */ 086 public static final int RT_NUMBER = 8; 087 088 /** Index of last WARC record type (zero indexed). */ 089 public static final int RT_INDEX_OF_LAST = RT_NUMBER+1; 090 091 /* 092 * WARC field names. 093 */ 094 095 /** Warc-type field name. */ 096 public static final String FN_WARC_TYPE = "WARC-Type"; 097 /** Warc-record-id field name. */ 098 public static final String FN_WARC_RECORD_ID = "WARC-Record-ID"; 099 /** Warc-date field name. */ 100 public static final String FN_WARC_DATE = "WARC-Date"; 101 /** Content-length field name. */ 102 public static final String FN_CONTENT_LENGTH = "Content-Length"; 103 /** Content-type field name. */ 104 public static final String FN_CONTENT_TYPE = "Content-Type"; 105 /** Warc-concurrent-to field name. */ 106 public static final String FN_WARC_CONCURRENT_TO = "WARC-Concurrent-To"; 107 /** Warc-block-digest field name. */ 108 public static final String FN_WARC_BLOCK_DIGEST = "WARC-Block-Digest"; 109 /** Warc-payload-digest field name. */ 110 public static final String FN_WARC_PAYLOAD_DIGEST = "WARC-Payload-Digest"; 111 /** Warc-ip-address field name. */ 112 public static final String FN_WARC_IP_ADDRESS = "WARC-IP-Address"; 113 /** Warc-refers-to field name. */ 114 public static final String FN_WARC_REFERS_TO = "WARC-Refers-To"; 115 /** Warc-target-uri field name. */ 116 public static final String FN_WARC_TARGET_URI = "WARC-Target-URI"; 117 /** Warc-truncated field name. */ 118 public static final String FN_WARC_TRUNCATED = "WARC-Truncated"; 119 /** Warc-warcinfo-id field name. */ 120 public static final String FN_WARC_WARCINFO_ID = "WARC-Warcinfo-ID"; 121 /** Warc-filename field name. */ 122 public static final String FN_WARC_FILENAME = "WARC-Filename"; 123 /** Warc-profile field name. */ 124 public static final String FN_WARC_PROFILE = "WARC-Profile"; 125 /** Warc-identified-payload-type field name. */ 126 public static final String FN_WARC_IDENTIFIED_PAYLOAD_TYPE = "WARC-Identified-Payload-Type"; 127 /** Warc-segment-origin-id field name. */ 128 public static final String FN_WARC_SEGMENT_ORIGIN_ID = "WARC-Segment-Origin-ID"; 129 /** Warc-segment-number field name. */ 130 public static final String FN_WARC_SEGMENT_NUMBER = "WARC-Segment-Number"; 131 /** Warc-segment-totalt-length field name. */ 132 public static final String FN_WARC_SEGMENT_TOTAL_LENGTH = "WARC-Segment-Total-Length"; 133 /** WARC-Refers-To-Target-URI field name. */ 134 public static final String FN_WARC_REFERS_TO_TARGET_URI = "WARC-Refers-To-Target-URI"; 135 /** WARC-Refers-To-Date field name. */ 136 public static final String FN_WARC_REFERS_TO_DATE = "WARC-Refers-To-Date"; 137 138 /** WARC field name id to field name mapping table. 139 * Zero indexed array with all indexes used > 1. (Index 0 is unused) */ 140 public static final String[] FN_IDX_STRINGS = { 141 null, 142 FN_WARC_TYPE, 143 FN_WARC_RECORD_ID, 144 FN_WARC_DATE, 145 FN_CONTENT_LENGTH, 146 FN_CONTENT_TYPE, 147 FN_WARC_CONCURRENT_TO, 148 FN_WARC_BLOCK_DIGEST, 149 FN_WARC_PAYLOAD_DIGEST, 150 FN_WARC_IP_ADDRESS, 151 FN_WARC_REFERS_TO, 152 FN_WARC_TARGET_URI, 153 FN_WARC_TRUNCATED, 154 FN_WARC_WARCINFO_ID, 155 FN_WARC_FILENAME, 156 FN_WARC_PROFILE, 157 FN_WARC_IDENTIFIED_PAYLOAD_TYPE, 158 FN_WARC_SEGMENT_ORIGIN_ID, 159 FN_WARC_SEGMENT_NUMBER, 160 FN_WARC_SEGMENT_TOTAL_LENGTH, 161 FN_WARC_REFERS_TO_TARGET_URI, 162 FN_WARC_REFERS_TO_DATE 163 }; 164 165 /** Warc reader warc-type field name id. */ 166 public static final int FN_IDX_WARC_TYPE = 1; 167 /** Warc reader warc-record-id field name id. */ 168 public static final int FN_IDX_WARC_RECORD_ID = 2; 169 /** Warc reader warc-date field name id. */ 170 public static final int FN_IDX_WARC_DATE = 3; 171 /** Warc reader content-length field name id. */ 172 public static final int FN_IDX_CONTENT_LENGTH = 4; 173 /** Warc reader content-type field name id. */ 174 public static final int FN_IDX_CONTENT_TYPE = 5; 175 /** Warc reader warc-concurrent-to field name id. */ 176 public static final int FN_IDX_WARC_CONCURRENT_TO = 6; 177 /** Warc reader warc-block-digest field name id. */ 178 public static final int FN_IDX_WARC_BLOCK_DIGEST = 7; 179 /** Warc reader warc-payload-digest field name id. */ 180 public static final int FN_IDX_WARC_PAYLOAD_DIGEST = 8; 181 /** Warc reader warc-ip-address field name id. */ 182 public static final int FN_IDX_WARC_IP_ADDRESS = 9; 183 /** Warc reader warc-refers-to field name id. */ 184 public static final int FN_IDX_WARC_REFERS_TO = 10; 185 /** Warc reader warc-target-uri field name id. */ 186 public static final int FN_IDX_WARC_TARGET_URI = 11; 187 /** Warc reader warc-truncated field name id. */ 188 public static final int FN_IDX_WARC_TRUNCATED = 12; 189 /** Warc reader warc-warcinfo-id field name id. */ 190 public static final int FN_IDX_WARC_WARCINFO_ID = 13; 191 /** Warc reader warc-filename field name id. */ 192 public static final int FN_IDX_WARC_FILENAME = 14; // warcinfo only 193 /** Warc reader warc-profile field name id. */ 194 public static final int FN_IDX_WARC_PROFILE = 15; // revisit only 195 /** Warc reader warc-identified-payload-type field name id. */ 196 public static final int FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE = 16; 197 /** Warc reader warc-segment-origin-id field name id. */ 198 public static final int FN_IDX_WARC_SEGMENT_ORIGIN_ID = 17; // continuation only 199 /** Warc reader warc-segment-number field name id. */ 200 public static final int FN_IDX_WARC_SEGMENT_NUMBER = 18; 201 /** Warc reader warc-segment-totalt-length field name id. */ 202 public static final int FN_IDX_WARC_SEGMENT_TOTAL_LENGTH = 19; //continuation only 203 /** WARC-Refers-To-Target-URI field name id. */ 204 public static final int FN_IDX_WARC_REFERS_TO_TARGET_URI = 20; 205 /** WARC-Refers-To-Date field name id. */ 206 public static final int FN_IDX_WARC_REFERS_TO_DATE = 21; 207 208 /** Map used to identify known warc field names. */ 209 public static final Map<String, Integer> fieldNameIdxMap = new HashMap<String, Integer>(); 210 211 /** 212 * Populate map of known WARC field names. 213 */ 214 static { 215 fieldNameIdxMap.put(FN_WARC_TYPE.toLowerCase(), FN_IDX_WARC_TYPE); 216 fieldNameIdxMap.put(FN_WARC_RECORD_ID.toLowerCase(), FN_IDX_WARC_RECORD_ID); 217 fieldNameIdxMap.put(FN_WARC_DATE.toLowerCase(), FN_IDX_WARC_DATE); 218 fieldNameIdxMap.put(FN_CONTENT_LENGTH.toLowerCase(), FN_IDX_CONTENT_LENGTH); 219 fieldNameIdxMap.put(FN_CONTENT_TYPE.toLowerCase(), FN_IDX_CONTENT_TYPE); 220 fieldNameIdxMap.put(FN_WARC_CONCURRENT_TO.toLowerCase(), FN_IDX_WARC_CONCURRENT_TO); 221 fieldNameIdxMap.put(FN_WARC_BLOCK_DIGEST.toLowerCase(), FN_IDX_WARC_BLOCK_DIGEST); 222 fieldNameIdxMap.put(FN_WARC_PAYLOAD_DIGEST.toLowerCase(), FN_IDX_WARC_PAYLOAD_DIGEST); 223 fieldNameIdxMap.put(FN_WARC_IP_ADDRESS.toLowerCase(), FN_IDX_WARC_IP_ADDRESS); 224 fieldNameIdxMap.put(FN_WARC_REFERS_TO.toLowerCase(), FN_IDX_WARC_REFERS_TO); 225 fieldNameIdxMap.put(FN_WARC_TARGET_URI.toLowerCase(), FN_IDX_WARC_TARGET_URI); 226 fieldNameIdxMap.put(FN_WARC_TRUNCATED.toLowerCase(), FN_IDX_WARC_TRUNCATED); 227 fieldNameIdxMap.put(FN_WARC_WARCINFO_ID.toLowerCase(), FN_IDX_WARC_WARCINFO_ID); 228 fieldNameIdxMap.put(FN_WARC_FILENAME.toLowerCase(), FN_IDX_WARC_FILENAME); 229 fieldNameIdxMap.put(FN_WARC_PROFILE.toLowerCase(), FN_IDX_WARC_PROFILE); 230 fieldNameIdxMap.put(FN_WARC_IDENTIFIED_PAYLOAD_TYPE.toLowerCase(), FN_IDX_WARC_IDENTIFIED_PAYLOAD_TYPE); 231 fieldNameIdxMap.put(FN_WARC_SEGMENT_ORIGIN_ID.toLowerCase(), FN_IDX_WARC_SEGMENT_ORIGIN_ID); 232 fieldNameIdxMap.put(FN_WARC_SEGMENT_NUMBER.toLowerCase(), FN_IDX_WARC_SEGMENT_NUMBER); 233 fieldNameIdxMap.put(FN_WARC_SEGMENT_TOTAL_LENGTH.toLowerCase(), FN_IDX_WARC_SEGMENT_TOTAL_LENGTH); 234 fieldNameIdxMap.put(FN_WARC_REFERS_TO_TARGET_URI.toLowerCase(), FN_IDX_WARC_REFERS_TO_TARGET_URI); 235 fieldNameIdxMap.put(FN_WARC_REFERS_TO_DATE.toLowerCase(), FN_IDX_WARC_REFERS_TO_DATE); 236 } 237 238 /** WARC String field datatype identifier. */ 239 public static final int FDT_STRING = 0; 240 /** WARC Integer field datatype identifier. */ 241 public static final int FDT_INTEGER = 1; 242 /** WARC Long field datatype identifier. */ 243 public static final int FDT_LONG = 2; 244 /** WARC Digest field datatype identifier. */ 245 public static final int FDT_DIGEST = 3; 246 /** WARC ContentType field datatype identifier. */ 247 public static final int FDT_CONTENTTYPE = 4; 248 /** WARC Date field datatype identifier. */ 249 public static final int FDT_DATE = 5; 250 /** WARC InetAddress field datatype identifier. */ 251 public static final int FDT_INETADDRESS = 6; 252 /** WARC URI field datatype identifier. */ 253 public static final int FDT_URI = 7; 254 255 /** WARC field datatype id to field datatype name mapping table. */ 256 public static final String[] FDT_IDX_STRINGS = { 257 "String", 258 "Integer", 259 "Long", 260 "Digest", 261 "ContentType", 262 "Date", 263 "InetAddress", 264 "URI" 265 }; 266 267 /** Array to lookup WARC field datatypes. */ 268 public static final int[] FN_IDX_DT = { 269 -1, 270 FDT_STRING, 271 FDT_URI, 272 FDT_DATE, 273 FDT_LONG, 274 FDT_CONTENTTYPE, 275 FDT_URI, 276 FDT_DIGEST, 277 FDT_DIGEST, 278 FDT_INETADDRESS, 279 FDT_URI, 280 FDT_URI, 281 FDT_STRING, 282 FDT_URI, 283 FDT_STRING, 284 FDT_URI, 285 FDT_CONTENTTYPE, 286 FDT_URI, 287 FDT_INTEGER, 288 FDT_LONG, 289 FDT_URI, 290 FDT_DATE 291 }; 292 293 /* 294 * WARC fields that can have multiple occurrences in a Warc header. 295 */ 296 297 /** Lookup table of Warc fields that can have multiple occurrences. */ 298 public static final boolean[] fieldNamesRepeatableLookup = new boolean[FN_INDEX_OF_LAST]; 299 300 /** 301 * Populate multiple occurrences lookup table. 302 */ 303 static { 304 fieldNamesRepeatableLookup[FN_IDX_WARC_CONCURRENT_TO] = true; 305 } 306 307 /* 308 * WARC record types. 309 */ 310 311 /** WARC-Type warcinfo id. */ 312 public static final String RT_WARCINFO = "warcinfo"; 313 /** WARC-Type response id. */ 314 public static final String RT_RESPONSE = "response"; 315 /** WARC-Type resource id. */ 316 public static final String RT_RESOURCE = "resource"; 317 /** WARC-Type request id. */ 318 public static final String RT_REQUEST = "request"; 319 /** WARC-Type metadata id. */ 320 public static final String RT_METADATA = "metadata"; 321 /** WARC-Type revisit id. */ 322 public static final String RT_REVISIT = "revisit"; 323 /** WARC-Type conversion id. */ 324 public static final String RT_CONVERSION = "conversion"; 325 /** WARC-Type continuation id. */ 326 public static final String RT_CONTINUATION = "continuation"; 327 328 /** WARC type id to field name mapping table. 329 * Zero indexed array with all indexes used > 1. (Index 0 is unused) */ 330 public static final String[] RT_IDX_STRINGS = { 331 null, 332 RT_WARCINFO, 333 RT_RESPONSE, 334 RT_RESOURCE, 335 RT_REQUEST, 336 RT_METADATA, 337 RT_REVISIT, 338 RT_CONVERSION, 339 RT_CONTINUATION 340 }; 341 342 /** Warc reader unknown warc record type id. */ 343 public static final int RT_IDX_UNKNOWN = 0; 344 /** Warc reader warcinfo warc record type id. */ 345 public static final int RT_IDX_WARCINFO = 1; 346 /** Warc reader response warc record type id. */ 347 public static final int RT_IDX_RESPONSE = 2; 348 /** Warc reader resource warc record type id. */ 349 public static final int RT_IDX_RESOURCE = 3; 350 /** Warc reader request warc record type id. */ 351 public static final int RT_IDX_REQUEST = 4; 352 /** Warc reader metadata warc record type id. */ 353 public static final int RT_IDX_METADATA = 5; 354 /** Warc reader revisit warc record type id. */ 355 public static final int RT_IDX_REVISIT = 6; 356 /** Warc reader conversion warc record type id. */ 357 public static final int RT_IDX_CONVERSION = 7; 358 /** Warc reader continuation warc record type id. */ 359 public static final int RT_IDX_CONTINUATION = 8; 360 361 /** WARC-Type lookup map. */ 362 public static final Map<String, Integer> recordTypeIdxMap = new HashMap<String, Integer>(); 363 364 /** 365 * Populate WARC-Type lookup map. 366 */ 367 static { 368 recordTypeIdxMap.put(RT_WARCINFO.toLowerCase(), RT_IDX_WARCINFO); 369 recordTypeIdxMap.put(RT_RESPONSE.toLowerCase(), RT_IDX_RESPONSE); 370 recordTypeIdxMap.put(RT_RESOURCE.toLowerCase(), RT_IDX_RESOURCE); 371 recordTypeIdxMap.put(RT_REQUEST.toLowerCase(), RT_IDX_REQUEST); 372 recordTypeIdxMap.put(RT_METADATA.toLowerCase(), RT_IDX_METADATA); 373 recordTypeIdxMap.put(RT_REVISIT.toLowerCase(), RT_IDX_REVISIT); 374 recordTypeIdxMap.put(RT_CONVERSION.toLowerCase(), RT_IDX_CONVERSION); 375 recordTypeIdxMap.put(RT_CONTINUATION.toLowerCase(), RT_IDX_CONTINUATION); 376 } 377 378 /* 379 * Truncation reason types. 380 */ 381 382 /** WARC-Truncated length id. */ 383 public static final String TT_LENGTH = "length"; 384 /** WARC-Truncated time id*/ 385 public static final String TT_TIME = "time"; 386 /** WARC-Truncated disconnect id. */ 387 public static final String TT_DISCONNECT = "disconnect"; 388 /** WARC-Truncated unspecified id. */ 389 public static final String TT_UNSPECIFIED = "unspecified"; 390 391 /** WARC truncation reason id to field name mapping table. 392 * Zero indexed array with all indexes used > 1. (Index 0 is unused) */ 393 public static final String[] TT_IDX_STRINGS = { 394 null, 395 TT_LENGTH, 396 TT_TIME, 397 TT_DISCONNECT, 398 TT_UNSPECIFIED 399 }; 400 401 /** Warc reader future reason id. */ 402 public static final int TT_IDX_FUTURE_REASON = 0; 403 /** Warc reader length reason id. */ 404 public static final int TT_IDX_LENGTH = 1; 405 /** Warc reader time reason id. */ 406 public static final int TT_IDX_TIME = 2; 407 /** Warc reader disconnect reason id. */ 408 public static final int TT_IDX_DISCONNECT = 3; 409 /** Warc reader unspecified reason id. */ 410 public static final int TT_IDX_UNSPECIFIED = 4; 411 412 /** Lookup map for known truncation reason id's. */ 413 public static final Map<String, Integer> truncatedTypeIdxMap = new HashMap<String, Integer>(); 414 415 /** 416 * Populate truncation reason id lookup map. 417 */ 418 static { 419 truncatedTypeIdxMap.put(TT_LENGTH.toLowerCase(), TT_IDX_LENGTH); 420 truncatedTypeIdxMap.put(TT_TIME.toLowerCase(), TT_IDX_TIME); 421 truncatedTypeIdxMap.put(TT_DISCONNECT.toLowerCase(), TT_IDX_DISCONNECT); 422 truncatedTypeIdxMap.put(TT_UNSPECIFIED.toLowerCase(), TT_IDX_UNSPECIFIED); 423 } 424 425 /* 426 * Warc revisit profile ids used in the WARC-Profile header (See ISO). 427 */ 428 429 /** Revisit WARC-Profile id for identical payload digest (1.0). */ 430 public static final String WARC10_PROFILE_IDENTICAL_PAYLOAD_DIGEST = 431 "http://netpreserve.org/warc/1.0/revisit/identical-payload-digest"; 432 433 /** Revisit WARC-Profile id for server not modified (1.0). */ 434 public static final String WARC10_PROFILE_SERVER_NOT_MODIFIED = 435 "http://netpreserve.org/warc/1.0/revisit/server-not-modified"; 436 437 /** Revisit WARC-Profile id for identical payload digest (1.1). */ 438 public static final String WARC11_PROFILE_IDENTICAL_PAYLOAD_DIGEST = 439 "http://netpreserve.org/warc/1.1/revisit/identical-payload-digest"; 440 441 /** Revisit WARC-Profile id for server not modified (1.1). */ 442 public static final String WARC11_PROFILE_SERVER_NOT_MODIFIED = 443 "http://netpreserve.org/warc/1.1/revisit/server-not-modified"; 444 445 /** WARC profile id to field name mapping table. 446 * Zero indexed array with all indexes used > 1. (Index 0 is unused) */ 447 public static final String[] P_IDX_STRINGS = { 448 null, 449 WARC10_PROFILE_IDENTICAL_PAYLOAD_DIGEST, 450 WARC10_PROFILE_SERVER_NOT_MODIFIED, 451 WARC11_PROFILE_IDENTICAL_PAYLOAD_DIGEST, 452 WARC11_PROFILE_SERVER_NOT_MODIFIED 453 }; 454 455 /* 456 * Warc revisit profile ids returned by the warc reader. 457 * The raw value is also available in case of unknown profiles. 458 */ 459 460 /** WARC reader id for unknown profile. */ 461 public static final int WARC_PROFILE_IDX_UNKNOWN = 0; 462 /** WARC/1.0 reader id for identical payload digest profile. */ 463 public static final int WARC10_PROFILE_IDX_IDENTICAL_PAYLOAD_DIGEST = 1; 464 /** WARC/1.0 reader id for server not modified profile. */ 465 public static final int WARC10_PROFILE_IDX_SERVER_NOT_MODIFIED = 2; 466 /** WARC/1.1 reader id for identical payload digest profile. */ 467 public static final int WARC11_PROFILE_IDX_IDENTICAL_PAYLOAD_DIGEST = 3; 468 /** WARC/1.1 reader id for server not modified profile. */ 469 public static final int WARC11_PROFILE_IDX_SERVER_NOT_MODIFIED = 4; 470 471 /** Profile lookup map used to identify WARC-Profile values. */ 472 public static final Map<String, Integer> profileIdxMap = new HashMap<String, Integer>(); 473 474 /** 475 * Populate the lookup map with known WARC-Profile ids. 476 */ 477 static { 478 profileIdxMap.put(WARC10_PROFILE_IDENTICAL_PAYLOAD_DIGEST.toLowerCase(), 479 WARC10_PROFILE_IDX_IDENTICAL_PAYLOAD_DIGEST); 480 profileIdxMap.put(WARC10_PROFILE_SERVER_NOT_MODIFIED.toLowerCase(), 481 WARC10_PROFILE_IDX_SERVER_NOT_MODIFIED); 482 profileIdxMap.put(WARC11_PROFILE_IDENTICAL_PAYLOAD_DIGEST.toLowerCase(), 483 WARC11_PROFILE_IDX_IDENTICAL_PAYLOAD_DIGEST); 484 profileIdxMap.put(WARC11_PROFILE_SERVER_NOT_MODIFIED.toLowerCase(), 485 WARC11_PROFILE_IDX_SERVER_NOT_MODIFIED); 486 } 487 488 /* 489 * The different requirement levels as per RFC 2119. 490 * (See http://www.ietf.org/rfc/rfc2119.txt) 491 */ 492 493 /** Warc header can be ignored. */ 494 public static final int POLICY_IGNORE = 0; 495 /** Warc header is mandatory (equal to shall). */ 496 public static final int POLICY_MANDATORY = 1; 497 /** Warc header must be present. */ 498 public static final int POLICY_SHALL = 2; 499 /** Warc header must not be present. */ 500 public static final int POLICY_SHALL_NOT = 3; 501 /** Warc header can be present. */ 502 public static final int POLICY_MAY = 4; 503 /** Warc header should not be present. */ 504 public static final int POLICY_MAY_NOT = 5; 505 506}