org.jwat.warc
Class WarcHeader

java.lang.Object
  org.jwat.warc.WarcHeader

public class WarcHeader
extends Object

Field Summary

Long	contentLength
String	contentLengthStr
ContentType	contentType
String	contentTypeStr
protected Diagnostics<Diagnosis>	diagnostics Diagnostics used to report diagnoses.
protected WarcFieldParsers	fieldParser WARC field parser used.
byte[]	headerBytes Raw WARC header byte array.
protected ByteArrayOutputStream	headerBytesOut Raw WARC header output stream.
protected List<HeaderLine>	headerList List of parsed header fields.
protected Map<String,HeaderLine>	headerMap Map of parsed header fields.
protected WarcReader	reader Associated WarcReader context.
protected boolean[]	seen
protected long	startOffset
WarcDigest	warcBlockDigest
String	warcBlockDigestStr
List<WarcConcurrentTo>	warcConcurrentToList
Date	warcDate
protected DateFormat	warcDateFormat WARC DateFormat as specified by the WARC ISO standard.
String	warcDateStr
String	warcFilename
ContentType	warcIdentifiedPayloadType
String	warcIdentifiedPayloadTypeStr
InetAddress	warcInetAddress
String	warcIpAddress
WarcDigest	warcPayloadDigest
String	warcPayloadDigestStr
Integer	warcProfileIdx
String	warcProfileStr
String	warcRecordIdStr
URI	warcRecordIdUri
String	warcRefersToStr
URI	warcRefersToUri
Integer	warcSegmentNumber
String	warcSegmentNumberStr
String	warcSegmentOriginIdStr
URI	warcSegmentOriginIdUrl
Long	warcSegmentTotalLength
String	warcSegmentTotalLengthStr
String	warcTargetUriStr
URI	warcTargetUriUri
Integer	warcTruncatedIdx
String	warcTruncatedStr
Integer	warcTypeIdx
String	warcTypeStr
String	warcWarcinfoIdStr
URI	warcWarcInfoIdUri

Constructor Summary

protected

WarcHeader() Non public constructor to allow unit testing.

Method Summary

protected void	addErrorDiagnosis(DiagnosisType type, String entity, String... information) Add an error diagnosis of the given type on a specific entity with optional extra information.
protected void	addHeader(HeaderLine headerLine) Identify a (WARC) header name, validate the value and set the header.
HeaderLine	addHeader(String fieldName, ContentType contentTypeFieldValue, String fieldValueStr)
HeaderLine	addHeader(String fieldName, Date dateFieldValue, String fieldValueStr)
HeaderLine	addHeader(String fieldName, InetAddress inetAddrFieldValue, String fieldValueStr)
HeaderLine	addHeader(String fieldName, Integer integerFieldValue, String fieldValueStr)
HeaderLine	addHeader(String fieldName, Long longFieldValue, String fieldValueStr)
HeaderLine	addHeader(String fieldName, String fieldValue)
HeaderLine	addHeader(String fieldName, String fieldValueStr, int dt, Integer integerFieldValue, Long longFieldValue, WarcDigest digestFieldValue, ContentType contentTypeFieldValue, Date dateFieldValue, InetAddress inetAddrFieldValue, URI uriFieldValue)
HeaderLine	addHeader(String fieldName, URI uriFieldValue, String fieldValueStr)
HeaderLine	addHeader(String fieldName, WarcDigest digestFieldValue, String fieldValueStr)
protected void	addWarningDiagnosis(DiagnosisType type, String entity, String... information) Add a warning diagnosis of the given type on a specific entity with optional extra information.
protected void	checkFieldPolicy(int rtype, int ftype, Object fieldObj, String valueStr) Given a WARC record type and a WARC field looks up the policy in a matrix build from the WARC ISO standard.
protected void	checkFields() Validate the WARC header relative to the WARC-Type and according the WARC ISO standard.
static WarcHeader	initHeader(WarcReader reader, long startOffset, Diagnostics<Diagnosis> diagnostics)
static WarcHeader	initHeader(WarcWriter writer, Diagnostics<Diagnosis> diagnostics)
boolean	parseHeader(ByteCountingPushBackInputStream in)
protected void	parseHeaders(ByteCountingPushBackInputStream in) Reads WARC header lines one line at a time until an empty line is encountered.
protected boolean	parseVersion(ByteCountingPushBackInputStream in) Looks forward in the inputstream for a valid WARC version line.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

reader

protected WarcReader reader

Associated WarcReader context. Must be set prior to calling the various methods.

diagnostics

protected Diagnostics<Diagnosis> diagnostics

Diagnostics used to report diagnoses. Must be set prior to calling the various methods.

fieldParser

protected WarcFieldParsers fieldParser

WARC field parser used. Must be set prior to calling the various methods.

warcDateFormat

protected DateFormat warcDateFormat

WARC DateFormat as specified by the WARC ISO standard.

startOffset

protected long startOffset

seen

protected boolean[] seen

warcTypeStr

public String warcTypeStr

warcTypeIdx

public Integer warcTypeIdx

warcFilename

public String warcFilename

warcRecordIdStr

public String warcRecordIdStr

warcRecordIdUri

public URI warcRecordIdUri

warcDateStr

public String warcDateStr

warcDate

public Date warcDate

contentLengthStr

public String contentLengthStr

contentLength

public Long contentLength

contentTypeStr

public String contentTypeStr

contentType

public ContentType contentType

warcTruncatedStr

public String warcTruncatedStr

warcTruncatedIdx

public Integer warcTruncatedIdx

warcIpAddress

public String warcIpAddress

warcInetAddress

public InetAddress warcInetAddress

warcConcurrentToList

public List<WarcConcurrentTo> warcConcurrentToList

warcRefersToStr

public String warcRefersToStr

warcRefersToUri

public URI warcRefersToUri

warcTargetUriStr

public String warcTargetUriStr

warcTargetUriUri

public URI warcTargetUriUri

warcWarcinfoIdStr

public String warcWarcinfoIdStr

warcWarcInfoIdUri

public URI warcWarcInfoIdUri

warcBlockDigestStr

public String warcBlockDigestStr

warcBlockDigest

public WarcDigest warcBlockDigest

warcPayloadDigestStr

public String warcPayloadDigestStr

warcPayloadDigest

public WarcDigest warcPayloadDigest

warcIdentifiedPayloadTypeStr

public String warcIdentifiedPayloadTypeStr

warcIdentifiedPayloadType

public ContentType warcIdentifiedPayloadType

warcProfileStr

public String warcProfileStr

warcProfileIdx

public Integer warcProfileIdx

warcSegmentNumberStr

public String warcSegmentNumberStr

warcSegmentNumber

public Integer warcSegmentNumber

warcSegmentOriginIdStr

public String warcSegmentOriginIdStr

warcSegmentOriginIdUrl

public URI warcSegmentOriginIdUrl

warcSegmentTotalLengthStr

public String warcSegmentTotalLengthStr

warcSegmentTotalLength

public Long warcSegmentTotalLength

headerBytesOut

protected ByteArrayOutputStream headerBytesOut

Raw WARC header output stream.

headerBytes

public byte[] headerBytes

Raw WARC header byte array.

headerList

protected List<HeaderLine> headerList

List of parsed header fields.

headerMap

protected Map<String,HeaderLine> headerMap

Map of parsed header fields.

Constructor Detail

WarcHeader

protected WarcHeader()

Non public constructor to allow unit testing.

Method Detail

initHeader

public static WarcHeader initHeader(WarcWriter writer,
                                    Diagnostics<Diagnosis> diagnostics)

initHeader

public static WarcHeader initHeader(WarcReader reader,
                                    long startOffset,
                                    Diagnostics<Diagnosis> diagnostics)

addErrorDiagnosis

protected void addErrorDiagnosis(DiagnosisType type,
                                 String entity,
                                 String... information)

Add an error diagnosis of the given type on a specific entity with optional extra information. The information varies according to the diagnosis type.

Parameters:

type - diagnosis type

entity - entity examined

information - optional extra information

addWarningDiagnosis

protected void addWarningDiagnosis(DiagnosisType type,
                                   String entity,
                                   String... information)

Add a warning diagnosis of the given type on a specific entity with optional extra information. The information varies according to the diagnosis type.

Parameters:

type - diagnosis type

entity - entity examined

information - optional extra information

parseHeader

public boolean parseHeader(ByteCountingPushBackInputStream in)
                    throws IOException

Throws:

IOException

parseVersion

protected boolean parseVersion(ByteCountingPushBackInputStream in)
                        throws IOException

Looks forward in the inputstream for a valid WARC version line.

Parameters:

in - data inputstream

Returns:

true, if magic WARC header found

Throws:

IOException - if an error occurs while reading version data

parseHeaders

protected void parseHeaders(ByteCountingPushBackInputStream in)
                     throws IOException

Reads WARC header lines one line at a time until an empty line is encountered.

Parameters:

in - header input stream

Throws:

IOException - if an error occurs while reading the WARC header

addHeader

protected void addHeader(HeaderLine headerLine)

Identify a (WARC) header name, validate the value and set the header.

Parameters:

fieldName - header name

fieldValue - header value

seen - array of headers seen so far used for duplication check

addHeader

public HeaderLine addHeader(String fieldName,
                            String fieldValue)

addHeader

public HeaderLine addHeader(String fieldName,
                            Integer integerFieldValue,
                            String fieldValueStr)

addHeader

public HeaderLine addHeader(String fieldName,
                            Long longFieldValue,
                            String fieldValueStr)

addHeader

public HeaderLine addHeader(String fieldName,
                            WarcDigest digestFieldValue,
                            String fieldValueStr)

addHeader

public HeaderLine addHeader(String fieldName,
                            ContentType contentTypeFieldValue,
                            String fieldValueStr)

addHeader

public HeaderLine addHeader(String fieldName,
                            Date dateFieldValue,
                            String fieldValueStr)

addHeader

public HeaderLine addHeader(String fieldName,
                            InetAddress inetAddrFieldValue,
                            String fieldValueStr)

addHeader

public HeaderLine addHeader(String fieldName,
                            URI uriFieldValue,
                            String fieldValueStr)

addHeader

public HeaderLine addHeader(String fieldName,
                            String fieldValueStr,
                            int dt,
                            Integer integerFieldValue,
                            Long longFieldValue,
                            WarcDigest digestFieldValue,
                            ContentType contentTypeFieldValue,
                            Date dateFieldValue,
                            InetAddress inetAddrFieldValue,
                            URI uriFieldValue)

checkFields

protected void checkFields()

Validate the WARC header relative to the WARC-Type and according the WARC ISO standard.

checkFieldPolicy

protected void checkFieldPolicy(int rtype,
                                int ftype,
                                Object fieldObj,
                                String valueStr)

Given a WARC record type and a WARC field looks up the policy in a matrix build from the WARC ISO standard.

Parameters:

rtype - WARC record type id

ftype - WARC field type id

fieldObj - WARC field

valueStr - WARC raw field value