Data.Warc.Header

Parsing

header :: Parser RecordHeader

A WARC header

Encoding

encodeHeader :: RecordHeader -> Builder

WARC Version

data Version

Constructors

Version

Fields

• versionMajor, versionMinor :: !Int

Instances

Eq Version

Methods

(==) :: Version -> Version -> Bool

(/=) :: Version -> Version -> Bool

Ord Version

Methods

compare :: Version -> Version -> Ordering

(<) :: Version -> Version -> Bool

(<=) :: Version -> Version -> Bool

(>) :: Version -> Version -> Bool

(>=) :: Version -> Version -> Bool

max :: Version -> Version -> Version

min :: Version -> Version -> Version

Read Version

Methods

readsPrec :: Int -> ReadS Version

readList :: ReadS [Version]

readPrec :: ReadPrec Version

readListPrec :: ReadPrec [Version]

Show Version

Methods

showsPrec :: Int -> Version -> ShowS

show :: Version -> String

showList :: [Version] -> ShowS

warc0_16 :: Version

Types

data RecordHeader

Constructors

RecordHeader

Fields

• _recWarcVersion :: Version
• _recHeaders :: HashMap FieldName ByteString

Instances

Show RecordHeader

Methods

showsPrec :: Int -> RecordHeader -> ShowS

show :: RecordHeader -> String

showList :: [RecordHeader] -> ShowS

data WarcType

Constructors

WarcInfo

Response

Resource

Request

Metadata

Revisit

Conversion

Continuation

FutureType !Text

Instances

Eq WarcType

Methods

(==) :: WarcType -> WarcType -> Bool

(/=) :: WarcType -> WarcType -> Bool

Ord WarcType

Methods

compare :: WarcType -> WarcType -> Ordering

(<) :: WarcType -> WarcType -> Bool

(<=) :: WarcType -> WarcType -> Bool

(>) :: WarcType -> WarcType -> Bool

(>=) :: WarcType -> WarcType -> Bool

max :: WarcType -> WarcType -> WarcType

min :: WarcType -> WarcType -> WarcType

Read WarcType

Methods

readsPrec :: Int -> ReadS WarcType

readList :: ReadS [WarcType]

readPrec :: ReadPrec WarcType

readListPrec :: ReadPrec [WarcType]

Show WarcType

Methods

showsPrec :: Int -> WarcType -> ShowS

show :: WarcType -> String

showList :: [WarcType] -> ShowS

newtype RecordId

Constructors

RecordId Uri

Instances

Eq RecordId

Methods

(==) :: RecordId -> RecordId -> Bool

(/=) :: RecordId -> RecordId -> Bool

Ord RecordId

Methods

compare :: RecordId -> RecordId -> Ordering

(<) :: RecordId -> RecordId -> Bool

(<=) :: RecordId -> RecordId -> Bool

(>) :: RecordId -> RecordId -> Bool

(>=) :: RecordId -> RecordId -> Bool

max :: RecordId -> RecordId -> RecordId

min :: RecordId -> RecordId -> RecordId

Read RecordId

Methods

readsPrec :: Int -> ReadS RecordId

readList :: ReadS [RecordId]

readPrec :: ReadPrec RecordId

readListPrec :: ReadPrec [RecordId]

Show RecordId

Methods

showsPrec :: Int -> RecordId -> ShowS

show :: RecordId -> String

showList :: [RecordId] -> ShowS

data TruncationReason

Constructors

TruncLength

TruncTime

TruncDisconnect

TruncUnspecified

TruncOther !Text

Instances

Eq TruncationReason

Methods

(==) :: TruncationReason -> TruncationReason -> Bool

(/=) :: TruncationReason -> TruncationReason -> Bool

Ord TruncationReason

Methods

compare :: TruncationReason -> TruncationReason -> Ordering

(<) :: TruncationReason -> TruncationReason -> Bool

(<=) :: TruncationReason -> TruncationReason -> Bool

(>) :: TruncationReason -> TruncationReason -> Bool

(>=) :: TruncationReason -> TruncationReason -> Bool

max :: TruncationReason -> TruncationReason -> TruncationReason

min :: TruncationReason -> TruncationReason -> TruncationReason

Read TruncationReason

Methods

readsPrec :: Int -> ReadS TruncationReason

readList :: ReadS [TruncationReason]

readPrec :: ReadPrec TruncationReason

readListPrec :: ReadPrec [TruncationReason]

Show TruncationReason

Methods

showsPrec :: Int -> TruncationReason -> ShowS

show :: TruncationReason -> String

showList :: [TruncationReason] -> ShowS

data Digest

Constructors

Digest

Fields

• digestAlgorithm, digestHash :: !ByteString

Instances

Eq Digest

Methods

(==) :: Digest -> Digest -> Bool

(/=) :: Digest -> Digest -> Bool

Ord Digest

Methods

compare :: Digest -> Digest -> Ordering

(<) :: Digest -> Digest -> Bool

(<=) :: Digest -> Digest -> Bool

(>) :: Digest -> Digest -> Bool

(>=) :: Digest -> Digest -> Bool

max :: Digest -> Digest -> Digest

min :: Digest -> Digest -> Digest

Read Digest

Methods

readsPrec :: Int -> ReadS Digest

readList :: ReadS [Digest]

readPrec :: ReadPrec Digest

readListPrec :: ReadPrec [Digest]

Show Digest

Methods

showsPrec :: Int -> Digest -> ShowS

show :: Digest -> String

showList :: [Digest] -> ShowS

newtype Uri

Constructors

Uri ByteString

Instances

Eq Uri

Methods

(==) :: Uri -> Uri -> Bool

(/=) :: Uri -> Uri -> Bool

Ord Uri

Methods

compare :: Uri -> Uri -> Ordering

(<) :: Uri -> Uri -> Bool

(<=) :: Uri -> Uri -> Bool

(>) :: Uri -> Uri -> Bool

(>=) :: Uri -> Uri -> Bool

max :: Uri -> Uri -> Uri

min :: Uri -> Uri -> Uri

Read Uri

Methods

readsPrec :: Int -> ReadS Uri

readList :: ReadS [Uri]

readPrec :: ReadPrec Uri

readListPrec :: ReadPrec [Uri]

Show Uri

Methods

showsPrec :: Int -> Uri -> ShowS

show :: Uri -> String

showList :: [Uri] -> ShowS

Header field types

data Field a

Constructors

Field

Fields

• fieldName :: FieldName
• encode :: a -> Builder
• decode :: Parser a

newtype FieldName

Constructors

FieldName

Fields

• getFieldName :: Text

Instances

Eq FieldName

Methods

(==) :: FieldName -> FieldName -> Bool

(/=) :: FieldName -> FieldName -> Bool

Ord FieldName

Methods

compare :: FieldName -> FieldName -> Ordering

(<) :: FieldName -> FieldName -> Bool

(<=) :: FieldName -> FieldName -> Bool

(>) :: FieldName -> FieldName -> Bool

(>=) :: FieldName -> FieldName -> Bool

max :: FieldName -> FieldName -> FieldName

min :: FieldName -> FieldName -> FieldName

Read FieldName

Methods

readsPrec :: Int -> ReadS FieldName

readList :: ReadS [FieldName]

readPrec :: ReadPrec FieldName

readListPrec :: ReadPrec [FieldName]

Show FieldName

Methods

showsPrec :: Int -> FieldName -> ShowS

show :: FieldName -> String

showList :: [FieldName] -> ShowS

IsString FieldName

Methods

fromString :: String -> FieldName

Hashable FieldName

Methods

hashWithSalt :: Int -> FieldName -> Int

hash :: FieldName -> Int

field :: Field a -> Traversal' RecordHeader a

A lens-y means of querying Fields.

lookupField :: RecordHeader -> Field a -> Maybe (Either String a)

Lookup the value of a field. Returns Nothing if the field is not present, Just (Left err) in the event of a parse error, and Just (Right v) on success.

addField :: Field a -> a -> RecordHeader -> RecordHeader

mapField :: (a -> b) -> (b -> a) -> Field a -> Field b

rawField :: FieldName -> Field ByteString

Standard fields

warcRecordId :: Field RecordId

contentLength :: Field Integer

warcDate :: Field UTCTime

warcType :: Field WarcType

contentType :: Field ByteString

warcConcurrentTo :: Field RecordId

warcBlockDigest :: Field Digest

warcPayloadDigest :: Field Digest

warcIpAddress :: Field ByteString

warcRefersTo :: Field Uri

warcTargetUri :: Field Uri

warcTruncated :: Field TruncationReason

warcWarcinfoID :: Field RecordId

warcFilename :: Field Text

warcProfile :: Field Uri

warcSegmentNumber :: Field Integer

warcSegmentTotalLength :: Field Integer

Lenses

recWarcVersion :: Lens' RecordHeader Version

recHeaders :: Lens' RecordHeader (HashMap FieldName ByteString)