Spark.Core.Internal.TypesStructures

The structures of data types in Krapsh.

For a detailed description of the supported types, see http:/spark.apache.orgdocslatestsql-programming-guide.html#data-types

At a high-level, Spark DataFrames and Datasets are equivalent to lists of objects whose type can be mapped to the same StructType: Dataset a ~ ArrayType StructType (...) Columns of a dataset are equivalent to lists of object whose type can be mapped to the same DataType (either Strict or Nullable) Local data (or "blobs") are single elements whose type can be mapped to a DataType (either strict or nullable)

Documentation

data StrictDataType

The data types that are guaranteed to not be null: evaluating them will return a value.

Constructors

IntType

StringType

Struct !StructType

ArrayType

Fields

• elementType :: !DataType

Instances

Eq StrictDataType

Methods

(==) :: StrictDataType -> StrictDataType -> Bool

(/=) :: StrictDataType -> StrictDataType -> Bool

Show StrictDataType

Methods

showsPrec :: Int -> StrictDataType -> ShowS

show :: StrictDataType -> String

showList :: [StrictDataType] -> ShowS

Arbitrary StrictDataType

Methods

arbitrary :: Gen StrictDataType

shrink :: StrictDataType -> [StrictDataType]

ToJSON StrictDataType

Methods

toJSON :: StrictDataType -> Value

toEncoding :: StrictDataType -> Encoding

toJSONList :: [StrictDataType] -> Value

toEncodingList :: [StrictDataType] -> Encoding

data DataType

All the data types supported by the Spark engine. The data types can either be nullable (they may contain null values) or strict (all the values are present). There are a couple of differences with the algebraic data types in Haskell: Maybe (Maybe a) ~ Maybe a which implies that arbitrary nesting of values will be flattened to a top-level Nullable Similarly, [[]] ~ []

Constructors

StrictType !StrictDataType

NullableType !StrictDataType

Instances

Eq DataType

Methods

(==) :: DataType -> DataType -> Bool

(/=) :: DataType -> DataType -> Bool

Show DataType

Methods

showsPrec :: Int -> DataType -> ShowS

show :: DataType -> String

showList :: [DataType] -> ShowS

Arbitrary DataType

Methods

arbitrary :: Gen DataType

shrink :: DataType -> [DataType]

ToJSON DataType

Methods

toJSON :: DataType -> Value

toEncoding :: DataType -> Encoding

toJSONList :: [DataType] -> Value

toEncodingList :: [DataType] -> Encoding

data StructField

A field in a structure

Constructors

StructField

Fields

• structFieldName :: !FieldName
• structFieldType :: !DataType

Instances

Eq StructField

Methods

(==) :: StructField -> StructField -> Bool

(/=) :: StructField -> StructField -> Bool

Show StructField

Methods

showsPrec :: Int -> StructField -> ShowS

show :: StructField -> String

showList :: [StructField] -> ShowS

Arbitrary StructField

Methods

arbitrary :: Gen StructField

shrink :: StructField -> [StructField]

data StructType

The main structure of a dataframe or a dataset

Constructors

StructType

Fields

• structFields :: !(Vector StructField)

Instances

Eq StructType

Methods

(==) :: StructType -> StructType -> Bool

(/=) :: StructType -> StructType -> Bool

Show StructType

Methods

showsPrec :: Int -> StructType -> ShowS

show :: StructType -> String

showList :: [StructType] -> ShowS

Arbitrary StructType

Methods

arbitrary :: Gen StructType

shrink :: StructType -> [StructType]

ToJSON StructType

Methods

toJSON :: StructType -> Value

toEncoding :: StructType -> Encoding

toJSONList :: [StructType] -> Value

toEncodingList :: [StructType] -> Encoding

data Nullable

Represents the choice between a strict and a nullable field

Constructors

CanNull

NoNull

Instances

Eq Nullable

Methods

(==) :: Nullable -> Nullable -> Bool

(/=) :: Nullable -> Nullable -> Bool

Show Nullable

Methods

showsPrec :: Int -> Nullable -> ShowS

show :: Nullable -> String

showList :: [Nullable] -> ShowS

data NullableDataType

Encodes the type of all the nullable data types

Constructors

NullableDataType !StrictDataType

Instances

Eq NullableDataType

Methods

(==) :: NullableDataType -> NullableDataType -> Bool

(/=) :: NullableDataType -> NullableDataType -> Bool

data SQLType a

A tagged datatype that encodes the sql types This is the main type information that should be used by users.

Constructors

SQLType

Fields

• unSQLType :: !DataType

  The underlying data type.

Instances

Eq (SQLType a)

Methods

(==) :: SQLType a -> SQLType a -> Bool

(/=) :: SQLType a -> SQLType a -> Bool

Show (SQLType a)

Methods

showsPrec :: Int -> SQLType a -> ShowS

show :: SQLType a -> String

showList :: [SQLType a] -> ShowS

Generic (SQLType a)

Associated Types

type Rep (SQLType a) :: * -> *

Methods

from :: SQLType a -> Rep (SQLType a) x

to :: Rep (SQLType a) x -> SQLType a

type Rep (SQLType a)

type Rep (SQLType a) = D1 (MetaData "SQLType" "Spark.Core.Internal.TypesStructures" "krapsh-0.1.6.1-24YumfFSIoWDNUVRZjuyVp" False) (C1 (MetaCons "SQLType" PrefixI True) (S1 (MetaSel (Just Symbol "unSQLType") NoSourceUnpackedness SourceStrict DecidedStrict) (Rec0 DataType)))

_fieldToJson :: StructField -> (Text, Value)