Frames.InCore

Efficient in-memory (in-core) storage of tabular data.

Documentation

type family VectorFor t :: * -> *

The most efficient vector type for each column data type.

Instances

type VectorFor Bool

type VectorFor Bool = Vector

type VectorFor Double

type VectorFor Double = Vector

type VectorFor Float

type VectorFor Float = Vector

type VectorFor Int

type VectorFor Int = Vector

type VectorFor String

type VectorFor String = Vector

type VectorFor Text

type VectorFor Text = Vector

type VectorMFor a = Mutable (VectorFor a)

The mutable version of VectorFor a particular type.

initialCapacity :: Int

Since we stream into the in-memory representation, we use an exponential growth strategy to resize arrays as more data is read in. This is the initial capacity of each column.

type family VectorMs m rs where ...

Mutable vector types for each column in a row.

Equations

VectorMs m '[] = '[]

VectorMs m ((s :-> a) ': rs) = (s :-> VectorMFor a (PrimState m) a) ': VectorMs m rs

type family Vectors rs where ...

Immutable vector types for each column in a row.

Equations

Vectors '[] = '[]

Vectors ((s :-> a) ': rs) = (s :-> VectorFor a a) ': Vectors rs

class RecVec rs where

Tooling to allocate, grow, write to, freeze, and index into records of vectors.

Minimal complete definition

allocRec, freezeRec, growRec, writeRec, indexRec, produceRec

Methods

allocRec :: PrimMonad m => proxy rs -> m (Record (VectorMs m rs))

freezeRec :: PrimMonad m => proxy rs -> Int -> Record (VectorMs m rs) -> m (Record (Vectors rs))

growRec :: PrimMonad m => proxy rs -> Record (VectorMs m rs) -> m (Record (VectorMs m rs))

writeRec :: PrimMonad m => proxy rs -> Int -> Record (VectorMs m rs) -> Record rs -> m ()

indexRec :: proxy rs -> Int -> Record (Vectors rs) -> Record rs

produceRec :: proxy rs -> Record (Vectors rs) -> Rec ((->) Int) rs

Instances

RecVec ([] *)

Methods

allocRec :: PrimMonad m => proxy [*] -> m (Record (VectorMs m [*]))

freezeRec :: PrimMonad m => proxy [*] -> Int -> Record (VectorMs m [*]) -> m (Record (Vectors [*]))

growRec :: PrimMonad m => proxy [*] -> Record (VectorMs m [*]) -> m (Record (VectorMs m [*]))

writeRec :: PrimMonad m => proxy [*] -> Int -> Record (VectorMs m [*]) -> Record [*] -> m ()

indexRec :: proxy [*] -> Int -> Record (Vectors [*]) -> Record [*]

produceRec :: proxy [*] -> Record (Vectors [*]) -> Rec * ((->) Int) [*]

(MVector (VectorMFor a) a, Vector (VectorFor a) a, RecVec rs) => RecVec ((:) * ((:->) s a) rs)

Methods

allocRec :: PrimMonad m => proxy ((* ': (s :-> a)) rs) -> m (Record (VectorMs m ((* ': (s :-> a)) rs)))

freezeRec :: PrimMonad m => proxy ((* ': (s :-> a)) rs) -> Int -> Record (VectorMs m ((* ': (s :-> a)) rs)) -> m (Record (Vectors ((* ': (s :-> a)) rs)))

growRec :: PrimMonad m => proxy ((* ': (s :-> a)) rs) -> Record (VectorMs m ((* ': (s :-> a)) rs)) -> m (Record (VectorMs m ((* ': (s :-> a)) rs)))

writeRec :: PrimMonad m => proxy ((* ': (s :-> a)) rs) -> Int -> Record (VectorMs m ((* ': (s :-> a)) rs)) -> Record ((* ': (s :-> a)) rs) -> m ()

indexRec :: proxy ((* ': (s :-> a)) rs) -> Int -> Record (Vectors ((* ': (s :-> a)) rs)) -> Record ((* ': (s :-> a)) rs)

produceRec :: proxy ((* ': (s :-> a)) rs) -> Record (Vectors ((* ': (s :-> a)) rs)) -> Rec * ((->) Int) ((* ': (s :-> a)) rs)

inCoreSoA :: forall m rs. (PrimMonad m, RecVec rs) => Producer (Record rs) m () -> m (Int, Rec ((->) Int) rs)

Stream a finite sequence of rows into an efficient in-memory representation for further manipulation. Each column of the input table will be stored optimally based on its type, making use of the resulting generators a matter of indexing into a densely packed representation. Returns the number of rows and a record of column indexing functions. See toAoS to convert the result to a Frame which provides an easier-to-use function that indexes into the table in a row-major fashion.

inCoreAoS :: (PrimMonad m, RecVec rs) => Producer (Record rs) m () -> m (FrameRec rs)

Stream a finite sequence of rows into an efficient in-memory representation for further manipulation. Each column of the input table will be stored optimally based on its type, making use of the resulting generators a matter of indexing into a densely packed representation. Returns a Frame that provides a function to index into the table.

inCoreAoS' :: (PrimMonad m, RecVec rs) => (Rec ((->) Int) rs -> Rec ((->) Int) ss) -> Producer (Record rs) m () -> m (FrameRec ss)

Like inCoreAoS, but applies the provided function to the record of columns before building the Frame.

toAoS :: Int -> Rec ((->) Int) rs -> FrameRec rs

Convert a structure-of-arrays to an array-of-structures. This can simplify usage of an in-memory representation.

inCore :: forall m n rs. (PrimMonad m, RecVec rs, Monad n) => Producer (Record rs) m () -> m (Producer (Record rs) n ())

Stream a finite sequence of rows into an efficient in-memory representation for further manipulation. Each column of the input table will be stored optimally based on its type, making use of the resulting generator a matter of indexing into a densely packed representation.

toFrame :: (Foldable f, RecVec rs) => f (Record rs) -> Frame (Record rs)

Build a Frame from a collection of Records using efficient column-based storage.

filterFrame :: RecVec rs => (Record rs -> Bool) -> FrameRec rs -> FrameRec rs

Keep only those rows of a FrameRec that satisfy a predicate.