Control.MapReduce.Engines.Streaming

map-reduce engine (fold builder) using Streaming.Streams as its intermediate type. Because Streaming.Stream does not end with the type of data data in the Stream, we wrap this type in StreamResult for the purposes of the output type of the fold.

Helper Types

newtype StreamResult m d

Wrap Stream (Of d) m () in a type which has d as its last parameter

Constructors

StreamResult

Fields

• unRes :: Stream (Of d) m ()

Engines

streamingEngine :: (Foldable g, Functor g) => (forall z r. Stream (Of (k, z)) Identity r -> Stream (Of (k, g z)) Identity r) -> MapReduceFold y k c (StreamResult Identity) x d

map-reduce-fold builder returning a StreamResult

streamingEngineM :: (Monad m, Traversable g) => (forall z r. Stream (Of (k, z)) m r -> Stream (Of (k, g z)) m r) -> MapReduceFoldM m y k c (StreamResult m) x d

effectful map-reduce-fold builder returning a StreamResult

Result Extractors

resultToList :: Monad m => StreamResult m d -> m [d]

get a [] result from a Stream

concatStream :: (Monad m, Monoid a) => StreamResult m a -> m a

mappend all elements of a StreamResult of monoids

concatStreamFold :: Monoid b => Fold a (StreamResult Identity b) -> Fold a b

apply monoidal stream-concatenation to a fold returning a stream to produce a fold returning the monoid

concatStreamFoldM :: (Monad m, Monoid b) => FoldM m a (StreamResult m b) -> FoldM m a b

apply monoidal stream-concatenation to an effectful fold returning a stream to produce an effectful fold returning the monoid

groupBy functions

groupByHashableKey :: forall m k c r. (Monad m, Hashable k, Eq k) => Stream (Of (k, c)) m r -> Stream (Of (k, Seq c)) m r

group the mapped and assigned values by key using a Data.HashMap.Strict

groupByOrderedKey :: forall m k c r. (Monad m, Ord k) => Stream (Of (k, c)) m r -> Stream (Of (k, Seq c)) m r

group the mapped and assigned values by key using a Data.Map.Strict