module Spark.Core.Internal.OpFunctions(
simpleShowOp,
extraNodeOpData,
hashUpdateNodeOp,
prettyShowColOp,
) where
import qualified Data.Text as T
import qualified Data.Aeson as A
import qualified Data.Vector as V
import qualified Data.ByteString as BS
import qualified Data.ByteString.Lazy as LBS
import Data.Aeson((.=), toJSON)
import Data.Char(isSymbol)
import qualified Crypto.Hash.SHA256 as SHA
import Spark.Core.Internal.OpStructures
import Spark.Core.Internal.Utilities
simpleShowOp :: NodeOp -> T.Text
simpleShowOp (NodeLocalOp op) = soName op
simpleShowOp (NodeDistributedOp op) = soName op
simpleShowOp (NodeLocalLit _ _) = T.pack "org.spark.LocalConstant"
simpleShowOp (NodeOpaqueAggregator op) = soName op
simpleShowOp (NodeUniversalAggregator ua) = soName . uaoMergeBuffer $ ua
simpleShowOp (NodeStructuredTransform _) = T.pack "org.spark.Select"
simpleShowOp (NodeDistributedLit _ _) = T.pack "org.spark.Constant"
prettyShowColOp :: ColOp -> T.Text
prettyShowColOp (ColExtraction fpath) = T.pack (show fpath)
prettyShowColOp (ColFunction txt cols) =
_prettyShowColFun txt (V.toList cols)
prettyShowColOp (ColLit _ cell) = T.pack (show cell)
prettyShowColOp (ColStruct s) =
"struct(" <> T.intercalate "," (prettyShowColOp . tfValue <$> V.toList s) <> ")"
extraNodeOpData :: NodeOp -> A.Value
extraNodeOpData (NodeLocalLit dt cell) =
A.object [ "type" .= toJSON dt,
"content" .= toJSON cell]
extraNodeOpData (NodeStructuredTransform st) = toJSON st
extraNodeOpData (NodeDistributedLit dt lst) =
A.object [ "cellType" .= toJSON dt,
"content" .= toJSON lst]
extraNodeOpData _ = A.Null
hashUpdateNodeOp :: SHA.Ctx -> NodeOp -> SHA.Ctx
hashUpdateNodeOp ctx op = _hashUpdateJson ctx $ A.object [
"op" .= simpleShowOp op,
"extra" .= extraNodeOpData op]
_prettyShowColFun :: T.Text -> [ColOp] -> T.Text
_prettyShowColFun txt [col] | _isSym txt =
T.concat [txt, prettyShowColOp col]
_prettyShowColFun txt [col1, col2] | _isSym txt =
T.concat [prettyShowColOp col1, txt, prettyShowColOp col2]
_prettyShowColFun txt cols =
let vals = T.intercalate ", " (prettyShowColOp <$> cols) in
T.concat [txt, "(", vals, ")"]
_isSym :: T.Text -> Bool
_isSym txt = all isSymbol (T.unpack txt)
instance A.ToJSON ColOp where
toJSON (ColExtraction fp) = A.object [
"colOp" .= T.pack "extraction",
"field" .= toJSON fp]
toJSON (ColFunction txt cols) = A.object [
"colOp" .= T.pack "fun",
"function" .= txt,
"args" .= (toJSON <$> cols)]
toJSON (ColLit _ cell) = A.object [
"colOp" .= T.pack "literal",
"lit" .= toJSON cell]
toJSON (ColStruct v) =
let fun (TransformField fn colOp) =
A.object ["name" .= T.pack (show fn), "op" .= toJSON colOp]
in A.Array $ fun <$> v
_hashUpdateJson :: SHA.Ctx -> A.Value -> SHA.Ctx
_hashUpdateJson ctx val = SHA.update ctx bs where
bs = BS.concat . LBS.toChunks . encodeDeterministicPretty $ val