module Test.Chell.Main
( defaultMain
) where
import Control.Applicative
import Control.Monad (forM, forM_, when)
import Control.Monad.Trans.Class (lift)
import qualified Control.Monad.Trans.State as State
import qualified Control.Monad.Trans.Writer as Writer
import Data.Char (ord)
import Data.List (isPrefixOf)
import System.Exit (exitSuccess, exitFailure)
import System.IO (hPutStr, hPutStrLn, hIsTerminalDevice, stderr, stdout, withBinaryFile, IOMode(..))
import System.Random (randomIO)
import Text.Printf (printf)
import Options
import Test.Chell.Output
import Test.Chell.Types
data MainOptions = MainOptions
{ optVerbose :: Bool
, optXmlReport :: String
, optJsonReport :: String
, optTextReport :: String
, optSeed :: Maybe Int
, optTimeout :: Maybe Int
, optColor :: ColorMode
}
optionType_ColorMode :: OptionType ColorMode
optionType_ColorMode = optionType "ColorMode" ColorModeAuto parseMode showMode where
parseMode s = case s of
"always" -> Right ColorModeAlways
"never" -> Right ColorModeNever
"auto" -> Right ColorModeAuto
_ -> Left (show s ++ " is not in {\"always\", \"never\", \"auto\"}.")
showMode mode = case mode of
ColorModeAlways -> "always"
ColorModeNever -> "never"
ColorModeAuto -> "auto"
instance Options MainOptions where
defineOptions = pure MainOptions
<*> defineOption optionType_bool (\o -> o
{ optionShortFlags = ['v']
, optionLongFlags = ["verbose"]
, optionDefault = False
, optionDescription = "Print more output."
})
<*> simpleOption "xml-report" ""
"Write a parsable report to a given path, in XML."
<*> simpleOption "json-report" ""
"Write a parsable report to a given path, in JSON."
<*> simpleOption "text-report" ""
"Write a human-readable report to a given path."
<*> simpleOption "seed" Nothing
"The seed used for random numbers in (for example) quickcheck."
<*> simpleOption "timeout" Nothing
"The maximum duration of a test, in milliseconds."
<*> defineOption optionType_ColorMode (\o -> o
{ optionLongFlags = ["color"]
, optionDefault = ColorModeAuto
, optionDescription = "Whether to enable color ('always', 'auto', or 'never')."
})
-- | A simple default main function, which runs a list of tests and logs
-- statistics to stdout.
defaultMain :: [Suite] -> IO ()
defaultMain suites = runCommand $ \opts args -> do
-- validate/sanitize test options
seed <- case optSeed opts of
Just s -> return s
Nothing -> randomIO
timeout <- case optTimeout opts of
Nothing -> return Nothing
Just t -> if toInteger t * 1000 > toInteger (maxBound :: Int)
then do
hPutStrLn stderr "Test.Chell.defaultMain: Ignoring --timeout because it is too large."
return Nothing
else return (Just t)
let testOptions = defaultTestOptions
{ testOptionSeed = seed
, testOptionTimeout = timeout
}
-- find which tests to run
let allTests = concatMap suiteTests suites
let tests = if null args
then allTests
else filter (matchesFilter args) allTests
-- output mode
output <- case optColor opts of
ColorModeNever -> return (plainOutput (optVerbose opts))
ColorModeAlways -> return (colorOutput (optVerbose opts))
ColorModeAuto -> do
isTerm <- hIsTerminalDevice stdout
return $ if isTerm
then colorOutput (optVerbose opts)
else plainOutput (optVerbose opts)
-- run tests
results <- forM tests $ \t -> do
outputStart output t
result <- runTest t testOptions
outputResult output t result
return (t, result)
-- generate reports
let reports = getReports opts
forM_ reports $ \(path, fmt, toText) ->
withBinaryFile path WriteMode $ \h -> do
when (optVerbose opts) $ do
putStrLn ("Writing " ++ fmt ++ " report to " ++ show path)
hPutStr h (toText results)
let stats = resultStatistics results
let (_, _, failed, aborted) = stats
putStrLn (formatResultStatistics stats)
if failed == 0 && aborted == 0
then exitSuccess
else exitFailure
matchesFilter :: [String] -> Test -> Bool
matchesFilter filters = check where
check t = any (matchName (testName t)) filters
matchName name f = f == name || isPrefixOf (f ++ ".") name
type Report = [(Test, TestResult)] -> String
getReports :: MainOptions -> [(String, String, Report)]
getReports opts = concat [xml, json, text] where
xml = case optXmlReport opts of
"" -> []
path -> [(path, "XML", xmlReport)]
json = case optJsonReport opts of
"" -> []
path -> [(path, "JSON", jsonReport)]
text = case optTextReport opts of
"" -> []
path -> [(path, "text", textReport)]
jsonReport :: [(Test, TestResult)] -> String
jsonReport results = Writer.execWriter writer where
tell = Writer.tell
writer = do
tell "{\"test-runs\": ["
commas results tellResult
tell "]}"
tellResult (t, result) = case result of
TestPassed notes -> do
tell "{\"test\": \""
tell (escapeJSON (testName t))
tell "\", \"result\": \"passed\""
tellNotes notes
tell "}"
TestSkipped -> do
tell "{\"test\": \""
tell (escapeJSON (testName t))
tell "\", \"result\": \"skipped\"}"
TestFailed notes fs -> do
tell "{\"test\": \""
tell (escapeJSON (testName t))
tell "\", \"result\": \"failed\", \"failures\": ["
commas fs $ \f -> do
tell "{\"message\": \""
tell (escapeJSON (failureMessage f))
tell "\""
case failureLocation f of
Just loc -> do
tell ", \"location\": {\"module\": \""
tell (escapeJSON (locationModule loc))
tell "\", \"file\": \""
tell (escapeJSON (locationFile loc))
case locationLine loc of
Just line -> do
tell "\", \"line\": "
tell (show line)
Nothing -> tell "\""
tell "}"
Nothing -> return ()
tell "}"
tell "]"
tellNotes notes
tell "}"
TestAborted notes msg -> do
tell "{\"test\": \""
tell (escapeJSON (testName t))
tell "\", \"result\": \"aborted\", \"abortion\": {\"message\": \""
tell (escapeJSON msg)
tell "\"}"
tellNotes notes
tell "}"
_ -> return ()
escapeJSON = concatMap (\c -> case c of
'"' -> "\\\""
'\\' -> "\\\\"
_ | ord c <= 0x1F -> printf "\\u%04X" (ord c)
_ -> [c])
tellNotes notes = do
tell ", \"notes\": ["
commas notes $ \(key, value) -> do
tell "{\"key\": \""
tell (escapeJSON key)
tell "\", \"value\": \""
tell (escapeJSON value)
tell "\"}"
tell "]"
commas xs block = State.evalStateT (commaState xs block) False
commaState xs block = forM_ xs $ \x -> do
let tell' = lift . Writer.tell
needComma <- State.get
if needComma
then tell' "\n, "
else tell' "\n "
State.put True
lift (block x)
xmlReport :: [(Test, TestResult)] -> String
xmlReport results = Writer.execWriter writer where
tell = Writer.tell
writer = do
tell "\n"
tell "\n"
mapM_ tellResult results
tell ""
tellResult (t, result) = case result of
TestPassed notes -> do
tell "\t\n"
tellNotes notes
tell "\t\n"
TestSkipped -> do
tell "\t\n"
TestFailed notes fs -> do
tell "\t\n"
forM_ fs $ \f -> do
tell "\t\t\n"
tell "\t\t\t\n"
tell "\t\t\n"
Nothing -> tell "'/>\n"
tellNotes notes
tell "\t\n"
TestAborted notes msg -> do
tell "\t\n"
tell "\t\t\n"
tellNotes notes
tell "\t\n"
_ -> return ()
escapeXML = concatMap (\c -> case c of
'&' -> "&"
'<' -> "<"
'>' -> ">"
'"' -> """
'\'' -> "'"
_ -> [c])
tellNotes notes = forM_ notes $ \(key, value) -> do
tell "\t\t\n"
textReport :: [(Test, TestResult)] -> String
textReport results = Writer.execWriter writer where
tell = Writer.tell
writer = do
forM_ results tellResult
let stats = resultStatistics results
tell (formatResultStatistics stats)
tellResult (t, result) = case result of
TestPassed notes -> do
tell (replicate 70 '=')
tell "\n"
tell "PASSED: "
tell (testName t)
tell "\n"
tellNotes notes
tell "\n\n"
TestSkipped -> do
tell (replicate 70 '=')
tell "\n"
tell "SKIPPED: "
tell (testName t)
tell "\n\n"
TestFailed notes fs -> do
tell (replicate 70 '=')
tell "\n"
tell "FAILED: "
tell (testName t)
tell "\n"
tellNotes notes
tell (replicate 70 '-')
tell "\n"
forM_ fs $ \f -> do
case failureLocation f of
Just loc -> do
tell (locationFile loc)
case locationLine loc of
Just line -> do
tell ":"
tell (show line)
Nothing -> return ()
tell "\n"
Nothing -> return ()
tell (failureMessage f)
tell "\n\n"
TestAborted notes msg -> do
tell (replicate 70 '=')
tell "\n"
tell "ABORTED: "
tell (testName t)
tell "\n"
tellNotes notes
tell (replicate 70 '-')
tell "\n"
tell msg
tell "\n\n"
_ -> return ()
tellNotes notes = forM_ notes $ \(key, value) -> do
tell key
tell "="
tell value
tell "\n"
formatResultStatistics :: (Integer, Integer, Integer, Integer) -> String
formatResultStatistics stats = Writer.execWriter writer where
writer = do
let (passed, skipped, failed, aborted) = stats
if failed == 0 && aborted == 0
then Writer.tell "PASS: "
else Writer.tell "FAIL: "
let putNum comma n what = Writer.tell $ if n == 1
then comma ++ "1 test " ++ what
else comma ++ show n ++ " tests " ++ what
let total = sum [passed, skipped, failed, aborted]
putNum "" total "run"
(putNum ", " passed "passed")
when (skipped > 0) (putNum ", " skipped "skipped")
when (failed > 0) (putNum ", " failed "failed")
when (aborted > 0) (putNum ", " aborted "aborted")
resultStatistics :: [(Test, TestResult)] -> (Integer, Integer, Integer, Integer)
resultStatistics results = State.execState state (0, 0, 0, 0) where
state = forM_ results $ \(_, result) -> case result of
TestPassed{} -> State.modify (\(p, s, f, a) -> (p+1, s, f, a))
TestSkipped{} -> State.modify (\(p, s, f, a) -> (p, s+1, f, a))
TestFailed{} -> State.modify (\(p, s, f, a) -> (p, s, f+1, a))
TestAborted{} -> State.modify (\(p, s, f, a) -> (p, s, f, a+1))
_ -> return ()