{-# LANGUAGE BangPatterns, NamedFieldPuns, RecordWildCards #-}

module Data.SearchEngine.Query (

    -- * Querying
    query,
    ResultsFilter(..),

    -- * Explain mode for query result rankings
    queryExplain,
    BM25F.Explanation(..),
    setRankParams,

    -- ** Utils used by autosuggest
    relevanceScore,
    indexDocToBM25Doc,
    expandTransformedQueryTerm,
  ) where

import Data.SearchEngine.Types
import qualified Data.SearchEngine.SearchIndex as SI
import qualified Data.SearchEngine.DocIdSet    as DocIdSet
import qualified Data.SearchEngine.DocTermIds  as DocTermIds
import qualified Data.SearchEngine.DocFeatVals as DocFeatVals
import qualified Data.SearchEngine.BM25F       as BM25F

import Data.Ix
import Data.List
import Data.Function
import Data.Maybe


-- | Execute a normal query. Find the documents in which one or more of
-- the search terms appear and return them in ranked order.
--
-- The number of documents returned is limited by the 'paramResultsetSoftLimit'
-- and 'paramResultsetHardLimit' paramaters. This also limits the cost of the
-- query (which is primarily the cost of scoring each document).
--
-- The given terms are all assumed to be complete (as opposed to prefixes
-- like with 'queryAutosuggest').
--
query :: (Ix field, Bounded field, Ix feature, Bounded feature) =>
         SearchEngine doc key field feature ->
         [Term] -> [key]
query :: forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature -> [Term] -> [key]
query se :: SearchEngine doc key field feature
se@SearchEngine{ SearchIndex key field feature
searchIndex :: SearchIndex key field feature
searchIndex :: forall doc key field feature.
SearchEngine doc key field feature -> SearchIndex key field feature
searchIndex,
                       searchRankParams :: forall doc key field feature.
SearchEngine doc key field feature
-> SearchRankParameters field feature
searchRankParams = SearchRankParameters{Float
Int
field -> Float
feature -> Float
feature -> FeatureFunction
paramResultsetSoftLimit :: forall field feature. SearchRankParameters field feature -> Int
paramResultsetHardLimit :: forall field feature. SearchRankParameters field feature -> Int
paramK1 :: Float
paramB :: field -> Float
paramFieldWeights :: field -> Float
paramFeatureWeights :: feature -> Float
paramFeatureFunctions :: feature -> FeatureFunction
paramResultsetSoftLimit :: Int
paramResultsetHardLimit :: Int
paramAutosuggestPrefilterLimit :: Int
paramAutosuggestPostfilterLimit :: Int
paramK1 :: forall field feature. SearchRankParameters field feature -> Float
paramB :: forall field feature.
SearchRankParameters field feature -> field -> Float
paramFieldWeights :: forall field feature.
SearchRankParameters field feature -> field -> Float
paramFeatureWeights :: forall field feature.
SearchRankParameters field feature -> feature -> Float
paramFeatureFunctions :: forall field feature.
SearchRankParameters field feature -> feature -> FeatureFunction
paramAutosuggestPrefilterLimit :: forall field feature. SearchRankParameters field feature -> Int
paramAutosuggestPostfilterLimit :: forall field feature. SearchRankParameters field feature -> Int
..} }
      [Term]
terms =

  let -- Start by transforming/normalising all the query terms.
      -- This can be done differently for each field we search by.
      lookupTerms :: [Term]
      lookupTerms :: [Term]
lookupTerms = (Term -> [Term]) -> [Term] -> [Term]
forall (t :: * -> *) a b. Foldable t => (a -> [b]) -> t a -> [b]
concatMap (SearchEngine doc key field feature -> Term -> [Term]
forall field doc key feature.
(Ix field, Bounded field) =>
SearchEngine doc key field feature -> Term -> [Term]
expandTransformedQueryTerm SearchEngine doc key field feature
se) [Term]
terms

      -- Then we look up all the normalised terms in the index.
      rawresults :: [Maybe (TermId, DocIdSet)]
      rawresults :: [Maybe (TermId, DocIdSet)]
rawresults = (Term -> Maybe (TermId, DocIdSet))
-> [Term] -> [Maybe (TermId, DocIdSet)]
forall a b. (a -> b) -> [a] -> [b]
map (SearchIndex key field feature -> Term -> Maybe (TermId, DocIdSet)
forall key field feature.
SearchIndex key field feature -> Term -> Maybe (TermId, DocIdSet)
SI.lookupTerm SearchIndex key field feature
searchIndex) [Term]
lookupTerms

      -- For the terms that occur in the index, this gives us the term's id
      -- and the set of documents that the term occurs in.
      termids   :: [TermId]
      docidsets :: [DocIdSet]
      ([TermId]
termids, [DocIdSet]
docidsets) = [(TermId, DocIdSet)] -> ([TermId], [DocIdSet])
forall a b. [(a, b)] -> ([a], [b])
unzip ([Maybe (TermId, DocIdSet)] -> [(TermId, DocIdSet)]
forall a. [Maybe a] -> [a]
catMaybes [Maybe (TermId, DocIdSet)]
rawresults)

      -- We looked up the documents that *any* of the term occur in (not all)
      -- so this could be rather a lot of docs if the user uses a few common
      -- terms. Scoring these result docs is a non-trivial cost so we want to
      -- limit the number that we have to score. The standard trick is to
      -- consider the doc sets in the order of size, smallest to biggest. Once
      -- we have gone over a certain threshold of docs then don't bother with
      -- the doc sets for the remaining terms. This tends to work because the
      -- scoring gives lower weight to terms that occur in many documents.
      unrankedResults :: DocIdSet
      unrankedResults :: DocIdSet
unrankedResults = Int -> Int -> [DocIdSet] -> DocIdSet
pruneRelevantResults
                          Int
paramResultsetSoftLimit
                          Int
paramResultsetHardLimit
                          [DocIdSet]
docidsets

      --TODO: technically this isn't quite correct. Because each field can
      -- be normalised differently, we can end up with different termids for
      -- the same original search term, and then we score those as if they
      -- were different terms, which makes a difference when the term appears
      -- in multiple fields (exactly the case BM25F is supposed to deal with).
      -- What we ought to have instead is an Array (Int, field) TermId, and
      -- make the scoring use the appropriate termid for each field, but to
      -- consider them the "same" term.
   in SearchEngine doc key field feature -> [TermId] -> [DocId] -> [key]
forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature -> [TermId] -> [DocId] -> [key]
rankResults SearchEngine doc key field feature
se [TermId]
termids (DocIdSet -> [DocId]
DocIdSet.toList DocIdSet
unrankedResults)

-- | Before looking up a term in the main index we need to normalise it
-- using the 'transformQueryTerm'. Of course the transform can be different
-- for different fields, so we have to collect all the forms (eliminating
-- duplicates).
--
expandTransformedQueryTerm :: (Ix field, Bounded field) =>
                              SearchEngine doc key field feature ->
                              Term -> [Term]
expandTransformedQueryTerm :: forall field doc key feature.
(Ix field, Bounded field) =>
SearchEngine doc key field feature -> Term -> [Term]
expandTransformedQueryTerm SearchEngine{SearchConfig doc key field feature
searchConfig :: SearchConfig doc key field feature
searchConfig :: forall doc key field feature.
SearchEngine doc key field feature
-> SearchConfig doc key field feature
searchConfig} Term
term =
    [Term] -> [Term]
forall a. Eq a => [a] -> [a]
nub [ field -> Term
transformForField field
field
        | let transformForField :: field -> Term
transformForField = SearchConfig doc key field feature -> Term -> field -> Term
forall doc key field feature.
SearchConfig doc key field feature -> Term -> field -> Term
transformQueryTerm SearchConfig doc key field feature
searchConfig Term
term
        , field
field <- (field, field) -> [field]
forall a. Ix a => (a, a) -> [a]
range (field
forall a. Bounded a => a
minBound, field
forall a. Bounded a => a
maxBound) ]


rankResults :: (Ix field, Bounded field, Ix feature, Bounded feature) =>
               SearchEngine doc key field feature ->
               [TermId] -> [DocId] -> [key]
rankResults :: forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature -> [TermId] -> [DocId] -> [key]
rankResults se :: SearchEngine doc key field feature
se@SearchEngine{SearchIndex key field feature
searchIndex :: forall doc key field feature.
SearchEngine doc key field feature -> SearchIndex key field feature
searchIndex :: SearchIndex key field feature
searchIndex} [TermId]
queryTerms [DocId]
docids =
    ((Float, key) -> key) -> [(Float, key)] -> [key]
forall a b. (a -> b) -> [a] -> [b]
map (Float, key) -> key
forall a b. (a, b) -> b
snd
  ([(Float, key)] -> [key]) -> [(Float, key)] -> [key]
forall a b. (a -> b) -> a -> b
$ ((Float, key) -> (Float, key) -> Ordering)
-> [(Float, key)] -> [(Float, key)]
forall a. (a -> a -> Ordering) -> [a] -> [a]
sortBy ((Float -> Float -> Ordering) -> Float -> Float -> Ordering
forall a b c. (a -> b -> c) -> b -> a -> c
flip Float -> Float -> Ordering
forall a. Ord a => a -> a -> Ordering
compare (Float -> Float -> Ordering)
-> ((Float, key) -> Float)
-> (Float, key)
-> (Float, key)
-> Ordering
forall b c a. (b -> b -> c) -> (a -> b) -> a -> a -> c
`on` (Float, key) -> Float
forall a b. (a, b) -> a
fst)
      [ (SearchEngine doc key field feature
-> [TermId] -> DocTermIds field -> DocFeatVals feature -> Float
forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature
-> [TermId] -> DocTermIds field -> DocFeatVals feature -> Float
relevanceScore SearchEngine doc key field feature
se [TermId]
queryTerms DocTermIds field
doctermids DocFeatVals feature
docfeatvals, key
dockey)
      | DocId
docid <- [DocId]
docids
      , let (key
dockey, DocTermIds field
doctermids, DocFeatVals feature
docfeatvals) = SearchIndex key field feature
-> DocId -> (key, DocTermIds field, DocFeatVals feature)
forall key field feature.
SearchIndex key field feature
-> DocId -> (key, DocTermIds field, DocFeatVals feature)
SI.lookupDocId SearchIndex key field feature
searchIndex DocId
docid ]

relevanceScore :: (Ix field, Bounded field, Ix feature, Bounded feature) =>
                  SearchEngine doc key field feature ->
                  [TermId] -> DocTermIds field -> DocFeatVals feature -> Float
relevanceScore :: forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature
-> [TermId] -> DocTermIds field -> DocFeatVals feature -> Float
relevanceScore SearchEngine{Context TermId field feature
bm25Context :: Context TermId field feature
bm25Context :: forall doc key field feature.
SearchEngine doc key field feature -> Context TermId field feature
bm25Context} [TermId]
queryTerms DocTermIds field
doctermids DocFeatVals feature
docfeatvals =
    Context TermId field feature
-> Doc TermId field feature -> [TermId] -> Float
forall field feature term.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
Context term field feature
-> Doc term field feature -> [term] -> Float
BM25F.score Context TermId field feature
bm25Context Doc TermId field feature
doc [TermId]
queryTerms
  where
    doc :: Doc TermId field feature
doc = DocTermIds field -> DocFeatVals feature -> Doc TermId field feature
forall field feature.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
DocTermIds field -> DocFeatVals feature -> Doc TermId field feature
indexDocToBM25Doc DocTermIds field
doctermids DocFeatVals feature
docfeatvals

indexDocToBM25Doc :: (Ix field, Bounded field, Ix feature, Bounded feature) =>
                     DocTermIds field ->
                     DocFeatVals feature ->
                     BM25F.Doc TermId field feature
indexDocToBM25Doc :: forall field feature.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
DocTermIds field -> DocFeatVals feature -> Doc TermId field feature
indexDocToBM25Doc DocTermIds field
doctermids DocFeatVals feature
docfeatvals =
    BM25F.Doc {
      docFieldLength :: field -> Int
BM25F.docFieldLength        = DocTermIds field -> field -> Int
forall field.
(Ix field, Bounded field) =>
DocTermIds field -> field -> Int
DocTermIds.fieldLength    DocTermIds field
doctermids,
      docFieldTermFrequency :: field -> TermId -> Int
BM25F.docFieldTermFrequency = DocTermIds field -> field -> TermId -> Int
forall field.
(Ix field, Bounded field) =>
DocTermIds field -> field -> TermId -> Int
DocTermIds.fieldTermCount DocTermIds field
doctermids,
      docFeatureValue :: feature -> Float
BM25F.docFeatureValue       = DocFeatVals feature -> feature -> Float
forall feature.
(Ix feature, Bounded feature) =>
DocFeatVals feature -> feature -> Float
DocFeatVals.featureValue DocFeatVals feature
docfeatvals
    }

pruneRelevantResults :: Int -> Int -> [DocIdSet] -> DocIdSet
pruneRelevantResults :: Int -> Int -> [DocIdSet] -> DocIdSet
pruneRelevantResults Int
softLimit Int
hardLimit =
    -- Look at the docsets starting with the smallest ones. Smaller docsets
    -- correspond to the rarer terms, which are the ones that score most highly.
    DocIdSet -> [DocIdSet] -> DocIdSet
go DocIdSet
DocIdSet.empty ([DocIdSet] -> DocIdSet)
-> ([DocIdSet] -> [DocIdSet]) -> [DocIdSet] -> DocIdSet
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (DocIdSet -> DocIdSet -> Ordering) -> [DocIdSet] -> [DocIdSet]
forall a. (a -> a -> Ordering) -> [a] -> [a]
sortBy (Int -> Int -> Ordering
forall a. Ord a => a -> a -> Ordering
compare (Int -> Int -> Ordering)
-> (DocIdSet -> Int) -> DocIdSet -> DocIdSet -> Ordering
forall b c a. (b -> b -> c) -> (a -> b) -> a -> a -> c
`on` DocIdSet -> Int
DocIdSet.size)
  where
    go :: DocIdSet -> [DocIdSet] -> DocIdSet
go !DocIdSet
acc [] = DocIdSet
acc
    go !DocIdSet
acc (DocIdSet
d:[DocIdSet]
ds)
        -- If this is the first one, we add it anyway, otherwise we're in
        -- danger of returning no results at all.
      | DocIdSet -> Bool
DocIdSet.null DocIdSet
acc = DocIdSet -> [DocIdSet] -> DocIdSet
go DocIdSet
d [DocIdSet]
ds
        -- We consider the size our docset would be if we add this extra one...
        -- If it puts us over the hard limit then stop.
      | Int
size Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
hardLimit  = DocIdSet
acc
        -- If it puts us over soft limit then we add it and stop
      | Int
size Int -> Int -> Bool
forall a. Ord a => a -> a -> Bool
> Int
softLimit  = DocIdSet -> DocIdSet -> DocIdSet
DocIdSet.union DocIdSet
acc DocIdSet
d
        -- Otherwise we can add it and carry on to consider the remainder
      | Bool
otherwise         = DocIdSet -> [DocIdSet] -> DocIdSet
go (DocIdSet -> DocIdSet -> DocIdSet
DocIdSet.union DocIdSet
acc DocIdSet
d) [DocIdSet]
ds
      where
        size :: Int
size = DocIdSet -> Int
DocIdSet.size DocIdSet
acc Int -> Int -> Int
forall a. Num a => a -> a -> a
+ DocIdSet -> Int
DocIdSet.size DocIdSet
d


--------------------------------
-- Normal query with explanation
--

queryExplain :: (Ix field, Bounded field, Ix feature, Bounded feature) =>
                SearchEngine doc key field feature ->
                [Term] -> [(BM25F.Explanation field feature Term, key)]
queryExplain :: forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature
-> [Term] -> [(Explanation field feature Term, key)]
queryExplain se :: SearchEngine doc key field feature
se@SearchEngine{ SearchIndex key field feature
searchIndex :: forall doc key field feature.
SearchEngine doc key field feature -> SearchIndex key field feature
searchIndex :: SearchIndex key field feature
searchIndex,
                              searchConfig :: forall doc key field feature.
SearchEngine doc key field feature
-> SearchConfig doc key field feature
searchConfig     = SearchConfig{Term -> field -> Term
transformQueryTerm :: forall doc key field feature.
SearchConfig doc key field feature -> Term -> field -> Term
transformQueryTerm :: Term -> field -> Term
transformQueryTerm},
                              searchRankParams :: forall doc key field feature.
SearchEngine doc key field feature
-> SearchRankParameters field feature
searchRankParams = SearchRankParameters{Float
Int
field -> Float
feature -> Float
feature -> FeatureFunction
paramResultsetSoftLimit :: forall field feature. SearchRankParameters field feature -> Int
paramResultsetHardLimit :: forall field feature. SearchRankParameters field feature -> Int
paramK1 :: forall field feature. SearchRankParameters field feature -> Float
paramB :: forall field feature.
SearchRankParameters field feature -> field -> Float
paramFieldWeights :: forall field feature.
SearchRankParameters field feature -> field -> Float
paramFeatureWeights :: forall field feature.
SearchRankParameters field feature -> feature -> Float
paramFeatureFunctions :: forall field feature.
SearchRankParameters field feature -> feature -> FeatureFunction
paramAutosuggestPrefilterLimit :: forall field feature. SearchRankParameters field feature -> Int
paramAutosuggestPostfilterLimit :: forall field feature. SearchRankParameters field feature -> Int
paramK1 :: Float
paramB :: field -> Float
paramFieldWeights :: field -> Float
paramFeatureWeights :: feature -> Float
paramFeatureFunctions :: feature -> FeatureFunction
paramResultsetSoftLimit :: Int
paramResultsetHardLimit :: Int
paramAutosuggestPrefilterLimit :: Int
paramAutosuggestPostfilterLimit :: Int
..} }
      [Term]
terms =

  -- See 'query' above for explanation. Really we ought to combine them.
  let lookupTerms :: [Term]
      lookupTerms :: [Term]
lookupTerms = [ Term
term'
                    | Term
term  <- [Term]
terms
                    , let transformForField :: field -> Term
transformForField = Term -> field -> Term
transformQueryTerm Term
term
                    , Term
term' <- [Term] -> [Term]
forall a. Eq a => [a] -> [a]
nub [ field -> Term
transformForField field
field
                                   | field
field <- (field, field) -> [field]
forall a. Ix a => (a, a) -> [a]
range (field
forall a. Bounded a => a
minBound, field
forall a. Bounded a => a
maxBound) ]
                    ]

      rawresults :: [Maybe (TermId, DocIdSet)]
      rawresults :: [Maybe (TermId, DocIdSet)]
rawresults = (Term -> Maybe (TermId, DocIdSet))
-> [Term] -> [Maybe (TermId, DocIdSet)]
forall a b. (a -> b) -> [a] -> [b]
map (SearchIndex key field feature -> Term -> Maybe (TermId, DocIdSet)
forall key field feature.
SearchIndex key field feature -> Term -> Maybe (TermId, DocIdSet)
SI.lookupTerm SearchIndex key field feature
searchIndex) [Term]
lookupTerms

      termids   :: [TermId]
      docidsets :: [DocIdSet]
      ([TermId]
termids, [DocIdSet]
docidsets) = [(TermId, DocIdSet)] -> ([TermId], [DocIdSet])
forall a b. [(a, b)] -> ([a], [b])
unzip ([Maybe (TermId, DocIdSet)] -> [(TermId, DocIdSet)]
forall a. [Maybe a] -> [a]
catMaybes [Maybe (TermId, DocIdSet)]
rawresults)

      unrankedResults :: DocIdSet
      unrankedResults :: DocIdSet
unrankedResults = Int -> Int -> [DocIdSet] -> DocIdSet
pruneRelevantResults
                          Int
paramResultsetSoftLimit
                          Int
paramResultsetHardLimit
                          [DocIdSet]
docidsets

   in SearchEngine doc key field feature
-> [TermId] -> [DocId] -> [(Explanation field feature Term, key)]
forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature
-> [TermId] -> [DocId] -> [(Explanation field feature Term, key)]
rankExplainResults SearchEngine doc key field feature
se [TermId]
termids (DocIdSet -> [DocId]
DocIdSet.toList DocIdSet
unrankedResults)

rankExplainResults :: (Ix field, Bounded field, Ix feature, Bounded feature) =>
                      SearchEngine doc key field feature ->
                      [TermId] ->
                      [DocId] ->
                      [(BM25F.Explanation field feature Term, key)]
rankExplainResults :: forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature
-> [TermId] -> [DocId] -> [(Explanation field feature Term, key)]
rankExplainResults se :: SearchEngine doc key field feature
se@SearchEngine{SearchIndex key field feature
searchIndex :: forall doc key field feature.
SearchEngine doc key field feature -> SearchIndex key field feature
searchIndex :: SearchIndex key field feature
searchIndex} [TermId]
queryTerms [DocId]
docids =
    ((Explanation field feature Term, key)
 -> (Explanation field feature Term, key) -> Ordering)
-> [(Explanation field feature Term, key)]
-> [(Explanation field feature Term, key)]
forall a. (a -> a -> Ordering) -> [a] -> [a]
sortBy ((Float -> Float -> Ordering) -> Float -> Float -> Ordering
forall a b c. (a -> b -> c) -> b -> a -> c
flip Float -> Float -> Ordering
forall a. Ord a => a -> a -> Ordering
compare (Float -> Float -> Ordering)
-> ((Explanation field feature Term, key) -> Float)
-> (Explanation field feature Term, key)
-> (Explanation field feature Term, key)
-> Ordering
forall b c a. (b -> b -> c) -> (a -> b) -> a -> a -> c
`on` (Explanation field feature Term -> Float
forall field feature term. Explanation field feature term -> Float
BM25F.overallScore (Explanation field feature Term -> Float)
-> ((Explanation field feature Term, key)
    -> Explanation field feature Term)
-> (Explanation field feature Term, key)
-> Float
forall b c a. (b -> c) -> (a -> b) -> a -> c
. (Explanation field feature Term, key)
-> Explanation field feature Term
forall a b. (a, b) -> a
fst))
      [ (SearchEngine doc key field feature
-> [TermId]
-> DocTermIds field
-> DocFeatVals feature
-> Explanation field feature Term
forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature
-> [TermId]
-> DocTermIds field
-> DocFeatVals feature
-> Explanation field feature Term
explainRelevanceScore SearchEngine doc key field feature
se [TermId]
queryTerms DocTermIds field
doctermids DocFeatVals feature
docfeatvals, key
dockey)
      | DocId
docid <- [DocId]
docids
      , let (key
dockey, DocTermIds field
doctermids, DocFeatVals feature
docfeatvals) = SearchIndex key field feature
-> DocId -> (key, DocTermIds field, DocFeatVals feature)
forall key field feature.
SearchIndex key field feature
-> DocId -> (key, DocTermIds field, DocFeatVals feature)
SI.lookupDocId SearchIndex key field feature
searchIndex DocId
docid ]


explainRelevanceScore :: (Ix field, Bounded field, Ix feature, Bounded feature) =>
                         SearchEngine doc key field feature ->
                         [TermId] ->
                         DocTermIds field ->
                         DocFeatVals feature ->
                         BM25F.Explanation field feature Term
explainRelevanceScore :: forall field feature doc key.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
SearchEngine doc key field feature
-> [TermId]
-> DocTermIds field
-> DocFeatVals feature
-> Explanation field feature Term
explainRelevanceScore SearchEngine{Context TermId field feature
bm25Context :: forall doc key field feature.
SearchEngine doc key field feature -> Context TermId field feature
bm25Context :: Context TermId field feature
bm25Context, SearchIndex key field feature
searchIndex :: forall doc key field feature.
SearchEngine doc key field feature -> SearchIndex key field feature
searchIndex :: SearchIndex key field feature
searchIndex}
                      [TermId]
queryTerms DocTermIds field
doctermids DocFeatVals feature
docfeatvals =
    (TermId -> Term)
-> Explanation field feature TermId
-> Explanation field feature Term
forall a b.
(a -> b)
-> Explanation field feature a -> Explanation field feature b
forall (f :: * -> *) a b. Functor f => (a -> b) -> f a -> f b
fmap (SearchIndex key field feature -> TermId -> Term
forall key field feature.
SearchIndex key field feature -> TermId -> Term
SI.getTerm SearchIndex key field feature
searchIndex) (Context TermId field feature
-> Doc TermId field feature
-> [TermId]
-> Explanation field feature TermId
forall field feature term.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
Context term field feature
-> Doc term field feature
-> [term]
-> Explanation field feature term
BM25F.explain Context TermId field feature
bm25Context Doc TermId field feature
doc [TermId]
queryTerms)
  where
    doc :: Doc TermId field feature
doc = DocTermIds field -> DocFeatVals feature -> Doc TermId field feature
forall field feature.
(Ix field, Bounded field, Ix feature, Bounded feature) =>
DocTermIds field -> DocFeatVals feature -> Doc TermId field feature
indexDocToBM25Doc DocTermIds field
doctermids DocFeatVals feature
docfeatvals


setRankParams :: SearchRankParameters field feature ->
                 SearchEngine doc key field feature ->
                 SearchEngine doc key field feature
setRankParams :: forall field feature doc key.
SearchRankParameters field feature
-> SearchEngine doc key field feature
-> SearchEngine doc key field feature
setRankParams params :: SearchRankParameters field feature
params@SearchRankParameters{Float
Int
field -> Float
feature -> Float
feature -> FeatureFunction
paramResultsetSoftLimit :: forall field feature. SearchRankParameters field feature -> Int
paramResultsetHardLimit :: forall field feature. SearchRankParameters field feature -> Int
paramK1 :: forall field feature. SearchRankParameters field feature -> Float
paramB :: forall field feature.
SearchRankParameters field feature -> field -> Float
paramFieldWeights :: forall field feature.
SearchRankParameters field feature -> field -> Float
paramFeatureWeights :: forall field feature.
SearchRankParameters field feature -> feature -> Float
paramFeatureFunctions :: forall field feature.
SearchRankParameters field feature -> feature -> FeatureFunction
paramAutosuggestPrefilterLimit :: forall field feature. SearchRankParameters field feature -> Int
paramAutosuggestPostfilterLimit :: forall field feature. SearchRankParameters field feature -> Int
paramK1 :: Float
paramB :: field -> Float
paramFieldWeights :: field -> Float
paramFeatureWeights :: feature -> Float
paramFeatureFunctions :: feature -> FeatureFunction
paramResultsetSoftLimit :: Int
paramResultsetHardLimit :: Int
paramAutosuggestPrefilterLimit :: Int
paramAutosuggestPostfilterLimit :: Int
..} SearchEngine doc key field feature
se =
    SearchEngine doc key field feature
se {
      searchRankParams = params,
      bm25Context      = (bm25Context se) {
        BM25F.paramK1         = paramK1,
        BM25F.paramB          = paramB,
        BM25F.fieldWeight     = paramFieldWeights,
        BM25F.featureWeight   = paramFeatureWeights,
        BM25F.featureFunction = paramFeatureFunctions
      }
    }


--------------------------------
-- Results filter
--

-- | In some applications it is necessary to enforce some security or
-- visibility rule about the query results (e.g. in a typical DB-based
-- application different users can see different data items). Typically
-- it would be too expensive to build different search indexes for the
-- different contexts and so the strategy is to use one index containing
-- everything and filter for visibility in the results. This means the
-- filter condition is different for different queries (e.g. performed
-- on behalf of different users).
--
-- Filtering the results after a query is possible but not the most efficient
-- thing to do because we've had to score all the not-visible documents.
-- The better thing to do is to filter as part of the query, this way we can
-- filter before the expensive scoring.
--
-- We provide one further optimisation: bulk predicates. In some applications
-- it can be quicker to check the security\/visibility of a whole bunch of
-- results all in one go.
--
data ResultsFilter key = NoFilter
                       | FilterPredicate     (key -> Bool)
                       | FilterBulkPredicate ([key] -> [Bool])
--TODO: allow filtering & non-feature score lookup in one bulk op