# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0.  If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright 1997 - July 2008 CWI, August 2008 - 2016 MonetDB B.V.

module tokenizer
comment "The tokenizer provides fast access to a large collection of strings
based on a vertical fragmented representation.";

command open(name:str):void
address TKNZRopen
comment "open the named tokenizer store, a new one is created if the specified name does not exist";

command close():void
address TKNZRclose
comment "close the current tokenizer store";

pattern take(i:oid):str
address TKNZRtakeOid
comment "reconstruct and returns the i-th string";

pattern locate(s:str):oid
address TKNZRlocate
comment "if the given string is in the store returns its oid, otherwise oid_nil";

command append(u:str):oid
address TKNZRappend
comment "tokenize a new string and append it to the tokenizer (duplicate elimination is performed)";

command depositFile(fnme:str):void
address TKNZRdepositFile
comment "batch insertion from a file of strings to tokenize, each string is separated by a new line";

command getLevel(i:int):bat[:str]
address TKNZRgetLevel
comment "administrative function that returns the bat on level i";

command getIndex():bat[:oid]
address TKNZRgetIndex
comment "administrative function that returns the INDEX bat";

command getCount():bat[:lng]
address TKNZRgetCount
comment "debugging function that returns the size of the bats at each level";

command getCardinality():bat[:lng]
address TKNZRgetCardinality
comment "debugging function that returns the unique tokens at each level";
