SKaiNET
Toggle table of contents
0.20.0-SNAPSHOT
common
Target filter
common
Switch theme
Search in API
Skip to content
SKaiNET
skainet-io-core
/
sk.ainet.io.tokenizer
/
SentencePieceTokenizer
/
SentencePieceTokenizer
Sentence
Piece
Tokenizer
constructor
(
tokens
:
List
<
String
>
,
scores
:
List
<
Float
>
,
unknownTokenId
:
Int
?
=
null
,
bosTokenId
:
Int
?
=
null
,
eosTokenId
:
Int
?
=
null
,
addSpacePrefix
:
Boolean
=
true
)
(
source
)