String s = "a天b";
ChineseTokenizer tokenizer = new ChineseTokenizer(new StringReader(s));
Token token;
int correctStartOffset = 0;
int correctEndOffset = 1;
while ((token = tokenizer.next()) != null)
{
assertEquals(correctStartOffset, token.startOffset());
assertEquals(correctEndOffset, token.endOffset());
correctStartOffset++;
correctEndOffset++;
}
