FileDocCategorySizeDatePackage
TestChineseTokenizer.javaAPI DocApache Lucene 1.9790Mon Feb 20 09:18:40 GMT 2006org.apache.lucene.analysis.cn

TestChineseTokenizer

public class TestChineseTokenizer extends TestCase
author
rayt

Fields Summary
Constructors Summary
Methods Summary
public voidtestOtherLetterOffset()

        String s = "a天b";
        ChineseTokenizer tokenizer = new ChineseTokenizer(new StringReader(s));
        Token token;

        int correctStartOffset = 0;
        int correctEndOffset = 1;
        while ((token = tokenizer.next()) != null)
        {
            assertEquals(correctStartOffset, token.startOffset());
            assertEquals(correctEndOffset, token.endOffset());
            correctStartOffset++;
            correctEndOffset++;
        }