if (thaiToken != null) {
String text = thaiToken.termText();
int start = breaker.current();
int end = breaker.next();
if (end != BreakIterator.DONE) {
return new Token(text.substring(start, end),
thaiToken.startOffset()+start, thaiToken.startOffset()+end, thaiToken.type());
}
thaiToken = null;
}
Token tk = input.next();
if (tk == null) {
return null;
}
String text = tk.termText();
if (UnicodeBlock.of(text.charAt(0)) != UnicodeBlock.THAI) {
return new Token(text.toLowerCase(), tk.startOffset(), tk.endOffset(), tk.type());
}
thaiToken = tk;
breaker.setText(text);
int end = breaker.next();
if (end != BreakIterator.DONE) {
return new Token(text.substring(0, end),
thaiToken.startOffset(), thaiToken.startOffset()+end, thaiToken.type());
}
return null;
