FileDocCategorySizeDatePackage
StandardTokenizer.javaAPI DocApache Lucene 1.95874Mon Feb 20 09:19:46 GMT 2006org.apache.lucene.analysis.standard

StandardTokenizer

public class StandardTokenizer extends Tokenizer implements StandardTokenizerConstants
A grammar-based tokenizer constructed with JavaCC.

This should be a good tokenizer for most European-language documents:

  • Splits words at punctuation characters, removing punctuation. However, a dot that's not followed by whitespace is considered part of a token.
  • Splits words at hyphens, unless there's a number in the token, in which case the whole token is interpreted as a product number and is not split.
  • Recognizes email addresses and internet hostnames as one token.

Many applications have specific tokenizer needs. If this tokenizer does not suit your application, please consider copying this source code directory to your project and maintaining your own grammar-based tokenizer.

Fields Summary
public StandardTokenizerTokenManager
token_source
public Token
token
public Token
jj_nt
private int
jj_ntk
private int
jj_gen
private final int[]
jj_la1
private static int[]
jj_la1_0
private Vector
jj_expentries
private int[]
jj_expentry
private int
jj_kind
Constructors Summary
public StandardTokenizer(Reader reader)
Constructs a tokenizer for this Reader.

    this(new FastCharStream(reader));
    this.input = reader;
public StandardTokenizer(CharStream stream)

    token_source = new StandardTokenizerTokenManager(stream);
    token = new Token();
    jj_ntk = -1;
    jj_gen = 0;
    for (int i = 0; i < 1; i++) jj_la1[i] = -1;
public StandardTokenizer(StandardTokenizerTokenManager tm)

    token_source = tm;
    token = new Token();
    jj_ntk = -1;
    jj_gen = 0;
    for (int i = 0; i < 1; i++) jj_la1[i] = -1;
Methods Summary
public voidReInit(CharStream stream)

    token_source.ReInit(stream);
    token = new Token();
    jj_ntk = -1;
    jj_gen = 0;
    for (int i = 0; i < 1; i++) jj_la1[i] = -1;
public voidReInit(StandardTokenizerTokenManager tm)

    token_source = tm;
    token = new Token();
    jj_ntk = -1;
    jj_gen = 0;
    for (int i = 0; i < 1; i++) jj_la1[i] = -1;
public final voiddisable_tracing()
public final voidenable_tracing()
public ParseExceptiongenerateParseException()


     
    jj_expentries.removeAllElements();
    boolean[] la1tokens = new boolean[16];
    for (int i = 0; i < 16; i++) {
      la1tokens[i] = false;
    }
    if (jj_kind >= 0) {
      la1tokens[jj_kind] = true;
      jj_kind = -1;
    }
    for (int i = 0; i < 1; i++) {
      if (jj_la1[i] == jj_gen) {
        for (int j = 0; j < 32; j++) {
          if ((jj_la1_0[i] & (1<<j)) != 0) {
            la1tokens[j] = true;
          }
        }
      }
    }
    for (int i = 0; i < 16; i++) {
      if (la1tokens[i]) {
        jj_expentry = new int[1];
        jj_expentry[0] = i;
        jj_expentries.addElement(jj_expentry);
      }
    }
    int[][] exptokseq = new int[jj_expentries.size()][];
    for (int i = 0; i < jj_expentries.size(); i++) {
      exptokseq[i] = (int[])jj_expentries.elementAt(i);
    }
    return new ParseException(token, exptokseq, tokenImage);
public final TokengetNextToken()

    if (token.next != null) token = token.next;
    else token = token.next = token_source.getNextToken();
    jj_ntk = -1;
    jj_gen++;
    return token;
public final TokengetToken(int index)

    Token t = token;
    for (int i = 0; i < index; i++) {
      if (t.next != null) t = t.next;
      else t = t.next = token_source.getNextToken();
    }
    return t;
private final Tokenjj_consume_token(int kind)

    Token oldToken;
    if ((oldToken = token).next != null) token = token.next;
    else token = token.next = token_source.getNextToken();
    jj_ntk = -1;
    if (token.kind == kind) {
      jj_gen++;
      return token;
    }
    token = oldToken;
    jj_kind = kind;
    throw generateParseException();
private static voidjj_la1_0()

   
      jj_la1_0();
   
      jj_la1_0 = new int[] {0x10ff,};
private final intjj_ntk()

    if ((jj_nt=token.next) == null)
      return (jj_ntk = (token.next=token_source.getNextToken()).kind);
    else
      return (jj_ntk = jj_nt.kind);
public final org.apache.lucene.analysis.Tokennext()
Returns the next token in the stream, or null at EOS.

The returned token's type is set to an element of {@link StandardTokenizerConstants#tokenImage}.

  Token token = null;
    switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
    case ALPHANUM:
      token = jj_consume_token(ALPHANUM);
      break;
    case APOSTROPHE:
      token = jj_consume_token(APOSTROPHE);
      break;
    case ACRONYM:
      token = jj_consume_token(ACRONYM);
      break;
    case COMPANY:
      token = jj_consume_token(COMPANY);
      break;
    case EMAIL:
      token = jj_consume_token(EMAIL);
      break;
    case HOST:
      token = jj_consume_token(HOST);
      break;
    case NUM:
      token = jj_consume_token(NUM);
      break;
    case CJ:
      token = jj_consume_token(CJ);
      break;
    case 0:
      token = jj_consume_token(0);
      break;
    default:
      jj_la1[0] = jj_gen;
      jj_consume_token(-1);
      throw new ParseException();
    }
      if (token.kind == EOF) {
        {if (true) return null;}
      } else {
        {if (true) return
          new org.apache.lucene.analysis.Token(token.image,
                                        token.beginColumn,token.endColumn,
                                        tokenImage[token.kind]);}
      }
    throw new Error("Missing return statement in function");