FileDocCategorySizeDatePackage
Token.javaAPI DocJava SE 5 API60397Fri Aug 26 14:55:50 BST 2005com.sun.org.apache.xerces.internal.impl.xpath.regex

Token

public class Token extends Object implements Serializable
This class represents a node in parse tree.
version
$Id: Token.java,v 1.7 2003/02/25 14:43:13 sandygao Exp $

Fields Summary
static final boolean
COUNTTOKENS
static int
tokens
static final int
CHAR
static final int
DOT
static final int
CONCAT
static final int
UNION
static final int
CLOSURE
static final int
RANGE
static final int
NRANGE
static final int
PAREN
static final int
EMPTY
static final int
ANCHOR
static final int
NONGREEDYCLOSURE
static final int
STRING
static final int
BACKREFERENCE
static final int
LOOKAHEAD
static final int
NEGATIVELOOKAHEAD
static final int
LOOKBEHIND
static final int
NEGATIVELOOKBEHIND
static final int
INDEPENDENT
static final int
MODIFIERGROUP
static final int
CONDITION
static final int
UTF16_MAX
int
type
static Token
token_dot
static Token
token_0to9
static Token
token_wordchars
static Token
token_not_0to9
static Token
token_not_wordchars
static Token
token_spaces
static Token
token_not_spaces
static Token
token_empty
static Token
token_linebeginning
static Token
token_linebeginning2
static Token
token_lineend
static Token
token_stringbeginning
static Token
token_stringend
static Token
token_stringend2
static Token
token_wordedge
static Token
token_not_wordedge
static Token
token_wordbeginning
static Token
token_wordend
static final int
FC_CONTINUE
static final int
FC_TERMINAL
static final int
FC_ANY
private static final Hashtable
categories
private static final Hashtable
categories2
private static final String[]
categoryNames
static final int
CHAR_INIT_QUOTE
static final int
CHAR_FINAL_QUOTE
static final int
CHAR_LETTER
static final int
CHAR_MARK
static final int
CHAR_NUMBER
static final int
CHAR_SEPARATOR
static final int
CHAR_OTHER
static final int
CHAR_PUNCTUATION
static final int
CHAR_SYMBOL
private static final String[]
blockNames
static final String
blockRanges
static final int[]
nonBMPBlockRanges
private static final int
NONBMP_BLOCK_START
static Hashtable
nonxs
static final String
viramaString
private static Token
token_grapheme
private static Token
token_ccs
Combing Character Sequence in Perl 5.6.
Constructors Summary
protected Token(int type)

        this.type = type;
    
Methods Summary
voidaddChild(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok)

        throw new RuntimeException("Not supported.");
    
protected voidaddRange(int start, int end)

        throw new RuntimeException("Not supported.");
    
final intanalyzeFirstCharacter(com.sun.org.apache.xerces.internal.impl.xpath.regex.RangeToken result, int options)

        switch (this.type) {
          case CONCAT:
            int ret = FC_CONTINUE;
            for (int i = 0;  i < this.size();  i ++)
                if ((ret = this.getChild(i).analyzeFirstCharacter(result, options)) != FC_CONTINUE)
                    break;
            return ret;

          case UNION:
            if (this.size() == 0)
                return FC_CONTINUE;
            /*
             *  a|b|c -> FC_TERMINAL
             *  a|.|c -> FC_ANY
             *  a|b|  -> FC_CONTINUE
             */
            int ret2 = FC_CONTINUE;
            boolean hasEmpty = false;
            for (int i = 0;  i < this.size();  i ++) {
                ret2 = this.getChild(i).analyzeFirstCharacter(result, options);
                if (ret2 == FC_ANY)
                    break;
                else if (ret2 == FC_CONTINUE)
                    hasEmpty = true;
            }
            return hasEmpty ? FC_CONTINUE : ret2;

          case CONDITION:
            int ret3 = this.getChild(0).analyzeFirstCharacter(result, options);
            if (this.size() == 1)  return FC_CONTINUE;
            if (ret3 == FC_ANY)  return ret3;
            int ret4 = this.getChild(1).analyzeFirstCharacter(result, options);
            if (ret4 == FC_ANY)  return ret4;
            return ret3 == FC_CONTINUE || ret4 == FC_CONTINUE ? FC_CONTINUE : FC_TERMINAL;

          case CLOSURE:
          case NONGREEDYCLOSURE:
            this.getChild(0).analyzeFirstCharacter(result, options);
            return FC_CONTINUE;

          case EMPTY:
          case ANCHOR:
            return FC_CONTINUE;

          case CHAR:
            int ch = this.getChar();
            result.addRange(ch, ch);
            if (ch < 0x10000 && isSet(options, RegularExpression.IGNORE_CASE)) {
                ch = Character.toUpperCase((char)ch);
                result.addRange(ch, ch);
                ch = Character.toLowerCase((char)ch);
                result.addRange(ch, ch);
            }
            return FC_TERMINAL;

          case DOT:                             // ****
            if (isSet(options, RegularExpression.SINGLE_LINE)) {
                return FC_CONTINUE;             // **** We can not optimize.
            } else {
                return FC_CONTINUE;
                /*
                result.addRange(0, RegularExpression.LINE_FEED-1);
                result.addRange(RegularExpression.LINE_FEED+1, RegularExpression.CARRIAGE_RETURN-1);
                result.addRange(RegularExpression.CARRIAGE_RETURN+1,
                                RegularExpression.LINE_SEPARATOR-1);
                result.addRange(RegularExpression.PARAGRAPH_SEPARATOR+1, UTF16_MAX);
                return 1;
                */
            }

          case RANGE:
            if (isSet(options, RegularExpression.IGNORE_CASE)) {
                result.mergeRanges(((RangeToken)this).getCaseInsensitiveToken());
            } else {
                result.mergeRanges(this);
            }
            return FC_TERMINAL;

          case NRANGE:                          // ****
            if (isSet(options, RegularExpression.IGNORE_CASE)) {
                result.mergeRanges(Token.complementRanges(((RangeToken)this).getCaseInsensitiveToken()));
            } else {
                result.mergeRanges(Token.complementRanges(this));
            }
            return FC_TERMINAL;

          case INDEPENDENT:
          case PAREN:
            return this.getChild(0).analyzeFirstCharacter(result, options);

          case MODIFIERGROUP:
            options |= ((ModifierToken)this).getOptions();
            options &= ~((ModifierToken)this).getOptionsMask();
            return this.getChild(0).analyzeFirstCharacter(result, options);

          case BACKREFERENCE:
            result.addRange(0, UTF16_MAX);  // **** We can not optimize.
            return FC_ANY;

          case STRING:
            int cha = this.getString().charAt(0);
            int ch2;
            if (REUtil.isHighSurrogate(cha)
                && this.getString().length() >= 2
                && REUtil.isLowSurrogate((ch2 = this.getString().charAt(1))))
                cha = REUtil.composeFromSurrogates(cha, ch2);
            result.addRange(cha, cha);
            if (cha < 0x10000 && isSet(options, RegularExpression.IGNORE_CASE)) {
                cha = Character.toUpperCase((char)cha);
                result.addRange(cha, cha);
                cha = Character.toLowerCase((char)cha);
                result.addRange(cha, cha);
            }
            return FC_TERMINAL;

          case LOOKAHEAD:
          case NEGATIVELOOKAHEAD:
          case LOOKBEHIND:
          case NEGATIVELOOKBEHIND:
            return FC_CONTINUE;

          default:
            throw new RuntimeException("Token#analyzeHeadCharacter(): Invalid Type: "+this.type);
        }
    
protected voidcompactRanges()

        throw new RuntimeException("Not supported.");
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.TokencomplementRanges(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok)

        return RangeToken.complementRanges(tok);
    
private static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$CharTokencreateAnchor(int ch)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.CharToken(Token.ANCHOR, ch);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$StringTokencreateBackReference(int refno)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.StringToken(Token.BACKREFERENCE, null, refno);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$CharTokencreateChar(int ch)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.CharToken(Token.CHAR, ch);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$ClosureTokencreateClosure(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.ClosureToken(Token.CLOSURE, tok);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$ConcatTokencreateConcat(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok1, com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok2)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.ConcatToken(tok1, tok2);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$UnionTokencreateConcat()

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.UnionToken(Token.CONCAT); // *** It is not a bug.
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$ConditionTokencreateCondition(int refno, com.sun.org.apache.xerces.internal.impl.xpath.regex.Token condition, com.sun.org.apache.xerces.internal.impl.xpath.regex.Token yespat, com.sun.org.apache.xerces.internal.impl.xpath.regex.Token nopat)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.ConditionToken(refno, condition, yespat, nopat);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.TokencreateEmpty()

        return Token.token_empty;
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$ParenTokencreateLook(int type, com.sun.org.apache.xerces.internal.impl.xpath.regex.Token child)

     
        Token.token_empty = new Token(Token.EMPTY);

        Token.token_linebeginning = Token.createAnchor('^");
        Token.token_linebeginning2 = Token.createAnchor('@");
        Token.token_lineend = Token.createAnchor('$");
        Token.token_stringbeginning = Token.createAnchor('A");
        Token.token_stringend = Token.createAnchor('z");
        Token.token_stringend2 = Token.createAnchor('Z");
        Token.token_wordedge = Token.createAnchor('b");
        Token.token_not_wordedge = Token.createAnchor('B");
        Token.token_wordbeginning = Token.createAnchor('<");
        Token.token_wordend = Token.createAnchor('>");

        Token.token_dot = new Token(Token.DOT);

        Token.token_0to9 = Token.createRange();
        Token.token_0to9.addRange('0", '9");
        Token.token_wordchars = Token.createRange();
        Token.token_wordchars.addRange('0", '9");
        Token.token_wordchars.addRange('A", 'Z");
        Token.token_wordchars.addRange('_", '_");
        Token.token_wordchars.addRange('a", 'z");
        Token.token_spaces = Token.createRange();
        Token.token_spaces.addRange('\t", '\t");
        Token.token_spaces.addRange('\n", '\n");
        Token.token_spaces.addRange('\f", '\f");
        Token.token_spaces.addRange('\r", '\r");
        Token.token_spaces.addRange(' ", ' ");

        Token.token_not_0to9 = Token.complementRanges(Token.token_0to9);
        Token.token_not_wordchars = Token.complementRanges(Token.token_wordchars);
        Token.token_not_spaces = Token.complementRanges(Token.token_spaces);
    
        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.ParenToken(type, child, 0);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$ModifierTokencreateModifierGroup(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token child, int add, int mask)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.ModifierToken(child, add, mask);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$ClosureTokencreateNGClosure(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.ClosureToken(Token.NONGREEDYCLOSURE, tok);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.RangeTokencreateNRange()

        if (COUNTTOKENS)  Token.tokens ++;
        return new RangeToken(Token.NRANGE);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$ParenTokencreateParen(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token child, int pnumber)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.ParenToken(Token.PAREN, child, pnumber);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.RangeTokencreateRange()

        if (COUNTTOKENS)  Token.tokens ++;
        return new RangeToken(Token.RANGE);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$StringTokencreateString(java.lang.String str)

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.StringToken(Token.STRING, str, 0);
    
static com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$UnionTokencreateUnion()

        if (COUNTTOKENS)  Token.tokens ++;
        return new Token.UnionToken(Token.UNION);
    
final voidfindFixedString(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token$FixedStringContainer container, int options)

        switch (this.type) {
          case CONCAT:
            Token prevToken = null;
            int prevOptions = 0;
            for (int i = 0;  i < this.size();  i ++) {
                this.getChild(i).findFixedString(container, options);
                if (prevToken == null || prevToken.isShorterThan(container.token)) {
                    prevToken = container.token;
                    prevOptions = container.options;
                }
            }
            container.token = prevToken;
            container.options = prevOptions;
            return;

          case UNION:
          case CLOSURE:
          case NONGREEDYCLOSURE:
          case EMPTY:
          case ANCHOR:
          case RANGE:
          case DOT:
          case NRANGE:
          case BACKREFERENCE:
          case LOOKAHEAD:
          case NEGATIVELOOKAHEAD:
          case LOOKBEHIND:
          case NEGATIVELOOKBEHIND:
          case CONDITION:
            container.token = null;
            return;

          case CHAR:                            // Ignore CHAR tokens.
            container.token = null;             // **
            return;                             // **

          case STRING:
            container.token = this;
            container.options = options;
            return;

          case INDEPENDENT:
          case PAREN:
            this.getChild(0).findFixedString(container, options);
            return;

          case MODIFIERGROUP:
            options |= ((ModifierToken)this).getOptions();
            options &= ~((ModifierToken)this).getOptionsMask();
            this.getChild(0).findFixedString(container, options);
            return;

          default:
            throw new RuntimeException("Token#findFixedString(): Invalid Type: "+this.type);
        }
    
intgetChar()

        return -1;
    
com.sun.org.apache.xerces.internal.impl.xpath.regex.TokengetChild(int index)

        return null;
    
static synchronized com.sun.org.apache.xerces.internal.impl.xpath.regex.TokengetCombiningCharacterSequence()

        
        if (Token.token_ccs != null)
            return Token.token_ccs;

        Token foo = Token.createClosure(Token.getRange("M", true)); // \pM*
        foo = Token.createConcat(Token.getRange("M", false), foo); // \PM + \pM*
        Token.token_ccs = foo;
        return Token.token_ccs;
    
static synchronized com.sun.org.apache.xerces.internal.impl.xpath.regex.TokengetGraphemePattern()

        
        if (Token.token_grapheme != null)
            return Token.token_grapheme;

        Token base_char = Token.createRange();  // [{ASSIGNED}]-[{M},{C}]
        base_char.mergeRanges(Token.getRange("ASSIGNED", true));
        base_char.subtractRanges(Token.getRange("M", true));
        base_char.subtractRanges(Token.getRange("C", true));

        Token virama = Token.createRange();
        for (int i = 0;  i < Token.viramaString.length();  i ++) {
            int ch = viramaString.charAt(i);
            virama.addRange(i, i);
        }

        Token combiner_wo_virama = Token.createRange();
        combiner_wo_virama.mergeRanges(Token.getRange("M", true));
        combiner_wo_virama.addRange(0x1160, 0x11ff); // hangul_medial and hangul_final
        combiner_wo_virama.addRange(0xff9e, 0xff9f); // extras

        Token left = Token.createUnion();       // base_char?
        left.addChild(base_char);
        left.addChild(Token.token_empty);

        Token foo = Token.createUnion();
        foo.addChild(Token.createConcat(virama, Token.getRange("L", true)));
        foo.addChild(combiner_wo_virama);

        foo = Token.createClosure(foo);

        foo = Token.createConcat(left, foo);

        Token.token_grapheme = foo;
        return Token.token_grapheme;
    
intgetMax()

                              // for CLOSURE
        return -1;
    
final intgetMaxLength()

        switch (this.type) {
          case CONCAT:
            int sum = 0;
            for (int i = 0;  i < this.size();  i ++) {
                int d = this.getChild(i).getMaxLength();
                if (d < 0)  return -1;
                sum += d;
            }
            return sum;

          case CONDITION:
          case UNION:
            if (this.size() == 0)
                return 0;
            int ret = this.getChild(0).getMaxLength();
            for (int i = 1;  ret >= 0 && i < this.size();  i ++) {
                int max = this.getChild(i).getMaxLength();
                if (max < 0) {                  // infinity
                    ret = -1;
                    break;
                }
                if (max > ret)  ret = max;
            }
            return ret;

          case CLOSURE:
          case NONGREEDYCLOSURE:
            if (this.getMax() >= 0)
                                                // When this.child.getMaxLength() < 0,
                                                // this returns minus value
                return this.getMax() * this.getChild(0).getMaxLength();
            return -1;

          case EMPTY:
          case ANCHOR:
            return 0;

          case CHAR:
            return 1;
          case DOT:
          case RANGE:
          case NRANGE:
            return 2;

          case INDEPENDENT:
          case PAREN:
          case MODIFIERGROUP:
            return this.getChild(0).getMaxLength();

          case BACKREFERENCE:
            return -1;                          // ******

          case STRING:
            return this.getString().length();

          case LOOKAHEAD:
          case NEGATIVELOOKAHEAD:
          case LOOKBEHIND:
          case NEGATIVELOOKBEHIND:
            return 0;                           // ***** Really?

          default:
            throw new RuntimeException("Token#getMaxLength(): Invalid Type: "+this.type);
        }
    
intgetMin()

                              // for CLOSURE
        return -1;
    
final intgetMinLength()
How many characters are needed?

        switch (this.type) {
          case CONCAT:
            int sum = 0;
            for (int i = 0;  i < this.size();  i ++)
                sum += this.getChild(i).getMinLength();
            return sum;

          case CONDITION:
          case UNION:
            if (this.size() == 0)
                return 0;
            int ret = this.getChild(0).getMinLength();
            for (int i = 1;  i < this.size();  i ++) {
                int min = this.getChild(i).getMinLength();
                if (min < ret)  ret = min;
            }
            return ret;

          case CLOSURE:
          case NONGREEDYCLOSURE:
            if (this.getMin() >= 0)
                return this.getMin() * this.getChild(0).getMinLength();
            return 0;

          case EMPTY:
          case ANCHOR:
            return 0;

          case DOT:
          case CHAR:
          case RANGE:
          case NRANGE:
            return 1;

          case INDEPENDENT:
          case PAREN:
          case MODIFIERGROUP:
            return this.getChild(0).getMinLength();

          case BACKREFERENCE:
            return 0;                           // *******

          case STRING:
            return this.getString().length();

          case LOOKAHEAD:
          case NEGATIVELOOKAHEAD:
          case LOOKBEHIND:
          case NEGATIVELOOKBEHIND:
            return 0;                           // ***** Really?

          default:
            throw new RuntimeException("Token#getMinLength(): Invalid Type: "+this.type);
        }
    
intgetParenNumber()

        return 0;
    
protected static com.sun.org.apache.xerces.internal.impl.xpath.regex.RangeTokengetRange(java.lang.String name, boolean positive)


           
        if (Token.categories.size() == 0) {
            synchronized (Token.categories) {
                Token[] ranges = new Token[Token.categoryNames.length];
                for (int i = 0;  i < ranges.length;  i ++) {
                    ranges[i] = Token.createRange();
                }
                int type;
                for (int i = 0;  i < 0x10000;  i ++) {
                    type = Character.getType((char)i);
                    if (type == Character.START_PUNCTUATION || 
                        type == Character.END_PUNCTUATION) {
                        //build table of Pi values
                        if (i == 0x00AB || i == 0x2018 || i == 0x201B || i == 0x201C ||
                            i == 0x201F || i == 0x2039) {
                            type = CHAR_INIT_QUOTE;
                        }
                        //build table of Pf values
                        if (i == 0x00BB || i == 0x2019 || i == 0x201D || i == 0x203A ) {
                            type = CHAR_FINAL_QUOTE;
                        }
                    }
                    ranges[type].addRange(i, i);
                    switch (type) {
                      case Character.UPPERCASE_LETTER:
                      case Character.LOWERCASE_LETTER:
                      case Character.TITLECASE_LETTER:
                      case Character.MODIFIER_LETTER:
                      case Character.OTHER_LETTER:
                        type = CHAR_LETTER;
                        break;
                      case Character.NON_SPACING_MARK:
                      case Character.COMBINING_SPACING_MARK:
                      case Character.ENCLOSING_MARK:
                        type = CHAR_MARK;
                        break;
                      case Character.DECIMAL_DIGIT_NUMBER:
                      case Character.LETTER_NUMBER:
                      case Character.OTHER_NUMBER:
                        type = CHAR_NUMBER;
                        break;
                      case Character.SPACE_SEPARATOR:
                      case Character.LINE_SEPARATOR:
                      case Character.PARAGRAPH_SEPARATOR:
                        type = CHAR_SEPARATOR;
                        break;
                      case Character.CONTROL:
                      case Character.FORMAT:
                      case Character.SURROGATE:
                      case Character.PRIVATE_USE:
                      case Character.UNASSIGNED:
                        type = CHAR_OTHER;
                        break;
                      case Character.CONNECTOR_PUNCTUATION:
                      case Character.DASH_PUNCTUATION:
                      case Character.START_PUNCTUATION:
                      case Character.END_PUNCTUATION:
                      case CHAR_INIT_QUOTE:
                      case CHAR_FINAL_QUOTE:
                      case Character.OTHER_PUNCTUATION:
                        type = CHAR_PUNCTUATION;
                        break;
                      case Character.MATH_SYMBOL:
                      case Character.CURRENCY_SYMBOL:
                      case Character.MODIFIER_SYMBOL:
                      case Character.OTHER_SYMBOL:
                        type = CHAR_SYMBOL;
                        break;
                      default:
                        throw new RuntimeException("com.sun.org.apache.xerces.internal.utils.regex.Token#getRange(): Unknown Unicode category: "+type);
                    }
                    ranges[type].addRange(i, i);
                } // for all characters
                ranges[Character.UNASSIGNED].addRange(0x10000, Token.UTF16_MAX);

                for (int i = 0;  i < ranges.length;  i ++) {
                    if (Token.categoryNames[i] != null) {
                        if (i == Character.UNASSIGNED) { // Unassigned
                            ranges[i].addRange(0x10000, Token.UTF16_MAX);
                        }
                        Token.categories.put(Token.categoryNames[i], ranges[i]);
                        Token.categories2.put(Token.categoryNames[i],
                                              Token.complementRanges(ranges[i]));
                    }
                }
                //REVISIT: do we really need to support block names as in Unicode 3.1
                //         or we can just create all the names in IsBLOCKNAME format (XML Schema REC)?
                //
                StringBuffer buffer = new StringBuffer(50);
                for (int i = 0;  i < Token.blockNames.length;  i ++) {
                    Token r1 = Token.createRange();
                    int location;
                    if (i < NONBMP_BLOCK_START) {
                        location = i*2;
                        int rstart = Token.blockRanges.charAt(location);
                        int rend = Token.blockRanges.charAt(location+1);
                        //DEBUGING
                        //System.out.println(n+" " +Integer.toHexString(rstart)
                        //                     +"-"+ Integer.toHexString(rend));
                        r1.addRange(rstart, rend);
                    } else {
                        location = (i - NONBMP_BLOCK_START) * 2;
                        r1.addRange(Token.nonBMPBlockRanges[location],
                                    Token.nonBMPBlockRanges[location + 1]);
                    }
                    String n = Token.blockNames[i];
                    if (n.equals("Specials"))
                        r1.addRange(0xfff0, 0xfffd);
                    if (n.equals("Private Use")) {
                        r1.addRange(0xF0000,0xFFFFD);
                        r1.addRange(0x100000,0x10FFFD);
                    }
                    Token.categories.put(n, r1);
                    Token.categories2.put(n, Token.complementRanges(r1));
                    buffer.setLength(0);
                    buffer.append("Is");
                    if (n.indexOf(' ") >= 0) {
                        for (int ci = 0;  ci < n.length();  ci ++)
                            if (n.charAt(ci) != ' ")  buffer.append((char)n.charAt(ci));
                    }
                    else {
                        buffer.append(n);
                    }
                    Token.setAlias(buffer.toString(), n, true);
                }

                // TR#18 1.2
                Token.setAlias("ASSIGNED", "Cn", false);
                Token.setAlias("UNASSIGNED", "Cn", true);
                Token all = Token.createRange();
                all.addRange(0, Token.UTF16_MAX);
                Token.categories.put("ALL", all);
                Token.categories2.put("ALL", Token.complementRanges(all));
                Token.registerNonXS("ASSIGNED");
                Token.registerNonXS("UNASSIGNED");
                Token.registerNonXS("ALL");

                Token isalpha = Token.createRange();
                isalpha.mergeRanges(ranges[Character.UPPERCASE_LETTER]); // Lu
                isalpha.mergeRanges(ranges[Character.LOWERCASE_LETTER]); // Ll
                isalpha.mergeRanges(ranges[Character.OTHER_LETTER]); // Lo
                Token.categories.put("IsAlpha", isalpha);
                Token.categories2.put("IsAlpha", Token.complementRanges(isalpha));
                Token.registerNonXS("IsAlpha");

                Token isalnum = Token.createRange();
                isalnum.mergeRanges(isalpha);   // Lu Ll Lo
                isalnum.mergeRanges(ranges[Character.DECIMAL_DIGIT_NUMBER]); // Nd
                Token.categories.put("IsAlnum", isalnum);
                Token.categories2.put("IsAlnum", Token.complementRanges(isalnum));
                Token.registerNonXS("IsAlnum");

                Token isspace = Token.createRange();
                isspace.mergeRanges(Token.token_spaces);
                isspace.mergeRanges(ranges[CHAR_SEPARATOR]); // Z
                Token.categories.put("IsSpace", isspace);
                Token.categories2.put("IsSpace", Token.complementRanges(isspace));
                Token.registerNonXS("IsSpace");

                Token isword = Token.createRange();
                isword.mergeRanges(isalnum);     // Lu Ll Lo Nd
                isword.addRange('_", '_");
                Token.categories.put("IsWord", isword);
                Token.categories2.put("IsWord", Token.complementRanges(isword));
                Token.registerNonXS("IsWord");

                Token isascii = Token.createRange();
                isascii.addRange(0, 127);
                Token.categories.put("IsASCII", isascii);
                Token.categories2.put("IsASCII", Token.complementRanges(isascii));
                Token.registerNonXS("IsASCII");

                Token isnotgraph = Token.createRange();
                isnotgraph.mergeRanges(ranges[CHAR_OTHER]);
                isnotgraph.addRange(' ", ' ");
                Token.categories.put("IsGraph", Token.complementRanges(isnotgraph));
                Token.categories2.put("IsGraph", isnotgraph);
                Token.registerNonXS("IsGraph");

                Token isxdigit = Token.createRange();
                isxdigit.addRange('0", '9");
                isxdigit.addRange('A", 'F");
                isxdigit.addRange('a", 'f");
                Token.categories.put("IsXDigit", Token.complementRanges(isxdigit));
                Token.categories2.put("IsXDigit", isxdigit);
                Token.registerNonXS("IsXDigit");

                Token.setAlias("IsDigit", "Nd", true);
                Token.setAlias("IsUpper", "Lu", true);
                Token.setAlias("IsLower", "Ll", true);
                Token.setAlias("IsCntrl", "C", true);
                Token.setAlias("IsPrint", "C", false);
                Token.setAlias("IsPunct", "P", true);
                Token.registerNonXS("IsDigit");
                Token.registerNonXS("IsUpper");
                Token.registerNonXS("IsLower");
                Token.registerNonXS("IsCntrl");
                Token.registerNonXS("IsPrint");
                Token.registerNonXS("IsPunct");

                Token.setAlias("alpha", "IsAlpha", true);
                Token.setAlias("alnum", "IsAlnum", true);
                Token.setAlias("ascii", "IsASCII", true);
                Token.setAlias("cntrl", "IsCntrl", true);
                Token.setAlias("digit", "IsDigit", true);
                Token.setAlias("graph", "IsGraph", true);
                Token.setAlias("lower", "IsLower", true);
                Token.setAlias("print", "IsPrint", true);
                Token.setAlias("punct", "IsPunct", true);
                Token.setAlias("space", "IsSpace", true);
                Token.setAlias("upper", "IsUpper", true);
                Token.setAlias("word", "IsWord", true); // Perl extension
                Token.setAlias("xdigit", "IsXDigit", true);
                Token.registerNonXS("alpha");
                Token.registerNonXS("alnum");
                Token.registerNonXS("ascii");
                Token.registerNonXS("cntrl");
                Token.registerNonXS("digit");
                Token.registerNonXS("graph");
                Token.registerNonXS("lower");
                Token.registerNonXS("print");
                Token.registerNonXS("punct");
                Token.registerNonXS("space");
                Token.registerNonXS("upper");
                Token.registerNonXS("word");
                Token.registerNonXS("xdigit");
            } // synchronized
        } // if null
        RangeToken tok = positive ? (RangeToken)Token.categories.get(name)
            : (RangeToken)Token.categories2.get(name);
        //if (tok == null) System.out.println(name);
        return tok;
    
protected static com.sun.org.apache.xerces.internal.impl.xpath.regex.RangeTokengetRange(java.lang.String name, boolean positive, boolean xs)

        RangeToken range = Token.getRange(name, positive);
        if (xs && range != null && Token.isRegisterNonXS(name))
            range = null;
        return range;
    
intgetReferenceNumber()

                  // for STRING
        return 0;
    
java.lang.StringgetString()

                        // for STRING
        return null;
    
protected voidintersectRanges(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok)

        throw new RuntimeException("Not supported.");
    
protected static booleanisRegisterNonXS(java.lang.String name)

        if (Token.nonxs == null)
            return false;
        //DEBUG
        //System.err.println("isRegisterNonXS: "+name);
        return Token.nonxs.containsKey(name);
    
private static final booleanisSet(int options, int flag)

            
        return (options & flag) == flag;
    
private final booleanisShorterThan(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok)

        if (tok == null)  return false;
        /*
        int mylength;
        if (this.type == STRING)  mylength = this.getString().length();
        else if (this.type == CHAR)  mylength = this.getChar() >= 0x10000 ? 2 : 1;
        else throw new RuntimeException("Internal Error: Illegal type: "+this.type);
        int otherlength;
        if (tok.type == STRING)  otherlength = tok.getString().length();
        else if (tok.type == CHAR)  otherlength = tok.getChar() >= 0x10000 ? 2 : 1;
        else throw new RuntimeException("Internal Error: Illegal type: "+tok.type);
        */
        int mylength;
        if (this.type == STRING)  mylength = this.getString().length();
        else throw new RuntimeException("Internal Error: Illegal type: "+this.type);
        int otherlength;
        if (tok.type == STRING)  otherlength = tok.getString().length();
        else throw new RuntimeException("Internal Error: Illegal type: "+tok.type);
        return mylength < otherlength;
    
booleanmatch(int ch)

        throw new RuntimeException("NFAArrow#match(): Internal error: "+this.type);
    
protected voidmergeRanges(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok)

        throw new RuntimeException("Not supported.");
    
protected static voidregisterNonXS(java.lang.String name)
This method is called by only getRange(). So this method need not MT-safe.

                      
         
        if (Token.nonxs == null)
            Token.nonxs = new Hashtable();
        Token.nonxs.put(name, name);
    
private static voidsetAlias(java.lang.String newName, java.lang.String name, boolean positive)

        Token t1 = (Token)Token.categories.get(name);
        Token t2 = (Token)Token.categories2.get(name);
        if (positive) {
            Token.categories.put(newName, t1);
            Token.categories2.put(newName, t2);
        } else {
            Token.categories2.put(newName, t1);
            Token.categories.put(newName, t2);
        }
    
voidsetMax(int max)

                      // for CLOSURE
    
voidsetMin(int min)

                      // for CLOSURE
    
intsize()
A number of children.

        return 0;
    
protected voidsortRanges()

        throw new RuntimeException("Not supported.");
    
protected voidsubtractRanges(com.sun.org.apache.xerces.internal.impl.xpath.regex.Token tok)

        throw new RuntimeException("Not supported.");
    
public java.lang.StringtoString()

        return this.toString(0);
    
public java.lang.StringtoString(int options)

        return this.type == Token.DOT ? "." : "";