File	Doc	Category	Size	Date	Package
RuleBasedBreakIterator.java	API Doc	Java SE 5 API	41572	Fri Aug 26 14:57:20 BST 2005	java.text

RuleBasedBreakIterator

• java.lang.Object
  • java.text.BreakIterator

Fields Summary
protected static final byte	IGNORE A token used as a character-category value to identify ignore characters
private static final short	START_STATE The state number of the starting state
private static final short	STOP_STATE The state-transition value indicating "stop"
static final byte[]	LABEL Magic number for the BreakIterator data file format.
static final int	LABEL_LENGTH
static final byte	supportedVersion Version number of the dictionary that was read in.
private static final int	HEADER_LENGTH Header size in byte count
private static final int	BMP_INDICES_LENGTH An array length of indices for BMP characters
private CompactByteArray	charCategoryTable Tables that indexes from character values to character category numbers
private SupplementaryCharacterData	supplementaryCharCategoryTable
private short[]	stateTable The table of state transitions used for forward iteration
private short[]	backwardsStateTable The table of state transitions used to sync up the iterator with the text in backwards and random-access iteration
private boolean[]	endStates A list of flags indicating which states in the state table are accepting ("end") states
private boolean[]	lookaheadStates A list of flags indicating which states in the state table are lookahead states (states which turn lookahead on and off)
private byte[]	additionalData A table for additional data. May be used by a subclass of RuleBasedBreakIterator.
private int	numCategories The number of character categories (and, thus, the number of columns in the state tables)
private CharacterIterator	text The character iterator through which this BreakIterator accesses the text
private long	checksum A CRC32 value of all data in datafile

Constructors Summary
public RuleBasedBreakIterator(String datafile) Constructs a RuleBasedBreakIterator according to the datafile provided. //======================================================================= // constructors //======================================================================= readTables(datafile);

Methods Summary
protected static final void	checkOffset(int offset, java.text.CharacterIterator text) Throw IllegalArgumentException unless begin <= offset < end. if (offset < text.getBeginIndex() || offset >= text.getEndIndex()) { throw new IllegalArgumentException("offset out of bounds"); }
public java.lang.Object	clone() Clones this iterator. return A newly-constructed RuleBasedBreakIterator with the same behavior as this one. RuleBasedBreakIterator result = (RuleBasedBreakIterator) super.clone(); if (text != null) { result.text = (CharacterIterator) text.clone(); } return result;
public int	current() Returns the current iteration position. return The current iteration position. return getText().getIndex();
public boolean	equals(java.lang.Object that) Returns true if both BreakIterators are of the same class, have the same rules, and iterate over the same text. try { if (that == null) { return false; } RuleBasedBreakIterator other = (RuleBasedBreakIterator) that; if (checksum != other.checksum) { return false; } if (text == null) { return other.text == null; } else { return text.equals(other.text); } } catch(ClassCastException e) { return false; }
public int	first() Sets the current iteration position to the beginning of the text. (i.e., the CharacterIterator's starting offset). return The offset of the beginning of the text. CharacterIterator t = getText(); t.first(); return t.getIndex();
public int	following(int offset) Sets the iterator to refer to the first boundary position following the specified position. offset The position from which to begin searching for a break position. return The position of the first break after the current position. CharacterIterator text = getText(); checkOffset(offset, text); // Set our internal iteration position (temporarily) // to the position passed in. If this is the _beginning_ position, // then we can just use next() to get our return value text.setIndex(offset); if (offset == text.getBeginIndex()) { return handleNext(); } // otherwise, we have to sync up first. Use handlePrevious() to back // us up to a known break position before the specified position (if // we can determine that the specified position is a break position, // we don't back up at all). This may or may not be the last break // position at or before our starting position. Advance forward // from here until we've passed the starting position. The position // we stop on will be the first break position after the specified one. int result = handlePrevious(); while (result != BreakIterator.DONE && result <= offset) { result = handleNext(); } return result;
byte[]	getAdditionalData() return additionalData;
int	getCurrent() Returns current character char c1 = text.current(); if (Character.isHighSurrogate(c1) && text.getIndex() < text.getEndIndex()) { char c2 = text.next(); text.previous(); if (Character.isLowSurrogate(c2)) { return Character.toCodePoint(c1, c2); } } return (int)c1;
private int	getCurrentCodePointCount() Returns the count of next character. char c1 = text.current(); if (Character.isHighSurrogate(c1) && text.getIndex() < text.getEndIndex()) { char c2 = text.next(); text.previous(); if (Character.isLowSurrogate(c2)) { return 2; } } return 1;
int	getNext() Returns next character int index = text.getIndex(); int endIndex = text.getEndIndex(); if (index == endIndex || (index = index + getCurrentCodePointCount()) >= endIndex) { return CharacterIterator.DONE; } text.setIndex(index); return getCurrent();
private int	getNextIndex() Returns the position of next character. int index = text.getIndex() + getCurrentCodePointCount(); int endIndex = text.getEndIndex(); if (index > endIndex) { return endIndex; } else { return index; }
private int	getPrevious() Returns previous character char c2 = text.previous(); if (Character.isLowSurrogate(c2) && text.getIndex() > text.getBeginIndex()) { char c1 = text.previous(); if (Character.isHighSurrogate(c1)) { return Character.toCodePoint(c1, c2); } else { text.next(); } } return (int)c2;
public java.text.CharacterIterator	getText() Return a CharacterIterator over the text being analyzed. This version of this method returns the actual CharacterIterator we're using internally. Changing the state of this iterator can have undefined consequences. If you need to change it, clone it first. return An iterator over the text being analyzed. // The iterator is initialized pointing to no text at all, so if this // function is called while we're in that state, we have to fudge an // iterator to return. if (text == null) { text = new StringCharacterIterator(""); } return text;
protected int	handleNext() This method is the actual implementation of the next() method. All iteration vectors through here. This method initializes the state machine to state 1 and advances through the text character by character until we reach the end of the text or the state machine transitions to state 0. We update our return value every time the state machine passes through a possible end state. // if we're already at the end of the text, return DONE. CharacterIterator text = getText(); if (text.getIndex() == text.getEndIndex()) { return BreakIterator.DONE; } // no matter what, we always advance at least one character forward int result = getNextIndex(); int lookaheadResult = 0; // begin in state 1 int state = START_STATE; int category; int c = getCurrent(); // loop until we reach the end of the text or transition to state 0 while (c != CharacterIterator.DONE && state != STOP_STATE) { // look up the current character's character category (which tells us // which column in the state table to look at) category = lookupCategory(c); // if the character isn't an ignore character, look up a state // transition in the state table if (category != IGNORE) { state = lookupState(state, category); } // if the state we've just transitioned to is a lookahead state, // (but not also an end state), save its position. If it's // both a lookahead state and an end state, update the break position // to the last saved lookup-state position if (lookaheadStates[state]) { if (endStates[state]) { result = lookaheadResult; } else { lookaheadResult = getNextIndex(); } } // otherwise, if the state we've just transitioned to is an accepting // state, update the break position to be the current iteration position else { if (endStates[state]) { result = getNextIndex(); } } c = getNext(); } // if we've run off the end of the text, and the very last character took us into // a lookahead state, advance the break position to the lookahead position // (the theory here is that if there are no characters at all after the lookahead // position, that always matches the lookahead criteria) if (c == CharacterIterator.DONE && lookaheadResult == text.getEndIndex()) { result = lookaheadResult; } text.setIndex(result); return result;
protected int	handlePrevious() This method backs the iterator back up to a "safe position" in the text. This is a position that we know, without any context, must be a break position. The various calling methods then iterate forward from this safe position to the appropriate position to return. (For more information, see the description of buildBackwardsStateTable() in RuleBasedBreakIterator.Builder.) CharacterIterator text = getText(); int state = START_STATE; int category = 0; int lastCategory = 0; int c = getCurrent(); // loop until we reach the beginning of the text or transition to state 0 while (c != CharacterIterator.DONE && state != STOP_STATE) { // save the last character's category and look up the current // character's category lastCategory = category; category = lookupCategory(c); // if the current character isn't an ignore character, look up a // state transition in the backwards state table if (category != IGNORE) { state = lookupBackwardState(state, category); } // then advance one character backwards c = getPrevious(); } // if we didn't march off the beginning of the text, we're either one or two // positions away from the real break position. (One because of the call to // previous() at the end of the loop above, and another because the character // that takes us into the stop state will always be the character BEFORE // the break position.) if (c != CharacterIterator.DONE) { if (lastCategory != IGNORE) { getNext(); getNext(); } else { getNext(); } } return text.getIndex();
public int	hashCode() Compute a hashcode for this BreakIterator return A hash code return (int)checksum;
public boolean	isBoundary(int offset) Returns true if the specfied position is a boundary position. As a side effect, leaves the iterator pointing to the first boundary position at or after "offset". param offset the offset to check. return True if "offset" is a boundary position. CharacterIterator text = getText(); checkOffset(offset, text); if (offset == text.getBeginIndex()) { return true; } // to check whether this is a boundary, we can use following() on the // position before the specified one and return true if the position we // get back is the one the user specified else { return following(offset - 1) == offset; }
public int	last() Sets the current iteration position to the end of the text. (i.e., the CharacterIterator's ending offset). return The text's past-the-end offset. CharacterIterator t = getText(); // I'm not sure why, but t.last() returns the offset of the last character, // rather than the past-the-end offset t.setIndex(t.getEndIndex()); return t.getIndex();
protected int	lookupBackwardState(int state, int category) Given a current state and a character category, looks up the next state to transition to in the backwards state table. return backwardsStateTable[state * numCategories + category];
protected int	lookupCategory(int c) Looks up a character's category (i.e., its category for breaking purposes, not its Unicode category) if (c < Character.MIN_SUPPLEMENTARY_CODE_POINT) { return charCategoryTable.elementAt((char)c); } else { return supplementaryCharCategoryTable.getValue(c); }
protected int	lookupState(int state, int category) Given a current state and a character category, looks up the next state to transition to in the state table. return stateTable[state * numCategories + category];
public int	next(int n) Advances the iterator either forward or backward the specified number of steps. Negative values move backward, and positive values move forward. This is equivalent to repeatedly calling next() or previous(). param n The number of steps to move. The sign indicates the direction (negative is backwards, and positive is forwards). return The character offset of the boundary position n boundaries away from the current one. int result = current(); while (n > 0) { result = handleNext(); --n; } while (n < 0) { result = previous(); ++n; } return result;
public int	next() Advances the iterator to the next boundary position. return The position of the first boundary after this one. return handleNext();
public int	preceding(int offset) Sets the iterator to refer to the last boundary position before the specified position. offset The position to begin searching for a break from. return The position of the last boundary before the starting position. // if we start by updating the current iteration position to the // position specified by the caller, we can just use previous() // to carry out this operation CharacterIterator text = getText(); checkOffset(offset, text); text.setIndex(offset); return previous();
public int	previous() Advances the iterator backwards, to the last boundary preceding this one. return The position of the last boundary position preceding this one. // if we're already sitting at the beginning of the text, return DONE CharacterIterator text = getText(); if (current() == text.getBeginIndex()) { return BreakIterator.DONE; } // set things up. handlePrevious() will back us up to some valid // break position before the current position (we back our internal // iterator up one step to prevent handlePrevious() from returning // the current position), but not necessarily the last one before // where we started int start = current(); getPrevious(); int lastResult = handlePrevious(); int result = lastResult; // iterate forward from the known break position until we pass our // starting point. The last break position before the starting // point is our return value while (result != BreakIterator.DONE && result < start) { lastResult = result; result = handleNext(); } // set the current iteration position to be the last break position // before where we started, and then return that value text.setIndex(lastResult); return lastResult;
protected byte[]	readFile(java.lang.String datafile) BufferedInputStream is; try { is = (BufferedInputStream)AccessController.doPrivileged( new PrivilegedExceptionAction() { public Object run() throws Exception { return new BufferedInputStream(getClass().getResourceAsStream("/sun/text/resources/" + datafile)); } } ); } catch (PrivilegedActionException e) { throw new InternalError(e.toString()); } int offset = 0; /* First, read magic, version, and header_info. */ int len = LABEL_LENGTH + 5; byte[] buf = new byte[len]; if (is.read(buf) != len) { throw new MissingResourceException("Wrong header length", datafile, ""); } /* Validate the magic number. */ for (int i = 0; i < LABEL_LENGTH; i++, offset++) { if (buf[offset] != LABEL[offset]) { throw new MissingResourceException("Wrong magic number", datafile, ""); } } /* Validate the version number. */ if (buf[offset] != supportedVersion) { throw new MissingResourceException("Unsupported version(" + buf[offset] + ")", datafile, ""); } /* Read data: totalDataSize + 8(for checksum) */ len = BreakIterator.getInt(buf, ++offset); buf = new byte[len]; if (is.read(buf) != len) { throw new MissingResourceException("Wrong data length", datafile, ""); } is.close(); return buf;
protected void	readTables(java.lang.String datafile) Read datafile. The datafile's format is as follows: BreakIteratorData { u1 magic[7]; u1 version; u4 totalDataSize; header_info header; body value; } totalDataSize is the summation of the size of header_info and body in byte count. In header, each field except for checksum implies the length of each field. Since BMPdataLength is a fixed-length data(512 entries), its length isn't included in header. checksum is a CRC32 value of all in body. header_info { u4 stateTableLength; u4 backwardsStateTableLength; u4 endStatesLength; u4 lookaheadStatesLength; u4 BMPdataLength; u4 nonBMPdataLength; u4 additionalDataLength; u8 checksum; } Finally, BMPindices and BMPdata are set to charCategoryTable. nonBMPdata is set to supplementaryCharCategoryTable. body { u2 stateTable[stateTableLength]; u2 backwardsStateTable[backwardsStateTableLength]; u1 endStates[endStatesLength]; u1 lookaheadStates[lookaheadStatesLength]; u2 BMPindices[512]; u1 BMPdata[BMPdataLength]; u4 nonBMPdata[numNonBMPdataLength]; u1 additionalData[additionalDataLength]; } byte[] buffer = readFile(datafile); /* Read header_info. */ int stateTableLength = BreakIterator.getInt(buffer, 0); int backwardsStateTableLength = BreakIterator.getInt(buffer, 4); int endStatesLength = BreakIterator.getInt(buffer, 8); int lookaheadStatesLength = BreakIterator.getInt(buffer, 12); int BMPdataLength = BreakIterator.getInt(buffer, 16); int nonBMPdataLength = BreakIterator.getInt(buffer, 20); int additionalDataLength = BreakIterator.getInt(buffer, 24); checksum = BreakIterator.getLong(buffer, 28); /* Read stateTable[numCategories * numRows] */ stateTable = new short[stateTableLength]; int offset = HEADER_LENGTH; for (int i = 0; i < stateTableLength; i++, offset+=2) { stateTable[i] = BreakIterator.getShort(buffer, offset); } /* Read backwardsStateTable[numCategories * numRows] */ backwardsStateTable = new short[backwardsStateTableLength]; for (int i = 0; i < backwardsStateTableLength; i++, offset+=2) { backwardsStateTable[i] = BreakIterator.getShort(buffer, offset); } /* Read endStates[numRows] */ endStates = new boolean[endStatesLength]; for (int i = 0; i < endStatesLength; i++, offset++) { endStates[i] = buffer[offset] == 1; } /* Read lookaheadStates[numRows] */ lookaheadStates = new boolean[lookaheadStatesLength]; for (int i = 0; i < lookaheadStatesLength; i++, offset++) { lookaheadStates[i] = buffer[offset] == 1; } /* Read a category table and indices for BMP characters. */ short[] temp1 = new short[BMP_INDICES_LENGTH]; // BMPindices for (int i = 0; i < BMP_INDICES_LENGTH; i++, offset+=2) { temp1[i] = BreakIterator.getShort(buffer, offset); } byte[] temp2 = new byte[BMPdataLength]; // BMPdata System.arraycopy(buffer, offset, temp2, 0, BMPdataLength); offset += BMPdataLength; charCategoryTable = new CompactByteArray(temp1, temp2); /* Read a category table for non-BMP characters. */ int[] temp3 = new int[nonBMPdataLength]; for (int i = 0; i < nonBMPdataLength; i++, offset+=4) { temp3[i] = BreakIterator.getInt(buffer, offset); } supplementaryCharCategoryTable = new SupplementaryCharacterData(temp3); /* Read additional data */ if (additionalDataLength > 0) { additionalData = new byte[additionalDataLength]; System.arraycopy(buffer, offset, additionalData, 0, additionalDataLength); } /* Set numCategories */ numCategories = stateTable.length / endStates.length;
void	setAdditionalData(byte[] b) additionalData = b;
public void	setText(java.text.CharacterIterator newText) Set the iterator to analyze a new piece of text. This function resets the current iteration position to the beginning of the text. param newText An iterator over the text to analyze. // Test iterator to see if we need to wrap it in a SafeCharIterator. // The correct behavior for CharacterIterators is to allow the // position to be set to the endpoint of the iterator. Many // CharacterIterators do not uphold this, so this is a workaround // to permit them to use this class. int end = newText.getEndIndex(); boolean goodIterator; try { newText.setIndex(end); // some buggy iterators throw an exception here goodIterator = newText.getIndex() == end; } catch(IllegalArgumentException e) { goodIterator = false; } if (goodIterator) { text = newText; } else { text = new SafeCharIterator(newText); } text.first();
public java.lang.String	toString() Returns text StringBuffer sb = new StringBuffer(); sb.append('["); sb.append("checksum=0x" + Long.toHexString(checksum)); sb.append(']"); return sb.toString();