| Methods Summary |
|---|
| Tokenizer | create(java.lang.String input, boolean multipleDelimsCountAsOne)
return( new TokenizerImpl( input,
WHITE_SPACE, multipleDelimsCountAsOne, ESCAPE_CHAR, ESCAPABLE_CHARS) );
|
| Tokenizer | create(java.lang.String input)
return( new TokenizerImpl( input,
WHITE_SPACE, false, ESCAPE_CHAR, ESCAPABLE_CHARS) );
|
| protected void | setUp()
|
| protected void | tearDown()
|
| public void | testEmpty()
final Tokenizer tk = create( "" );
assertEquals( "expecting no tokens", 0, tk.getTokens().length );
try
{
tk.iterator().next();
fail( "expecting exception trying to get token" );
}
catch( Exception e )
{
}
|
| public void | testEscapedNewlineCR()
final String TEST = "test" + BACKSLASH + "n" + BACKSLASH + "r";
final String [] tokens = create( TEST ).getTokens();
assertEquals( 1, tokens.length );
assertEquals( "test\n\r", tokens[ 0 ] );
|
| public void | testEscaping()
// create a String which each escapable character is represented
final StringBuffer b = new StringBuffer();
for( int i = 0; i < ESCAPABLE_CHARS.length(); ++i )
{
b.append( "\\" + ESCAPABLE_CHARS.charAt( i ) );
}
final String [] tokens = create( b.toString() ).getTokens();
assertEquals( "expecting 1 token", 1, tokens.length );
assertEquals( "expecting match", ESCAPABLE_CHARS, tokens[ 0 ] );
|
| public void | testLeadingDelim()
final String input = WHITE_SPACE.charAt( 0 ) + "hello";
final String [] tokens = create( input, false ).getTokens();
assertEquals( 2, tokens.length );
|
| public void | testMultipleDelimsAsOne()
final String input = "hello" + WHITE_SPACE + "there" + WHITE_SPACE;
final String [] tokens = create( input, true ).getTokens();
assertEquals( 3, tokens.length );
|
| public void | testMultipleDelimsWithNoData()
final String input = "" + WHITE_SPACE.charAt( 0 );
final String [] tokens = create( input, false ).getTokens();
assertEquals( input.length() + 1, tokens.length );
|
| public void | testMultipleTokens()
final String input = "hello there 1 2 3 4 5";
final Iterator iter = create( input ).iterator();
int count = 0;
String temp = "";
while ( iter.hasNext() )
{
++count;
temp = temp + " " + iter.next();
}
assertEquals( "expecting 7 tokens", 7, count );
assertEquals( "expecting match", input, temp.substring( 1, temp.length() ));
|
| public void | testQuotedString()
final String input = "\"hello there\" \"another\" \"3\" \"words\" \"\"";
final String [] tokens = create( input ).getTokens();
assertEquals( 5, tokens.length );
assertEquals( "hello there", tokens[ 0 ] );
assertEquals( "another", tokens[ 1 ] );
assertEquals( "3", tokens[ 2 ] );
assertEquals( "words", tokens[ 3 ] );
assertEquals( "", tokens[ 4 ] );
|
| public void | testSingleDelimOnly()
final String input = "" + WHITE_SPACE.charAt( 0 );
final String [] tokens = create( input, false ).getTokens();
assertEquals( 2, tokens.length );
|
| public void | testSingleToken()
final String input = "hello";
final String [] tokens = create( input, false ).getTokens();
assertEquals( "expecting 1 token", 1, tokens.length );
assertEquals( "expecting " + input, input, tokens[ 0 ] );
|
| public void | testTrailingDelimiter1()
final String input = "hello" + WHITE_SPACE.charAt( 0 );
final String [] tokens = create( input, false ).getTokens();
assertEquals( 2, tokens.length );
|
| public void | testTrailingDelimiter2()
final String input = "hello" + WHITE_SPACE.charAt( 0 ) +
"\"there\"" + WHITE_SPACE.charAt( 0 );
final String [] tokens = create( input, false ).getTokens();
assertEquals( 3, tokens.length );
|
| public void | testUnescapableChar()
final String TEST = "test" + BACKSLASH + "xyz";
final String [] tokens = create( TEST ).getTokens();
assertEquals( 1, tokens.length );
assertEquals( "test\\xyz", tokens[ 0 ] );
|
| public void | testWhiteSpaceEquality()
final String input1 = "hello there 1 2 3 4 5";
final String input2 = "hello\tthere 1\t2 3 4\t5";
final Iterator iter1 = create( input1 ).iterator();
final Iterator iter2 = create( input2 ).iterator();
while ( iter1.hasNext() )
{
assertEquals( "expecting equal results from different white space",
iter1.next(), iter2.next() );
}
|
