| Methods Summary |
|---|
| Tokenizer | create(java.lang.String input, boolean multipleDelimsCountAsOne)
return( new TokenizerImpl( input,
WHITE_SPACE, multipleDelimsCountAsOne, ESCAPE_CHAR, ESCAPABLE_CHARS) );
|
| Tokenizer | create(java.lang.String input)
return( new TokenizerImpl( input,
WHITE_SPACE, false, ESCAPE_CHAR, ESCAPABLE_CHARS) );
|
| static java.lang.String | quote(java.lang.String s)
return( QUOTE + s + QUOTE );
|
| protected void | setUp()
|
| protected void | tearDown()
|
| public void | testAdjacentQuotedStrings()
final String input = quote( "xxx" ) + quote( "yyy" ) + quote( "" );
final String [] tokens = create( input ).getTokens();
assertEquals( 1, tokens.length );
assertEquals( "xxxyyy", tokens[ 0 ] );
|
| public void | testEmpty()
final Tokenizer tk = create( "" );
assertEquals( "expecting no tokens", 0, tk.getTokens().length );
|
| public void | testEmptyQuotedString()
final String input = quote( "" );
final String [] tokens = create( input ).getTokens();
assertEquals( 1, tokens.length );
assertEquals( "", tokens[ 0 ] );
|
| public void | testEmptyQuotedStrings()
final String input = quote( "" ) + quote( "" ) + quote( "" ) + quote( "" );
final String [] tokens = create( input ).getTokens();
assertEquals( 1, tokens.length );
assertEquals( "", tokens[ 0 ] );
|
| public void | testEmptyQuotedStringsSeparatedByDelim()
final String input = quote( "" ) + WHITE_SPACE.charAt( 0 ) + quote( "" );
final String [] tokens = create( input ).getTokens();
assertEquals( 2, tokens.length );
|
| public void | testEscapedNewlineCR()
final String TEST = "test" + BACKSLASH + "n" + BACKSLASH + "r" + BACKSLASH + "t";
final String [] tokens = create( TEST ).getTokens();
assertEquals( 1, tokens.length );
assertEquals( "test\n\r\t", tokens[ 0 ] );
|
| public void | testEscaping()
// create a String which each escapable character is represented
final StringBuffer b = new StringBuffer();
for( int i = 0; i < ESCAPABLE_CHARS.length(); ++i )
{
b.append( "\\" + ESCAPABLE_CHARS.charAt( i ) );
}
final String [] tokens = create( b.toString() ).getTokens();
assertEquals( "expecting 1 token", 1, tokens.length );
assertEquals( "expecting match", ESCAPABLE_CHARS, tokens[ 0 ] );
|
| public void | testIllegalEscapeSequence()
final String input = BACKSLASH + "x";
try
{
final String [] tokens = create( input ).getTokens();
fail( "expected to fail: " + input);
}
catch( TokenizerException e )
{
}
|
| public void | testIllegalUnicodeSequence()
final String input1 = BACKSLASH + "u";
final String input2 = input1 + "zzzz";
final String input3 = input1 + "abcx";
try
{
create( input1 ).getTokens();
fail( "expected to fail: " + input1);
create( input2 ).getTokens();
fail( "expected to fail: " + input2);
create( input3 ).getTokens();
fail( "expected to fail: " + input3);
}
catch( TokenizerException e )
{
}
|
| public void | testLeadingDelim()
final String input = WHITE_SPACE.charAt( 0 ) + "hello";
final String [] tokens = create( input, false ).getTokens();
assertEquals( 2, tokens.length );
assertEquals( "", tokens[ 0 ] );
assertEquals( "hello", tokens[ 1 ] );
|
| public void | testLegalUnicodeSequence()
final String input = BACKSLASH + "u0020"; // unicode for the space char
final String[] tokens = create( input ).getTokens();
assertEquals( 1, tokens.length );
assertEquals( " ", tokens[ 0 ] );
|
| public void | testMultipleDelimsAsOne()
final String input = "HELLO" + WHITE_SPACE + "THERE" + WHITE_SPACE + WHITE_SPACE;
final String [] tokens = create( input, true ).getTokens();
assertEquals( 3, tokens.length );
|
| public void | testMultipleDelimsWithNoData()
final String input = "" + WHITE_SPACE.charAt( 0 );
final String [] tokens = create( input, false ).getTokens();
assertEquals( input.length() + 1, tokens.length );
|
| public void | testMultipleTokens()
final String input = "hello there 1 2 3 4 5";
final String[] tokens = create( input ).getTokens();
assertEquals( "expecting 7 tokens", 7, tokens.length );
assertEquals( tokens[ 0 ], "hello" );
assertEquals( tokens[ 1 ], "there" );
assertEquals( tokens[ 2 ], "1" );
assertEquals( tokens[ 3 ], "2" );
assertEquals( tokens[ 4 ], "3" );
assertEquals( tokens[ 5 ], "4" );
assertEquals( tokens[ 6 ], "5" );
|
| public void | testOnlyDelims()
final String input = WHITE_SPACE + WHITE_SPACE + WHITE_SPACE;
final String [] tokens = create( input, false ).getTokens();
assertEquals( (WHITE_SPACE.length() * 3) + 1, tokens.length );
|
| public void | testQuotedDelim()
final String input = quote( " " ) + " " + quote( " " ) + BACKSLASH + " ";
final String [] tokens = create( input ).getTokens();
assertEquals( 2, tokens.length );
assertEquals( " ", tokens[ 0 ] );
assertEquals( " ", tokens[ 1 ] );
|
| public void | testQuotedString()
final String input = quote( "hello there" ) + " " + quote( "another" ) + " " +
quote( "3" ) + " " + quote( "words" ) + " ";
final String [] tokens = create( input ).getTokens();
assertEquals( 5, tokens.length );
assertEquals( "hello there", tokens[ 0 ] );
assertEquals( "another", tokens[ 1 ] );
assertEquals( "3", tokens[ 2 ] );
assertEquals( "words", tokens[ 3 ] );
assertEquals( "", tokens[ 4 ] );
|
| public void | testSingleDelimOnly()
final String input = "" + WHITE_SPACE.charAt( 0 );
final String [] tokens = create( input, false ).getTokens();
assertEquals( 2, tokens.length );
assertEquals( "", tokens[ 0 ] );
assertEquals( "", tokens[ 1 ] );
|
| public void | testSingleToken()
final String input = "hello";
final String [] tokens = create( input, false ).getTokens();
assertEquals( "expecting 1 token", 1, tokens.length );
assertEquals( "expecting " + input, input, tokens[ 0 ] );
|
| public void | testTrailingDelimiter1()
final String input = "hello" + WHITE_SPACE.charAt( 0 );
final String [] tokens = create( input, false ).getTokens();
assertEquals( 2, tokens.length );
|
| public void | testTrailingDelimiter2()
final String input = "hello" + WHITE_SPACE.charAt( 0 ) +
"\"there\"" + WHITE_SPACE.charAt( 0 );
final String [] tokens = create( input, false ).getTokens();
assertEquals( 3, tokens.length );
|
| public void | testTrailingEscapedDelim()
final String input = "x." + BACKSLASH + ".";
final String[] tokens = new TokenizerImpl( input,
".", false, BACKSLASH, BACKSLASH + "." ).getTokens();
assertEquals( 2, tokens.length );
assertEquals( "x", tokens[ 0 ] );
assertEquals( ".", tokens[ 1 ] );
|
| public void | testUnterminatedLiteralString()
final String input1 = "" + QUOTE;
final String input2 = QUOTE + "xxx";
final String input3 = "xxx" + QUOTE;
try
{
create( input1 ).getTokens();
fail( "expected to fail: " + input1);
create( input2 ).getTokens();
fail( "expected to fail: " + input2);
create( input3 ).getTokens();
fail( "expected to fail: " + input3);
}
catch( TokenizerException e )
{
}
|
| public void | testWhiteSpaceEquality()
final String input1 = "hello there 1 2 3 4 5";
final String input2 = "hello\tthere 1\t2 3 4\t5";
final String[] tokens1 = create( input1 ).getTokens();
final String[] tokens2 = create( input2 ).getTokens();
assertEquals( tokens1.length, tokens2.length );
for( int i = 0; i < tokens1.length; ++i )
{
assertEquals( tokens1[ i ], tokens2[ i ] );
}
|
