StringTokeniser.java
import java.util.*;

/**
 * StringTokeniser allows client code to break a specified String up into an
 * array of 'tokens' using a specified delimiter. The important differences
 * between this class and <code>java.util.StringTokenizer</code> are:
 * <ul>
 *   <li>In order to qualify as a token, a substring must both begin and end
 *       with the specified delimiter.
 *   <li>The delimiter marking the end of a token is not considered to be the
 *       delimiter marking the beginning of the next token.
 *   <li>This class does not ignore the empty String (""), which is treated as
 *       a token just like any other
 *   <li>This class breaks the String up into an array of tokens and returns the
 *       entire set of tokens to the caller in one go, rather than 1 token at a
 *       time.
 * </ul>
 * <p>
 * An example of the differing results of tokenising a String using this class
 * as opposed to <code>java.util.StringTokenizer</code> can be seen from the
 * following example:
 * <p>
 * <pre><code>
 * stringToTokenise = "\"Hello\",\"There\",\"\",\" \",\"Michael\"";
 * delimiter = "\"";
 * </pre></code>
 * <p>
 * When the above String is tokenised by both classes using the specified delimiter,
 * the tokens created by each class are as follows:
 * <p>
 * <strong><code>java.util.StringTokenizer</code> tokens:</strong>
 * <ul>
 *   <li>Token 0 : Hello
 *   <li>Token 1 : ,
 *   <li>Token 2 : There
 *   <li>Token 3 : ,
 *   <li>Token 4 : ,
 *   <li>Token 5 :
 *   <li>Token 6 : ,
 *   <li>Token 7 : Michael
 *   <li>Token 8 :
 * </ul>
 * <p>
 * <strong><code>StringTokeniser</code> tokens:</strong>
 * <ul>
 *   <li>Token 0 : Hello
 *   <li>Token 1 : There
 *   <li>Token 2 :
 *   <li>Token 3 :
 *   <li>Token 4 : Michael
 * </ul>
 *
 * Note that the caller has the option to append the delimiters to the tokens if
 * required, hence Token 0 of the above example would be <code>"Hello"</code> as
 * opposed to <code>Hello</code> if this option was exercised.
 *
 * @author Michael Fitzmaurice, November 2002
 */
public class StringTokeniser
{
    private String      m_stringToTokenise;
    private String      m_tokenDelimiter;

    /**
     * Creates a new instance of <code>StringTokeniser</code> to parse the
     * specified string.
     *
     * @param str The string to be parsed.
     * @param delim The string that will be used to mark the beginning and end
     * of a token. In order to be recognised as a token, a substring must both
     * begin and end with this delimiting sequence. The end delimiter of one
     * token is not considered to be the start delimiter of the next token.
     */
    public StringTokeniser(String str, String delimiter)
    {
        m_stringToTokenise  = str;
        m_tokenDelimiter    = delimiter;
    }

    /**
     * Splits this tokeniser's raw string into a collection of tokens using
     * the delimiter specified at construction time.
     *
     * @param includeDelimiters If true, each token will have its beginning and
     * end markers included as part of the token itself
     *
     * @return An array of Strings, each element of which represents one token.
     * If no tokens were present, a 0 length String array will be returned
     */
    public String[] getTokens(boolean includeDelimiters)
    {
        List delimiterIndices = new ArrayList();

        int delimiterCount   = 0;
        int tokenCount       = 0;
        int delimiterIndex   = m_stringToTokenise.indexOf(m_tokenDelimiter);

        while (delimiterIndex != -1)
        {
            delimiterCount++;
            // every 2 delimiters means another token
            if (delimiterCount % 2 == 0)
                tokenCount++;

            delimiterIndices.add( new Integer(delimiterIndex) );
            delimiterIndex =
                m_stringToTokenise.indexOf(m_tokenDelimiter, delimiterIndex + 1);
        }

        String[] tokens       = new String[tokenCount];
        int nextTokenIndex    = 0;

        for (int i = 0; i < delimiterCount; i++)
        {
            int firstDelimiterIndex =
                ( (Integer)delimiterIndices.get(i) ).intValue();

            if ( ++i < delimiterCount )
            {
                int secondDelimeterIndex =
                    ( (Integer)delimiterIndices.get(i) ).intValue();

                // do we append the delimiters to the token or not?
                if (includeDelimiters)
                    secondDelimeterIndex += 1;
                else
                    firstDelimiterIndex += 1;

                String nextToken =
                        m_stringToTokenise.substring(   firstDelimiterIndex,
                                                        secondDelimeterIndex);

                tokens[nextTokenIndex++] = nextToken;
            }
        }

        return tokens;
    }

    /**
     * Basic unit test. Takes a String to be tokenised and a String to be used
     * as a token delimiter and prints the contents of the array returned by the
     * <code>getTokens</code> method of the <code>StringTokeniser</code> object
     * created. The result of tokenising the same String using the <code>
     * java.util.StringTokenizer</code> class is also printed to the console
     * for comparison.
     * <p>
     * Parameters required:
     * <ul>
     *   <li>Flag to specifiy whether or not delimiters should be appended to
     *       tokens. A value of 1 denotes true, 0 denotes false
     *   <li>String to use as token delimiter
     *   <li><i>n</i> Strings to be interpreted as a single String to tokenise
     * </ul>
     */
    public static void main(String[] args)
    {
        int includeDelims       = Integer.parseInt(args[0]);
        String delimiter        = args[1];
        String stringToTokenise = "";

        for (int i = 2; i < args.length; i++)
        {
            stringToTokenise += args[i];
            stringToTokenise += " ";
        }

        StringTokeniser tokeniser   = new StringTokeniser(stringToTokenise, delimiter);
        boolean includeDelimiters   = includeDelims == 1;
        String[] tokens             = tokeniser.getTokens(includeDelimiters);

        System.out.println("Original String: " + stringToTokenise);
        System.out.println("Delimiter: " + delimiter);
        System.out.println("\nTOKENS:\n");

        for (int i = 0; i < tokens.length; i++)
        {
            System.out.println("\tToken " + i + " : " + tokens[i]);
        }

        System.out.println("-----------------------------------------------");

        StringTokenizer tokenizer = new StringTokenizer(  stringToTokenise,
                                                          delimiter);

        System.out.println("\nJAVA TOKENS:\n");

        int tokenCount = 0;

        while ( tokenizer.hasMoreTokens() )
        {
            System.out.println( "\tToken " + tokenCount + " : " +
                                tokenizer.nextToken() );
            tokenCount++;
        }

    }
}

StringTokeniser.java