/* 
 * Parser.java --
 *
 *	This class contains methods that parse Tcl scripts.  They
 *	do so in a general-purpose fashion that can be used for many
 *	different purposes, including compilation, direct execution,
 *	code analysis, etc.  This class also includes a few additional
 *	procedures such as evalObjv, eval, and eval2, which allow 
 *	scripts to be evaluated directly, without compiling.
 *
 * Copyright (c) 1998 by Sun Microsystems, Inc.
 *
 * See the file "license.terms" for information on usage and redistribution
 * of this file, and for a DISCLAIMER OF ALL WARRANTIES.
 *
 * RCS: @(#) $Id: Parser.java,v 1.1.1.1 1998/10/14 21:09:19 cvsadmin Exp $
 */

package tcl.lang;

class Parser 
{

/*
 *----------------------------------------------------------------------
 *
 * parseCommand --
 *
 *	Given a script, this procedure parses the first Tcl command
 *	in the string and returns information about the structure of
 *	the command.
 *
 * Results:
 *	The return value is TclParse object that contains information 
 *	about the parsed command.  If the command was parsed 
 *	successfully, the TclParse object's result variable is set to
 *	TCL_OK and TCL_ERROR otherwise.  If an error occurs and interp
 *	isn't null then an error message is left in its result.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static TclParse
parseCommand(
    Interp interp,		// Interpreter to use for error reporting;
				// if null, then no error message is
				// provided. 

    char[]  script_array,       // References the script and contains an 
    int     script_index,       // index to the next character to parse.


    int numBytes,		// Total number of bytes in string.  If < 0,
				// the script consists of all bytes up to the
				// first null character. 
    String fileName,		// Name of file from which script was
				// loaded.  Null means there is no
				// known file for the script.  Used for
				// error messages. 
    int lineNum,		// Line number of first byte in string.
				// Used for error messages. 
    boolean nested)		// True means that this is a nested command:
				// close bracket should be considered
				// a command terminator.  If false, then close
				// bracket has no special meaning. 
{

    char cur;                   //the char we are currently parsing
    int type;			// Result returned by charType(src.charAt()). 
    TclToken token; 		// Pointer to token being filled in. 
    int wordIndex;		// Index of word token for current word. 
    int level;			// Nesting level of curly braces: gives
				// number of right braces we must find to
				// end word. 
    TclParse parse;     	// Return value to fill in with information
				// about the parsed command.
    int terminators;		// charType() bits that indicate the end
				// of a command. 
    BackSlashResult bs;		// Result of a call to backslash(...).
    int endIndex;		// Index that points to the character after
				// the last character to be parsed.
    char savedChar;		// To terminate the parsing correctly, the
				// character at endIndex is set to \0.  This
				// stores the value to return when finished.


    int saved_script_index = script_index; //save the original index
    

    int script_length = script_array.length - 1;


    if (numBytes < 0) {
	numBytes = script_length - script_index;
    }
    endIndex = script_index + numBytes;
    if (endIndex > script_length) {
	endIndex = script_length;
    }

    savedChar = script_array[endIndex];
    script_array[endIndex] = '\0';

    parse = new TclParse(interp, script_array, endIndex, fileName, lineNum);

    if (nested) {
	terminators = TYPE_COMMAND_END | TYPE_CLOSE_BRACK;
    } else {
	terminators = TYPE_COMMAND_END;
    }

    /*
     * Parse any leading space and comments before the first word of the
     * command.
     */


    try {

	while (true) {

	  cur = script_array[script_index];
	  
	  while ((typeTable[cur] == TYPE_SPACE) || (cur == '\n')) {
	    cur = script_array[++script_index];
	  }
	  
	  if ((cur == '\\') && (script_array[script_index + 1] == '\n')) {
	    
	    // Skip backslash-newline sequence: it should be treated
	    // just like white space.
	    
	    if ((script_index + 2) == parse.endIndex) {
	      parse.incomplete = true;
	    }

	    //this will add 2 to the offset and return to the top
	    //of the while(true) loop which will get the next cur

	    script_index += 2;
	    continue;
	  }
	  
	  //if we have found the start of a command goto the word parsing loop
	  if (cur != '#') {
	    break;
	  }

	  //record the index where the comment starts
	  if (parse.commentStart < 0) {
	    parse.commentStart = script_index;
	  }

	  while (true) {
	    cur = script_array[script_index];
	    if (script_index == parse.endIndex) {
	      parse.incomplete = true;
	      parse.commentSize = script_index - parse.commentStart;
	      break;
	    } else if (cur == '\\') {
	      if ((script_array[script_index + 1] == '\n') 
		  && ((script_index + 2) == parse.endIndex)) {
		parse.incomplete = true;
	      }
	      bs = backslash(script_array, script_index);
	      script_index = bs.nextIndex;
	    } else if (cur == '\n') {
	      script_index++;
	      parse.commentSize = script_index - parse.commentStart;
	      break;
	    } else {
	      script_index++;
	    }
	  }
	}



	// The following loop parses the words of the command, one word
	// in each iteration through the loop.

	parse.commandStart = script_index;

	while (true) {
 	    
	    // Create the token for  the word.
	    wordIndex = parse.numTokens;

	    token = parse.getToken(wordIndex);
	    token.type = TCL_TOKEN_WORD;

	    // Skip white space before the word. Also skip a backslash-newline
	    // sequence: it should be treated just like white space.


	    while (true) {
	      cur = script_array[script_index];
	      type = typeTable[cur];
	      
	      if (type == TYPE_SPACE) {
		script_index++;
		continue;
	      } else if ((cur == '\\') &&
			 (script_array[script_index + 1] == '\n')) {
		if ((script_index + 2) == parse.endIndex) {
		  parse.incomplete = true;
		}
		bs = backslash(script_array,script_index);
		script_index = bs.nextIndex;
		continue;
	      }
	      break;
	    }

	    if ((type & terminators) != 0) {
		script_index++;
		break;
	    }

	    if (script_index == parse.endIndex) {
		break;
	    }

	    token.script_array = script_array;
	    token.script_index = script_index;

	    parse.numTokens++;
	    parse.numWords++;


	    // At this point the word can have one of three forms: something
	    // enclosed in quotes, something enclosed in braces, or an
	    // unquoted word (anything else).

	    cur = script_array[script_index];
	    
	    if (cur == '"') {
	        script_index++;
		parse = parseTokens(script_array,script_index, TYPE_QUOTE, parse);
		if (parse.result != TCL.OK) {
		    throw new TclException(parse.result);
		}
		if (parse.string[parse.termIndex] != '"') {
		    parse.termIndex = script_index - 1;
		    parse.incomplete = true;
		    throw new TclException(parse.interp, "missing \"");
		}
		script_index = parse.termIndex + 1;
	    } else if (cur == '{') {
		/*
		 * Find the matching right brace that terminates the word,
		 * then generate a single token for everything between the
		 * braces.
		 */

		script_index++;
		token = parse.getToken(parse.numTokens);
		token.type = TCL_TOKEN_TEXT;
		token.script_array = script_array;
		token.script_index = script_index;
		token.numComponents = 0;
		level = 1;
		while (true) {
		    cur = script_array[script_index];

		    //get the current char in the array and lookup its type in array
		    while (typeTable[script_array[script_index]] == TYPE_NORMAL) {
		      script_index++;
		    }
		    if (script_array[script_index] == '}') {
			level--;
			if (level == 0) {
			    break;
			}
			script_index++;
		    } else if (script_array[script_index] == '{') {
			level++;
			script_index++;
		    } else if (script_array[script_index] == '\\') {
			bs = backslash(script_array,script_index);
			if (script_array[script_index + 1] == '\n') {
			    /*
			     * A backslash-newline sequence requires special
			     * treatment: it must be collapsed, even inside
			     * braces, so we have to split the word into
			     * multiple tokens so that the backslash-newline
			     * can be represented explicitly.
			     */

			    if ((script_index + 2) == parse.endIndex) {
				parse.incomplete = true;
			    }
			    token.size = script_index - token.script_index;
			    if (token.size != 0) {
				parse.numTokens++;
			    }
			    token = parse.getToken(parse.numTokens);
			    token.type = TCL_TOKEN_BS;
			    token.script_array = script_array;
			    token.script_index = script_index;
			    token.size = bs.nextIndex - script_index;
			    token.numComponents = 0;
			    parse.numTokens++;
			    script_index = bs.nextIndex;
			    token = parse.getToken(parse.numTokens);
			    token.type = TCL_TOKEN_TEXT;
			    token.script_array = script_array;
			    token.script_index = script_index;
			    token.numComponents = 0;
			} else {
			    script_index = bs.nextIndex;
			}
		    } else if (script_index == parse.endIndex) {
			parse.termIndex = 
			    parse.getToken(wordIndex).script_index; 
			parse.incomplete = true;
			throw new TclException(interp, "missing close-brace");
		    } else {
			script_index++;
		    }
		}
		if ((script_index != token.script_index)
			|| (parse.numTokens == (wordIndex + 1))) {
		    token.size = script_index - token.script_index;
		    parse.numTokens++;
		}
		script_index++;
	    } else {
		/*
		 * This is an unquoted word.  Call parseTokens and let it do
		 * all of the work.
		 */

		parse = parseTokens(script_array, script_index, TYPE_SPACE|terminators, 
			parse);
		if (parse.result != TCL.OK) {
		    throw new TclException(parse.result);
		}
		script_index = parse.termIndex;
	    }
	    
	    /*
	     * Finish filling in the token for the word and check for the
	     * special case of a word consisting of a single range of
	     * literal text.
	     */
	    
	    token = parse.getToken(wordIndex);
	    token.size = script_index - token.script_index;
	    token.numComponents = parse.numTokens - (wordIndex + 1);
	    if ((token.numComponents == 1) && 
		    (parse.getToken(wordIndex + 1).type == TCL_TOKEN_TEXT)) {
		token.type = TCL_TOKEN_SIMPLE_WORD;
	    }

	    /*
	     * Do two additional checks: (a) make sure we're really at the
	     * end of a word (there might have been garbage left after a
	     * quoted or braced word), and (b) check for the end of the
	     * command.
	     */
	    

	    cur = script_array[script_index];
	    type = typeTable[cur];

	    if (type == TYPE_SPACE) {
		script_index++;
		continue;
	    } else {
		/*
		 * Backslash-newline (and any following white space) must be
		 * treated as if it were a space character.
		 */

		if ((cur == '\\') && (script_array[script_index + 1] == '\n')) {
		    if ((script_index + 2) == parse.endIndex) {
			parse.incomplete = true;
		    }
		    bs = backslash(script_array,script_index);
		    script_index = bs.nextIndex;
		    continue;
		}
	    }

	    if ((type & terminators) != 0) {
		script_index++;
		break;
	    }
	    if (script_index == parse.endIndex) {
		break;
	    }
	    parse.termIndex = script_index;
	    if (script_array[script_index - 1] == '"') { 
		throw new TclException(interp, 
			"extra characters after close-quote");
	    } else {
		throw new TclException(interp, 
			"extra characters after close-brace");
	    }
	}

    } catch (TclException e) {
	script_array[endIndex] = savedChar;
	if (parse.commandStart < 0) {
	    parse.commandStart = saved_script_index;
	}
	parse.commandSize = parse.termIndex - parse.commandStart;
	parse.result = TCL.ERROR;
	return parse;
    }

    script_array[endIndex] = savedChar;
    parse.commandSize = script_index - parse.commandStart;
    parse.result = TCL.OK;
    return parse;
}












/*
 *----------------------------------------------------------------------
 *
 * parseTokens --
 *
 *	This procedure forms the heart of the Tcl parser.  It parses one
 *	or more tokens from a string, up to a termination point
 *	specified by the caller.  This procedure is used to parse
 *	unquoted command words (those not in quotes or braces), words in
 *	quotes, and array indices for variables.
 *
 * Results:
 *	Tokens are added to parse and parse.termIndex is filled in
 *	with the index of the character that terminated the parse (the
 *	first one whose charType() matched mask or the character at
 *	parse.endIndex).  The return value is TclParse object that 
 *	contains information about the parsed command.  If the 
 *	command was parsed successfully, the TclParse object's result 
 *	variable is set to TCL_OK and TCL_ERROR otherwise.  If an error 
 *	occurs and interp isn't null then an error message is left in 
 *	its result.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */


static TclParse
parseTokens(
    char[] script_array, //the text to parse
    int    script_index, // Index points to the first  character to parse.

    int mask,			// Specifies when to stop parsing.  The
				// parse stops at the first unquoted
				// character whose charType() contains
				// any of the bits in mask. 
    TclParse parse)     	// Information about parse in progress.
				// Updated with additional tokens and
				// termination information. 
{
    char cur;
    int type, originalTokens, varToken;
    TclToken token;
    TclParse nested;
    BackSlashResult bs;

    /*
     * Each iteration through the following loop adds one token of
     * type TCL_TOKEN_TEXT, TCL_TOKEN_BS, TCL_TOKEN_COMMAND, or
     * TCL_TOKEN_VARIABLE to parsePtr.  For TCL_TOKEN_VARIABLE additional,
     * tokens tokens are added for the parsed variable name.
     */

    originalTokens = parse.numTokens;
    while (true) {
	token = parse.getToken(parse.numTokens);
	token.script_array = script_array;
	token.script_index = script_index;
	token.numComponents = 0;
	
	cur = script_array[script_index];
	type = typeTable[cur];
	    
	if ((type & mask) != 0) {
	    break;
	}

	if ((type & TYPE_SUBS) == 0) {
	    /*
	     * This is a simple range of characters.  Scan to find the end
	     * of the range.
	     */

	    while (true) {
	        script_index++;
		if (((typeTable[script_array[script_index]]) & (mask | TYPE_SUBS)) != 0) {
		    break;
		}
	    }
	    token.type = TCL_TOKEN_TEXT;
	    token.size = script_index - token.script_index;
	    parse.numTokens++;
	} else if (cur == '$') {
	    /*
	     * This is a variable reference.  Call parseVarName to do
	     * all the dirty work of parsing the name.
	     */

	    varToken = parse.numTokens;
	    parse = parseVarName(parse.interp, script_array, script_index,
				 parse.endIndex - script_index, parse, true);
	    if (parse.result != TCL.OK) {
		return parse;
	    }
	    script_index += parse.getToken(varToken).size;
	} else if (cur == '[') {
	    /*
	     * Command substitution.  Call parseCommand recursively
	     * (and repeatedly) to parse the nested command(s), then
	     * throw away the parse information.
	     */

	    script_index++;
	    while (true) {
		nested = parseCommand(parse.interp, script_array, script_index, 
			parse.endIndex - script_index, parse.fileName,
			parse.lineNum, true);
		if (nested.result != TCL.OK) {
		    parse.termIndex = nested.termIndex;
		    parse.incomplete = nested.incomplete;
		    parse.result = nested.result;
		    return parse;
		}
		script_index = nested.commandStart + nested.commandSize;
		if ((script_array[script_index - 1] == ']') && !nested.incomplete) {
		    break;
		}
		if (script_index == parse.endIndex) {
		    if (parse.interp != null) {
			parse.interp.setResult("missing close-bracket");
		    }
		    parse.termIndex = token.script_index;
		    parse.incomplete = true;
		    parse.result = TCL.ERROR;
		    return parse;
		}
	    }
	    token.type = TCL_TOKEN_COMMAND;
	    token.size = script_index - token.script_index;
	    parse.numTokens++;
	} else if (cur == '\\') {
	    /*
	     * Backslash substitution.
	     */

	    if (script_array[script_index + 1] == '\n') {
		if ((script_index + 2) == parse.endIndex) {
		    parse.incomplete = true;
		}
		
		/*
		 * Note: backslash-newline is special in that it is
		 * treated the same as a space character would be.  This
		 * means that it could terminate the token.
		 */

		if ((mask & TYPE_SPACE) != 0) {
		    break;
		}
	    }
	    token.type = TCL_TOKEN_BS;
	    bs = backslash(script_array,script_index);
	    token.size = bs.nextIndex - script_index;
	    parse.numTokens++;
	    script_index += token.size;
	} else if (cur == '\0') {
	    /*
	     * We encountered a null character.  If it is the null
	     * character at the end of the string, then return.
	     * Otherwise generate a text token for the single
	     * character.
	     */

	    if (script_index == parse.endIndex) {
		 break;
	    }

	    token.type = TCL_TOKEN_TEXT;
	    token.size = 1;
	    parse.numTokens++;
	    script_index++;
	} else {
	    throw new TclRuntimeError(
		    "parseTokens encountered unknown character");
	}
    }
    if (parse.numTokens == originalTokens) {
	/*
	 * There was nothing in this range of text.  Add an empty token
	 * for the empty range, so that there is always at least one
	 * token added.
	 */

	token.type = TCL_TOKEN_TEXT;
	token.size = 0;
	parse.numTokens++;
    }
    parse.termIndex = script_index;
    parse.result = TCL.OK;
    return parse;
}





/*
 *----------------------------------------------------------------------
 *
 * evalObjv --
 *
 *	This procedure evaluates a Tcl command that has already been
 *	parsed into words, with one TclObject holding each word.
 *
 * Results:
 *	A result or error message is left in interp's result.  If an 
 *	error occurs, this procedure does NOT add any information to
 *	the errorInfo variable.
 *
 * Side effects:
 *	Depends on the command.
 *
 *----------------------------------------------------------------------
 */

static void
evalObjv(
    Interp interp,		// Interpreter in which to evaluate the
				// command.  Also used for error
				// reporting. 
    TclObject[] objv,        	// An array of pointers to objects that are
				// the words that make up the command. 

    int length,			// Number of bytes in command; if -1, all
				// characters up to the first null byte are
				// used. 
    int flags)			// Collection of OR-ed bits that control
				// the evaluation of the script.  Only
				// TCL_EVAL_GLOBAL is currently
				// supported. 
throws 
    TclException
{
    Command cmd;
    TclObject[] newObjv;
    int i;
    CallFrame savedVarFrame;	//Saves old copy of iPtr->varFramePtr
                                // in case TCL_EVAL_GLOBAL was set.

    interp.resetResult();
    if (objv.length == 0) {
	return;
    }
    
    /*
     * If the interpreter was deleted, return an error.
     */
    
    if (interp.deleted){
	TclString.append(interp.getResult(),
		"attempt to call eval in deleted interpreter");
	interp.setErrorCode(TclString.newInstance(
		"CORE IDELETE {attempt to call eval in deleted interpreter}"));
	throw new TclException(TCL.ERROR);
    }

    /*
     * Check depth of nested calls to eval:  if this gets too large,
     * it's probably because of an infinite loop somewhere.
     */

    if (interp.nestLevel >= interp.maxNestingDepth) {
	throw new TclException(interp, 
		"too many nested calls to eval (infinite loop?)");
    }
    interp.nestLevel++;

    try {
	/*
	 * Find the procedure to execute this command. If there isn't one,
	 * then see if there is a command "unknown".  If so, create a new
	 * word array with "unknown" as the first word and the original
	 * command words as arguments.  Then call ourselves recursively
	 * to execute it.
	 */
    
	cmd = interp.getCommand(objv[0].toString());
	if (cmd == null) {
	    newObjv = new TclObject[objv.length + 1];
	    for (i = (objv.length - 1); i >= 0; i--) {
		newObjv[i+1] = objv[i];
	    }
	    newObjv[0] = TclString.newInstance("unknown");
	    newObjv[0].preserve();
	    cmd = interp.getCommand("unknown");
	    if (cmd == null) {
		throw new TclException(interp, "invalid command name \""
			+ objv[0].toString() + "\"");
	    } else {
		evalObjv(interp, newObjv, length, 0);
	    }
	    newObjv[0].release();
	    return;
	}
    
	/*
	 * Finally, invoke the Command's cmdProc.
	 */
	
	interp.cmdCount++;
	savedVarFrame = interp.varFrame;
	if ((flags & TCL_EVAL_GLOBAL) != 0) {
	    interp.varFrame = interp.globalFrame;
	}
	
	cmd.cmdProc(interp, objv);
	interp.varFrame = savedVarFrame;
	
	// TO DO...
	//
	//if (AsyncReady()) {
	//    code = AsyncInvoke(interp, code);
	//}

    } finally {
	interp.nestLevel--;
    }
}

/*
 *----------------------------------------------------------------------
 *
 * logCommandInfo --
 *
 *	This procedure is invoked after an error occurs in an interpreter.
 *	It adds information to the "errorInfo" variable to describe the
 *	command that was being executed when the error occurred.
 *
 * Results:
 *	None.
 *
 * Side effects:
 *	Information about the command is added to errorInfo and the
 *	line number stored internally in the interpreter is set.  If this
 *	is the first call to this procedure or interp.addErrorInfo since
 *	an error occurred, then old information in errorInfo is
 *	deleted.
 *
 *----------------------------------------------------------------------
 */

static void
logCommandInfo(
    Interp interp, 		// Interpreter in which to log information. 
    char[] script_array,        // The script to be logged
    int    script_index,        // First character in script containing
                                // command (must be <= cmdIndex).

    int cmdIndex,		// First character in command that
				// generated the error. 
    int length,			// Number of bytes in command (-1 means
				// use all bytes up to first null byte). 
    TclException e)		// The exception caused by the script 
				// evaluation. 
{
    String ellipsis;
    String msg;
    int offset;
    int pIndex;

    if (interp.errAlreadyLogged) {
	/*
	 * Someone else has already logged error information for this
	 * command; we shouldn't add anything more.
	 */

	return;
    }

    // Compute the line number where the error occurred.
    // Note: The script array must be accessed directly
    // because we want to count from the beginning of
    // the script, not the current index.

    interp.errorLine = 1;

    for (pIndex = 0; pIndex < cmdIndex; pIndex++) {
	if (script_array[pIndex] == '\n') {
	    interp.errorLine++;
	}
    }


    //Create an error message to add to errorInfo, including up to a
    //maximum number of characters of the command.

    if (length < 0) {
        //take into account the trailing '\0'
        int script_length = script_array.length - 1;

	length = script_length - cmdIndex;
	//length = script.length() - cmdIndex;
    }
    if (length > 150) {
	offset = 150;
	ellipsis = "...";
    } else {
	offset = length;
	ellipsis = "";
    }

    msg = new String(script_array, cmdIndex, offset);
    if (!(interp.errInProgress)) {
   	interp.addErrorInfo("\n    while executing\n\"" +
   		msg + ellipsis + "\"");
    } else {
   	interp.addErrorInfo("\n    invoked from within\n\"" +
   		msg + ellipsis + "\"");
    }
    interp.errAlreadyLogged = false;
    e.errIndex = cmdIndex + offset;
}

/*
 *----------------------------------------------------------------------
 *
 * evalTokens --
 *
 *	Given an array of tokens parsed from a Tcl command (e.g., the
 *	tokens that make up a word or the index for an array variable)
 *	this procedure evaluates the tokens and concatenates their
 *	values to form a single result value.
 *
 * Results:
 *	The return value is a pointer to a newly allocated TclObject
 *	containing the value of the array of tokens.  The reference
 *	count of the returned object has been incremented.  If an error
 *	occurs in evaluating the tokens then a TclException is 
 *	generated.
 *
 * Side effects:
 *	A new object is allocated to hold the result.
 *
 *----------------------------------------------------------------------
 */

static TclObject 
evalTokens(
    Interp interp,		// Interpreter in which to lookup
				// variables, execute nested commands,
				// and report errors. 
    TclToken[] tokenList,       // Token list from a TclParse object.
    int tIndex,     		// Index to first token to evaluate
				// and concatenate. 
    int count)			// Number of tokens to consider at tIndex.
				// Must be at least 1. 
throws 
    TclException
{
    TclObject result, index, value;
    TclToken token;
    String p = null;
    String varName;
    BackSlashResult bs;

    /*
     * The only tricky thing about this procedure is that it attempts to
     * avoid object creation and string copying whenever possible.  For
     * example, if the value is just a nested command, then use the
     * command's result object directly.
     */

    result = null;
    for ( ; count > 0; count--) {
	token = tokenList[tIndex];

	/*
	 * The switch statement below computes the next value to be
	 * concat to the result, as either a range of text or an
	 * object.
	 */

	value = null;
	switch (token.type) {
	    case TCL_TOKEN_TEXT:
		p = token.getTokenString();
		break;

	    case TCL_TOKEN_BS:
	        bs = backslash(token.script_array,token.script_index);
		if (bs.isWordSep) {
		    p = "\\" + bs.c;
		} else {
		    Character ch = new Character(bs.c);
		    p = ch.toString();
		}
		break;

	    case TCL_TOKEN_COMMAND:
		interp.evalFlags |= Parser.TCL_BRACKET_TERM;
		token.script_index++;

//should the nest level be changed???
		//interp.nestLevel++;

		eval2(interp, token.script_array, token.script_index,
		      token.size-2, 0);

		token.script_index--;
		//interp.nestLevel--;
		value = interp.getResult();
		break;

	    case TCL_TOKEN_VARIABLE:
		if (token.numComponents == 1) {
		    index = null;
		} else {
		    index = evalTokens(interp, tokenList,
			    tIndex + 2, token.numComponents - 1);
		    if (index == null) {
			return null;
		    }
		}
		varName = tokenList[tIndex + 1].getTokenString();

		/*
		 * In order to get the existing expr parser to work with the
		 * new Parser, we test the interp.noEval flag which is set
		 * by the expr parser.  If it is != 0, then we do not evaluate 
		 * the variable.  This should be removed when the new expr
		 * parser is implemented.
		 */

		if (interp.noEval == 0) {
		    if (index != null) {
			try {
			    value = interp.getVar(varName, index.toString(),0);
			} finally {
			    index.release();
			}
		    } else {
			value = interp.getVar(varName, null, 0);
		    }
		} else {
		    value = TclString.newInstance("");
		    value.preserve();
		}
		count -= token.numComponents;
		tIndex += token.numComponents;
		break;

	    default:
		throw new TclRuntimeError(
		        "unexpected token type in evalTokens");
	}

	/*
	 * If value isn't null, the next piece of text comes from that
	 * object; otherwise, take value of p.
	 */

	if (result == null) {
	    if (value != null) {
		result = value;
	    } else {
		result = TclString.newInstance(p);
	    }
	    result.preserve();
	} else {
	    result = result.takeExclusive();
	    if (value != null) {
		p = value.toString();
	    }
	    TclString.append(result, p);
	}
	tIndex++;
    }
    return result;
}

/*
 *----------------------------------------------------------------------
 *
 * eval2 --
 *
 *	This procedure evaluates a Tcl script without using the compiler
 *	or byte-code interpreter.  It just parses the script, creates
 *	values for each word of each command, then calls evalObjv
 *	to execute each command.
 *
 * Results:
 *	A result or error message is left in interp's result.
 *
 * Side effects:
 *	Depends on the script.
 *
 *----------------------------------------------------------------------
 */

static void
eval2(
    Interp interp,		// Interpreter in which to evaluate the
				// script.  Also used for error reporting. 
    char[] script_array,        //the array of charcters
    int    script_index,        //the starting index into this array

    int numBytes,		// Number of bytes in script.  If < 0, the
				// script consists of all bytes up to the
				// first end of script. 
    int flags)			// Collection of OR-ed bits that control
				// the evaluation of the script.  Only
				// TCL_EVAL_GLOBAL is currently
				// supported. 
throws
    TclException
{
    int i;
    int objUsed = 0;
    int pIndex, nextIndex, wordIndex, tokenIndex;
    int commandLength, bytesLeft;
    boolean nested;
    TclObject[] objv;
    TclObject obj;
    TclParse parse = null;
    TclToken token;

    // Saves old copy of interp.varFrame in case TCL_EVAL_GLOBAL was set
    CallFrame savedVarFrame;

    //take into account the trailing '\0'
    int script_length = script_array.length - 1;


    //these are modified instead of script_array and script_index
    char[] src_array = script_array;
    int    src_index = script_index;

    //System.out.println("call to eval2");

    if (numBytes < 0) {
	numBytes = script_length - script_index;
    }
    interp.resetResult();
    savedVarFrame = interp.varFrame;
    if ((flags & TCL_EVAL_GLOBAL) != 0) {
	interp.varFrame = interp.globalFrame;
    }

    //Each iteration through the following loop parses the next
    //command from the script and then executes it.

    bytesLeft = numBytes;

    //init objv with the most commonly used array size
    objv = grabObjv(interp,3);

    if ((interp.evalFlags & TCL_BRACKET_TERM) != 0) {
	nested = true;
    } else {
	nested = false;
    }
    interp.evalFlags &= ~TCL_BRACKET_TERM;

    try {

    do {
	parse = parseCommand(interp, src_array, src_index,
			     bytesLeft, null, 0, nested);

	if (parse.result != TCL.OK) {
	    throw new TclException(parse.result);
	}

	/*
	 * The test on noEval is temporary.  As soon as the new expr
	 * parser is implemented it should be removed.
	 */

	if (parse.numWords > 0 && interp.noEval == 0) {	    
	    /*
	     * Generate an array of objects for the words of the command.
	     */
	    
	    try {
		tokenIndex = 0;
		token = parse.getToken(tokenIndex);
				
		// Test to see if new space needs to be allocated.  If objv
		// is the EXACT size of parse.numWords, then no allocation
		// needs to be performed.
		 
		if (objv.length != parse.numWords) {
                  //System.out.println("need new size " + objv.length);
		  releaseObjv(interp, objv); //let go of resource
		  objv = grabObjv(interp, parse.numWords); //get new resource
		} else {
		  //System.out.println("reusing size " + objv.length);
 		}

		for (objUsed = 0; objUsed < parse.numWords; objUsed++) {
		    obj = evalTokens(interp, parse.tokenList, 
			    tokenIndex + 1, token.numComponents);
		    if (obj == null) {
			throw new TclException(TCL.ERROR);
		    } else {
		      objv[objUsed] = obj;
                    }
		    tokenIndex += (token.numComponents + 1);
		    token = parse.getToken(tokenIndex);
		}
		
		//Execute the command and free the objects for its words.
		try {
		    evalObjv(interp, objv, /*src,*/ bytesLeft, 0);
		} catch (java.lang.StackOverflowError e) {
		    interp.setResult("too many nested calls" +
			    " to eval (infinite loop?)");
		    throw new TclException(TCL.ERROR); 
		}

	    } catch (TclException e) {
		/*
		 * Generate various pieces of error information, such 
		 * as the line number where the error occurred and 
		 * information to add to the errorInfo variable.  Then 
		 * free resources that had been allocated
		 * to the command.
		 */
		
		if ( e.getCompletionCode()== TCL.ERROR &&
		     !(interp.errAlreadyLogged)) {
		    commandLength = parse.commandSize;
		    if ((parse.commandStart + commandLength) 
			    != (script_index + numBytes)) {
			/*
			 * The command where the error occurred didn't end 
			 * at the end of the script (i.e. it ended at a 
			 * terminator character such as ";".  Reduce the 
			 * length by one so that the error message
			 * doesn't include the terminator character.
			 */
			
			commandLength -= 1;
		    }
		    interp.varFrame = savedVarFrame;
		    logCommandInfo(interp, script_array, script_index,
			    parse.commandStart, commandLength, e);
		}
		throw e;
	    } finally {
		for (i = 0; i < objUsed; i++) {
		    objv[i].release();
		}
                objUsed = 0;

		parse.release(); //cleanup parser resources
	    }
	}


	//Advance to the next command in the script.
	
	nextIndex = parse.commandStart + parse.commandSize;
	bytesLeft -= (nextIndex - src_index);
	src_index = nextIndex;
	if (nested && (src_index > 1) &&
	    (src_array[src_index-1] == ']')) {
	    /*
	     * We get here in the special case where the TCL_BRACKET_TERM
	     * flag was set in the interpreter and we reached a close
	     * bracket in the script.  Return immediately.
	     */

	    interp.termOffset = (src_index - 1) - script_index;
	    interp.varFrame = savedVarFrame;
	    return;
	}
    } while (bytesLeft > 0);

    } finally {
      parse.release(); //let go of parser resources
      releaseObjv(interp, objv); //let go of objv buffer
    }

    interp.termOffset = src_index - script_index;
    interp.varFrame = savedVarFrame;
    return;
}

/*
 *----------------------------------------------------------------------
 *
 * parseVarName --
 *
 *	Given a string starting with a $ sign, parse off a variable
 *	name and return information about the parse.
 *
 * Results:
 *	The return value is TclParse object that contains information 
 *	about the parsed command.  If the command was parsed 
 *	successfully, the TclParse object's result variable is set to
 *	TCL_OK and TCL_ERROR otherwise.  If an error occurs and interp
 *	isn't null then an error message is left in its result. 
 *	On a successful return, tokenList and numTokens fields of
 *	TclParse are filled in with information about the variable name
 *	that was parsed.  The "size" field of the first new token gives
 *	the total number of bytes in the variable name.  Other fields in
 *	TclParse are undefined.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */


static TclParse
parseVarName(
    Interp interp,		// Interpreter to use for error reporting;
				// if NULL, then no error message is
				// provided. 

    char[]      script_array,   // String containing variable name.  First
    int         script_index,   // character must be "$".

    int numBytes,		// Total number of bytes in string.  If < 0,
				// the string consists of all bytes up to the
				// first null character. 
    TclParse parse, 		// Structure to fill in with information
				// about the variable name. 
    boolean append)		// Non-zero means append tokens to existing
				// information in parsePtr; zero means ignore
				// existing tokens in parsePtr and reinitialize
				// it. 
{
    char cur;
    TclToken token, startToken;
    int endIndex,  varIndex;


    if (numBytes >= 0) {
	endIndex = script_index + numBytes;
    } else {
	endIndex = script_array.length - 1;
    }
    if (!append) {
	parse = new TclParse(interp, script_array, endIndex, null, -1);
    }

    /*
     * Generate one token for the variable, an additional token for the
     * name, plus any number of additional tokens for the index, if
     * there is one.
     */

    token = parse.getToken(parse.numTokens);
    token.type = TCL_TOKEN_VARIABLE;
    token.script_array = script_array;
    token.script_index = script_index;
    varIndex = parse.numTokens;
    parse.numTokens++;
    script_index++;
    if (script_index >= endIndex) {
	/*
	 * The dollar sign isn't followed by a variable name.
	 * replace the TCL_TOKEN_VARIABLE token with a
	 * TCL_TOKEN_TEXT token for the dollar sign.
	 */

	token.type = TCL_TOKEN_TEXT;
	token.size = 1;
	token.numComponents = 0;
	parse.result = TCL.OK;
	return parse;
    }
    startToken = token;
    token = parse.getToken(parse.numTokens);

    /*
     * The name of the variable can have three forms:
     * 1. The $ sign is followed by an open curly brace.  Then 
     *    the variable name is everything up to the next close
     *    curly brace, and the variable is a scalar variable.
     * 2. The $ sign is not followed by an open curly brace.  Then
     *    the variable name is everything up to the next
     *    character that isn't a letter, digit, or underscore.
     *    :: sequences are also considered part of the variable
     *    name, in order to support namespaces. If the following
     *    character is an open parenthesis, then the information
     *    between parentheses is the array element name.
     * 3. The $ sign is followed by something that isn't a letter,
     *    digit, or underscore:  in this case, there is no variable
     *    name and the token is just "$".
     */

    if (script_array[script_index] == '{') {
	script_index++;
	token.type = TCL_TOKEN_TEXT;
	token.script_array = script_array;
	token.script_index = script_index;
	token.numComponents = 0;

	while (true) {
	    if (script_index == endIndex) {
		if (interp != null) {
		    interp.setResult("missing close-brace for variable name");
		}
		parse.termIndex = token.script_index - 1;
		parse.incomplete = true;
		parse.result = TCL.ERROR;
		return parse;
	    }
	    if (script_array[script_index] == '}') {
		break;
	    }
	    script_index++;
	}
	token.size = script_index - token.script_index;
	startToken.size = script_index - startToken.script_index;
	parse.numTokens++;
	script_index++;
    } else {
	token.type = TCL_TOKEN_TEXT;
	token.script_array = script_array;
	token.script_index = script_index;
	token.numComponents = 0;
	while (script_index != endIndex) {
	    cur = script_array[script_index];
	    if ((Character.isLetterOrDigit(cur)) || 
		    (cur == '_')) {
		script_index++;
		continue;
	    }
	    if ((cur == ':') && 
		(((script_index + 1) != endIndex) && (script_array[script_index + 1] == ':'))) {
		script_index += 2;
		while ((script_index != endIndex) && (script_array[script_index] == ':')) {
		    script_index++;
		}
		continue;
	    }
	    break;
	}
	token.size = script_index - token.script_index;
	if (token.size == 0) {
	    /*
	     * The dollar sign isn't followed by a variable name.
	     * replace the TCL_TOKEN_VARIABLE token with a
	     * TCL_TOKEN_TEXT token for the dollar sign.
	     */
	    
	    startToken.type = TCL_TOKEN_TEXT;
	    startToken.size = 1;
	    startToken.numComponents = 0;
	    parse.result = TCL.OK;
	    return parse;
	}
	parse.numTokens++;
	if ((script_index != endIndex) && (script_array[script_index] == '(')) {
	    /*
	     * This is a reference to an array element.  Call
	     * parseTokens recursively to parse the element name,
	     * since it could contain any number of substitutions.
	     */

	    script_index++;
	    parse = parseTokens(script_array,script_index, TYPE_CLOSE_PAREN, parse);
	    if (parse.result != TCL.OK) {
		return parse;
	    }
	    if ((parse.termIndex == endIndex) || 
		    (parse.string[parse.termIndex] != ')')) { 
		if (interp != null) {
		    interp.setResult("missing )");
		}
		parse.termIndex = script_index - 1;
		parse.incomplete = true;
		parse.result = TCL.ERROR;
		return parse;
	    }
	    script_index = parse.termIndex + 1;
	}
    }

    startToken.size = script_index - startToken.script_index;
    startToken.numComponents = parse.numTokens - (varIndex + 1);
    parse.result = TCL.OK;
    return parse;
}




/*
 *----------------------------------------------------------------------
 *
 * parseVar --
 *
 *	Given a string starting with a $ sign, parse off a variable
 *	name and return its value.
 *
 * Results:
 *	The return value is a ParseResult object that contains the 
 *	contents of the variable given by the leading characters of
 *	string and a index to the character just after the last one
 *	in the variable specifier.  If the variable doesn't exist, 
 *	then a TclException is generated.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static ParseResult
parseVar(
    Interp interp,			// Context for looking up variable.
    String string)			// String containing variable name.
					// First character must be "$".
throws
    TclException
{
    TclParse parse;
    TclObject obj;
    CharPointer src = new CharPointer(string);
    parse = parseVarName(interp, src.array, src.index, -1, null, false);

    try {
      if (parse.numTokens == 1) {
        //There isn't a variable name after all: the $ is just a $.
	return new ParseResult("$", 1);
      }

      obj = evalTokens(interp, parse.tokenList, 0, parse.numTokens);
      if (!obj.isShared()) {
	throw new TclRuntimeError(
	    "parseVar got temporary object from evalTokens");
      }
      obj.release();

      return new ParseResult(obj, parse.tokenList[0].size);
    } finally {
      parse.release();  //release parser resources
    }
}

/*
 *----------------------------------------------------------------------
 *
 * commandComplete --
 *
 *	This procedure is shared by interp.commandComplete and
 *	objCommandComplete; it does all the real work of seeing
 *	whether a script is complete.
 *
 * Results:
 *	True is returned if the script is complete, false if there are
 *	open delimiters such as " or (. True is also returned if there
 *	is a parse error in the script other than unmatched delimiters.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static boolean
commandComplete(
    String string,			// Script to check.
    int length)				// Number of bytes in script.
{
    TclParse parse;
    String p;

    CharPointer src = new CharPointer(string);
    int end = (src.index + length);
    
    do {
	parse = parseCommand(null, src.array, src.index, end-src.index, 
		null, 0, false);
	src.index = parse.commandStart + parse.commandSize;
	if (src.index >= end) {
	    break;
	}
	src.index++;
    } while (parse.result == TCL.OK);

    parse.release(); //release parser resources

    if (parse.incomplete) {
	return false;
    }
    return true;
}

/*
 *----------------------------------------------------------------------
 *
 * objCommandComplete --
 *
 *	Given a partial or complete Tcl command in a Tcl object, this
 *	procedure determines whether the command is complete in the 
 *	sense of having matched braces and quotes and brackets.
 *
 * Results:
 *	True is returned if the command is complete, false otherwise.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static boolean
objCommandComplete(
    TclObject obj)			// Points to object holding script
					// to check.
{
    String string = obj.toString();
    return commandComplete(string, string.length());
}

/*
 *----------------------------------------------------------------------
 *
 * backslash --
 *
 *	Figure out how to handle a backslash sequence.  The index
 *	of the script must be pointing to the first /.
 *
 * Results:
 *	The return value is an instance of BackSlashResult that 
 *	contains the character that should be substituted in place 
 *	of the backslash sequence that starts at src.index, and
 *	an index to the next character after the backslash sequence.
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static BackSlashResult
backslash(
    char[] script_array,
    int script_index)
{
    int result;

    script_index++;
    int endIndex = script_array.length - 1;

    if (script_index == endIndex) {
	return new BackSlashResult('\\', script_index);
    }

    char c = script_array[script_index];
    switch (c) {
    case 'a':
	return new BackSlashResult((char)0x7, script_index+1);
    case 'b':
	return new BackSlashResult((char)0x8, script_index+1);
    case 'f':
	return new BackSlashResult((char)0xc, script_index+1);
    case 'n':
	return new BackSlashResult('\n', script_index+1);
    case 'r':
	return new BackSlashResult('\r', script_index+1);
    case 't':
	return new BackSlashResult('\t', script_index+1);
    case 'v':
	return new BackSlashResult((char)0xb, script_index+1);
    case 'x':
	script_index++;
	if (script_index < endIndex) {
	    c = script_array[script_index];

	    if (((c >= '0') && (c <= '9')) ||
		    ((c >= 'A') && (c <= 'F')) ||
		    ((c >= 'a') && (c <= 'f'))) {
		
		String str = new String(script_array, script_index, 
			endIndex - script_index);
		StrtoulResult res = Util.strtoul(str, 0, 16);
		if (res.errno == 0) {
		    /*
		     * We force res.value to be a 8-bit (ASCII) character
		     * so that it compatible with Tcl 7.6.
		     */
		    
		    byte b = (byte)(res.value & 0xff);
		    return new BackSlashResult((char)(b), script_index + 
			    res.index);
		}
	    }
	}
	return new BackSlashResult('x', script_index);
    case 'u':
	    int count, n;
	    result = 0;
	    for (count = 0; count < 4; count++) {
		script_index++;
		c = script_array[script_index];
		if (((c >= '0') && (c <= '9'))
			|| ((c >= 'a') && (c <= 'f'))
			|| ((c >= 'A') && (c <= 'F'))) {
		    n = c - '0';
		    if (n > 9) {
			n = n + '0' + 10 - 'A';
		    }
		    if (n > 16) {
			n = n + 'A' - 'a';
		    }
		    result = (result << 4) + n;
		} else {
		    break;
		}
	    }
	    if (count == 0) {
		result = 'u';
	    }
	    return new BackSlashResult((char)result, script_index);
    case '\r':
    case '\n':
	if (c == '\r') {
	    if ((script_index + 1) < endIndex) {
		if (script_array[script_index + 1] == '\n') {
		    script_index++;
		}
	    }
	}
	do {
	    script_index++;
	    c = script_array[script_index];
	} while ((script_index < endIndex) && 
		((c == ' ') || (c == '\t')));
	return new BackSlashResult((char)' ', script_index);
    case 0:
	return new BackSlashResult((char)'\\', script_index+1);
    default:
	if ((c >= '0') && (c <= '9')) {
	    /*
	     * Convert it to an octal number. This implementation is
	     * compatible with tcl 7.6 - characters 8 and 9 are allowed.
	     */
	    
	    result = c - '0';
	    script_index++;
	    
	getoctal: {
		if (script_index == endIndex) {
		    break getoctal;
		}
		c = script_array[script_index];
		if (!((c >= '0') && (c <= '9'))) {
		    break getoctal;
		}
		result = (result * 8) + (c - '0');
		script_index++;
		
		if (script_index == endIndex) {
		    break getoctal;
		}
		c = script_array[script_index];
		if (!((c >= '0') && (c <= '9'))) {
		    break getoctal;
		}
		result = (result * 8) + (c - '0');
		script_index++;
	    }
	    
	    /*
	     * We force result to be a 8-bit (ASCII) character so
	     * that it compatible with Tcl 7.6.
	     */
	    
	    return new BackSlashResult((char)(result & 0xff), script_index);
	} else {
	    return new BackSlashResult(c, script_index+1);
	}
    }
}







/*
 *----------------------------------------------------------------------
 *
 * charType --
 *
 *	Looks into the typeTable to determine the character type. 
 * 	Possible types are TYPE_NORMAL, TYPE_SPACE, TYPE_COMMAND_END,
 * 	TYPE_SUBS, etc.  See below for more detail.
 *
 * Results:
 *	A char that specifies the character type
 *
 * Side effects:
 *	None.
 *
 *----------------------------------------------------------------------
 */

static char 
charType(
    char c)
{
    return (typeTable[c]);
}

/*
 * The following table provides parsing information about each possible
 * character.  
 *
 * The method charType is used to index into the table and return
 * information about its character argument.  The following return
 * values are defined.
 *
 * TYPE_NORMAL -	All characters that don't have special significance
 *			to the Tcl parser.
 * TYPE_SPACE -		The character is a whitespace character other
 *			than newline.
 * TYPE_COMMAND_END -	Character is newline or semicolon.
 * TYPE_SUBS -		Character begins a substitution or has other
 *			special meaning in parseTokens: backslash, dollar
 *			sign, open bracket, or null.
 * TYPE_QUOTE -		Character is a double quote.
 * TYPE_CLOSE_PAREN -	Character is a right parenthesis.
 * TYPE_CLOSE_BRACK -	Character is a right square bracket.
 * TYPE_BRACE -		Character is a curly brace (either left or right).
 */

static final char TYPE_NORMAL		= 0;
static final char TYPE_SPACE		= 0x1;
static final char TYPE_COMMAND_END	= 0x2;
static final char TYPE_SUBS		= 0x4;
static final char TYPE_QUOTE		= 0x8;
static final char TYPE_CLOSE_PAREN	= 0x10;
static final char TYPE_CLOSE_BRACK	= 0x20;
static final char TYPE_BRACE		= 0x40;


static char typeTable[] = {
    /*
     * Character values, from 0-127:
     */

    TYPE_SUBS,        TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_SPACE,       TYPE_COMMAND_END, TYPE_SPACE,
    TYPE_SPACE,       TYPE_SPACE,       TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_SPACE,       TYPE_NORMAL,      TYPE_QUOTE,       TYPE_NORMAL,
    TYPE_SUBS,        TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_CLOSE_PAREN, TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_COMMAND_END,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_SUBS,
    TYPE_SUBS,        TYPE_CLOSE_BRACK, TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_BRACE,
    TYPE_NORMAL,      TYPE_BRACE,       TYPE_NORMAL,      TYPE_NORMAL,

    /*
     * Character values, from 128-255:
     */

    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
    TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,      TYPE_NORMAL,
};

/*
 * Type values defined for TclToken structures.  These values are
 * defined as mask bits so that it's easy to check for collections of
 * types.
 *
 * TCL_TOKEN_WORD -		The token describes one word of a command,
 *				from the first non-blank character of
 *				the word (which may be " or {) up to but
 *				not including the space, semicolon, or
 *				bracket that terminates the word. 
 *				NumComponents counts the total number of
 *				sub-tokens that make up the word.  This
 *				includes, for example, sub-tokens of
 *				TCL_TOKEN_VARIABLE tokens.
 * TCL_TOKEN_SIMPLE_WORD -	This token is just like TCL_TOKEN_WORD
 *				except that the word is guaranteed to
 *				consist of a single TCL_TOKEN_TEXT
 *				sub-token.
 * TCL_TOKEN_TEXT -		The token describes a range of literal
 *				text that is part of a word. 
 *				NumComponents is always 0.
 * TCL_TOKEN_BS -		The token describes a backslash sequence
 *				that must be collapsed.  NumComponents
 *				is always 0.
 * TCL_TOKEN_COMMAND -		The token describes a command whose result
 *				must be substituted into the word.  The
 *				token includes the enclosing brackets. 
 *				NumComponents is always 0.
 * TCL_TOKEN_VARIABLE -		The token describes a variable
 *				substitution, including the dollar sign,
 *				variable name, and array index (if there
 *				is one) up through the right
 *				parentheses.  NumComponents tells how
 *				many additional tokens follow to
 *				represent the variable name.  The first
 *				token will be a TCL_TOKEN_TEXT token
 *				that describes the variable name.  If
 *				the variable is an array reference then
 *				there will be one or more additional
 *				tokens, of type TCL_TOKEN_TEXT,
 *				TCL_TOKEN_BS, TCL_TOKEN_COMMAND, and
 *				TCL_TOKEN_VARIABLE, that describe the
 *				array index; numComponents counts the
 *				total number of nested tokens that make
 *				up the variable reference, including
 *				sub-tokens of TCL_TOKEN_VARIABLE tokens.
 */

static final int TCL_TOKEN_WORD		= 1;
static final int TCL_TOKEN_SIMPLE_WORD	= 2;
static final int TCL_TOKEN_TEXT		= 4;
static final int TCL_TOKEN_BS		= 8;
static final int TCL_TOKEN_COMMAND	= 16;
static final int TCL_TOKEN_VARIABLE	= 32;


/*
 * Note: Most of the variables below will not be used until the
 * Compilier is implemented, but are left for consistency.
 */


/*
 * Flag values passed to recordAndEval and/or evalObj.
 * WARNING: these bit choices must not conflict with the bit choices
 * for evalFlag bits in tclInt.h!!
 */

static final int TCL_NO_EVAL		= 0x10000;
static final int TCL_EVAL_GLOBAL	= 0x20000;
static final int TCL_EVAL_DIRECT	= 0x40000;

/*
 * A structure of the following type is filled in by parseCommand.
 * It describes a single command parsed from an input string.
 */

/*
 * evalFlag bits for Interp structures:
 *
 * TCL_BRACKET_TERM	1 means that the current script is terminated by
 *			a close bracket rather than the end of the string.
 * TCL_ALLOW_EXCEPTIONS	1 means it's OK for the script to terminate with
 *			a code other than TCL_OK or TCL_ERROR;  0 means
 *			codes other than these should be turned into errors.
 */

static final int TCL_BRACKET_TERM	  = 1;
static final int TCL_ALLOW_EXCEPTIONS	  = 4;

/*
 * Flag bits for Interp structures:
 *
 * DELETED:		Non-zero means the interpreter has been deleted:
 *			don't process any more commands for it, and destroy
 *			the structure as soon as all nested invocations of
 *			Tcl_Eval are done.
 * ERR_IN_PROGRESS:	Non-zero means an error unwind is already in
 *			progress. Zero means a command proc has been
 *			invoked since last error occured.
 * ERR_ALREADY_LOGGED:	Non-zero means information has already been logged
 *			in $errorInfo for the current Tcl_Eval instance,
 *			so Tcl_Eval needn't log it (used to implement the
 *			"error message log" command).
 * ERROR_CODE_SET:	Non-zero means that Tcl_SetErrorCode has been
 *			called to record information for the current
 *			error.  Zero means Tcl_Eval must clear the
 *			errorCode variable if an error is returned.
 * EXPR_INITIALIZED:	Non-zero means initialization specific to
 *			expressions has	been carried out.
 * DONT_COMPILE_CMDS_INLINE: Non-zero means that the bytecode compiler
 *			should not compile any commands into an inline
 *			sequence of instructions. This is set 1, for
 *			example, when command traces are requested.
 * RAND_SEED_INITIALIZED: Non-zero means that the randSeed value of the
 *			interp has not be initialized.  This is set 1
 *			when we first use the rand() or srand() functions.
 * SAFE_INTERP:         Non zero means that the current interp is a
 *                      safe interp (ie it has only the safe commands
 *                      installed, less priviledge than a regular interp).
 * USE_EVAL_DIRECT:	Non-zero means don't use the compiler or byte-code
 *			interpreter; instead, have Tcl_EvalObj call
 *			Tcl_EvalDirect.  Used primarily for testing the
 *			new parser.
 */

static final int DELETED			   = 1;
static final int ERR_IN_PROGRESS		   = 2;
static final int ERR_ALREADY_LOGGED		   = 4;
static final int ERROR_CODE_SET			   = 8;
static final int EXPR_INITIALIZED		= 0x10;
static final int DONT_COMPILE_CMDS_INLINE	= 0x20;
static final int RAND_SEED_INITIALIZED		= 0x40;
static final int SAFE_INTERP              	= 0x80;
static final int USE_EVAL_DIRECT		= 0x100;






//these are private read only values that are used by the parser
//class to implement a TclObject[] cache

//max size of array to cache (1..N)
private static final int OBJV_CACHE_MAX = 10;

//the number of array to cache for each size
//for example if the number of 3 elements is set to 5
//an array of 5 TclObject[] objects
//which will each be 3 elements long

private static final int[] OBJV_CACHE_SIZES = {0,4,4,10,4,4,4,4,4,4};

//use test results
//1 373
//2 2424
//3 11889
//4 840
//5 1374
//6 926
//7 0
//8 74
//9 0


static void init(Interp interp) {
  //System.out.println("called Parser.init()");

  TclObject[][][] OBJV  = new TclObject[OBJV_CACHE_MAX][][];
  int[] USED = new int[OBJV_CACHE_MAX];

  int i,j,size;

  for (i=0; i < OBJV_CACHE_MAX; i++) {
    size = OBJV_CACHE_SIZES[i];
    //System.out.println("size " + i + " has " + size + " cache blocks");
    OBJV[i] = new TclObject[size][];
    USED[i] = 0;
    for (j=0;j<size;j++) {
      OBJV[i][j] = new TclObject[i];
    }
  }

  interp.parserObjv = OBJV;
  interp.parserObjvUsed = USED;
}


private static TclObject[] grabObjv(Interp interp, int size) {

  if (size >= OBJV_CACHE_MAX) {
    //System.out.println("allocate for big objv of size " + size);
    return new TclObject[size];
  }

  //get array of used markers for this size
  final int OPEN = interp.parserObjvUsed[size];

  if (OPEN < OBJV_CACHE_SIZES[size]) {
    //we found an open cache slot
    //System.out.println("cache hit for objv of size " + size);
    interp.parserObjvUsed[size] += 1;
    return interp.parserObjv[size][OPEN];
  } else {
    //we did not find a free cache array of this size
    //System.out.println("cache miss for objv of size " + size);
    return new TclObject[size];
  }

}


private static void releaseObjv(Interp interp, TclObject[] objv) { 
  final int size = objv.length;
  
  if (size >= OBJV_CACHE_MAX) {
    //System.out.println("release for big objv of size " + size);
    return;
  }

  int OPEN = interp.parserObjvUsed[size];

  if (OPEN > 0) {
    OPEN--;
    interp.parserObjvUsed[size] = OPEN;
    interp.parserObjv[size][OPEN] = objv;
    //System.out.println("released objv of size " + size);
  }
  /*
  else {
    System.out.println("no release for objv of size " + size);
  }
  */

  return;
}


} // end class Parser