Coverage Report

Coverage Report - org.kuali.rice.krad.uif.util.Tokenizer

Classes in this File

Line Coverage

Branch Coverage

Complexity

Tokenizer

0/248

0/279

4.103

Tokenizer$Token

0/22

0/16

4.103

Tokenizer$TokenKind

0/20

0/4

4.103

 /**
  * Copyright 2005-2011 The Kuali Foundation
  *
  * Licensed under the Educational Community License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.opensource.org/licenses/ecl2.php
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.kuali.rice.krad.uif.util;
 
 import org.springframework.expression.spel.InternalParseException;
 import org.springframework.expression.spel.SpelMessage;
 import org.springframework.expression.spel.SpelParseException;
 import org.springframework.util.Assert;
 
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
 /**
  * @author Kuali Rice Team (rice.collab@kuali.org)
  */
 public class Tokenizer {
     String expressionString;
     char[] toProcess;
     int pos;
     int max;
     List<Token> tokens = new ArrayList<Token>();
 
     protected Tokenizer(String inputdata) {
         for (int ch = '0'; ch <= '9'; ch++) {
             flags[ch] |= IS_DIGIT | IS_HEXDIGIT;
         }
         for (int ch = 'A'; ch <= 'F'; ch++) {
             flags[ch] |= IS_HEXDIGIT;
         }
         for (int ch = 'a'; ch <= 'f'; ch++) {
             flags[ch] |= IS_HEXDIGIT;
         }
         for (int ch = 'A'; ch <= 'Z'; ch++) {
             flags[ch] |= IS_ALPHA;
         }
         for (int ch = 'a'; ch <= 'z'; ch++) {
             flags[ch] |= IS_ALPHA;
         }
 
         this.expressionString = inputdata;
         this.toProcess = (inputdata + "\0").toCharArray();
         this.max = toProcess.length;
         this.pos = 0;
         process();
     }
 
     public void process() {
         while (pos < max) {
             char ch = toProcess[pos];
             if (isAlphabetic(ch)) {
                 lexIdentifier();
             } else {
                 switch (ch) {
                     case '+':
                         pushCharToken(TokenKind.PLUS);
                         break;
                     case '_': // the other way to start an identifier
                         lexIdentifier();
                         break;
                     case '-':
                         pushCharToken(TokenKind.MINUS);
                         break;
                     case ':':
                         pushCharToken(TokenKind.COLON);
                         break;
                     case '.':
                         pushCharToken(TokenKind.DOT);
                         break;
                     case ',':
                         pushCharToken(TokenKind.COMMA);
                         break;
                     case '*':
                         pushCharToken(TokenKind.STAR);
                         break;
                     case '/':
                         pushCharToken(TokenKind.DIV);
                         break;
                     case '%':
                         pushCharToken(TokenKind.MOD);
                         break;
                     case '(':
                         pushCharToken(TokenKind.LPAREN);
                         break;
                     case ')':
                         pushCharToken(TokenKind.RPAREN);
                         break;
                     case '[':
                         pushCharToken(TokenKind.LSQUARE);
                         break;
                     case '#':
                         pushCharToken(TokenKind.HASH);
                         break;
                     case ']':
                         pushCharToken(TokenKind.RSQUARE);
                         break;
                     case '{':
                         pushCharToken(TokenKind.LCURLY);
                         break;
                     case '}':
                         pushCharToken(TokenKind.RCURLY);
                         break;
                     case '@':
                         pushCharToken(TokenKind.BEAN_REF);
                         break;
                     case '^':
                         if (isTwoCharToken(TokenKind.SELECT_FIRST)) {
                             pushPairToken(TokenKind.SELECT_FIRST);
                         } else {
                             pushCharToken(TokenKind.POWER);
                         }
                         break;
                     case '!':
                         if (isTwoCharToken(TokenKind.NE)) {
                             pushPairToken(TokenKind.NE);
                         } else if (isTwoCharToken(TokenKind.PROJECT)) {
                             pushPairToken(TokenKind.PROJECT);
                         } else {
                             pushCharToken(TokenKind.NOT);
                         }
                         break;
                     case '=':
                         if (isTwoCharToken(TokenKind.EQ)) {
                             pushPairToken(TokenKind.EQ);
                         } else {
                             pushCharToken(TokenKind.ASSIGN);
                         }
                         break;
                     case '?':
                         if (isTwoCharToken(TokenKind.SELECT)) {
                             pushPairToken(TokenKind.SELECT);
                         } else if (isTwoCharToken(TokenKind.ELVIS)) {
                             pushPairToken(TokenKind.ELVIS);
                         } else if (isTwoCharToken(TokenKind.SAFE_NAVI)) {
                             pushPairToken(TokenKind.SAFE_NAVI);
                         } else {
                             pushCharToken(TokenKind.QMARK);
                         }
                         break;
                     case '$':
                         if (isTwoCharToken(TokenKind.SELECT_LAST)) {
                             pushPairToken(TokenKind.SELECT_LAST);
                         } else {
                             lexIdentifier();
                         }
                         break;
                     case '>':
                         if (isTwoCharToken(TokenKind.GE)) {
                             pushPairToken(TokenKind.GE);
                         } else {
                             pushCharToken(TokenKind.GT);
                         }
                         break;
                     case '<':
                         if (isTwoCharToken(TokenKind.LE)) {
                             pushPairToken(TokenKind.LE);
                         } else {
                             pushCharToken(TokenKind.LT);
                         }
                         break;
                     case '0':
                     case '1':
                     case '2':
                     case '3':
                     case '4':
                     case '5':
                     case '6':
                     case '7':
                     case '8':
                     case '9':
                         lexNumericLiteral(ch == '0');
                         break;
                     case ' ':
                     case '\t':
                     case '\r':
                     case '\n':
                         // drift over white space
                         pos++;
                         break;
                     case '\'':
                         lexQuotedStringLiteral();
                         break;
                     case '"':
                         lexDoubleQuotedStringLiteral();
                         break;
                     case 0:
                         // hit sentinel at end of value
                         pos++; // will take us to the end
                         break;
                     default:
                         throw new IllegalStateException("Cannot handle ("
                                 + Integer.valueOf(ch)
                                 + ") '"
                                 + ch
                                 + "', in expression: "
                                 + expressionString);
                 }
             }
         }
     }
 
     public List<Token> getTokens() {
         return tokens;
     }
 
     // STRING_LITERAL: '\''! (APOS|~'\'')* '\''!;
     private void lexQuotedStringLiteral() {
         int start = pos;
         boolean terminated = false;
         while (!terminated) {
             pos++;
             char ch = toProcess[pos];
             if (ch == '\'') {
                 // may not be the end if the char after is also a '
                 if (toProcess[pos + 1] == '\'') {
                     pos++; // skip over that too, and continue
                 } else {
                     terminated = true;
                 }
             }
             if (ch == 0) {
                 throw new InternalParseException(new SpelParseException(expressionString, start,
                         SpelMessage.NON_TERMINATING_QUOTED_STRING));
             }
         }
         pos++;
         tokens.add(new Token(TokenKind.LITERAL_STRING, subarray(start, pos), start, pos));
     }
 
     // DQ_STRING_LITERAL:        '"'! (~'"')* '"'!;
     private void lexDoubleQuotedStringLiteral() {
         int start = pos;
         boolean terminated = false;
         while (!terminated) {
             pos++;
             char ch = toProcess[pos];
             if (ch == '"') {
                 terminated = true;
             }
             if (ch == 0) {
                 throw new InternalParseException(new SpelParseException(expressionString, start,
                         SpelMessage.NON_TERMINATING_DOUBLE_QUOTED_STRING));
             }
         }
         pos++;
         tokens.add(new Token(TokenKind.LITERAL_STRING, subarray(start, pos), start, pos));
     }
 
     //        REAL_LITERAL :
     //          ('.' (DECIMAL_DIGIT)+ (EXPONENT_PART)? (REAL_TYPE_SUFFIX)?) |
     //                ((DECIMAL_DIGIT)+ '.' (DECIMAL_DIGIT)+ (EXPONENT_PART)? (REAL_TYPE_SUFFIX)?) |
     //                ((DECIMAL_DIGIT)+ (EXPONENT_PART) (REAL_TYPE_SUFFIX)?) |
     //                ((DECIMAL_DIGIT)+ (REAL_TYPE_SUFFIX));
     //        fragment INTEGER_TYPE_SUFFIX : ( 'L' | 'l' );
     //        fragment HEX_DIGIT : '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9'|'A'|'B'|'C'|'D'|'E'|'F'|'a'|'b'|'c'|'d'|'e'|'f';
     //
     //        fragment EXPONENT_PART : 'e'  (SIGN)*  (DECIMAL_DIGIT)+ | 'E'  (SIGN)*  (DECIMAL_DIGIT)+ ;
     //        fragment SIGN :        '+' | '-' ;
     //        fragment REAL_TYPE_SUFFIX : 'F' | 'f' | 'D' | 'd';
     //        INTEGER_LITERAL
     //        : (DECIMAL_DIGIT)+ (INTEGER_TYPE_SUFFIX)?;
 
     private void lexNumericLiteral(boolean firstCharIsZero) {
         boolean isReal = false;
         int start = pos;
         char ch = toProcess[pos + 1];
         boolean isHex = ch == 'x' || ch == 'X';
 
         // deal with hexadecimal
         if (firstCharIsZero && isHex) {
             pos = pos + 1;
             do {
                 pos++;
             } while (isHexadecimalDigit(toProcess[pos]));
             if (isChar('L', 'l')) {
                 pushHexIntToken(subarray(start + 2, pos), true, start, pos);
                 pos++;
             } else {
                 pushHexIntToken(subarray(start + 2, pos), false, start, pos);
             }
             return;
         }
 
         // real numbers must have leading digits
 
         // Consume first part of number
         do {
             pos++;
         } while (isDigit(toProcess[pos]));
 
         // a '.' indicates this number is a real
         ch = toProcess[pos];
         if (ch == '.') {
             isReal = true;
             // carry on consuming digits
             do {
                 pos++;
             } while (isDigit(toProcess[pos]));
         }
 
         int endOfNumber = pos;
 
         // Now there may or may not be an exponent
 
         // is it a long ?
         if (isChar('L', 'l')) {
             if (isReal) { // 3.4L - not allowed
                 throw new InternalParseException(new SpelParseException(expressionString, start,
                         SpelMessage.REAL_CANNOT_BE_LONG));
             }
             pushIntToken(subarray(start, endOfNumber), true, start, endOfNumber);
             pos++;
         } else if (isExponentChar(toProcess[pos])) {
             isReal = true; // if it wasnt before, it is now
             pos++;
             char possibleSign = toProcess[pos];
             if (isSign(possibleSign)) {
                 pos++;
             }
 
             // exponent digits
             do {
                 pos++;
             } while (isDigit(toProcess[pos]));
             boolean isFloat = false;
             if (isFloatSuffix(toProcess[pos])) {
                 isFloat = true;
                 endOfNumber = ++pos;
             } else if (isDoubleSuffix(toProcess[pos])) {
                 endOfNumber = ++pos;
             }
             pushRealToken(subarray(start, pos), isFloat, start, pos);
         } else {
             ch = toProcess[pos];
             boolean isFloat = false;
             if (isFloatSuffix(ch)) {
                 isReal = true;
                 isFloat = true;
                 endOfNumber = ++pos;
             } else if (isDoubleSuffix(ch)) {
                 isReal = true;
                 endOfNumber = ++pos;
             }
             if (isReal) {
                 pushRealToken(subarray(start, endOfNumber), isFloat, start, endOfNumber);
             } else {
                 pushIntToken(subarray(start, endOfNumber), false, start, endOfNumber);
             }
         }
     }
 
     // if this is changed, it must remain sorted
     private String[] alternativeOperatorNames = {"DIV", "EQ", "GE", "GT", "LE", "LT", "MOD", "NE", "NOT"};
 
     private void lexIdentifier() {
         int start = pos;
         do {
             pos++;
         } while (isIdentifier(toProcess[pos]));
         char[] subarray = subarray(start, pos);
 
         // Check if this is the alternative (textual) representation of an operator (see alternativeOperatorNames)
         if ((pos - start) == 2 || (pos - start) == 3) {
             String asString = new String(subarray).toUpperCase();
             int idx = Arrays.binarySearch(alternativeOperatorNames, asString);
             if (idx >= 0) {
                 pushOneCharOrTwoCharToken(TokenKind.valueOf(asString), start);
                 return;
             }
         }
         tokens.add(new Token(TokenKind.IDENTIFIER, subarray, start, pos));
     }
 
     private void pushIntToken(char[] data, boolean isLong, int start, int end) {
         if (isLong) {
             tokens.add(new Token(TokenKind.LITERAL_LONG, data, start, end));
         } else {
             tokens.add(new Token(TokenKind.LITERAL_INT, data, start, end));
         }
     }
 
     private void pushHexIntToken(char[] data, boolean isLong, int start, int end) {
         if (data.length == 0) {
             if (isLong) {
                 throw new InternalParseException(new SpelParseException(expressionString, start, SpelMessage.NOT_A_LONG,
                         expressionString.substring(start, end + 1)));
             } else {
                 throw new InternalParseException(new SpelParseException(expressionString, start,
                         SpelMessage.NOT_AN_INTEGER, expressionString.substring(start, end)));
             }
         }
         if (isLong) {
             tokens.add(new Token(TokenKind.LITERAL_HEXLONG, data, start, end));
         } else {
             tokens.add(new Token(TokenKind.LITERAL_HEXINT, data, start, end));
         }
     }
 
     private void pushRealToken(char[] data, boolean isFloat, int start, int end) {
         if (isFloat) {
             tokens.add(new Token(TokenKind.LITERAL_REAL_FLOAT, data, start, end));
         } else {
             tokens.add(new Token(TokenKind.LITERAL_REAL, data, start, end));
         }
     }
 
     private char[] subarray(int start, int end) {
         char[] result = new char[end - start];
         System.arraycopy(toProcess, start, result, 0, end - start);
         return result;
     }
 
     /**
      * Check if this might be a two character token.
      */
     private boolean isTwoCharToken(TokenKind kind) {
         Assert.isTrue(kind.tokenChars.length == 2);
         Assert.isTrue(toProcess[pos] == kind.tokenChars[0]);
         return toProcess[pos + 1] == kind.tokenChars[1];
     }
 
     /**
      * Push a token of just one character in length.
      */
     private void pushCharToken(TokenKind kind) {
         tokens.add(new Token(kind, pos, pos + 1));
         pos++;
     }
 
     /**
      * Push a token of two characters in length.
      */
     private void pushPairToken(TokenKind kind) {
         tokens.add(new Token(kind, pos, pos + 2));
         pos += 2;
     }
 
     private void pushOneCharOrTwoCharToken(TokenKind kind, int pos) {
         tokens.add(new Token(kind, pos, pos + kind.getLength()));
     }
 
     //        ID:        ('a'..'z'|'A'..'Z'|'_'|'$') ('a'..'z'|'A'..'Z'|'_'|'$'|'0'..'9'|DOT_ESCAPED)*;
     private boolean isIdentifier(char ch) {
         return isAlphabetic(ch) || isDigit(ch) || ch == '_' || ch == '$';
     }
 
     private boolean isChar(char a, char b) {
         char ch = toProcess[pos];
         return ch == a || ch == b;
     }
 
     private boolean isExponentChar(char ch) {
         return ch == 'e' || ch == 'E';
     }
 
     private boolean isFloatSuffix(char ch) {
         return ch == 'f' || ch == 'F';
     }
 
     private boolean isDoubleSuffix(char ch) {
         return ch == 'd' || ch == 'D';
     }
 
     private boolean isSign(char ch) {
         return ch == '+' || ch == '-';
     }
 
     private boolean isDigit(char ch) {
         if (ch > 255) {
             return false;
         }
         return (flags[ch] & IS_DIGIT) != 0;
     }
 
     private boolean isAlphabetic(char ch) {
         if (ch > 255) {
             return false;
         }
         return (flags[ch] & IS_ALPHA) != 0;
     }
 
     private boolean isHexadecimalDigit(char ch) {
         if (ch > 255) {
             return false;
         }
         return (flags[ch] & IS_HEXDIGIT) != 0;
     }
 
     private final byte flags[] = new byte[256];
     private static final byte IS_DIGIT = 0x01;
     private static final byte IS_HEXDIGIT = 0x02;
     private static final byte IS_ALPHA = 0x04;
 
     public class Token {
         TokenKind kind;
         String data;
         int startpos; // index of first character
         int endpos;   // index of char after the last character
 
         /**
          * Constructor for use when there is no particular data for the token (eg. TRUE or '+')
          *
          * @param startpos the exact start
          * @param endpos the index to the last character
          */
         public Token(TokenKind tokenKind, int startpos, int endpos) {
             this.kind = tokenKind;
             this.startpos = startpos;
             this.endpos = endpos;
         }
 
         Token(TokenKind tokenKind, char[] tokenData, int pos, int endpos) {
             this(tokenKind, pos, endpos);
             this.data = new String(tokenData);
         }
 
         public TokenKind getKind() {
             return kind;
         }
 
         public String toString() {
             StringBuilder s = new StringBuilder();
             s.append("[").append(kind.toString());
             if (kind.hasPayload()) {
                 s.append(":").append(data);
             }
             s.append("]");
             s.append("(").append(startpos).append(",").append(endpos).append(")");
             return s.toString();
         }
 
         public boolean isIdentifier() {
             return kind == TokenKind.IDENTIFIER;
         }
 
         public boolean isNumericRelationalOperator() {
             return kind == TokenKind.GT
                     || kind == TokenKind.GE
                     || kind == TokenKind.LT
                     || kind == TokenKind.LE
                     || kind == TokenKind.EQ
                     || kind == TokenKind.NE;
         }
 
         public String stringValue() {
             return data;
         }
 
         public Token asInstanceOfToken() {
             return new Token(TokenKind.INSTANCEOF, startpos, endpos);
         }
 
         public Token asMatchesToken() {
             return new Token(TokenKind.MATCHES, startpos, endpos);
         }
 
         public Token asBetweenToken() {
             return new Token(TokenKind.BETWEEN, startpos, endpos);
         }
     }
 
     public enum TokenKind {
         // ordered by priority - operands first
         LITERAL_INT, LITERAL_LONG, LITERAL_HEXINT, LITERAL_HEXLONG, LITERAL_STRING, LITERAL_REAL, LITERAL_REAL_FLOAT,
         LPAREN("("), RPAREN(")"), COMMA(","), IDENTIFIER,
         COLON(":"), HASH("#"), RSQUARE("]"), LSQUARE("["),
         LCURLY("{"), RCURLY("}"),
         DOT("."), PLUS("+"), STAR("*"), DIV("/"), NOT("!"), MINUS("-"), SELECT_FIRST("^["), SELECT_LAST("$["), QMARK(
                 "?"), PROJECT("!["),
         GE(">="), GT(">"), LE("<="), LT("<"), EQ("=="), NE("!="), ASSIGN("="), INSTANCEOF("instanceof"), MATCHES(
                 "matches"), BETWEEN("between"),
         SELECT("?["), MOD("%"), POWER("^"),
         ELVIS("?:"), SAFE_NAVI("?."), BEAN_REF("@");
 
         char[] tokenChars;
         private boolean hasPayload; // is there more to this token than simply the kind
 
         private TokenKind(String tokenString) {
             tokenChars = tokenString.toCharArray();
             hasPayload = tokenChars.length == 0;
         }
 
         private TokenKind() {
             this("");
         }
 
         public String toString() {
             return this.name() + (tokenChars.length != 0 ? "(" + new String(tokenChars) + ")" : "");
         }
 
         public boolean hasPayload() {
             return hasPayload;
         }
 
         public int getLength() {
             return tokenChars.length;
         }
     }
 }

1		/**
2		* Copyright 2005-2011 The Kuali Foundation
3		*
4		* Licensed under the Educational Community License, Version 2.0 (the "License");
5		* you may not use this file except in compliance with the License.
6		* You may obtain a copy of the License at
7		*
8		* http://www.opensource.org/licenses/ecl2.php
9		*
10		* Unless required by applicable law or agreed to in writing, software
11		* distributed under the License is distributed on an "AS IS" BASIS,
12		* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13		* See the License for the specific language governing permissions and
14		* limitations under the License.
15		*/
16		package org.kuali.rice.krad.uif.util;
17
18		import org.springframework.expression.spel.InternalParseException;
19		import org.springframework.expression.spel.SpelMessage;
20		import org.springframework.expression.spel.SpelParseException;
21		import org.springframework.util.Assert;
22
23		import java.util.ArrayList;
24		import java.util.Arrays;
25		import java.util.List;
26
27		/**
28		* @author Kuali Rice Team (rice.collab@kuali.org)
29		*/
30		public class Tokenizer {
31		String expressionString;
32		char[] toProcess;
33		int pos;
34		int max;
35	0	List<Token> tokens = new ArrayList<Token>();
36
37	0	protected Tokenizer(String inputdata) {
38	0	for (int ch = '0'; ch <= '9'; ch++) {
39	0	flags[ch] \|= IS_DIGIT \| IS_HEXDIGIT;
40		}
41	0	for (int ch = 'A'; ch <= 'F'; ch++) {
42	0	flags[ch] \|= IS_HEXDIGIT;
43		}
44	0	for (int ch = 'a'; ch <= 'f'; ch++) {
45	0	flags[ch] \|= IS_HEXDIGIT;
46		}
47	0	for (int ch = 'A'; ch <= 'Z'; ch++) {
48	0	flags[ch] \|= IS_ALPHA;
49		}
50	0	for (int ch = 'a'; ch <= 'z'; ch++) {
51	0	flags[ch] \|= IS_ALPHA;
52		}
53
54	0	this.expressionString = inputdata;
55	0	this.toProcess = (inputdata + "\0").toCharArray();
56	0	this.max = toProcess.length;
57	0	this.pos = 0;
58	0	process();
59	0	}
60
61		public void process() {
62	0	while (pos < max) {
63	0	char ch = toProcess[pos];
64	0	if (isAlphabetic(ch)) {
65	0	lexIdentifier();
66		} else {
67	0	switch (ch) {
68		case '+':
69	0	pushCharToken(TokenKind.PLUS);
70	0	break;
71		case '_': // the other way to start an identifier
72	0	lexIdentifier();
73	0	break;
74		case '-':
75	0	pushCharToken(TokenKind.MINUS);
76	0	break;
77		case ':':
78	0	pushCharToken(TokenKind.COLON);
79	0	break;
80		case '.':
81	0	pushCharToken(TokenKind.DOT);
82	0	break;
83		case ',':
84	0	pushCharToken(TokenKind.COMMA);
85	0	break;
86		case '*':
87	0	pushCharToken(TokenKind.STAR);
88	0	break;
89		case '/':
90	0	pushCharToken(TokenKind.DIV);
91	0	break;
92		case '%':
93	0	pushCharToken(TokenKind.MOD);
94	0	break;
95		case '(':
96	0	pushCharToken(TokenKind.LPAREN);
97	0	break;
98		case ')':
99	0	pushCharToken(TokenKind.RPAREN);
100	0	break;
101		case '[':
102	0	pushCharToken(TokenKind.LSQUARE);
103	0	break;
104		case '#':
105	0	pushCharToken(TokenKind.HASH);
106	0	break;
107		case ']':
108	0	pushCharToken(TokenKind.RSQUARE);
109	0	break;
110		case '{':
111	0	pushCharToken(TokenKind.LCURLY);
112	0	break;
113		case '}':
114	0	pushCharToken(TokenKind.RCURLY);
115	0	break;
116		case '@':
117	0	pushCharToken(TokenKind.BEAN_REF);
118	0	break;
119		case '^':
120	0	if (isTwoCharToken(TokenKind.SELECT_FIRST)) {
121	0	pushPairToken(TokenKind.SELECT_FIRST);
122		} else {
123	0	pushCharToken(TokenKind.POWER);
124		}
125	0	break;
126		case '!':
127	0	if (isTwoCharToken(TokenKind.NE)) {
128	0	pushPairToken(TokenKind.NE);
129	0	} else if (isTwoCharToken(TokenKind.PROJECT)) {
130	0	pushPairToken(TokenKind.PROJECT);
131		} else {
132	0	pushCharToken(TokenKind.NOT);
133		}
134	0	break;
135		case '=':
136	0	if (isTwoCharToken(TokenKind.EQ)) {
137	0	pushPairToken(TokenKind.EQ);
138		} else {
139	0	pushCharToken(TokenKind.ASSIGN);
140		}
141	0	break;
142		case '?':
143	0	if (isTwoCharToken(TokenKind.SELECT)) {
144	0	pushPairToken(TokenKind.SELECT);
145	0	} else if (isTwoCharToken(TokenKind.ELVIS)) {
146	0	pushPairToken(TokenKind.ELVIS);
147	0	} else if (isTwoCharToken(TokenKind.SAFE_NAVI)) {
148	0	pushPairToken(TokenKind.SAFE_NAVI);
149		} else {
150	0	pushCharToken(TokenKind.QMARK);
151		}
152	0	break;
153		case '$':
154	0	if (isTwoCharToken(TokenKind.SELECT_LAST)) {
155	0	pushPairToken(TokenKind.SELECT_LAST);
156		} else {
157	0	lexIdentifier();
158		}
159	0	break;
160		case '>':
161	0	if (isTwoCharToken(TokenKind.GE)) {
162	0	pushPairToken(TokenKind.GE);
163		} else {
164	0	pushCharToken(TokenKind.GT);
165		}
166	0	break;
167		case '<':
168	0	if (isTwoCharToken(TokenKind.LE)) {
169	0	pushPairToken(TokenKind.LE);
170		} else {
171	0	pushCharToken(TokenKind.LT);
172		}
173	0	break;
174		case '0':
175		case '1':
176		case '2':
177		case '3':
178		case '4':
179		case '5':
180		case '6':
181		case '7':
182		case '8':
183		case '9':
184	0	lexNumericLiteral(ch == '0');
185	0	break;
186		case ' ':
187		case '\t':
188		case '\r':
189		case '\n':
190		// drift over white space
191	0	pos++;
192	0	break;
193		case '\'':
194	0	lexQuotedStringLiteral();
195	0	break;
196		case '"':
197	0	lexDoubleQuotedStringLiteral();
198	0	break;
199		case 0:
200		// hit sentinel at end of value
201	0	pos++; // will take us to the end
202	0	break;
203		default:
204	0	throw new IllegalStateException("Cannot handle ("
205		+ Integer.valueOf(ch)
206		+ ") '"
207		+ ch
208		+ "', in expression: "
209		+ expressionString);
210		}
211		}
212	0	}
213	0	}
214
215		public List<Token> getTokens() {
216	0	return tokens;
217		}
218
219		// STRING_LITERAL: '\''! (APOS\|~'\'')* '\''!;
220		private void lexQuotedStringLiteral() {
221	0	int start = pos;
222	0	boolean terminated = false;
223	0	while (!terminated) {
224	0	pos++;
225	0	char ch = toProcess[pos];
226	0	if (ch == '\'') {
227		// may not be the end if the char after is also a '
228	0	if (toProcess[pos + 1] == '\'') {
229	0	pos++; // skip over that too, and continue
230		} else {
231	0	terminated = true;
232		}
233		}
234	0	if (ch == 0) {
235	0	throw new InternalParseException(new SpelParseException(expressionString, start,
236		SpelMessage.NON_TERMINATING_QUOTED_STRING));
237		}
238	0	}
239	0	pos++;
240	0	tokens.add(new Token(TokenKind.LITERAL_STRING, subarray(start, pos), start, pos));
241	0	}
242
243		// DQ_STRING_LITERAL: '"'! (~'"')* '"'!;
244		private void lexDoubleQuotedStringLiteral() {
245	0	int start = pos;
246	0	boolean terminated = false;
247	0	while (!terminated) {
248	0	pos++;
249	0	char ch = toProcess[pos];
250	0	if (ch == '"') {
251	0	terminated = true;
252		}
253	0	if (ch == 0) {
254	0	throw new InternalParseException(new SpelParseException(expressionString, start,
255		SpelMessage.NON_TERMINATING_DOUBLE_QUOTED_STRING));
256		}
257	0	}
258	0	pos++;
259	0	tokens.add(new Token(TokenKind.LITERAL_STRING, subarray(start, pos), start, pos));
260	0	}
261
262		// REAL_LITERAL :
263		// ('.' (DECIMAL_DIGIT)+ (EXPONENT_PART)? (REAL_TYPE_SUFFIX)?) \|
264		// ((DECIMAL_DIGIT)+ '.' (DECIMAL_DIGIT)+ (EXPONENT_PART)? (REAL_TYPE_SUFFIX)?) \|
265		// ((DECIMAL_DIGIT)+ (EXPONENT_PART) (REAL_TYPE_SUFFIX)?) \|
266		// ((DECIMAL_DIGIT)+ (REAL_TYPE_SUFFIX));
267		// fragment INTEGER_TYPE_SUFFIX : ( 'L' \| 'l' );
268		// fragment HEX_DIGIT : '0'\|'1'\|'2'\|'3'\|'4'\|'5'\|'6'\|'7'\|'8'\|'9'\|'A'\|'B'\|'C'\|'D'\|'E'\|'F'\|'a'\|'b'\|'c'\|'d'\|'e'\|'f';
269		//
270		// fragment EXPONENT_PART : 'e' (SIGN)* (DECIMAL_DIGIT)+ \| 'E' (SIGN)* (DECIMAL_DIGIT)+ ;
271		// fragment SIGN : '+' \| '-' ;
272		// fragment REAL_TYPE_SUFFIX : 'F' \| 'f' \| 'D' \| 'd';
273		// INTEGER_LITERAL
274		// : (DECIMAL_DIGIT)+ (INTEGER_TYPE_SUFFIX)?;
275
276		private void lexNumericLiteral(boolean firstCharIsZero) {
277	0	boolean isReal = false;
278	0	int start = pos;
279	0	char ch = toProcess[pos + 1];
280	0	boolean isHex = ch == 'x' \|\| ch == 'X';
281
282		// deal with hexadecimal
283	0	if (firstCharIsZero && isHex) {
284	0	pos = pos + 1;
285		do {
286	0	pos++;
287	0	} while (isHexadecimalDigit(toProcess[pos]));
288	0	if (isChar('L', 'l')) {
289	0	pushHexIntToken(subarray(start + 2, pos), true, start, pos);
290	0	pos++;
291		} else {
292	0	pushHexIntToken(subarray(start + 2, pos), false, start, pos);
293		}
294	0	return;
295		}
296
297		// real numbers must have leading digits
298
299		// Consume first part of number
300		do {
301	0	pos++;
302	0	} while (isDigit(toProcess[pos]));
303
304		// a '.' indicates this number is a real
305	0	ch = toProcess[pos];
306	0	if (ch == '.') {
307	0	isReal = true;
308		// carry on consuming digits
309		do {
310	0	pos++;
311	0	} while (isDigit(toProcess[pos]));
312		}
313
314	0	int endOfNumber = pos;
315
316		// Now there may or may not be an exponent
317
318		// is it a long ?
319	0	if (isChar('L', 'l')) {
320	0	if (isReal) { // 3.4L - not allowed
321	0	throw new InternalParseException(new SpelParseException(expressionString, start,
322		SpelMessage.REAL_CANNOT_BE_LONG));
323		}
324	0	pushIntToken(subarray(start, endOfNumber), true, start, endOfNumber);
325	0	pos++;
326	0	} else if (isExponentChar(toProcess[pos])) {
327	0	isReal = true; // if it wasnt before, it is now
328	0	pos++;
329	0	char possibleSign = toProcess[pos];
330	0	if (isSign(possibleSign)) {
331	0	pos++;
332		}
333
334		// exponent digits
335		do {
336	0	pos++;
337	0	} while (isDigit(toProcess[pos]));
338	0	boolean isFloat = false;
339	0	if (isFloatSuffix(toProcess[pos])) {
340	0	isFloat = true;
341	0	endOfNumber = ++pos;
342	0	} else if (isDoubleSuffix(toProcess[pos])) {
343	0	endOfNumber = ++pos;
344		}
345	0	pushRealToken(subarray(start, pos), isFloat, start, pos);
346	0	} else {
347	0	ch = toProcess[pos];
348	0	boolean isFloat = false;
349	0	if (isFloatSuffix(ch)) {
350	0	isReal = true;
351	0	isFloat = true;
352	0	endOfNumber = ++pos;
353	0	} else if (isDoubleSuffix(ch)) {
354	0	isReal = true;
355	0	endOfNumber = ++pos;
356		}
357	0	if (isReal) {
358	0	pushRealToken(subarray(start, endOfNumber), isFloat, start, endOfNumber);
359		} else {
360	0	pushIntToken(subarray(start, endOfNumber), false, start, endOfNumber);
361		}
362		}
363	0	}
364
365		// if this is changed, it must remain sorted
366	0	private String[] alternativeOperatorNames = {"DIV", "EQ", "GE", "GT", "LE", "LT", "MOD", "NE", "NOT"};
367
368		private void lexIdentifier() {
369	0	int start = pos;
370		do {
371	0	pos++;
372	0	} while (isIdentifier(toProcess[pos]));
373	0	char[] subarray = subarray(start, pos);
374
375		// Check if this is the alternative (textual) representation of an operator (see alternativeOperatorNames)
376	0	if ((pos - start) == 2 \|\| (pos - start) == 3) {
377	0	String asString = new String(subarray).toUpperCase();
378	0	int idx = Arrays.binarySearch(alternativeOperatorNames, asString);
379	0	if (idx >= 0) {
380	0	pushOneCharOrTwoCharToken(TokenKind.valueOf(asString), start);
381	0	return;
382		}
383		}
384	0	tokens.add(new Token(TokenKind.IDENTIFIER, subarray, start, pos));
385	0	}
386
387		private void pushIntToken(char[] data, boolean isLong, int start, int end) {
388	0	if (isLong) {
389	0	tokens.add(new Token(TokenKind.LITERAL_LONG, data, start, end));
390		} else {
391	0	tokens.add(new Token(TokenKind.LITERAL_INT, data, start, end));
392		}
393	0	}
394
395		private void pushHexIntToken(char[] data, boolean isLong, int start, int end) {
396	0	if (data.length == 0) {
397	0	if (isLong) {
398	0	throw new InternalParseException(new SpelParseException(expressionString, start, SpelMessage.NOT_A_LONG,
399		expressionString.substring(start, end + 1)));
400		} else {
401	0	throw new InternalParseException(new SpelParseException(expressionString, start,
402		SpelMessage.NOT_AN_INTEGER, expressionString.substring(start, end)));
403		}
404		}
405	0	if (isLong) {
406	0	tokens.add(new Token(TokenKind.LITERAL_HEXLONG, data, start, end));
407		} else {
408	0	tokens.add(new Token(TokenKind.LITERAL_HEXINT, data, start, end));
409		}
410	0	}
411
412		private void pushRealToken(char[] data, boolean isFloat, int start, int end) {
413	0	if (isFloat) {
414	0	tokens.add(new Token(TokenKind.LITERAL_REAL_FLOAT, data, start, end));
415		} else {
416	0	tokens.add(new Token(TokenKind.LITERAL_REAL, data, start, end));
417		}
418	0	}
419
420		private char[] subarray(int start, int end) {
421	0	char[] result = new char[end - start];
422	0	System.arraycopy(toProcess, start, result, 0, end - start);
423	0	return result;
424		}
425
426		/**
427		* Check if this might be a two character token.
428		*/
429		private boolean isTwoCharToken(TokenKind kind) {
430	0	Assert.isTrue(kind.tokenChars.length == 2);
431	0	Assert.isTrue(toProcess[pos] == kind.tokenChars[0]);
432	0	return toProcess[pos + 1] == kind.tokenChars[1];
433		}
434
435		/**
436		* Push a token of just one character in length.
437		*/
438		private void pushCharToken(TokenKind kind) {
439	0	tokens.add(new Token(kind, pos, pos + 1));
440	0	pos++;
441	0	}
442
443		/**
444		* Push a token of two characters in length.
445		*/
446		private void pushPairToken(TokenKind kind) {
447	0	tokens.add(new Token(kind, pos, pos + 2));
448	0	pos += 2;
449	0	}
450
451		private void pushOneCharOrTwoCharToken(TokenKind kind, int pos) {
452	0	tokens.add(new Token(kind, pos, pos + kind.getLength()));
453	0	}
454
455		// ID: ('a'..'z'\|'A'..'Z'\|'_'\|'$') ('a'..'z'\|'A'..'Z'\|'_'\|'$'\|'0'..'9'\|DOT_ESCAPED)*;
456		private boolean isIdentifier(char ch) {
457	0	return isAlphabetic(ch) \|\| isDigit(ch) \|\| ch == '_' \|\| ch == '$';
458		}
459
460		private boolean isChar(char a, char b) {
461	0	char ch = toProcess[pos];
462	0	return ch == a \|\| ch == b;
463		}
464
465		private boolean isExponentChar(char ch) {
466	0	return ch == 'e' \|\| ch == 'E';
467		}
468
469		private boolean isFloatSuffix(char ch) {
470	0	return ch == 'f' \|\| ch == 'F';
471		}
472
473		private boolean isDoubleSuffix(char ch) {
474	0	return ch == 'd' \|\| ch == 'D';
475		}
476
477		private boolean isSign(char ch) {
478	0	return ch == '+' \|\| ch == '-';
479		}
480
481		private boolean isDigit(char ch) {
482	0	if (ch > 255) {
483	0	return false;
484		}
485	0	return (flags[ch] & IS_DIGIT) != 0;
486		}
487
488		private boolean isAlphabetic(char ch) {
489	0	if (ch > 255) {
490	0	return false;
491		}
492	0	return (flags[ch] & IS_ALPHA) != 0;
493		}
494
495		private boolean isHexadecimalDigit(char ch) {
496	0	if (ch > 255) {
497	0	return false;
498		}
499	0	return (flags[ch] & IS_HEXDIGIT) != 0;
500		}
501
502	0	private final byte flags[] = new byte[256];
503		private static final byte IS_DIGIT = 0x01;
504		private static final byte IS_HEXDIGIT = 0x02;
505		private static final byte IS_ALPHA = 0x04;
506
507		public class Token {
508		TokenKind kind;
509		String data;
510		int startpos; // index of first character
511		int endpos; // index of char after the last character
512
513		/**
514		* Constructor for use when there is no particular data for the token (eg. TRUE or '+')
515		*
516		* @param startpos the exact start
517		* @param endpos the index to the last character
518		*/
519	0	public Token(TokenKind tokenKind, int startpos, int endpos) {
520	0	this.kind = tokenKind;
521	0	this.startpos = startpos;
522	0	this.endpos = endpos;
523	0	}
524
525		Token(TokenKind tokenKind, char[] tokenData, int pos, int endpos) {
526	0	this(tokenKind, pos, endpos);
527	0	this.data = new String(tokenData);
528	0	}
529
530		public TokenKind getKind() {
531	0	return kind;
532		}
533
534		public String toString() {
535	0	StringBuilder s = new StringBuilder();
536	0	s.append("[").append(kind.toString());
537	0	if (kind.hasPayload()) {
538	0	s.append(":").append(data);
539		}
540	0	s.append("]");
541	0	s.append("(").append(startpos).append(",").append(endpos).append(")");
542	0	return s.toString();
543		}
544
545		public boolean isIdentifier() {
546	0	return kind == TokenKind.IDENTIFIER;
547		}
548
549		public boolean isNumericRelationalOperator() {
550	0	return kind == TokenKind.GT
551		\|\| kind == TokenKind.GE
552		\|\| kind == TokenKind.LT
553		\|\| kind == TokenKind.LE
554		\|\| kind == TokenKind.EQ
555		\|\| kind == TokenKind.NE;
556		}
557
558		public String stringValue() {
559	0	return data;
560		}
561
562		public Token asInstanceOfToken() {
563	0	return new Token(TokenKind.INSTANCEOF, startpos, endpos);
564		}
565
566		public Token asMatchesToken() {
567	0	return new Token(TokenKind.MATCHES, startpos, endpos);
568		}
569
570		public Token asBetweenToken() {
571	0	return new Token(TokenKind.BETWEEN, startpos, endpos);
572		}
573		}
574
575	0	public enum TokenKind {
576		// ordered by priority - operands first
577	0	LITERAL_INT, LITERAL_LONG, LITERAL_HEXINT, LITERAL_HEXLONG, LITERAL_STRING, LITERAL_REAL, LITERAL_REAL_FLOAT,
578	0	LPAREN("("), RPAREN(")"), COMMA(","), IDENTIFIER,
579	0	COLON(":"), HASH("#"), RSQUARE("]"), LSQUARE("["),
580	0	LCURLY("{"), RCURLY("}"),
581	0	DOT("."), PLUS("+"), STAR("*"), DIV("/"), NOT("!"), MINUS("-"), SELECT_FIRST("^["), SELECT_LAST("$["), QMARK(
582	0	"?"), PROJECT("!["),
583	0	GE(">="), GT(">"), LE("<="), LT("<"), EQ("=="), NE("!="), ASSIGN("="), INSTANCEOF("instanceof"), MATCHES(
584	0	"matches"), BETWEEN("between"),
585	0	SELECT("?["), MOD("%"), POWER("^"),
586	0	ELVIS("?:"), SAFE_NAVI("?."), BEAN_REF("@");
587
588		char[] tokenChars;
589		private boolean hasPayload; // is there more to this token than simply the kind
590
591	0	private TokenKind(String tokenString) {
592	0	tokenChars = tokenString.toCharArray();
593	0	hasPayload = tokenChars.length == 0;
594	0	}
595
596		private TokenKind() {
597	0	this("");
598	0	}
599
600		public String toString() {
601	0	return this.name() + (tokenChars.length != 0 ? "(" + new String(tokenChars) + ")" : "");
602		}
603
604		public boolean hasPayload() {
605	0	return hasPayload;
606		}
607
608		public int getLength() {
609	0	return tokenChars.length;
610		}
611		}
612		}