false, iff there was new input.
- *
- * @exception java.io.IOException if any I/O-Error occurs
- */
- private boolean zzRefill() throws java.io.IOException {
- return true;
- }
-
-
- /**
- * Returns the current lexical state.
- */
- public final int yystate() {
- return zzLexicalState;
- }
-
-
- /**
- * Enters a new lexical state
- *
- * @param newState the new lexical state
- */
- public final void yybegin(int newState) {
- zzLexicalState = newState;
- }
-
-
- /**
- * Returns the text matched by the current regular expression.
- */
- public final CharSequence yytext() {
- return zzBuffer.subSequence(zzStartRead, zzMarkedPos);
- }
-
-
- /**
- * Returns the character at position pos from the
- * matched text.
- *
- * It is equivalent to yytext().charAt(pos), but faster
- *
- * @param pos the position of the character to fetch.
- * A value from 0 to yylength()-1.
- *
- * @return the character at position pos
- */
- public final char yycharat(int pos) {
- return zzBufferArray != null ? zzBufferArray[zzStartRead+pos]:zzBuffer.charAt(zzStartRead+pos);
- }
-
-
- /**
- * Returns the length of the matched text region.
- */
- public final int yylength() {
- return zzMarkedPos-zzStartRead;
- }
-
-
- /**
- * Reports an error that occured while scanning.
- *
- * In a wellformed scanner (no or only correct usage of
- * yypushback(int) and a match-all fallback rule) this method
- * will only be called with things that "Can't Possibly Happen".
- * If this method is called, something is seriously wrong
- * (e.g. a JFlex bug producing a faulty scanner etc.).
- *
- * Usual syntax/scanner level error handling should be done
- * in error fallback rules.
- *
- * @param errorCode the code of the errormessage to display
- */
---- zzScanError declaration
- String message;
- try {
- message = ZZ_ERROR_MSG[errorCode];
- }
- catch (ArrayIndexOutOfBoundsException e) {
- message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
- }
-
---- throws clause
- }
-
-
- /**
- * Pushes the specified amount of characters back into the input stream.
- *
- * They will be read again by then next call of the scanning method
- *
- * @param number the number of characters to be read again.
- * This number must not be greater than yylength()!
- */
---- yypushback decl (contains zzScanError exception)
- if ( number > yylength() )
- zzScanError(ZZ_PUSHBACK_2BIG);
-
- zzMarkedPos -= number;
- }
-
-
---- zzDoEOF
- /**
- * Resumes scanning until the next regular expression is matched,
- * the end of input is encountered or an I/O-Error occurs.
- *
- * @return the next token
- * @exception java.io.IOException if any I/O-Error occurs
- */
---- yylex declaration
- int zzInput;
- int zzAction;
-
- // cached fields:
- int zzCurrentPosL;
- int zzMarkedPosL;
- int zzEndReadL = zzEndRead;
- CharSequence zzBufferL = zzBuffer;
- char[] zzBufferArrayL = zzBufferArray;
- char [] zzCMapL = ZZ_CMAP;
-
---- local declarations
-
- while (true) {
- zzMarkedPosL = zzMarkedPos;
-
---- start admin (line, char, col count)
- zzAction = -1;
-
- zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
-
---- start admin (lexstate etc)
-
- zzForAction: {
- while (true) {
-
---- next input, line, col, char count, next transition, isFinal action
- zzAction = zzState;
- zzMarkedPosL = zzCurrentPosL;
---- line count update
- }
-
- }
- }
-
- // store back cached position
- zzMarkedPos = zzMarkedPosL;
---- char count update
-
---- actions
- default:
- if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
- zzAtEOF = true;
---- eofvalue
- }
- else {
---- no match
- }
- }
- }
- }
-
---- main
-
-}
diff --git a/simple_language_plugin/src/com/simpleplugin/Simple.flex b/simple_language_plugin/src/com/simpleplugin/Simple.flex
index 89dabf207..0c505eb80 100644
--- a/simple_language_plugin/src/com/simpleplugin/Simple.flex
+++ b/simple_language_plugin/src/com/simpleplugin/Simple.flex
@@ -15,13 +15,13 @@ import com.intellij.psi.TokenType;
%eof{ return;
%eof}
-CRLF= \n|\r|\r\n
+CRLF=\n
WHITE_SPACE=[\ \t\f]
-FIRST_VALUE_CHARACTER=[^ \n\r\f\\] | "\\"{CRLF} | "\\".
-VALUE_CHARACTER=[^\n\r\f\\] | "\\"{CRLF} | "\\".
-END_OF_LINE_COMMENT=("#"|"!")[^\r\n]*
+FIRST_VALUE_CHARACTER=[^ \n\f\\] | "\\"{CRLF} | "\\".
+VALUE_CHARACTER=[^\n\f\\] | "\\"{CRLF} | "\\".
+END_OF_LINE_COMMENT=("#"|"!")[^\n]*
SEPARATOR=[:=]
-KEY_CHARACTER=[^:=\ \n\r\t\f\\] | "\\ "
+KEY_CHARACTER=[^:=\ \n\t\f\\] | "\\ "
%state WAITING_VALUE
@@ -39,7 +39,7 @@ KEY_CHARACTER=[^:=\ \n\r\t\f\\] | "\\ "
aState
*/
- private static final int[] ZZ_ATTRIBUTE = zzUnpackAttribute();
+ private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
- "\2\0\4\1\1\11\4\1\2\0\1\1";
+ "\2\0\5\1\1\11\5\1\2\0\1\1";
- private static int[] zzUnpackAttribute() {
- int[] result = new int[14];
+ private static int [] zzUnpackAttribute() {
+ int [] result = new int[16];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
}
- private static int zzUnpackAttribute(String packed, int offset, int[] result) {
+ private static int zzUnpackAttribute(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
@@ -181,51 +189,30 @@ class SimpleLexer implements FlexLexer {
return j;
}
- /**
- * the current state of the DFA
- */
+ /** the input device */
+ private java.io.Reader zzReader;
+
+ /** the current state of the DFA */
private int zzState;
- /**
- * the current lexical state
- */
+ /** the current lexical state */
private int zzLexicalState = YYINITIAL;
- /**
- * this buffer contains the current text to be matched and is
- * the source of the yytext() string
- */
+ /** this buffer contains the current text to be matched and is
+ the source of the yytext() string */
private CharSequence zzBuffer = "";
- /**
- * this buffer may contains the current text array to be matched when it is cheap to acquire it
- */
- private char[] zzBufferArray;
-
- /**
- * the textposition at the last accepting state
- */
+ /** the textposition at the last accepting state */
private int zzMarkedPos;
- /**
- * the textposition at the last state to be included in yytext
- */
- private int zzPushbackPos;
-
- /**
- * the current text position in the buffer
- */
+ /** the current text position in the buffer */
private int zzCurrentPos;
- /**
- * startRead marks the beginning of the yytext() string in the buffer
- */
+ /** startRead marks the beginning of the yytext() string in the buffer */
private int zzStartRead;
- /**
- * endRead marks the last character in the buffer, that has been read
- * from input
- */
+ /** endRead marks the last character in the buffer, that has been read
+ from input */
private int zzEndRead;
/**
@@ -233,43 +220,39 @@ class SimpleLexer implements FlexLexer {
*/
private boolean zzAtBOL = true;
- /**
- * zzAtEOF == true <=> the scanner is at the EOF
- */
+ /** zzAtEOF == true <=> the scanner is at the EOF */
private boolean zzAtEOF;
- /**
- * denotes if the user-EOF-code has already been executed
- */
+ /** denotes if the user-EOF-code has already been executed */
private boolean zzEOFDone;
+ /**
+ * Creates a new scanner
+ *
+ * @param in the java.io.Reader to read input from.
+ */
SimpleLexer(java.io.Reader in) {
this.zzReader = in;
}
- /**
- * Creates a new scanner.
- * There is also java.io.Reader version of this constructor.
- *
- * @param in the java.io.Inputstream to read input from.
- */
- SimpleLexer(java.io.InputStream in) {
- this(new java.io.InputStreamReader(in));
- }
- /**
+ /**
* Unpacks the compressed character translation table.
*
- * @param packed the packed character translation table
- * @return the unpacked character translation table
+ * @param packed the packed character translation table
+ * @return the unpacked character translation table
*/
- private static char[] zzUnpackCMap(String packed) {
- char[] map = new char[0x10000];
+ private static char [] zzUnpackCMap(String packed) {
+ int size = 0;
+ for (int i = 0, length = packed.length(); i < length; i += 2) {
+ size += packed.charAt(i);
+ }
+ char[] map = new char[size];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
- while (i < 36) {
- int count = packed.charAt(i++);
+ while (i < packed.length()) {
+ int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
}
@@ -286,10 +269,8 @@ class SimpleLexer implements FlexLexer {
public void reset(CharSequence buffer, int start, int end, int initialState) {
zzBuffer = buffer;
- zzBufferArray = com.intellij.util.text.CharArrayUtil.fromSequenceWithoutCopying(buffer);
zzCurrentPos = zzMarkedPos = zzStartRead = start;
- zzPushbackPos = 0;
- zzAtEOF = false;
+ zzAtEOF = false;
zzAtBOL = true;
zzEndRead = end;
yybegin(initialState);
@@ -298,8 +279,9 @@ class SimpleLexer implements FlexLexer {
/**
* Refills the input buffer.
*
- * @return false, iff there was new input.
- * @throws java.io.IOException if any I/O-Error occurs
+ * @return false, iff there was new input.
+ *
+ * @exception java.io.IOException if any I/O-Error occurs
*/
private boolean zzRefill() throws java.io.IOException {
return true;
@@ -335,15 +317,16 @@ class SimpleLexer implements FlexLexer {
/**
* Returns the character at position pos from the
* matched text.
- * + * * It is equivalent to yytext().charAt(pos), but faster * * @param pos the position of the character to fetch. * A value from 0 to yylength()-1. + * * @return the character at position pos */ public final char yycharat(int pos) { - return zzBufferArray != null ? zzBufferArray[zzStartRead + pos] : zzBuffer.charAt(zzStartRead + pos); + return zzBuffer.charAt(zzStartRead+pos); } @@ -351,29 +334,30 @@ class SimpleLexer implements FlexLexer { * Returns the length of the matched text region. */ public final int yylength() { - return zzMarkedPos - zzStartRead; + return zzMarkedPos-zzStartRead; } /** * Reports an error that occured while scanning. - *
+ * * In a wellformed scanner (no or only correct usage of * yypushback(int) and a match-all fallback rule) this method * will only be called with things that "Can't Possibly Happen". * If this method is called, something is seriously wrong * (e.g. a JFlex bug producing a faulty scanner etc.). - *
+ * * Usual syntax/scanner level error handling should be done * in error fallback rules. * - * @param errorCode the code of the errormessage to display + * @param errorCode the code of the errormessage to display */ private void zzScanError(int errorCode) { String message; try { message = ZZ_ERROR_MSG[errorCode]; - } catch (ArrayIndexOutOfBoundsException e) { + } + catch (ArrayIndexOutOfBoundsException e) { message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR]; } @@ -383,14 +367,14 @@ class SimpleLexer implements FlexLexer { /** * Pushes the specified amount of characters back into the input stream. - *
+ *
* They will be read again by then next call of the scanning method
*
- * @param number the number of characters to be read again.
- * This number must not be greater than yylength()!
+ * @param number the number of characters to be read again.
+ * This number must not be greater than yylength()!
*/
- public void yypushback(int number) {
- if (number > yylength())
+ public void yypushback(int number) {
+ if ( number > yylength() )
zzScanError(ZZ_PUSHBACK_2BIG);
zzMarkedPos -= number;
@@ -404,7 +388,7 @@ class SimpleLexer implements FlexLexer {
private void zzDoEOF() {
if (!zzEOFDone) {
zzEOFDone = true;
-
+
}
}
@@ -413,8 +397,8 @@ class SimpleLexer implements FlexLexer {
* Resumes scanning until the next regular expression is matched,
* the end of input is encountered or an I/O-Error occurs.
*
- * @return the next token
- * @throws java.io.IOException if any I/O-Error occurs
+ * @return the next token
+ * @exception java.io.IOException if any I/O-Error occurs
*/
public IElementType advance() throws java.io.IOException {
int zzInput;
@@ -425,12 +409,10 @@ class SimpleLexer implements FlexLexer {
int zzMarkedPosL;
int zzEndReadL = zzEndRead;
CharSequence zzBufferL = zzBuffer;
- char[] zzBufferArrayL = zzBufferArray;
- char[] zzCMapL = ZZ_CMAP;
- int[] zzTransL = ZZ_TRANS;
- int[] zzRowMapL = ZZ_ROWMAP;
- int[] zzAttrL = ZZ_ATTRIBUTE;
+ int [] zzTransL = ZZ_TRANS;
+ int [] zzRowMapL = ZZ_ROWMAP;
+ int [] zzAttrL = ZZ_ATTRIBUTE;
while (true) {
zzMarkedPosL = zzMarkedPos;
@@ -441,42 +423,52 @@ class SimpleLexer implements FlexLexer {
zzState = ZZ_LEXSTATE[zzLexicalState];
+ // set up zzAction for empty match case:
+ int zzAttributes = zzAttrL[zzState];
+ if ( (zzAttributes & 1) == 1 ) {
+ zzAction = zzState;
+ }
- zzForAction:
- {
+
+ zzForAction: {
while (true) {
- if (zzCurrentPosL < zzEndReadL)
- zzInput = (zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++] : zzBufferL.charAt(zzCurrentPosL++));
+ if (zzCurrentPosL < zzEndReadL) {
+ zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL/*, zzEndReadL*/);
+ zzCurrentPosL += Character.charCount(zzInput);
+ }
else if (zzAtEOF) {
zzInput = YYEOF;
break zzForAction;
- } else {
+ }
+ else {
// store back cached positions
- zzCurrentPos = zzCurrentPosL;
- zzMarkedPos = zzMarkedPosL;
+ zzCurrentPos = zzCurrentPosL;
+ zzMarkedPos = zzMarkedPosL;
boolean eof = zzRefill();
// get translated positions and possibly new buffer
- zzCurrentPosL = zzCurrentPos;
- zzMarkedPosL = zzMarkedPos;
- zzBufferL = zzBuffer;
- zzEndReadL = zzEndRead;
+ zzCurrentPosL = zzCurrentPos;
+ zzMarkedPosL = zzMarkedPos;
+ zzBufferL = zzBuffer;
+ zzEndReadL = zzEndRead;
if (eof) {
zzInput = YYEOF;
break zzForAction;
- } else {
- zzInput = (zzBufferArrayL != null ? zzBufferArrayL[zzCurrentPosL++] : zzBufferL.charAt(zzCurrentPosL++));
+ }
+ else {
+ zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL/*, zzEndReadL*/);
+ zzCurrentPosL += Character.charCount(zzInput);
}
}
- int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
+ int zzNext = zzTransL[ zzRowMapL[zzState] + ZZ_CMAP(zzInput) ];
if (zzNext == -1) break zzForAction;
zzState = zzNext;
- int zzAttributes = zzAttrL[zzState];
- if ((zzAttributes & 1) == 1) {
+ zzAttributes = zzAttrL[zzState];
+ if ( (zzAttributes & 1) == 1 ) {
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
- if ((zzAttributes & 8) == 8) break zzForAction;
+ if ( (zzAttributes & 8) == 8 ) break zzForAction;
}
}
@@ -485,54 +477,46 @@ class SimpleLexer implements FlexLexer {
// store back cached position
zzMarkedPos = zzMarkedPosL;
- switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
- case 6: {
- yybegin(YYINITIAL);
- return SimpleTypes.VALUE;
- }
- case 8:
- break;
- case 5: {
- yybegin(WAITING_VALUE);
- return SimpleTypes.SEPARATOR;
- }
- case 9:
- break;
- case 4: {
- yybegin(YYINITIAL);
- return SimpleTypes.COMMENT;
- }
- case 10:
- break;
- case 3: {
- return TokenType.BAD_CHARACTER;
- }
- case 11:
- break;
- case 2: {
- yybegin(YYINITIAL);
- return TokenType.WHITE_SPACE;
- }
- case 12:
- break;
- case 7: {
- yybegin(WAITING_VALUE);
- return TokenType.WHITE_SPACE;
- }
- case 13:
- break;
- case 1: {
- yybegin(YYINITIAL);
- return SimpleTypes.KEY;
- }
- case 14:
- break;
- default:
- if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
- zzAtEOF = true;
- zzDoEOF();
- return null;
- } else {
+ if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
+ zzAtEOF = true;
+ zzDoEOF();
+ return null;
+ }
+ else {
+ switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
+ case 1:
+ { yybegin(YYINITIAL); return SimpleTypes.KEY;
+ }
+ case 9: break;
+ case 2:
+ { yybegin(YYINITIAL); return SimpleTypes.CRLF;
+ }
+ case 10: break;
+ case 3:
+ { yybegin(YYINITIAL); return TokenType.WHITE_SPACE;
+ }
+ case 11: break;
+ case 4:
+ { return TokenType.BAD_CHARACTER;
+ }
+ case 12: break;
+ case 5:
+ { yybegin(YYINITIAL); return SimpleTypes.COMMENT;
+ }
+ case 13: break;
+ case 6:
+ { yybegin(WAITING_VALUE); return SimpleTypes.SEPARATOR;
+ }
+ case 14: break;
+ case 7:
+ { yybegin(YYINITIAL); return SimpleTypes.VALUE;
+ }
+ case 15: break;
+ case 8:
+ { yybegin(WAITING_VALUE); return TokenType.WHITE_SPACE;
+ }
+ case 16: break;
+ default:
zzScanError(ZZ_NO_MATCH);
}
}
diff --git a/simple_language_plugin/testData/ParsingTestData.txt b/simple_language_plugin/testData/ParsingTestData.txt
index ce7aa544e..01012c591 100644
--- a/simple_language_plugin/testData/ParsingTestData.txt
+++ b/simple_language_plugin/testData/ParsingTestData.txt
@@ -1,63 +1,63 @@
Simple File(0,492)
PsiComment(SimpleTokenType.COMMENT)('# You are reading the ".properties" entry.')(0,42)
- PsiWhiteSpace('\n')(42,43)
+ PsiElement(SimpleTokenType.CRLF)('\n')(42,43)
PsiComment(SimpleTokenType.COMMENT)('! The exclamation mark can also mark text as comments.')(43,97)
- PsiWhiteSpace('\n')(97,98)
+ PsiElement(SimpleTokenType.CRLF)('\n')(97,98)
SimplePropertyImpl(PROPERTY)(98,132)
PsiElement(SimpleTokenType.KEY)('website')(98,105)
PsiWhiteSpace(' ')(105,106)
PsiElement(SimpleTokenType.SEPARATOR)('=')(106,107)
PsiWhiteSpace(' ')(107,108)
PsiElement(SimpleTokenType.VALUE)('http://en.wikipedia.org/')(108,132)
- PsiWhiteSpace('\n\n')(132,134)
+ PsiElement(SimpleTokenType.CRLF)('\n\n')(132,134)
SimplePropertyImpl(PROPERTY)(134,152)
PsiElement(SimpleTokenType.KEY)('language')(134,142)
PsiWhiteSpace(' ')(142,143)
PsiElement(SimpleTokenType.SEPARATOR)('=')(143,144)
PsiWhiteSpace(' ')(144,145)
PsiElement(SimpleTokenType.VALUE)('English')(145,152)
- PsiWhiteSpace('\n')(152,153)
+ PsiElement(SimpleTokenType.CRLF)('\n')(152,153)
PsiComment(SimpleTokenType.COMMENT)('# The backslash below tells the application to continue reading')(153,216)
- PsiWhiteSpace('\n')(216,217)
+ PsiElement(SimpleTokenType.CRLF)('\n')(216,217)
PsiComment(SimpleTokenType.COMMENT)('# the value onto the next line.')(217,248)
- PsiWhiteSpace('\n')(248,249)
+ PsiElement(SimpleTokenType.CRLF)('\n')(248,249)
SimplePropertyImpl(PROPERTY)(249,292)
PsiElement(SimpleTokenType.KEY)('message')(249,256)
PsiWhiteSpace(' ')(256,257)
PsiElement(SimpleTokenType.SEPARATOR)('=')(257,258)
PsiWhiteSpace(' ')(258,259)
PsiElement(SimpleTokenType.VALUE)('Welcome to \\n Wikipedia!')(259,292)
- PsiWhiteSpace('\n')(292,293)
+ PsiElement(SimpleTokenType.CRLF)('\n')(292,293)
PsiComment(SimpleTokenType.COMMENT)('# Add spaces to the key')(293,316)
- PsiWhiteSpace('\n')(316,317)
+ PsiElement(SimpleTokenType.CRLF)('\n')(316,317)
SimplePropertyImpl(PROPERTY)(317,410)
PsiElement(SimpleTokenType.KEY)('key\ with\ spaces')(317,334)
PsiWhiteSpace(' ')(334,335)
PsiElement(SimpleTokenType.SEPARATOR)('=')(335,336)
PsiWhiteSpace(' ')(336,337)
PsiElement(SimpleTokenType.VALUE)('This is the value that could be looked up with the key "key with spaces".')(337,410)
- PsiWhiteSpace('\n')(410,411)
+ PsiElement(SimpleTokenType.CRLF)('\n')(410,411)
PsiComment(SimpleTokenType.COMMENT)('# Unicode')(411,420)
- PsiWhiteSpace('\n')(420,421)
+ PsiElement(SimpleTokenType.CRLF)('\n')(420,421)
SimplePropertyImpl(PROPERTY)(421,433)
PsiElement(SimpleTokenType.KEY)('tab')(421,424)
PsiWhiteSpace(' ')(424,425)
PsiElement(SimpleTokenType.SEPARATOR)(':')(425,426)
PsiWhiteSpace(' ')(426,427)
PsiElement(SimpleTokenType.VALUE)('\u0009')(427,433)
- PsiWhiteSpace('\n')(433,434)
+ PsiElement(SimpleTokenType.CRLF)('\n')(433,434)
PsiComment(SimpleTokenType.COMMENT)('# test for illegal key attempt')(434,464)
- PsiWhiteSpace('\n')(464,465)
+ PsiElement(SimpleTokenType.CRLF)('\n')(464,465)
SimplePropertyImpl(PROPERTY)(465,468)
PsiElement(SimpleTokenType.KEY)('key')(465,468)
PsiErrorElement: