public Token(Token copy) { Text = copy.Text; Line = copy.Line; Position = copy.Position; }
/// <summary> /// Eval a token depending on the parser context /// </summary> /// <param name="evalContext">The token could be an identifier</param> /// <param name="evalKeyword">The token could be a keyword</param> /// <param name="position">Parser position</param> /// <returns>A keyword was found</returns> private bool EvalToken(bool evalContext, bool evalKeyword, int position) { bool hadContext = (context != 0); bool hadKeyword = (foundKeyword != 0); foundKeyword = 0; string token = curToken.Text; Debug.WriteLine("\t\t\t\t\t'"+token+"' @"+curToken.Position+" ("+evalKeyword+") after "+context); /* KEYWORD EVALUATION */ if (evalKeyword && (token.Length > 2)) { // members if (token == "var") { foundKeyword = FlagType.Variable; } else if (token == "function") { foundKeyword = FlagType.Function; } else if (haXe && token == "property") { foundKeyword = FlagType.Function; modifiers |= FlagType.HXProperty; } else if (version == 3 && token == "const") { foundKeyword = FlagType.Variable; modifiers |= FlagType.Constant; } else if (version == 3 && token == "namespace") { foundKeyword = FlagType.Namespace; } // class declaration else if (tryPackage && token == "package") { foundKeyword = FlagType.Package; if (version < 3) { version = 3; model.Namespaces.Insert(1, "protected"); model.Namespaces.Add("internal"); } } else if (token == "class") { foundKeyword = FlagType.Class; if (version == 1) version = 2; } else if (token == "interface") { foundKeyword = FlagType.Class; modifiers |= FlagType.Interface; if (version == 1) version = 2; } else if (haXe && token == "enum") { foundKeyword = FlagType.Enum; } // head declarations else if (token == "import") { foundKeyword = FlagType.Import; } // modifiers else { if (context == FlagType.Class) { if (token == "extends") { foundKeyword = FlagType.Class; curModifiers = FlagType.Extends; return true; } else if (token == "implements") { foundKeyword = FlagType.Class; curModifiers = FlagType.Implements; return true; } } // modifiers else if (context == FlagType.Function) { if (token == "get") { foundKeyword = FlagType.Function; curModifiers |= FlagType.Getter; return true; } else if (token == "set") { foundKeyword = FlagType.Function; curModifiers |= FlagType.Setter; return true; } } FlagType foundModifier = 0; // namespace modifier foreach(string ns in model.Namespaces) if (token == ns) { curNamespace = token; foundModifier = FlagType.Namespace; } // other modifiers if (foundModifier == 0) if (token == "static") { foundModifier = FlagType.Static; } else if (version <= 3 && token == "intrinsic") { foundModifier = FlagType.Intrinsic; } else if (version == 3 && token == "override") { foundModifier = FlagType.Override; } else if (version == 3 && token == "native") { foundModifier = FlagType.Intrinsic | FlagType.Native; } else if (version == 4 && token == "extern") { foundModifier = FlagType.Intrinsic | FlagType.Extern; } else if (token == "dynamic") { foundModifier = FlagType.Dynamic; } // a declaration modifier was recognized if (foundModifier != 0) { if (inParams && inValue) valueKeyword = new Token(curToken); inParams = false; inEnum = false; inValue = false; inType = false; foundColon = false; if (context != 0) { modifiers = 0; context = 0; } if (modifiers == 0) { modifiersLine = curToken.Line; //modifiersPos = curToken.Position; } modifiers |= foundModifier; return true; } } } // a declaration keyword was recognized if (foundKeyword != 0) { if (inParams && inValue) valueKeyword = new Token(curToken); inParams = false; inEnum = false; inValue = false; foundColon = false; context = foundKeyword; curModifiers = modifiers; curComment = lastComment; if (foundKeyword != FlagType.Import) lastComment = null; modifiers = 0; curMember = null; return true; } else if (!evalContext) return false; /* EVAL DECLARATION */ if (foundColon && curMember != null) { foundColon = false; //Debug.WriteLine(" "+curMember.Name+":/*"+lastComment+"*/"+curToken); curMember.Type = curToken.Text; curMember.LineTo = curToken.Line; // Typed Arrays if (curMember.Type == "Array" && lastComment != null && re_typedArray.IsMatch(lastComment)) { curMember.Type = "/*"+lastComment+"*/Array"; } } else if (hadContext && (hadKeyword || inParams)) { //if (curModifiers != 0) //Debug.WriteLine(" @"+modifiersLine+" "+modifiersPos+" *"); //else //Debug.WriteLine(" @"+curToken.Line+" "+curToken.Position); if (curComment != null && context != FlagType.Import) { curComment = curComment.Trim(); Match mTip = re_tip.Match(curComment+"@"); if (mTip.Success) { //Debug.WriteLine("@tiptext: "+re_cleanTip.Replace(mTip.Groups["tip"].Value, " ").Trim()); } ////Debug.WriteLine("\t/**"+curComment+"*/"); } MemberModel member; switch (context) { case FlagType.Package: //Debug.WriteLine("package "+token); // if (prevToken.Text == "package") { model.Package = token; } break; case FlagType.Namespace: //Debug.WriteLine("namespace "+token); // if (prevToken.Text == "namespace") { model.Namespaces.Add(token); } break; case FlagType.Import: //Debug.WriteLine("import "+token+" ("+prevToken.Text+")"); // if (prevToken.Text == "import") { member = new MemberModel(); member.Name = LastStringToken(token, "."); member.Type = token; model.Imports.Add(member); } break; case FlagType.Class: if (curModifiers == FlagType.Extends) { //Debug.WriteLine("extends "+token); // TODO Get extended class from context /*if (curClass != null) { ClassModel exClass = new ClassModel(); exClass.ClassName = token; curClass.Extends = exClass; }*/ } else if (curModifiers == FlagType.Implements) { //Debug.WriteLine("implements "+token); /*if (curClass != null) { if (curClass.Implements == null) curClass.Implements = new ArrayList(); // TODO Get implemented class from context ClassModel impClass = new ClassModel(); impClass.ClassName = token; curClass.Implements.Add(impClass); }*/ } else if (prevToken.Text == "class" || prevToken.Text == "interface") { //Debug.WriteLine(curNamespace+" "+curModifiers+" class "+token); // if (curClass != null) { curClass.LineTo = (curModifiers != 0) ? modifiersLine-1 : curToken.Line-1; } // check classname int p = token.LastIndexOf('.'); if (p > 0) { if (version < 3) { string package = token.Substring(0, p); model.Package = package; token = token.Substring(p+1); } else { //TODO Error: AS3 & haXe classes are qualified by their package declaration } } curClass = new ClassModel(); model.Classes.Add(curClass); curClass.InFile = model; curClass.Comments = curComment; curClass.ClassName = token; curClass.Constructor = (haXe) ? "new" : token; curClass.Flags = curModifiers; curClass.Namespace = curNamespace; curClass.LineFrom = (curModifiers != 0) ? modifiersLine : curToken.Line; curClass.LineTo = curToken.Line; } else { context = 0; modifiers = 0; } break; case FlagType.Enum: if (inEnum) { //Debug.WriteLine("value "+token); // member = new MemberModel(); member.Comments = curComment; member.Name = token; member.Flags = curModifiers | FlagType.Variable | FlagType.Enum; member.Namespace = curNamespace; member.LineFrom = member.LineTo = curToken.Line; //curClass.Vars.Add(member); curClass.Members.Add(member); // curMember = member; } else { //Debug.WriteLine(curNamespace+" "+curModifiers+" enum "+token); // if (curClass != null) { curClass.LineTo = (curModifiers != 0) ? modifiersLine-1 : curToken.Line-1; } curClass = new ClassModel(); model.Enums.Add(curClass); curClass.InFile = model; curClass.Comments = curComment; curClass.ClassName = token; curClass.Flags = curModifiers | FlagType.Enum; curClass.Namespace = curNamespace; curClass.LineFrom = (curModifiers != 0) ? modifiersLine : curToken.Line; curClass.LineTo = curToken.Line; } break; case FlagType.Variable: /*if (inParams && curMethod != null) { if (inEnum) Debug.WriteLine(" *"+curMethod.Name+"("+token); else Debug.WriteLine(" "+curMethod.Name+"("+token); } else Debug.WriteLine(curNamespace+" "+curModifiers+" var "+token);*/ // // haXe type: T -> T if (haXe && curMember != null && curMember.Type != null && curMember.Type.EndsWith("->")) { curMember.Type += " "+token; return false; } else { member = new MemberModel(); member.Comments = curComment; member.Name = token; member.Flags = curModifiers | FlagType.Variable; member.Namespace = curNamespace; member.LineFrom = (curModifiers != 0) ? modifiersLine : curToken.Line; member.LineTo = curToken.Line; // // method parameter if (inParams && curMethod != null) { if (inEnum) member.Flags |= FlagType.Enum; if (curMethod.Parameters == null) curMethod.Parameters = new ArrayList(); curMethod.Parameters.Add(member); } // class member else if (curClass != null) { //curClass.Vars.Add(member); curClass.Members.Add(member); } // package member else model.Members.Add(member); // curMember = member; } break; case FlagType.Function: //Debug.WriteLine("{"+curNamespace+" "+curModifiers+" function "+token); // member = new MemberModel(); member.Comments = curComment; member.Name = token; member.Flags = curModifiers | FlagType.Function; member.Namespace = curNamespace; member.LineFrom = (curModifiers != 0) ? modifiersLine : curToken.Line; member.LineTo = curToken.Line; // if (curClass != null) { if (token == curClass.Constructor) member.Flags |= FlagType.Constructor; //if ((curModifiers & (FlagType.Getter | FlagType.Setter)) > 0) curClass.Properties.Add(member); //else curClass.Methods.Add(member); curClass.Members.Add(member); } else model.Members.Add(member); // curMember = member; break; } if (context != FlagType.Function && !inParams) curMethod = null; curComment = null; curNamespace = "public"; modifiers = 0; tryPackage = false; } return false; }
private void ParseSrc(FileModel fileModel, string ba) { model = fileModel; if (model.haXe) model.Enums.Clear(); model.Classes.Clear(); model.Members.Clear(); // state int len = ba.Length; if (len < 0) return; int i = 0; int line = 0; // when parsing cache file including multiple files resetParser: char c1; char c2; int matching = 0; bool isInString = false; int inString = 0; int braceCount = 0; bool inCode = true; // comments char[] commentBuffer = new char[COMMENTS_BUFFER]; int commentLength = 0; lastComment = null; curComment = null; // tokenisation tryPackage = true; haXe = model.haXe; version = (haXe)? 4 : 1; curToken = new Token(); prevToken = new Token(); int tokPos = 0; int tokLine = 0; curMethod = null; curMember = null; valueKeyword = null; curModifiers = 0; curNamespace = "public"; char[] buffer = new char[TOKEN_BUFFER]; int length = 0; char[] valueBuffer = new char[VALUE_BUFFER]; int valueLength = 0; int paramBraceCount = 0; int paramTempCount = 0; int paramParCount = 0; int paramSqCount = 0; bool hadWS = true; bool hadDot = false; inParams = false; inEnum = false; inValue = false; inType = false; bool addChar = false; int evalToken = 0; bool evalKeyword = true; context = 0; modifiers = 0; foundColon = false; while (i < len) { c1 = ba[i++]; isInString = (inString > 0); /* MATCH COMMENTS / STRING LITTERALS */ switch (matching) { // look for comment block/line and preprocessor commands case 0: if (!isInString) { // new comment if (c1 == '/') { c2 = ba[i]; if (c2 == '/') { matching = 1; inCode = false; i++; continue; } else if (c2 == '*') { matching = 2; inCode = false; i++; while (i < len-1) { c2 = ba[i]; if (c2 == '*' && ba[i+1] != '/') i++; else break; } continue; } } // don't look for comments in strings else if (c1 == '"') { isInString = true; inString = 1; } else if (c1 == '\'') { isInString = true; inString = 2; } // preprocessor statements else if (c1 == '#') { c2 = ba[i]; if (i < 2 || ba[i-2] < 33 && c2 >= 'a' && c2 <= 'z') { matching = 3; inCode = false; continue; } } } // end of string else if (isInString) { if (c1 == '\\') i++; else if ((inString == 1) && (c1 == '"')) inString = 0; else if ((inString == 2) && (c1 == '\'')) inString = 0; } break; // skip commented line case 1: if (c1 == 10 || c1 == 13) { // ignore single comments commentLength = 0; inCode = true; matching = 0; } break; // skip commented block case 2: if (c1 == '*') { bool end = false; while (i < len) { c2 = ba[i]; if (c2 == '/') { end = true; break; } else if (c2 == '*') i++; else break; } if (end) { lastComment = (commentLength > 0) ? new string(commentBuffer, 0, commentLength) : null; // TODO parse for TODO statements? commentLength = 0; inCode = true; matching = 0; i++; continue; } } break; // directive/preprocessor statement case 3: if (c1 == 10 || c1 == 13) { if (commentLength > 0) { string directive = new string(commentBuffer, 0, commentLength); // TODO Store haXe directives in model // FD cache custom directive if (parentPath != null && directive.StartsWith("file-cache ")) { model.Version = version; model = new FileModel(directive.Substring(11), cacheLastWriteTime); parentPath.AddFile(model); goto resetParser; } } commentLength = 0; inCode = true; matching = 0; } break; } /* LINE/COLUMN NUMBER */ if (c1 == 10 || c1 == 13) { line++; if (c1 == 13 && ba[i] == 10) i++; } /* SKIP CONTENT */ if (!inCode) { // store comments if (matching == 2 || (matching == 3 && (haXe || parentPath != null))) { if (commentLength < COMMENTS_BUFFER) commentBuffer[commentLength++] = c1; } continue; } else if (isInString) { // store parameter default value if (inParams && inValue && valueLength < VALUE_BUFFER) valueBuffer[valueLength++] = c1; continue; } if (braceCount > 0) { if (c1 == '}') { braceCount--; if (braceCount == 0 && curMethod != null) { //Debug.WriteLine("} "+curMethod.Name+" @"+line); curMethod.LineTo = line; curMethod = null; } } else if (c1 == '{') braceCount++; continue; } /* PARSE DECLARATION VALUES/TYPES */ if (inValue) { if (c1 == '{') paramBraceCount++; else if (c1 == '}'&& paramBraceCount > 0) { paramBraceCount--; if (paramBraceCount == 0 && paramParCount == 0 && paramSqCount == 0 && valueLength < VALUE_BUFFER) valueBuffer[valueLength++] = '}'; } else if (c1 == '(') paramParCount++; else if (c1 == ')'&& paramParCount > 0) paramParCount--; else if (c1 == '(') paramSqCount++; else if (c1 == ')' && paramSqCount > 0) paramSqCount--; else if (paramTempCount > 0) { if (c1 == '<') paramTempCount++; else if (c1 == '>') paramTempCount--; } // end of value if ( paramBraceCount == 0 && paramParCount == 0 && paramSqCount == 0 && paramTempCount == 0 && (c1 == ',' || c1 == ';' || c1 == '}' || (inParams && c1 == ')') || inType) ) { inValue = false; length = 0; if (inType) valueBuffer[valueLength++] = c1; } // in params, store the default value else if ((inParams || inType) && valueLength < VALUE_BUFFER) { if (c1 < 32) { if (valueLength > 0 && valueBuffer[valueLength-1] != ' ') valueBuffer[valueLength++] = ' '; } else valueBuffer[valueLength++] = c1; } // only try to detect keywords if (c1 < 'a' || c1 > 'z') { hadWS = true; continue; } } // store parameter value if (!inValue && valueLength > 0) { string param = new string(valueBuffer, 0, valueLength); if (valueKeyword != null) { int p = param.LastIndexOf(valueKeyword.Text); if (p > 0) param = param.Substring(0,p).TrimEnd(); } // if (curMember == null) { if (inType) { prevToken.Text = curToken.Text; prevToken.Line = curToken.Line; prevToken.Position = curToken.Position; curToken.Text = param; curToken.Line = tokLine; curToken.Position = tokPos; EvalToken(true, false, i-1-valueLength); evalKeyword = true; evalToken = 0; } } else if (inType) { //Debug.WriteLine(" : "+param); // foundColon = false; curMember.Type = param; } else { //Debug.WriteLine(" = "+param); // curMember.Parameters = new ArrayList(); curMember.Parameters.Add(param); } // valueLength = 0; length = 0; } /* TOKENIZATION */ // whitespace if (c1 <= 32) { hadWS = true; continue; } // a dot can be in an identifier if (c1 == '.') { if (length > 0 || (inParams && version == 3)) { hadWS = false; hadDot = true; addChar = true; } else continue; } else { // should we evaluate the token? if (hadWS && !hadDot && length > 0) { evalToken = (evalKeyword) ? 1 : 2; } hadWS = false; hadDot = false; // valid char for keyword if (c1 >= 'a' && c1 <= 'z') { addChar = true; } else { // valid chars for identifiers if (c1 >= 'A' && c1 <= 'Z') { addChar = true; } else if (c1 == '$' || c1 == '_') { addChar = true; } else if (length > 0) { if (c1 >= '0' && c1 <= '9') { addChar = true; } else if (c1 == '*' && context == FlagType.Import) { addChar = true; } // haXe template types else if (haXe && c1 == '<') { inValue = true; inType = true; valueLength = 0; for(int j=0; j<length; j++) valueBuffer[valueLength++] = buffer[j]; valueBuffer[valueLength++] = c1; length = 0; paramBraceCount = 0; paramParCount = 0; paramSqCount = 0; paramTempCount = 1; continue; } else { evalToken = (evalKeyword) ? 1 : 2; } } // star is valid in import statements else if (c1 == '*' && version == 3 && !foundColon) { addChar = true; } // these chars are not valid in keywords if (length > 0) evalKeyword = false; } // eval this word if (evalToken > 0) { prevToken.Text = curToken.Text; prevToken.Line = curToken.Line; prevToken.Position = curToken.Position; curToken.Text = new string(buffer, 0, length); curToken.Line = tokLine; curToken.Position = tokPos; EvalToken(!inValue, (evalToken == 1), i-1-length); length = 0; evalKeyword = true; evalToken = 0; } // start of block if (c1 == '{') { if (context == FlagType.Package || context == FlagType.Class) { } else if (context == FlagType.Enum) { inEnum = true; } else if (foundColon && haXe && length == 0) { inValue = true; inType = true; valueLength = 0; valueBuffer[valueLength++] = c1; paramBraceCount = 1; paramParCount = 0; paramSqCount = 0; paramTempCount = 0; continue; } else { braceCount++; //Debug.WriteLine("{"+braceCount); } } // end of block else if (c1 == '}') { // outside of a method, the '}' ends the current class if (curClass != null) { //Debug.WriteLine("} class "+curClass+" @"+line); if (curClass != null) curClass.LineTo = line; curClass = null; tryPackage = true; } } // member type declaration else if (c1 == ':') { foundColon = true; } // next variable declaration else if (c1 == ',') { if (context == FlagType.Variable) foundKeyword = FlagType.Variable; } else if (c1 == '(') { // beginning of method parameters if (context == FlagType.Function) { context = FlagType.Variable; inParams = true; if (curMember == null) { context = FlagType.Function; if ((curModifiers & FlagType.Getter) > 0) { curModifiers -= FlagType.Getter; EvalToken(true, false, i); curMethod = curMember; } else if ((curModifiers & FlagType.Setter) > 0) { curModifiers -= FlagType.Setter; EvalToken(true, false, i); curMethod = curMember; } else { inParams = false; context = 0; } } else { curMethod = curMember; Debug.WriteLine("{ "+curMember.Name); } } // an Enum value with parameters else if (inEnum && curToken != null) { //Debug.WriteLine("********** enum construc "+curToken); context = FlagType.Variable; inParams = true; curMethod = new MemberModel(); curMethod.Name = curToken.Text; curMethod.Flags = curModifiers | FlagType.Function | FlagType.Enum; // if (curClass != null) curClass.Members.Add(curMethod); } } // end of statement else if (c1 == ';') { context = 0; modifiers = 0; inParams = false; curMember = null; //curMethod = null; } // end of method parameters else if (c1 == ')' && inParams) { context = (inEnum) ? FlagType.Enum : 0; modifiers = 0; inParams = false; curMember = curMethod; } // skip value of a declared variable else if (c1 == '=' && context == FlagType.Variable) { if (!inValue) { inValue = true; inType = false; paramBraceCount = 0; paramParCount = 0; paramSqCount = 0; paramTempCount = 0; valueLength = 0; } } // special haXe types: T -> T else if (haXe && c1 == '-' && curMember != null) { if (ba[i] == '>') { curMember.Type += " ->"; } } } // put in buffer if (addChar) { if (length < TOKEN_BUFFER) buffer[length++] = c1; if (length == 1) { tokPos = i-1; tokLine = line; } addChar = false; } } // parsing done! model.Version = version; }