private void AddToken(ParticleEmitterToken token) { if (token.StringValue != null) { if (token.StringValue.Length > 0) { DetermineTokenType(ref token); TokenVector.Add(token); } } }
private void DetermineTokenType(ref ParticleEmitterToken token) { token.Type = TokenType.Unknown; // these things are easy to see... if (char.IsDigit(token.StringValue[0]) || token.StringValue[0] == '-') { token.Type = TokenType.RealNumber; return; } if (token.StringValue[0] == '=') { token.Type = TokenType.Equals; return; } if (token.StringValue[0] == ',') { token.Type = TokenType.Comma; return; } if (token.StringValue[0] == '(') { token.Type = TokenType.OpenParen; return; } if (token.StringValue[0] == ')') { token.Type = TokenType.CloseParen; return; } if (token.StringValue[0] == '{') { token.Type = TokenType.OpenBrace; return; } if (token.StringValue[0] == '}') { token.Type = TokenType.CloseBrace; return; } if (token.StringValue[0] == '\"') { token.Type = TokenType.Quote; return; } // if we got here, it's not a quote... so convert it to uppercase. token.StringValue = token.StringValue.ToUpper(); // keywords are easy to figure out, just look for the text... if (token.StringValue.Contains("PARTICLESYSTEM")) { token.Type = TokenType.KeywordParticleSystem; return; } if (token.StringValue.Contains("EVENTSEQUENCE")) { token.Type = TokenType.KeywordEventSequence; return; } if (token.StringValue.Contains("RANDOM")) { token.Type = TokenType.KeywordRandom; return; } if (token.StringValue.Contains("XYZ")) { token.Type = TokenType.KeywordXYZ; return; } if (token.StringValue.Contains("RGBA")) { token.Type = TokenType.KeywordColor; return; } if (token.StringValue.Contains("FADE")) { token.Type = TokenType.KeywordFade; return; } if (token.StringValue.Contains("INITIAL")) { token.Type = TokenType.KeywordInitial; return; } if (token.StringValue.Contains("FINAL")) { token.Type = TokenType.KeywordFinal; return; } if (token.StringValue.Contains("TEXTURE")) { token.Type = TokenType.KeywordTexture; return; } // these two keywords are embedded in other words, so we've got to be careful. if (token.StringValue.Contains("SO") && token.StringValue.Length == 2) { token.Type = TokenType.KeywordSo; return; } if (token.StringValue.Contains("AT") && token.StringValue.Length == 2) { token.Type = TokenType.KeywordAt; return; } // now it gets harder... if (token.StringValue.Contains("D3DBLEND_")) { token.Type = TokenType.AlphaBlendMode; return; } if (token.StringValue.Contains("SOURCEBLENDMODE") || token.StringValue.Contains("DESTBLENDMODE")) { token.Type = TokenType.SeqAlphaBlendModeProp; return; } if (token.StringValue.Contains("LIFETIME") || token.StringValue.Contains("EMITRATE") || token.StringValue.Contains("NUMPARTICLES") || token.StringValue.Contains("LOOPS")) { token.Type = TokenType.SeqNumericProp; return; } if (token.StringValue.Contains("SPAWNDIR") || token.StringValue.Contains("GRAVITY") || token.StringValue.Contains("EMITRADIUS")) { token.Type = TokenType.SeqVectorProp; return; } if (token.StringValue.Contains("POSITION")) { token.Type = TokenType.SysVectorProp; return; } if (token.StringValue.Contains("SIZE") || token.StringValue.Contains("EVENTTIMER")) { token.Type = TokenType.ParticleNumericProp; return; } if (token.StringValue.Contains("VELOCITY")) { token.Type = TokenType.ParticleVectorProp; return; } if (token.StringValue.Contains("COLOR")) { token.Type = TokenType.ParticleColorProp; return; } }
public void Tokenize(string str) { var cs = TokenizerState.InWhiteSpace; string remainingText = str; var curCharacter = 0; ParticleEmitterToken token = new ParticleEmitterToken(); while (curCharacter < remainingText.Length) { switch (cs) { case TokenizerState.InWhiteSpace: { // if this letter is not whitespace, if (!char.IsWhiteSpace(str[curCharacter]) && !char.IsControl(str[curCharacter])) { // add it to the running buffer token.StringValue += str[curCharacter]; // switch to appropriate case if (str[curCharacter] == '/' && str[curCharacter + 1] == '/') { cs = TokenizerState.InComment; } else { cs = (str[curCharacter] == '\"') ? TokenizerState.InQuote : TokenizerState.InText; } } } break; case TokenizerState.InText: { // if this letter is whitespace if (char.IsWhiteSpace(str[curCharacter])) { // add the completed token to the vector AddToken(token); // Reset the token token = new ParticleEmitterToken(); // switch to whitespace case cs = TokenizerState.InWhiteSpace; } else { // if this letter is a token terminator if (str[curCharacter] == '(' || str[curCharacter] == ')' || str[curCharacter] == ',' || str[curCharacter] == '\"' || str[curCharacter] == '{' || str[curCharacter] == '}' || str[curCharacter] == '/') { if (str[curCharacter] == '/' && str[curCharacter + 1] == '/') { cs = TokenizerState.InComment; } else { // add the completed token to the vector AddToken(token); // Reset the token token = new ParticleEmitterToken(); // if it was a quote, transition to InQuote state if (str[curCharacter] == '\"') { cs = TokenizerState.InQuote; } // otherwise, process this one char as a token else { token.StringValue = str[curCharacter].ToString(); AddToken(token); token = new ParticleEmitterToken { StringValue = string.Empty }; } } } else { // add this letter to the work in progress token token.StringValue += str[curCharacter]; } } } break; case TokenizerState.InComment: { // C++ style comments - skip everything until end of line if (str[curCharacter] == '\n') { token.StringValue = string.Empty; cs = TokenizerState.InWhiteSpace; } } break; case TokenizerState.InQuote: { // unconditionally add this letter to the token until we hit a close quote token.StringValue += str[curCharacter]; if (str[curCharacter] == '\"') { AddToken(token); // Reset the token token = new ParticleEmitterToken(); // go back to whitespace cs = TokenizerState.InWhiteSpace; } } break; } curCharacter++; } AddToken(token); }