Tokenize() public method

Tokenizes the given input and returns the list of tokens found
public Tokenize ( String str, String source ) : IList
str String
source String
return IList
Exemplo n.º 1
0
		/// <summary>
		/// Loads the requested script and converts it to an AST
		/// </summary>
		/// <param name="name"></param>
		/// <returns></returns>
		private IList<AbstractNode> _loadImportPath( string name )
		{
			IList<AbstractNode> retval = null;
			IList<ConcreteNode> nodes = null;

			if ( this.OnImportFile != null )
				this.OnImportFile( this, name );

			if ( nodes != null && ResourceGroupManager.Instance != null )
			{
				using ( Stream stream = ResourceGroupManager.Instance.OpenResource( name, _resourceGroup ) )
				{
					if ( stream != null )
					{
						ScriptLexer lexer = new ScriptLexer();
						ScriptParser parser = new ScriptParser();
						IList<ScriptToken> tokens = null;
						using ( StreamReader reader = new StreamReader( stream ) )
						{
							tokens = lexer.Tokenize( reader.ReadToEnd(), name );
						}
						nodes = parser.Parse( tokens );
					}
				}
			}

			if ( nodes != null )
				retval = _convertToAST( nodes );

			return retval;
		}
Exemplo n.º 2
0
		/// <summary>
		/// Takes in a string of script code and compiles it into resources
		/// </summary>
		/// <param name="script">The script code</param>
		/// <param name="source">The source of the script code (e.g. a script file)</param>
		/// <param name="group">The resource group to place the compiled resources into</param>
		/// <returns></returns>
		public bool Compile( String script, String source, String group )
		{
			ScriptLexer lexer = new ScriptLexer();
			ScriptParser parser = new ScriptParser();
			IList<ScriptToken> tokens = lexer.Tokenize( script, source );
			IList<ConcreteNode> nodes = parser.Parse( tokens );
			return Compile( nodes, group );
		}
Exemplo n.º 3
0
		/// <summary>
		/// Handles processing the variables
		/// </summary>
		/// <param name="nodes"></param>
		private void _processVariables( ref IList<AbstractNode> nodes )
		{
			for ( int i = 0; i < nodes.Count; ++i )
			{
				AbstractNode cur = nodes[ i ];

				if ( cur is ObjectAbstractNode )
				{
					// Only process if this object is not abstract
					ObjectAbstractNode obj = (ObjectAbstractNode)cur;
					if ( !obj.IsAbstract )
					{
						_processVariables( ref obj.Children );
						_processVariables( ref obj.Values );
					}
				}
				else if ( cur is PropertyAbstractNode )
				{
					PropertyAbstractNode prop = (PropertyAbstractNode)cur;
					_processVariables( ref prop.Values );
				}
				else if ( cur is VariableGetAbstractNode )
				{
					VariableGetAbstractNode var = (VariableGetAbstractNode)cur;

					// Look up the enclosing scope
					ObjectAbstractNode scope = null;
					AbstractNode temp = var.Parent;
					while ( temp != null )
					{
						if ( temp is ObjectAbstractNode )
						{
							scope = (ObjectAbstractNode)temp;
							break;
						}
						temp = temp.Parent;
					}

					// Look up the variable in the environment
					KeyValuePair<bool, string> varAccess = new KeyValuePair<bool, string>( false, string.Empty );
					if ( scope != null )
						varAccess = scope.GetVariable( var.Name );

					if ( scope == null || !varAccess.Key )
					{
						bool found = _environment.ContainsKey( var.Name );
						if ( found )
							varAccess = new KeyValuePair<bool, string>( true, _environment[ var.Name ] );
						else
							varAccess = new KeyValuePair<bool, string>( false, varAccess.Value );
					}

					if ( varAccess.Key )
					{
						// Found the variable, so process it and insert it into the tree
						ScriptLexer lexer = new ScriptLexer();
						IList<ScriptToken> tokens = lexer.Tokenize( varAccess.Value, var.File );
						ScriptParser parser = new ScriptParser();
						IList<ConcreteNode> cst = parser.ParseChunk( tokens );
						IList<AbstractNode> ast = _convertToAST( cst );

						// Set up ownership for these nodes
						foreach ( AbstractNode currentNode in ast )
							currentNode.Parent = var.Parent;

						// Recursively handle variable accesses within the variable expansion
						_processVariables( ref ast );

						// Insert the nodes in place of the variable
						for ( int j = 0; j < ast.Count; j++ )
							nodes.Insert( j, ast[ j ] );
					}
					else
					{
						// Error
						AddError( CompileErrorCode.UndefinedVariable, var.File, var.Line );
					}

					// Remove the variable node
					nodes.RemoveAt( i );
					i--;
				}
			}
		}
Exemplo n.º 4
0
		/// <summary>
		/// Takes in a string of script code and compiles it into resources
		/// </summary>
		/// <param name="script">The script code</param>
		/// <param name="source">The source of the script code (e.g. a script file)</param>
		/// <param name="group">The resource group to place the compiled resources into</param>
		/// <returns></returns>
		public bool Compile( String script, String source, String group )
		{
			var lexer = new ScriptLexer();
			var parser = new ScriptParser();
			var tokens = lexer.Tokenize( script, source );
			var nodes = parser.Parse( tokens );
			return Compile( nodes, group );
		}