This class supplies character escape utilities for project.
示例#1
0
        internal Terminal LookupOrDefineTerminal(Token token, string name, string alias, LexSpan span)
        {
            bool     isIdent = (token == Token.ident);
            Terminal result  = null;

            // Canonicalize escaped char-literals
            if (!isIdent)
            {
                name = CharacterUtilities.CanonicalizeCharacterLiteral(name, 1);
            }
            else if (alias != null)
            {
                alias = CharacterUtilities.CanonicalizeAlias(alias);
            }
            // Check if already present in dictionary
            if (!terminals.ContainsKey(name))      // terminal not already known
            {
                result          = new Terminal(isIdent, name, alias);
                terminals[name] = result;
                if (alias != null)
                {
                    CheckAndSetAlias(alias, result, span);
                }
            }
            else
            {
                result = terminals[name];
                if (alias != null)
                {
                    CheckAlias(alias, result, span);
                }
            }
            return(result);
        }
示例#2
0
        internal static bool BumpsMax(string str)
        {
            string num = CharacterUtilities.CanonicalizeCharacterLiteral(str, 1);
            int    ord = CharacterUtilities.OrdinalOfCharacterLiteral(str, 1);

            return(ord > Terminal.max);
        }
示例#3
0
        internal static void InsertMaxDummyTerminalInDictionary(Dictionary <string, Terminal> table)
        {
            Terminal maxTerm = null;

            if (Terminal.Max != 0)
            {
                string maxChr = CharacterUtilities.QuoteMap(Terminal.Max);   // FIXME
                maxTerm = table[maxChr];
            }
            table["@Max@"] = maxTerm;
        }
示例#4
0
 public override string ToString()
 {
     if (this.alias != null)
     {
         return(CharacterUtilities.QuoteAndCanonicalize(this.alias));
     }
     else
     {
         return(base.ToString());
     }
 }
示例#5
0
        internal Terminal LookupTerminal(Token token, string name)
        {
            bool isIdent = (token == Token.ident);

            // Canonicalize escaped char-literals
            if (!isIdent)
            {
                name = CharacterUtilities.CanonicalizeCharacterLiteral(name, 1);
            }
            // Check if already present in dictionary
            if (!terminals.ContainsKey(name))  // else insert ...
            {
                terminals[name] = new Terminal(isIdent, name);
            }
            return(terminals[name]);
        }
示例#6
0
 /// <summary>
 /// If name is an escaped char-lit, it must already be
 /// canonicalized according to some convention. In this
 /// application CharUtils.Canonicalize().
 /// </summary>
 /// <param name="symbolic">Means "is an ident"</param>
 /// <param name="name">string representation of symbol</param>
 internal Terminal(bool symbolic, string name)
     : base(name)
 {
     this.symbolic = symbolic;
     if (symbolic)
     {
         this.n = ++count;
     }
     else
     {
         this.n = CharacterUtilities.OrdinalOfCharacterLiteral(name, 1);
         if (n > max)
         {
             max = n;
         }
     }
 }
示例#7
0
        internal Terminal LookupOrDefineTerminal(Token token, string name, string alias)
        {
            bool isIdent = (token == Token.ident);

            // Canonicalize escaped char-literals
            if (!isIdent)
            {
                name = CharacterUtilities.Canonicalize(name, 1);
            }
            // Check if already present in dictionary
            if (!terminals.ContainsKey(name))     // else insert ...
            {
                Terminal newTerm = new Terminal(isIdent, name, alias);
                terminals[name] = newTerm;
                if (alias != null)
                {
                    aliasTerms[alias] = newTerm;
                }
            }
            return(terminals[name]);
        }
示例#8
0
        /// <summary>
        /// This static method expands characters in a literal
        /// string, returning a modified string with character
        /// escapes replaced by the character which they denote.
        /// This includes characters outside the BMP which
        /// return a pair of surrogate characters.
        /// </summary>
        /// <param name="source">the input string</param>
        /// <returns>interpreted version of the string</returns>
        public static string InterpretCharacterEscapes(string source)
        {
            int sLen = source.Length;

            if (sLen == 0)
            {
                return(source);
            }
            char[] arr  = new char[sLen];
            int    sNxt = 0;
            int    aNxt = 0;
            char   chr  = source[sNxt++];

            for (; ; chr = source[sNxt++])
            {
                if (chr != '\\')
                {
                    arr[aNxt++] = chr;
                }
                else
                {
                    int codePoint = EscapedChar(source, ref sNxt);
                    if (codePoint > 0xFFFF)
                    {
                        arr[aNxt++] = CharacterUtilities.HiSurrogate(codePoint);
                        arr[aNxt++] = CharacterUtilities.LoSurrogate(codePoint);
                    }
                    else
                    {
                        arr[aNxt++] = (char)codePoint;
                    }
                }
                if (sNxt == sLen)
                {
                    return(new String(arr, 0, aNxt));
                }
            }
        }
        private void GenerateInitializeMethod(
            List <AutomatonState> states,
            List <Production> productions,
            Dictionary <string, NonTerminal> nonTerminals)
        {
            // warning 649 : this field never assigned to.
            Console.WriteLine("#pragma warning disable 649");
            Console.WriteLine("  private static Dictionary<int, string> aliasses;");
            Console.WriteLine("#pragma warning restore 649");
            Console.WriteLine("  private static Rule[] rules = new Rule[{0}];", productions.Count + 1);
            Console.WriteLine("  private static State[] states = new State[{0}];", states.Count);
            Console.WriteLine("  private static string[] nonTerms = new string[] {");

            int length = 0;

            Console.Write("      ");
            foreach (NonTerminal nonTerminal in nonTerminals.Values)
            {
                string ss = String.Format(CultureInfo.InvariantCulture, "\"{0}\", ", nonTerminal.ToString());
                length += ss.Length;
                Console.Write(ss);
                if (length > 70)
                {
                    Console.WriteLine();
                    Console.Write("      ");
                    length = 0;
                }
            }
            Console.WriteLine("};");
            Console.WriteLine();

            Console.WriteLine("  static {0}() {{", grammar.ParserName);
            int state_nr = 0;

            foreach (AutomatonState state in states)
            {
                GenerateShiftReduceMachineState(state_nr++, state);
            }
            Console.WriteLine();

            Console.WriteLine("    for (int sNo = 0; sNo < states.Length; sNo++) states[sNo].number = sNo;");

            Console.WriteLine();
            foreach (Production production in productions)
            {
                GenerateShiftReduceMachineRule(production);
            }

            List <Terminal> aliasList = new List <Terminal>();

            foreach (KeyValuePair <string, Terminal> pair in grammar.terminals)
            {
                Terminal term = pair.Value;
                if (term.Alias != null)
                {
                    aliasList.Add(term);
                }
            }
            if (aliasList.Count > 0)
            {
                Console.WriteLine();
                Console.WriteLine("    aliasses = new Dictionary<int, string>();");
                foreach (Terminal termWithAlias in aliasList)
                {
                    Console.WriteLine("    aliasses.Add({0}, {1});",
                                      termWithAlias.num,
                                      CharacterUtilities.QuoteMap(termWithAlias.Alias));
                }
            }
            Console.WriteLine("  }");
            Console.WriteLine();

            Console.WriteLine("  protected override void Initialize() {");
            Console.WriteLine("    this.InitSpecialTokens((int){0}.error, (int){0}.EOF);", grammar.TokenName);
            Console.WriteLine("    this.InitStates(states);");
            Console.WriteLine("    this.InitRules(rules);");
            Console.WriteLine("    this.InitNonTerminals(nonTerms);");
            Console.WriteLine("  }");
            Console.WriteLine();
        }
示例#10
0
文件: Main.cs 项目: kzyg/spark
        private static void Main(string[] args)
        {
            Stream       inputFile = null;
            Grammar      grammar   = null;
            ErrorHandler handler   = new ErrorHandler();

            Lexers.Scanner scanner = null;
            Parser.Parser  parser  = null;

            Assembly assm = Assembly.GetExecutingAssembly();
            object   info = Attribute.GetCustomAttribute(assm, typeof(AssemblyFileVersionAttribute));

            versionInfo = ((AssemblyFileVersionAttribute)info).Version;

            try
            {
                string filename = ProcessOptions(args);

                if (filename == null)
                {
                    return;
                }

                try
                {
                    inputFile = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read);
                }
                catch (IOException)
                {
                    inputFile = null;
                    string message = String.Format(CultureInfo.InvariantCulture, "Source file <{0}> not found{1}", filename, Environment.NewLine);
                    handler.AddError(message, null); // aast.AtStart;
                    throw;
                }

                scanner = new Lexers.Scanner(inputFile);
                scanner.SetHandler(handler);

                parser = new Parser.Parser(filename, scanner, handler);
                //
                // If the parse is successful, then process the grammar.
                // Otherwise just report the errors that have been listed.
                //
                if (parser.Parse())
                {
                    grammar = parser.Grammar;

                    if (Terminal.Max > 255)
                    {
                        handler.ListError(null, 103, CharacterUtilities.Map(Terminal.Max), '\'');
                    }

                    LALRGenerator         generator = new LALRGenerator(grammar);
                    List <AutomatonState> states    = generator.BuildStates();
                    generator.ComputeLookAhead();
                    generator.BuildParseTable();
                    if (!grammar.CheckGrammar(handler))
                    {
                        throw new ArgumentException("Non-terminating grammar");
                    }
                    //
                    // If the grammar has non-terminating non-terms we cannot
                    // create a diagnostic report as the grammar is incomplete.
                    //
                    bool DoDiagnose = Diagnose && !grammar.HasNonTerminatingNonTerms;

                    if (Report || DoDiagnose)
                    {
                        string htmlName = System.IO.Path.ChangeExtension(filename, ".report.html");
                        try
                        {
                            System.IO.FileStream   htmlFile   = new System.IO.FileStream(htmlName, System.IO.FileMode.Create);
                            System.IO.StreamWriter htmlWriter = new System.IO.StreamWriter(htmlFile);
                            Grammar.HtmlHeader(htmlWriter, filename);

                            if (Report && DoDiagnose)
                            {
                                grammar.GenerateCompoundReport(htmlWriter, filename, states);
                            }
                            else if (Report)
                            {
                                grammar.GenerateReport(htmlWriter, filename, states);
                            }

                            Grammar.HtmlTrailer(htmlWriter);

                            if (htmlFile != null)
                            {
                                htmlWriter.Flush();
                                htmlFile.Close();
                            }
                        }
                        catch (System.IO.IOException)
                        {
                            Console.Error.WriteLine("Cannot create html output file {0}", htmlName);
                        }
                    }
                    else if (!handler.Errors)
                    {
                        CodeGenerator code = new CodeGenerator();
                        code.Generate(states, grammar);
                    }
                }
            }
            catch (System.Exception e)
            {
                Console.Error.WriteLine("Unexpected Error {0}", e.Message);
                throw; // Now rethrow the caught exception.
            }
            finally
            {
                if ((handler.Errors || handler.Warnings) && scanner != null)
                {
                    handler.DumpAll(scanner.Buffer, Console.Error);
                }
                if (Listing || handler.Errors || handler.Warnings)
                {
                    string       listName   = parser.ListfileName;
                    StreamWriter listStream = ListingFile(listName);
                    if (listStream != null)
                    {
                        handler.MakeListing(scanner.Buffer, listStream, parser.SourceFileName, versionInfo);
                    }
                }
            }
        }
示例#11
0
文件: Main.cs 项目: traviolia/gppg
        private static int Main(string[] args)
        {
            Stream inputFile = null;

            Grammar      grammar       = null;
            ErrorHandler handler       = new ErrorHandler();
            string       inputFileInfo = null; // Filename plus revision time.

            Lexers.Scanner scanner = null;
            Parser.Parser  parser  = null;
            Assembly       assm    = Assembly.GetExecutingAssembly();
            object         info    = Attribute.GetCustomAttribute(assm, typeof(AssemblyFileVersionAttribute));

            versionInfo = ((AssemblyFileVersionAttribute)info).Version;

            try {
                string filename = ProcessOptions(args);

                if (filename == null)
                {
                    return(MC_OK);
                }

                try {
                    inputFile     = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read);
                    inputFileInfo = filename + " - " + File.GetLastWriteTime(filename).ToString();
                }
                catch (IOException x) {
                    string message;
                    inputFile = null;
                    if (x is FileNotFoundException)
                    {
                        message = String.Format(CultureInfo.InvariantCulture,
                                                "Source file <{0}> not found{1}",
                                                filename, Environment.NewLine);
                    }
                    else
                    {
                        message = String.Format(CultureInfo.InvariantCulture,
                                                "Source file <{0}> could not be opened{1}",
                                                filename, Environment.NewLine);
                    }
                    handler.AddError(4, message, null);   // aast.AtStart;
                    return(MC_FILEERROR);
                }

                scanner = new Lexers.Scanner(inputFile);
                scanner.SetHandler(handler);

                parser = new Parser.Parser(filename, inputFileInfo, scanner, handler);
                //
                // If the parse is successful, then process the grammar.
                // Otherwise just report the errors that have been listed.
                //
                if (parser.Parse() && !handler.Errors)
                {
                    grammar = parser.Grammar;

                    if (Terminal.Max > 255)
                    {
                        // No ambiguating context possible since result appears in delimited error message
                        handler.ListError(null, 103, CharacterUtilities.MapCodepointToDisplayForm(Terminal.Max), '\'');
                    }

                    LALRGenerator         generator = new LALRGenerator(grammar);
                    List <AutomatonState> states    = generator.BuildStates();
                    generator.ComputeLookAhead();
                    generator.BuildParseTable();
                    if (!grammar.CheckGrammar())
                    {
                        throw new ArgumentException("Non-terminating grammar");
                    }
                    //
                    // If the grammar has non-terminating non-terms we cannot
                    // create a diagnostic report as the grammar is incomplete.
                    //
                    if (!handler.Errors)
                    {
                        CodeGenerator emitter = new CodeGenerator(grammar);
                        emitter.Generate(states);
                    }

                    bool DoDiagnose = Diagnose && !grammar.HasNonTerminatingNonTerms;
                    if (Report || DoDiagnose)
                    {
                        string htmlName = System.IO.Path.ChangeExtension(filename, ".report.html");
                        try {
                            System.IO.FileStream   htmlFile   = new System.IO.FileStream(htmlName, System.IO.FileMode.Create);
                            System.IO.StreamWriter htmlWriter = new System.IO.StreamWriter(htmlFile);
                            Grammar.HtmlHeader(htmlWriter, filename);

                            if (Report && DoDiagnose)
                            {
                                grammar.GenerateCompoundReport(htmlWriter, inputFileInfo, states);
                            }
                            else if (Report)
                            {
                                grammar.GenerateReport(htmlWriter, inputFileInfo, states);
                            }

                            Grammar.HtmlTrailer(htmlWriter);

                            if (htmlFile != null)
                            {
                                htmlWriter.Flush();
                                htmlFile.Close();
                            }
                        }
                        catch (System.IO.IOException) {
                            Console.Error.WriteLine("Cannot create html output file {0}", htmlName);
                        }
                    }
                }
            }
            catch (System.Exception e) {
                if (e is TooManyErrorsException)
                {
                    return(MC_TOOMANYERRORS);
                }
                Console.Error.WriteLine("Unexpected Error {0}", e.Message);

                if (NoThrowOnError)
                {
                    // report the error, do not let it go into the void
                    Console.Error.WriteLine(e);
                    return(MC_EXCEPTION);
                }
            }
            finally {
                if (handler.Errors || handler.Warnings)
                {
                    handler.DumpAll((scanner == null ? null : scanner.Buffer), Console.Error);
                }
                if ((Listing || handler.Errors || handler.Warnings) && parser != null)
                {
                    string       listName   = parser.ListfileName;
                    StreamWriter listStream = ListingFile(listName);
                    if (listStream != null)
                    {
                        handler.MakeListing(scanner.Buffer, listStream, parser.SourceFileInfo, versionInfo);
                    }
                }
            }
            return(MC_OK);
        }
示例#12
0
        private void GenerateInitializeMethod(
            List <AutomatonState> states,
            List <Production> productions,
            Dictionary <string, NonTerminal> nonTerminals)
        {
            Console.WriteLine("#pragma warning disable 649");
            Console.WriteLine("    private Dictionary<int, string> aliasses;");
            Console.WriteLine("#pragma warning restore 649");
            Console.WriteLine();
            Console.WriteLine("  protected override void Initialize()");
            Console.WriteLine("  {");

            //Console.WriteLine("    this.errorToken = (int){0}.error;", grammar.TokenName);
            //Console.WriteLine("    this.endOfFileToken = (int){0}.EOF;", grammar.TokenName);

            Console.WriteLine("    this.InitSpecialTokens((int){0}.error, (int){0}.EOF);", grammar.TokenName);
            Console.WriteLine();

            Console.WriteLine("    this.InitStateTable({0});", states.Count);
            int state_nr = 0;

            foreach (AutomatonState state in states)
            {
                GenerateState(state_nr++, state);
            }

            Console.WriteLine();

            Console.WriteLine("    Rule[] rules=new Rule[{0}];", productions.Count + 1);
            foreach (Production production in productions)
            {
                GenerateRule(production);
            }
            Console.WriteLine("    this.InitRules(rules);");
            Console.WriteLine();

            Console.Write("    this.InitNonTerminals(new string[] {\"\", ");
            int length = 37;

            foreach (NonTerminal nonTerminal in nonTerminals.Values)
            {
                string ss = String.Format(CultureInfo.InvariantCulture, "\"{0}\", ", nonTerminal.ToString());
                length += ss.Length;
                Console.Write(ss);
                if (length > 70)
                {
                    Console.WriteLine();
                    Console.Write("      ");
                    length = 0;
                }
            }
            Console.WriteLine("});");

            //
            //  Now initialize the terminal alias list, if needed.
            //
            List <Terminal> aliasList = new List <Terminal>();

            foreach (KeyValuePair <string, Terminal> pair in grammar.terminals)
            {
                Terminal term = pair.Value;
                if (term.Alias != null)
                {
                    aliasList.Add(term);
                }
            }
            if (aliasList.Count > 0)
            {
                Console.WriteLine();
                Console.WriteLine("    aliasses = new Dictionary<int, string>();");
                foreach (Terminal termWithAlias in aliasList)
                {
                    Console.WriteLine("    aliasses.Add({0}, {1});",
                                      termWithAlias.num,
                                      CharacterUtilities.QuoteMap(termWithAlias.Alias));
                }
            }
            Console.WriteLine("  }");

            Console.WriteLine();
        }