public void AffixSpacing()
        {
            Normalizer normalizer = new Normalizer(false, false, true);

            string input, expected, actual;

            input = "خانه ی پدری";
            expected = "خانه‌ی پدری";
            actual = normalizer.Run(input);
            Assert.AreEqual(expected, actual, "Failed to affix spacing of '" + input + "'");

            input = "فاصله میان پیشوند ها و پسوند ها را اصلاح می کند.";
            expected = "فاصله میان پیشوند‌ها و پسوند‌ها را اصلاح می‌کند.";
            actual = normalizer.Run(input);
            Assert.AreEqual(expected, actual, "Failed to affix spacing of '" + input + "'");

            input = "می روم";
            expected = "می‌روم";
            actual = normalizer.Run(input);
            Assert.AreEqual(expected, actual, "Failed to affix spacing of '" + input + "'");

            input = "حرفه ای";
            expected = "حرفه‌ای";
            actual = normalizer.Run(input);
            Assert.AreEqual(expected, actual, "Failed to affix spacing of '" + input + "'");
        }
Exemple #2
0
 public BijankhanReader(string bijankhanFile, bool joinedVerbParts, string posMap)
 {
     this._bijankhanFile = bijankhanFile;
     this._joinedVerbParts = joinedVerbParts;
     this._posMap = posMap;
     this.normalizer = new Normalizer(true, false, true);
     this.tokenizer = new WordTokenizer();
 }
        public void PunctuationSpacing()
        {
            Normalizer normalizer = new Normalizer(false, true, false);

            string input, expected, actual;

            input = "اصلاح ( پرانتزها ) در متن .";
            expected = "اصلاح (پرانتزها) در متن.";
            actual = normalizer.Run(input);
            Assert.AreEqual(expected, actual, "Failed to punctuation spacing of '" + input + "'");
        }
        public void CharacterRefinementTest()
        {
            Normalizer normalizer = new Normalizer(true, false, false);

            string input, expected, actual;

            input = "اصلاح كاف و ياي عربي";
            expected = "اصلاح کاف و یای عربی";
            actual = normalizer.Run(input);
            Assert.AreEqual(expected, actual, "Failed to character refinement of '" + input + "'");

            input = "رمــــان";
            expected = "رمان";
            actual = normalizer.Run(input);
            Assert.AreEqual(expected, actual, "Failed to character refinement of '" + input + "'");

            input = "1,2,3,...";
            expected = "۱,۲,۳, …";
            actual = normalizer.Run(input);
            Assert.AreEqual(expected, actual, "Failed to character refinement of '" + input + "'");
        }
 public abstract void TheOthers(Normalizer normalizer, char c);
 public override void SpecialChars(Normalizer normalizer, char c)
 {
     normalizer._stringBuilder.Append(' ');
     normalizer._stringBuilder.Append(c);
     normalizer._state = Normalizer._specialCharsState;
 }
 public override void SpecialChars(Normalizer normalizer, char c)
 {
     if (normalizer.IsNotEmpty)
     {
         normalizer._stringBuilder.Append(' ');
     }
     normalizer._stringBuilder.Append(c);
 }
 public abstract void Space(Normalizer normalizer, char c);
        private void Save()
        {
            try
            {
                var webConfigPath = this.FilePathTextbox.Text.Trim(" \"".ToCharArray());
                webConfigPath = ConfigBuilderProxy.ConvertLinkToRealFile(webConfigPath);
                var buildWebConfigResult        = this.BuildWebConfigResult.Checked;
                var normalizeOutput             = this.NormalizeOutput.Checked;
                var requireDefaultConfiguration = this.RequireDefaultConfiguration.Checked;
                var selectedItem = (ComboboxItem)this.SitecoreVersionComboBox.SelectedItem;
                var releaseInfo  = selectedItem.Value;

                var outputShowConfigFile = this.GetShowConfigFilePath(ShowconfigFileName);

                if (string.IsNullOrEmpty(outputShowConfigFile))
                {
                    return;
                }

                var outputWebConfigFile = string.Empty;
                if (buildWebConfigResult)
                {
                    outputWebConfigFile = this.GetShowConfigFilePath(WebConfigResultFileName);
                    Assert.IsNotNull(outputWebConfigFile, "outputWebConfigFile");
                }

                ConfigBuilderProxy.Build(webConfigPath, false, false).Save(outputShowConfigFile);
                if (normalizeOutput)
                {
                    ConfigBuilderProxy.Build(webConfigPath, false, true).Save(GetNormalizedPath(outputShowConfigFile));
                }

                if (buildWebConfigResult)
                {
                    ConfigBuilderProxy.Build(webConfigPath, true, false).Save(outputWebConfigFile);
                    if (normalizeOutput)
                    {
                        ConfigBuilderProxy.Build(webConfigPath, true, true).Save(GetNormalizedPath(outputWebConfigFile));
                    }
                }

                if (requireDefaultConfiguration)
                {
                    var websiteFolder = Path.GetDirectoryName(webConfigPath);
                    if (string.Equals(websiteFolder, Path.GetDirectoryName(outputShowConfigFile)))
                    {
                        websiteFolder       += " " + releaseInfo.Version.MajorMinorUpdate;
                        webConfigPath        = Path.Combine(websiteFolder, Path.GetFileName(webConfigPath));
                        outputShowConfigFile = Path.Combine(websiteFolder, Path.GetFileName(outputShowConfigFile));

                        Directory.Delete(websiteFolder, true);

                        {
                            var filesZipPath = Path.GetTempFileName();
                            new WebClient()
                            .DownloadFile(releaseInfo.DefaultDistribution.Defaults.Configs.FilesUrl, filesZipPath);

                            var tempFolder = Path.GetTempFileName();
                            File.Delete(tempFolder);
                            Directory.CreateDirectory(tempFolder);

                            ZipFile.ExtractToDirectory(filesZipPath, tempFolder);

                            tempFolder =
                                (
                                    Directory.GetDirectories(tempFolder, "Website", SearchOption.AllDirectories).First());
                            Directory.Move(tempFolder, websiteFolder);

                            var outputWebConfigFile1 = string.Empty;
                            if (buildWebConfigResult)
                            {
                                outputWebConfigFile1 = Path.Combine(websiteFolder, "web.config.result.xml");
                                Assert.IsNotNull(outputWebConfigFile1, "outputWebConfigFile");
                            }

                            ConfigBuilderProxy.Build(webConfigPath, false, false).Save(outputShowConfigFile);
                            if (normalizeOutput)
                            {
                                ConfigBuilderProxy.Build(webConfigPath, false, true).Save(GetNormalizedPath(outputShowConfigFile));
                            }

                            if (buildWebConfigResult)
                            {
                                ConfigBuilderProxy.Build(webConfigPath, true, false).Save(outputWebConfigFile1);
                                if (normalizeOutput)
                                {
                                    ConfigBuilderProxy.Build(webConfigPath, true, true).Save(GetNormalizedPath(outputWebConfigFile1));
                                }
                            }
                        }
                    }
                    else
                    {
                        try
                        {
                            var release           = releaseInfo.Version.MajorMinorUpdate;
                            var defaultShowConfig = outputShowConfigFile + "." + release + ".xml";
                            var defaults          = releaseInfo.DefaultDistribution.Defaults;
                            Assert.IsNotNull(defaults, $"Defaults are not available for {release}");

                            defaults.Configs.ShowConfig.Save(defaultShowConfig);
                            var normalizer = new Normalizer();
                            normalizer.Normalize(defaultShowConfig, GetNormalizedPath(defaultShowConfig));
                            if (buildWebConfigResult)
                            {
                                var defaultWebConfigResult = outputWebConfigFile + "." + release + ".xml";
                                defaults.Configs.Configuration.Save(defaultWebConfigResult);
                                normalizer.Normalize(defaultWebConfigResult, GetNormalizedPath(defaultWebConfigResult));
                            }
                        }
                        catch (Exception ex)
                        {
                            // Log.Error()
                        }
                    }
                }

                if (this.OpenFolder.Checked && File.Exists(outputWebConfigFile))
                {
                    string argument = @"/select, """ + outputWebConfigFile + @"""";
                    Process.Start("explorer.exe", argument);
                }

                if (this.CloseWhenDone.Checked)
                {
                    this.Close();
                }
            }
            catch (Exception ex)
            {
                MessageBox.Show("The action failed with exception. " + ex.Message + Environment.NewLine + "Find details in the ConfigBuilder.ConfigBuilder.dll.log file", "Sitecore ConfigBuilder");
                File.AppendAllText("ConfigBuilder.ConfigBuilder.dll.log", DateTime.Now.ToString("yyyy-MM-dd hh:mm:ss") + " ERROR " + ex.GetType().FullName + Environment.NewLine + "Message: " + ex.Message + Environment.NewLine + "Stack trace:" + Environment.NewLine + ex.StackTrace + Environment.NewLine);
            }
        }
        public void FindProperModifiers()
        {
            int     i       = 0;
            Dialect dialect = Dialect.LooseyGoosey;

            dialect.InferCompoundsPrepositionsForeignText = false;
            ParserUtils pu = new ParserUtils(dialect);

            Normalizer               norm   = new Normalizer(dialect);
            CorpusFileReader         reader = new CorpusFileReader();
            Dictionary <string, int> words  = new Dictionary <string, int>(500);

            SentenceSplitter ss = new SentenceSplitter(dialect);

            foreach (string s in reader.NextFile())
            {
                string[] sentences = ss.ParseIntoNonNormalizedSentences(s);
                foreach (string original in sentences)
                {
                    try
                    {
                        string   normalized = norm.NormalizeText(original);
                        Sentence structured = pu.ParsedSentenceFactory(normalized, original);
                        //string diag = structured.ToString("b");


                        string stringified = structured.Subjects.ToString();
                        if (!stringified.Contains(" "))
                        {
                            continue;                            //single word
                        }
                        if (stringified.Contains(@""""))
                        {
                            continue;                             //foreign
                        }
                        if (stringified.StartsWith(@"nanpa"))
                        {
                            continue;                                  //implicit number
                        }
                        if (stringified.StartsWith(@"#"))
                        {
                            continue;                              //explicit number by punctuation
                        }
                        if (stringified.ContainsLetter(Token.AlphabetUpper))
                        {
                            if (words.ContainsKey(stringified))
                            {
                                words[stringified] = words[stringified] + 1;
                            }
                            else
                            {
                                words.Add(stringified, 1);
                                Console.WriteLine(i + " : " + stringified);
                            }
                        }
                    }
                    catch (Exception)
                    {
                        i++;
                    }
                }
            }
            foreach (KeyValuePair <string, int> pair in words.OrderBy(x => x.Value))
            {
                Console.WriteLine(pair.Key + " : " + pair.Value);
            }
        }
Exemple #11
0
    public override Node CompileParseTree(Node node, Scope scope, Module targetModule, ErrorNodeList errorNodes){
      TrivialHashtable ambiguousTypes = new TrivialHashtable();
      TrivialHashtable referencedLabels = new TrivialHashtable();
      TrivialHashtable scopeFor = new TrivialHashtable();
      ErrorHandler errorHandler = new ErrorHandler(errorNodes);
      SpecSharpCompilation ssCompilation = new SpecSharpCompilation();

      // Setting the state
      TypeNode thisType = null;
      Method   currentMethod = null;
      BlockScope blockScope = scope as BlockScope;
      if (blockScope != null){
        Class baseScope = blockScope;
        MethodScope methodScope = null;
        while (baseScope != null){
          methodScope = baseScope.BaseClass as MethodScope;
          if (methodScope != null) break;
          baseScope = baseScope.BaseClass;
        }
        if (methodScope != null){
          thisType = methodScope.ThisType;
          if (thisType == null && methodScope.BaseClass is TypeScope){
            thisType = ((TypeScope) methodScope.BaseClass).Type;

          }
          currentMethod = methodScope.DeclaringMethod;
        }
      }

      //Attach scope to namespaces and types
      scopeFor[node.UniqueKey] = scope;
      Scoper scoper = new Scoper(scopeFor);
      scoper.currentScope = scope;
      node = scoper.Visit(node);

      //Walk IR looking up names
      Looker looker = new Looker(scope, errorHandler, scopeFor, ambiguousTypes, referencedLabels);
      // begin change by drunje (this is called from debugger only)
      looker.AllowPointersToManagedStructures = true;
      // end change by drunje
      if (blockScope != null)
      {
        looker.currentType = thisType;
        looker.currentMethod = currentMethod;
      }
      looker.currentAssembly = targetModule as AssemblyNode;
      looker.currentModule = targetModule;
      node = looker.Visit(node);
      
      //Walk IR inferring types and resolving overloads
      TypeSystem typeSystem = new TypeSystem(errorHandler);
      Resolver resolver = new Resolver(errorHandler, typeSystem);
      if (blockScope != null){
        resolver.currentType = thisType;
        resolver.currentMethod = currentMethod;
      }
      resolver.currentAssembly = targetModule as AssemblyNode;
      resolver.currentModule = targetModule;
      node = resolver.Visit(node);
      
      //TODO:  Need to set the state of the checker for compiling Expression, STOP using this method when the shift is complete
      //Walk IR checking for semantic errors and repairing it so that the next walk will work
      Checker checker = new Checker(ssCompilation, errorHandler, typeSystem, scopeFor, ambiguousTypes, referencedLabels);
      if (blockScope != null){
        checker.currentType = thisType;
        checker.currentMethod = currentMethod;
      }
      checker.currentAssembly = targetModule as AssemblyNode;
      checker.currentModule = targetModule;
      node = checker.Visit(node);

      //Walk IR reducing it to nodes that have predefined mappings to MD+IL
      Normalizer normalizer = new Normalizer(typeSystem);
      if (blockScope != null){
        normalizer.currentType = thisType;
        normalizer.currentMethod = currentMethod;
        normalizer.WrapToBlockExpression = false;
      }
      normalizer.currentModule = targetModule;
      node = normalizer.Visit(node);

      return node;
    }
Exemple #12
0
    /// <summary>
    /// 获得甲方
    /// </summary>
    /// <returns></returns>
    public string GetJiaFang()
    {
        //最高置信度的抽取
        EntityProperty e = new EntityProperty();

        e.ExcludeContainsWordList = new string[] { "招标代理" };
        e.LeadingColonKeyWordList = new string[] {
            "甲方:", "合同买方:",
            "发包人:", "发包单位:", "发包方:", "发包机构:", "发包人名称:",
            "招标人:", "招标单位:", "招标方:", "招标机构:", "招标人名称:",
            "业主:", "业主单位:", "业主方:", "业主机构:", "业主名称:",
            "采购单位:", "采购单位名称:", "采购人:", "采购人名称:", "采购方:", "采购方名称:"
        };
        e.CandidatePreprocess = (x =>
        {
            x = Normalizer.ClearTrailing(x);
            return(CompanyNameLogic.AfterProcessFullName(x).secFullName);
        });
        e.MaxLength = ContractTraning.JiaFangES.MaxLength;
        e.MaxLengthCheckPreprocess = Utility.TrimEnglish;
        e.MinLength = 3;
        e.Extract(this);

        //这里不直接做Distinct,出现频次越高,则可信度越高
        //多个甲方的时候,可能意味着没有甲方!
        if (e.LeadingColonKeyWordCandidate.Distinct().Count() > 1)
        {
            foreach (var candidate in e.LeadingColonKeyWordCandidate)
            {
                Program.Logger.WriteLine("发现多个甲方:" + candidate);
            }
        }
        if (e.LeadingColonKeyWordCandidate.Count > 0)
        {
            return(e.LeadingColonKeyWordCandidate[0]);
        }


        //招标
        var Extractor     = new ExtractPropertyByHTML();
        var CandidateWord = new List <String>();
        var StartArray    = new string[] { "招标单位", "业主", "收到", "接到" };
        var EndArray      = new string[] { "发来", "发出", "的中标" };

        Extractor.StartEndFeature = Utility.GetStartEndStringArray(StartArray, EndArray);
        Extractor.Extract(root);
        foreach (var item in Extractor.CandidateWord)
        {
            var JiaFang = CompanyNameLogic.AfterProcessFullName(item.Value.Trim());
            if (JiaFang.secFullName.Contains("招标代理"))
            {
                continue;                                       //特殊业务规则
            }
            JiaFang.secFullName = JiaFang.secFullName.Replace("业主", String.Empty).Trim();
            JiaFang.secFullName = JiaFang.secFullName.Replace("招标单位", String.Empty).Trim();
            if (Utility.TrimEnglish(JiaFang.secFullName).Length > ContractTraning.JiaFangES.MaxLength)
            {
                continue;
            }
            if (JiaFang.secFullName.Length < 3)
            {
                continue;                                     //使用实际长度排除全英文的情况
            }
            if (!Program.IsMultiThreadMode)
            {
                Program.Logger.WriteLine("甲方候补词(招标):[" + JiaFang.secFullName + "]");
            }
            CandidateWord.Add(JiaFang.secFullName);
        }

        //合同
        Extractor  = new ExtractPropertyByHTML();
        StartArray = new string[] { "与", "与业主" };
        EndArray   = new string[] { "签署", "签订" };
        Extractor.StartEndFeature = Utility.GetStartEndStringArray(StartArray, EndArray);
        Extractor.Extract(root);
        foreach (var item in Extractor.CandidateWord)
        {
            var JiaFang = CompanyNameLogic.AfterProcessFullName(item.Value.Trim());
            JiaFang.secFullName = JiaFang.secFullName.Replace("业主", String.Empty).Trim();
            if (JiaFang.secFullName.Contains("招标代理"))
            {
                continue;                                       //特殊业务规则
            }
            if (Utility.TrimEnglish(JiaFang.secFullName).Length > ContractTraning.JiaFangES.MaxLength)
            {
                continue;
            }
            if (JiaFang.secFullName.Length < 3)
            {
                continue;                                     //使用实际长度排除全英文的情况
            }
            if (!Program.IsMultiThreadMode)
            {
                Program.Logger.WriteLine("甲方候补词(合同):[" + JiaFang.secFullName + "]");
            }
            CandidateWord.Add(JiaFang.secFullName);
        }
        return(CompanyNameLogic.MostLikeCompanyName(CandidateWord));
    }
 public PreprocessorAndFeatureExtractor(float sampleRate)
 {
     extractor     = new LpcFeaturesExtractor(sampleRate, 20);
     voiceDetector = new AutocorrellatedVoiceActivityDetector();
     normalizer    = new Normalizer();
 }
Exemple #14
0
    public override void ParseAndAnalyzeCompilationUnit(string fname, string text, int line, int col, ErrorNodeList errors, Compilation compilation, AuthoringSink sink) {
      if (fname == null || text == null || errors == null || compilation == null){Debug.Assert(false); return;}
      if (compilation != null && compilation.CompilerParameters is SpecSharpCompilerOptions)
        this.allowSpecSharpExtensions = !((SpecSharpCompilerOptions)compilation.CompilerParameters).Compatibility;
      CompilationUnitList compilationUnitSnippets = compilation.CompilationUnits;
      if (compilationUnitSnippets == null){Debug.Assert(false); return;}
      //Fix up the CompilationUnitSnippet corresponding to fname with the new source text
      CompilationUnitSnippet cuSnippet = this.GetCompilationUnitSnippet(compilation, fname);
      if (cuSnippet == null) return;
      Compiler compiler = new Compiler();
      compiler.CurrentCompilation = compilation;
      cuSnippet.SourceContext.Document = compiler.CreateDocument(fname, 1, new DocumentText(text));
      cuSnippet.SourceContext.EndPos = text.Length;
      //Parse all of the compilation unit snippets
      Module symbolTable = compilation.TargetModule = compiler.CreateModule(compilation.CompilerParameters, errors);
      AttributeList assemblyAttributes = symbolTable is AssemblyNode ? symbolTable.Attributes : null;
      AttributeList moduleAttributes = symbolTable is AssemblyNode ? ((AssemblyNode)symbolTable).ModuleAttributes : symbolTable.Attributes;
      int n = compilationUnitSnippets.Count;
      for (int i = 0; i < n; i++){
        CompilationUnitSnippet compilationUnitSnippet = compilationUnitSnippets[i] as CompilationUnitSnippet;
        if (compilationUnitSnippet == null){Debug.Assert(false); continue;}
        Document doc = compilationUnitSnippet.SourceContext.Document;
        doc = compilationUnitSnippet.SourceContext.Document;
        if (doc == null || doc.Text == null){Debug.Assert(false); continue;}
        IParserFactory factory = compilationUnitSnippet.ParserFactory;
        if (factory == null) continue;
        compilationUnitSnippet.Nodes = null;
        compilationUnitSnippet.PreprocessorDefinedSymbols = null;
        IParser p = factory.CreateParser(doc.Name, doc.LineNumber, doc.Text, symbolTable, compilationUnitSnippet == cuSnippet ? errors : new ErrorNodeList(), compilation.CompilerParameters);
        if (p == null){Debug.Assert(false); continue;}
        if (p is ResgenCompilerStub) continue;
        Parser specSharpParser = p as Parser;
        if (specSharpParser == null)
          p.ParseCompilationUnit(compilationUnitSnippet);
        else
          specSharpParser.ParseCompilationUnit(compilationUnitSnippet, compilationUnitSnippet != cuSnippet, false);
        //TODO: this following is a good idea only if the files will not be frequently reparsed from source
        //StringSourceText stringSourceText = doc.Text.TextProvider as StringSourceText;
        //if (stringSourceText != null && stringSourceText.IsSameAsFileContents)
        //  doc.Text.TextProvider = new CollectibleSourceText(doc.Name, doc.Text.Length);
      }
      //Construct symbol table for entire project
      ErrorHandler errorHandler = new ErrorHandler(errors);
      SpecSharpCompilation ssCompilation = new SpecSharpCompilation();
      TrivialHashtable ambiguousTypes = new TrivialHashtable();
      TrivialHashtable referencedLabels = new TrivialHashtable();
      TrivialHashtable scopeFor = this.scopeFor = new TrivialHashtable();
      Scoper scoper = new Scoper(scopeFor);
      scoper.currentModule = symbolTable;
      Looker symLooker = new Looker(null, new ErrorHandler(new ErrorNodeList(0)), scopeFor, ambiguousTypes, referencedLabels);
      symLooker.currentAssembly = (symLooker.currentModule = symbolTable) as AssemblyNode;
      Looker looker = new Looker(null, errorHandler, scopeFor, ambiguousTypes, referencedLabels);
      looker.currentAssembly = (looker.currentModule = symbolTable) as AssemblyNode;
      looker.VisitAttributeList(assemblyAttributes);
      bool dummyCompilation = compilation.CompilerParameters is SpecSharpCompilerOptions && ((SpecSharpCompilerOptions)compilation.CompilerParameters).DummyCompilation;
      if (dummyCompilation){
        //This happens when there is no project. In this case, semantic errors should be ignored since the references and options are unknown.
        //But proceed with the full analysis anyway so that some measure of Intellisense can still be provided.
        errorHandler.Errors = new ErrorNodeList(0);
      }
      for (int i = 0; i < n; i++){
        CompilationUnit cUnit = compilationUnitSnippets[i];
        scoper.VisitCompilationUnit(cUnit);
      }
      for (int i = 0; i < n; i++){
        CompilationUnit cUnit = compilationUnitSnippets[i];
        if (cUnit == cuSnippet)
          looker.VisitCompilationUnit(cUnit); //Uses real error message list and populate the identifier info lists
        else
          symLooker.VisitCompilationUnit(cUnit); //Errors are discarded
      }
      //Run resolver over symbol table so that custom attributes on member signatures are known and can be used
      //to error check the the given file.
      TypeSystem typeSystem = new TypeSystem(errorHandler);
      Resolver resolver = new Resolver(errorHandler, typeSystem);
      resolver.currentAssembly = (resolver.currentModule = symbolTable) as AssemblyNode;
      Resolver symResolver = new Resolver(new ErrorHandler(new ErrorNodeList(0)), typeSystem);
      symResolver.currentAssembly = resolver.currentAssembly;
      symResolver.VisitAttributeList(assemblyAttributes);
      for (int i = 0; i < n; i++) {
        CompilationUnit cUnit = compilationUnitSnippets[i];
        if (cUnit == cuSnippet)
          resolver.VisitCompilationUnit(cUnit); //Uses real error message list and populate the identifier info lists
        else
          symResolver.VisitCompilationUnit(cUnit); //Errors are discarded
      }
      if (dummyCompilation) return;
      //Now analyze the given file for errors
      Checker checker = new Checker(ssCompilation, errorHandler, typeSystem, scopeFor, ambiguousTypes, referencedLabels);
      checker.currentAssembly = (checker.currentModule = symbolTable) as AssemblyNode;
      checker.VisitAttributeList(assemblyAttributes, checker.currentAssembly);
      checker.VisitModuleAttributes(moduleAttributes);
      checker.VisitCompilationUnit(cuSnippet);
       
      MemberFinder finder = new MemberFinder(line, col);
      finder.VisitCompilationUnit(cuSnippet);
      Node node = finder.Member;
      if (node == null){
        if (line == 0 && col == 0) 
          node = cuSnippet;
        else
          return;
      }

      SpecSharpCompilerOptions options = (SpecSharpCompilerOptions) compilation.CompilerParameters;
      if (options.IsContractAssembly) return;
      ssCompilation.RunPlugins(node, errorHandler);
      Normalizer normalizer = new Normalizer(typeSystem);
      normalizer.Visit(node);
      Analyzer analyzer = new Analyzer(typeSystem, compilation);
      analyzer.Visit(node);

      if (options.RunProgramVerifierWhileEditing)
        ssCompilation.AddProgramVerifierPlugin(typeSystem, compilation);
      ssCompilation.analyzer = analyzer; // make the analyzer available to plugins for access to method CFGs
      ssCompilation.RunPlugins(node, errorHandler);
      ssCompilation.analyzer = null;
      analyzer = null;
    }
Exemple #15
0
    /// <summary>
    /// 获得甲方
    /// </summary>
    /// <returns></returns>
    string GetJiaFang(String YiFang)
    {
        //最高置信度的抽取
        EntityProperty e = new EntityProperty();

        e.ExcludeContainsWordList = new string[] { "招标代理" };
        e.LeadingColonKeyWordList = new string[] {
            "甲方:", "合同买方:",
            "发包人:", "发包单位:", "发包方:", "发包机构:", "发包人名称:",
            "招标人:", "招标单位:", "招标方:", "招标机构:", "招标人名称:", "项目招标人:",
            "业主:", "业主单位:", "业主方:", "业主机构:", "业主名称:",
            "采购单位:", "采购单位名称:", "采购人:", "采购人名称:", "采购方:", "采购方名称:"
        };
        e.CandidatePreprocess = (x =>
        {
            x = Normalizer.ClearTrailing(x);
            return(CompanyNameLogic.AfterProcessFullName(x).secFullName);
        });
        e.MaxLength = 32;
        e.MaxLengthCheckPreprocess = Utility.TrimEnglish;
        e.MinLength = 3;
        e.Extract(this);
        //这里不直接做Distinct,出现频次越高,则可信度越高
        //多个甲方的时候,可能意味着没有甲方!
        if (e.LeadingColonKeyWordCandidate.Distinct().Count() > 1)
        {
            foreach (var candidate in e.LeadingColonKeyWordCandidate)
            {
                Program.Logger.WriteLine("发现多个甲方:" + candidate);
            }
        }
        if (e.LeadingColonKeyWordCandidate.Count > 0)
        {
            return(e.LeadingColonKeyWordCandidate[0]);
        }

        var ner    = SearchJiaFang();
        var NerJia = String.Empty;

        if (!String.IsNullOrEmpty(ner))
        {
            foreach (var cn in companynamelist)
            {
                if (cn.secShortName == ner)
                {
                    ner = cn.secFullName;
                }
            }
            if (String.IsNullOrEmpty(YiFang))
            {
                NerJia = ner;
            }
            if (!YiFang.Equals(ner))
            {
                NerJia = ner;
            }
        }

        //招标
        var Extractor     = new ExtractPropertyByHTML();
        var CandidateWord = new List <String>();
        var StartArray    = new string[] { "招标单位", "业主", "收到", "接到" };
        var EndArray      = new string[] { "发来", "发出", "的中标" };

        Extractor.StartEndFeature = Utility.GetStartEndStringArray(StartArray, EndArray);
        Extractor.Extract(root);
        foreach (var item in Extractor.CandidateWord)
        {
            var JiaFang = CompanyNameLogic.AfterProcessFullName(item.Value.Trim());
            if (JiaFang.secFullName.Contains("招标代理"))
            {
                continue;                                       //特殊业务规则
            }
            JiaFang.secFullName = JiaFang.secFullName.Replace("业主", String.Empty).Trim();
            JiaFang.secFullName = JiaFang.secFullName.Replace("招标单位", String.Empty).Trim();
            if (Utility.TrimEnglish(JiaFang.secFullName).Length > 32)
            {
                continue;
            }
            if (JiaFang.secFullName.Length < 3)
            {
                continue;                                     //使用实际长度排除全英文的情况
            }
            if (!Program.IsMultiThreadMode)
            {
                Program.Logger.WriteLine("甲方候补词(招标):[" + JiaFang.secFullName + "]");
            }
            CandidateWord.Add(JiaFang.secFullName);
        }

        //合同
        Extractor  = new ExtractPropertyByHTML();
        StartArray = new string[] { "与", "与业主" };
        EndArray   = new string[] { "签署", "签订" };
        Extractor.StartEndFeature = Utility.GetStartEndStringArray(StartArray, EndArray);
        Extractor.Extract(root);
        foreach (var item in Extractor.CandidateWord)
        {
            var JiaFang = CompanyNameLogic.AfterProcessFullName(item.Value.Trim());
            JiaFang.secFullName = JiaFang.secFullName.Replace("业主", String.Empty).Trim();
            if (JiaFang.secFullName.Contains("招标代理"))
            {
                continue;                                       //特殊业务规则
            }
            if (Utility.TrimEnglish(JiaFang.secFullName).Length > 32)
            {
                continue;
            }
            if (JiaFang.secFullName.Length < 3)
            {
                continue;                                     //使用实际长度排除全英文的情况
            }
            if (!Program.IsMultiThreadMode)
            {
                Program.Logger.WriteLine("甲方候补词(合同):[" + JiaFang.secFullName + "]");
            }
            CandidateWord.Add(JiaFang.secFullName);
        }
        if (!String.IsNullOrEmpty(NerJia))
        {
            //原则上,有NER中提取的甲方,则使用甲方
            foreach (var c in CandidateWord)
            {
                //但是,这里有可能是正确的解答,例如
                //NER:(集团)有限公司 实际上应该是 XXXX(集团)有限公司
                if (c.EndsWith(NerJia))
                {
                    return(c);
                }
            }
            return(NerJia);
        }
        else
        {
            return(CompanyNameLogic.MostLikeCompanyName(CandidateWord));
        }
    }
Exemple #16
0
        /// <summary>
        /// Creates a Formula from a string that consists of a standard infix expression composed
        /// from non-negative floating-point numbers (using C#-like syntax for double/int literals),
        /// variable symbols (a letter followed by zero or more letters and/or digits), left and right
        /// parentheses, and the four binary operator symbols +, -, *, and /.  White space is
        /// permitted between tokens, but is not required.
        ///
        /// Examples of a valid parameter to this constructor are:
        ///     "2.5e9 + x5 / 17"
        ///     "(5 * 2) + 8"
        ///     "x*y-2+35/9"
        ///
        /// Examples of invalid parameters are:
        ///     "_"
        ///     "-5.3"
        ///     "2 5 + 3"
        ///
        /// If the formula is syntacticaly invalid, throws a FormulaFormatException with an
        /// explanatory Message.
        /// </summary>
        public Formula(String formula)
        {
            if (formula == null)
            {
                throw new ArgumentNullException("formula");
            }
            IEnumerable <string> tokens = GetTokens(formula);

            tokensList = new List <string>();
            Normalizer N = (s => s);
            Validator  V = (s => true);

            foreach (string s in tokens)
            {
                tokensList.Add(s);
            }
            if (tokensList.Count == 0)
            {
                throw new FormulaFormatException("The formula is empty.");
            }
            foreach (string s in tokensList)
            {
                if (!Regex.IsMatch(N(s), @"\(") && !Regex.IsMatch(N(s), @"\)") && !Regex.IsMatch(N(s), @"[\+\-*/]") && !Regex.IsMatch(N(s), @"[a-zA-Z][0-9a-zA-Z]*") && !Regex.IsMatch(N(s), @"(?: \d+\.\d* | \d*\.\d+ | \d+ ) (?: e[\+-]?\d+)?", RegexOptions.IgnorePatternWhitespace) && V(N(s)))
                {
                    throw new FormulaFormatException("There is an invalid character in the formula.");
                }
            }
            int left  = 0;
            int right = 0;

            foreach (string s in tokensList)
            {
                if (s == "(")
                {
                    left++;
                }
                if (s == ")")
                {
                    right++;
                    if (right > left)
                    {
                        throw new FormulaFormatException("Closing parenthesis with no opening parenthesis.");
                    }
                }
            }
            if (left != right)
            {
                throw new FormulaFormatException("Parentheses imbalance.");
            }
            if (Regex.IsMatch(tokensList[0], @"[\+\-*/]") && !double.TryParse(tokensList[0], out double n))
            {
                throw new FormulaFormatException("Formula cannot start with an operator");
            }
            if (Regex.IsMatch(tokensList[tokensList.Count - 1], @"[\+\-*/]") && !double.TryParse(tokensList[tokensList.Count - 1], out n))
            {
                throw new FormulaFormatException("Formula cannot end with an operator");
            }
            for (int x = 0; x < tokensList.Count - 1; x++)
            {
                if (Regex.IsMatch(tokensList[x], @"[\+\-*/]") | Regex.IsMatch(tokensList[x], @"\(") && !double.TryParse(tokensList[x], out n))
                {
                    if ((Regex.IsMatch(tokensList[x + 1], @"[\+\-*/]") | Regex.IsMatch(tokensList[x + 1], @"\)")) && !double.TryParse(tokensList[x + 1], out n))
                    {
                        throw new FormulaFormatException("There is an operator or closing parenthesis immediately following an operator or opening parenthesis.");
                    }
                }
                if (!Regex.IsMatch(tokensList[x], @"[\+\-*/]") && !Regex.IsMatch(tokensList[x], @"\("))
                {
                    if (!Regex.IsMatch(tokensList[x + 1], @"[\+\-*/]") && !Regex.IsMatch(tokensList[x + 1], @"\)"))
                    {
                        throw new FormulaFormatException("There is a number, variable, or opening parenthesis immediately following a number, variable, or closing parenthesis.");
                    }
                }
            }
        }
Exemple #17
0
        private void SeedCategory(
            Category parent, JToken categoryToken, IList <int> numbers,
            string instanceTitle = null)
        {
            int repeatCount = 1;
            var repeat      = categoryToken["Repeat"];

            if (repeat != null)
            {
                repeatCount = (int)repeat;
            }

            for (int j = 0; j < repeatCount; j++)
            {
                int    id   = dataContainer.NextCategoryId();
                string name = PrepareText((string)categoryToken["Name"], numbers);

                string thisMaterialTypeTitle = (string)categoryToken["MaterialTypeTitle"] ?? instanceTitle;

                Category category = new Category
                {
                    Id                = id,
                    ParentId          = parent?.Id,
                    Name              = name,
                    NameNormalized    = Normalizer.Normalize(name),
                    Title             = PrepareText((string)categoryToken["Title"], numbers),
                    Header            = PrepareText((string)categoryToken["Header"], numbers),
                    MaterialTypeTitle = thisMaterialTypeTitle,
                    SortNumber        = id
                };

                if (categoryToken["IsMaterialsContainer"] != null)
                {
                    category.IsMaterialsContainer = (bool)categoryToken["IsMaterialsContainer"];
                }

                if (categoryToken["LayoutName"] != null)
                {
                    category.LayoutName = (string)categoryToken["LayoutName"];
                }

                if (categoryToken["IsMaterialsNameEditable"] != null)
                {
                    category.IsMaterialsNameEditable = (bool)categoryToken["IsMaterialsNameEditable"];
                }

                if (categoryToken["IsMaterialsSubTitleEditable"] != null)
                {
                    category.IsMaterialsSubTitleEditable = (bool)categoryToken["IsMaterialsSubTitleEditable"];
                }


                dataContainer.Categories.Add(category);

                if (categoryToken["SubCategories"] != null)
                {
                    var numbers1 = new List <int> {
                        1
                    };
                    numbers1.AddRange(numbers);

                    foreach (JToken subCategoryToken in (JArray)categoryToken["SubCategories"])
                    {
                        SeedCategory(category, subCategoryToken, numbers1, thisMaterialTypeTitle);
                    }
                }

                numbers[0]++;
            }
        }
Exemple #18
0
        /// <summary>
        /// See if the decomposition of cp2 is at segment starting at <paramref name="segmentPos"/>
        /// (with canonical rearrangment!).
        /// If so, take the remainder, and return the equivalents.
        /// </summary>
        /// <param name="comp"></param>
        /// <param name="segment"></param>
        /// <param name="segmentPos"></param>
        /// <param name="buf"></param>
        /// <returns></returns>
        private ISet <string> Extract(int comp, string segment, int segmentPos, StringBuffer buf)
        {
            if (PROGRESS)
            {
                Console.Out.WriteLine(" extract: " + Utility.Hex(UTF16.ValueOf(comp))
                                      + ", " + Utility.Hex(segment.Substring(segmentPos)));
            }

            string decomp = nfcImpl.GetDecomposition(comp);

            if (decomp == null)
            {
                decomp = UTF16.ValueOf(comp);
            }

            // See if it matches the start of segment (at segmentPos)
            bool ok = false;
            int  cp;
            int  decompPos = 0;
            int  decompCp  = UTF16.CharAt(decomp, 0);

            decompPos += UTF16.GetCharCount(decompCp); // adjust position to skip first char
                                                       //int decompClass = getClass(decompCp);
            buf.Length = 0;                            // initialize working buffer, shared among callees

            for (int i = segmentPos; i < segment.Length; i += UTF16.GetCharCount(cp))
            {
                cp = UTF16.CharAt(segment, i);
                if (cp == decompCp)
                { // if equal, eat another cp from decomp
                    if (PROGRESS)
                    {
                        Console.Out.WriteLine("  matches: " + Utility.Hex(UTF16.ValueOf(cp)));
                    }
                    if (decompPos == decomp.Length)
                    {                                                              // done, have all decomp characters!
                        buf.Append(segment.Substring(i + UTF16.GetCharCount(cp))); // add remaining segment chars
                        ok = true;
                        break;
                    }
                    decompCp   = UTF16.CharAt(decomp, decompPos);
                    decompPos += UTF16.GetCharCount(decompCp);
                    //decompClass = getClass(decompCp);
                }
                else
                {
                    if (PROGRESS)
                    {
                        Console.Out.WriteLine("  buffer: " + Utility.Hex(UTF16.ValueOf(cp)));
                    }
                    // brute force approach
                    UTF16.Append(buf, cp);

                    /* TODO: optimize
                     * // since we know that the classes are monotonically increasing, after zero
                     * // e.g. 0 5 7 9 0 3
                     * // we can do an optimization
                     * // there are only a few cases that work: zero, less, same, greater
                     * // if both classes are the same, we fail
                     * // if the decomp class < the segment class, we fail
                     *
                     * segClass = getClass(cp);
                     * if (decompClass <= segClass) return null;
                     */
                }
            }
            if (!ok)
            {
                return(null);     // we failed, characters left over
            }
            if (PROGRESS)
            {
                Console.Out.WriteLine("Matches");
            }
            if (buf.Length == 0)
            {
                return(SET_WITH_NULL_STRING);                 // succeed, but no remainder
            }
            string remainder = buf.ToString();

            // brute force approach
            // to check to make sure result is canonically equivalent

            /*
             * String trial = Normalizer.normalize(UTF16.valueOf(comp) + remainder, Normalizer.DECOMP, 0);
             * if (!segment.regionMatches(segmentPos, trial, 0, segment.length() - segmentPos)) return null;
             */

            if (0 != Normalizer.Compare(UTF16.ValueOf(comp) + remainder, segment.Substring(segmentPos), 0))
            {
                return(null);
            }

            // get the remaining combinations
            return(GetEquivalents2(remainder));
        }
Exemple #19
0
 public NormDocument Normalize(string format)
 {
     Normalizer norm = new Normalizer (this, format);
     return norm.CreateNormDocument ();
 }
Exemple #20
0
        public override Statement VisitWhile(While While)
        {
            if (While == null || While.Condition == null)
                return While;

            if (insideAtomicBlock && IsCoalescableWhileStmt(While))
            {
                CurrentContinuation =
                    AddBlock(new BasicBlock(While, CurrentContinuation));
                return While;
            }

            Normalizer normalizer = new Normalizer(splicer, null, false);
            BasicBlock testBlock = new BasicBlock(null);
            testBlock.SourceContext = While.Condition.SourceContext;

            PushContinuationStack(testBlock);
            this.Visit(While.Body);
            BasicBlock bodyBlock = PopContinuationStack();

            testBlock.ConditionalExpression = normalizer.VisitExpression(While.Condition);
            testBlock.ConditionalTarget = bodyBlock;
            testBlock.UnconditionalTarget = CurrentContinuation;

            testBlock.ConditionalExpression.SourceContext = testBlock.SourceContext;
            CurrentContinuation = AddBlock(testBlock);

            return While;
        }
Exemple #21
0
    /// <summary>
    /// 获得合同名
    /// </summary>
    /// <returns></returns>
    string GetContractName()
    {
        var e = new EntityProperty();

        e.PropertyName = "合同名称";
        e.PropertyType = EntityProperty.enmType.NER;
        e.MaxLength    = ContractTraning.ContractES.MaxLength;
        e.MinLength    = ContractTraning.ContractES.MinLength;

        /* 训练模式下
         * e.LeadingColonKeyWordList = ContractTraning.ContractNameLeadingDict
         *                          .Where((x) => { return x.Value >= 40; })    //阈值40%以上
         *                          .Select((x) => { return x.Key + ":"; }).ToArray();
         */
        e.LeadingColonKeyWordList   = new string[] { "合同名称:" };
        e.QuotationTrailingWordList = new string[] { "协议书", "合同书", "确认书", "合同", "协议" };
        e.QuotationTrailingWordList_IsSkipBracket = true;   //暂时只能选True
        var KeyList = new List <ExtractPropertyByDP.DPKeyWord>();

        KeyList.Add(new ExtractPropertyByDP.DPKeyWord()
        {
            StartWord    = new string[] { "签署", "签订" }, //通过SRL训练获得
            StartDPValue = new string[] { LTPTrainingDP.核心关系, LTPTrainingDP.定中关系, LTPTrainingDP.并列关系 },
            EndWord      = new string[] { "补充协议", "合同书", "合同", "协议书", "协议", },
            EndDPValue   = new string[] { LTPTrainingDP.核心关系, LTPTrainingDP.定中关系, LTPTrainingDP.并列关系, LTPTrainingDP.动宾关系, LTPTrainingDP.主谓关系 }
        });
        e.DpKeyWordList = KeyList;

        var StartArray = new string[] { "签署了", "签订了" };   //通过语境训练获得
        var EndArray   = new string[] { "合同" };

        e.ExternalStartEndStringFeature = Utility.GetStartEndStringArray(StartArray, EndArray);
        e.ExternalStartEndStringFeatureCandidatePreprocess = (x) => { return(x + "合同"); };
        e.MaxLengthCheckPreprocess = str =>
        {
            return(Utility.TrimEnglish(str));
        };
        //最高级别的置信度,特殊处理器
        e.LeadingColonKeyWordCandidatePreprocess = str =>
        {
            var c = Normalizer.ClearTrailing(TrimEndJianCheng(str));
            return(c);
        };

        e.CandidatePreprocess = str =>
        {
            var c             = Normalizer.ClearTrailing(TrimEndJianCheng(str));
            var RightQMarkIdx = c.IndexOf("”");
            if (!(RightQMarkIdx != -1 && RightQMarkIdx != c.Length - 1))
            {
                //对于"XXX"合同,有右边引号,但不是最后的时候,不用做
                c = c.TrimStart("“".ToCharArray());
            }
            c = c.TrimStart("《".ToCharArray());
            c = c.TrimEnd("》".ToCharArray()).TrimEnd("”".ToCharArray());
            return(c);
        };
        e.ExcludeContainsWordList = new string[] { "日常经营重大合同" };
        //下面这个列表的根据不足
        e.ExcludeEqualsWordList = new string[] { "合同", "重大合同", "项目合同", "终止协议", "经营合同", "特别重大合同", "相关项目合同" };
        e.Extract(this);

        //是否所有的候选词里面包括(测试集无法使用)
        var contractlist = TraningDataset.ContractList.Where((x) => { return(x.Id == this.Id); });

        if (contractlist.Count() > 0)
        {
            var contract     = contractlist.First();
            var contractname = contract.ContractName;
            if (!String.IsNullOrEmpty(contractname))
            {
                e.CheckIsCandidateContainsTarget(contractname);
            }
        }
        //置信度
        e.Confidence = ContractTraning.ContractES.GetStardardCI();
        return(e.EvaluateCI());
    }
Exemple #22
0
    public override void CompileParseTree(Compilation compilation, ErrorNodeList errorNodes){
      if (compilation == null || compilation.CompilationUnits == null || compilation.TargetModule == null){Debug.Assert(false); return;}
      if (compilation.CompilationUnits.Count == 0) return;
      TrivialHashtable ambiguousTypes = new TrivialHashtable();
      TrivialHashtable referencedLabels = new TrivialHashtable();
      TrivialHashtable scopeFor = new TrivialHashtable(64);
      ErrorHandler errorHandler = new ErrorHandler(errorNodes);
      SpecSharpCompilation ssCompilation = new SpecSharpCompilation();
      SpecSharpCompilerOptions options = (SpecSharpCompilerOptions)compilation.CompilerParameters;

      //Attach scopes to namespaces and types so that forward references to base types can be looked up in the appropriate namespace scope
      Scoper scoper = new Scoper(scopeFor);
      scoper.VisitCompilation(compilation);
      scoper = null;

      if (options.NoStandardLibrary && compilation.TargetModule is AssemblyNode) {
        if (compilation.TargetModule.IsValidTypeName(StandardIds.System, StandardIds.CapitalObject)) {
          SystemAssemblyLocation.ParsedAssembly = (AssemblyNode)compilation.TargetModule;
          SystemCompilerRuntimeAssemblyLocation.ParsedAssembly = (AssemblyNode)compilation.TargetModule; //So that mscorlib can have contracts but no reference to another assembly
        } else if (compilation.TargetModule.IsValidTypeName(Identifier.For("System.Compiler"), Identifier.For("ComposerAttribute")))
          SystemCompilerRuntimeAssemblyLocation.ParsedAssembly = (AssemblyNode)compilation.TargetModule;
        else if (compilation.TargetModule.IsValidTypeName(Identifier.For("Microsoft.SpecSharp"), Identifier.For("dummy")))
          RuntimeAssemblyLocation.ParsedAssembly = (AssemblyNode)compilation.TargetModule;
      }
      object ObjectType = SystemTypes.Object;
      if (ObjectType == null) return; //system types did not initialize

      //Walk IR looking up names
      Looker looker = new Looker(compilation.GlobalScope, errorHandler, scopeFor, ambiguousTypes, referencedLabels);
      if (options != null && options.EmitSourceContextsOnly)
      {
        looker.DontInjectDefaultConstructors = true;
      }

      // begin change by drunje
      looker.AllowPointersToManagedStructures = options.AllowPointersToManagedStructures;
      // end change by drunje
      looker.VisitCompilation(compilation);
      looker = null;

      if (options != null && options.EmitSourceContextsOnly) return; // stop after looker to have resolved types

      //Walk IR inferring types and resolving overloads
      TypeSystem typeSystem = new TypeSystem(errorHandler);
      Resolver resolver = new Resolver(errorHandler, typeSystem);
      resolver.VisitCompilation(compilation);
      resolver = null;

      //Walk IR checking for semantic errors and repairing it so that the next walk will work
      Checker checker = new Checker(ssCompilation, errorHandler, typeSystem, scopeFor, ambiguousTypes, referencedLabels);
      checker.VisitCompilation(compilation);
      checker = null;
      scopeFor = null;
      ambiguousTypes = null;
      referencedLabels = null;

      if (!options.IsContractAssembly) {
        if (options.RunProgramVerifier)
          ssCompilation.AddProgramVerifierPlugin(typeSystem, compilation);

        //Allow third party extensions to analyze AST IR for further errors
        ssCompilation.RunPlugins(compilation, errorHandler);
      }

      //Walk IR reducing it to nodes that have predefined mappings to MD+IL
      Normalizer normalizer = new Normalizer(typeSystem);
      normalizer.VisitCompilation(compilation);
      normalizer = null;

      if (options.IsContractAssembly) return;
      //Walk normalized IR instrumenting accesses of fields of guarded classes with checks
      CompilationUnit cu = compilation.CompilationUnits[0];
      if (cu != null && cu.PreprocessorDefinedSymbols != null && cu.PreprocessorDefinedSymbols.ContainsKey("GuardedFieldAccessChecks")){
        if (errorNodes.Count == 0){
          GuardedFieldAccessInstrumenter instrumenter = new GuardedFieldAccessInstrumenter();
          instrumenter.VisitCompilation(compilation);
          instrumenter = null;
        }
      }

      //Walk normalized IR doing code analysis
      Analyzer analyzer = new Analyzer(typeSystem, compilation);
      analyzer.Visit(compilation);

      //Allow third party extensions to analyze normalized IR for further errors
      ssCompilation.analyzer = analyzer; // make the analyzer available to plugins for access to method CFGs
      ssCompilation.RunPlugins(compilation, errorHandler);
      ssCompilation.analyzer = null;
      ssCompilation = null;
      analyzer = null;
      errorHandler = null;

      //Walk IR to optimize code further after analyses were performed, eg. to remove debug only code
      Optimizer optimizer = new Optimizer();
      optimizer.Visit(compilation);
      optimizer = null;
    }
Exemple #23
0
        public void DBN()
        {
            var dbn = DeepBeliefNetwork.CreateGaussianBernoulli(1024, 10);
            new GaussianWeights(dbn).Randomize();

            dbn.UpdateVisibleWeights();

            var dblNode = new UnsupervisedLearning(new DeepBeliefNetworkLearning(dbn));
            var dataNorm = new Normalizer();
            var fromFile = new TextReader();

            var dataFile = new System.IO.FileInfo(@"C:\Users\Dan\Documents\Visual Studio 2015\Projects\AccordNETSamples\framework-master\Samples\Neuro\Deep Learning\Resources\optdigits-tra.txt");

            // POST DataFile -> TextReader -> ? -> Normalizer -> UnsupervisedLearning
            var dataProc = new TransformBlock<Instance<string>, IObservable<List<Instance<double>>>>(inst =>
            {
                var reader = new System.IO.StringReader(inst.Data);
                var samples = new Subject<List<Instance<double>>>();

                var epochSamples = new List<MachineLearningData<double[], double>>();
                var buffer = new char[(32 + 1) * 32]; // 32 chars + \n
                var count = 0;

                while (true)
                {
                    var read = reader.ReadBlock(buffer, 0, buffer.Length);
                    var label = reader.ReadLine();

                    if (read < buffer.Length || label == null)
                        break;

                    var currMLS = new MachineLearningData<double[], double>();
                    currMLS.Class = Convert.ToInt32(label);

                    count++;
                }

                return samples.AsObservable();
            });
        }
Exemple #24
0
 public void TestInitialize()
 {
     _normalizer = new Normalizer();
 }
Exemple #25
0
        internal Expression ContextAttributeConstructor(AttributeList attrs)
        {
            AttributeNode contextAttr = this.GetContextAttribute(attrs);

            if (contextAttr == null)
                return new Literal(null, SystemTypes.Object);

            Duplicator duplicator = new Duplicator(null, null);
            AttributeNode dupAttr = duplicator.VisitAttributeNode(contextAttr);

            Construct cons = (Construct)Templates.GetExpressionTemplate("ContextAttributeConstructor");

            Replacer.Replace(cons, "_AttributeName", dupAttr.Type.Name);

            Normalizer normalizer = new Normalizer(false);
            cons.Operands = normalizer.VisitExpressionList(dupAttr.Expressions);

            return cons;
        }
Exemple #26
0
 public Expression ImplicitCoercionAdapter(Expression source, TypeNode targetType, TypeViewer typeViewer){
   ErrorHandler savedErrorHandler = this.ErrorHandler;
   this.ErrorHandler = null;
   Expression e = null;
   try
   {
     e = this.ImplicitCoercion(source, targetType, typeViewer);
     if (e is CoerceTuple || e is BlockExpression || e is ConstructTuple)
     {
       Normalizer n = new Normalizer(this);
       e = n.VisitExpression(e);
     }
   }
   finally
   {
     this.ErrorHandler = savedErrorHandler;
   };
   return e;
 }
Exemple #27
0
        private void AddBlockMethod(ZMethod zMethod, Class methodClass, BasicBlock block, List<Scope> nonTrivialScopes)
        {
            Method blockMethod = (Method)Templates.GetTypeTemplateByName("BlockMethod").Members[0];
            blockMethod.Name = new Identifier(block.Name);
            methodClass.Members.Add(blockMethod);
            blockMethod.DeclaringType = methodClass;

            // Generate the appropriate closing statements for the block. Indicate if the
            // block terminates an atomic region and establish the transfer of control to
            // the next block(s) or out of the method.

            if ((ZingCompilerOptions.IsPreemtive && !block.MiddleOfTransition && !block.IsReturn) || (block.Yields))
            {
                // p.MiddleOfTransition = false;
                blockMethod.Body.Statements.Add(
                    new ExpressionStatement(
                        new AssignmentExpression(
                            new AssignmentStatement(
                                new QualifiedIdentifier(Identifier.For("p"), Identifier.For("MiddleOfTransition")),
                                new Literal(false, SystemTypes.Boolean)
                            )
                        )
                    )
                );
            }

            // p.AtomicityLevel = this.SavedAtomicityLevel + X;
            blockMethod.Body.Statements.Add(
                new ExpressionStatement(
                    new AssignmentExpression(
                        new AssignmentStatement(
                            new QualifiedIdentifier(Identifier.For("p"), Identifier.For("AtomicityLevel")),
                            new BinaryExpression(
                                new QualifiedIdentifier(new This(), Identifier.For("SavedAtomicityLevel")),
                                new Literal(block.RelativeAtomicLevel, SystemTypes.Int32),
                                NodeType.Add
                            )
                        )
                    )
                )
            );

            #if false
            //
            // The following code was added for summarization, but isn't quite right. It
            // updates the nextBlock too early for some blocks. -- Tony
            //
            //
            // when generating summaries of type MaxCall, we need to
            // know the value of nextBlock before we invoke p.Call().
            // the first of the two basic blocks of a Zing method call
            // is guaranteed to fall through, so we only need to lift
            // the assignment of nextBlock for fall-through blocks.
            if (block.ConditionalTarget == null && block.UnconditionalTarget != null)
            {
                stmt = Templates.GetStatementTemplate("UnconditionalBlockTransfer");
                Replacer.Replace(stmt, "_UnconditionalBlock",
                                 new Identifier(block.UnconditionalTarget.Name));
                blockMethod.Body.Statements.Add(stmt);
            }
            #endif

            if (block.Attributes != null)
            {
                Duplicator duplicator = new Duplicator(null, null);

                for (int i = 0, n = block.Attributes.Count; i < n; i++)
                {
                    if (block.Attributes[i] == null)
                        continue;

                    AttributeNode dupAttr = duplicator.VisitAttributeNode(block.Attributes[i]);

                    Normalizer normalizer = new Normalizer(false);
                    ExpressionList attrParams = normalizer.VisitExpressionList(dupAttr.Expressions);

                    // application.Trace(_context, _contextAttr, new Z.Attributes._attrName(...) );
                    ExpressionStatement traceStmt = new ExpressionStatement(
                        new MethodCall(
                            new QualifiedIdentifier(Identifier.For("application"), Identifier.For("Trace")),
                            new ExpressionList(
                                SourceContextConstructor(dupAttr.SourceContext),
                                new Literal(null, SystemTypes.Object),
                                new Construct(
                                    new MemberBinding(
                                        null,
                                        new TypeExpression(
                                            new QualifiedIdentifier(
                                                new QualifiedIdentifier(Identifier.For("Z"), Identifier.For("Attributes")),
                                                dupAttr.Type.Name
                                            )
                                        )
                                    ),
                                    attrParams
                                )
                            )
                        )
                    );
                    blockMethod.Body.Statements.Add(traceStmt);
                }
            }

            if (block.Statement != null)
            {
                if (block.SkipNormalizer)
                    blockMethod.Body.Statements.Add(block.Statement);
                else
                {
                    // Do statement-level code-gen pass on the block's statement
                    Normalizer normalizer = new Normalizer(this, block.Attributes, block.SecondOfTwo);

                    blockMethod.Body.Statements.Add((Statement)normalizer.Visit(block.Statement));
                }
            }

            if (block.ConditionalTarget != null && block.ConditionalExpression != null)
            {
                Block trueBlock, falseBlock;

                // if (_conditionalExpression)
                //     nextBlock = Blocks._conditionalTarget;
                // else
                //     nextBlock = Blocks._unconditionalTarget;
                blockMethod.Body.Statements.Add(
                    new If(
                        block.ConditionalExpression,
                        trueBlock = new Block(new StatementList(
                            new ExpressionStatement(
                                new AssignmentExpression(
                                    new AssignmentStatement(
                                        Identifier.For("nextBlock"),
                                        new QualifiedIdentifier(
                                            Identifier.For("Blocks"),
                                            Identifier.For(block.ConditionalTarget.Name)
                                        )
                                    )
                                )
                            )
                        )),
                        falseBlock = new Block(new StatementList(
                            new ExpressionStatement(
                                new AssignmentExpression(
                                    new AssignmentStatement(
                                        Identifier.For("nextBlock"),
                                        new QualifiedIdentifier(
                                            Identifier.For("Blocks"),
                                            Identifier.For(block.UnconditionalTarget.Name)
                                        )
                                    )
                                )
                            )
                        ))
                    )
                );

                AddScopeCleanupCalls(trueBlock.Statements, block, block.ConditionalTarget, nonTrivialScopes);
                AddScopeCleanupCalls(falseBlock.Statements, block, block.UnconditionalTarget, nonTrivialScopes);
            }
            else if (block.UnconditionalTarget != null)
            {
                // nextBlock = Blocks.X;
                blockMethod.Body.Statements.Add(
                    new ExpressionStatement(
                        new AssignmentExpression(
                            new AssignmentStatement(
                                Identifier.For("nextBlock"),
                                new QualifiedIdentifier(Identifier.For("Blocks"), Identifier.For(block.UnconditionalTarget.Name))
                            )
                        )
                    )
                );
                AddScopeCleanupCalls(blockMethod.Body.Statements, block, block.UnconditionalTarget, nonTrivialScopes);
            }
            else if (block.IsReturn)
            {
                Debug.Assert(block.UnconditionalTarget == null);

                Statement returnCall = Templates.GetStatementTemplate("ReturnBlockTransfer");
                SourceContext context;
                Return ret = block.Statement as Return;
                if (ret != null)
                {
                    context = ret.SourceContext;
                }
                else
                {
                    // If not a return stmt, the context is the closing brace of the method
                    context = zMethod.SourceContext;
                    context.StartPos = context.EndPos - 1;
                }

                Replacer.Replace(returnCall, "_context", SourceContextConstructor(context));
                Replacer.Replace(returnCall, "_contextAttr", ContextAttributeConstructor(block.Attributes));

                blockMethod.Body.Statements.Add(returnCall);

                // StateImpl.IsReturn = true;
                blockMethod.Body.Statements.Add(
                    new ExpressionStatement(
                        new AssignmentExpression(
                            new AssignmentStatement(
                                new QualifiedIdentifier(Identifier.For("StateImpl"), Identifier.For("IsReturn")),
                                new Literal(true, SystemTypes.Boolean)
                            )
                        )
                    )
                );
            }
        }
Exemple #28
0
        private void PatchIsAtomicEntryMethod(Class methodClass, List<BasicBlock> basicBlocks)
        {
            Method atomicEntryMethod = (Method)
                Templates.GetMemberByName(methodClass.Members, "IsAtomicEntryBlock");
            Debug.Assert(atomicEntryMethod.Body.Statements[0] is System.Compiler.Switch);
            System.Compiler.Switch switchStmt = (System.Compiler.Switch)atomicEntryMethod.Body.Statements[0];

            Normalizer normalizer = new Normalizer(this, null, false);

            foreach (BasicBlock block in basicBlocks)
            {
                SwitchCase newCase;
                Expression trueExpr;

                if (block.IsAtomicEntry)
                {
                    Debug.Assert(block.RelativeAtomicLevel == 1);

                    trueExpr = new Literal(true, SystemTypes.Boolean);
                    newCase = ((System.Compiler.Switch)Templates.GetStatementTemplate("RunnableSwitchSelect")).Cases[0];
                    Replacer.Replace(newCase, "_BlockName", new Identifier(block.Name));
                    Replacer.Replace(newCase, "_expr", trueExpr);
                    switchStmt.Cases.Add(newCase);
                }
            }
        }
Exemple #29
0
        private void PatchRunnableMethod(Class methodClass, List<BasicBlock> basicBlocks)
        {
            Method runnableJSMethod = (Method)
                Templates.GetMemberByName(methodClass.Members, "GetRunnableJoinStatements");
            Debug.Assert(runnableJSMethod.Body.Statements[0] is System.Compiler.Switch);
            System.Compiler.Switch switchStmt = (System.Compiler.Switch)runnableJSMethod.Body.Statements[0];

            Normalizer normalizer = new Normalizer(this, null, false);

            foreach (BasicBlock block in basicBlocks)
            {
                SwitchCase newCase;

                if (block.selectStmt != null)
                {
                    Expression runnableExpr = null;

                    for (int i = 0, n = block.selectStmt.joinStatementList.Length; i < n; i++)
                    {
                        Expression jsRunnable = normalizer.GetRunnablePredicate(block.selectStmt.joinStatementList[i]);

                        if (jsRunnable == null)
                            continue;

                        Expression jsRunnableBit = Templates.GetExpressionTemplate("JoinStatementRunnableBit");
                        Replacer.Replace(jsRunnableBit, "_jsRunnableBoolExpr", jsRunnable);
                        Replacer.Replace(jsRunnableBit, "_jsBitMask", (Expression)new Literal((ulong)(1 << i), SystemTypes.UInt64));

                        if (runnableExpr == null)
                            runnableExpr = jsRunnableBit;
                        else
                            runnableExpr = new BinaryExpression(runnableExpr, jsRunnableBit, NodeType.Or, block.selectStmt.SourceContext);
                    }

                    if (runnableExpr != null)
                    {
                        newCase = ((System.Compiler.Switch)Templates.GetStatementTemplate("RunnableSwitchSelect")).Cases[0];
                        Replacer.Replace(newCase, "_BlockName", new Identifier(block.Name));
                        Replacer.Replace(newCase, "_expr", runnableExpr);
                        switchStmt.Cases.Add(newCase);
                    }

                    //
                    // Now check for blocks that flow atomically into this one and add cases for them
                    // as well. This can happen when the target of a "goto" is a label preceding a
                    // select statement.
                    //
                    foreach (BasicBlock previousBlock in basicBlocks)
                    {
                        if (previousBlock.UnconditionalTarget == block && previousBlock.ConditionalTarget == null &&
                            previousBlock.Statement == null && previousBlock.MiddleOfTransition)
                        {
                            newCase = ((System.Compiler.Switch)Templates.GetStatementTemplate("RelatedSwitchSelect")).Cases[0];
                            Replacer.Replace(newCase, "_BlockName", new Identifier(previousBlock.Name));
                            Replacer.Replace(newCase, "_TargetName", new Identifier(block.Name));
                            switchStmt.Cases.Add(newCase);
                        }
                    }
                }
            }
        }
 public override void Space(Normalizer normalizer, char c)
 {
 }
Exemple #31
0
        private void ProcessGlobals(MemberList globals)
        {
            // Locate the "Globals" struct so we can add fields to it.
            Class globalsClass = (Class)Templates.GetMemberByName(appClass.Members,
                                                                   "GlobalVars");

            // Locate the "initialization" constructor so we can add initialization
            // statements to it.
            Debug.Assert(appClass.Members[1].Name.Name == ".ctor");
            Method initCtor = (Method)appClass.Members[1];

            // Locate the WriteString method so we can add statements to write out the globals
            Method writer =
                (Method)Templates.GetMemberByName(globalsClass.Members, "WriteString");
            Method copier =
                (Method)Templates.GetMemberByName(globalsClass.Members, "CopyContents");
            Method traverser = (Method)Templates.GetMemberByName(globalsClass.Members, "TraverseFields");

            Normalizer normalizer = new Normalizer(false);

            Method getValue = (Method)Templates.GetMemberByName(globalsClass.Members, "GetValue");
            Method setValue = (Method)Templates.GetMemberByName(globalsClass.Members, "SetValue");
            System.Compiler.Switch switchGetValue =
                (System.Compiler.Switch)Templates.GetStatementTemplate("GetFieldInfoSwitch");
            getValue.Body.Statements.Add(switchGetValue);
            System.Compiler.Switch switchSetValue =
                (System.Compiler.Switch)Templates.GetStatementTemplate("SetFieldInfoSwitch");
            setValue.Body.Statements.Add(switchSetValue);

            for (int i = 0, n = globals.Count; i < n; i++)
            {
                // Make a shallow copy of the field since we're going to tinker with it
                Field f = (Field)((Field)globals[i]).Clone();
                string name = f.DeclaringType.Name.Name + "_" + f.Name.Name;

                TypeNode zingType = f.Type;

                if (GetTypeClassification(f.Type) == TypeClassification.Heap)
                    f.Type = this.ZingPtrType;
                else if (!IsPredefinedType(f.Type))
                    f.Type = new TypeExpression(new QualifiedIdentifier(
                        new Identifier("Application"), zingType.Name), f.Type.SourceContext);

                // Mangle the name to guarantee uniqueness across the globals
                f.Name = new Identifier("priv_" + name);
                f.Flags &= ~FieldFlags.Static;
                // Sriram: I have made this into a public field so that
                // the fieldInfo below works. If the field is internal
                // then fieldInfo gets set to null (there is some access
                // permission problem)
                // Need to check with Tony as to
                // whether this is tbe best solution

                // The following two lines are Tony's code
                //-----------------------------------------------------------
                // f.Flags &= (FieldFlags)(~TypeFlags.VisibilityMask);
                //f.Flags |= FieldFlags.Assembly;
                //-----------------------------------------------------------

                //This is my replacement
                f.Flags |= FieldFlags.Public;

                /*
                Identifier idFieldName = new Identifier("id_" + name);
                System.Compiler.Expression idTypeExpr =
                    new QualifiedIdentifier(
                        new QualifiedIdentifier(new Identifier("System"), new Identifier("Reflection")),
                        new Identifier("FieldInfo"));
                System.Compiler.TypeNode idfType =  new System.Compiler.TypeExpression(idTypeExpr);
                Field idf = new Field(globalsClass, null, FieldFlags.Public|FieldFlags.Static,
                                       idFieldName,
                                       idfType, null);
                idf.Initializer = Templates.GetExpressionTemplate("GetFieldInfo");
                Replacer.Replace(idf.Initializer, "_class", globalsClass.Name);
                Replacer.Replace(idf.Initializer, "_fieldName", new Literal(f.Name.Name, SystemTypes.String));
                */

                Identifier idFieldName = new Identifier("id_" + name);
                Field idf = new Field(globalsClass, null, FieldFlags.Public | FieldFlags.Static,
                    idFieldName,
                    SystemTypes.Int32, null);
                idf.Initializer = new Literal(i, SystemTypes.Int32);
                SwitchCase getCase = ((System.Compiler.Switch)Templates.GetStatementTemplate("GetFieldInfoCase")).Cases[0];
                Replacer.Replace(getCase, "_fieldId", new Literal(i, SystemTypes.Int32));
                Replacer.Replace(getCase, "_fieldName", new Identifier(f.Name.Name));
                switchGetValue.Cases.Add(getCase);

                SwitchCase setCase = ((System.Compiler.Switch)Templates.GetStatementTemplate("SetFieldInfoCase")).Cases[0];
                Replacer.Replace(setCase, "_fieldId", new Literal(i, SystemTypes.Int32));
                Replacer.Replace(setCase, "_fieldName", new Identifier(f.Name.Name));
                TypeExpression tn = f.Type as TypeExpression;
                Replacer.Replace(setCase, "_fieldType", tn != null ? tn.Expression : new Identifier(f.Type.Name.Name));
                switchSetValue.Cases.Add(setCase);

                //The last argument to the call below is a dont care
                Property accessor = GetAccessorProperty("globalAccessor", f.Type,
                                                        new Identifier(name), f.Name, idFieldName, f.Name);

                globalsClass.Members.Add(f);
                globalsClass.Members.Add(idf);
                globalsClass.Members.Add(accessor);
                f.DeclaringType = globalsClass;
                idf.DeclaringType = globalsClass;
                accessor.DeclaringType = globalsClass;
                if (accessor.Getter != null)
                {
                    globalsClass.Members.Add(accessor.Getter);
                    accessor.Getter.DeclaringType = globalsClass;
                }
                if (accessor.Setter != null)
                {
                    globalsClass.Members.Add(accessor.Setter);
                    accessor.Setter.DeclaringType = globalsClass;
                }

                if (zingType is Struct && !zingType.IsPrimitive && f.Type != SystemTypes.Decimal)
                    collectStructAccessors(true, (Struct)zingType, f.Name,
                                           name, globalsClass);

                if (f.Initializer != null)
                {
                    Statement stmt = Templates.GetStatementTemplate("InitializeGlobal");
                    Replacer.Replace(stmt, "_FieldName", f.Name);
                    Replacer.Replace(stmt, "_FieldInitializer", normalizer.VisitFieldInitializer(f.Initializer));
                    initCtor.Body.Statements.Add(stmt);
                    f.Initializer = null;
                }

                writer.Body.Statements.Add(GetWriterStatement(null, zingType, f.Name));
                copier.Body.Statements.Add(GetCopyStatement(f.Name));
                traverser.Body.Statements.Add(GetTraverserStatement(null, zingType, f.Name));
                /*if(GetTypeClassification(f.Type) == TypeClassification.Heap)
                {
                    refTraverser.Body.Statements.Add(GetTraverserStatement(null, zingType, f.Name));
                }
                */
            }
        }
 public override void TheOthers(Normalizer normalizer, char c)
 {
     normalizer._stringBuilder.Append(' ');
     normalizer._stringBuilder.Append(c);
     normalizer._state = Normalizer._theOthersState;
 }
Exemple #33
0
        private void GenerateClass(Class c)
        {
            // The following code added by Jiri Adamek
            // Do not generate any code for native ZOM classes (they were manually written
            // and their code is placed in another assmbly)
            if (c is NativeZOM) return;
            // END of added code

            TypeNode newClass = Templates.GetTypeTemplateByName("Class");
            if (c.Interfaces != null)
            {
                for (int i = 0, n = c.Interfaces.Count; i < n; i++)
                {
                    string iname = c.Interfaces[i].Name.Name;
                    QualifiedIdentifier id = new QualifiedIdentifier(new Identifier(iname), new Identifier("CreateMethods"));
                    newClass.Interfaces.Add(new InterfaceExpression(id));
                }
            }

            Method writer = (Method)Templates.GetMemberByName(newClass.Members, "WriteString");
            Method traverser = (Method)Templates.GetMemberByName(newClass.Members, "TraverseFields");

            // Replace all references to the class name
            Replacer.Replace(newClass, newClass.Name, c.Name);
            SetTypeId(newClass);

            Block cloneFields = new Block();
            cloneFields.Statements = new StatementList();

            Method getValue = (Method)Templates.GetMemberByName(newClass.Members, "GetValue");
            Method setValue = (Method)Templates.GetMemberByName(newClass.Members, "SetValue");
            System.Compiler.Switch switchGetValue =
                (System.Compiler.Switch)Templates.GetStatementTemplate("GetFieldInfoSwitch");
            getValue.Body.Statements.Add(switchGetValue);
            System.Compiler.Switch switchSetValue =
                (System.Compiler.Switch)Templates.GetStatementTemplate("SetFieldInfoSwitch");
            setValue.Body.Statements.Add(switchSetValue);

            // Transfer non-static fields to the emitted class
            for (int i = 0, n = c.Members.Count; i < n; i++)
            {
                Field f = c.Members[i] as Field;

                if (f != null && f.Type != null && !f.IsStatic)
                {
                    // Clone the field since we might tinker with it
                    Field newField = (Field)f.Clone();
                    // change name of the field, so that the accessor can be named appropriately
                    newField.Name = new Identifier("priv_" + f.Name.Name);

                    if (GetTypeClassification(f.Type) == TypeClassification.Heap)
                        newField.Type = this.ZingPtrType;
                    else if (!IsPredefinedType(f.Type))
                        newField.Type = new TypeExpression(new QualifiedIdentifier(
                            new Identifier("Application"), f.Type.Name), f.Type.SourceContext);

                    if (newField.Initializer != null)
                    {
                        // Move the initialization to our constructor.
                        Expression initializer = newField.Initializer;
                        newField.Initializer = null;

                        Statement initStmt = Templates.GetStatementTemplate("InitComplexInstanceField");
                        Replacer.Replace(initStmt, "_FieldName", newField.Name);
                        Normalizer normalizer = new Normalizer(false);
                        Replacer.Replace(initStmt, "_expr", normalizer.VisitFieldInitializer(initializer));
                        Method ctor = (Method)newClass.Members[0];
                        Debug.Assert(ctor.Parameters.Count == 1);
                        ctor.Body.Statements.Add(initStmt);
                    }

                    Identifier idFieldName = new Identifier("id_" + f.Name.Name);
                    Field idf = new Field(newClass, null, FieldFlags.Public | FieldFlags.Static,
                        idFieldName,
                        SystemTypes.Int32, null);
                    idf.Initializer = new Literal(i, SystemTypes.Int32);
                    SwitchCase getCase = ((System.Compiler.Switch)Templates.GetStatementTemplate("GetFieldInfoCase")).Cases[0];
                    Replacer.Replace(getCase, "_fieldId", new Literal(i, SystemTypes.Int32));
                    Replacer.Replace(getCase, "_fieldName", new Identifier(newField.Name.Name));
                    switchGetValue.Cases.Add(getCase);

                    SwitchCase setCase = ((System.Compiler.Switch)Templates.GetStatementTemplate("SetFieldInfoCase")).Cases[0];
                    Replacer.Replace(setCase, "_fieldId", new Literal(i, SystemTypes.Int32));
                    Replacer.Replace(setCase, "_fieldName", new Identifier(newField.Name.Name));
                    TypeExpression tn = newField.Type as TypeExpression;
                    Replacer.Replace(setCase, "_fieldType", tn != null ? tn.Expression : new Identifier(newField.Type.Name.Name));
                    switchSetValue.Cases.Add(setCase);

                    newClass.Members.Add(newField);
                    newField.DeclaringType = newClass;
                    newClass.Members.Add(idf);
                    idf.DeclaringType = newClass;

                    // add property for the field
                    Property accessor = GetFieldAccessorProperty(f.Type, newField.Type, f.Name, newField.Name, idFieldName);
                    newClass.Members.Add(accessor);
                    accessor.DeclaringType = newClass;
                    if (accessor.Getter != null)
                    {
                        newClass.Members.Add(accessor.Getter);
                        accessor.Getter.DeclaringType = newClass;
                    }
                    if (accessor.Setter != null)
                    {
                        newClass.Members.Add(accessor.Setter);
                        accessor.Setter.DeclaringType = newClass;
                    }

                    writer.Body.Statements.Add(GetWriterStatement("this", f.Type, newField.Name));

                    traverser.Body.Statements.Add(GetTraverserStatement("this", f.Type, newField.Name));
                    /*if(GetTypeClassification(f.Type) == TypeClassification.Heap)
                    {
                        refTraverser.Body.Statements.Add(GetTraverserStatement("this", f.Type, newField.Name));
                    }
                    */

                    Statement cloneStmt = GetCloneStatement(newField.Name);
                    cloneFields.Statements.Add(cloneStmt);
                }

                ZMethod zMethod = c.Members[i] as ZMethod;
                if (zMethod != null)
                {
                    InterfaceList xs = FindMatchingInterfaces(c, zMethod);
                    Interface x = null;
                    if (xs.Count != 0)
                    {
                        // TODO: Handle the case when one method implements methods declared in multiple interfaces
                        Debug.Assert(xs.Count == 1);
                        x = xs[0];
                    }
                    Class methodClass = GenerateClassMethod(zMethod, x);
                    newClass.Members.Add(methodClass);
                    methodClass.DeclaringType = newClass;
                    methodClass.Flags = (methodClass.Flags & ~TypeFlags.VisibilityMask) | TypeFlags.NestedFamORAssem;

                    if (x != null)
                    {
                        TypeNode tn = Templates.GetTypeTemplateByName("ClassExtras");
                        Method member = (Method)Templates.GetMemberByName(tn.Members, "__CreateInterfaceMethod");
                        member.Name = new Identifier("Create" + methodClass.Name.Name);
                        member.DeclaringType = newClass;
                        Replacer.Replace(member,
                            new Identifier("__InterfaceMethod"),
                            new QualifiedIdentifier(x.Name, methodClass.Name));
                        Replacer.Replace(member,
                            new Identifier("__ClassMethod"),
                            new QualifiedIdentifier(c.Name, methodClass.Name));
                        newClass.Members.Add(member);
                    }
                }
            }

            // Splice the cloning assignment statements into the class's Clone method
            // at the appropriate place.
            Method cloner = (Method)Templates.GetMemberByName(newClass.Members, "Clone");
            Replacer.Replace(cloner.Body, "cloneFields", cloneFields);

            // Add the emitted class to our Zing application class
            InstallType(newClass);
        }
 public abstract void SpecialChars(Normalizer normalizer, char c);
Exemple #35
0
        public override Statement VisitForEach(ForEach forEach)
        {
            Normalizer normalizer = new Normalizer(false);
            Identifier incrVar = new Identifier("____" + forEach.UniqueKey.ToString(CultureInfo.InvariantCulture));

            Expression sourceEnumerable = normalizer.VisitExpression(forEach.SourceEnumerable);

            Statement incrStmt = Templates.GetStatementTemplate("foreachIncrementer");
            Replacer.Replace(incrStmt, "_iterator", incrVar);
            BasicBlock incrBlock = new BasicBlock(incrStmt);
            AddBlock(incrBlock);
            incrBlock.MiddleOfTransition = true;
            incrBlock.SkipNormalizer = true;
            incrBlock.SourceContext = forEach.SourceContext;

            PushContinuationStack(incrBlock);
            this.Visit(forEach.Body);
            BasicBlock bodyBlock = PopContinuationStack();

            Statement derefStmt = Templates.GetStatementTemplate("foreachDeref");
            Replacer.Replace(derefStmt, "_tmpVar",
                             normalizer.VisitExpression(forEach.TargetVariable));
            Replacer.Replace(derefStmt, "_collectionExpr", sourceEnumerable);
            Replacer.Replace(derefStmt, "_collectionType",
                             new Identifier(forEach.SourceEnumerable.Type.FullName));
            Replacer.Replace(derefStmt, "_iterator", incrVar);
            BasicBlock derefBlock = new BasicBlock(derefStmt, bodyBlock);
            AddBlock(derefBlock);
            derefBlock.MiddleOfTransition = true;
            derefBlock.SkipNormalizer = true;

            Expression testExpr = Templates.GetExpressionTemplate("foreachTest");
            Replacer.Replace(testExpr, "_iterator", incrVar);
            Replacer.Replace(testExpr, "_sourceEnumerable", sourceEnumerable);
            BasicBlock testBlock = new BasicBlock(null, testExpr, derefBlock, CurrentContinuation);
            AddBlock(testBlock);
            testBlock.SkipNormalizer = true;

            incrBlock.UnconditionalTarget = testBlock;

            Statement initStmt = Templates.GetStatementTemplate("foreachInit");
            Replacer.Replace(initStmt, "_iterator", incrVar);
            BasicBlock initBlock = new BasicBlock(initStmt, testBlock);
            AddBlock(initBlock);
            initBlock.MiddleOfTransition = true;
            initBlock.SkipNormalizer = true;
            initBlock.SourceContext = forEach.SourceContext;

            CurrentContinuation = initBlock;

            return forEach;
        }
 public override void Space(Normalizer normalizer, char c)
 {
     normalizer._state = Normalizer._spaceState;
 }
Exemple #37
0
        public override Statement VisitIf(If If)
        {
            if (insideAtomicBlock && IsCoalescableIfStmt(If))
            {
                CurrentContinuation =
                    AddBlock(new BasicBlock(If, CurrentContinuation));
                return If;
            }

            PushContinuationStack();
            this.Visit(If.TrueBlock);
            BasicBlock trueBlock = PopContinuationStack();

            PushContinuationStack();
            this.Visit(If.FalseBlock);
            BasicBlock falseBlock = PopContinuationStack();

            SourceContext savedConditionalContext = If.SourceContext;
            if (If.Condition != null)
                savedConditionalContext = If.Condition.SourceContext;

            // We normalize all of the conditional expressions here to make life
            // easier for "select"
            Normalizer normalizer = new Normalizer(splicer, null, false);

            CurrentContinuation = AddBlock(new BasicBlock(null,
                normalizer.VisitExpression(If.Condition), trueBlock, falseBlock));

            CurrentContinuation.SourceContext = savedConditionalContext;
            if (CurrentContinuation.ConditionalExpression != null)
                CurrentContinuation.ConditionalExpression.SourceContext = savedConditionalContext;

            return If;
        }
 public override void TheOthers(Normalizer normalizer, char c)
 {
     normalizer._stringBuilder.Append(c);
 }
Exemple #39
0
        public override int DivideUpDictionaryRange(CharacterIterator inText, int startPos, int endPos,
                                                    DequeI foundBreaks)
        {
            if (startPos >= endPos)
            {
                return(0);
            }

            inText.SetIndex(startPos);

            int inputLength = endPos - startPos;

            int[]        charPositions = new int[inputLength + 1];
            StringBuffer s             = new StringBuffer("");

            inText.SetIndex(startPos);
            while (inText.Index < endPos)
            {
                s.Append(inText.Current);
                inText.Next();
            }
            string prenormstr   = s.ToString();
            bool   isNormalized = Normalizer.QuickCheck(prenormstr, Normalizer.NFKC) == Normalizer.YES ||
                                  Normalizer.IsNormalized(prenormstr, Normalizer.NFKC, 0);
            CharacterIterator text;
            int numChars = 0;

            if (isNormalized)
            {
                text = new StringCharacterIterator(prenormstr);
                int index = 0;
                charPositions[0] = 0;
                while (index < prenormstr.Length)
                {
                    int codepoint = prenormstr.CodePointAt(index);
                    index += Character.CharCount(codepoint);
                    numChars++;
                    charPositions[numChars] = index;
                }
            }
            else
            {
                string normStr = Normalizer.Normalize(prenormstr, Normalizer.NFKC);
                text          = new StringCharacterIterator(normStr);
                charPositions = new int[normStr.Length + 1];
                Normalizer normalizer = new Normalizer(prenormstr, Normalizer.NFKC, 0);
                int        index      = 0;
                charPositions[0] = 0;
                while (index < normalizer.EndIndex)
                {
                    normalizer.Next();
                    numChars++;
                    index = normalizer.Index;
                    charPositions[numChars] = index;
                }
            }

            // From here on out, do the algorithm. Note that our indices
            // refer to indices within the normalized string.
            int[] bestSnlp = new int[numChars + 1];
            bestSnlp[0] = 0;
            for (int i = 1; i <= numChars; i++)
            {
                bestSnlp[i] = kint32max;
            }

            int[] prev = new int[numChars + 1];
            for (int i = 0; i <= numChars; i++)
            {
                prev[i] = -1;
            }

            int maxWordSize = 20;

            int[] values  = new int[numChars];
            int[] lengths = new int[numChars];
            // dynamic programming to find the best segmentation
            bool is_prev_katakana = false;

            for (int i = 0; i < numChars; i++)
            {
                text.SetIndex(i);
                if (bestSnlp[i] == kint32max)
                {
                    continue;
                }

                int   maxSearchLength = (i + maxWordSize < numChars) ? maxWordSize : (numChars - i);
                int[] count_          = new int[1];
                fDictionary.Matches(text, maxSearchLength, lengths, count_, maxSearchLength, values);
                int count = count_[0];

                // if there are no single character matches found in the dictionary
                // starting with this character, treat character as a 1-character word
                // with the highest value possible (i.e. the least likely to occur).
                // Exclude Korean characters from this treatment, as they should be
                // left together by default.
                text.SetIndex(i); // fDictionary.matches() advances the text position; undo that.
                if ((count == 0 || lengths[0] != 1) && CharacterIteration.Current32(text) != CharacterIteration.DONE32 && !fHangulWordSet.Contains(CharacterIteration.Current32(text)))
                {
                    values[count]  = maxSnlp;
                    lengths[count] = 1;
                    count++;
                }

                for (int j = 0; j < count; j++)
                {
                    int newSnlp = bestSnlp[i] + values[j];
                    if (newSnlp < bestSnlp[lengths[j] + i])
                    {
                        bestSnlp[lengths[j] + i] = newSnlp;
                        prev[lengths[j] + i]     = i;
                    }
                }

                // In Japanese, single-character Katakana words are pretty rare.
                // So we apply the following heuristic to Katakana: any continuous
                // run of Katakana characters is considered a candidate word with
                // a default cost specified in the katakanaCost table according
                // to its length.
                bool is_katakana = IsKatakana(CharacterIteration.Current32(text));
                if (!is_prev_katakana && is_katakana)
                {
                    int j = i + 1;
                    CharacterIteration.Next32(text);
                    while (j < numChars && (j - i) < kMaxKatakanaGroupLength && IsKatakana(CharacterIteration.Current32(text)))
                    {
                        CharacterIteration.Next32(text);
                        ++j;
                    }

                    if ((j - i) < kMaxKatakanaGroupLength)
                    {
                        int newSnlp = bestSnlp[i] + GetKatakanaCost(j - i);
                        if (newSnlp < bestSnlp[j])
                        {
                            bestSnlp[j] = newSnlp;
                            prev[j]     = i;
                        }
                    }
                }
                is_prev_katakana = is_katakana;
            }

            int[] t_boundary = new int[numChars + 1];
            int   numBreaks  = 0;

            if (bestSnlp[numChars] == kint32max)
            {
                t_boundary[numBreaks] = numChars;
                numBreaks++;
            }
            else
            {
                for (int i = numChars; i > 0; i = prev[i])
                {
                    t_boundary[numBreaks] = i;
                    numBreaks++;
                }
                Assert.Assrt(prev[t_boundary[numBreaks - 1]] == 0);
            }

            if (foundBreaks.Count == 0 || foundBreaks.Peek() < startPos)
            {
                t_boundary[numBreaks++] = 0;
            }

            int correctedNumBreaks = 0;

            for (int i = numBreaks - 1; i >= 0; i--)
            {
                int pos = charPositions[t_boundary[i]] + startPos;
                if (!(foundBreaks.Contains(pos) || pos == startPos))
                {
                    foundBreaks.Push(charPositions[t_boundary[i]] + startPos);
                    correctedNumBreaks++;
                }
            }

            if (!foundBreaks.IsEmpty && foundBreaks.Peek() == endPos)
            {
                foundBreaks.Pop();
                correctedNumBreaks--;
            }
            if (!foundBreaks.IsEmpty)
            {
                inText.SetIndex(foundBreaks.Peek());
            }
            return(correctedNumBreaks);
        }