public void VisitThrowsOnNullVisitor() { ParserVisitor target = null; ParserResults results = new ParserResults(new BlockBuilder() { Type = BlockType.Comment }.Build(), new List<RazorError>()); Assert.Throws<ArgumentNullException>("self", () => target.Visit(results)); }
public void VisitThrowsOnNullVisitor() { ParserVisitor target = null; var errorSink = new ParserErrorSink(); var results = new ParserResults(new BlockBuilder() { Type = BlockType.Comment }.Build(), Enumerable.Empty<TagHelperDescriptor>(), errorSink); Assert.Throws<ArgumentNullException>("self", () => target.Visit(results)); }
public void VisitSendsDocumentToVisitor() { // Arrange Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>(); Block root = new BlockBuilder() { Type = BlockType.Comment }.Build(); ParserResults results = new ParserResults(root, new List<RazorError>()); // Act targetMock.Object.Visit(results); // Assert targetMock.Verify(v => v.VisitBlock(root)); }
public void VisitSendsDocumentToVisitor() { // Arrange Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>(); var root = new BlockBuilder() { Type = BlockType.Comment }.Build(); var errorSink = new ErrorSink(); var results = new ParserResults(root, Enumerable.Empty<TagHelperDescriptor>(), errorSink); // Act targetMock.Object.Visit(results); // Assert targetMock.Verify(v => v.VisitBlock(root)); }
public void VisitCallsOnCompleteWhenAllNodesHaveBeenVisited() { // Arrange Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>(); var root = new BlockBuilder() { Type = BlockType.Comment }.Build(); var errorSink = new ErrorSink(); errorSink.OnError(new RazorError("Foo", new SourceLocation(1, 0, 1), length: 3)); errorSink.OnError(new RazorError("Bar", new SourceLocation(2, 0, 2), length: 3)); var results = new ParserResults(root, Enumerable.Empty<TagHelperDescriptor>(), errorSink); // Act targetMock.Object.Visit(results); // Assert targetMock.Verify(v => v.OnComplete()); }
public void VisitCallsOnCompleteWhenAllNodesHaveBeenVisited() { // Arrange Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>(); Block root = new BlockBuilder() { Type = BlockType.Comment }.Build(); List<RazorError> errors = new List<RazorError>() { new RazorError("Foo", 1, 0, 1), new RazorError("Bar", 2, 0, 2) }; ParserResults results = new ParserResults(root, errors); // Act targetMock.Object.Visit(results); // Assert targetMock.Verify(v => v.OnComplete()); }
public static void Visit(this ParserVisitor self, ParserResults result) { if (self == null) { throw new ArgumentNullException("self"); } if (result == null) { throw new ArgumentNullException("result"); } result.Document.Accept(self); foreach (RazorError error in result.ParserErrors) { self.VisitError(error); } self.OnComplete(); }
public void VisitSendsErrorsToVisitor() { // Arrange Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>(); Block root = new BlockBuilder() { Type = BlockType.Comment }.Build(); List<RazorError> errors = new List<RazorError>() { new RazorError("Foo", 1, 0, 1), new RazorError("Bar", 2, 0, 2) }; ParserResults results = new ParserResults(root, errors); // Act targetMock.Object.Visit(results); // Assert targetMock.Verify(v => v.VisitError(errors[0])); targetMock.Verify(v => v.VisitError(errors[1])); }
public void VisitCallsOnCompleteWhenAllNodesHaveBeenVisited() { // Arrange Mock <ParserVisitor> targetMock = new Mock <ParserVisitor>(); Block root = new BlockBuilder() { Type = BlockType.Comment }.Build(); List <RazorError> errors = new List <RazorError>() { new RazorError("Foo", 1, 0, 1), new RazorError("Bar", 2, 0, 2) }; ParserResults results = new ParserResults(root, errors); // Act targetMock.Object.Visit(results); // Assert targetMock.Verify(v => v.OnComplete()); }
public void ParseMethodUsesProvidedParserListenerIfSpecified() { var factory = SpanFactory.CreateCsHtml(); // Arrange RazorParser parser = new RazorParser(new CSharpCodeParser(), new HtmlMarkupParser()); // Act ParserResults results = parser.Parse(new StringReader("foo @bar baz")); // Assert ParserTestBase.EvaluateResults(results, new MarkupBlock( factory.Markup("foo "), new ExpressionBlock( factory.CodeTransition(), factory.Code("bar") .AsImplicitExpression(CSharpCodeParser.DefaultKeywords) .Accepts(AcceptedCharacters.NonWhiteSpace)), factory.Markup(" baz"))); }
protected virtual ParserResults RunParse(string document, Func <ParserBase, Action> parserActionSelector, bool designTimeParser, Func <ParserContext, ParserBase> parserSelector = null, ErrorSink errorSink = null) { parserSelector = parserSelector ?? (c => c.ActiveParser); errorSink = errorSink ?? new ErrorSink(); // Create the source ParserResults results = null; using (var reader = new SeekableTextReader(document)) { try { var codeParser = CreateCodeParser(); var markupParser = CreateMarkupParser(); var context = CreateParserContext(reader, codeParser, markupParser, errorSink); context.DesignTimeMode = designTimeParser; codeParser.Context = context; markupParser.Context = context; // Run the parser parserActionSelector(parserSelector(context))(); results = context.CompleteParse(); } finally { if (results != null && results.Document != null) { WriteTraceLine(string.Empty); WriteTraceLine("Actual Parse Tree:"); WriteNode(0, results.Document); } } } return(results); }
public static double CalculateBiasAndVariance(ParserResults trainingData, List <List <bool> > predictorsPredictions, out double bias) { double variance = 0; bias = 0; // Calculate biar and variance for all trees for (int trainingDataIndex = 0; trainingDataIndex < trainingData.Values.Count; trainingDataIndex++) { var trainingDataValue = trainingData.Values[trainingDataIndex]; double realValue = Transformer.BoolToDouble(trainingDataValue.Output); List <double> allPredictions = new List <double>(predictorsPredictions.Count); foreach (var predictorPredictions in predictorsPredictions) { bool prediction = predictorPredictions[trainingDataIndex]; allPredictions.Add(Transformer.BoolToDouble(prediction)); } double mode = ModeFinder.FindMode(allPredictions); double averagePrediction = allPredictions.Average(); // Now that we have the mode, realValue, and average of all predictions, we can calculate variance and bias double varianceForThisDataPoint = 0; double diffOfRealValueAndAveragePrediction = (realValue - averagePrediction); double biasForThisDataPoint = diffOfRealValueAndAveragePrediction * diffOfRealValueAndAveragePrediction; foreach (var prediction in allPredictions) { var diffForModeAndPrediction = prediction - mode; varianceForThisDataPoint += diffForModeAndPrediction * diffForModeAndPrediction; } varianceForThisDataPoint = varianceForThisDataPoint / allPredictions.Count; // Accumulate variance += varianceForThisDataPoint; bias += biasForThisDataPoint; } variance = variance / trainingData.Values.Count; bias = bias / trainingData.Values.Count; return(variance); }
/// <summary> /// Instantiates a new <see cref="GeneratorResults"/> instance. /// </summary> /// <param name="parserResults">The results of parsing a document.</param> /// <param name="codeGeneratorResult">The results of generating code for the document.</param> /// <param name="chunkTree">A <see cref="ChunkTree"/> for the document.</param> public GeneratorResults(ParserResults parserResults, CodeGeneratorResult codeGeneratorResult, ChunkTree chunkTree) : this(parserResults.Document, parserResults.TagHelperDescriptors, parserResults.ErrorSink, codeGeneratorResult, chunkTree) { if (parserResults == null) { throw new ArgumentNullException(nameof(parserResults)); } if (codeGeneratorResult == null) { throw new ArgumentNullException(nameof(codeGeneratorResult)); } if (chunkTree == null) { throw new ArgumentNullException(nameof(chunkTree)); } }
public virtual CompilerResult Compile(string razorTemplate) { if (razorTemplate == null) { throw new ArgumentNullException("razorTemplate"); } if (String.IsNullOrWhiteSpace(razorTemplate)) { throw new ArgumentException("razorTemplate cannot be empty or white-space!", "razorTemplate"); } ParserResults parserResults = this._templateParser.ParseTemplate(razorTemplate); if (!parserResults.Success) { return(new CompilerResult(parserResults.ParserErrors)); } this._documentTranslator.Translate(parserResults.Document, this._templateBuilder); return(this._templateBuilder.Build()); }
public void VisitSendsErrorsToVisitor() { // Arrange Mock <ParserVisitor> targetMock = new Mock <ParserVisitor>(); Block root = new BlockBuilder() { Type = BlockType.Comment }.Build(); List <RazorError> errors = new List <RazorError>() { new RazorError("Foo", 1, 0, 1), new RazorError("Bar", 2, 0, 2) }; ParserResults results = new ParserResults(root, errors); // Act targetMock.Object.Visit(results); // Assert targetMock.Verify(v => v.VisitError(errors[0])); targetMock.Verify(v => v.VisitError(errors[1])); }
public static double CalculateBiasAndVariance(ParserResults trainingData, List <List <DecisionTreeLevel> > listOfTreesToRunTestOn, out double bias) { double variance = 0; bias = 0; // Calculate biar and variance for all trees foreach (var trainingDataValue in trainingData.Values) { double realValue = Transformer.BoolToDouble(trainingDataValue.Output); List <double> allPredictions = new List <double>(listOfTreesToRunTestOn.Count); foreach (var bagger in listOfTreesToRunTestOn) { bool prediction = DecisionTreeScorer.CalculatePrediction(bagger, trainingDataValue); allPredictions.Add(Transformer.BoolToDouble(prediction)); } double mode = ModeFinder.FindMode(allPredictions); double averagePrediction = allPredictions.Average(); // Now that we have the mode, realValue, and average of all predictions, we can calculate variance and bias double varianceForThisDataPoint = 0; double diffOfRealValueAndAveragePrediction = (realValue - averagePrediction); double biasForThisDataPoint = diffOfRealValueAndAveragePrediction * diffOfRealValueAndAveragePrediction; foreach (var prediction in allPredictions) { var diffForModeAndPrediction = prediction - mode; varianceForThisDataPoint += diffForModeAndPrediction * diffForModeAndPrediction; } varianceForThisDataPoint = varianceForThisDataPoint / allPredictions.Count; // Accumulate variance += varianceForThisDataPoint; bias += biasForThisDataPoint; } variance = variance / trainingData.Values.Count; bias = bias / trainingData.Values.Count; return(variance); }
public List <Veranstaltung> ParseEventTabellen(ParserResults parserResults) { if (parserResults.EventText.Contains("Es gibt keine bevorstehenden Veranstaltungen.")) { return(new List <Veranstaltung>(0)); } var eventTexte = SepariereVeranstalltungen(parserResults.EventText); var events = new List <Veranstaltung>(); for (var i = 0; i < eventTexte.Count; i++) { try { var eventTitle = ExtrahiereEventTitle(parserResults.LinkTexte[i]); events.Add(ParseEineVeranstalltung(eventTexte[i], eventTitle)); } catch (Exception e) { _logger.LogError(e.ToString()); } } return(events); }
public void ConditionalAttributeCollapserDoesNotRemoveUrlAttributeValues() { // Act ParserResults results = ParseDocument("<a href='~/Foo/Bar' />"); Block rewritten = new ConditionalAttributeCollapser(new HtmlMarkupParser().BuildSpan).Rewrite(results.Document); rewritten = new MarkupCollapser(new HtmlMarkupParser().BuildSpan).Rewrite(rewritten); // Assert Assert.Equal(0, results.ParserErrors.Count); EvaluateParseTree(rewritten, new MarkupBlock( Factory.Markup("<a"), new MarkupBlock(new AttributeBlockCodeGenerator(name: "href", prefix: new LocationTagged <string>(" href='", 2, 0, 2), suffix: new LocationTagged <string>("'", 18, 0, 18)), Factory.Markup(" href='").With(SpanCodeGenerator.Null), Factory.Markup("~/Foo/Bar") .WithEditorHints(EditorHints.VirtualPath) .With(new LiteralAttributeCodeGenerator( new LocationTagged <string>(String.Empty, 9, 0, 9), new LocationTagged <SpanCodeGenerator>(new ResolveUrlCodeGenerator(), 9, 0, 9))), Factory.Markup("'").With(SpanCodeGenerator.Null)), Factory.Markup(" />"))); }
protected virtual void RunParseTest( string document, Func <ParserBase, Action> parserActionSelector, Block expectedRoot, IList <RazorError> expectedErrors, bool designTimeParser, Func <ParserContext, ParserBase> parserSelector = null ) { // Create the source ParserResults results = RunParse( document, parserActionSelector, designTimeParser, parserSelector ); // Evaluate the results if (!ReferenceEquals(expectedRoot, IgnoreOutput)) { EvaluateResults(results, expectedRoot, expectedErrors); } }
private static List <Span> ParseDocument(string documentContents, IList <RazorError> errors = null) { errors = errors ?? new List <RazorError>(); var markupParser = new HtmlMarkupParser(); var codeParser = new TestMvcCSharpRazorCodeParser(); var context = new ParserContext( new SeekableTextReader(documentContents), codeParser, markupParser, markupParser, new ErrorSink()); codeParser.Context = context; markupParser.Context = context; markupParser.ParseDocument(); ParserResults results = context.CompleteParse(); foreach (RazorError error in results.ParserErrors) { errors.Add(error); } return(results.Document.Flatten().ToList()); }
private static void EvaluateNaiveBayesPredictor(ParserResults testData, NaiveBayesCalculator naiveBayesPredictor) { Console.WriteLine("Making predictions..."); uint hits = 0, misses = 0; uint falsePositives = 0, falseNegatives = 0; foreach (var testExample in testData.Values) { var isOnePrediction = naiveBayesPredictor.CalculatePrediction(testExample); if (isOnePrediction && testExample.Output) { hits++; } else if (!isOnePrediction && !testExample.Output) { hits++; } else if (isOnePrediction && !testExample.Output) { misses++; falsePositives++; } else if (!isOnePrediction && testExample.Output) { misses++; falseNegatives++; } else { throw new InvalidOperationException(); } } Console.WriteLine("Score: {0}%. Hits: {1}, Misses: {2}", 100.0 * hits / (misses + hits), hits, misses); Console.WriteLine("FalsePositives: {0}. FalseNegatives: {1}", falsePositives, falseNegatives); }
public void VisitSendsErrorsToVisitor() { // Arrange Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>(); var root = new BlockBuilder() { Type = BlockType.Comment }.Build(); var errorSink = new ErrorSink(); List<RazorError> errors = new List<RazorError> { new RazorError("Foo", new SourceLocation(1, 0, 1), length: 3), new RazorError("Bar", new SourceLocation(2, 0, 2), length: 3), }; foreach (var error in errors) { errorSink.OnError(error); } var results = new ParserResults(root, Enumerable.Empty<TagHelperDescriptor>(), errorSink); // Act targetMock.Object.Visit(results); // Assert targetMock.Verify(v => v.VisitError(errors[0])); targetMock.Verify(v => v.VisitError(errors[1])); }
static void Main(string[] args) { string errorMessage = ""; if (!File.Exists(DataSetPath)) { errorMessage += $"Failed to find file ${DataSetPath} - please update variable ${nameof(DataSetPath)} or create that file.\n"; } if (!File.Exists(TestSetPath)) { errorMessage += $"Failed to find file ${TestSetPath} - please update variable ${nameof(TestSetPath)} or create that file.\n"; } if (errorMessage != "") { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Not all files available - not running!"); Console.WriteLine(errorMessage); Console.ResetColor(); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); return; } Random rnd = new Random(); Console.WriteLine("Reading training data..."); ParserResults trainingData = ParserUtils.ParseData(DataSetPath, convertContinuousValues: true); Console.WriteLine("Validating training set"); DataSetCleaner.ValidateDataSet(trainingData.Attributes, trainingData.Values); Console.WriteLine("Getting test set..."); ParserResults testData = ParserUtils.ParseData(TestSetPath, trainingData.Attributes, convertContinuousValues: true); Console.WriteLine("Validating test set"); DataSetCleaner.ValidateDataSet(testData.Attributes, testData.Values); const string svmTrainingPath = "svmtraining"; const string svmOutputPath = "output"; List <int> kernelsToRunIn = new List <int>() { 0, 1, 2, 3 }; string originalTrainingFile = "originalTraining.txt"; LibSvmConverter.ConvertToLibSvm(trainingData.Values, "originalTraining.txt"); // Run all kernels in parallel kernelsToRunIn.Select((kernel) => { // Run all iterations in parallel Enumerable.Range(0, TotalSamplesForBiasAndVariance).Select((i) => { Console.WriteLine("Doing loop {0} for kernel {1}", i, kernel); var postFix = string.Format("K{0}-I{1}.txt", kernel, i); string trainingPath = svmTrainingPath + postFix; string trainingModelPath = trainingPath + ".model"; string outputPath = svmOutputPath + postFix; if (!File.Exists(trainingModelPath)) { List <List <DataSetValue> > differentTrainingData = Bagging.ProduceDifferentDataSets(trainingData.Values, 1, rnd); LibSvmConverter.ConvertToLibSvm(differentTrainingData.Single(), trainingPath); RunTrainingExe(trainingPath, kernel); } if (!File.Exists(outputPath)) { RunEvaluateExe(originalTrainingFile, trainingModelPath, outputPath); } return(0); }).ToList(); return(0); }).ToList(); // Evaluate bias and variance foreach (var kernel in kernelsToRunIn) { List <List <bool> > allPredictions = new List <List <bool> >(); for (int i = 0; i < TotalSamplesForBiasAndVariance; i++) { Console.WriteLine("Evaluating loop {0} for kernel {1}", i, kernel); var postFix = string.Format("K{0}-I{1}.txt", kernel, i); string trainingPath = svmTrainingPath + postFix; string outputPath = svmOutputPath + postFix; if (!File.Exists(trainingPath) || !File.Exists(outputPath)) { continue; } List <bool> predictions = GetPredictionsFromOutputPath(outputPath); allPredictions.Add(predictions); } if (allPredictions.Count == 0) { Console.WriteLine("Not enough information to evaluate kernel {0}", kernel); continue; } double bias; double variance = BiasAndVarianceCalculator.CalculateBiasAndVariance(trainingData, allPredictions, out bias); Console.WriteLine("Bias:{0:0.00000} Variance:{1:0.00000}", bias, variance); } Console.WriteLine("Press any key to quit..."); Console.ReadKey(); }
static void Main(string[] args) { string errorMessage = ""; if (!File.Exists(DataSetPath)) { errorMessage += $"Failed to find file ${DataSetPath} - please update variable ${nameof(DataSetPath)} or create that file.\n"; } if (!File.Exists(TestSetPath)) { errorMessage += $"Failed to find file ${TestSetPath} - please update variable ${nameof(TestSetPath)} or create that file.\n"; } if (errorMessage != "") { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Not all files available - not running!"); Console.WriteLine(errorMessage); Console.ResetColor(); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); return; } var startTime = DateTime.Now; Console.WriteLine(startTime); Random rnd = new Random(); Console.WriteLine("Reading training data..."); ParserResults trainingData = ParserUtils.ParseData(DataSetPath, convertContinuousValues: true); Console.WriteLine("Validating training set"); DataSetCleaner.ValidateDataSet(trainingData.Attributes, trainingData.Values); Console.WriteLine("Getting test set..."); ParserResults testData = ParserUtils.ParseData(TestSetPath, trainingData.Attributes, convertContinuousValues: true); Console.WriteLine("Validating test set"); DataSetCleaner.ValidateDataSet(testData.Attributes, testData.Values); List <List <DataSetValue> > differentDataSets = Bagging.ProduceDifferentDataSets(trainingData.Values, TotalSamplesForBiasAndVariance, rnd); List <IPredictor> naiveBayesPredictors = new List <IPredictor>(); foreach (var differentTrainingDataSet in differentDataSets) { var naiveBayesPredictor = TrainNaiveBayes(differentTrainingDataSet); naiveBayesPredictors.Add(naiveBayesPredictor); } double bias; double variance = BiasAndVarianceCalculator.CalculateBiasAndVariance(trainingData, naiveBayesPredictors, out bias); Console.WriteLine($"Bias:{bias} Variance:{variance}"); var originalNaiveBayesPredictor = TrainNaiveBayes(trainingData.Values); EvaluateNaiveBayesPredictor(testData, originalNaiveBayesPredictor); var endTime = DateTime.Now; Console.WriteLine(endTime); var totalMinutes = (endTime - startTime).TotalMinutes; Console.WriteLine("Took {0} minutes.", totalMinutes); Console.WriteLine("Press any key to quit..."); Console.ReadKey(); }
public void ConditionalAttributesDoNotCreateExtraDataForEntirelyLiteralAttribute() { // Arrange const string code = #region Big Block o' code @"<div class=""sidebar""> <h1>Title</h1> <p> As the author, you can <a href=""/Photo/Edit/photoId"">edit</a> or <a href=""/Photo/Remove/photoId"">remove</a> this photo. </p> <dl> <dt class=""description"">Description</dt> <dd class=""description""> The uploader did not provide a description for this photo. </dd> <dt class=""uploaded-by"">Uploaded by</dt> <dd class=""uploaded-by""><a href=""/User/View/user.UserId"">user.DisplayName</a></dd> <dt class=""upload-date"">Upload date</dt> <dd class=""upload-date"">photo.UploadDate</dd> <dt class=""part-of-gallery"">Gallery</dt> <dd><a href=""/View/gallery.Id"" title=""View gallery.Name gallery"">gallery.Name</a></dd> <dt class=""tags"">Tags</dt> <dd class=""tags""> <ul class=""tags""> <li>This photo has no tags.</li> </ul> <a href=""/Photo/EditTags/photoId"">edit tags</a> </dd> </dl> <p> <a class=""download"" href=""/Photo/Full/photoId"" title=""Download: (photo.FileTitle + photo.FileExtension)"">Download full photo</a> ((photo.FileSize / 1024) KB) </p> </div> <div class=""main""> <img class=""large-photo"" alt=""photo.FileTitle"" src=""/Photo/Thumbnail"" /> <h2>Nobody has commented on this photo</h2> <ol class=""comments""> <li> <h3 class=""comment-header""> <a href=""/User/View/comment.UserId"" title=""View comment.DisplayName's profile"">comment.DisplayName</a> commented at comment.CommentDate: </h3> <p class=""comment-body"">comment.CommentText</p> </li> </ol> <form method=""post"" action=""""> <fieldset id=""addComment""> <legend>Post new comment</legend> <ol> <li> <label for=""newComment"">Comment</label> <textarea id=""newComment"" name=""newComment"" title=""Your comment"" rows=""6"" cols=""70""></textarea> </li> </ol> <p class=""form-actions""> <input type=""submit"" title=""Add comment"" value=""Add comment"" /> </p> </fieldset> </form> </div>"; #endregion // Act ParserResults results = ParseDocument(code); Block rewritten = new ConditionalAttributeCollapser(new HtmlMarkupParser().BuildSpan).Rewrite(results.Document); rewritten = new MarkupCollapser(new HtmlMarkupParser().BuildSpan).Rewrite(rewritten); // Assert Assert.Equal(0, results.ParserErrors.Count); EvaluateParseTree(rewritten, new MarkupBlock(Factory.Markup(code))); }
private static void RunWithTreeLevels(ParserResults trainingData, Random rnd, int treeDepth, int sizeOfBaggers, ParserResults testData) { List <List <List <DataSetValue> > > dataSetValuesForBagging = new List <List <List <DataSetValue> > >(); for (int i = 0; i < TotalSamplesForBiasAndVariance; i++) { // Two layer sampling List <DataSetValue> layer1Sampling = Bagging.ProduceDifferentDataSets(trainingData.Values, 1, rnd).Single(); if (sizeOfBaggers == 1) { dataSetValuesForBagging.Add(new List <List <DataSetValue> >() { layer1Sampling }); } else { dataSetValuesForBagging.Add(Bagging.ProduceDifferentDataSets(layer1Sampling, sizeOfBaggers, rnd)); } } // Initialize the required trees List <List <DecisionTreeLevel> > listOfTreesToRunTestOn = new List <List <DecisionTreeLevel> >(); foreach (var dataSetForBagging in dataSetValuesForBagging) { // Foe each bagger, for each dataset, create a new tree listOfTreesToRunTestOn.Add( dataSetForBagging.Select( dataSet => new DecisionTreeLevel(0, trainingData.Attributes, dataSet, maximumDepth: treeDepth)).ToList()); } Parallel.ForEach(listOfTreesToRunTestOn.SelectMany(s => s), l => l.D3()); Parallel.ForEach(listOfTreesToRunTestOn.SelectMany(s => s), l => l.TrimTree()); //string sampleSerializedTree = listOfTreesToRunTestOn[0][0].SerializeDecisionTree(); //Console.WriteLine("Evaluating trees against test data..."); double totalScoreAgainstTrainingData = 0; double totalScoreAgainstTestData = 0; foreach (List <DecisionTreeLevel> baggingSetOfTrees in listOfTreesToRunTestOn) { DecisionTreeScore scoreAgainstTrainingData = DecisionTreeScorer.ScoreWithTreeWithTestSet(baggingSetOfTrees, trainingData.Values); DecisionTreeScore scoreAgainstTestData = DecisionTreeScorer.ScoreWithTreeWithTestSet(baggingSetOfTrees, testData.Values); //score.PrintTotalScore(); totalScoreAgainstTrainingData += scoreAgainstTrainingData.GetTotalScore(); totalScoreAgainstTestData += scoreAgainstTestData.GetTotalScore(); } totalScoreAgainstTrainingData = totalScoreAgainstTrainingData / listOfTreesToRunTestOn.Count; totalScoreAgainstTestData = totalScoreAgainstTestData / listOfTreesToRunTestOn.Count; double bias; double variance = BiasAndVarianceCalculator.CalculateBiasAndVariance(trainingData, listOfTreesToRunTestOn, out bias); Console.WriteLine("Variance: {0:0.00000}. Bias: {1:0.00000}. ScoreTraining : {2:0.00000}, ScoreTest : {3:0.00000}", variance, bias, totalScoreAgainstTrainingData, totalScoreAgainstTestData); //Console.WriteLine(bias); //Console.WriteLine(variance); //Console.WriteLine(totalScoreAgainstTrainingData); //Console.WriteLine(totalScoreAgainstTestData); }
public static void EvaluateResults(ParserResults results, Block expectedRoot, IList <RazorError> expectedErrors) { EvaluateResults(null, results, expectedRoot, expectedErrors); }
public static void EvaluateResults(ParserResults results, Block expectedRoot, IList <RazorError> expectedErrors) { EvaluateParseTree(results.Document, expectedRoot); EvaluateRazorErrors(results.ParserErrors, expectedErrors); }
public static void EvaluateResults(TestContext context, ParserResults results, Block expectedRoot) { EvaluateResults(context, results, expectedRoot, null); }
static void Main(string[] args) { string errorMessage = ""; if (!File.Exists(DataSetPath)) { errorMessage += $"Failed to find file ${DataSetPath} - please update variable ${nameof(DataSetPath)} or create that file.\n"; } if (!File.Exists(TestSetPath)) { errorMessage += $"Failed to find file ${TestSetPath} - please update variable ${nameof(TestSetPath)} or create that file.\n"; } if (errorMessage != "") { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Not all files available - not running!"); Console.WriteLine(errorMessage); Console.ResetColor(); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); return; } Random rnd = new Random(); Console.WriteLine("Reading training data..."); ParserResults trainingData = ParserUtils.ParseData(DataSetPath); Console.WriteLine("Validating data set"); DataSetCleaner.ValidateDataSet(trainingData.Attributes, trainingData.Values); List <List <List <DataSetValue> > > dataSetValuesForBagging = new List <List <List <DataSetValue> > >() { Bagging.ProduceDifferentDataSets(trainingData.Values, 1, rnd), Bagging.ProduceDifferentDataSets(trainingData.Values, 3, rnd), Bagging.ProduceDifferentDataSets(trainingData.Values, 5, rnd), Bagging.ProduceDifferentDataSets(trainingData.Values, 10, rnd), Bagging.ProduceDifferentDataSets(trainingData.Values, 20, rnd), }; // Initialize the required trees List <List <DecisionTreeLevel> > listOfTreesToRunTestOn = new List <List <DecisionTreeLevel> >(); foreach (var dataSetForBagging in dataSetValuesForBagging) { listOfTreesToRunTestOn.Add(dataSetForBagging.Select(x => new DecisionTreeLevel(ChiTestLimit, trainingData.Attributes, x)).ToList()); } Console.WriteLine("Runnind D3 on all trees in parallel..."); Parallel.ForEach(listOfTreesToRunTestOn.SelectMany(s => s), l => l.D3()); Console.WriteLine("Deleting unecessary nodes..."); Parallel.ForEach(listOfTreesToRunTestOn.SelectMany(s => s), l => l.TrimTree()); Console.WriteLine("Getting test data set..."); ParserResults testData = ParserUtils.ParseData(TestSetPath); Console.WriteLine("Evaluating trees against test data..."); foreach (List <DecisionTreeLevel> baggingSetOfTrees in listOfTreesToRunTestOn) { DecisionTreeScore score = DecisionTreeScorer.ScoreWithTreeWithTestSet(baggingSetOfTrees, testData.Values); score.PrintTotalScore(); } Console.WriteLine("Press any key to quit..."); Console.ReadKey(); }
public static void EvaluateResults(ParserResults results, Block expectedRoot, IList<RazorError> expectedErrors) { EvaluateResults(null, results, expectedRoot, expectedErrors); }
public static void EvaluateResults(ParserResults results, Block expectedRoot) { EvaluateResults(results, expectedRoot, null); }
static void Main(string[] args) { string errorMessage = ""; if (!File.Exists(DataSetPath)) { errorMessage += $"Failed to find file ${DataSetPath} - please update variable ${nameof(DataSetPath)} or create that file.\n"; } if (!File.Exists(TestSetPath)) { errorMessage += $"Failed to find file ${TestSetPath} - please update variable ${nameof(TestSetPath)} or create that file.\n"; } if (errorMessage != "") { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Not all files available - not running!"); Console.WriteLine(errorMessage); Console.ResetColor(); Console.WriteLine("Press any key to continue..."); Console.ReadKey(); return; } Console.WriteLine("Reading training data..."); ParserResults trainingData = ParserUtils.ParseData(DataSetPath); // Optimizations are optional // DataSetOptimizerForExtraCredit.OptimizeDataSetForExtraCredit(trainingData.Attributes, trainingData.Values); Console.WriteLine("Validating data set"); DataSetCleaner.ValidateDataSet(trainingData.Attributes, trainingData.Values); // Initialize the required trees with their respective chiTestLimits List <DecisionTreeLevel> listOfTreesToRunTestOn = new List <DecisionTreeLevel>() { new DecisionTreeLevel(chiTestLimit: 0.99), new DecisionTreeLevel(chiTestLimit: 0.95), new DecisionTreeLevel(chiTestLimit: 0), }; Console.WriteLine("Runnind D3..."); Parallel.ForEach(listOfTreesToRunTestOn, l => l.D3(trainingData.Attributes, trainingData.Values)); Console.WriteLine("Deleting unecessary nodes..."); Parallel.ForEach(listOfTreesToRunTestOn, l => l.TrimTree()); Console.WriteLine("Getting test data set..."); ParserResults testData = ParserUtils.ParseData(TestSetPath); // Optimizations are optional // DataSetOptimizerForExtraCredit.OptimizeDataSetForExtraCredit(testData.Attributes, testData.Values); Console.WriteLine("Evaluating trees against test data..."); List <DecisionTreeScore> scores = listOfTreesToRunTestOn.AsParallel().Select(t => DecisionTreeScorer.ScoreWithTreeWithTestSet(t, testData.Values)).ToList(); //Console.WriteLine("Writing trees to text files (for debugging/visualization)..."); // Dump the trees to a txt file for debugging/visualization // NOTE: This won't work the the Chi=0 case - the JSON file generated is too big // Parallel.ForEach(listOfTreesToRunTestOn, l => File.WriteAllText("Chi" + Convert.ToInt64(l.ChiTestLimit * 10000000000000) + ".json", l.SerializeDecisionTree())); List <DecisionTreeScore> trainingDataScores = listOfTreesToRunTestOn.AsParallel().Select(t => DecisionTreeScorer.ScoreWithTreeWithTestSet(t, trainingData.Values)).ToList(); // Print the results to console foreach (var score in scores) { score.PrintTotalScore(); } Console.WriteLine("Evaluating trees against training data:"); foreach (var score in trainingDataScores) { score.PrintTotalScore(); } Console.WriteLine("Press any key to quit..."); Console.ReadKey(); }
public RazorCSharpModelTypeLocater(ITextBuffer textBuffer) { ParserResults results = ParseTemplate(textBuffer); ModelTypeName = GetModelTypeName(results); }
public object Process(Node node, bool extractDeleted, bool extractSource) { ParserResults pr = new ParserResults(); if (node is FileSystem fs) //这个插件期望node是文件系统 { MetaData meta = new MetaData(); var ds = fs.DataStore; try { //analyze_csidata //private/var/root/Library/Lockdown/data_ark.plist var data_ark = fs.Search("/Library/Lockdown/data_ark.plist$").FirstOrDefault(); if (data_ark != null && data_ark.Type == NodeType.File) { var plist = PlistHelper.ReadPlist <NSDictionary>(data_ark); FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.Language.ToString(), "com.apple.international-Language"); FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.BuildVersion.ToString(), "com.apple.who-uptodate_build"); FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.LastBackupComputerName.ToString(), "com.apple.iTunes.backup-LastBackupComputerName"); FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.DeviceName.ToString(), "-DeviceName"); FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.TimeZone.ToString(), "-TimeZone"); FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.ActivationStateAcknowledged.ToString(), "-ActivationStateAcknowledged"); FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.Locale.ToString(), "com.apple.international-Locale"); FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.LastBackupComputerType.ToString(), "com.apple.iTunes.backup-LastBackupComputerType"); } //System/Library/LaunchDaemons/com.apple.atc.plist var atcNode = fs.Search("/Preferences/com.apple.atc.plist$").FirstOrDefault(); if (atcNode != null && atcNode.Type == NodeType.File) { var plist = PlistHelper.ReadPlist <NSDictionary>(atcNode); if (plist != null) { var diskUsage = plist.SafeGetObject <NSDictionary>("DiskUsage"); FileSize GetDiskUsageSize(NSDictionary dict, string key) { var subdict = dict.SafeGetObject <NSDictionary>(key); if (subdict != null) { return(new FileSize(subdict.SafeGetInt64("_PhysicalSize"))); } return(new FileSize(0)); } ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.VoiceMemoSize.ToString(), GetDiskUsageSize(diskUsage, "VoiceMemo"))); ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.UserDataSize.ToString(), GetDiskUsageSize(diskUsage, "UserData"))); ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.BookDataSize.ToString(), GetDiskUsageSize(diskUsage, "Book"))); ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.MediaDataSize.ToString(), GetDiskUsageSize(diskUsage, "Media"))); ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.ApplicationSize.ToString(), GetDiskUsageSize(diskUsage, "Application"))); ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.LogsSize.ToString(), GetDiskUsageSize(diskUsage, "Logs"))); ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.RingtoneSize.ToString(), GetDiskUsageSize(diskUsage, "Ringtone"))); ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.TotalDiskSize.ToString(), new FileSize(diskUsage.SafeGetInt64("_PhysicalSize")))); ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.FreeDiskSize.ToString(), new FileSize(diskUsage.SafeGetInt64("_FreeSize")))); } } var projFile = ds.ProjectState.ProjectFile; if (!string.IsNullOrEmpty(projFile) && File.Exists(projFile)) { var plist = PlistHelper.ReadPlist <NSDictionary>(projFile); if (plist != null) { var deviceInfo = plist.SafeGetObject <NSDictionary>("Device"); if (deviceInfo != null) { foreach (var p in deviceInfo) { var key = p.Key; var obj = p.Value; if (obj is NSString nss) { ds.DeviceInfo.Add(new MetaDataField(key, nss.Content)); } else if (obj is NSNumber nsn) { if (nsn.isBoolean()) { ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToBool())); } else if (nsn.isInteger()) { ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToLong())); } else if (nsn.isReal()) { ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToDouble())); } } } } } } var lockd = fs.GetByPath("/private/Lockdown.plist"); if (lockd != null && lockd.Type == NodeType.File) { var plist = PlistHelper.ReadPlist <NSDictionary>(lockd); if (plist != null) { var keys = Enum.GetNames(typeof(DeviceInfoKeys)); foreach (var key in keys) { if (plist.ContainsKey(key)) { var obj = plist.SafeGetObject <NSObject>(key); if (obj is NSString nss) { ds.DeviceInfo.Add(new MetaDataField(key, nss.Content)); } else if (obj is NSNumber nsn) { if (nsn.isBoolean()) { ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToBool())); } else if (nsn.isInteger()) { ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToLong())); } else if (nsn.isReal()) { ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToDouble())); } } } } } } } catch (Exception e) { TraceService.TraceError(e); } } //返回空的pr,实际的数据已经存在ds.DeviceInfo里面了,这是特殊情况,一般情况下你应该将数据通过pr存储返回给主程序 return(pr); }
public override void OnComplete() { base.OnComplete(); Results = new ParserResults(_rootBlock, _errorList); }
public static void BuilderTemplate(ITemplateInfo templateinfo, ITemplateHost host) { Type type = host.BaseType; string typename = "_" + Guid.NewGuid().ToString("N"); RazorTemplateEngine engine = CreateHost(type, "_" + type.Name, typename); AddNamespace(type.Namespace); GeneratorResults razorResults = null; using (System.IO.StreamReader reader = templateinfo.GetCode()) { razorResults = engine.GenerateCode(reader); CSharpCodeProvider codeProvider = new CSharpCodeProvider(); CodeGeneratorOptions options = new CodeGeneratorOptions(); string LastGeneratedCode = null; using (StringWriter writer = new StringWriter()) { codeProvider.GenerateCodeFromCompileUnit(razorResults.GeneratedCode, writer, options); templateinfo.Host.LastCompileSource = writer.ToString(); } CompilerParameters compilerParameters = new CompilerParameters(new string[0]); compilerParameters.OutputAssembly = "tmp_assembly" + typename; compilerParameters.GenerateInMemory = false; AddDomainAssemblies(); foreach (string item in ReferencedAssemblies) { compilerParameters.ReferencedAssemblies.Add(item); } compilerParameters.GenerateInMemory = true; CompilerResults compilerResults = codeProvider.CompileAssemblyFromDom(compilerParameters, razorResults.GeneratedCode); if (compilerResults.Errors.Count > 0) { string errormessage = null; int throwindex = 1; System.Text.StringBuilder sb = new System.Text.StringBuilder(); reader.BaseStream.Seek(0, SeekOrigin.Begin); using (System.IO.StreamReader line = new StreamReader(reader.BaseStream, reader.CurrentEncoding)) { ParserResults presult = engine.ParseTemplate(line); if (presult.ParserErrors.Count > 0) { throwindex = presult.ParserErrors[0].Location.LineIndex + 1; errormessage = presult.ParserErrors[0].Message; reader.BaseStream.Seek(0, SeekOrigin.Begin); using (System.IO.StreamReader readcode = new StreamReader(reader.BaseStream, reader.CurrentEncoding)) { string code = readcode.ReadLine(); while (code != null) { sb.AppendLine(code); code = readcode.ReadLine(); } } } else { throwindex = compilerResults.Errors[0].Line; errormessage = compilerResults.Errors[0].ErrorText; sb.Append(LastGeneratedCode); } } templateinfo.CompileError = errormessage; throw new RazorException(templateinfo.CompileError); } templateinfo.Assembly = compilerResults.CompiledAssembly; templateinfo.TemplateType = compilerResults.CompiledAssembly.GetType("_" + type.Name + "." + typename); } }
public static void EvaluateResults(ParserResults results, Block expectedRoot, IList<RazorError> expectedErrors) { EvaluateParseTree(results.Document, expectedRoot); EvaluateRazorErrors(results.ParserErrors, expectedErrors); }
void ParseTemplate(string html) { var parser = new RazorParser(codeLanguage.CreateCodeParser(), new HtmlMarkupParser()); parserResults = parser.Parse(new StringReader(html)); }