public void VisitThrowsOnNullVisitor()
{
ParserVisitor target = null;
ParserResults results = new ParserResults(new BlockBuilder() { Type = BlockType.Comment }.Build(), new List<RazorError>());
Assert.Throws<ArgumentNullException>("self", () => target.Visit(results));
}
public void VisitThrowsOnNullVisitor()
{
ParserVisitor target = null;
var errorSink = new ParserErrorSink();
var results = new ParserResults(new BlockBuilder() { Type = BlockType.Comment }.Build(),
Enumerable.Empty<TagHelperDescriptor>(),
errorSink);
Assert.Throws<ArgumentNullException>("self", () => target.Visit(results));
}
public void VisitSendsDocumentToVisitor()
{
// Arrange
Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>();
Block root = new BlockBuilder() { Type = BlockType.Comment }.Build();
ParserResults results = new ParserResults(root, new List<RazorError>());
// Act
targetMock.Object.Visit(results);
// Assert
targetMock.Verify(v => v.VisitBlock(root));
}
public void VisitSendsDocumentToVisitor()
{
// Arrange
Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>();
var root = new BlockBuilder() { Type = BlockType.Comment }.Build();
var errorSink = new ErrorSink();
var results = new ParserResults(root,
Enumerable.Empty<TagHelperDescriptor>(),
errorSink);
// Act
targetMock.Object.Visit(results);
// Assert
targetMock.Verify(v => v.VisitBlock(root));
}
public void VisitCallsOnCompleteWhenAllNodesHaveBeenVisited()
{
// Arrange
Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>();
var root = new BlockBuilder() { Type = BlockType.Comment }.Build();
var errorSink = new ErrorSink();
errorSink.OnError(new RazorError("Foo", new SourceLocation(1, 0, 1), length: 3));
errorSink.OnError(new RazorError("Bar", new SourceLocation(2, 0, 2), length: 3));
var results = new ParserResults(root, Enumerable.Empty<TagHelperDescriptor>(), errorSink);
// Act
targetMock.Object.Visit(results);
// Assert
targetMock.Verify(v => v.OnComplete());
}
public void VisitCallsOnCompleteWhenAllNodesHaveBeenVisited()
{
// Arrange
Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>();
Block root = new BlockBuilder() { Type = BlockType.Comment }.Build();
List<RazorError> errors = new List<RazorError>() {
new RazorError("Foo", 1, 0, 1),
new RazorError("Bar", 2, 0, 2)
};
ParserResults results = new ParserResults(root, errors);
// Act
targetMock.Object.Visit(results);
// Assert
targetMock.Verify(v => v.OnComplete());
}
public static void Visit(this ParserVisitor self, ParserResults result)
{
if (self == null)
{
throw new ArgumentNullException("self");
}
if (result == null)
{
throw new ArgumentNullException("result");
}
result.Document.Accept(self);
foreach (RazorError error in result.ParserErrors)
{
self.VisitError(error);
}
self.OnComplete();
}
public void VisitSendsErrorsToVisitor()
{
// Arrange
Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>();
Block root = new BlockBuilder() { Type = BlockType.Comment }.Build();
List<RazorError> errors = new List<RazorError>() {
new RazorError("Foo", 1, 0, 1),
new RazorError("Bar", 2, 0, 2)
};
ParserResults results = new ParserResults(root, errors);
// Act
targetMock.Object.Visit(results);
// Assert
targetMock.Verify(v => v.VisitError(errors[0]));
targetMock.Verify(v => v.VisitError(errors[1]));
}
public void VisitCallsOnCompleteWhenAllNodesHaveBeenVisited()
{
// Arrange
Mock <ParserVisitor> targetMock = new Mock <ParserVisitor>();
Block root = new BlockBuilder()
{
Type = BlockType.Comment
}.Build();
List <RazorError> errors = new List <RazorError>()
{
new RazorError("Foo", 1, 0, 1),
new RazorError("Bar", 2, 0, 2)
};
ParserResults results = new ParserResults(root, errors);
// Act
targetMock.Object.Visit(results);
// Assert
targetMock.Verify(v => v.OnComplete());
}
public void ParseMethodUsesProvidedParserListenerIfSpecified()
{
var factory = SpanFactory.CreateCsHtml();
// Arrange
RazorParser parser = new RazorParser(new CSharpCodeParser(), new HtmlMarkupParser());
// Act
ParserResults results = parser.Parse(new StringReader("foo @bar baz"));
// Assert
ParserTestBase.EvaluateResults(results,
new MarkupBlock(
factory.Markup("foo "),
new ExpressionBlock(
factory.CodeTransition(),
factory.Code("bar")
.AsImplicitExpression(CSharpCodeParser.DefaultKeywords)
.Accepts(AcceptedCharacters.NonWhiteSpace)),
factory.Markup(" baz")));
}
protected virtual ParserResults RunParse(string document,
Func <ParserBase, Action> parserActionSelector,
bool designTimeParser,
Func <ParserContext, ParserBase> parserSelector = null,
ErrorSink errorSink = null)
{
parserSelector = parserSelector ?? (c => c.ActiveParser);
errorSink = errorSink ?? new ErrorSink();
// Create the source
ParserResults results = null;
using (var reader = new SeekableTextReader(document))
{
try
{
var codeParser = CreateCodeParser();
var markupParser = CreateMarkupParser();
var context = CreateParserContext(reader, codeParser, markupParser, errorSink);
context.DesignTimeMode = designTimeParser;
codeParser.Context = context;
markupParser.Context = context;
// Run the parser
parserActionSelector(parserSelector(context))();
results = context.CompleteParse();
}
finally
{
if (results != null && results.Document != null)
{
WriteTraceLine(string.Empty);
WriteTraceLine("Actual Parse Tree:");
WriteNode(0, results.Document);
}
}
}
return(results);
}
public static double CalculateBiasAndVariance(ParserResults trainingData, List <List <bool> > predictorsPredictions, out double bias)
{
double variance = 0;
bias = 0;
// Calculate biar and variance for all trees
for (int trainingDataIndex = 0; trainingDataIndex < trainingData.Values.Count; trainingDataIndex++)
{
var trainingDataValue = trainingData.Values[trainingDataIndex];
double realValue = Transformer.BoolToDouble(trainingDataValue.Output);
List <double> allPredictions = new List <double>(predictorsPredictions.Count);
foreach (var predictorPredictions in predictorsPredictions)
{
bool prediction = predictorPredictions[trainingDataIndex];
allPredictions.Add(Transformer.BoolToDouble(prediction));
}
double mode = ModeFinder.FindMode(allPredictions);
double averagePrediction = allPredictions.Average();
// Now that we have the mode, realValue, and average of all predictions, we can calculate variance and bias
double varianceForThisDataPoint = 0;
double diffOfRealValueAndAveragePrediction = (realValue - averagePrediction);
double biasForThisDataPoint = diffOfRealValueAndAveragePrediction * diffOfRealValueAndAveragePrediction;
foreach (var prediction in allPredictions)
{
var diffForModeAndPrediction = prediction - mode;
varianceForThisDataPoint += diffForModeAndPrediction * diffForModeAndPrediction;
}
varianceForThisDataPoint = varianceForThisDataPoint / allPredictions.Count;
// Accumulate
variance += varianceForThisDataPoint;
bias += biasForThisDataPoint;
}
variance = variance / trainingData.Values.Count;
bias = bias / trainingData.Values.Count;
return(variance);
}
/// <summary>
/// Instantiates a new <see cref="GeneratorResults"/> instance.
/// </summary>
/// <param name="parserResults">The results of parsing a document.</param>
/// <param name="codeGeneratorResult">The results of generating code for the document.</param>
/// <param name="chunkTree">A <see cref="ChunkTree"/> for the document.</param>
public GeneratorResults(ParserResults parserResults,
CodeGeneratorResult codeGeneratorResult,
ChunkTree chunkTree)
: this(parserResults.Document,
parserResults.TagHelperDescriptors,
parserResults.ErrorSink,
codeGeneratorResult,
chunkTree)
{
if (parserResults == null)
{
throw new ArgumentNullException(nameof(parserResults));
}
if (codeGeneratorResult == null)
{
throw new ArgumentNullException(nameof(codeGeneratorResult));
}
if (chunkTree == null)
{
throw new ArgumentNullException(nameof(chunkTree));
}
}
/// <summary>
/// Instantiates a new <see cref="GeneratorResults"/> instance.
/// </summary>
/// <param name="parserResults">The results of parsing a document.</param>
/// <param name="codeGeneratorResult">The results of generating code for the document.</param>
/// <param name="chunkTree">A <see cref="ChunkTree"/> for the document.</param>
public GeneratorResults(ParserResults parserResults,
CodeGeneratorResult codeGeneratorResult,
ChunkTree chunkTree)
: this(parserResults.Document,
parserResults.TagHelperDescriptors,
parserResults.ErrorSink,
codeGeneratorResult,
chunkTree)
{
if (parserResults == null)
{
throw new ArgumentNullException(nameof(parserResults));
}
if (codeGeneratorResult == null)
{
throw new ArgumentNullException(nameof(codeGeneratorResult));
}
if (chunkTree == null)
{
throw new ArgumentNullException(nameof(chunkTree));
}
}
public virtual CompilerResult Compile(string razorTemplate)
{
if (razorTemplate == null)
{
throw new ArgumentNullException("razorTemplate");
}
if (String.IsNullOrWhiteSpace(razorTemplate))
{
throw new ArgumentException("razorTemplate cannot be empty or white-space!", "razorTemplate");
}
ParserResults parserResults = this._templateParser.ParseTemplate(razorTemplate);
if (!parserResults.Success)
{
return(new CompilerResult(parserResults.ParserErrors));
}
this._documentTranslator.Translate(parserResults.Document, this._templateBuilder);
return(this._templateBuilder.Build());
}
public void VisitSendsErrorsToVisitor()
{
// Arrange
Mock <ParserVisitor> targetMock = new Mock <ParserVisitor>();
Block root = new BlockBuilder()
{
Type = BlockType.Comment
}.Build();
List <RazorError> errors = new List <RazorError>()
{
new RazorError("Foo", 1, 0, 1),
new RazorError("Bar", 2, 0, 2)
};
ParserResults results = new ParserResults(root, errors);
// Act
targetMock.Object.Visit(results);
// Assert
targetMock.Verify(v => v.VisitError(errors[0]));
targetMock.Verify(v => v.VisitError(errors[1]));
}
public static double CalculateBiasAndVariance(ParserResults trainingData, List <List <DecisionTreeLevel> > listOfTreesToRunTestOn, out double bias)
{
double variance = 0;
bias = 0;
// Calculate biar and variance for all trees
foreach (var trainingDataValue in trainingData.Values)
{
double realValue = Transformer.BoolToDouble(trainingDataValue.Output);
List <double> allPredictions = new List <double>(listOfTreesToRunTestOn.Count);
foreach (var bagger in listOfTreesToRunTestOn)
{
bool prediction = DecisionTreeScorer.CalculatePrediction(bagger, trainingDataValue);
allPredictions.Add(Transformer.BoolToDouble(prediction));
}
double mode = ModeFinder.FindMode(allPredictions);
double averagePrediction = allPredictions.Average();
// Now that we have the mode, realValue, and average of all predictions, we can calculate variance and bias
double varianceForThisDataPoint = 0;
double diffOfRealValueAndAveragePrediction = (realValue - averagePrediction);
double biasForThisDataPoint = diffOfRealValueAndAveragePrediction * diffOfRealValueAndAveragePrediction;
foreach (var prediction in allPredictions)
{
var diffForModeAndPrediction = prediction - mode;
varianceForThisDataPoint += diffForModeAndPrediction * diffForModeAndPrediction;
}
varianceForThisDataPoint = varianceForThisDataPoint / allPredictions.Count;
// Accumulate
variance += varianceForThisDataPoint;
bias += biasForThisDataPoint;
}
variance = variance / trainingData.Values.Count;
bias = bias / trainingData.Values.Count;
return(variance);
}
public List <Veranstaltung> ParseEventTabellen(ParserResults parserResults)
{
if (parserResults.EventText.Contains("Es gibt keine bevorstehenden Veranstaltungen."))
{
return(new List <Veranstaltung>(0));
}
var eventTexte = SepariereVeranstalltungen(parserResults.EventText);
var events = new List <Veranstaltung>();
for (var i = 0; i < eventTexte.Count; i++)
{
try {
var eventTitle = ExtrahiereEventTitle(parserResults.LinkTexte[i]);
events.Add(ParseEineVeranstalltung(eventTexte[i], eventTitle));
}
catch (Exception e) {
_logger.LogError(e.ToString());
}
}
return(events);
}
public void ConditionalAttributeCollapserDoesNotRemoveUrlAttributeValues()
{
// Act
ParserResults results = ParseDocument("<a href='~/Foo/Bar' />");
Block rewritten = new ConditionalAttributeCollapser(new HtmlMarkupParser().BuildSpan).Rewrite(results.Document);
rewritten = new MarkupCollapser(new HtmlMarkupParser().BuildSpan).Rewrite(rewritten);
// Assert
Assert.Equal(0, results.ParserErrors.Count);
EvaluateParseTree(rewritten,
new MarkupBlock(
Factory.Markup("<a"),
new MarkupBlock(new AttributeBlockCodeGenerator(name: "href", prefix: new LocationTagged <string>(" href='", 2, 0, 2), suffix: new LocationTagged <string>("'", 18, 0, 18)),
Factory.Markup(" href='").With(SpanCodeGenerator.Null),
Factory.Markup("~/Foo/Bar")
.WithEditorHints(EditorHints.VirtualPath)
.With(new LiteralAttributeCodeGenerator(
new LocationTagged <string>(String.Empty, 9, 0, 9),
new LocationTagged <SpanCodeGenerator>(new ResolveUrlCodeGenerator(), 9, 0, 9))),
Factory.Markup("'").With(SpanCodeGenerator.Null)),
Factory.Markup(" />")));
}
protected virtual void RunParseTest(
string document,
Func <ParserBase, Action> parserActionSelector,
Block expectedRoot,
IList <RazorError> expectedErrors,
bool designTimeParser,
Func <ParserContext, ParserBase> parserSelector = null
)
{
// Create the source
ParserResults results = RunParse(
document,
parserActionSelector,
designTimeParser,
parserSelector
);
// Evaluate the results
if (!ReferenceEquals(expectedRoot, IgnoreOutput))
{
EvaluateResults(results, expectedRoot, expectedErrors);
}
}
private static List <Span> ParseDocument(string documentContents, IList <RazorError> errors = null)
{
errors = errors ?? new List <RazorError>();
var markupParser = new HtmlMarkupParser();
var codeParser = new TestMvcCSharpRazorCodeParser();
var context = new ParserContext(
new SeekableTextReader(documentContents),
codeParser,
markupParser,
markupParser,
new ErrorSink());
codeParser.Context = context;
markupParser.Context = context;
markupParser.ParseDocument();
ParserResults results = context.CompleteParse();
foreach (RazorError error in results.ParserErrors)
{
errors.Add(error);
}
return(results.Document.Flatten().ToList());
}
private static void EvaluateNaiveBayesPredictor(ParserResults testData, NaiveBayesCalculator naiveBayesPredictor)
{
Console.WriteLine("Making predictions...");
uint hits = 0, misses = 0;
uint falsePositives = 0, falseNegatives = 0;
foreach (var testExample in testData.Values)
{
var isOnePrediction = naiveBayesPredictor.CalculatePrediction(testExample);
if (isOnePrediction && testExample.Output)
{
hits++;
}
else if (!isOnePrediction && !testExample.Output)
{
hits++;
}
else if (isOnePrediction && !testExample.Output)
{
misses++;
falsePositives++;
}
else if (!isOnePrediction && testExample.Output)
{
misses++;
falseNegatives++;
}
else
{
throw new InvalidOperationException();
}
}
Console.WriteLine("Score: {0}%. Hits: {1}, Misses: {2}", 100.0 * hits / (misses + hits), hits, misses);
Console.WriteLine("FalsePositives: {0}. FalseNegatives: {1}", falsePositives, falseNegatives);
}
public void VisitSendsErrorsToVisitor()
{
// Arrange
Mock<ParserVisitor> targetMock = new Mock<ParserVisitor>();
var root = new BlockBuilder() { Type = BlockType.Comment }.Build();
var errorSink = new ErrorSink();
List<RazorError> errors = new List<RazorError>
{
new RazorError("Foo", new SourceLocation(1, 0, 1), length: 3),
new RazorError("Bar", new SourceLocation(2, 0, 2), length: 3),
};
foreach (var error in errors)
{
errorSink.OnError(error);
}
var results = new ParserResults(root, Enumerable.Empty<TagHelperDescriptor>(), errorSink);
// Act
targetMock.Object.Visit(results);
// Assert
targetMock.Verify(v => v.VisitError(errors[0]));
targetMock.Verify(v => v.VisitError(errors[1]));
}
static void Main(string[] args)
{
string errorMessage = "";
if (!File.Exists(DataSetPath))
{
errorMessage += $"Failed to find file ${DataSetPath} - please update variable ${nameof(DataSetPath)} or create that file.\n";
}
if (!File.Exists(TestSetPath))
{
errorMessage += $"Failed to find file ${TestSetPath} - please update variable ${nameof(TestSetPath)} or create that file.\n";
}
if (errorMessage != "")
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Not all files available - not running!");
Console.WriteLine(errorMessage);
Console.ResetColor();
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
return;
}
Random rnd = new Random();
Console.WriteLine("Reading training data...");
ParserResults trainingData = ParserUtils.ParseData(DataSetPath, convertContinuousValues: true);
Console.WriteLine("Validating training set");
DataSetCleaner.ValidateDataSet(trainingData.Attributes, trainingData.Values);
Console.WriteLine("Getting test set...");
ParserResults testData = ParserUtils.ParseData(TestSetPath, trainingData.Attributes, convertContinuousValues: true);
Console.WriteLine("Validating test set");
DataSetCleaner.ValidateDataSet(testData.Attributes, testData.Values);
const string svmTrainingPath = "svmtraining";
const string svmOutputPath = "output";
List <int> kernelsToRunIn = new List <int>()
{
0, 1, 2, 3
};
string originalTrainingFile = "originalTraining.txt";
LibSvmConverter.ConvertToLibSvm(trainingData.Values, "originalTraining.txt");
// Run all kernels in parallel
kernelsToRunIn.Select((kernel) =>
{
// Run all iterations in parallel
Enumerable.Range(0, TotalSamplesForBiasAndVariance).Select((i) =>
{
Console.WriteLine("Doing loop {0} for kernel {1}", i, kernel);
var postFix = string.Format("K{0}-I{1}.txt", kernel, i);
string trainingPath = svmTrainingPath + postFix;
string trainingModelPath = trainingPath + ".model";
string outputPath = svmOutputPath + postFix;
if (!File.Exists(trainingModelPath))
{
List <List <DataSetValue> > differentTrainingData = Bagging.ProduceDifferentDataSets(trainingData.Values, 1, rnd);
LibSvmConverter.ConvertToLibSvm(differentTrainingData.Single(), trainingPath);
RunTrainingExe(trainingPath, kernel);
}
if (!File.Exists(outputPath))
{
RunEvaluateExe(originalTrainingFile, trainingModelPath, outputPath);
}
return(0);
}).ToList();
return(0);
}).ToList();
// Evaluate bias and variance
foreach (var kernel in kernelsToRunIn)
{
List <List <bool> > allPredictions = new List <List <bool> >();
for (int i = 0; i < TotalSamplesForBiasAndVariance; i++)
{
Console.WriteLine("Evaluating loop {0} for kernel {1}", i, kernel);
var postFix = string.Format("K{0}-I{1}.txt", kernel, i);
string trainingPath = svmTrainingPath + postFix;
string outputPath = svmOutputPath + postFix;
if (!File.Exists(trainingPath) || !File.Exists(outputPath))
{
continue;
}
List <bool> predictions = GetPredictionsFromOutputPath(outputPath);
allPredictions.Add(predictions);
}
if (allPredictions.Count == 0)
{
Console.WriteLine("Not enough information to evaluate kernel {0}", kernel);
continue;
}
double bias;
double variance = BiasAndVarianceCalculator.CalculateBiasAndVariance(trainingData, allPredictions, out bias);
Console.WriteLine("Bias:{0:0.00000} Variance:{1:0.00000}", bias, variance);
}
Console.WriteLine("Press any key to quit...");
Console.ReadKey();
}
static void Main(string[] args)
{
string errorMessage = "";
if (!File.Exists(DataSetPath))
{
errorMessage += $"Failed to find file ${DataSetPath} - please update variable ${nameof(DataSetPath)} or create that file.\n";
}
if (!File.Exists(TestSetPath))
{
errorMessage += $"Failed to find file ${TestSetPath} - please update variable ${nameof(TestSetPath)} or create that file.\n";
}
if (errorMessage != "")
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Not all files available - not running!");
Console.WriteLine(errorMessage);
Console.ResetColor();
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
return;
}
var startTime = DateTime.Now;
Console.WriteLine(startTime);
Random rnd = new Random();
Console.WriteLine("Reading training data...");
ParserResults trainingData = ParserUtils.ParseData(DataSetPath, convertContinuousValues: true);
Console.WriteLine("Validating training set");
DataSetCleaner.ValidateDataSet(trainingData.Attributes, trainingData.Values);
Console.WriteLine("Getting test set...");
ParserResults testData = ParserUtils.ParseData(TestSetPath, trainingData.Attributes, convertContinuousValues: true);
Console.WriteLine("Validating test set");
DataSetCleaner.ValidateDataSet(testData.Attributes, testData.Values);
List <List <DataSetValue> > differentDataSets = Bagging.ProduceDifferentDataSets(trainingData.Values, TotalSamplesForBiasAndVariance, rnd);
List <IPredictor> naiveBayesPredictors = new List <IPredictor>();
foreach (var differentTrainingDataSet in differentDataSets)
{
var naiveBayesPredictor = TrainNaiveBayes(differentTrainingDataSet);
naiveBayesPredictors.Add(naiveBayesPredictor);
}
double bias;
double variance = BiasAndVarianceCalculator.CalculateBiasAndVariance(trainingData, naiveBayesPredictors, out bias);
Console.WriteLine($"Bias:{bias} Variance:{variance}");
var originalNaiveBayesPredictor = TrainNaiveBayes(trainingData.Values);
EvaluateNaiveBayesPredictor(testData, originalNaiveBayesPredictor);
var endTime = DateTime.Now;
Console.WriteLine(endTime);
var totalMinutes = (endTime - startTime).TotalMinutes;
Console.WriteLine("Took {0} minutes.", totalMinutes);
Console.WriteLine("Press any key to quit...");
Console.ReadKey();
}
public void ConditionalAttributesDoNotCreateExtraDataForEntirelyLiteralAttribute()
{
// Arrange
const string code =
#region Big Block o' code
@"<div class=""sidebar"">
<h1>Title</h1>
<p>
As the author, you can <a href=""/Photo/Edit/photoId"">edit</a>
or <a href=""/Photo/Remove/photoId"">remove</a> this photo.
</p>
<dl>
<dt class=""description"">Description</dt>
<dd class=""description"">
The uploader did not provide a description for this photo.
</dd>
<dt class=""uploaded-by"">Uploaded by</dt>
<dd class=""uploaded-by""><a href=""/User/View/user.UserId"">user.DisplayName</a></dd>
<dt class=""upload-date"">Upload date</dt>
<dd class=""upload-date"">photo.UploadDate</dd>
<dt class=""part-of-gallery"">Gallery</dt>
<dd><a href=""/View/gallery.Id"" title=""View gallery.Name gallery"">gallery.Name</a></dd>
<dt class=""tags"">Tags</dt>
<dd class=""tags"">
<ul class=""tags"">
<li>This photo has no tags.</li>
</ul>
<a href=""/Photo/EditTags/photoId"">edit tags</a>
</dd>
</dl>
<p>
<a class=""download"" href=""/Photo/Full/photoId"" title=""Download: (photo.FileTitle + photo.FileExtension)"">Download full photo</a> ((photo.FileSize / 1024) KB)
</p>
</div>
<div class=""main"">
<img class=""large-photo"" alt=""photo.FileTitle"" src=""/Photo/Thumbnail"" />
<h2>Nobody has commented on this photo</h2>
<ol class=""comments"">
<li>
<h3 class=""comment-header"">
<a href=""/User/View/comment.UserId"" title=""View comment.DisplayName's profile"">comment.DisplayName</a> commented at comment.CommentDate:
</h3>
<p class=""comment-body"">comment.CommentText</p>
</li>
</ol>
<form method=""post"" action="""">
<fieldset id=""addComment"">
<legend>Post new comment</legend>
<ol>
<li>
<label for=""newComment"">Comment</label>
<textarea id=""newComment"" name=""newComment"" title=""Your comment"" rows=""6"" cols=""70""></textarea>
</li>
</ol>
<p class=""form-actions"">
<input type=""submit"" title=""Add comment"" value=""Add comment"" />
</p>
</fieldset>
</form>
</div>";
#endregion
// Act
ParserResults results = ParseDocument(code);
Block rewritten = new ConditionalAttributeCollapser(new HtmlMarkupParser().BuildSpan).Rewrite(results.Document);
rewritten = new MarkupCollapser(new HtmlMarkupParser().BuildSpan).Rewrite(rewritten);
// Assert
Assert.Equal(0, results.ParserErrors.Count);
EvaluateParseTree(rewritten, new MarkupBlock(Factory.Markup(code)));
}
private static void RunWithTreeLevels(ParserResults trainingData, Random rnd, int treeDepth, int sizeOfBaggers, ParserResults testData)
{
List <List <List <DataSetValue> > > dataSetValuesForBagging = new List <List <List <DataSetValue> > >();
for (int i = 0; i < TotalSamplesForBiasAndVariance; i++)
{
// Two layer sampling
List <DataSetValue> layer1Sampling = Bagging.ProduceDifferentDataSets(trainingData.Values, 1, rnd).Single();
if (sizeOfBaggers == 1)
{
dataSetValuesForBagging.Add(new List <List <DataSetValue> >()
{
layer1Sampling
});
}
else
{
dataSetValuesForBagging.Add(Bagging.ProduceDifferentDataSets(layer1Sampling, sizeOfBaggers, rnd));
}
}
// Initialize the required trees
List <List <DecisionTreeLevel> > listOfTreesToRunTestOn = new List <List <DecisionTreeLevel> >();
foreach (var dataSetForBagging in dataSetValuesForBagging)
{
// Foe each bagger, for each dataset, create a new tree
listOfTreesToRunTestOn.Add(
dataSetForBagging.Select(
dataSet => new DecisionTreeLevel(0, trainingData.Attributes, dataSet, maximumDepth: treeDepth)).ToList());
}
Parallel.ForEach(listOfTreesToRunTestOn.SelectMany(s => s), l => l.D3());
Parallel.ForEach(listOfTreesToRunTestOn.SelectMany(s => s), l => l.TrimTree());
//string sampleSerializedTree = listOfTreesToRunTestOn[0][0].SerializeDecisionTree();
//Console.WriteLine("Evaluating trees against test data...");
double totalScoreAgainstTrainingData = 0;
double totalScoreAgainstTestData = 0;
foreach (List <DecisionTreeLevel> baggingSetOfTrees in listOfTreesToRunTestOn)
{
DecisionTreeScore scoreAgainstTrainingData = DecisionTreeScorer.ScoreWithTreeWithTestSet(baggingSetOfTrees, trainingData.Values);
DecisionTreeScore scoreAgainstTestData = DecisionTreeScorer.ScoreWithTreeWithTestSet(baggingSetOfTrees, testData.Values);
//score.PrintTotalScore();
totalScoreAgainstTrainingData += scoreAgainstTrainingData.GetTotalScore();
totalScoreAgainstTestData += scoreAgainstTestData.GetTotalScore();
}
totalScoreAgainstTrainingData = totalScoreAgainstTrainingData / listOfTreesToRunTestOn.Count;
totalScoreAgainstTestData = totalScoreAgainstTestData / listOfTreesToRunTestOn.Count;
double bias;
double variance = BiasAndVarianceCalculator.CalculateBiasAndVariance(trainingData, listOfTreesToRunTestOn, out bias);
Console.WriteLine("Variance: {0:0.00000}. Bias: {1:0.00000}. ScoreTraining : {2:0.00000}, ScoreTest : {3:0.00000}", variance, bias, totalScoreAgainstTrainingData, totalScoreAgainstTestData);
//Console.WriteLine(bias);
//Console.WriteLine(variance);
//Console.WriteLine(totalScoreAgainstTrainingData);
//Console.WriteLine(totalScoreAgainstTestData);
}
public static void EvaluateResults(ParserResults results, Block expectedRoot, IList <RazorError> expectedErrors)
{
EvaluateResults(null, results, expectedRoot, expectedErrors);
}
public static void EvaluateResults(ParserResults results, Block expectedRoot, IList <RazorError> expectedErrors)
{
EvaluateParseTree(results.Document, expectedRoot);
EvaluateRazorErrors(results.ParserErrors, expectedErrors);
}
public static void EvaluateResults(TestContext context, ParserResults results, Block expectedRoot) {
EvaluateResults(context, results, expectedRoot, null);
}
static void Main(string[] args)
{
string errorMessage = "";
if (!File.Exists(DataSetPath))
{
errorMessage += $"Failed to find file ${DataSetPath} - please update variable ${nameof(DataSetPath)} or create that file.\n";
}
if (!File.Exists(TestSetPath))
{
errorMessage += $"Failed to find file ${TestSetPath} - please update variable ${nameof(TestSetPath)} or create that file.\n";
}
if (errorMessage != "")
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Not all files available - not running!");
Console.WriteLine(errorMessage);
Console.ResetColor();
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
return;
}
Random rnd = new Random();
Console.WriteLine("Reading training data...");
ParserResults trainingData = ParserUtils.ParseData(DataSetPath);
Console.WriteLine("Validating data set");
DataSetCleaner.ValidateDataSet(trainingData.Attributes, trainingData.Values);
List <List <List <DataSetValue> > > dataSetValuesForBagging = new List <List <List <DataSetValue> > >()
{
Bagging.ProduceDifferentDataSets(trainingData.Values, 1, rnd),
Bagging.ProduceDifferentDataSets(trainingData.Values, 3, rnd),
Bagging.ProduceDifferentDataSets(trainingData.Values, 5, rnd),
Bagging.ProduceDifferentDataSets(trainingData.Values, 10, rnd),
Bagging.ProduceDifferentDataSets(trainingData.Values, 20, rnd),
};
// Initialize the required trees
List <List <DecisionTreeLevel> > listOfTreesToRunTestOn = new List <List <DecisionTreeLevel> >();
foreach (var dataSetForBagging in dataSetValuesForBagging)
{
listOfTreesToRunTestOn.Add(dataSetForBagging.Select(x => new DecisionTreeLevel(ChiTestLimit, trainingData.Attributes, x)).ToList());
}
Console.WriteLine("Runnind D3 on all trees in parallel...");
Parallel.ForEach(listOfTreesToRunTestOn.SelectMany(s => s), l => l.D3());
Console.WriteLine("Deleting unecessary nodes...");
Parallel.ForEach(listOfTreesToRunTestOn.SelectMany(s => s), l => l.TrimTree());
Console.WriteLine("Getting test data set...");
ParserResults testData = ParserUtils.ParseData(TestSetPath);
Console.WriteLine("Evaluating trees against test data...");
foreach (List <DecisionTreeLevel> baggingSetOfTrees in listOfTreesToRunTestOn)
{
DecisionTreeScore score = DecisionTreeScorer.ScoreWithTreeWithTestSet(baggingSetOfTrees, testData.Values);
score.PrintTotalScore();
}
Console.WriteLine("Press any key to quit...");
Console.ReadKey();
}
public static void EvaluateResults(ParserResults results, Block expectedRoot, IList<RazorError> expectedErrors) {
EvaluateResults(null, results, expectedRoot, expectedErrors);
}
public static void EvaluateResults(ParserResults results, Block expectedRoot)
{
EvaluateResults(results, expectedRoot, null);
}
static void Main(string[] args)
{
string errorMessage = "";
if (!File.Exists(DataSetPath))
{
errorMessage += $"Failed to find file ${DataSetPath} - please update variable ${nameof(DataSetPath)} or create that file.\n";
}
if (!File.Exists(TestSetPath))
{
errorMessage += $"Failed to find file ${TestSetPath} - please update variable ${nameof(TestSetPath)} or create that file.\n";
}
if (errorMessage != "")
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Not all files available - not running!");
Console.WriteLine(errorMessage);
Console.ResetColor();
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
return;
}
Console.WriteLine("Reading training data...");
ParserResults trainingData = ParserUtils.ParseData(DataSetPath);
// Optimizations are optional
// DataSetOptimizerForExtraCredit.OptimizeDataSetForExtraCredit(trainingData.Attributes, trainingData.Values);
Console.WriteLine("Validating data set");
DataSetCleaner.ValidateDataSet(trainingData.Attributes, trainingData.Values);
// Initialize the required trees with their respective chiTestLimits
List <DecisionTreeLevel> listOfTreesToRunTestOn = new List <DecisionTreeLevel>()
{
new DecisionTreeLevel(chiTestLimit: 0.99),
new DecisionTreeLevel(chiTestLimit: 0.95),
new DecisionTreeLevel(chiTestLimit: 0),
};
Console.WriteLine("Runnind D3...");
Parallel.ForEach(listOfTreesToRunTestOn, l => l.D3(trainingData.Attributes, trainingData.Values));
Console.WriteLine("Deleting unecessary nodes...");
Parallel.ForEach(listOfTreesToRunTestOn, l => l.TrimTree());
Console.WriteLine("Getting test data set...");
ParserResults testData = ParserUtils.ParseData(TestSetPath);
// Optimizations are optional
// DataSetOptimizerForExtraCredit.OptimizeDataSetForExtraCredit(testData.Attributes, testData.Values);
Console.WriteLine("Evaluating trees against test data...");
List <DecisionTreeScore> scores = listOfTreesToRunTestOn.AsParallel().Select(t => DecisionTreeScorer.ScoreWithTreeWithTestSet(t, testData.Values)).ToList();
//Console.WriteLine("Writing trees to text files (for debugging/visualization)...");
// Dump the trees to a txt file for debugging/visualization
// NOTE: This won't work the the Chi=0 case - the JSON file generated is too big
// Parallel.ForEach(listOfTreesToRunTestOn, l => File.WriteAllText("Chi" + Convert.ToInt64(l.ChiTestLimit * 10000000000000) + ".json", l.SerializeDecisionTree()));
List <DecisionTreeScore> trainingDataScores = listOfTreesToRunTestOn.AsParallel().Select(t => DecisionTreeScorer.ScoreWithTreeWithTestSet(t, trainingData.Values)).ToList();
// Print the results to console
foreach (var score in scores)
{
score.PrintTotalScore();
}
Console.WriteLine("Evaluating trees against training data:");
foreach (var score in trainingDataScores)
{
score.PrintTotalScore();
}
Console.WriteLine("Press any key to quit...");
Console.ReadKey();
}
public RazorCSharpModelTypeLocater(ITextBuffer textBuffer)
{
ParserResults results = ParseTemplate(textBuffer);
ModelTypeName = GetModelTypeName(results);
}
public static void EvaluateResults(TestContext context, ParserResults results, Block expectedRoot)
{
EvaluateResults(context, results, expectedRoot, null);
}
public static void EvaluateResults(ParserResults results, Block expectedRoot)
{
EvaluateResults(results, expectedRoot, null);
}
public object Process(Node node, bool extractDeleted, bool extractSource)
{
ParserResults pr = new ParserResults();
if (node is FileSystem fs) //这个插件期望node是文件系统
{
MetaData meta = new MetaData();
var ds = fs.DataStore;
try
{
//analyze_csidata
//private/var/root/Library/Lockdown/data_ark.plist
var data_ark = fs.Search("/Library/Lockdown/data_ark.plist$").FirstOrDefault();
if (data_ark != null && data_ark.Type == NodeType.File)
{
var plist = PlistHelper.ReadPlist <NSDictionary>(data_ark);
FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.Language.ToString(), "com.apple.international-Language");
FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.BuildVersion.ToString(), "com.apple.who-uptodate_build");
FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.LastBackupComputerName.ToString(), "com.apple.iTunes.backup-LastBackupComputerName");
FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.DeviceName.ToString(), "-DeviceName");
FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.TimeZone.ToString(), "-TimeZone");
FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.ActivationStateAcknowledged.ToString(), "-ActivationStateAcknowledged");
FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.Locale.ToString(), "com.apple.international-Locale");
FetchDeviceInfo(ds.DeviceInfo, plist, DeviceInfoKeys.LastBackupComputerType.ToString(), "com.apple.iTunes.backup-LastBackupComputerType");
}
//System/Library/LaunchDaemons/com.apple.atc.plist
var atcNode = fs.Search("/Preferences/com.apple.atc.plist$").FirstOrDefault();
if (atcNode != null && atcNode.Type == NodeType.File)
{
var plist = PlistHelper.ReadPlist <NSDictionary>(atcNode);
if (plist != null)
{
var diskUsage = plist.SafeGetObject <NSDictionary>("DiskUsage");
FileSize GetDiskUsageSize(NSDictionary dict, string key)
{
var subdict = dict.SafeGetObject <NSDictionary>(key);
if (subdict != null)
{
return(new FileSize(subdict.SafeGetInt64("_PhysicalSize")));
}
return(new FileSize(0));
}
ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.VoiceMemoSize.ToString(),
GetDiskUsageSize(diskUsage, "VoiceMemo")));
ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.UserDataSize.ToString(),
GetDiskUsageSize(diskUsage, "UserData")));
ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.BookDataSize.ToString(),
GetDiskUsageSize(diskUsage, "Book")));
ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.MediaDataSize.ToString(),
GetDiskUsageSize(diskUsage, "Media")));
ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.ApplicationSize.ToString(),
GetDiskUsageSize(diskUsage, "Application")));
ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.LogsSize.ToString(),
GetDiskUsageSize(diskUsage, "Logs")));
ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.RingtoneSize.ToString(),
GetDiskUsageSize(diskUsage, "Ringtone")));
ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.TotalDiskSize.ToString(),
new FileSize(diskUsage.SafeGetInt64("_PhysicalSize"))));
ds.DeviceInfo.Add(new MetaDataField(DeviceInfoKeys.FreeDiskSize.ToString(),
new FileSize(diskUsage.SafeGetInt64("_FreeSize"))));
}
}
var projFile = ds.ProjectState.ProjectFile;
if (!string.IsNullOrEmpty(projFile) && File.Exists(projFile))
{
var plist = PlistHelper.ReadPlist <NSDictionary>(projFile);
if (plist != null)
{
var deviceInfo = plist.SafeGetObject <NSDictionary>("Device");
if (deviceInfo != null)
{
foreach (var p in deviceInfo)
{
var key = p.Key;
var obj = p.Value;
if (obj is NSString nss)
{
ds.DeviceInfo.Add(new MetaDataField(key, nss.Content));
}
else if (obj is NSNumber nsn)
{
if (nsn.isBoolean())
{
ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToBool()));
}
else if (nsn.isInteger())
{
ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToLong()));
}
else if (nsn.isReal())
{
ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToDouble()));
}
}
}
}
}
}
var lockd = fs.GetByPath("/private/Lockdown.plist");
if (lockd != null && lockd.Type == NodeType.File)
{
var plist = PlistHelper.ReadPlist <NSDictionary>(lockd);
if (plist != null)
{
var keys = Enum.GetNames(typeof(DeviceInfoKeys));
foreach (var key in keys)
{
if (plist.ContainsKey(key))
{
var obj = plist.SafeGetObject <NSObject>(key);
if (obj is NSString nss)
{
ds.DeviceInfo.Add(new MetaDataField(key, nss.Content));
}
else if (obj is NSNumber nsn)
{
if (nsn.isBoolean())
{
ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToBool()));
}
else if (nsn.isInteger())
{
ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToLong()));
}
else if (nsn.isReal())
{
ds.DeviceInfo.Add(new MetaDataField(key, nsn.ToDouble()));
}
}
}
}
}
}
}
catch (Exception e)
{
TraceService.TraceError(e);
}
}
//返回空的pr,实际的数据已经存在ds.DeviceInfo里面了,这是特殊情况,一般情况下你应该将数据通过pr存储返回给主程序
return(pr);
}
public override void OnComplete() {
base.OnComplete();
Results = new ParserResults(_rootBlock, _errorList);
}
public static void BuilderTemplate(ITemplateInfo templateinfo, ITemplateHost host)
{
Type type = host.BaseType;
string typename = "_" + Guid.NewGuid().ToString("N");
RazorTemplateEngine engine = CreateHost(type, "_" + type.Name, typename);
AddNamespace(type.Namespace);
GeneratorResults razorResults = null;
using (System.IO.StreamReader reader = templateinfo.GetCode())
{
razorResults = engine.GenerateCode(reader);
CSharpCodeProvider codeProvider = new CSharpCodeProvider();
CodeGeneratorOptions options = new CodeGeneratorOptions();
string LastGeneratedCode = null;
using (StringWriter writer = new StringWriter())
{
codeProvider.GenerateCodeFromCompileUnit(razorResults.GeneratedCode, writer, options);
templateinfo.Host.LastCompileSource = writer.ToString();
}
CompilerParameters compilerParameters = new CompilerParameters(new string[0]);
compilerParameters.OutputAssembly = "tmp_assembly" + typename;
compilerParameters.GenerateInMemory = false;
AddDomainAssemblies();
foreach (string item in ReferencedAssemblies)
{
compilerParameters.ReferencedAssemblies.Add(item);
}
compilerParameters.GenerateInMemory = true;
CompilerResults compilerResults = codeProvider.CompileAssemblyFromDom(compilerParameters, razorResults.GeneratedCode);
if (compilerResults.Errors.Count > 0)
{
string errormessage = null;
int throwindex = 1;
System.Text.StringBuilder sb = new System.Text.StringBuilder();
reader.BaseStream.Seek(0, SeekOrigin.Begin);
using (System.IO.StreamReader line = new StreamReader(reader.BaseStream, reader.CurrentEncoding))
{
ParserResults presult = engine.ParseTemplate(line);
if (presult.ParserErrors.Count > 0)
{
throwindex = presult.ParserErrors[0].Location.LineIndex + 1;
errormessage = presult.ParserErrors[0].Message;
reader.BaseStream.Seek(0, SeekOrigin.Begin);
using (System.IO.StreamReader readcode = new StreamReader(reader.BaseStream, reader.CurrentEncoding))
{
string code = readcode.ReadLine();
while (code != null)
{
sb.AppendLine(code);
code = readcode.ReadLine();
}
}
}
else
{
throwindex = compilerResults.Errors[0].Line;
errormessage = compilerResults.Errors[0].ErrorText;
sb.Append(LastGeneratedCode);
}
}
templateinfo.CompileError = errormessage;
throw new RazorException(templateinfo.CompileError);
}
templateinfo.Assembly = compilerResults.CompiledAssembly;
templateinfo.TemplateType = compilerResults.CompiledAssembly.GetType("_" + type.Name + "." + typename);
}
}
public static void EvaluateResults(ParserResults results, Block expectedRoot, IList<RazorError> expectedErrors)
{
EvaluateParseTree(results.Document, expectedRoot);
EvaluateRazorErrors(results.ParserErrors, expectedErrors);
}
void ParseTemplate(string html)
{
var parser = new RazorParser(codeLanguage.CreateCodeParser(), new HtmlMarkupParser());
parserResults = parser.Parse(new StringReader(html));
}
