public void Run()
{
if (HasRunBeenCalled())
{
throw new InvalidOperationException("Can only call run once.");
}
Console.WriteLine($"Running for {RunPeriod.TotalSeconds}sec.");
//Write the headers, but no histograms (as we don't have any yet).
_logWriter.Write(DateTime.Now);
//ThreadSafe-writes require a Concurrent implementation of a Histogram
//ThreadSafe-reads require a recorder
var recorder = HistogramFactory
.With64BitBucketSize() //LongHistogram
.WithValuesFrom(1) //Default value
.WithValuesUpTo(TimeStamp.Minutes(10)) //Default value
.WithPrecisionOf(3) //Default value
.WithThreadSafeWrites() //Switches internal imp to concurrent version i.e. LongConcurrentHistogram
.WithThreadSafeReads() //returns a Recorder that wraps the LongConcurrentHistogram
.Create();
var outputThread = new Thread(ts => WriteToDisk((Recorder)ts));
outputThread.Start(recorder);
RecordMeasurements(recorder);
//Wait for the output thread to complete writing.
outputThread.Join();
}
// Using a DependencyProperty as the backing store for MassMzImage. This enables animation, styling, binding, etc...
private void CreatePlots()
{
if (AlignmentData == null || AlignmentData.AlignmentData == null)
{
return;
}
var alignmentData = AlignmentData.AlignmentData;
var heatmap = HeatmapFactory.CreateAlignedHeatmap(alignmentData.HeatScores, alignmentData.BaselineIsAmtDB);
var feature = ScatterPlotFactory.CreateFeatureMassScatterPlot(AlignmentData.AligneeFeatures);
var netHistogram = HistogramFactory.CreateHistogram(alignmentData.NetErrorHistogram, "NET Error", "NET Error");
var massHistogram = HistogramFactory.CreateHistogram(alignmentData.NetErrorHistogram, "Mass Error", "Mass Error (ppm)");
var residuals = alignmentData.ResidualData;
var netResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Net, residuals.LinearCustomNet,
residuals.LinearNet, "NET Residuals", "Scans", "NET");
var massMzResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Mz, residuals.MzMassError,
residuals.MzMassErrorCorrected, "Mass Residuals", "m/z", "Mass Errors");
var massScanResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Net, residuals.MzMassError,
residuals.MzMassErrorCorrected, "Mass Residuals", "Scan", "Mass Errors");
NetScanImage = ImageConverter.ConvertImage(PlotImageUtility.CreateImage(netResidual));
MassHistogram = ImageConverter.ConvertImage(PlotImageUtility.CreateImage(massHistogram));
NetHistogram = ImageConverter.ConvertImage(PlotImageUtility.CreateImage(netHistogram));
HeatmapImage = ImageConverter.ConvertImage(PlotImageUtility.CreateImage(heatmap));
MassMzImage = ImageConverter.ConvertImage(PlotImageUtility.CreateImage(massMzResidual));
MassScanImage = ImageConverter.ConvertImage(PlotImageUtility.CreateImage(massScanResidual));
FeaturePlotImage = ImageConverter.ConvertImage(PlotImageUtility.CreateImage(feature));
}
public void CanCreateShortHistogram()
{
var actual = HistogramFactory.With16BitBucketSize()
.Create();
Assert.IsAssignableFrom <ShortHistogram>(actual);
}
public void CanCreateIntHistogram()
{
var actual = HistogramFactory.With32BitBucketSize()
.Create();
Assert.IsInstanceOf <IntHistogram>(actual);
}
public void CanCreateLongHistogram()
{
var actual = HistogramFactory.With64BitBucketSize()
.Create();
Assert.IsAssignableFrom <LongHistogram>(actual);
}
public void CanCreateIntConcurrentHistogram()
{
var actual = HistogramFactory.With32BitBucketSize()
.WithThreadSafeWrites()
.Create();
Assert.IsAssignableFrom <IntConcurrentHistogram>(actual);
}
protected override HistogramBase Create(long highestTrackableValue, int numberOfSignificantDigits)
{
//return new IntHistogram(highestTrackableValue, numberOfSignificantDigits);
return(HistogramFactory.With32BitBucketSize()
.WithValuesUpTo(highestTrackableValue)
.WithPrecisionOf(numberOfSignificantDigits)
.Create());
}
public void CanCreateLongConcurrentHistogram()
{
var actual = HistogramFactory.With64BitBucketSize()
.WithThreadSafeWrites()
.Create();
Assert.IsInstanceOf <LongConcurrentHistogram>(actual);
}
protected override HistogramBase CreateHistogram(long id, long min, long max, int sf)
{
return(HistogramFactory.With64BitBucketSize()
.WithValuesFrom(min)
.WithValuesUpTo(max)
.WithPrecisionOf(sf)
.WithThreadSafeWrites()
.Create());
}
protected override HistogramBase CreateHistogram(long id, long min, long max, int sf)
{
//return new ShortHistogram(id, min, max, sf);
return(HistogramFactory.With16BitBucketSize()
.WithValuesFrom(min)
.WithValuesUpTo(max)
.WithPrecisionOf(sf)
.Create());
}
internal override HistogramBase Create(long highestTrackableValue, int numberOfSignificantValueDigits)
{
//return new IntConcurrentHistogram(1, highestTrackableValue, numberOfSignificantValueDigits);
return(HistogramFactory.With32BitBucketSize()
.WithValuesUpTo(highestTrackableValue)
.WithPrecisionOf(numberOfSignificantValueDigits)
.WithThreadSafeWrites()
.Create());
}
protected override Recorder Create(long min, long max, int sf)
{
return(HistogramFactory.With16BitBucketSize()
.WithValuesFrom(min)
.WithValuesUpTo(max)
.WithPrecisionOf(sf)
.WithThreadSafeReads()
.Create());
}
internal override HistogramBase Create(long lowestTrackableValue, long highestTrackableValue, int numberOfSignificantValueDigits)
{
//return new ShortHistogram(lowestTrackableValue, highestTrackableValue, numberOfSignificantValueDigits);
return(HistogramFactory.With16BitBucketSize()
.WithValuesFrom(lowestTrackableValue)
.WithValuesUpTo(highestTrackableValue)
.WithPrecisionOf(numberOfSignificantValueDigits)
.Create());
}
protected override HistogramBase CreateHistogram(long id, long min, long max, int sf)
{
//return new IntConcurrentHistogram(id, min, max, sf);
return(HistogramFactory.With32BitBucketSize()
.WithValuesFrom(min)
.WithValuesUpTo(max)
.WithPrecisionOf(sf)
.WithThreadSafeWrites()
.Create());
}
protected override HistogramBase Create(long lowestTrackableValue, long highestTrackableValue, int numberOfSignificantValueDigits)
{
//return new LongConcurrentHistogram(lowestTrackableValue, highestTrackableValue, numberOfSignificantValueDigits);
return(HistogramFactory.With64BitBucketSize()
.WithValuesFrom(lowestTrackableValue)
.WithValuesUpTo(highestTrackableValue)
.WithPrecisionOf(numberOfSignificantValueDigits)
.WithThreadSafeWrites()
.Create());
}
static HdrHistogramReservoir()
{
HdrHistogramReservoir._emptySnapshot = new HdrSnapshot(
HistogramFactory
.With64BitBucketSize()
.WithThreadSafeReads()
.Create()
.GetIntervalHistogram(),
0,
null,
0,
null);
}
public void CanCreateIntHistogramWithSpecifiedRangeValues(long min, long max, int sf)
{
var actual = HistogramFactory.With32BitBucketSize()
.WithValuesFrom(min)
.WithValuesUpTo(max)
.WithPrecisionOf(sf)
.Create();
Assert.IsAssignableFrom <IntHistogram>(actual);
Assert.Equal(min, actual.LowestTrackableValue);
Assert.Equal(max, actual.HighestTrackableValue);
Assert.Equal(sf, actual.NumberOfSignificantValueDigits);
}
public void IntConcurrentHistogramWithSpecifiedRangeValues(long min, long max, int sf)
{
var actual = HistogramFactory.With32BitBucketSize()
.WithValuesFrom(min)
.WithValuesUpTo(max)
.WithPrecisionOf(sf)
.WithThreadSafeWrites()
.Create();
Assert.IsInstanceOf <IntConcurrentHistogram>(actual);
Assert.AreEqual(min, actual.LowestTrackableValue);
Assert.AreEqual(max, actual.HighestTrackableValue);
Assert.AreEqual(sf, actual.NumberOfSignificantValueDigits);
}
/// <summary>
/// Creates the final analysis plots.
/// </summary>
public void CreateChargePlots(Dictionary <int, int> chargeMap)
{
Logger.PrintMessage("Creating Charge State Plots");
var report = Config.Report;
report.PushTextHeader("Cluster Charge States");
report.PushStartTable();
var path = Path.Combine(Config.AnalysisPath, PlotPath, "clusters_chargestates.png");
var histogram = HistogramFactory.CreateHistogram(chargeMap, "Charge States", "Charge States");
PlotImageUtility.SaveImage(histogram, path);
report.PushImageColumn(path);
report.PushEndTable();
}
public void CanCreateShortHistogramRecorder(long min, long max, int sf)
{
var actual = HistogramFactory.With16BitBucketSize()
.WithValuesFrom(min)
.WithValuesUpTo(max)
.WithPrecisionOf(sf)
.WithThreadSafeReads()
.Create();
var histogram = actual.GetIntervalHistogram();
Assert.IsAssignableFrom <ShortHistogram>(histogram);
Assert.Equal(min, histogram.LowestTrackableValue);
Assert.Equal(max, histogram.HighestTrackableValue);
Assert.Equal(sf, histogram.NumberOfSignificantValueDigits);
}
public void CanCreateLongConcurrentHistogramRecorder(long min, long max, int sf)
{
var actual = HistogramFactory.With64BitBucketSize()
.WithValuesFrom(min)
.WithValuesUpTo(max)
.WithPrecisionOf(sf)
.WithThreadSafeWrites()
.WithThreadSafeReads()
.Create();
var histogram = actual.GetIntervalHistogram();
Assert.IsInstanceOf <LongConcurrentHistogram>(histogram);
Assert.AreEqual(min, histogram.LowestTrackableValue);
Assert.AreEqual(max, histogram.HighestTrackableValue);
Assert.AreEqual(sf, histogram.NumberOfSignificantValueDigits);
}
public AlignmentPlotCreator(AlignmentData alignment)
{
var residuals = alignment.ResidualData;
Heatmap = HeatmapFactory.CreateAlignedHeatmap(alignment.HeatScores, alignment.BaselineIsAmtDB);
NetResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Net, residuals.LinearCustomNet,
residuals.LinearNet, "NET Residuals", "Scans", "NET");
MassHistogram = HistogramFactory.CreateHistogram(alignment.MassErrorHistogram, "Mass Error", "Mass Error (ppm)");
NetHistogram = HistogramFactory.CreateHistogram(alignment.NetErrorHistogram, "NET Error", "NET Error");
MassMzResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Mz, residuals.MzMassError,
residuals.MzMassErrorCorrected, "Mass Residuals", "m/z", "Mass Errors");
MassScanResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Net, residuals.MzMassError,
residuals.MzMassErrorCorrected, "Mass Residuals", "Scan", "Mass Errors");
}
/// <summary>
/// Initializes a new instance of the <see cref="HdrHistogramReservoir" /> class.
/// </summary>
public HdrHistogramReservoir(long lowestTrackableValue, long highestTrackableValue, int numberOfSignificantValueDigits)
{
_highestTrackableValue = highestTrackableValue;
var recorder = HistogramFactory
.With64BitBucketSize()
.WithValuesFrom(lowestTrackableValue)
.WithValuesUpTo(highestTrackableValue)
.WithPrecisionOf(numberOfSignificantValueDigits)
.WithThreadSafeWrites()
.WithThreadSafeReads()
.Create();
_recorder = recorder;
_intervalHistogram = recorder.GetIntervalHistogram();
_runningTotals = new LongHistogram(lowestTrackableValue, highestTrackableValue, _intervalHistogram.NumberOfSignificantValueDigits);
}
/// <summary>
/// Initializes a new instance of the <see cref="HdrHistogramReservoir" /> class.
/// </summary>
public HdrHistogramReservoir(
long lowestTrackableValue, long highestTrackableValue, int numberOfSignificantValueDigits, long refreshIntervalMilliseconds)
{
_highestTrackableValue = highestTrackableValue;
_refreshIntervalTicks = TimeSpan.FromMilliseconds(refreshIntervalMilliseconds).Ticks;
var recorder = HistogramFactory
.With64BitBucketSize()
.WithValuesFrom(lowestTrackableValue)
.WithValuesUpTo(highestTrackableValue)
.WithPrecisionOf(numberOfSignificantValueDigits)
.WithThreadSafeWrites()
.WithThreadSafeReads()
.Create();
_recorder = recorder;
_intervalHistogram = recorder.GetIntervalHistogram();
UnsafeRefresh(DateTime.UtcNow.Ticks, true);
}
public void CreateClusterPlots(FeaturesClusteredEventArgs args)
{
// Plots:
// Dataset member
// Total members
// Cluster scatter plot
var clusters = args.Clusters;
var report = Config.Report;
Logger.PrintMessage("Creating Charge State Plots");
report.PushStartTable();
report.PushStartTableRow();
report.PushTextHeader("Cluster Statistics");
report.PushEndTableRow();
report.PushStartTableRow();
var path = Path.Combine(Config.AnalysisPath, PlotPath, "clusters_scatterplot.png");
var clusterScatterPlot = ScatterPlotFactory.CreateClusterMassScatterPlot(clusters);
PlotImageUtility.SaveImage(clusterScatterPlot, path);
report.PushImageColumn(path, WIDTH, HEIGHT);
report.PushEndTableRow();
report.PushStartTableRow();
path = Path.Combine(Config.AnalysisPath, PlotPath, "clusters_datasetMembersHistogram.png");
var datasetMemberHistogram =
HistogramFactory.CreateHistogram(clusters.CreateClusterDatasetMemberSizeHistogram(), "Dataset Members", "Dataset Count");
PlotImageUtility.SaveImage(datasetMemberHistogram, path);
report.PushImageColumn(path, WIDTH, HEIGHT);
path = Path.Combine(Config.AnalysisPath, PlotPath, "clusters_clusterSizeHistogram.png");
var clusterSizeHistogram = HistogramFactory.CreateHistogram(clusters.CreateClusterSizeHistogram(), "Cluster Members", "Cluster Size");
PlotImageUtility.SaveImage(clusterSizeHistogram, path);
report.PushImageColumn(path, WIDTH, HEIGHT);
report.PushEndTableRow();
report.PushEndTable();
}
public static void Run()
{
_outputStream = File.Create(LogPath);
_logWriter = new HistogramLogWriter(_outputStream);
_logWriter.Write(DateTime.Now);
var recorder = HistogramFactory
.With64BitBucketSize()
?.WithValuesFrom(1)
?.WithValuesUpTo(2345678912345)
?.WithPrecisionOf(3)
?.WithThreadSafeWrites()
?.WithThreadSafeReads()
?.Create();
var accumulatingHistogram = new LongHistogram(2345678912345, 3);
var size = accumulatingHistogram.GetEstimatedFootprintInBytes();
RILogManager.Default?.SendDebug("Histogram size = {0} bytes ({1:F2} MB)", size, size / 1024.0 / 1024.0);
RILogManager.Default?.SendDebug("Recorded latencies [in system clock ticks]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.None, useCsvFormat: true);
Console.WriteLine();
RILogManager.Default?.SendDebug("Recorded latencies [in usec]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMicroseconds, useCsvFormat: true);
Console.WriteLine();
RILogManager.Default?.SendDebug("Recorded latencies [in msec]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds, useCsvFormat: true);
Console.WriteLine();
RILogManager.Default?.SendDebug("Recorded latencies [in sec]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToSeconds, useCsvFormat: true);
DocumentResults(accumulatingHistogram, recorder);
RILogManager.Default?.SendDebug("Build Vocabulary.");
DocumentResults(accumulatingHistogram, recorder);
Vocabulary vocabulary = new Vocabulary();
DocumentResults(accumulatingHistogram, recorder);
string trainPath = InternetFileDownloader.Download(DOWNLOAD_URL + TRAIN_FILE, TRAIN_FILE);
DocumentResults(accumulatingHistogram, recorder);
string validPath = InternetFileDownloader.Download(DOWNLOAD_URL + VALID_FILE, VALID_FILE);
DocumentResults(accumulatingHistogram, recorder);
string testPath = InternetFileDownloader.Download(DOWNLOAD_URL + TEST_FILE, TEST_FILE);
DocumentResults(accumulatingHistogram, recorder);
int[] trainData = vocabulary.LoadData(trainPath);
DocumentResults(accumulatingHistogram, recorder);
int[] validData = vocabulary.LoadData(validPath);
DocumentResults(accumulatingHistogram, recorder);
int[] testData = vocabulary.LoadData(testPath);
DocumentResults(accumulatingHistogram, recorder);
int nVocab = vocabulary.Length;
RILogManager.Default?.SendDebug("Network Initializing.");
FunctionStack model = new FunctionStack("Test10",
new EmbedID(nVocab, N_UNITS, name: "l1 EmbedID"),
new Dropout(),
new LSTM(true, N_UNITS, N_UNITS, name: "l2 LSTM"),
new Dropout(),
new LSTM(true, N_UNITS, N_UNITS, name: "l3 LSTM"),
new Dropout(),
new Linear(true, N_UNITS, nVocab, name: "l4 Linear")
);
DocumentResults(accumulatingHistogram, recorder);
// Do not cease at the given threshold, correct the rate by taking the rate from L2Norm of all parameters
GradientClipping gradientClipping = new GradientClipping(threshold: GRAD_CLIP);
SGD sgd = new SGD(learningRate: 1);
model.SetOptimizer(gradientClipping, sgd);
DocumentResults(accumulatingHistogram, recorder);
Real wholeLen = trainData.Length;
int jump = (int)Math.Floor(wholeLen / BATCH_SIZE);
int epoch = 0;
Stack <NdArray[]> backNdArrays = new Stack <NdArray[]>();
RILogManager.Default?.SendDebug("Train Start.");
double dVal;
NdArray x = new NdArray(new[] { 1 }, BATCH_SIZE, (Function)null);
NdArray t = new NdArray(new[] { 1 }, BATCH_SIZE, (Function)null);
for (int i = 0; i < jump * N_EPOCH; i++)
{
for (int j = 0; j < BATCH_SIZE; j++)
{
x.Data[j] = trainData[(int)((jump * j + i) % wholeLen)];
t.Data[j] = trainData[(int)((jump * j + i + 1) % wholeLen)];
}
NdArray[] result = model.Forward(true, x);
Real sumLoss = new SoftmaxCrossEntropy().Evaluate(result, t);
backNdArrays.Push(result);
RILogManager.Default?.SendDebug("[{0}/{1}] Loss: {2}", i + 1, jump, sumLoss);
//Run truncated BPTT
if ((i + 1) % BPROP_LEN == 0)
{
for (int j = 0; backNdArrays.Count > 0; j++)
{
RILogManager.Default?.SendDebug("backward" + backNdArrays.Count);
model.Backward(true, backNdArrays.Pop());
}
model.Update();
model.ResetState();
}
if ((i + 1) % jump == 0)
{
epoch++;
RILogManager.Default?.SendDebug("evaluate");
dVal = Evaluate(model, validData);
RILogManager.Default?.SendDebug($"validation perplexity: {dVal}");
if (epoch >= 6)
{
sgd.LearningRate /= 1.2;
RILogManager.Default?.SendDebug("learning rate =" + sgd.LearningRate);
}
}
DocumentResults(accumulatingHistogram, recorder);
}
RILogManager.Default?.SendDebug("test start");
dVal = Evaluate(model, testData);
RILogManager.Default?.SendDebug("test perplexity:" + dVal);
DocumentResults(accumulatingHistogram, recorder);
_logWriter.Dispose();
_outputStream.Dispose();
RILogManager.Default?.SendDebug("Log contents");
RILogManager.Default?.SendDebug(File.ReadAllText(LogPath));
Console.WriteLine();
RILogManager.Default?.SendDebug("Percentile distribution (values reported in milliseconds)");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds, useCsvFormat: true);
RILogManager.Default?.SendDebug("Mean: " + BytesToString(accumulatingHistogram.GetMean()) + ", StdDev: " +
BytesToString(accumulatingHistogram.GetStdDeviation()));
}
public void TestMsFeatureScatterPlot(string path1, string path2, string pngPath)
{
// Convert relative paths to absolute paths
path1 = GetPath(path1);
path2 = GetPath(path2);
pngPath = GetPath(pngPath);
var fiOutput = new FileInfo(pngPath);
var didirectory = fiOutput.Directory;
if (didirectory == null)
{
throw new DirectoryNotFoundException(pngPath);
}
if (!didirectory.Exists)
{
didirectory.Create();
}
var aligner = new LcmsWarpFeatureAligner(new LcmsWarpAlignmentOptions());
var isosFilterOptions = new DeconToolsIsosFilterOptions();
var baselineMs = UmcLoaderFactory.LoadMsFeatureData(path1, isosFilterOptions);
var aligneeMs = UmcLoaderFactory.LoadMsFeatureData(path2, isosFilterOptions);
var finder = FeatureFinderFactory.CreateFeatureFinder(FeatureFinderType.TreeBased);
var tolerances = new FeatureTolerances
{
FragmentationWindowSize = .5,
Mass = 13,
DriftTime = .3,
Net = .01
};
var options = new LcmsFeatureFindingOptions(tolerances);
options.MaximumNetRange = .002;
var baseline = finder.FindFeatures(baselineMs, options, null);
var alignee = finder.FindFeatures(aligneeMs, options, null);
var alignmentResults = aligner.Align(baseline, alignee);
var plotModel1 = new PlotModel
{
Subtitle = "Interpolated, cartesian axes",
Title = "HeatMapSeries"
};
var palette = OxyPalettes.Hot(200);
var linearColorAxis1 = new LinearColorAxis
{
InvalidNumberColor = OxyColors.Gray,
Position = AxisPosition.Right,
Palette = palette
};
plotModel1.Axes.Add(linearColorAxis1);
// linearColorAxis1.
var linearAxis1 = new LinearAxis {
Position = AxisPosition.Bottom
};
plotModel1.Axes.Add(linearAxis1);
var linearAxis2 = new LinearAxis();
plotModel1.Axes.Add(linearAxis2);
var heatMapSeries1 = new HeatMapSeries
{
X0 = 0,
X1 = 1,
Y0 = 0,
Y1 = 1,
FontSize = .2
};
var scores = alignmentResults.HeatScores;
var width = scores.GetLength(0);
var height = scores.GetLength(1);
heatMapSeries1.Data = new double[width, height];
var seriesData = heatMapSeries1.Data;
for (var i = 0; i < width; i++)
{
for (var j = 0; j < height; j++)
{
seriesData[i, j] = Convert.ToDouble(scores[i, j]);
}
}
plotModel1.Series.Add(heatMapSeries1);
var svg = new SvgExporter();
var svgString = svg.ExportToString(plotModel1);
var xml = new XmlDocument();
xml.LoadXml(svgString);
var x = SvgDocument.Open(xml); // Svg.SvgDocument();
var bmp = x.Draw();
bmp.Save(pngPath);
var heatmap = HeatmapFactory.CreateAlignedHeatmap(alignmentResults.HeatScores, false);
var netHistogram = HistogramFactory.CreateHistogram(alignmentResults.NetErrorHistogram, "NET Error", "NET Error");
var massHistogram = HistogramFactory.CreateHistogram(alignmentResults.MassErrorHistogram, "Mass Error", "Mass Error (ppm)");
var baseName = Path.Combine(didirectory.FullName, Path.GetFileNameWithoutExtension(fiOutput.Name));
var encoder = new SvgEncoder();
PlotImageUtility.SaveImage(heatmap, baseName + "_heatmap.svg", encoder);
PlotImageUtility.SaveImage(netHistogram, baseName + "_netHistogram.svg", encoder);
PlotImageUtility.SaveImage(massHistogram, baseName + "_massHistogram.svg", encoder);
}
static void Main(string[] args)
{
//Comment out here if you want to run all on .Net Framework
//Weaver.Initialize(ComputeDeviceTypes.Gpu);
//Weaver.Initialize(ComputeDeviceTypes.Cpu, 0); //Subscript required if there are multiple devices
var recorder = HistogramFactory
.With64BitBucketSize()
?.WithValuesFrom(1)
?.WithValuesUpTo(2345678912345)
?.WithPrecisionOf(3)
?.WithThreadSafeWrites()
?.WithThreadSafeReads()
?.Create();
//Learning XOR with MLP
//recorder.Record(() => Test1.Run());
//Learning XOR with MLP 【Returned version】
//Test2.Run();
//Learning of Sin function by MLP
//recorder.Record(() => Test3.Run());
//Learning of MNIST (Handwritten Characters) by MLP
//Test4.Run();
//Reproduction of Excel CNN
//recorder.Record(() => Test5.Run());
//Learning of MNIST by 5-layer CNN
//recorder.Record(() => Test6.Run());
//Learning of MNIST by 15-tier MLP using BatchNorm
//Test7.Run();
//Learning of Sin function by LSTM
//Test8.Run();
//RNNLM with Simple RNN
//Test9.Run();
//RNNLM by LSTM
//Test10.Run();
//Decoupled Neural Interfaces using Synthetic Gradients by learning MNIST
//Test11.Run();
//DNI of Test 11 was defined as cDNI
//Test12.Run();
//Test of Deconvolution 2D(Winform)
//new Test13WinForm().ShowDialog();
//Concatenate Test 6 and execute
//Test14.Run();
//Test to read VGA 16 of Caffe model and classify images
//Test15.Run();
//Load and execute the same content as Test 5 of Chainer model
//Test16.Run();
//Test that reads RESNET of Caffe model and classifies images
//Test17.Run(Test17.ResnetModel.ResNet152); //Please select any Resnet model
//Learn CIFAR-10 with 5-layer CNN
//Test18.Run();
//Partial execution of Linear
//TestX.Run();
// LeakyReLu and PolynomialApproximantSteep combination network
//Test19.Run();
// 1000 layer neural network
//Test20.Run();
// MNIST (Handwritten Characters) by MLP Accuracy Tester 99.79 is goal
Test21.Run();
//benchmark
//SingleBenchmark.Run(true);
RILogManager.Default?.SendDebug("Test Done...");
Console.WriteLine("Test Complete, press any key to end");
Console.Read();
}
private static void TestNow(string relativeBaselinePath, string relativeAligneePath, string relativeOutput, string name)
{
string baselinePath = GetPath(relativeBaselinePath);
var aligneePath = GetPath(relativeAligneePath);
var aligner = new LcmsWarpFeatureAligner();
var rawBaselineData = File.ReadAllLines(baselinePath);
var rawFeaturesData = File.ReadAllLines(aligneePath);
var outputPath = GetOutputPath(relativeOutput);
var delimiter = new[] { TextDelimiter };
if (!Directory.Exists(outputPath))
{
Directory.CreateDirectory(outputPath);
}
var baseline = (from line in rawBaselineData
where line != ""
select line.Split(delimiter, StringSplitOptions.RemoveEmptyEntries)
into parsed
select new UMCLight
{
Net = Convert.ToDouble(parsed[0]),
ChargeState = Convert.ToInt32(parsed[1]),
Mz = Convert.ToDouble(parsed[2]),
Scan = Convert.ToInt32(parsed[3]),
MassMonoisotopic = Convert.ToDouble(parsed[4]),
MassMonoisotopicAligned = Convert.ToDouble(parsed[5]),
Id = Convert.ToInt32(parsed[6]),
ScanStart = Convert.ToInt32(parsed[7]),
ScanEnd = Convert.ToInt32(parsed[8]),
ScanAligned = Convert.ToInt32(parsed[9])
}).ToList();
var features = (from line in rawFeaturesData
where line != ""
select line.Split(delimiter, StringSplitOptions.RemoveEmptyEntries)
into parsed
select new UMCLight
{
Net = Convert.ToDouble(parsed[0]),
ChargeState = Convert.ToInt32(parsed[1]),
Mz = Convert.ToDouble(parsed[2]),
Scan = Convert.ToInt32(parsed[3]),
MassMonoisotopic = Convert.ToDouble(parsed[4]),
MassMonoisotopicAligned = Convert.ToDouble(parsed[5]),
Id = Convert.ToInt32(parsed[6]),
ScanStart = Convert.ToInt32(parsed[7]),
ScanEnd = Convert.ToInt32(parsed[8]),
ScanAligned = Convert.ToInt32(parsed[9])
}).ToList();
var maxd = features.Max(x => x.Net);
var mind = features.Min(x => x.Net);
if (maxd - mind < double.Epsilon)
{
throw new Exception("There is something wrong with the features NET values");
}
aligner.Options.AlignType = AlignmentType.NET_MASS_WARP;
var outputData = aligner.Align(baseline, features);
var residuals = outputData.ResidualData;
using (var writer = new StreamWriter(@"E:\MultiAlignTest\Results\heat-scores-old.txt"))
{
foreach (var oovarscore in outputData.heatScores)
{
writer.WriteLine(oovarscore);
}
}
var heatmap = HeatmapFactory.CreateAlignedHeatmap(outputData.heatScores);
var netHistogram = HistogramFactory.CreateHistogram(outputData.netErrorHistogram, "NET Error Histogram", "NET Error");
var massHistogram = HistogramFactory.CreateHistogram(outputData.massErrorHistogram, "Mass Error Histogram", "Mass Error (ppm)");
var netResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Scan, residuals.LinearCustomNet,
residuals.LinearNet, "NET Residuals", "Scans", "NET");
var massMzResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Mz, residuals.MzMassError,
residuals.MzMassErrorCorrected, "Mass Residuals", "m/z", "Mass Errors");
var massScanResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Scan, residuals.MzMassError,
residuals.MzMassErrorCorrected, "Mass Residuals", "Scan", "Mass Errors");
var directory = Path.Combine(outputPath, name);
var encoder = new SvgEncoder();
PlotImageUtility.SaveImage(heatmap, directory + "_heatmap.svg", encoder);
PlotImageUtility.SaveImage(netResidual, directory + "_netResidual.svg", encoder);
PlotImageUtility.SaveImage(massMzResidual, directory + "_massMzResidual.svg", encoder);
PlotImageUtility.SaveImage(massScanResidual, directory + "_massScanResidual.svg", encoder);
//PlotImageUtility.SaveImage(netHistogram, directory + "_netHistogram.svg", encoder);
PlotImageUtility.SaveImage(massHistogram, directory + "_massHistogram.svg", encoder);
}
/// <summary>
/// Creates alignment plots for the HTML output.
/// </summary>
public void CreateAlignmentPlots(FeaturesAlignedEventArgs e)
{
var name = e.AligneeDatasetInformation.DatasetName;
var alignmentData = e.AlignmentData;
if (alignmentData == null)
{
return;
}
var directory = Path.Combine(Config.AnalysisPath, PlotPath, name);
var heatmap = HeatmapFactory.CreateAlignedHeatmap(alignmentData.HeatScores, alignmentData.BaselineIsAmtDB);
var feature = ScatterPlotFactory.CreateFeatureMassScatterPlot(e.AligneeFeatures);
var netHistogram = HistogramFactory.CreateHistogram(alignmentData.NetErrorHistogram, "NET Error", "NET Error");
var massHistogram = HistogramFactory.CreateHistogram(alignmentData.MassErrorHistogram, "Mass Error", "Mass Error (ppm)");
var residuals = alignmentData.ResidualData;
var netResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Net, residuals.LinearCustomNet,
residuals.LinearNet, "NET Residuals", "Scans", "NET");
var massMzResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Mz, residuals.MzMassError,
residuals.MzMassErrorCorrected, "Mass Residuals", "m/z", "Mass Errors");
var massScanResidual = ScatterPlotFactory.CreateResidualPlot(residuals.Net, residuals.MzMassError,
residuals.MzMassErrorCorrected, "Mass Residuals", "Scan", "Mass Errors");
var report = Config.Report;
report.PushLargeText(string.Format("Alignee Statistics - {0}", name));
report.PushStartTable();
// Features
report.PushStartTableRow();
report.PushTextHeader("Features");
report.PushEndTableRow();
report.PushStartTableRow();
Config.Report.PushImageColumn(directory + "_features.png", WIDTH, HEIGHT);
Config.Report.PushImageColumn(directory + "_heatmap.png", WIDTH, HEIGHT);
PlotImageUtility.SaveImage(feature, directory + "_features.png");
PlotImageUtility.SaveImage(heatmap, directory + "_heatmap.png");
report.PushEndTableRow();
// Histograms
report.PushStartTableRow();
report.PushTextHeader("Histograms");
report.PushEndTableRow();
report.PushStartTableRow();
Config.Report.PushImageColumn(directory + "_netHistogram.png");
Config.Report.PushImageColumn(directory + "_massHistogram.png");
PlotImageUtility.SaveImage(netHistogram, directory + "_netHistogram.png");
PlotImageUtility.SaveImage(massHistogram, directory + "_massHistogram.png");
report.PushEndTableRow();
// Residuals
report.PushStartTableRow();
report.PushTextHeader("Histograms");
report.PushEndTableRow();
report.PushStartTableRow();
Config.Report.PushImageColumn(directory + "_netResidual.png");
Config.Report.PushImageColumn(directory + "_massMzResidual.png");
Config.Report.PushImageColumn(directory + "_massScanResidual.png");
PlotImageUtility.SaveImage(netResidual, directory + "_netResidual.png");
PlotImageUtility.SaveImage(massMzResidual, directory + "_massMzResidual.png");
PlotImageUtility.SaveImage(massScanResidual, directory + "_massScanResidual.png");
report.PushEndTableRow();
report.PushEndTable();
}
