public void AddDataIncreasesUpperBound() { var h = new Histogram(new[] { 1.0, 5.0, 10.0 }, 2); h.AddData(20.0); Assert.AreEqual(2, h[1].Count); }
public void AddDataDecreasesLowerBound() { var h = new Histogram(new[] { 1.0, 5.0, 10.0 }, 2); h.AddData(0.0); Assert.AreEqual(3, h[0].Count); }
public void CanAddDataSingle() { var h = new Histogram(new[] { 1.0, 5.0, 10.0 }, 2); h.AddData(7.0); Assert.AreEqual(2, h[1].Count); }
public void CanAddDataList() { var h = new Histogram(new[] { 1.0, 5.0, 10.0 }, 2); h.AddData(new[] { 7.0, 8.0 }); Assert.AreEqual(3, h[1].Count); }
public void AddDataEqualToLowerBound() { var h = new Histogram(new[] { 1.0, 5.0, 10.0 }, 3, 0.0, 10.0); Assert.DoesNotThrow(() => h.AddData(0.0)); Assert.AreEqual(2, h[0].Count); }
public void buttonAddValue_Click(object sender, EventArgs e) { int dataV = 0; string inValue; inValue = textBoxDataV.Text; dataV = int.Parse(inValue); one.AddData(dataV); //using the object }
private void MakeOneAnalysis([NotNull] List <double> areas, int bucketSize, [NotNull] string baseName, [NotNull] string sectionDescription, [NotNull] ScenarioSliceParameters slice) { { var maxArea = areas.Max(); var histogram = new Histogram(); for (var i = 0; i < bucketSize * 10; i += bucketSize) { histogram.AddBucket(new Bucket(i, i + bucketSize)); } if (maxArea > bucketSize * 10) { histogram.AddBucket(new Bucket(bucketSize * 10, maxArea)); } histogram.AddData(areas); string histogramfilename = MakeAndRegisterFullFilename(baseName + "_Histogram.png", Name, sectionDescription, slice); var bs = BarSeriesEntry.MakeBarSeriesEntry(histogram, out var colNames); var bss = new List <BarSeriesEntry> { bs }; Services.PlotMaker.MakeBarChart(histogramfilename, "Anzahl von Haushalten mit Fläche in diesem Bereich", bss, colNames); } { string dstFileName2 = MakeAndRegisterFullFilename(baseName + "_SortedAreas.png", Name, sectionDescription, slice); var lse = new LineSeriesEntry("Sorted"); var sorted = areas.ToList(); sorted.Sort(); for (var i = 0; i < sorted.Count; i++) { lse.Values.Add(new Point(i, sorted[i])); } var lss = new List <LineSeriesEntry> { lse }; Services.PlotMaker.MakeLineChart(dstFileName2, "", lss, new List <PlotMaker.AnnotationEntry>()); } { string dstFileName2 = MakeAndRegisterFullFilename(baseName + "_Kumulativ.png", Name, sectionDescription, slice); var lse = new LineSeriesEntry("Kumulativ"); var sorted = areas.ToList(); sorted.Sort(); double tempSum = 0; for (var i = 0; i < sorted.Count; i++) { tempSum += sorted[i]; lse.Values.Add(new Point(i, tempSum)); } var lss = new List <LineSeriesEntry> { lse }; Services.PlotMaker.MakeLineChart(dstFileName2, "", lss, new List <PlotMaker.AnnotationEntry>()); } }
// NOTE: _numbers 를 사용하게 되면, Parallel이 불가능하기 때문에, numbers를 인자로 받고, Thread-Safe 하게 사용할 수 있도록 했다. // private static void DisplayDistribution(string name, Histogram histogram, double[] numbers) { log.Debug(@"-------------------------------------------"); histogram.ResetData(); histogram.AddData(numbers); log.Debug(@"{0} Histogram:\r\n{1}", name, histogram.StemLeaf(50)); double avg, stdev; numbers.AverageAndStDev(out avg, out stdev); log.Debug(@"Avg = {0} ; StDev = {1}", avg, stdev); log.Debug(@"-------------------------------------------"); log.Debug(string.Empty); }
public void CanAddDataList() { var h = new Histogram(new[] {1.0, 5.0, 10.0}, 2); h.AddData(new[] {7.0, 8.0}); Assert.AreEqual(3, h[1].Count); }
public void AddDataIncreasesUpperBound() { var h = new Histogram(new[] {1.0, 5.0, 10.0}, 2); h.AddData(20.0); Assert.AreEqual(2, h[1].Count); }
public void AddDataDecreasesLowerBound() { var h = new Histogram(new[] {1.0, 5.0, 10.0}, 2); h.AddData(0.0); Assert.AreEqual(3, h[0].Count); }
public void CanAddDataSingle() { var h = new Histogram(new[] {1.0, 5.0, 10.0}, 2); h.AddData(7.0); Assert.AreEqual(2, h[1].Count); }
public static void Run() { RILogManager.Default?.SendDebug("MNIST Data Loading..."); MnistData mnistData = new MnistData(28); RILogManager.Default?.SendDebug("Training Start..."); int neuronCount = 28; FunctionStack nn = new FunctionStack("Test19", new Linear(true, neuronCount * neuronCount, N, name: "l1 Linear"), // L1 new BatchNormalization(true, N, name: "l1 BatchNorm"), new LeakyReLU(slope: 0.000001, name: "l1 LeakyReLU"), new Linear(true, N, N, name: "l2 Linear"), // L2 new BatchNormalization(true, N, name: "l2 BatchNorm"), new LeakyReLU(slope: 0.000001, name: "l2 LeakyReLU"), new Linear(true, N, N, name: "l3 Linear"), // L3 new BatchNormalization(true, N, name: "l3 BatchNorm"), new LeakyReLU(slope: 0.000001, name: "l3 LeakyReLU"), new Linear(true, N, N, name: "l4 Linear"), // L4 new BatchNormalization(true, N, name: "l4 BatchNorm"), new LeakyReLU(slope: 0.000001, name: "l4 LeakyReLU"), new Linear(true, N, N, name: "l5 Linear"), // L5 new BatchNormalization(true, N, name: "l5 BatchNorm"), new LeakyReLU(slope: 0.000001, name: "l5 LeakyReLU"), new Linear(true, N, N, name: "l6 Linear"), // L6 new BatchNormalization(true, N, name: "l6 BatchNorm"), new LeakyReLU(slope: 0.000001, name: "l6 LeakyReLU"), new Linear(true, N, N, name: "l7 Linear"), // L7 new BatchNormalization(true, N, name: "l7 BatchNorm"), new LeakyReLU(slope: 0.000001, name: "l7 ReLU"), new Linear(true, N, N, name: "l8 Linear"), // L8 new BatchNormalization(true, N, name: "l8 BatchNorm"), new LeakyReLU(slope: 0.000001, name: "l8 LeakyReLU"), new Linear(true, N, N, name: "l9 Linear"), // L9 new BatchNormalization(true, N, name: "l9 BatchNorm"), new PolynomialApproximantSteep(slope: 0.000001, name: "l9 PolynomialApproximantSteep"), new Linear(true, N, N, name: "l10 Linear"), // L10 new BatchNormalization(true, N, name: "l10 BatchNorm"), new PolynomialApproximantSteep(slope: 0.000001, name: "l10 PolynomialApproximantSteep"), new Linear(true, N, N, name: "l11 Linear"), // L11 new BatchNormalization(true, N, name: "l11 BatchNorm"), new PolynomialApproximantSteep(slope: 0.000001, name: "l11 PolynomialApproximantSteep"), new Linear(true, N, N, name: "l12 Linear"), // L12 new BatchNormalization(true, N, name: "l12 BatchNorm"), new PolynomialApproximantSteep(slope: 0.000001, name: "l12 PolynomialApproximantSteep"), new Linear(true, N, N, name: "l13 Linear"), // L13 new BatchNormalization(true, N, name: "l13 BatchNorm"), new PolynomialApproximantSteep(slope: 0.000001, name: "l13 PolynomialApproximantSteep"), new Linear(true, N, N, name: "l14 Linear"), // L14 new BatchNormalization(true, N, name: "l14 BatchNorm"), new PolynomialApproximantSteep(slope: 0.000001, name: "l14 PolynomialApproximantSteep"), new Linear(true, N, 10, name: "l15 Linear") // L15 ); nn.SetOptimizer(new AdaGrad()); //nn.SetOptimizer(new Adam()); RunningStatistics stats = new RunningStatistics(); Histogram lossHistogram = new Histogram(); Histogram accuracyHistogram = new Histogram(); Real totalLoss = 0; long totalLossCounter = 0; Real highestAccuracy = 0; Real bestLocalLoss = 0; Real bestTotalLoss = 0; // First skeleton save ModelIO.Save(nn, nn.Name); for (int epoch = 0; epoch < 1; epoch++) { RILogManager.Default?.SendDebug("epoch " + (epoch + 1)); RILogManager.Default?.ViewerSendWatch("epoch", (epoch + 1)); for (int i = 1; i < TRAIN_DATA_COUNT + 1; i++) { RILogManager.Default?.SendInformation("batch count " + i + "/" + TRAIN_DATA_COUNT); TestDataSet datasetX = mnistData.GetRandomXSet(BATCH_DATA_COUNT, 28, 28); Real sumLoss = Trainer.Train(nn, datasetX.Data, datasetX.Label, new SoftmaxCrossEntropy()); totalLoss += sumLoss; totalLossCounter++; stats.Push(sumLoss); lossHistogram.AddBucket(new Bucket(-10, 10)); accuracyHistogram.AddBucket(new Bucket(-10.0, 10)); if (sumLoss < bestLocalLoss && sumLoss != Double.NaN) { bestLocalLoss = sumLoss; } if (stats.Mean < bestTotalLoss && sumLoss != Double.NaN) { bestTotalLoss = stats.Mean; } try { lossHistogram.AddData(sumLoss); } catch (Exception) { } if (i % 20 == 0) { RILogManager.Default?.SendDebug("\nbatch count " + i + "/" + TRAIN_DATA_COUNT); RILogManager.Default?.SendDebug("Total/Mean loss " + stats.Mean); RILogManager.Default?.SendDebug("local loss " + sumLoss); RILogManager.Default?.SendInformation("batch count " + i + "/" + TRAIN_DATA_COUNT); RILogManager.Default?.ViewerSendWatch("batch count", i); RILogManager.Default?.ViewerSendWatch("Total/Mean loss", stats.Mean); RILogManager.Default?.ViewerSendWatch("local loss", sumLoss); RILogManager.Default?.SendDebug(""); RILogManager.Default?.SendDebug("Testing..."); TestDataSet datasetY = mnistData.GetRandomYSet(TEST_DATA_COUNT, 28); Real accuracy = Trainer.Accuracy(nn, datasetY.Data, datasetY.Label); if (accuracy > highestAccuracy) { highestAccuracy = accuracy; } RILogManager.Default?.SendDebug("Accuracy: " + accuracy); RILogManager.Default?.ViewerSendWatch("Accuracy", accuracy); try { accuracyHistogram.AddData(accuracy); } catch (Exception) { } } } } RILogManager.Default?.SendDebug("Best Accuracy: " + highestAccuracy); RILogManager.Default?.SendDebug("Best Total Loss " + bestTotalLoss); RILogManager.Default?.SendDebug("Best Local Loss " + bestLocalLoss); RILogManager.Default?.ViewerSendWatch("Best Accuracy:", highestAccuracy); RILogManager.Default?.ViewerSendWatch("Best Total Loss", bestTotalLoss); RILogManager.Default?.ViewerSendWatch("Best Local Loss", bestLocalLoss); // Save all with training data ModelIO.Save(nn, nn.Name); }
protected override void MakeVisualization([NotNull] ScenarioSliceParameters slice, bool isPresent) { var dbHouse = Services.SqlConnectionPreparer.GetDatabaseConnection(Stage.Houses, slice); var houses = dbHouse.Fetch <House>(); var households = dbHouse.Fetch <Household>(); if (households.Count == 0) { throw new FlaException("Not a single household"); } MakeAgeHistogram(); MakePeopleCountMap2(); MakeFamilySizeSankey(); MakePeopleCountMap(); MakeAgeMap(); void MakeAgeHistogram() { List <Occupant> occupants = households.SelectMany(x => x.Occupants).ToList(); double maxAge = occupants.Max(x => x.Age); var histogram = new Histogram(); const int bucketSize = 5; for (var i = 0; i < maxAge; i += bucketSize) { histogram.AddBucket(new Bucket(i, i + bucketSize)); } var ages = occupants.Select(x => (double)x.Age).ToList(); histogram.AddData(ages); var filename = MakeAndRegisterFullFilename("Altersverteilung.png", slice); var bs = BarSeriesEntry.MakeBarSeriesEntry(histogram, out var colNames); var bss = new List <BarSeriesEntry> { bs }; Services.PlotMaker.MakeBarChart(filename, "Anzahl von Personen in diesem Altersbereich", bss, colNames); } void MakeFamilySizeSankey() { var ssa = new SingleSankeyArrow("Households", 1000, MyStage, SequenceNumber, Name, slice, Services); ssa.AddEntry(new SankeyEntry("Households", houses.Count, 5000, Orientation.Straight)); var counts = households.Select(x => x.Occupants.Count).ToList(); var maxSize = counts.Max(); var filename = MakeAndRegisterFullFilename("HouseholdSizeHistogram.png", slice); var names = new List <string>(); var barSeries = new List <BarSeriesEntry>(); for (var i = 0; i < maxSize + 1; i++) { names.Add(i.ToString()); var j = i; // because of closure var count = counts.Count(x => x == j); barSeries.Add(BarSeriesEntry.MakeBarSeriesEntry(i + "Personen", count, i)); } Services.PlotMaker.MakeBarChart(filename, "HouseholdSizeHistogram", barSeries, names); } void MakePeopleCountMap() { RGB GetColor(House h) { var hhs = households.Where(x => x.HouseGuid == h.Guid).ToList(); var peopleCount = hhs.Select(x => x.Occupants.Count).Sum(); if (peopleCount == 0) { return(new RGB(128, 128, 128)); } if (peopleCount == 1) { return(Constants.Green); } if (peopleCount == 2) { return(Constants.Blue); } return(Constants.Red); } var mapPoints = houses.Select(x => x.GetMapPoint(GetColor)).ToList(); var filename = MakeAndRegisterFullFilename("AnzahlPersonenProHaushalt.svg", slice); var legendEntries = new List <MapLegendEntry> { new MapLegendEntry("Keine Einwohner", new RGB(128, 128, 128)), new MapLegendEntry("Ein Einwohner", Constants.Green), new MapLegendEntry("Zwei Einwohner", Constants.Blue), new MapLegendEntry("Drei+ Einwohner", Constants.Red) }; Services.PlotMaker.MakeMapDrawer(filename, Name, mapPoints, legendEntries); } void MakePeopleCountMap2() { var maxCount = 0; foreach (var house in houses) { var hhs = households.Where(x => x.HouseGuid == house.Guid).ToList(); var sumCount = hhs.Sum(x => x.Occupants.Count); if (maxCount < sumCount) { maxCount = sumCount; } } var colorStep = 250.0 / maxCount; double color = 0; var colorDict = new Dictionary <int, int>(); for (var i = 1; i <= maxCount; i++) { colorDict.Add(i, (int)color); color += colorStep; } RGBWithSize GetColor(House h) { var hhs = households.Where(x => x.HouseGuid == h.Guid).ToList(); var peopleCount = hhs.Select(x => x.Occupants.Count).Sum(); if (peopleCount == 0) { return(new RGBWithSize(128, 128, 128, 10)); } var red = colorDict[peopleCount]; var size = 10; if (peopleCount > size) { size = peopleCount; } return(new RGBWithSize(red, 0, 0, size)); } var mapPoints = houses.Select(x => x.GetMapPointWithSize(GetColor)).ToList(); var filename = MakeAndRegisterFullFilename("AnzahlPersonenProHaushaltRelative.svg", slice); var legendEntries = new List <MapLegendEntry> { new MapLegendEntry("Keine Einwohner", new RGB(128, 128, 128)) }; Services.PlotMaker.MakeMapDrawer(filename, Name, mapPoints, legendEntries); } void MakeAgeMap() { RGB GetColor(House h) { var hhs = households.Where(x => x.HouseGuid == h.Guid).ToList(); var houseoccupants = hhs.SelectMany(x => x.Occupants).ToList(); double averageAge = 0; if (houseoccupants.Count > 0) { averageAge = houseoccupants.Average(x => x.Age); } if (houseoccupants.Count == 0) { return(new RGB(128, 128, 128)); } var agefactor = averageAge / 100; if (agefactor > 1) { agefactor = 1; } return(new RGB((int)(255.0 * agefactor), 0, 0)); } var mapPoints = houses.Select(x => x.GetMapPoint(GetColor)).ToList(); var filename = MakeAndRegisterFullFilename("Durchschnittsalter.svg", slice); var legendEntries = new List <MapLegendEntry> { new MapLegendEntry("Keine Einwohner", new RGB(128, 128, 128)), new MapLegendEntry("0 Jahre Durchschnittsalter", new RGB(0, 0, 0)), new MapLegendEntry("100 Jahre Durchschnittsalter", new RGB(255, 0, 0)) }; Services.PlotMaker.MakeMapDrawer(filename, Name, mapPoints, legendEntries); } }
public static void Run() { int neuronCount = 28; RILogManager.Default?.SendDebug("MNIST Data Loading..."); MnistData mnistData = new MnistData(neuronCount); RILogManager.Default.SendInformation("Training Start, creating function stack."); SortedFunctionStack nn = new SortedFunctionStack(); SortedList <Function> functions = new SortedList <Function>(); ParallelOptions po = new ParallelOptions(); po.MaxDegreeOfParallelism = 4; for (int x = 0; x < numLayers; x++) { Application.DoEvents(); functions.Add(new Linear(true, neuronCount * neuronCount, N, name: $"l{x} Linear")); functions.Add(new BatchNormalization(true, N, name: $"l{x} BatchNorm")); functions.Add(new ReLU(name: $"l{x} ReLU")); RILogManager.Default.ViewerSendWatch("Total Layers", (x + 1)); } ; RILogManager.Default.SendInformation("Adding Output Layer"); Application.DoEvents(); nn.Add(new Linear(true, N, 10, noBias: false, name: $"l{numLayers + 1} Linear")); RILogManager.Default.ViewerSendWatch("Total Layers", numLayers); RILogManager.Default.SendInformation("Setting Optimizer to AdaGrad"); nn.SetOptimizer(new AdaGrad()); Application.DoEvents(); RunningStatistics stats = new RunningStatistics(); Histogram lossHistogram = new Histogram(); Histogram accuracyHistogram = new Histogram(); Real totalLoss = 0; long totalLossCounter = 0; Real highestAccuracy = 0; Real bestLocalLoss = 0; Real bestTotalLoss = 0; for (int epoch = 0; epoch < 3; epoch++) { RILogManager.Default?.SendDebug("epoch " + (epoch + 1)); RILogManager.Default.SendInformation("epoch " + (epoch + 1)); RILogManager.Default.ViewerSendWatch("epoch", (epoch + 1)); Application.DoEvents(); for (int i = 1; i < TRAIN_DATA_COUNT + 1; i++) { Application.DoEvents(); TestDataSet datasetX = mnistData.GetRandomXSet(BATCH_DATA_COUNT, neuronCount, neuronCount); Real sumLoss = Trainer.Train(nn, datasetX.Data, datasetX.Label, new SoftmaxCrossEntropy()); totalLoss += sumLoss; totalLossCounter++; stats.Push(sumLoss); lossHistogram.AddBucket(new Bucket(-10, 10)); accuracyHistogram.AddBucket(new Bucket(-10.0, 10)); if (sumLoss < bestLocalLoss && !double.IsNaN(sumLoss)) { bestLocalLoss = sumLoss; } if (stats.Mean < bestTotalLoss && !double.IsNaN(sumLoss)) { bestTotalLoss = stats.Mean; } try { lossHistogram.AddData(sumLoss); } catch (Exception) { } if (i % 20 == 0) { RILogManager.Default.ViewerSendWatch("Batch Count ", i); RILogManager.Default.ViewerSendWatch("Total/Mean loss", stats.Mean); RILogManager.Default.ViewerSendWatch("Local loss", sumLoss); RILogManager.Default.SendInformation("Batch Count " + i + "/" + TRAIN_DATA_COUNT + ", epoch " + epoch + 1); RILogManager.Default.SendInformation("Total/Mean loss " + stats.Mean); RILogManager.Default.SendInformation("Local loss " + sumLoss); Application.DoEvents(); RILogManager.Default?.SendDebug("Testing..."); TestDataSet datasetY = mnistData.GetRandomYSet(TEST_DATA_COUNT, 28); Real accuracy = Trainer.Accuracy(nn, datasetY?.Data, datasetY.Label); if (accuracy > highestAccuracy) { highestAccuracy = accuracy; } RILogManager.Default?.SendDebug("Accuracy: " + accuracy); RILogManager.Default.ViewerSendWatch("Best Accuracy: ", highestAccuracy); RILogManager.Default.ViewerSendWatch("Best Total Loss ", bestTotalLoss); RILogManager.Default.ViewerSendWatch("Best Local Loss ", bestLocalLoss); Application.DoEvents(); try { accuracyHistogram.AddData(accuracy); } catch (Exception) { } } } } ModelIO.Save(nn, Application.StartupPath + "\\test20.nn"); RILogManager.Default?.SendDebug("Best Accuracy: " + highestAccuracy); RILogManager.Default?.SendDebug("Best Total Loss " + bestTotalLoss); RILogManager.Default?.SendDebug("Best Local Loss " + bestLocalLoss); RILogManager.Default.ViewerSendWatch("Best Accuracy: ", highestAccuracy); RILogManager.Default.ViewerSendWatch("Best Total Loss ", bestTotalLoss); RILogManager.Default.ViewerSendWatch("Best Local Loss ", bestLocalLoss); }