public void TestGetEstimatedFootprintInBytes2()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var largestValueWithSingleUnitResolution = 2 * (long)Math.Pow(10, NumberOfSignificantValueDigits);
var subBucketCountMagnitude = (int)Math.Ceiling(Math.Log(largestValueWithSingleUnitResolution) / Math.Log(2));
var subBucketSize = (int)Math.Pow(2, (subBucketCountMagnitude));
var bucketCount = GetBucketsNeededToCoverValue(subBucketSize, HighestTrackableValue);
var header = 512;
var width = sizeof (long);
var length = (bucketCount + 1) * (subBucketSize / 2);
var expectedSize = header + (width * length);
Assert.AreEqual(expectedSize, longHistogram.GetEstimatedFootprintInBytes());
}
public void TestGetEstimatedFootprintInBytes2()
{
var longHistogram = new LongHistogram(HighestTrackableValue, NumberOfSignificantValueDigits);
var largestValueWithSingleUnitResolution = 2 * (long)Math.Pow(10, NumberOfSignificantValueDigits);
var subBucketCountMagnitude = (int)Math.Ceiling(Math.Log(largestValueWithSingleUnitResolution) / Math.Log(2));
var subBucketSize = (int)Math.Pow(2, (subBucketCountMagnitude));
var bucketCount = GetBucketsNeededToCoverValue(subBucketSize, HighestTrackableValue);
var header = 512;
var width = sizeof(long);
var length = (bucketCount + 1) * (subBucketSize / 2);
var expectedSize = header + (width * length);
Assert.AreEqual(expectedSize, longHistogram.GetEstimatedFootprintInBytes());
}
/// <summary>
/// Write to the console the memory footprint of the histogram instance and
/// the percentile distribution of all the recorded values.
/// </summary>
private static void OutputMeasurements()
{
var size = Histogram.GetEstimatedFootprintInBytes();
Console.WriteLine("Histogram size = {0} bytes ({1:F2} MB)", size, size / 1024.0 / 1024.0);
Console.WriteLine("Recorded latencies [in system clock ticks] for Create+Close of a DatagramSocket:");
Histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.None);
Console.WriteLine();
Console.WriteLine("Recorded latencies [in usec] for Create+Close of a DatagramSocket:");
Histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMicroseconds);
Console.WriteLine();
Console.WriteLine("Recorded latencies [in msec] for Create+Close of a DatagramSocket:");
Histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds);
Console.WriteLine();
Console.WriteLine("Recorded latencies [in sec] for Create+Close of a DatagramSocket:");
Histogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToSeconds);
}
public static void Run()
{
_outputStream = File.Create(LogPath);
_logWriter = new HistogramLogWriter(_outputStream);
_logWriter.Write(DateTime.Now);
var recorder = HistogramFactory
.With64BitBucketSize()
?.WithValuesFrom(1)
?.WithValuesUpTo(2345678912345)
?.WithPrecisionOf(3)
?.WithThreadSafeWrites()
?.WithThreadSafeReads()
?.Create();
var accumulatingHistogram = new LongHistogram(2345678912345, 3);
var size = accumulatingHistogram.GetEstimatedFootprintInBytes();
RILogManager.Default?.SendDebug("Histogram size = {0} bytes ({1:F2} MB)", size, size / 1024.0 / 1024.0);
RILogManager.Default?.SendDebug("Recorded latencies [in system clock ticks]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.None, useCsvFormat: true);
Console.WriteLine();
RILogManager.Default?.SendDebug("Recorded latencies [in usec]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMicroseconds, useCsvFormat: true);
Console.WriteLine();
RILogManager.Default?.SendDebug("Recorded latencies [in msec]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds, useCsvFormat: true);
Console.WriteLine();
RILogManager.Default?.SendDebug("Recorded latencies [in sec]");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToSeconds, useCsvFormat: true);
DocumentResults(accumulatingHistogram, recorder);
RILogManager.Default?.SendDebug("Build Vocabulary.");
DocumentResults(accumulatingHistogram, recorder);
Vocabulary vocabulary = new Vocabulary();
DocumentResults(accumulatingHistogram, recorder);
string trainPath = InternetFileDownloader.Download(DOWNLOAD_URL + TRAIN_FILE, TRAIN_FILE);
DocumentResults(accumulatingHistogram, recorder);
string validPath = InternetFileDownloader.Download(DOWNLOAD_URL + VALID_FILE, VALID_FILE);
DocumentResults(accumulatingHistogram, recorder);
string testPath = InternetFileDownloader.Download(DOWNLOAD_URL + TEST_FILE, TEST_FILE);
DocumentResults(accumulatingHistogram, recorder);
int[] trainData = vocabulary.LoadData(trainPath);
DocumentResults(accumulatingHistogram, recorder);
int[] validData = vocabulary.LoadData(validPath);
DocumentResults(accumulatingHistogram, recorder);
int[] testData = vocabulary.LoadData(testPath);
DocumentResults(accumulatingHistogram, recorder);
int nVocab = vocabulary.Length;
RILogManager.Default?.SendDebug("Network Initializing.");
FunctionStack model = new FunctionStack("Test10",
new EmbedID(nVocab, N_UNITS, name: "l1 EmbedID"),
new Dropout(),
new LSTM(true, N_UNITS, N_UNITS, name: "l2 LSTM"),
new Dropout(),
new LSTM(true, N_UNITS, N_UNITS, name: "l3 LSTM"),
new Dropout(),
new Linear(true, N_UNITS, nVocab, name: "l4 Linear")
);
DocumentResults(accumulatingHistogram, recorder);
// Do not cease at the given threshold, correct the rate by taking the rate from L2Norm of all parameters
GradientClipping gradientClipping = new GradientClipping(threshold: GRAD_CLIP);
SGD sgd = new SGD(learningRate: 1);
model.SetOptimizer(gradientClipping, sgd);
DocumentResults(accumulatingHistogram, recorder);
Real wholeLen = trainData.Length;
int jump = (int)Math.Floor(wholeLen / BATCH_SIZE);
int epoch = 0;
Stack <NdArray[]> backNdArrays = new Stack <NdArray[]>();
RILogManager.Default?.SendDebug("Train Start.");
double dVal;
NdArray x = new NdArray(new[] { 1 }, BATCH_SIZE, (Function)null);
NdArray t = new NdArray(new[] { 1 }, BATCH_SIZE, (Function)null);
for (int i = 0; i < jump * N_EPOCH; i++)
{
for (int j = 0; j < BATCH_SIZE; j++)
{
x.Data[j] = trainData[(int)((jump * j + i) % wholeLen)];
t.Data[j] = trainData[(int)((jump * j + i + 1) % wholeLen)];
}
NdArray[] result = model.Forward(true, x);
Real sumLoss = new SoftmaxCrossEntropy().Evaluate(result, t);
backNdArrays.Push(result);
RILogManager.Default?.SendDebug("[{0}/{1}] Loss: {2}", i + 1, jump, sumLoss);
//Run truncated BPTT
if ((i + 1) % BPROP_LEN == 0)
{
for (int j = 0; backNdArrays.Count > 0; j++)
{
RILogManager.Default?.SendDebug("backward" + backNdArrays.Count);
model.Backward(true, backNdArrays.Pop());
}
model.Update();
model.ResetState();
}
if ((i + 1) % jump == 0)
{
epoch++;
RILogManager.Default?.SendDebug("evaluate");
dVal = Evaluate(model, validData);
RILogManager.Default?.SendDebug($"validation perplexity: {dVal}");
if (epoch >= 6)
{
sgd.LearningRate /= 1.2;
RILogManager.Default?.SendDebug("learning rate =" + sgd.LearningRate);
}
}
DocumentResults(accumulatingHistogram, recorder);
}
RILogManager.Default?.SendDebug("test start");
dVal = Evaluate(model, testData);
RILogManager.Default?.SendDebug("test perplexity:" + dVal);
DocumentResults(accumulatingHistogram, recorder);
_logWriter.Dispose();
_outputStream.Dispose();
RILogManager.Default?.SendDebug("Log contents");
RILogManager.Default?.SendDebug(File.ReadAllText(LogPath));
Console.WriteLine();
RILogManager.Default?.SendDebug("Percentile distribution (values reported in milliseconds)");
accumulatingHistogram.OutputPercentileDistribution(Console.Out, outputValueUnitScalingRatio: OutputScalingFactor.TimeStampToMilliseconds, useCsvFormat: true);
RILogManager.Default?.SendDebug("Mean: " + BytesToString(accumulatingHistogram.GetMean()) + ", StdDev: " +
BytesToString(accumulatingHistogram.GetStdDeviation()));
}
