/// <summary>
/// Constructor
/// </summary>
/// <param name="baseFolder">Base folder with MNIST dataset files</param>
/// <param name="imagesCnt">How many images to load</param>
/// <param name="exact">If TRUE, you will get exactly imagesCnt images. If FALSE, you will get AT MOST imagesCnt images. It's here for performance reason and BC</param>
/// <param name="afterLastImage">What to do, after the last values has been sent</param>
public MyMNISTManager(string baseFolder, int trainingExamplesPerDigitCnt = int.MaxValue, int testExamplesPerDigitCnt = int.MaxValue,
bool exact = false, MNISTLastImageMethod afterLastImage = MNISTLastImageMethod.ResetToStart)
{
m_baseFolder = baseFolder;
m_trainingImages = new ArrayList();
m_testImages = new ArrayList();
m_afterLastImage = afterLastImage;
m_trainingExamplesPerDigitCnt = trainingExamplesPerDigitCnt;
m_testExamplesPerDigitCnt = testExamplesPerDigitCnt;
m_imagesServed = 0;
m_sequenceIterator = 0;
m_definedOrder = false;
// START TIME MEASUREMENT ----------------------------------------------------------
//Stopwatch sw = new Stopwatch();
//sw.Start();
ReadMnistSet("train-images.idx3-ubyte", "train-labels.idx1-ubyte", m_trainingExamplesPerDigitCnt, m_trainingImages);
ReadMnistSet("t10k-images.idx3-ubyte", "t10k-labels.idx1-ubyte", m_testExamplesPerDigitCnt, m_testImages);
byte[,] data = new byte[28, 28];
m_blankImage = new MyMNISTImage(data, 0);
m_trainingImagesEnumerator = m_trainingImages.GetEnumerator();
m_testImagesEnumerator = m_testImages.GetEnumerator();
//sw.Stop();
//Console.WriteLine("Elapsed={0}", sw.Elapsed);
}
public override void Execute()
{
Owner.EOFSignal.Drop();
Owner.MNISTManager.RandomEnumerate = RandomEnumerate;
Owner.MNISTManager.m_definedOrder = SequenceOrdered;
//if ((SimulationStep <= ExpositionTime * Owner.ImagesCnt) && (SimulationStep % ExpositionTime == 0))
if ((SimulationStep + ExpositionTimeOffset) % ExpositionTime == 0)
{
MyMNISTImage im = Owner.MNISTManager.GetNextImage(m_numsToSend, m_setType);
if (Owner.BinaryPixels)
{
im.ToBinary();
}
Array.Copy(im.Data1D, Owner.Bitmap.Host, 784); //28 * 28
Owner.Bitmap.SafeCopyToDevice();
if (Owner.Binary)
{
Array.Clear(Owner.Label.Host, 0, 10);
Owner.Label.Host[im.Label] = 1;
}
else
{
Owner.Label.Host[0] = im.Label;
}
Owner.Label.SafeCopyToDevice();
Owner.EOFSignal.Raise();
}
}
/// <summary>
/// Gets the next values
/// </summary>
/// <param name="validNumbers">Array of integers, you want the selection restrict to.</param>
/// <returns>Array of arrays of floats, in which the values is encoded.</returns>
public MyMNISTImage GetNextImage(int[] validNumbers)
{
if (RandomEnumerate)
{
MyMNISTImage im = (MyMNISTImage)m_images[rand.Next(Math.Min(m_imagesDemand, m_images.Count))];
if (m_definedOrder && im.Label != validNumbers[m_sequenceIterator] || !validNumbers.Contains(im.Label))
{
return(this.GetNextImage(validNumbers));
}
m_sequenceIterator = (m_sequenceIterator + 1) % validNumbers.Length;
return(im);
}
else if (m_imageEnumerator.MoveNext() && m_imagesServed < m_imagesDemand)
{
MyMNISTImage im = (MyMNISTImage)m_imageEnumerator.Current;
m_imagesServed++;
if (m_definedOrder)
{
if (im.Label != validNumbers[m_sequenceIterator])
{
return(this.GetNextImage(validNumbers));
}
m_sequenceIterator = (m_sequenceIterator + 1) % validNumbers.Length;
}
else
{
if (!validNumbers.Contains(im.Label))
{
return(this.GetNextImage(validNumbers));
}
}
return(im);
}
else
{
switch (m_afterLastImage)
{
case MNISTLastImageMethod.ResetToStart:
{
m_imageEnumerator.Reset();
m_imagesServed = 0; // Hack
return(GetNextImage(validNumbers));
}
case MNISTLastImageMethod.SendNothing:
{
return(m_blankImage);
}
default:
{
return(GetNextImage(validNumbers));
}
}
}
}
private void ReadMnistSet(String imagesInputFile, String labelsInputFile, int numDigitClasses, ArrayList images)
{
FileStream ifsLabels = new FileStream(m_baseFolder + labelsInputFile, FileMode.Open, FileAccess.Read);
FileStream ifsImages = new FileStream(m_baseFolder + imagesInputFile, FileMode.Open, FileAccess.Read);
BinaryReader brLabels = new BinaryReader(ifsLabels);
BinaryReader brImages = new BinaryReader(ifsImages);
//Magic number
brLabels.ReadInt32();
brImages.ReadInt32();
int numImagesLables = brLabels.ReadInt32();
numImagesLables = ReverseBytes(numImagesLables);
int numImages = brImages.ReadInt32();
numImages = ReverseBytes(numImages);
int numRowsImages = brImages.ReadInt32();
numRowsImages = ReverseBytes(numRowsImages);
int numColsImages = brImages.ReadInt32();
numColsImages = ReverseBytes(numColsImages);
int numOfLoadedDigitClasses = 0;
int[] digitCounts = new int[10]; // helper array used for loading a given number of each digit
if (numImagesLables == numImages)
{
for (int i = 0; i < numImages; ++i)
{
MyMNISTImage mImage = new MyMNISTImage(brImages, brLabels, numColsImages, numRowsImages);
if (digitCounts[mImage.Label] < numDigitClasses)
{
images.Add(mImage);
digitCounts[mImage.Label]++;
if (digitCounts[mImage.Label] == numDigitClasses)
{
numOfLoadedDigitClasses++;
}
}
if (numOfLoadedDigitClasses == 10)
{
break;
}
}
}
ifsImages.Close();
brImages.Close();
ifsLabels.Close();
brLabels.Close();
}
private void ReadMnistSet(String imagesInputFile, String labelsInputFile, int numDigitClasses, ArrayList images)
{
FileStream ifsLabels = new FileStream(m_baseFolder + labelsInputFile, FileMode.Open, FileAccess.Read);
FileStream ifsImages = new FileStream(m_baseFolder + imagesInputFile, FileMode.Open, FileAccess.Read);
BinaryReader brLabels = new BinaryReader(ifsLabels);
BinaryReader brImages = new BinaryReader(ifsImages);
//Magic number
brLabels.ReadInt32();
brImages.ReadInt32();
int numImagesLables = brLabels.ReadInt32();
numImagesLables = ReverseBytes(numImagesLables);
int numImages = brImages.ReadInt32();
numImages = ReverseBytes(numImages);
int numRowsImages = brImages.ReadInt32();
numRowsImages = ReverseBytes(numRowsImages);
int numColsImages = brImages.ReadInt32();
numColsImages = ReverseBytes(numColsImages);
int numOfLoadedDigitClasses = 0;
int[] digitCounts = new int[10]; // helper array used for loading a given number of each digit
if (numImagesLables == numImages)
{
for (int i = 0; i < numImages; ++i)
{
MyMNISTImage mImage = new MyMNISTImage(brImages, brLabels, numColsImages, numRowsImages);
if (digitCounts[mImage.Label] < numDigitClasses)
{
images.Add(mImage);
digitCounts[mImage.Label]++;
if (digitCounts[mImage.Label] == numDigitClasses)
{
numOfLoadedDigitClasses++;
}
}
if (numOfLoadedDigitClasses == 10)
{
break;
}
}
}
ifsImages.Close();
brImages.Close();
ifsLabels.Close();
brLabels.Close();
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="baseFolder">Base folder with MNIST dataset files</param>
/// <param name="imagesCnt">How many images to load</param>
/// <param name="exact">If TRUE, you will get exactly imagesCnt images. If FALSE, you will get AT MOST imagesCnt images. It's here for performance reason and BC</param>
/// <param name="afterLastImage">What to do, after the last values has been sent</param>
public MyMNISTManager(string baseFolder, int trainingExamplesPerDigitCnt = int.MaxValue, int testExamplesPerDigitCnt = int.MaxValue,
bool exact = false, MNISTLastImageMethod afterLastImage = MNISTLastImageMethod.ResetToStart)
{
m_baseFolder = baseFolder;
m_trainingImages = new ArrayList();
m_testImages = new ArrayList();
m_afterLastImage = afterLastImage;
m_trainingExamplesPerDigitCnt = trainingExamplesPerDigitCnt;
m_testExamplesPerDigitCnt = testExamplesPerDigitCnt;
m_imagesServed = 0;
m_sequenceIterator = 0;
m_definedOrder = false;
// START TIME MEASUREMENT ----------------------------------------------------------
//Stopwatch sw = new Stopwatch();
//sw.Start();
ReadMnistSet("train-images.idx3-ubyte", "train-labels.idx1-ubyte", m_trainingExamplesPerDigitCnt, m_trainingImages);
ReadMnistSet("t10k-images.idx3-ubyte", "t10k-labels.idx1-ubyte", m_testExamplesPerDigitCnt, m_testImages);
byte[,] data = new byte[28, 28];
m_blankImage = new MyMNISTImage(data, 0);
m_trainingImagesEnumerator = m_trainingImages.GetEnumerator();
m_testImagesEnumerator = m_testImages.GetEnumerator();
//sw.Stop();
//Console.WriteLine("Elapsed={0}", sw.Elapsed);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="baseFolder">Base folder with MNIST dataset files</param>
/// <param name="imagesCnt">How many images to load</param>
/// <param name="exact">If TRUE, you will get exactly imagesCnt images. If FALSE, you will get AT MOST imagesCnt images. It's here for performance reason and BC</param>
/// <param name="afterLastImage">What to do, after the last values has been sent</param>
public MyMNISTManager(string baseFolder, int imagesCnt = int.MaxValue, bool exact = false, MNISTLastImageMethod afterLastImage = MNISTLastImageMethod.ResetToStart)
{
m_baseFolder = baseFolder;
m_images = new ArrayList();
m_afterLastImage = afterLastImage;
m_imagesDemand = imagesCnt;
m_imagesServed = 0;
m_sequenceIterator = 0;
m_definedOrder = false;
FileStream ifsTrainLabels = new FileStream(m_baseFolder + "train-labels.idx1-ubyte", FileMode.Open, FileAccess.Read);
FileStream ifsTrainImages = new FileStream(m_baseFolder + "train-images.idx3-ubyte", FileMode.Open, FileAccess.Read);
BinaryReader brTrainLabels = new BinaryReader(ifsTrainLabels);
BinaryReader brTrainImages = new BinaryReader(ifsTrainImages);
//Magic number
brTrainLabels.ReadInt32();
brTrainImages.ReadInt32();
int numImagesTrainLables = brTrainLabels.ReadInt32();
numImagesTrainLables = ReverseBytes(numImagesTrainLables);
int numImagesTrainImages = brTrainImages.ReadInt32();
numImagesTrainImages = ReverseBytes(numImagesTrainImages);
int numRowsTrainImages = brTrainImages.ReadInt32();
numRowsTrainImages = ReverseBytes(numRowsTrainImages);
int numColsTrainImages = brTrainImages.ReadInt32();
numColsTrainImages = ReverseBytes(numColsTrainImages);
int maxImages;
if (exact)
{
// numImagesTrainImages = 60000
maxImages = numImagesTrainImages;
}
else
{
// value of 2000 is a compromise between long loading-time for maximum of 60k images and minimum of user set "imagesCnt"
// this also brings more flexibility for changing the m_imagesDemand during simulation (up to max(2000,imagesCnt) instead of imagesCnt)
maxImages = Math.Max(2000, Math.Min(numImagesTrainImages, imagesCnt));
}
byte[][] data = new byte[numColsTrainImages][];
for (int i = 0; i < data.Length; ++i)
{
data[i] = new byte[numRowsTrainImages];
}
for (int i = 0; i < maxImages; ++i)
{
for (int j = 0; j < numColsTrainImages; ++j)
{
for (int k = 0; k < numRowsTrainImages; ++k)
{
byte b = brTrainImages.ReadByte();
data[j][k] = b;
}
}
byte label = brTrainLabels.ReadByte();
MyMNISTImage mImage = new MyMNISTImage(data, label, numColsTrainImages, numRowsTrainImages);
m_images.Add(mImage);
}
ifsTrainImages.Close();
brTrainImages.Close();
ifsTrainLabels.Close();
brTrainLabels.Close();
m_imageEnumerator = m_images.GetEnumerator();
for (int i = 0; i < numColsTrainImages; ++i)
{
for (int j = 0; j < numRowsTrainImages; ++j)
{
data[i][j] = 0;
}
}
m_blankImage = new MyMNISTImage(data, 0);
}
/// <summary>
/// Gets the next values
/// </summary>
/// <param name="validNumbers">Array of integers, you want the selection restrict to.</param>
/// <returns>Array of arrays of floats, in which the values is encoded.</returns>
public MyMNISTImage GetNextImage(int[] validNumbers, MNISTSetType setType)
{
ArrayList images = null;
IEnumerator enumerator = null;
int examplesPerDigitCnt = 0;
if (setType == MNISTSetType.Training)
{
images = m_trainingImages;
enumerator = m_trainingImagesEnumerator;
examplesPerDigitCnt = m_trainingExamplesPerDigitCnt;
}
else if (setType == MNISTSetType.Test)
{
images = m_testImages;
enumerator = m_testImagesEnumerator;
examplesPerDigitCnt = m_testExamplesPerDigitCnt;
}
if (RandomEnumerate)
{
MyMNISTImage im = (MyMNISTImage)images[rand.Next(Math.Min(examplesPerDigitCnt * 10, images.Count))];
// choose random image until it is in the validNumbers set;
// if "sequence ordered" is ON it also has to be the expected next number in given sequence
if (m_definedOrder && im.Label != validNumbers[m_sequenceIterator] || !validNumbers.Contains(im.Label))
{
return(this.GetNextImage(validNumbers, setType));
}
m_sequenceIterator = (m_sequenceIterator + 1) % validNumbers.Length;
return(im);
}
else if (enumerator.MoveNext() && m_imagesServed < images.Count)
{
MyMNISTImage im = (MyMNISTImage)enumerator.Current;
m_imagesServed++;
if (m_definedOrder)
{
if (im.Label != validNumbers[m_sequenceIterator])
{
return(GetNextImage(validNumbers, setType));
}
m_sequenceIterator = (m_sequenceIterator + 1) % validNumbers.Length;
}
else
{
if (!validNumbers.Contains(im.Label))
{
return(GetNextImage(validNumbers, setType));
}
}
return(im);
}
else
{
switch (m_afterLastImage)
{
case MNISTLastImageMethod.ResetToStart:
{
enumerator.Reset();
m_imagesServed = 0; // Hack
return(GetNextImage(validNumbers, setType));
}
case MNISTLastImageMethod.SendNothing:
{
return(m_blankImage);
}
default:
{
return(GetNextImage(validNumbers, setType));
}
}
}
}
/// <summary>
/// Gets the next values
/// </summary>
/// <param name="validNumbers">Array of integers, you want the selection restrict to.</param>
/// <returns>Array of arrays of floats, in which the values is encoded.</returns>
public MyMNISTImage GetNextImage(int[] validNumbers, MNISTSetType setType)
{
MyMNISTImage imageToReturn = null;
ArrayList images = null;
IEnumerator enumerator = null;
int examplesPerDigitCnt = 0;
if (setType == MNISTSetType.Training)
{
images = m_trainingImages;
enumerator = m_trainingImagesEnumerator;
examplesPerDigitCnt = m_trainingExamplesPerDigitCnt;
}
else if (setType == MNISTSetType.Test)
{
images = m_testImages;
enumerator = m_testImagesEnumerator;
examplesPerDigitCnt = m_testExamplesPerDigitCnt;
}
CheckIfSomethingChaged(setType, validNumbers, RandomEnumerate, m_definedOrder);
if (!initialized)
{
initialized = true;
//indicates whether this particular image was already shown. This prevents the situation that some numbers start to repeat while others do not appear on the output
alreadyShown = new bool[examplesPerDigitCnt * 10];
for (int i = 0; i < alreadyShown.Length; i++)
{
alreadyShown[i] = false;
}
//how many images to show until the set starts to repeat again
howManyLeft = examplesPerDigitCnt * validNumbers.Length;
m_sequenceIterator = 0;
m_lastServedImage = -1;
}
if (RandomEnumerate)
{
m_lastServedImage = rand.Next(Math.Min(examplesPerDigitCnt * 10, images.Count));
MyMNISTImage im = (MyMNISTImage)images[m_lastServedImage];
// choose random image until it is in the validNumbers set;
// if "sequence ordered" is ON it also has to be the expected next number in given sequence
if ((m_definedOrder && im.Label != validNumbers[m_sequenceIterator]) || !validNumbers.Contains(im.Label))
{
return(this.GetNextImage(validNumbers, setType));
}
imageToReturn = im;
}
else
{
MyMNISTImage im = null;
while (enumerator.MoveNext() && m_lastServedImage < images.Count - 1)
{
im = (MyMNISTImage)enumerator.Current;
m_lastServedImage++;
if (m_definedOrder)
{
if (im.Label == validNumbers[m_sequenceIterator])
{
imageToReturn = im;
break;
}
}
else
{
if (validNumbers.Contains(im.Label))
{
imageToReturn = im;
break;
}
}
}
if (imageToReturn == null)
{
switch (m_afterLastImage)
{
case MNISTLastImageMethod.ResetToStart:
{
enumerator.Reset();
m_lastServedImage = -1;
return(GetNextImage(validNumbers, setType));
}
case MNISTLastImageMethod.SendNothing:
{
return(m_blankImage);
}
default:
{
return(GetNextImage(validNumbers, setType));
}
}
}
}
if (alreadyShown[m_lastServedImage])
{
return(this.GetNextImage(validNumbers, setType));
}
else
{
DecreaseLeftCounter(m_lastServedImage, examplesPerDigitCnt * validNumbers.Length);
m_sequenceIterator = (m_sequenceIterator + 1) % validNumbers.Length;
return(imageToReturn);
}
}
