//"t10k-labels.idx1-ubyte" --> test labels file
//"t10k-images.idx3-ubyte" --> test images file
//"train-labels-idx1-ubyte" --> train labels file
//"train-images-idx3-ubyte" --> train images file
public static Tuple <List <DigitImage>, List <DigitImage> > ReadData(string trainImagesFile, string trainLabelsFile, string testImagesFile, string testLabelsFile)
{
var Training = new List <DigitImage>();
var Testing = new List <DigitImage>();
MNISTReader.ReadMNIST(trainImagesFile, trainLabelsFile, testImagesFile, testLabelsFile, Training, Testing);
return(new Tuple <List <DigitImage>, List <DigitImage> >(Training, Testing));
}
static void InitNN(string[] args, StreamWriter writer)
{
var argsCounter = 1;
writer.WriteLine("=============== Data ===================");
trainImagesFile = args[argsCounter++];
writer.WriteLine("Training Image file: " + trainImagesFile);
trainLabelsFile = args[argsCounter++];
writer.WriteLine("Training Labels file: " + trainLabelsFile);
testImagesFile = args[argsCounter++];
writer.WriteLine("Testing Images file: " + testImagesFile);
testLabelsFile = args[argsCounter++];
writer.WriteLine("Testing Labels file: " + testLabelsFile);
trainingSize = int.Parse(args[argsCounter++]);
writer.WriteLine("Training Size: " + trainingSize);
testingSize = int.Parse(args[argsCounter++]);
writer.WriteLine("Testing Size: " + testingSize);
writer.WriteLine("=============== Network Parameters ===================");
batchSize = int.Parse(args[argsCounter++]);
writer.WriteLine("Training Mini-Batch Size: " + batchSize);
epochs = int.Parse(args[argsCounter++]);
writer.WriteLine("Training Epochs: " + epochs);
learningRate = double.Parse(args[argsCounter++]);
writer.WriteLine("Learning Rate: " + learningRate);
weightDecay = double.Parse(args[argsCounter++]);
writer.WriteLine("Weight Decay: " + weightDecay);
seed = int.Parse(args[argsCounter++]);
writer.WriteLine("Random Seed: " + seed);
problemType = args[argsCounter++]; //classification or regression
writer.WriteLine("Problem Type: " + problemType);
criterion = args[argsCounter++]; //mse or softmax
writer.WriteLine("Cost Criterion: " + criterion);
epsilonLeaky = double.Parse(args[argsCounter++]); //threshold for Leaky ReLU
writer.WriteLine("Leaky ReLU threshold: " + epsilonLeaky);
epsilonGradientCheck = double.Parse(args[argsCounter++]);
writer.WriteLine("Gradient Check Step: " + epsilonGradientCheck);
gradientCheckThreshold = double.Parse(args[argsCounter++]);
writer.WriteLine("Gradient Check Threshold: " + gradientCheckThreshold);
L = int.Parse(args[argsCounter++]);
layerFunctions = new string[L + 1];
for (int l = 1; l <= L; l++)
{
layerFunctions[l] = args[argsCounter++]; //Sigmoid, Tanh, ReLU, or LReLU (Leaky ReLU) for now
writer.WriteLine("Hidden Layer " + l + ": " + layerFunctions[l]);
}
writer.WriteLine("Number of Hidden Layers: " + L);
layerSizes = new int[L + 2];
W = new Matrix[L + 1];
dW = new Matrix[L + 1];
deltaW = new Matrix[L + 1];
b = new Matrix[L + 1];
db = new Matrix[L + 1];
deltab = new Matrix[L + 1];
errors = new Matrix[L + 2];
Loss = new double[epochs];
for (int i = 0; i <= L + 1; i++)
{
layerSizes[i] = int.Parse(args[argsCounter++]);
writer.WriteLine("Size of Layer No. " + (i + 1) + ": " + layerSizes[i]);
}
Data = MNISTReader.ReadData(trainImagesFile, trainLabelsFile, testImagesFile, testLabelsFile);
Training = Data.Item1;
Testing = Data.Item1;
acc_dW = new List <Matrix[]>();
acc_db = new List <Matrix[]>();
for (int i = 0; i <= L; i++)
{
var W_layer = new Matrix(layerSizes[i + 1], layerSizes[i]);
var b_layer = new Matrix(layerSizes[i + 1], 1);
var dW_layer = new Matrix(layerSizes[i + 1], layerSizes[i]);
var db_layer = new Matrix(layerSizes[i + 1], 1);
var deltaW_layer = new Matrix(layerSizes[i + 1], layerSizes[i]);
var deltab_layer = new Matrix(layerSizes[i + 1], 1);
var errors_layer = new Matrix(layerSizes[i + 1], 1);
W_layer.Ramdomize(-0.01, 0.01);
b_layer.Ramdomize(-0.01, 0.01);
W[i] = W_layer;
b[i] = b_layer;
dW[i] = dW_layer;
db[i] = db_layer;
deltaW[i] = deltaW_layer;
deltab[i] = deltab_layer;
errors[i + 1] = errors_layer;
for (int j = 0; j <= batchSize; j++)
{
acc_dW[j][i] = new Matrix(layerSizes[i + 1], layerSizes[i]);
acc_db[j][i] = new Matrix(layerSizes[i + 1], 1);
}
}
A = new Matrix[L + 2];
Z = new Matrix[L + 2];
label = Matrix.CreateColumnMatrix(new double[layerSizes[L + 1]]);
}