List <DataSequence> getData(string fileName)
{
List <DataSequence> result = new List <DataSequence>();
string row = "";
using (StreamReader sr = new StreamReader(fileName))
{
for (int rowNumber = 0; null != ((row = sr.ReadLine())); rowNumber++)
{
if (rowNumber % 1000 == 0)
{
Console.WriteLine(rowNumber);
}
DataSequence sequence = new DataSequence();
sequence.Steps = new List <DataStep>();
for (int i = 0; i < row.Length; i++)
{
DataStep step = new DataStep(FieldLetterTranslator.traslateToField(row[i]), FieldLetterTranslator.traslateToField((i + 1 < row.Length) ? row[i + 1] : '$'));
if (step.Input != null && step.TargetOutput != null)
{
sequence.Steps.Add(step);
}
}
result.Add(sequence);
}
}
return(result);
}
public static double Pass(double learningRate, INetwork network, List <DataSequence> sequences,
bool applyTraining, ILoss lossTraining, ILoss lossReporting)
{
double numerLoss = 0;
double denomLoss = 0;
RnnConfig rnnConfig = (RnnConfig)Serializer.Deserialize(NetworkBuilder.Config.RnnConfigFile);
foreach (DataSequence seq in sequences)
{
network.ResetState();
Graph g = new Graph(applyTraining);
network.GenerateDropout(applyTraining);
for (int i = 0; i < seq.Steps.Count; ++i)
{
DataStep step = seq.Steps[i];
// Generate in dropout
bool[] dropped = new bool[step.Input.W.Length];
for (int col = 0; col < dropped.Length; ++col)
{
dropped[col] = Math.Abs(rnnConfig.GetTransformed(0, i, col, step.Input.W[col])) < 0.0000001;
}
Matrix output = network.Activate(step.Input, g, dropped);
if (step.TargetOutput != null)
{
double loss = lossReporting.Measure(output, step.TargetOutput);
if (Double.IsNaN(loss) || Double.IsInfinity(loss))
{
return(loss);
}
numerLoss += loss;
denomLoss++;
if (applyTraining)
{
lossTraining.Backward(output, step.TargetOutput);
}
}
if (i % 10 == 0 && applyTraining)
{
g.Backward(); //backprop dw values
UpdateModelParams(network, learningRate); //update params
g = new Graph(applyTraining);
network.GenerateDropout(applyTraining);
}
}
}
return(numerLoss / denomLoss);
}
static List <DataSequence> GetSequences(NNValue[] inputs, NNValue[] outputs)
{
List <DataSequence> dataSequences = new List <DataSequence>();
for (int i = 0; i < inputs.Length; i++)
{
DataStep dataStep = new DataStep(inputs[i], outputs[i]);
DataSequence dataSequence = new DataSequence();
dataSequence.Steps.Add(dataStep);
dataSequences.Add(dataSequence);
}
return(dataSequences);
}
public static List <DataSequence> GetDataSequences(List <int[]> dataInp, List <int[]> dataOutp)
{
List <DataSequence> lds = new List <DataSequence>();
for (int i = 0; i < dataInp.Count; i++)
{
DataSequence sequence = new DataSequence();
sequence.Steps = new List <DataStep>();
for (int j = 0; j < dataInp[i].Length; j++)
{
DataStep ds = new DataStep(GetValue(dataInp[i][j], 11));
sequence.Steps.Add(ds);
}
for (int j = 0; j < dataOutp[i].Length; j++)
{
DataStep ds;
if (j == 0)
{
ds = new DataStep(GetValue(10, 11), GetValue(dataOutp[i][j]));
}
else
{
ds = new DataStep(GetValue(dataOutp[i][j - 1], 11), GetValue(dataOutp[i][j]));
}
sequence.Steps.Add(ds);
}
lds.Add(sequence);
}
return(lds);
}
public void Predict()
{
if (!Config.Reload &&
File.Exists(Config.RnnPredictedXFile) &&
File.Exists(Config.RnnPredictedYFile))
{
return;
}
Random rng = new Random(Config.Random.Next());
CustomDataSet data = new CustomDataSet(Config);
RnnConfig rnnConfig = (RnnConfig)Serializer.Deserialize(Config.RnnConfigFile);
int inputDimension = data.Training[0].Steps[0].Input.Rows;
int hiddenDimension = 30;
int outputDimension = data.Training[0].Steps[0].TargetOutput.Rows;
int hiddenLayers = 1;
double learningRate = 0.01;
double initParamsStdDev = 0.08;
double dropout = 0.5;
double inDropout = 0.8;
INetwork nn = NetworkBuilder.MakeLstm(inputDimension,
hiddenDimension,
hiddenLayers,
outputDimension,
new LinearUnit(),
initParamsStdDev, rng, dropout, inDropout, Config);
//nn = NetworkBuilder.MakeFeedForward(inputDimension,
// hiddenDimension,
// hiddenLayers,
// outputDimension,
// new SigmoidUnit(),
// new LinearUnit(),
// initParamsStdDev, rng, dropout, inDropout, Config);
int reportEveryNthEpoch = 10;
int trainingEpochs = 100;
Trainer.train <NeuralNetwork>(trainingEpochs, learningRate, nn, data, reportEveryNthEpoch, rng);
StreamWriter predictedXFile = new StreamWriter(Config.RnnPredictedXFile);
StreamWriter predictedYFile = new StreamWriter(Config.RnnPredictedYFile);
for (int i = 0; i < data.Testing.First().Steps.Count; ++i)
{
DataStep ds = data.Testing.First().Steps[i];
Graph g = new Graph(false);
// Generate in dropout
bool[] dropped = new bool[ds.Input.W.Length];
for (int col = 0; col < dropped.Length; ++col)
{
dropped[col] = Math.Abs(rnnConfig.GetTransformed(0, i, col, ds.Input.W[col])) < 0.0000001;
}
Matrix input = new Matrix(ds.Input.W);
Matrix output = nn.Activate(input, g, dropped);
// Write into file
string line1 = "";
string line2 = "";
foreach (double d in output.W)
{
line1 += d + ";";
}
foreach (double d in ds.TargetOutput.W)
{
line2 += d + ";";
}
predictedXFile.WriteLine(line1.Substring(0, line1.Length - 1));
predictedYFile.WriteLine(line2.Substring(0, line2.Length - 1));
}
predictedXFile.Close();
predictedYFile.Close();
}
