public static void Run()
{
Console.WriteLine("Build Vocabulary.");
Vocabulary vocabulary = new Vocabulary();
string trainPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TRAIN_FILE, TRAIN_FILE, TRAIN_FILE_HASH);
string validPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + VALID_FILE, VALID_FILE, VALID_FILE_HASH);
string testPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TEST_FILE, TEST_FILE, TEST_FILE_HASH);
int[] trainData = vocabulary.LoadData(trainPath);
int[] validData = vocabulary.LoadData(validPath);
int[] testData = vocabulary.LoadData(testPath);
int nVocab = vocabulary.Length;
Console.WriteLine("Network Initilizing.");
FunctionStack model = new FunctionStack(
new EmbedID(nVocab, N_UNITS, name: "l1 EmbedID"),
new Dropout(),
new LSTM(N_UNITS, N_UNITS, name: "l2 LSTM"),
new Dropout(),
new LSTM(N_UNITS, N_UNITS, name: "l3 LSTM"),
new Dropout(),
new Linear(N_UNITS, nVocab, name: "l4 Linear")
);
//与えられたthresholdで頭打ちではなく、全パラメータのL2Normからレートを取り補正を行う
GradientClipping gradientClipping = new GradientClipping(threshold: GRAD_CLIP);
SGD sgd = new SGD(learningRate: 1);
model.SetOptimizer(gradientClipping, sgd);
Real wholeLen = trainData.Length;
int jump = (int)Math.Floor(wholeLen / BATCH_SIZE);
int epoch = 0;
Stack <NdArray[]> backNdArrays = new Stack <NdArray[]>();
Console.WriteLine("Train Start.");
for (int i = 0; i < jump * N_EPOCH; i++)
{
NdArray x = new NdArray(new[] { 1 }, BATCH_SIZE);
NdArray t = new NdArray(new[] { 1 }, BATCH_SIZE);
for (int j = 0; j < BATCH_SIZE; j++)
{
x.Data[j] = trainData[(int)((jump * j + i) % wholeLen)];
t.Data[j] = trainData[(int)((jump * j + i + 1) % wholeLen)];
}
NdArray[] result = model.Forward(x);
Real sumLoss = new SoftmaxCrossEntropy().Evaluate(result, t);
backNdArrays.Push(result);
Console.WriteLine("[{0}/{1}] Loss: {2}", i + 1, jump, sumLoss);
//Run truncated BPTT
if ((i + 1) % BPROP_LEN == 0)
{
for (int j = 0; backNdArrays.Count > 0; j++)
{
Console.WriteLine("backward" + backNdArrays.Count);
model.Backward(backNdArrays.Pop());
}
model.Update();
model.ResetState();
}
if ((i + 1) % jump == 0)
{
epoch++;
Console.WriteLine("evaluate");
Console.WriteLine("validation perplexity: {0}", Evaluate(model, validData));
if (epoch >= 6)
{
sgd.LearningRate /= 1.2;
Console.WriteLine("learning rate =" + sgd.LearningRate);
}
}
}
Console.WriteLine("test start");
Console.WriteLine("test perplexity:" + Evaluate(model, testData));
}
public static void Run()
{
Console.WriteLine("Build Vocabulary.");
Vocabulary vocabulary = new Vocabulary();
string trainPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TRAIN_FILE, TRAIN_FILE, TRAIN_FILE_HASH);
string testPath = InternetFileDownloader.Donwload(DOWNLOAD_URL + TEST_FILE, TEST_FILE, TEST_FILE_HASH);
int[] trainData = vocabulary.LoadData(trainPath);
int[] testData = vocabulary.LoadData(testPath);
int nVocab = vocabulary.Length;
Console.WriteLine("Done.");
Console.WriteLine("Network Initilizing.");
FunctionStack model = new FunctionStack(
new EmbedID(nVocab, N_UNITS, name: "l1 EmbedID"),
new Linear(N_UNITS, N_UNITS, name: "l2 Linear"),
new TanhActivation("l2 Tanh"),
new Linear(N_UNITS, nVocab, name: "l3 Linear"),
new Softmax("l3 Sonftmax")
);
model.SetOptimizer(new Adam());
List <int> s = new List <int>();
Console.WriteLine("Train Start.");
SoftmaxCrossEntropy softmaxCrossEntropy = new SoftmaxCrossEntropy();
for (int epoch = 0; epoch < TRAINING_EPOCHS; epoch++)
{
for (int pos = 0; pos < trainData.Length; pos++)
{
NdArray h = new NdArray(new Real[N_UNITS]);
int id = trainData[pos];
s.Add(id);
if (id == vocabulary.EosID)
{
Real accumloss = 0;
Stack <NdArray> tmp = new Stack <NdArray>();
for (int i = 0; i < s.Count; i++)
{
int tx = i == s.Count - 1 ? vocabulary.EosID : s[i + 1];
//l1 EmbedID
NdArray l1 = model.Functions[0].Forward(s[i])[0];
//l2 Linear
NdArray l2 = model.Functions[1].Forward(h)[0];
//Add
NdArray xK = l1 + l2;
//l2 Tanh
h = model.Functions[2].Forward(xK)[0];
//l3 Linear
NdArray h2 = model.Functions[3].Forward(h)[0];
Real loss = softmaxCrossEntropy.Evaluate(h2, tx);
tmp.Push(h2);
accumloss += loss;
}
Console.WriteLine(accumloss);
for (int i = 0; i < s.Count; i++)
{
model.Backward(tmp.Pop());
}
model.Update();
s.Clear();
}
if (pos % 100 == 0)
{
Console.WriteLine(pos + "/" + trainData.Length + " finished");
}
}
}
Console.WriteLine("Test Start.");
Real sum = 0;
int wnum = 0;
List <int> ts = new List <int>();
bool unkWord = false;
for (int pos = 0; pos < 1000; pos++)
{
int id = testData[pos];
ts.Add(id);
if (id > trainData.Length)
{
unkWord = true;
}
if (id == vocabulary.EosID)
{
if (!unkWord)
{
Console.WriteLine("pos" + pos);
Console.WriteLine("tsLen" + ts.Count);
Console.WriteLine("sum" + sum);
Console.WriteLine("wnum" + wnum);
sum += CalPs(model, ts);
wnum += ts.Count - 1;
}
else
{
unkWord = false;
}
ts.Clear();
}
}
Console.WriteLine(Math.Pow(2.0, sum / wnum));
}
