private void TrainElementWithNegativeSampling(TrainingElement trainset) { foreach (var contextWordIndex in trainset.ContextWordIndexList) { var deltaW1 = Vector <double> .Build.Dense(Ip2VecSettings.NeuronCount, 0); for (int i = 0; i < this.Ip2VecSettings.NegativeSampling + 1; i++) { int targetNegativeWordIndex; int label; if (i == 0) { targetNegativeWordIndex = contextWordIndex; label = 1; } else { targetNegativeWordIndex = this.DiceTable[random.Next(0, tableSize)]; if (targetNegativeWordIndex == contextWordIndex) { continue; } label = 0; } this.NegativeSamplingIteration(contextWordIndex, label, targetNegativeWordIndex, ref deltaW1); } this.Ip2VecModel.W1.SetRow(contextWordIndex, this.Ip2VecModel.W1.Row(contextWordIndex).Add(deltaW1)); } }
public static TrainingElement NewContainer(int count, int contextCount) { var container = new TrainingElement { Count = count }.BuildTargetWordVector(0); for (int i = 0; i < contextCount; i++) { container = container.AddContextWordVector(0); } return(container); }
private void TrainElement(TrainingElement trainset, int subIter) { var sb = new StringBuilder("********************new iteration starts************************").AppendLine(); var h = this.Ip2VecModel.W1.Transpose().Multiply(trainset.TargetWordVector); var u = this.Ip2VecModel.W2.Multiply(h); var y_pred = this.Softmax(u); var EI = Vector <double> .Build.Dense(this.WordCount, 0); foreach (var item in trainset.ContextWordVectorList) { EI = EI.Add(y_pred.Subtract(item)); } sb.Append("h >>>").Append(h.ToString(int.MaxValue, int.MaxValue)).AppendLine(); sb.Append("u >>>").Append(u.ToString(int.MaxValue, int.MaxValue)).AppendLine(); sb.Append("y_p >>>").Append(y_pred.ToString(int.MaxValue, int.MaxValue)).AppendLine(); sb.Append("EI >>>").Append(EI.ToString(int.MaxValue, int.MaxValue)).AppendLine(); var delta_dw2 = h.OuterProduct(EI); var delta_dw1 = trainset.TargetWordVector.OuterProduct(this.Ip2VecModel.W2.Transpose().Multiply(EI.ToColumnMatrix()).Column(0)); this.Ip2VecModel.W1 -= delta_dw1.Multiply(this.Ip2VecSettings.LearningRate); this.Ip2VecModel.W2 = (this.Ip2VecModel.W2.Transpose() - delta_dw2.Multiply(this.Ip2VecSettings.LearningRate)).Transpose(); sb.AppendLine("W1 >>>").Append(Ip2VecModel.W1.ToString(int.MaxValue, int.MaxValue)).AppendLine(); sb.AppendLine("W2 >>>").Append(Ip2VecModel.W2.ToString(int.MaxValue, int.MaxValue)).AppendLine(); sb.AppendLine("************************iteration ends***************************"); #if DEBUG var data = new KeyValuePair <string, string>(string.Format("{0}-{1}", this.Iter.ToString(), subIter.ToString()), sb.ToString()); new Thread(Record).Start(data); #endif //this.Record(sb.ToString()); }
private void Train() { using (var status = StatusWrapper.NewStatus(@"Train", this.Data.Count() * this.Ip2VecSettings.EpochsCount)) { // Building training container var sets = new List <TrainingElement>(); foreach (var set in this.Ip2VecSettings.ContextRelationDict) { var target = TrainingElement.NewContainer(this.WordCount, set.Value.Count); sets.Add(target); } for (int i = 0; i < this.Ip2VecSettings.EpochsCount; i++) { this.Loss = 0d; foreach (var item in this.Data) { //this.StatusController.Preparesw.Start(); // Building training elements var c = 0; foreach (var set in this.Ip2VecSettings.ContextRelationDict) { sets[c].RebuildTargetWordVector(W2I(set.Key(item))); sets[c].SetContextWordVector(set.Value.Select(x => W2I(x(item))).ToArray()); c++; } //this.StatusController.Preparesw.Stop(); //this.StatusController.Calcsw.Start(); var index = 0; if (this.Ip2VecSettings.NegativeSampling > 0) { foreach (var set in sets) { this.TrainElementWithNegativeSampling(set); } } else { foreach (var set in sets) { this.TrainElement(set, index++); } } Iter++; status.PushProgress(); //this.StatusController.Calcsw.Stop(); //this.StatusController.SetCurrentProgress(); //if (Iter % 1000 == 0) //{ // this.StatusController.Refresh(); //} //this.Percentage.Value = Iter; //if (this.W1.ToRowMajorArray().Contains(double.NaN)) //{ // MessageBox.Show("NAN error"); //} } Console.WriteLine("loss = " + this.Loss); } #if DEBUG var sb = new StringBuilder(); foreach (var item in this.Ip2VecModel.Index2Word) { sb.AppendLine(string.Format("{0}\t{1}", item.Key, item.Value)); } (new Thread(Record)).Start(new KeyValuePair <string, string>("Dictionary", sb.ToString())); #endif } }