/// <summary>
/// the error deriv at top output layer must have been set up before call this method.
/// This process only do backprop computations. It does not update model weights at all.
/// Need to call update_weight afterwards to update models.
/// </summary>
/// <param name="input_batch"></param>
/// <param name="momentum"></param>
/// <param name="learning_rate"></param>
public void backward_propagate_deriv(BatchSample_Input input_batch)
{
// step 1, compute the derivatives for the output values of each layer
backward_calculate_layerout_deriv(input_batch.batchsize);
// step 2, compute the derivatives for the connections of each neural link layer
backward_calculate_weight_deriv(input_batch);
}
unsafe public void backward_calculate_weight_deriv(BatchSample_Input input_batch) //, int alpha_index) // float[] alpha)
{
int batchsize = input_batch.batchsize;
for (int i = 0; i < neurallinks.Count; i++)
{
neurallinks[i].ZeroDeriv();
if (ParameterSetting.UpdateBias)
{
MathOperatorManager.GlobalInstance.Matrix_Aggragate(neurallayers[i + 1].ErrorDeriv, neurallinks[i].BiasDeriv, input_batch.batchsize, neurallinks[i].NeuralLinkModel.Neural_Out.Number);
}
if (i == 0)
{
if (neurallinks[i].NeuralLinkModel.Nt == N_Type.Fully_Connected)
{
MathOperatorManager.GlobalInstance.SEQ_Sparse_Matrix_Transpose_Multiply_INTEX(input_batch, neurallinks[i].WeightDeriv, neurallayers[i + 1].ErrorDeriv, neurallinks[i].NeuralLinkModel.Neural_In.Number, neurallinks[i].NeuralLinkModel.Neural_Out.Number, neurallinks[i].NeuralLinkModel.N_Winsize);
}
else if (neurallinks[i].NeuralLinkModel.Nt == N_Type.Convolution_layer)
{
MathOperatorManager.GlobalInstance.Convolution_Sparse_Matrix_Product_INTEX(neurallayers[i + 1].ErrorDeriv, neurallinks[i].LayerMaxPooling_Index, input_batch, neurallinks[i].NeuralLinkModel.N_Winsize,
batchsize, neurallayers[i + 1].Number, neurallinks[i].WeightDeriv, neurallinks[i].NeuralLinkModel.Neural_In.Number);
}
}
else
{
MathOperatorManager.GlobalInstance.Matrix_Product(neurallayers[i].Output, neurallayers[i + 1].ErrorDeriv, neurallinks[i].WeightDeriv,
batchsize, neurallayers[i].Number, neurallayers[i + 1].Number);
}
}
}
unsafe public void Forward_CalSimilarityScore(BatchSample_Input query_batch, BatchSample_Input doc_batch)
{
/// forward (query doc, negdoc) streaming.
dnn_model_query.forward_activate(query_batch);
dnn_model_doc.forward_activate(doc_batch);
MathOperatorManager.GlobalInstance.Cosine_Similarity(dnn_model_query.neurallayers.Last().Output,
dnn_model_doc.neurallayers.Last().Output, alphaCudaPiece, ParameterSetting.NTRIAL + 1, ParameterSetting.BATCH_SIZE, 0,
query_batch.batchsize, dnn_model_query.OutputLayerSize, ParameterSetting.DSSMEpsilon); // float.Epsilon);
}
public void get_dimension(string fileName)
{
if (ParameterSetting.LoadInputBackwardCompatibleMode == "BOW")
{
// code for back-compatibility to the previous BOW bin format
mstream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
mreader = new BinaryReader(mstream);
mstream.Seek(-3 * sizeof(Int32), SeekOrigin.End);
Feature_Size = mreader.ReadInt32(); //// binary feature file stores feature dimension
total_Batch_Size = mreader.ReadInt32();
MAXELEMENTS_PERBATCH = mreader.ReadInt32();
MAXSEQUENCE_PERBATCH = ParameterSetting.BATCH_SIZE;
}
else if (ParameterSetting.LoadInputBackwardCompatibleMode == "SEQ")
{
// code for back-compatibility to the previous SEQ bin format, with unnecessary batch_size and feature_dim
mstream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
mreader = new BinaryReader(mstream);
mstream.Seek(-4 * sizeof(Int32), SeekOrigin.End);
Feature_Size = mreader.ReadInt32(); //// binary feature file stores feature dimension
total_Batch_Size = mreader.ReadInt32();
MAXSEQUENCE_PERBATCH = mreader.ReadInt32();
MAXELEMENTS_PERBATCH = mreader.ReadInt32();
}
else
{
mstream = new FileStream(fileName, FileMode.Open, FileAccess.Read);
mreader = new BinaryReader(mstream);
mstream.Seek(-5 * sizeof(Int32), SeekOrigin.End);
Feature_Size = mreader.ReadInt32(); //// binary feature file stores feature dimension
total_Batch_Size = mreader.ReadInt32();
MAXSEQUENCE_PERBATCH = mreader.ReadInt32();
MAXELEMENTS_PERBATCH = mreader.ReadInt32();
int batch_size = mreader.ReadInt32();
if (batch_size != ParameterSetting.BATCH_SIZE)
{
throw new Exception(string.Format(
"Batch_Size does not match between configuration and input data!\n\tFrom config: {0}.\n\tFrom data ({1}): {2}"
, ParameterSetting.BATCH_SIZE, fileName, batch_size)
);
}
}
Data = new BatchSample_Input(ParameterSetting.BATCH_SIZE, MAXSEQUENCE_PERBATCH, MAXELEMENTS_PERBATCH);
BATCH_NUM = (total_Batch_Size + ParameterSetting.BATCH_SIZE - 1) / ParameterSetting.BATCH_SIZE;
LAST_INCOMPLETE_BATCH_SIZE = total_Batch_Size % ParameterSetting.BATCH_SIZE;
BATCH_INDEX = 0;
}
public void CloseStream()
{
if (mstream != null)
{
mreader.Close();
mstream.Close();
mreader = null;
mstream = null;
}
if (Data != null)
{
Data.Dispose();
Data = null;
}
}
public override void ProcessBatch(BatchSample_Input input)
{
for (int sample = 0; sample < input.batchsize; ++sample)
{
int seg_begin = sample >= 1 ? input.Sample_Idx_Mem[sample - 1] : 0;
int seg_end = input.Sample_Idx_Mem[sample];
for (int seg_id = seg_begin; seg_id < seg_end; ++seg_id)
{
int fea_begin = seg_id >= 1 ? input.Seg_Idx_Mem[seg_id - 1] : 0;
int fea_end = input.Seg_Idx_Mem[seg_id];
for (int fea_id = fea_begin; fea_id < fea_end; ++fea_id)
{
int fea_key = input.Fea_Idx_Mem[fea_id];
float fea_value = input.Fea_Value_Mem[fea_id];
float new_fea_value = (fea_value - FeaMin[fea_key]) / (FeaMax[fea_key] - FeaMin[fea_key] + float.Epsilon);
input.Fea_Value_Mem[fea_id] = new_fea_value;
}
}
}
}
public override void AnalysisBatch(BatchSample_Input input)
{
SampleNum += input.batchsize;
for (int sample = 0; sample < input.batchsize; ++sample)
{
int seg_begin = sample >= 1 ? input.Sample_Idx_Mem[sample - 1] : 0;
int seg_end = input.Sample_Idx_Mem[sample];
for (int seg_id = seg_begin; seg_id < seg_end; ++seg_id)
{
int fea_begin = seg_id >= 1 ? input.Seg_Idx_Mem[seg_id - 1] : 0;
int fea_end = input.Seg_Idx_Mem[seg_id];
for (int fea_id = fea_begin; fea_id < fea_end; ++fea_id)
{
int fea_key = input.Fea_Idx_Mem[fea_id];
float fea_value = input.Fea_Value_Mem[fea_id];
if (FeaDict.ContainsKey(fea_key))
{
FeaDict[fea_key] += 1;
}
else
{
FeaDict[fea_key] = 1;
}
if (FeaMin[fea_key] > fea_value)
{
FeaMin[fea_key] = fea_value;
}
if (FeaMax[fea_key] < fea_value)
{
FeaMax[fea_key] = fea_value;
}
}
}
}
}
/// <summary>
/// given batch of input data. calculate the output.
/// </summary>
/// <param name="data"></param>
//unsafe public void forward_activate( BatchSample_Input data, List<Amplib.AMPArrayInternal> layerOutputs)
unsafe public void forward_activate(BatchSample_Input data)
{
int layerIndex = 0;
foreach (NeuralLinkData neurallinkData in neurallinks)
{
NeuralLink neurallink = neurallinkData.NeuralLinkModel;
///first layer.
if (layerIndex == 0)
{
if (neurallink.Nt == N_Type.Fully_Connected)
{
MathOperatorManager.GlobalInstance.SEQ_Sparse_Matrix_Multiply_INTEX(data, neurallink.weight, neurallayers[layerIndex + 1].Output,
neurallink.Neural_In.Number, neurallink.Neural_Out.Number, neurallink.N_Winsize);
}
else if (neurallink.Nt == N_Type.Convolution_layer)
{
MathOperatorManager.GlobalInstance.Convolution_Sparse_Matrix_Multiply_INTEX(data, neurallink.weight, neurallinkData.LayerPoolingOutput, neurallink.Neural_In.Number, neurallink.Neural_Out.Number, neurallink.N_Winsize);
MathOperatorManager.GlobalInstance.Max_Pooling(neurallinkData.LayerPoolingOutput, data, neurallayers[layerIndex + 1].Output, neurallinkData.LayerMaxPooling_Index, neurallink.Neural_Out.Number);
}
}
else
{
MathOperatorManager.GlobalInstance.Matrix_Multipy(neurallayers[layerIndex].Output, neurallink.weight, neurallayers[layerIndex + 1].Output, data.batchsize, neurallink.Neural_In.Number, neurallink.Neural_Out.Number, 0);
}
if (neurallink.Af == A_Func.Tanh)
{
MathOperatorManager.GlobalInstance.Matrix_Add_Tanh(neurallayers[layerIndex + 1].Output, neurallink.bias, data.batchsize, neurallink.Neural_Out.Number);
}
else if (neurallink.Af == A_Func.Linear)
{
MathOperatorManager.GlobalInstance.Matrix_Add_Vector(neurallayers[layerIndex + 1].Output, neurallink.bias, data.batchsize, neurallink.Neural_Out.Number);
}
layerIndex += 1;
}
}
/*return the loss using by feedstream */
//unsafe public float feedstream_batch( BatchSample_Input query_batch, BatchSample_Input doc_batch, List<BatchSample_Input> negdoc_batches, bool train_update)
unsafe public float feedstream_batch(BatchSample_Input query_batch, BatchSample_Input doc_batch, bool train_update, StreamReader srNCEProbDist)
{
/// forward (query doc, negdoc) streaming.
Forward_CalSimilarityScore(query_batch, doc_batch);
Negative_Sampling(query_batch.batchsize);
MathOperatorManager.GlobalInstance.Cosine_Similarity_EX_Full(dnn_model_query.neurallayers.Last().Output,
dnn_model_doc.neurallayers.Last().Output, GPU_negative_index_Array,
alphaCudaPiece, ParameterSetting.NTRIAL, ParameterSetting.BATCH_SIZE,
query_batch.batchsize, dnn_model_query.OutputLayerSize, ParameterSetting.DSSMEpsilon); //float.Epsilon);
float maxlogpD = 0;
if (ParameterSetting.reserved_settings.Contains("_maxlogpd_"))
{
string[] spl = ParameterSetting.reserved_settings.Split('_');
for (int i = 0; i < spl.Length; i++)
{
if (spl[i] == "maxlogpd" && i < spl.Length - 1)
{
maxlogpD = float.Parse(spl[i + 1]);
break;
}
}
}
if (srNCEProbDist != null)
{
string line = string.Empty;
for (int i = 0; i < ParameterSetting.BATCH_SIZE; i++)
{
line = srNCEProbDist.ReadLine();
if (line == null)
{
break;
}
float logprob = float.Parse(line.Trim());
if (logprob > maxlogpD)
{
logprob = maxlogpD;
}
doc_dist[i] = logprob;
//doc_dist[i] = -(i + 1);
}
}
else
{
for (int i = 0; i < ParameterSetting.BATCH_SIZE; i++)
{
doc_dist[i] = (float)(-Math.Log(LearningParameters.total_doc_num));
}
}
distCudaPiece.CopyIntoCuda(ParameterSetting.BATCH_SIZE); //this sets D+
MathOperatorManager.GlobalInstance.FillOut_Dist_NCE_Full(distCudaPiece, GPU_negative_index_Array, ParameterSetting.NTRIAL, ParameterSetting.BATCH_SIZE, doc_batch.batchsize);
MathOperatorManager.GlobalInstance.Calculate_Alpha_NCE(alphaCudaPiece, distCudaPiece, ParameterSetting.NTRIAL + 1, ParameterSetting.BATCH_SIZE, query_batch.batchsize, ParameterSetting.PARM_GAMMA);
float error = 0;
if (ParameterSetting.LOSS_REPORT == 1)
{
alphaCudaPiece.CopyOutFromCuda();
for (int i = 0; i < query_batch.batchsize; i++)
{
float mlambda = 0;
mlambda = -(float)Math.Log(Math.Max(float.Epsilon, 1 - first_alpha[i] / ParameterSetting.PARM_GAMMA));
for (int nt = 1; nt <= ParameterSetting.NTRIAL; nt++)
{
mlambda += -(float)Math.Log(Math.Max(float.Epsilon, 1 - first_alpha[nt * ParameterSetting.BATCH_SIZE + i] / ParameterSetting.PARM_GAMMA));
}
if (float.IsNaN(mlambda))
{
//Console.WriteLine("IsNaN");
throw new Exception("Error! NaN.");
}
if (float.IsInfinity(mlambda))
{
//Console.WriteLine("IsInfinity");
throw new Exception("Error! IsInfinity.");
}
error += mlambda;
}
}
if (train_update)
{
Negative_Sampling_Transpose(query_batch.batchsize);
/******* Calculate the error derivatives on the top layer outputs *****/
calculate_deltaQD_TOP(Pos_QD_Pair_TOP, query_batch.batchsize);
/// Only support GPU version now.
calculate_deltaQD_TOPEX_Full(Neg_QD_Pair_TOP, query_batch.batchsize);
// Query Derive Merge
MathOperatorManager.GlobalInstance.Matrix_WeightAdd(dnn_model_query.neurallayers.Last().ErrorDeriv, Pos_QD_Pair_TOP.cuda_layer_Deriv_Q,
query_batch.batchsize, dnn_model_query.OutputLayerSize, alphaCudaPiece, 0, 0);
MathOperatorManager.GlobalInstance.Matrix_WeightAdd_Full(dnn_model_query.neurallayers.Last().ErrorDeriv, Neg_QD_Pair_TOP.cuda_layer_Deriv_Q,
ParameterSetting.NTRIAL, ParameterSetting.BATCH_SIZE,
query_batch.batchsize, dnn_model_query.OutputLayerSize, alphaCudaPiece, ParameterSetting.BATCH_SIZE, -1);
// Doc Derive Merge
MathOperatorManager.GlobalInstance.Matrix_WeightAdd(dnn_model_doc.neurallayers.Last().ErrorDeriv, Pos_QD_Pair_TOP.cuda_layer_Deriv_D,
doc_batch.batchsize, dnn_model_doc.OutputLayerSize, alphaCudaPiece, 0, 0);
MathOperatorManager.GlobalInstance.Matrix_WeightAdd_EX_Full(dnn_model_doc.neurallayers.Last().ErrorDeriv, Neg_QD_Pair_TOP.cuda_layer_Deriv_D,
GPU_Inver_negative_index_Array,
GPU_Inver_negative_value_Array, ParameterSetting.NTRIAL, ParameterSetting.BATCH_SIZE, doc_batch.batchsize,
dnn_model_doc.OutputLayerSize, alphaCudaPiece, ParameterSetting.BATCH_SIZE, -1);
// back propagate
dnn_model_query.backward_propagate_deriv(query_batch);
dnn_model_doc.backward_propagate_deriv(doc_batch);
// update here
// here we have to do all the backprop computations before updating the model, because the model's actual weights will affect the backprop computation
dnn_model_query.update_weight(LearningParameters.momentum, LearningParameters.learning_rate * query_batch.batchsize / ParameterSetting.BATCH_SIZE);
dnn_model_doc.update_weight(LearningParameters.momentum, LearningParameters.learning_rate * query_batch.batchsize / ParameterSetting.BATCH_SIZE);
// and now it should support shared models
}
return(error);
}
public void Sparse2Dense_Matrix(BatchSample_Input data, CudaPieceFloat matrix, int batchsize, int outputDimension)
{
BasicMathlib.Sparse2Dense_Matrix(data.Seg_Idx_Mem, data.Fea_Idx_Mem, data.Fea_Value_Mem, matrix.MemPtr, batchsize, outputDimension);
}
public void Convolution_Sparse_Matrix_Product_INTEX(CudaPieceFloat upperOutputErrorDeriv, CudaPieceInt layerMaxPooling_Index, BatchSample_Input input_batch, int winSize, int batchsize, int outputDimension, CudaPieceFloat weightDeriv, int inputDimension)
{
BasicMathlib.Convolution_Sparse_Matrix_Product_INTEX(upperOutputErrorDeriv.MemPtr, layerMaxPooling_Index.MemPtr, input_batch.Seg_Idx_Mem, input_batch.Seg_Margin_Mem, input_batch.segsize, winSize,
batchsize, outputDimension, input_batch.Fea_Idx_Mem, input_batch.Fea_Value_Mem, weightDeriv.MemPtr, inputDimension);
}
public void Max_Pooling(CudaPieceFloat layerPoolingOutput, BatchSample_Input data, CudaPieceFloat output, CudaPieceInt layerMaxPooling_Index, int outputDimension)
{
BasicMathlib.Max_Pooling(layerPoolingOutput.MemPtr, data.Sample_Idx_Mem, data.batchsize, output.MemPtr, layerMaxPooling_Index.MemPtr, outputDimension);
}
public void Convolution_Sparse_Matrix_Multiply_INTEX(BatchSample_Input data, CudaPieceFloat weight, CudaPieceFloat layerPoolingOutput, int inputDimension, int outputDimension, int winSize)
{
BasicMathlib.Convolution_Sparse_Matrix_Multiply_INTEX(data.Sample_Idx_Mem, data.batchsize, data.Seg_Idx_Mem, data.Seg_Margin_Mem, data.Seg_Len_Mem, data.segsize, data.Fea_Idx_Mem, data.Fea_Value_Mem, data.elementsize,
weight.MemPtr, layerPoolingOutput.MemPtr, inputDimension, outputDimension, winSize);
}
public void Sparse2Dense_Matrix(BatchSample_Input data, CudaPieceFloat matrix, int batchsize, int outputDimension)
{
Cudalib.Sparse2Dense_Matrix(data.Seg_Idx, data.Fea_Idx, data.Fea_Value, matrix.CudaPtr, batchsize, outputDimension);
}
public void SEQ_Sparse_Matrix_Multiply_INTEX(BatchSample_Input data, CudaPieceFloat weight, CudaPieceFloat output, int inputDimension, int outputDimension, int winSize)
{
Cudalib.SEQ_Sparse_Matrix_Multiply_INTEX(data.Sample_Idx, data.batchsize, data.Seg_Idx, data.Seg_Margin, data.Seg_Len, data.segsize, data.Fea_Idx, data.Fea_Value, data.elementsize,
weight.CudaPtr, output.CudaPtr, inputDimension, outputDimension, winSize);
}
public virtual void AnalysisBatch(BatchSample_Input input)
{
}
public virtual void ProcessBatch(BatchSample_Input input)
{
}
public void SEQ_Sparse_Matrix_Transpose_Multiply_INTEX(BatchSample_Input input_batch, CudaPieceFloat weightDeriv, CudaPieceFloat upperOutputErrorDeriv, int inputDimension, int outputDimension, int winSize)
{
BasicMathlib.SEQ_Sparse_Matrix_Transpose_Multiply_INTEX(input_batch.Sample_Idx_Mem, input_batch.batchsize, input_batch.Seg_Idx_Mem, input_batch.Seg_Margin_Mem, input_batch.Seg_Len_Mem, input_batch.segsize, input_batch.Fea_Idx_Mem, input_batch.Fea_Value_Mem, input_batch.elementsize,
weightDeriv.MemPtr, upperOutputErrorDeriv.MemPtr, inputDimension, outputDimension, winSize);
}