public void Matrix_WeightAdd_EX_Full(CudaPieceFloat gpu_floats_a, CudaPieceFloat gpu_floats_b, CudaPieceInt inver_neg_index,
CudaPieceInt inver_neg_value, int nTrial, int BATCHSIZE, int batchsize, int dimension, CudaPieceFloat mweight,
int start, int keep)
{
BasicMathlib.Matrix_WeightAdd_EX_Full(gpu_floats_a.MemPtr, gpu_floats_b.MemPtr, inver_neg_index.MemPtr,
inver_neg_value.MemPtr, nTrial, BATCHSIZE, batchsize, dimension, mweight.MemPtr, start, keep);
}
/// <summary>
/// End of Batch Grad.
/// </summary>
public override void EndBatch()
{
if (GradIdx == 0)
{
MathOperatorManager.GlobalInstance.Zero(direction, Num);
MathOperatorManager.GlobalInstance.Matrix_Add_OFFSET(direction, 0, grad_list[GradIdx % GradHistory], 0, Num, 1);
}
else
{
//yk = gk+1−gk -yk = gk - gk+1
//βHS(k) = gT(k+1)y(k) / dT(k) y(k) (1952) in the original (linear) CG paper of Hestenes and Stiefel [59]
//Cudalib.Matrix_Add_OFFSET(grad_list[(GradIdx - 1) % GradHistory].CudaPtr, 0, grad_list[GradIdx % GradHistory].CudaPtr, 0, Num, -1);
/*grad_list[(GradIdx - 1) % GradHistory].CopyOutFromCuda();
* direction.CopyOutFromCuda();
* grad_list[GradIdx % GradHistory].CopyOutFromCuda();
*
* float g1 = BasicMathlib.VectorInnerProduct(grad_list[GradIdx % GradHistory].MemPtr, 0, grad_list[GradIdx % GradHistory].MemPtr, 0, Num);
* float g2 = BasicMathlib.VectorInnerProduct(grad_list[GradIdx % GradHistory].MemPtr, 0, grad_list[(GradIdx - 1) % GradHistory].MemPtr, 0, Num);
* float s1 = BasicMathlib.VectorInnerProduct(direction.MemPtr, 0, grad_list[(GradIdx ) % GradHistory].MemPtr, 0, Num);
* float s2 = BasicMathlib.VectorInnerProduct(direction.MemPtr, 0, grad_list[(GradIdx - 1) % GradHistory].MemPtr, 0, Num);
* //float dy = BasicMathlib.VectorInnerProduct(direction.MemPtr, 0, grad_list[(GradIdx - 1) % GradHistory].MemPtr, 0, Num);
*
* float beta = 0;
* if (Math.Abs(s1-s2) > float.Epsilon)
* {
* beta = (g1 - g2) / (s1 - s2);
* }
*/
grad_list[(GradIdx - 1) % GradHistory].CopyOutFromCuda();
grad_list[GradIdx % GradHistory].CopyOutFromCuda();
float gk = BasicMathlib.VectorInnerProduct(grad_list[GradIdx % GradHistory].MemPtr, 0, grad_list[GradIdx % GradHistory].MemPtr, 0, Num);
float gk_1 = BasicMathlib.VectorInnerProduct(grad_list[(GradIdx - 1) % GradHistory].MemPtr, 0, grad_list[(GradIdx - 1) % GradHistory].MemPtr, 0, Num);
float beta = gk * 1.0f / (gk_1 + float.Epsilon);
Console.WriteLine("Beta Value ....................." + beta.ToString());
//dk+1 =−gk+1 + βk dk, d0 =−g0
MathOperatorManager.GlobalInstance.Scale_Matrix(direction, 1, Num, beta); // grad_list[GradIdx % GradHistory].CudaPtr, 0, Num, 1);
MathOperatorManager.GlobalInstance.Matrix_Add(direction, grad_list[GradIdx % GradHistory], 1, Num, 1.0f);
}
//xk+1 = xk + αk dk
MathOperatorManager.GlobalInstance.Matrix_Add(parameters, direction, 1, Num, -LearnRate);
}
public void Matrix_Add_OFFSET(CudaPieceFloat a, int offset_a, CudaPieceFloat b, int offset_b, int len, float mweight)
{
BasicMathlib.Matrix_Add_OFFSET(a.MemPtr, offset_a, b.MemPtr, offset_b, len, mweight);
}
public void InnerProduct_Similarity(CudaPieceFloat a, CudaPieceFloat b, CudaPieceFloat c, int batchsize, int dimension)
{
BasicMathlib.InnerProduct_Similarity(a.MemPtr, b.MemPtr, c.MemPtr, batchsize, dimension);
}
public void SoftMax(CudaPieceFloat a, CudaPieceFloat b, int labelDim, int batchsize, float gamma)
{
BasicMathlib.SoftMax(a.MemPtr, b.MemPtr, labelDim, batchsize, gamma);
}
public void Matrix_Aggragate(CudaPieceFloat a, CudaPieceFloat b, int batchsize, int m)
{
BasicMathlib.Matrix_Aggragate(a.MemPtr, b.MemPtr, batchsize, m);
}
public void Matrix_Add(CudaPieceFloat matrix, CudaPieceFloat updates, int inputDimension, int outputDimnsion, float learning_rate)
{
BasicMathlib.Matrix_Add(matrix.MemPtr, updates.MemPtr, inputDimension, outputDimnsion, learning_rate);
}
public void Matrix_Product(CudaPieceFloat lowerOutput, CudaPieceFloat upperOutputErrorDeriv, CudaPieceFloat weightDeriv, int batchsize, int inputDimension, int outputDimension)
{
BasicMathlib.Matrix_Product(lowerOutput.MemPtr, upperOutputErrorDeriv.MemPtr, weightDeriv.MemPtr,
batchsize, inputDimension, outputDimension);
}
public void Calculate_Alpha_NCE2(CudaPieceFloat alpha, CudaPieceFloat dist, int nTrailPlus1, int BATCH_SIZE, int batchsize, float GAMMA)
{
BasicMathlib.Calculate_Alpha_NCE2(alpha.MemPtr, dist.MemPtr, nTrailPlus1, BATCH_SIZE, batchsize, GAMMA);
}
public void Cosine_Similarity_EX(CudaPieceFloat srcTopLayerOutput, CudaPieceFloat tgtTopLayerOutput, CudaPieceInt GPU_negative_index, CudaPieceFloat alpha, int nTrailPlus1, int BATCH_SIZE, int mIndex, int batchsize, int topLayerSize, float eps)
{
BasicMathlib.Cosine_Similarity_EX(srcTopLayerOutput.MemPtr,
tgtTopLayerOutput.MemPtr, GPU_negative_index.MemPtr, alpha.MemPtr, nTrailPlus1, BATCH_SIZE, mIndex,
batchsize, topLayerSize, eps);
}
public void Matrix_Rectified_Vector(CudaPieceFloat output, CudaPieceFloat bias, int batchsize, int outputDimension)
{
BasicMathlib.Matrix_Rectified_Vector(output.MemPtr, bias.MemPtr, batchsize, outputDimension);
}
public void Matrix_Add_Tanh(CudaPieceFloat output, CudaPieceFloat bias, int batchsize, int outputDimension)
{
BasicMathlib.Matrix_Add_Tanh(output.MemPtr, bias.MemPtr, batchsize, outputDimension);
}
public void Matrix_Multipy(CudaPieceFloat input, CudaPieceFloat weight, CudaPieceFloat output, int batchsize, int inputDimension, int outputDimension, int inverse)
{
BasicMathlib.Matrix_Multipy(input.MemPtr, weight.MemPtr, output.MemPtr, batchsize, inputDimension, outputDimension, inverse);
}
public void Max_Pooling(CudaPieceFloat layerPoolingOutput, BatchSample_Input data, CudaPieceFloat output, CudaPieceInt layerMaxPooling_Index, int outputDimension)
{
BasicMathlib.Max_Pooling(layerPoolingOutput.MemPtr, data.Sample_Idx_Mem, data.batchsize, output.MemPtr, layerMaxPooling_Index.MemPtr, outputDimension);
}
public void Convolution_Sparse_Matrix_Multiply_INTEX(BatchSample_Input data, CudaPieceFloat weight, CudaPieceFloat layerPoolingOutput, int inputDimension, int outputDimension, int winSize)
{
BasicMathlib.Convolution_Sparse_Matrix_Multiply_INTEX(data.Sample_Idx_Mem, data.batchsize, data.Seg_Idx_Mem, data.Seg_Margin_Mem, data.Seg_Len_Mem, data.segsize, data.Fea_Idx_Mem, data.Fea_Value_Mem, data.elementsize,
weight.MemPtr, layerPoolingOutput.MemPtr, inputDimension, outputDimension, winSize);
}
public void SEQ_Sparse_Matrix_Transpose_Multiply_INTEX(BatchSample_Input input_batch, CudaPieceFloat weightDeriv, CudaPieceFloat upperOutputErrorDeriv, int inputDimension, int outputDimension, int winSize)
{
BasicMathlib.SEQ_Sparse_Matrix_Transpose_Multiply_INTEX(input_batch.Sample_Idx_Mem, input_batch.batchsize, input_batch.Seg_Idx_Mem, input_batch.Seg_Margin_Mem, input_batch.Seg_Len_Mem, input_batch.segsize, input_batch.Fea_Idx_Mem, input_batch.Fea_Value_Mem, input_batch.elementsize,
weightDeriv.MemPtr, upperOutputErrorDeriv.MemPtr, inputDimension, outputDimension, winSize);
}
public void Convolution_Sparse_Matrix_Product_INTEX(CudaPieceFloat upperOutputErrorDeriv, CudaPieceInt layerMaxPooling_Index, BatchSample_Input input_batch, int winSize, int batchsize, int outputDimension, CudaPieceFloat weightDeriv, int inputDimension)
{
BasicMathlib.Convolution_Sparse_Matrix_Product_INTEX(upperOutputErrorDeriv.MemPtr, layerMaxPooling_Index.MemPtr, input_batch.Seg_Idx_Mem, input_batch.Seg_Margin_Mem, input_batch.segsize, winSize,
batchsize, outputDimension, input_batch.Fea_Idx_Mem, input_batch.Fea_Value_Mem, weightDeriv.MemPtr, inputDimension);
}
public void Calculate_Alpha_PAIRRANK(CudaPieceFloat alpha, int nTrailPlus1, int BATCH_SIZE, int batchsize, float GAMMA)
{
BasicMathlib.Calculate_Alpha_PAIRRANK(alpha.MemPtr, nTrailPlus1, BATCH_SIZE, batchsize, GAMMA);
}
public void Scale_Matrix(CudaPieceFloat matrix, int inputDimension, int outputDimnsion, float momentum)
{
BasicMathlib.Scale_Matrix(matrix.MemPtr, inputDimension, outputDimnsion, momentum);
}
public void FillOut_Dist_NCE(CudaPieceFloat dist, CudaPieceInt GPU_negative_index, int nTrailPlus1, int BATCH_SIZE, int mIndex, int batchsize)
{
BasicMathlib.FillOut_Dist_NCE(dist.MemPtr, GPU_negative_index.MemPtr, nTrailPlus1, BATCH_SIZE, mIndex, batchsize);
}
public void Sparse2Dense_Matrix(BatchSample_Input data, CudaPieceFloat matrix, int batchsize, int outputDimension)
{
BasicMathlib.Sparse2Dense_Matrix(data.Seg_Idx_Mem, data.Fea_Idx_Mem, data.Fea_Value_Mem, matrix.MemPtr, batchsize, outputDimension);
}
public void Derive_Cosine_Linear(CudaPieceFloat srcTopLayerOutput, CudaPieceFloat tgtTopLayerOutput, CudaPieceFloat srcTopLayerOutputDeriv, CudaPieceFloat tgtTopLayerOutputDeriv, int batchsize, int outputLayerSize, float eps)
{
BasicMathlib.Derive_Cosine_Linear(srcTopLayerOutput.MemPtr, tgtTopLayerOutput.MemPtr, srcTopLayerOutputDeriv.MemPtr, tgtTopLayerOutputDeriv.MemPtr, batchsize, outputLayerSize, eps);
}
public void Cosine_Similarity_SubSpace(CudaPieceFloat a, CudaPieceFloat b, CudaPieceFloat c, int labelDim, int BATCHSIZE, int batchsize, int subspaceDim, float eps)
{
BasicMathlib.Cosine_Similarity_SubSpace(a.MemPtr, b.MemPtr, c.MemPtr, labelDim, BATCHSIZE, batchsize, subspaceDim, eps);
}
public void Derive_Cosine_Rectified_EX(CudaPieceFloat srcTopLayerOutput, CudaPieceFloat tgtTopLayerOutput, CudaPieceInt GPU_negative_index, CudaPieceFloat srcTopLayerOutputDeriv, CudaPieceFloat tgtTopLayerOutputDeriv, int batchsize, int outputLayerSize, float eps)
{
BasicMathlib.Derive_Cosine_Rectified_EX(srcTopLayerOutput.MemPtr, tgtTopLayerOutput.MemPtr, GPU_negative_index.MemPtr, srcTopLayerOutputDeriv.MemPtr, tgtTopLayerOutputDeriv.MemPtr, batchsize, outputLayerSize, eps);
}
public void Deriv_Cosine_Subspace(CudaPieceFloat q, CudaPieceFloat d, CudaPieceFloat dcq, CudaPieceFloat dcd, CudaPieceFloat alpha, int act_type, int batchsize, int labelDim, int subspaceDim, float gamma, float eps)
{
BasicMathlib.Deriv_Cosine_Subspace(q.MemPtr, d.MemPtr, dcq.MemPtr, dcd.MemPtr, alpha.MemPtr, act_type, batchsize, labelDim, subspaceDim, gamma, eps);
}
public void Matrix_WeightAdd(CudaPieceFloat result, CudaPieceFloat addTerm, int batchsize, int outputLayerSize, CudaPieceFloat mweight, int start, int keep)
{
BasicMathlib.Matrix_WeightAdd(result.MemPtr, addTerm.MemPtr, batchsize, outputLayerSize, mweight.MemPtr, start, keep);
}
public void Deriv_InnerProduct(CudaPieceFloat q, CudaPieceFloat d, CudaPieceFloat dcq, CudaPieceFloat dcd, CudaPieceFloat alpha, int act_type, int batchsize, int Dim, float gamma, float eps)
{
BasicMathlib.Deriv_InnerProduct(q.MemPtr, d.MemPtr, dcq.MemPtr, dcd.MemPtr, alpha.MemPtr, act_type, batchsize, Dim, gamma, eps);
}
public void Matrix_WeightAdd_EX(CudaPieceFloat result, CudaPieceFloat addTerm, CudaPieceInt GPU_Inver_negative_index, CudaPieceInt GPU_Inver_negative_value, int batchsize, int outputLayerSize, CudaPieceFloat mweight, int start, int keep)
{
BasicMathlib.Matrix_WeightAdd_EX(result.MemPtr, addTerm.MemPtr, GPU_Inver_negative_index.MemPtr, GPU_Inver_negative_value.MemPtr, batchsize, outputLayerSize, mweight.MemPtr, start, keep);
}
public void Deriv_Rectified(CudaPieceFloat errorDeriv, CudaPieceFloat output, int batchsize, int inputDimension)
{
BasicMathlib.Deriv_Rectified(errorDeriv.MemPtr, output.MemPtr, batchsize, inputDimension);
}
public void Matrix_WeightAdd_Full(CudaPieceFloat gpu_floats_a, CudaPieceFloat gpu_floats_b, int nTrail, int BATCHSIZE, int batchsize, int dimension,
CudaPieceFloat mweight, int start, int keep)
{
BasicMathlib.Matrix_WeightAdd_Full(gpu_floats_a.MemPtr, gpu_floats_b.MemPtr, nTrail, BATCHSIZE, batchsize, dimension,
mweight.MemPtr, start, keep);
}
