public void Matrix_WeightAdd_EX_Full(CudaPieceFloat gpu_floats_a, CudaPieceFloat gpu_floats_b, CudaPieceInt inver_neg_index,
CudaPieceInt inver_neg_value, int nTrial, int BATCHSIZE, int batchsize, int dimension, CudaPieceFloat mweight,
int start, int keep)
{
Cudalib.Matrix_WeightAdd_EX_Full(gpu_floats_a.CudaPtr, gpu_floats_b.CudaPtr, inver_neg_index.CudaPtr,
inver_neg_value.CudaPtr, nTrial, BATCHSIZE, batchsize, dimension, mweight.CudaPtr, start, keep);
}
public void Dispose()
{
if (cudaPiecePointer != IntPtr.Zero)
{
Cudalib.CudaDeallocInt(cudaPiecePointer);
cudaPiecePointer = IntPtr.Zero;
}
}
public void Matrix_Multipy(CudaPieceFloat input, CudaPieceFloat weight, CudaPieceFloat output, int batchsize, int inputDimension, int outputDimension, int inverse)
{
if (ParameterSetting.CuBlasEnable)
{
Cudalib.CUBLAS_Matrix_Multipy(input.CudaPtr, weight.CudaPtr, output.CudaPtr, batchsize, inputDimension, outputDimension, inverse);
}
else
{
Cudalib.Matrix_Multipy(input.CudaPtr, weight.CudaPtr, output.CudaPtr, batchsize, inputDimension, outputDimension, inverse);
}
}
/// <summary>
///
/// </summary>
public void Zero()
{
if (cpuMemArray != null)
{
for (int i = 0; i < Size; ++i)
{
cpuMemArray[i] = 0;
}
}
if (cudaPiecePointer != IntPtr.Zero)
{
Cudalib.Zero(cudaPiecePointer, Size);
}
}
/// <summary>
/// Copy data from GPU to CPU
/// </summary>
unsafe public void CopyOutFromCuda(int SpecifiedSize)
{
if (cudaPiecePointer == IntPtr.Zero)
{
return;
}
if (cpuMemArray == null)
{
throw new Exception("Error! Must set needCpuMem=true for CopyOutFromCuda() operation!");
}
fixed(float *pb = &cpuMemArray[0])
{
Cudalib.CudaCopyOutFloat(cudaPiecePointer, (IntPtr)pb, SpecifiedSize);
}
}
unsafe public void CopyIntoCuda()
{
if (cudaPiecePointer == IntPtr.Zero)
{
return;
}
if (cpuMemArray == null)
{
throw new Exception("Error! Must set needCpuMem=true for CopyIntoCuda() operation!");
}
fixed(int *gpu_ptr = cpuMemArray)
{
Cudalib.CudaCopyInInt(cudaPiecePointer, (IntPtr)gpu_ptr, size);
//Amplib.CopyInInt(GPU_negative_index[i], (IntPtr)gpu_neg, batchSize);
}
}
/// <summary>
///
/// </summary>
/// <param name="length"></param>
/// <param name="needCpuMem"></param>
public CudaPieceFloat(int length, bool needCpuMem, bool needGpuMem)
{
// the input is given assuming MATH_LIB = gpu
// So if cpu is used, we will overwrite
if (ParameterSetting.MATH_LIB == MathLibType.cpu)
{
needCpuMem = true;
needGpuMem = false;
}
size = length;
if (needCpuMem)
{
cpuMemArray = new float[size];
}
if (needGpuMem)
{
if ((Int64)(cudaPiecePointer = Cudalib.CudaAllocFloat(size)) == 0)
{
throw new Exception("Out of GPU Memo, use a smaller model!");
}
}
}
public void Deriv_Cosine_Rectified_EX_Full(CudaPieceFloat q, CudaPieceFloat d, CudaPieceInt neg_list, CudaPieceFloat dcq, CudaPieceFloat dcd,
int nTrail, int BATCHSIZE, int batchsize, int m, float eps)
{
Cudalib.Deriv_Cosine_Rectified_EX_Full(q.CudaPtr, d.CudaPtr, neg_list.CudaPtr, dcq.CudaPtr, dcd.CudaPtr, nTrail, BATCHSIZE, batchsize, m, eps);
}
public void FillOut_Dist_NCE_Full(CudaPieceFloat dist, CudaPieceInt neg_list, int nTrail, int BATCH_SIZE, int batchsize)
{
Cudalib.FillOut_Dist_NCE_Full(dist.CudaPtr, neg_list.CudaPtr, nTrail, BATCH_SIZE, batchsize);
}
public void Cosine_Similarity_EX_Full(CudaPieceFloat a, CudaPieceFloat b, CudaPieceInt neg_list, CudaPieceFloat c, int nTrial, int BATCHSIZE,
int batchsize, int dimension, float eps)
{
Cudalib.Cosine_Similarity_EX_Full(a.CudaPtr, b.CudaPtr, neg_list.CudaPtr, c.CudaPtr, nTrial, BATCHSIZE, batchsize, dimension, eps);
}
public void Sparse2Dense_Matrix(BatchSample_Input data, CudaPieceFloat matrix, int batchsize, int outputDimension)
{
Cudalib.Sparse2Dense_Matrix(data.Seg_Idx, data.Fea_Idx, data.Fea_Value, matrix.CudaPtr, batchsize, outputDimension);
}
public static void LoadArgs(string conf_filename)
{
FileStream mstream = new FileStream(conf_filename, FileMode.Open, FileAccess.Read);
StreamReader mreader = new StreamReader(mstream);
while (!mreader.EndOfStream)
{
string[] cmds = mreader.ReadLine().Split('\t');
if (cmds.Length < 2)
{
continue;
}
if (cmds[0].Equals("CORPUS"))
{
CORPUS = cmds[1];
}
if (cmds[0].Equals("EMB_FILE"))
{
EMB_FILE = cmds[1];
}
if (cmds[0].Equals("FEA_LIST"))
{
string feaList = cmds[1];
string[] feas = feaList.Split(new char[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries);
if (feas.Contains("l3g"))
{
featureList.l3g = true;
}
if (feas.Contains("root"))
{
featureList.root = true;
}
if (feas.Contains("infl"))
{
featureList.infl = true;
}
}
//if (cmds[0].Equals("FEAFILE"))
//{
// FEAFILE = cmds[1];
//}
if (cmds[0].Equals("LINEAR_MAPPING"))
{
Linear_Mapping = int.Parse(cmds[1].Trim().ToUpper());
}
if (cmds[0].Equals("reserved_settings"))
{
reserved_settings = cmds[1];
}
if (cmds[0].Equals("CUBLAS"))
{
if (int.Parse(cmds[1]) == 1)
{
CuBlasEnable = true;
}
else
{
CuBlasEnable = false;
}
}
if (cmds[0].Equals("NTRIAL"))
{
NTRIAL = int.Parse(cmds[1]);
}
if (cmds[0].Equals("BATCHSIZE"))
{
BATCH_SIZE = int.Parse(cmds[1]);
}
if (cmds[0].Equals("PARM_GAMMA"))
{
PARM_GAMMA = float.Parse(cmds[1]);
}
if (cmds[0].Equals("MAX_ITER"))
{
MAX_ITER = int.Parse(cmds[1]);
}
if (cmds[0].Equals("SHALLOW_SOURCE"))
{
SHALLOW_SOURCE = cmds[1];
IS_SHALLOW = true;
}
if (cmds[0].Equals("SHALLOW_TARGET"))
{
SHALLOW_TARGET = cmds[1];
IS_SHALLOW = true;
}
if (cmds[0].Equals("DEVICE"))
{
device = int.Parse(cmds[1]);
Cudalib.CudaSetDevice(device);
}
if (cmds[0].Equals("LFILE"))
{
LFILE = cmds[1];
}
if (cmds[0].Equals("LEARNINGRATE"))
{
LearningParameters.lr_begin = float.Parse(cmds[1]);
LearningParameters.lr_mid = float.Parse(cmds[1]);
LearningParameters.lr_latter = float.Parse(cmds[1]);
LearningParameters.learning_rate = float.Parse(cmds[1]);
}
if (cmds[0].Equals("SEEDMODEL1"))
{
SEEDMODEL1 = cmds[1];
ISSEED = true;
//NOTrain = true;
}
if (cmds[0].Equals("SEEDMODEL2"))
{
SEEDMODEL2 = cmds[1];
ISSEED = true;
//NOTrain = true;
}
if (cmds[0].Equals("SEEDMODEL3"))
{
SEEDMODEL3 = cmds[1];
ISSEED = true;
}
if (cmds[0].Equals("SEEDMODEL4"))
{
SEEDMODEL4 = cmds[1];
ISSEED = true;
}
if (cmds[0].Equals("SOURCE_LAYER_DIM"))
{
string[] items = cmds[1].Split(',');
SOURCE_LAYER_DIM = new int[items.Length];
int i = 0;
foreach (string s in items)
{
SOURCE_LAYER_DIM[i] = int.Parse(s);
i++;
}
}
if (cmds[0].Equals("TARGET_LAYER_DIM"))
{
string[] items = cmds[1].Split(',');
TARGET_LAYER_DIM = new int[items.Length];
int i = 0;
foreach (string s in items)
{
TARGET_LAYER_DIM[i] = int.Parse(s);
i++;
}
}
if (cmds[0].Equals("VALIDATEQFILE"))
{
VALIDATE_QFILE = cmds[1];
}
if (cmds[0].Equals("VALIDATEDFILE"))
{
VALIDATE_DFILE = cmds[1];
}
if (cmds[0].Equals("VALIDATEPAIR"))
{
VALIDATE_QDPAIR = cmds[1];
}
if (cmds[0].Equals("VALIDATEPROCESS"))
{
VALIDATE_PROCESS = cmds[1];
ISVALIDATE = true;
}
if (cmds[0].Equals("VALIDATE_MODEL_ONLY"))
{
if (int.Parse(cmds[1]) == 1)
{
VALIDATE_MODEL_ONLY = true;
}
else
{
VALIDATE_MODEL_ONLY = false;
}
}
if (cmds[0].Equals("EVULATIONEXE"))
{
EVULATION_EXE = cmds[1];
}
if (cmds[0].Equals("LOAD_MODEL_OLD_FORMAT"))
{
if (!bool.TryParse(cmds[1], out LoadModelOldFormat))
{
LoadModelOldFormat = cmds[1].Trim().Equals("1");
}
}
if (cmds[0].Equals("LOAD_INPUT_BACKWARD_COMPATIBLE_MODE"))
{
if (cmds[1].Equals("BOW", StringComparison.OrdinalIgnoreCase))
{
LoadInputBackwardCompatibleMode = "BOW";
FeatureValueAsInt = true;
}
else if (cmds[1].Equals("SEQ", StringComparison.OrdinalIgnoreCase))
{
LoadInputBackwardCompatibleMode = "SEQ";
FeatureValueAsInt = true;
}
}
if (cmds[0].Equals("FEATURE_VALUE_AS_INT"))
{
if (!bool.TryParse(cmds[1], out FeatureValueAsInt))
{
FeatureValueAsInt = cmds[1].Trim().Equals("1");
}
}
if (cmds[0].Equals("MATH_LIB"))
{
if (cmds[1].Trim().ToUpper() == "CPU")
{
MATH_LIB = MathLibType.cpu;
}
else
{
MATH_LIB = MathLibType.gpu;
}
}
if (cmds[0].Equals("CPU_MATH_LIB_THREAD_NUM"))
{
ParameterSetting.BasicMathLibThreadNum = int.Parse(cmds[1]);
if (ParameterSetting.BasicMathLibThreadNum < 1)
{
throw new Exception("Error! CPU_MATH_LIB_THREAD_NUM should be >= 1");
}
}
if (cmds[0].Equals("RANDOM_SEED"))
{
RANDOM_SEED = int.Parse(cmds[1]);
if (RANDOM_SEED >= 0)
{
PSEUDO_RANDOM = true;
}
else
{
PSEUDO_RANDOM = false;
}
}
}
if (PSEUDO_RANDOM)
{
Random = new Random(RANDOM_SEED);
}
else
{
Random = new Random();
}
mreader.Close();
mstream.Close();
InflModel.wordListPath = DIC;
InflModel.infl2oriPath = FEAFILE + "infl2ori.txt";
InflModel.oriListpath = FEAFILE + "oriList.txt";
RootModel.rootListpath = FEAFILE + "rootList.txt";
RootModel.word2rootsPath = FEAFILE + "word2roots.txt";
Console.WriteLine(CORPUS);
}
public void Deriv_Cosine_Subspace(CudaPieceFloat q, CudaPieceFloat d, CudaPieceFloat dcq, CudaPieceFloat dcd, CudaPieceFloat alpha, int act_type, int batchsize, int labelDim, int subspaceDim, float gamma, float eps)
{
Cudalib.Deriv_Cosine_Subspace(q.CudaPtr, d.CudaPtr, dcq.CudaPtr, dcd.CudaPtr, alpha.CudaPtr, act_type, batchsize, labelDim, subspaceDim, gamma, eps);
}
public void Cosine_Similarity_SubSpace(CudaPieceFloat a, CudaPieceFloat b, CudaPieceFloat c, int labelDim, int BATCHSIZE, int batchsize, int subspaceDim, float eps)
{
Cudalib.Cosine_Similarity_SubSpace(a.CudaPtr, b.CudaPtr, c.CudaPtr, labelDim, BATCHSIZE, batchsize, subspaceDim, eps);
}
public void SEQ_Sparse_Matrix_Transpose_Multiply_INTEX(BatchSample_Input input_batch, CudaPieceFloat weightDeriv, CudaPieceFloat upperOutputErrorDeriv, int inputDimension, int outputDimension, int winSize)
{
Cudalib.SEQ_Sparse_Matrix_Transpose_Multiply_INTEX(input_batch.Sample_Idx, input_batch.batchsize, input_batch.Seg_Idx, input_batch.Seg_Margin, input_batch.Seg_Len, input_batch.segsize, input_batch.Fea_Idx, input_batch.Fea_Value, input_batch.elementsize,
weightDeriv.CudaPtr, upperOutputErrorDeriv.CudaPtr, inputDimension, outputDimension, winSize);
}
public void Deriv_Rectified(CudaPieceFloat errorDeriv, CudaPieceFloat output, int batchsize, int inputDimension)
{
Cudalib.Deriv_Rectified(errorDeriv.CudaPtr, output.CudaPtr, batchsize, inputDimension);
}
public void Matrix_WeightAdd_EX(CudaPieceFloat result, CudaPieceFloat addTerm, CudaPieceInt GPU_Inver_negative_index, CudaPieceInt GPU_Inver_negative_value, int batchsize, int outputLayerSize, CudaPieceFloat mweight, int start, int keep)
{
Cudalib.Matrix_WeightAdd_EX(result.CudaPtr, addTerm.CudaPtr, GPU_Inver_negative_index.CudaPtr, GPU_Inver_negative_value.CudaPtr, batchsize, outputLayerSize, mweight.CudaPtr, start, keep);
}
public void Matrix_WeightAdd(CudaPieceFloat result, CudaPieceFloat addTerm, int batchsize, int outputLayerSize, CudaPieceFloat mweight, int start, int keep)
{
Cudalib.Matrix_WeightAdd(result.CudaPtr, addTerm.CudaPtr, batchsize, outputLayerSize, mweight.CudaPtr, start, keep);
}
public void Derive_Cosine_Rectified_EX(CudaPieceFloat srcTopLayerOutput, CudaPieceFloat tgtTopLayerOutput, CudaPieceInt GPU_negative_index, CudaPieceFloat srcTopLayerOutputDeriv, CudaPieceFloat tgtTopLayerOutputDeriv, int batchsize, int outputLayerSize, float eps)
{
Cudalib.Derive_Cosine_Rectified_EX(srcTopLayerOutput.CudaPtr, tgtTopLayerOutput.CudaPtr, GPU_negative_index.CudaPtr, srcTopLayerOutputDeriv.CudaPtr, tgtTopLayerOutputDeriv.CudaPtr, batchsize, outputLayerSize, eps);
}
public void Matrix_WeightAdd_Full(CudaPieceFloat gpu_floats_a, CudaPieceFloat gpu_floats_b, int nTrail, int BATCHSIZE, int batchsize, int dimension,
CudaPieceFloat mweight, int start, int keep)
{
Cudalib.Matrix_WeightAdd_Full(gpu_floats_a.CudaPtr, gpu_floats_b.CudaPtr, nTrail, BATCHSIZE, batchsize, dimension,
mweight.CudaPtr, start, keep);
}
public void Convolution_Sparse_Matrix_Product_INTEX(CudaPieceFloat upperOutputErrorDeriv, CudaPieceInt layerMaxPooling_Index, BatchSample_Input input_batch, int winSize, int batchsize, int outputDimension, CudaPieceFloat weightDeriv, int inputDimension)
{
Cudalib.Convolution_Sparse_Matrix_Product_INTEX(upperOutputErrorDeriv.CudaPtr, layerMaxPooling_Index.CudaPtr, input_batch.Seg_Idx, input_batch.Seg_Margin, input_batch.segsize, winSize,
batchsize, outputDimension, input_batch.Fea_Idx, input_batch.Fea_Value, weightDeriv.CudaPtr, inputDimension);
}
public void Deriv_InnerProduct(CudaPieceFloat q, CudaPieceFloat d, CudaPieceFloat dcq, CudaPieceFloat dcd, CudaPieceFloat alpha, int act_type, int batchsize, int Dim, float gamma, float eps)
{
Cudalib.Deriv_InnerProduct(q.CudaPtr, d.CudaPtr, dcq.CudaPtr, dcd.CudaPtr, alpha.CudaPtr, act_type, batchsize, Dim, gamma, eps);
}
public void SoftMax(CudaPieceFloat a, CudaPieceFloat b, int labelDim, int batchsize, float gamma)
{
Cudalib.SoftMax(a.CudaPtr, b.CudaPtr, labelDim, batchsize, gamma);
}
public void Matrix_Product(CudaPieceFloat lowerOutput, CudaPieceFloat upperOutputErrorDeriv, CudaPieceFloat weightDeriv, int batchsize, int inputDimension, int outputDimension)
{
Cudalib.Matrix_Product(lowerOutput.CudaPtr, upperOutputErrorDeriv.CudaPtr, weightDeriv.CudaPtr,
batchsize, inputDimension, outputDimension);
}
public void InnerProduct_Similarity(CudaPieceFloat a, CudaPieceFloat b, CudaPieceFloat c, int batchsize, int dimension)
{
Cudalib.InnerProduct_Similarity(a.CudaPtr, b.CudaPtr, c.CudaPtr, batchsize, dimension);
}
public void Matrix_Aggragate(CudaPieceFloat a, CudaPieceFloat b, int batchsize, int m)
{
Cudalib.Matrix_Aggragate(a.CudaPtr, b.CudaPtr, batchsize, m);
}
public void Matrix_Add_OFFSET(CudaPieceFloat a, int offset_a, CudaPieceFloat b, int offset_b, int len, float mweight)
{
Cudalib.Matrix_Add_OFFSET(a.CudaPtr, offset_a, b.CudaPtr, offset_b, len, mweight);
}
public void Scale_Matrix(CudaPieceFloat matrix, int inputDimension, int outputDimnsion, float momentum)
{
Cudalib.Scale_Matrix(matrix.CudaPtr, inputDimension, outputDimnsion, momentum);
}
public static void Main()
{
try
{
string logDirecotry = new FileInfo(ParameterSetting.Log_FileName).Directory.FullName;
if (!Directory.Exists(logDirecotry))
{
Directory.CreateDirectory(logDirecotry);
}
log_stream = new FileStream(ParameterSetting.Log_FileName, FileMode.Append, FileAccess.Write);
log_writer = new StreamWriter(log_stream);
string modelDirectory = new FileInfo(ParameterSetting.MODEL_PATH).Directory.FullName;
if (!Directory.Exists(modelDirectory))
{
Directory.CreateDirectory(modelDirectory);
}
timer.Reset();
timer.Start();
Print("Loading doc Query Stream ....");
if (ParameterSetting.CuBlasEnable)
{
Cudalib.CUBLAS_Init();
}
//Load_Train_PairData(ParameterSetting.QFILE, ParameterSetting.DFILE);
DNN_Train dnnTrain = null;
/// 1. loading training dataset.
dnnTrain = new DSSM_Train();
dnnTrain.LoadTrainData(new string[] { ParameterSetting.QFILE, ParameterSetting.DFILE });
if (ParameterSetting.ISVALIDATE)
{
if (!ParameterSetting.VALIDATE_MODEL_ONLY)
{
dnnTrain.LoadValidateData(new string[] { ParameterSetting.VALIDATE_QFILE, ParameterSetting.VALIDATE_DFILE, ParameterSetting.VALIDATE_QDPAIR });
}
else
{
Program.Print("Validation process without stream; model only");
}
}
/// 2. loading config and start to train.
dnnTrain.ModelInit_FromConfig();
dnnTrain.Training();
dnnTrain.Dispose();
log_writer.Close();
log_stream.Close();
if (ParameterSetting.CuBlasEnable)
{
Cudalib.CUBLAS_Destroy();
}
}
catch (Exception exc)
{
Console.Error.WriteLine(exc.ToString());
Environment.Exit(0);
}
}
public void Matrix_Add(CudaPieceFloat matrix, CudaPieceFloat updates, int inputDimension, int outputDimnsion, float learning_rate)
{
Cudalib.Matrix_Add(matrix.CudaPtr, updates.CudaPtr, inputDimension, outputDimnsion, learning_rate);
}