/// <summary>
/// Train the model.
/// </summary>
/// <param name="bNewWts">Specifies whether to use new weights or load existing ones (if they exist).</param>
public void Train(bool bNewWts)
{
if (m_mycaffeTrain == null)
{
return;
}
byte[] rgWts = null;
if (!bNewWts)
{
rgWts = loadWeights();
}
if (rgWts == null)
{
Console.WriteLine("Starting with new weights...");
}
SolverParameter solver = createSolver();
NetParameter model = createModel();
string strModel = model.ToProto("root").ToString();
Console.WriteLine("Using Train Model:");
Console.WriteLine(strModel);
Console.WriteLine("Starting training...");
m_mycaffeTrain.LoadLite(Phase.TRAIN, solver.ToProto("root").ToString(), model.ToProto("root").ToString(), rgWts, false, false);
m_mycaffeTrain.SetOnTrainingStartOverride(new EventHandler(onTrainingStart));
m_mycaffeTrain.SetOnTestingStartOverride(new EventHandler(onTestingStart));
// Set clockwork weights.
if (m_param.LstmEngine != EngineParameter.Engine.CUDNN)
{
Net <float> net = m_mycaffeTrain.GetInternalNet(Phase.TRAIN);
Blob <float> lstm1 = net.parameters[2];
lstm1.SetData(1, m_param.Hidden, m_param.Hidden);
}
m_mycaffeTrain.Train(m_param.Iterations);
saveLstmState(m_mycaffeTrain);
Image img = SimpleGraphingControl.QuickRender(m_plots, 1000, 600);
showImage(img, "training.png");
saveWeights(m_mycaffeTrain.GetWeights());
}
public void TestCreateTrainingModel()
{
ModelBuilder builder = create();
NetParameter net_param = builder.CreateModel();
RawProto proto = net_param.ToProto("root");
string strNet = proto.ToString();
RawProto proto2 = RawProto.Parse(strNet);
NetParameter net_param2 = NetParameter.FromProto(proto2);
m_log.CHECK(net_param2.Compare(net_param), "The two net parameters should be the same!");
// verify creating the model.
SolverParameter solver = builder.CreateSolver();
RawProto protoSolver = solver.ToProto("root");
string strSolver = protoSolver.ToString();
SettingsCaffe settings = new SettingsCaffe();
CancelEvent evtCancel = new CancelEvent();
MyCaffeControl <T> mycaffe = new MyCaffeControl <T>(settings, m_log, evtCancel);
save(strNet, strSolver, false);
// mycaffe.LoadLite(Phase.TRAIN, strSolver, strNet, null);
mycaffe.Dispose();
}
public BeamSearchTest2(string strName, int nDeviceID, EngineParameter.Engine engine)
: base(strName, new List <int>() { 3, 2, 4, 1 }, nDeviceID)
{
m_engine = engine;
NetParameter net_param = new NetParameter();
LayerParameter input = new LayerParameter(LayerParameter.LayerType.INPUT);
input.input_param.shape.Add(new BlobShape(new List <int>()
{
1, 1, 1
}));
input.input_param.shape.Add(new BlobShape(new List <int>()
{
80, 1, 1
}));
input.input_param.shape.Add(new BlobShape(new List <int>()
{
80, 1, 1
}));
input.input_param.shape.Add(new BlobShape(new List <int>()
{
80, 1
}));
input.top.Add("dec");
input.top.Add("enc");
input.top.Add("encr");
input.top.Add("encc");
net_param.layer.Add(input);
string strModel = net_param.ToProto("root").ToString();
m_net = new Net <T>(m_cuda, m_log, net_param, new CancelEvent(), null);
InputLayerEx <T> layer = new InputLayerEx <T>(m_cuda, m_log, m_net.layers[0]);
layer.OnGetData += Layer_OnGetData;
m_net.layers[0] = layer;
m_rgTestSequences.Add("rdany but you can call me dany");
m_rgTestSequences.Add("rdany call me dany");
m_rgTestSequences.Add("rdany you can call me dany");
m_rgTestSequences.Add("my name is dany");
m_rgTestSequences.Add("call me dany");
m_rgrgTestSequenceIndexes = new List <List <int> >();
foreach (string strSequence in m_rgTestSequences)
{
string[] rgstrWords = strSequence.Split(' ');
List <int> rgIdx = new List <int>();
foreach (string strWord in rgstrWords)
{
int nIdx = layer.Vocabulary.WordToIndex(strWord);
rgIdx.Add(nIdx);
}
m_rgrgTestSequenceIndexes.Add(rgIdx);
}
}
/// <summary>
/// The ResizeModel method gives the custom trainer the opportunity to resize the model if needed.
/// </summary>
/// <param name="strModel">Specifies the model descriptor.</param>
/// <param name="rgVocabulary">Specifies the vocabulary.</param>
/// <param name="log">Specifies the output log.</param>
/// <returns>A new model discriptor is returned (or the same 'strModel' if no changes were made).</returns>
/// <remarks>Note, this method is called after PreloadData.</remarks>
string IXMyCaffeCustomTrainerRNN.ResizeModel(Log log, string strModel, BucketCollection rgVocabulary)
{
if (rgVocabulary == null || rgVocabulary.Count == 0)
{
return(strModel);
}
int nVocabCount = rgVocabulary.Count;
NetParameter p = NetParameter.FromProto(RawProto.Parse(strModel));
string strEmbedName = "";
EmbedParameter embed = null;
string strIpName = "";
InnerProductParameter ip = null;
foreach (LayerParameter layer in p.layer)
{
if (layer.type == LayerParameter.LayerType.EMBED)
{
strEmbedName = layer.name;
embed = layer.embed_param;
}
else if (layer.type == LayerParameter.LayerType.INNERPRODUCT)
{
strIpName = layer.name;
ip = layer.inner_product_param;
}
}
if (embed != null)
{
if (embed.input_dim != (uint)nVocabCount)
{
log.WriteLine("WARNING: Embed layer '" + strEmbedName + "' input dim changed from " + embed.input_dim.ToString() + " to " + nVocabCount.ToString() + " to accomodate for the vocabulary count.");
embed.input_dim = (uint)nVocabCount;
}
}
if (ip.num_output != (uint)nVocabCount)
{
log.WriteLine("WARNING: InnerProduct layer '" + strIpName + "' num_output changed from " + ip.num_output.ToString() + " to " + nVocabCount.ToString() + " to accomodate for the vocabulary count.");
ip.num_output = (uint)nVocabCount;
}
m_rgVocabulary = rgVocabulary;
RawProto proto = p.ToProto("root");
return(proto.ToString());
}
public void TestCreateDeployModel()
{
ModelBuilder builder = create();
NetParameter net_param = builder.CreateDeployModel();
RawProto proto = net_param.ToProto("root");
string strNet = proto.ToString();
RawProto proto2 = RawProto.Parse(strNet);
NetParameter net_param2 = NetParameter.FromProto(proto2);
m_log.CHECK(net_param2.Compare(net_param), "The two net parameters should be the same!");
// verify creating the model.
SettingsCaffe settings = new SettingsCaffe();
CancelEvent evtCancel = new CancelEvent();
MyCaffeControl <T> mycaffe = new MyCaffeControl <T>(settings, m_log, evtCancel);
save(strNet, null, true);
// mycaffe.LoadToRun(strNet, null, new BlobShape(1, 3, 300, 300));
mycaffe.Dispose();
}
/// <summary>
/// Replace the Data input layer with the MemoryData input layer.
/// </summary>
/// <param name="strModel">Specifies the model descriptor to change.</param>
/// <param name="nBatchSize">Specifies the batch size.</param>
/// <returns>The new model descriptor with the MemoryData layer is returned.</returns>
private string fixup_model(string strModel, int nBatchSize)
{
RawProto proto = RawProto.Parse(strModel);
NetParameter net_param = NetParameter.FromProto(proto);
for (int i = 0; i < net_param.layer.Count; i++)
{
if (net_param.layer[i].type == LayerParameter.LayerType.DATA)
{
LayerParameter layer = new LayerParameter(LayerParameter.LayerType.INPUT);
layer.name = net_param.layer[i].name;
layer.top = net_param.layer[i].top;
layer.bottom = net_param.layer[i].bottom;
layer.include = net_param.layer[i].include;
layer.input_param.shape.Add(new BlobShape(nBatchSize, 1, 28, 28));
layer.input_param.shape.Add(new BlobShape(nBatchSize, 1, 1, 1));
net_param.layer[i] = layer;
}
}
return(net_param.ToProto("root").ToString());
}
/// <summary>
/// The ResizeModel method gives the custom trainer the opportunity to resize the model if needed.
/// </summary>
/// <param name="strModel">Specifies the model descriptor.</param>
/// <param name="rgVocabulary">Specifies the vocabulary.</param>
/// <returns>A new model discriptor is returned (or the same 'strModel' if no changes were made).</returns>
/// <remarks>Note, this method is called after PreloadData.</remarks>
public string ResizeModel(string strModel, BucketCollection rgVocabulary)
{
if (rgVocabulary == null || rgVocabulary.Count == 0)
{
return(strModel);
}
int nVocabCount = rgVocabulary.Count;
NetParameter p = NetParameter.FromProto(RawProto.Parse(strModel));
EmbedParameter embed = null;
InnerProductParameter ip = null;
foreach (LayerParameter layer in p.layer)
{
if (layer.type == LayerParameter.LayerType.EMBED)
{
embed = layer.embed_param;
}
else if (layer.type == LayerParameter.LayerType.INNERPRODUCT)
{
ip = layer.inner_product_param;
}
}
if (embed != null)
{
embed.input_dim = (uint)nVocabCount;
}
ip.num_output = (uint)nVocabCount;
m_rgVocabulary = rgVocabulary;
RawProto proto = p.ToProto("root");
return(proto.ToString());
}
/// <summary>
/// The DoWork thread is the main tread used to train or run the model depending on the operation selected.
/// </summary>
/// <param name="sender">Specifies the sender</param>
/// <param name="e">specifies the arguments.</param>
private void m_bw_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker bw = sender as BackgroundWorker;
m_input = e.Argument as InputData;
SettingsCaffe s = new SettingsCaffe();
s.ImageDbLoadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL;
try
{
m_model.Batch = m_input.Batch;
m_mycaffe = new MyCaffeControl <float>(s, m_log, m_evtCancel);
// Train the model.
if (m_input.Operation == InputData.OPERATION.TRAIN)
{
m_model.Iterations = (int)((m_input.Epochs * 7000) / m_model.Batch);
m_log.WriteLine("Training for " + m_input.Epochs.ToString() + " epochs (" + m_model.Iterations.ToString("N0") + " iterations).", true);
m_log.WriteLine("INFO: " + m_model.Iterations.ToString("N0") + " iterations.", true);
m_log.WriteLine("Using hidden = " + m_input.HiddenSize.ToString() + ", and word size = " + m_input.WordSize.ToString() + ".", true);
// Load the Seq2Seq training model.
NetParameter netParam = m_model.CreateModel(m_input.InputFileName, m_input.TargetFileName, m_input.HiddenSize, m_input.WordSize, m_input.UseSoftmax, m_input.UseExternalIp);
string strModel = netParam.ToProto("root").ToString();
SolverParameter solverParam = m_model.CreateSolver(m_input.LearningRate);
string strSolver = solverParam.ToProto("root").ToString();
byte[] rgWts = loadWeights("sequence");
m_strModel = strModel;
m_strSolver = strSolver;
m_mycaffe.OnTrainingIteration += m_mycaffe_OnTrainingIteration;
m_mycaffe.OnTestingIteration += m_mycaffe_OnTestingIteration;
m_mycaffe.LoadLite(Phase.TRAIN, strSolver, strModel, rgWts, false, false);
if (!m_input.UseSoftmax)
{
MemoryLossLayer <float> lossLayerTraining = m_mycaffe.GetInternalNet(Phase.TRAIN).FindLayer(LayerParameter.LayerType.MEMORY_LOSS, "loss") as MemoryLossLayer <float>;
if (lossLayerTraining != null)
{
lossLayerTraining.OnGetLoss += LossLayer_OnGetLossTraining;
}
MemoryLossLayer <float> lossLayerTesting = m_mycaffe.GetInternalNet(Phase.TEST).FindLayer(LayerParameter.LayerType.MEMORY_LOSS, "loss") as MemoryLossLayer <float>;
if (lossLayerTesting != null)
{
lossLayerTesting.OnGetLoss += LossLayer_OnGetLossTesting;
}
}
m_blobProbs = new Blob <float>(m_mycaffe.Cuda, m_mycaffe.Log);
m_blobScale = new Blob <float>(m_mycaffe.Cuda, m_mycaffe.Log);
TextDataLayer <float> dataLayerTraining = m_mycaffe.GetInternalNet(Phase.TRAIN).FindLayer(LayerParameter.LayerType.TEXT_DATA, "data") as TextDataLayer <float>;
if (dataLayerTraining != null)
{
dataLayerTraining.OnGetData += DataLayerTraining_OnGetDataTraining;
}
// Train the Seq2Seq model.
m_plotsSequenceLoss = new PlotCollection("Sequence Loss");
m_plotsSequenceAccuracyTest = new PlotCollection("Sequence Accuracy Test");
m_plotsSequenceAccuracyTrain = new PlotCollection("Sequence Accuracy Train");
m_mycaffe.Train(m_model.Iterations);
saveWeights("sequence", m_mycaffe);
}
// Run a trained model.
else
{
NetParameter netParam = m_model.CreateModel(m_input.InputFileName, m_input.TargetFileName, m_input.HiddenSize, m_input.WordSize, m_input.UseSoftmax, m_input.UseExternalIp, Phase.RUN);
string strModel = netParam.ToProto("root").ToString();
byte[] rgWts = loadWeights("sequence");
strModel = m_model.PrependInput(strModel);
m_strModelRun = strModel;
int nN = m_model.TimeSteps;
m_mycaffe.LoadToRun(strModel, rgWts, new BlobShape(new List <int>()
{
nN, 1, 1, 1
}), null, null, false, false);
m_blobProbs = new Blob <float>(m_mycaffe.Cuda, m_mycaffe.Log);
m_blobScale = new Blob <float>(m_mycaffe.Cuda, m_mycaffe.Log);
runModel(m_mycaffe, bw, m_input.InputText);
}
}
catch (Exception excpt)
{
throw excpt;
}
finally
{
// Cleanup.
if (m_mycaffe != null)
{
m_mycaffe.Dispose();
m_mycaffe = null;
}
}
}
static void Main(string[] args)
{
if (!sqlCheck())
{
return;
}
Log log = new Log("test");
log.OnWriteLine += Log_OnWriteLine;
CancelEvent cancel = new CancelEvent();
SettingsCaffe settings = new SettingsCaffe();
// Load all images into memory before training.
settings.ImageDbLoadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL;
// Use GPU ID = 0
settings.GpuIds = "0";
// Load the descriptors from their respective files
string strSolver = load_file("C:\\ProgramData\\MyCaffe\\test_data\\models\\siamese\\mnist\\solver.prototxt");
string strModel = load_file("C:\\ProgramData\\MyCaffe\\test_data\\models\\siamese\\mnist\\train_val.prototxt");
RawProto proto = RawProto.Parse(strModel);
NetParameter net_param = NetParameter.FromProto(proto);
LayerParameter layer = net_param.FindLayer(LayerParameter.LayerType.DECODE);
layer.decode_param.target = DecodeParameter.TARGET.CENTROID;
proto = net_param.ToProto("root");
strModel = proto.ToString();
// Load the MNIST data descriptor.
DatasetFactory factory = new DatasetFactory();
DatasetDescriptor ds = factory.LoadDataset("MNIST");
// Create a test project with the dataset and descriptors
ProjectEx project = new ProjectEx("Test");
project.SetDataset(ds);
project.ModelDescription = strModel;
project.SolverDescription = strSolver;
// Crate the MyCaffeControl (with the 'float' base type)
string strCudaPath = "C:\\Program Files\\SignalPop\\MyCaffe\\cuda_11.3\\CudaDnnDll.11.3.dll";
MyCaffeControl <float> mycaffe = new MyCaffeControl <float>(settings, log, cancel, null, null, null, null, strCudaPath);
// Load the project, using the TRAIN phase.
mycaffe.Load(Phase.TRAIN, project);
// Train the model for 4000 iterations
// (which uses the internal solver and internal training net)
int nIterations = 4000;
mycaffe.Train(nIterations);
// Test the model for 100 iterations
// (which uses the internal testing net)
nIterations = 100;
double dfAccuracy = mycaffe.Test(nIterations);
// Report the testing accuracy.
log.WriteLine("Accuracy = " + dfAccuracy.ToString("P"));
mycaffe.Dispose();
Console.Write("Press any key...");
Console.ReadKey();
}
/// <summary>
/// Process the content image by applying the style to it that was learned from the style image.
/// </summary>
/// <param name="bmpStyle">Specifies the image used to train the what style to apply to the content.</param>
/// <param name="bmpContent">Specifies the content image to which the style is to be applied.</param>
/// <param name="nIterations">Specifies the number of training iterations.</param>
/// <param name="strResultDir">Optionally, specifies an output directory where intermediate images are stored.</param>
/// <param name="nIntermediateOutput">Optionally, specifies how often to output an intermediate image.</param>
/// <param name="dfTvLoss">Optionally, specifies the TV-Loss weight for smoothing (default = 0, which disables this loss).</param>
/// <returns>The resulting image is returned.</returns>
public Bitmap Process(Bitmap bmpStyle, Bitmap bmpContent, int nIterations, string strResultDir = null, int nIntermediateOutput = -1, double dfTvLoss = 0)
{
Solver <T> solver = null;
Net <T> net = null;
BlobCollection <T> colContentActivations = new BlobCollection <T>();
BlobCollection <T> colGramActivations = new BlobCollection <T>();
double dfLoss;
try
{
m_dfTVLossWeight = dfTvLoss;
m_nIterations = nIterations;
if (bmpStyle.Width != bmpContent.Width ||
bmpStyle.Height != bmpContent.Height)
{
bmpStyle = ImageTools.ResizeImage(bmpStyle, bmpContent.Width, bmpContent.Height);
}
m_log.WriteLine("Creating input network...");
m_log.Enable = false;
net = new Net <T>(m_cuda, m_log, m_param, m_evtCancel, null, Phase.TEST);
m_log.Enable = true;
if (m_rgWeights != null)
{
net.LoadWeights(m_rgWeights, m_persist);
}
//-----------------------------------------
// Get style and content activations.
//-----------------------------------------
prepare_data_blob(net, bmpStyle);
net.Forward(out dfLoss);
foreach (KeyValuePair <string, double> kvGram in m_rgLayers["gram"])
{
string strGram = kvGram.Key;
Blob <T> blobGram = net.blob_by_name(strGram);
colGramActivations.Add(blobGram.Clone());
}
prepare_data_blob(net, bmpContent);
net.Forward(out dfLoss);
foreach (KeyValuePair <string, double> kvContent in m_rgLayers["content"])
{
string strContent = kvContent.Key;
Blob <T> blobContent = net.blob_by_name(strContent);
colContentActivations.Add(blobContent.Clone());
}
//-----------------------------------------
// Prepare the network by adding new layers.
//-----------------------------------------
NetParameter net_param = m_param;
foreach (KeyValuePair <string, double> kvInput in m_rgLayers["input"])
{
string strName = kvInput.Key;
LayerParameter p = new LayerParameter(LayerParameter.LayerType.INPUT);
p.name = "input_" + strName;
p.top.Add(p.name);
Blob <T> blob = net.blob_by_name(strName);
p.input_param.shape.Add(new BlobShape(blob.shape()));
net_param.layer.Add(p);
}
foreach (KeyValuePair <string, double> kvContent in m_rgLayers["content"])
{
string strName = kvContent.Key;
string strScale1 = "input_" + strName;
string strScale2 = strName;
if (m_dfContentDataScale != 1.0)
{
strScale1 += "b";
LayerParameter ps1 = new LayerParameter(LayerParameter.LayerType.SCALAR);
ps1.scalar_param.value = m_dfContentDataScale;
ps1.scalar_param.operation = ScalarParameter.ScalarOp.MUL;
ps1.scalar_param.passthrough_gradient = true;
ps1.bottom.Add("input_" + strName);
ps1.top.Add(strScale1);
net_param.layer.Add(ps1);
strScale2 += "b";
LayerParameter ps2 = new LayerParameter(LayerParameter.LayerType.SCALAR);
ps2.scalar_param.value = m_dfContentDataScale;
ps2.scalar_param.operation = ScalarParameter.ScalarOp.MUL;
ps2.scalar_param.passthrough_gradient = true;
ps2.bottom.Add(strName);
ps2.top.Add(strScale2);
net_param.layer.Add(ps2);
}
LayerParameter event_param = new LayerParameter(LayerParameter.LayerType.EVENT);
event_param.name = "event_" + strName;
event_param.bottom.Add(strScale2);
event_param.bottom.Add(strScale1);
event_param.top.Add("event_" + strName);
net_param.layer.Add(event_param);
LayerParameter p = new LayerParameter(LayerParameter.LayerType.EUCLIDEAN_LOSS);
p.name = "loss_" + strName;
Blob <T> blobContent = colContentActivations[strName];
double dfScale = get_content_scale(blobContent);
p.loss_weight.Add(kvContent.Value * dfScale);
p.bottom.Add("event_" + strName);
p.bottom.Add(strScale1);
p.top.Add("loss_" + strName);
net_param.layer.Add(p);
}
foreach (KeyValuePair <string, double> kvGram in m_rgLayers["gram"].ToList())
{
string strGramName = kvGram.Key;
LayerParameter event_param = new LayerParameter(LayerParameter.LayerType.EVENT);
event_param.name = "event_" + strGramName;
event_param.bottom.Add(strGramName);
event_param.bottom.Add("input_" + strGramName);
event_param.top.Add("event_" + strGramName);
net_param.layer.Add(event_param);
LayerParameter p = new LayerParameter(LayerParameter.LayerType.EUCLIDEAN_LOSS);
p.name = "loss_" + strGramName;
Blob <T> blobGram = colGramActivations[strGramName];
double dfScale = get_style_scale(blobGram);
p.loss_weight.Add(kvGram.Value * dfScale);
p.bottom.Add("input_" + strGramName);
p.bottom.Add("event_" + strGramName);
p.top.Add("loss_" + strGramName);
net_param.layer.Add(p);
}
// Add TV Loss;
if (m_dfTVLossWeight != 0)
{
LayerParameter p = new LayerParameter(LayerParameter.LayerType.TV_LOSS);
p.name = "loss_tv";
double dfWeight = m_dfTVLossWeight;
p.loss_weight.Add(dfWeight);
p.bottom.Add("data");
p.top.Add("loss_tv");
net_param.layer.Add(p);
}
// Replace InputLayer with ParameterLayer,
// so that we'll be able to backprop into the image.
Blob <T> data = net.blob_by_name("data");
for (int i = 0; i < net_param.layer.Count; i++)
{
LayerParameter p = net_param.layer[i];
if (p.name == "input1")
{
net_param.layer[i].SetType(LayerParameter.LayerType.PARAMETER);
net_param.layer[i].parameter_param.shape = new BlobShape(data.shape());
break;
}
}
// Disable weights learning.
List <LayerParameter.LayerType> rgTypes = new List <LayerParameter.LayerType>();
rgTypes.Add(LayerParameter.LayerType.CONVOLUTION);
rgTypes.Add(LayerParameter.LayerType.DECONVOLUTION);
rgTypes.Add(LayerParameter.LayerType.INNERPRODUCT);
rgTypes.Add(LayerParameter.LayerType.PRELU);
rgTypes.Add(LayerParameter.LayerType.BIAS);
rgTypes.Add(LayerParameter.LayerType.EMBED);
rgTypes.Add(LayerParameter.LayerType.LSTM);
rgTypes.Add(LayerParameter.LayerType.LSTM_SIMPLE);
rgTypes.Add(LayerParameter.LayerType.RNN);
foreach (LayerParameter layer in net_param.layer)
{
if (rgTypes.Contains(layer.type))
{
layer.parameters = new List <ParamSpec>();
layer.parameters.Add(new ParamSpec(0, 0));
layer.parameters.Add(new ParamSpec(0, 0));
}
}
net.Dispose();
net = null;
//-----------------------------------------
// Create solver and assign inputs.
//-----------------------------------------
RawProto proto1 = net_param.ToProto("root");
string str = proto1.ToString();
SolverParameter solver_param = new SolverParameter();
solver_param.display = m_nDisplayEvery;
solver_param.train_net_param = net_param;
solver_param.test_iter.Clear();
solver_param.test_interval = 0;
solver_param.test_initialization = false;
solver_param.base_lr = m_dfLearningRate;
solver_param.type = m_solverType;
m_log.WriteLine("Creating " + m_solverType.ToString() + " solver with learning rate = " + m_dfLearningRate.ToString() + "...");
m_log.Enable = false;
if (m_solverType == SolverParameter.SolverType.LBFGS)
{
solver = new LBFGSSolver <T>(m_cuda, m_log, solver_param, m_evtCancel, null, null, null, m_persist);
}
else
{
solver = Solver <T> .Create(m_cuda, m_log, solver_param, m_evtCancel, null, null, null, m_persist);
}
m_log.Enable = true;
solver.OnSnapshot += Solver_OnSnapshot;
solver.OnTrainingIteration += Solver_OnTrainingIteration;
foreach (Layer <T> layer in solver.net.layers)
{
if (layer.type == LayerParameter.LayerType.EVENT)
{
EventLayer <T> eventLayer = layer as EventLayer <T>;
eventLayer.OnBackward += EventLayer_OnBackward;
}
}
prepare_input_param(solver.net, bmpContent);
foreach (KeyValuePair <string, double> kvContent in m_rgLayers["content"])
{
string strName = kvContent.Key;
Blob <T> blobDst = solver.net.blob_by_name("input_" + strName);
Blob <T> blobSrc = colContentActivations[strName];
blobDst.CopyFrom(blobSrc);
}
foreach (KeyValuePair <string, double> kvGram in m_rgLayers["gram"])
{
string strName = kvGram.Key;
Blob <T> blobDst = solver.net.blob_by_name("input_" + strName);
Blob <T> blobSrc = colGramActivations[strName];
blobDst.CopyFrom(blobSrc);
}
//-----------------------------------------
// Optimize.
//-----------------------------------------
int nIterations1 = m_nIterations;
if (strResultDir != null && nIntermediateOutput > 0)
{
nIterations1 /= nIntermediateOutput;
}
if (m_rgWeights != null)
{
Blob <T> blobInput = solver.net.learnable_parameters[0];
solver.net.learnable_parameters.RemoveAt(0);
solver.net.LoadWeights(m_rgWeights, m_persist);
solver.net.learnable_parameters.Insert(0, blobInput);
}
if (strResultDir != null)
{
strResultDir = strResultDir.TrimEnd('\\');
strResultDir += "\\";
}
for (int i = 0; i < nIterations1; i++)
{
if (m_evtCancel.WaitOne(0))
{
break;
}
solver.Step(nIntermediateOutput, TRAIN_STEP.NONE, true, true, true);
if (strResultDir != null)
{
Bitmap bmpTemp = save(solver.net);
string strFile = strResultDir + i.ToString() + "_temp.png";
if (File.Exists(strFile))
{
File.Delete(strFile);
}
bmpTemp.Save(strFile);
}
}
Bitmap bmpOutput = save(solver.net);
return(bmpOutput);
}
catch (Exception excpt)
{
throw excpt;
}
finally
{
if (net != null)
{
net.Dispose();
}
if (solver != null)
{
solver.Dispose();
}
colGramActivations.Dispose();
colContentActivations.Dispose();
}
}
/// <summary>
/// Run the trained model on the generated Sin curve.
/// </summary>
/// <returns>Returns <i>false</i> if no trained model found.</returns>
public bool Run()
{
// Load the run net with the previous weights.
byte[] rgWts = loadWeights();
if (rgWts == null)
{
Console.WriteLine("You must first train the network!");
return(false);
}
// Crate the model used to run indefinitely
NetParameter model = createModelInfiniteInput();
string strModel = model.ToProto("root").ToString();
Console.WriteLine("Using Run Model:");
Console.WriteLine(strModel);
// Load the model for running with the trained weights.
int nN = 1;
m_mycaffeRun.LoadToRun(strModel, rgWts, new BlobShape(new List <int>()
{
nN, 1, 1
}), null, null, false, false);
// Load the previously saved LSTM state (hy and cy) along with the previously
// trained weights.
loadLstmState(m_mycaffeRun);
// Get the internal RUN net and associated blobs.
Net <float> net = m_mycaffeRun.GetInternalNet(Phase.RUN);
Blob <float> blobData = net.FindBlob("data");
Blob <float> blobClip = net.FindBlob("clip2");
Blob <float> blobIp1 = net.FindBlob("ip1");
int nBatch = 1;
// Run on 3 different, randomly selected Sin curves.
for (int i = 0; i < 3; i++)
{
// Create the Sin data.
Dictionary <string, float[]> data = generateSample(i + 1.1337f, null, nBatch, m_param.Output, m_param.TimeSteps);
List <float> rgPrediction = new List <float>();
// Set the clip to 1 for we are continuing from the
// last training session and want start with the last
// cy and hy states.
blobClip.SetData(1);
float[] rgY = data["Y"];
float[] rgFY = data["FY"];
// Run the model on the data up to number of
// time steps.
for (int t = 0; t < m_param.TimeSteps; t++)
{
blobData.SetData(rgY[t]);
net.Forward();
rgPrediction.Add(blobIp1.GetData(0));
}
// Run the model on the last prediction for
// the number of predicted output steps.
for (int t = 0; t < m_param.Output; t++)
{
blobData.SetData(rgPrediction[rgPrediction.Count - 1]);
//blobData.SetData(rgFY[t]);
net.Forward();
rgPrediction.Add(blobIp1.GetData(0));
}
// Graph and show the resupts.
List <float> rgT2 = new List <float>(data["T"]);
rgT2.AddRange(data["FT"]);
// Plot the graph.
PlotCollection plotsY = createPlots("Y", rgT2.ToArray(), new List <float[]>()
{
data["Y"], data["FY"]
}, 0);
PlotCollection plotsTarget = createPlots("Target", rgT2.ToArray(), new List <float[]>()
{
data["Y"], data["FY"]
}, 1);
PlotCollection plotsPrediction = createPlots("Predicted", rgT2.ToArray(), new List <float[]>()
{
rgPrediction.ToArray()
}, 0);
PlotCollectionSet set = new PlotCollectionSet(new List <PlotCollection>()
{
plotsY, plotsTarget, plotsPrediction
});
// Create the graph image and display
Image img = SimpleGraphingControl.QuickRender(set, 2000, 600);
showImage(img, "result_" + i.ToString() + ".png");
}
return(true);
}
/// <summary>
/// Create the LeNet_train_test prototxt programmatically.
/// </summary>
/// <param name="strDataName">Specifies the dataset name.</param>
/// <param name="nBatchSize">Specifies the batch size.</param>
/// <returns>The model descriptor is returned as text.</returns>
private string create_model_descriptor_programmatically(string strDataName, int nBatchSize, LayerParameter.LayerType inputType)
{
if (!verifyInputType(inputType))
{
throw new Exception("The input type " + inputType.ToString() + " is not supported by this sample.");
}
NetParameter net_param = new NetParameter();
net_param.name = "LeNet";
if (inputType == LayerParameter.LayerType.INPUT)
{
LayerParameter input_param_train = new LayerParameter(LayerParameter.LayerType.INPUT);
input_param_train.name = strDataName;
input_param_train.top.Add("data");
input_param_train.top.Add("label");
input_param_train.include.Add(new NetStateRule(Phase.TRAIN));
input_param_train.transform_param = new TransformationParameter();
input_param_train.transform_param.scale = 1.0 / 256.0;
input_param_train.input_param.shape = new List <BlobShape>()
{
new BlobShape(nBatchSize, 1, 28, 28), // data (the images)
new BlobShape(nBatchSize, 1, 1, 1)
}; // label
net_param.layer.Add(input_param_train);
LayerParameter input_param_test = new LayerParameter(LayerParameter.LayerType.INPUT);
input_param_test.name = strDataName;
input_param_test.top.Add("data");
input_param_test.top.Add("label");
input_param_test.include.Add(new NetStateRule(Phase.TEST));
input_param_test.transform_param = new TransformationParameter();
input_param_test.transform_param.scale = 1.0 / 256.0;
input_param_train.input_param.shape = new List <BlobShape>()
{
new BlobShape(nBatchSize, 1, 28, 28), // data (the images)
new BlobShape(nBatchSize, 1, 1, 1)
}; // label
net_param.layer.Add(input_param_test);
}
else if (inputType == LayerParameter.LayerType.IMAGE_DATA)
{
LayerParameter input_param_train = new LayerParameter(LayerParameter.LayerType.IMAGE_DATA);
input_param_train.name = strDataName;
input_param_train.top.Add("data");
input_param_train.top.Add("label");
input_param_train.include.Add(new NetStateRule(Phase.TRAIN));
input_param_train.transform_param = new TransformationParameter();
input_param_train.transform_param.scale = 1.0 / 256.0;
input_param_train.data_param.batch_size = (uint)nBatchSize;
input_param_train.data_param.source = m_strImageDirTraining + "\\file_list.txt";
input_param_train.image_data_param.is_color = false;
net_param.layer.Add(input_param_train);
LayerParameter input_param_test = new LayerParameter(LayerParameter.LayerType.IMAGE_DATA);
input_param_test.name = strDataName;
input_param_test.top.Add("data");
input_param_test.top.Add("label");
input_param_test.include.Add(new NetStateRule(Phase.TEST));
input_param_test.transform_param = new TransformationParameter();
input_param_test.transform_param.scale = 1.0 / 256.0;
input_param_test.data_param.batch_size = (uint)nBatchSize;
input_param_test.data_param.source = m_strImageDirTesting + "\\file_list.txt";
input_param_test.image_data_param.is_color = false;
net_param.layer.Add(input_param_test);
}
LayerParameter conv1 = new LayerParameter(LayerParameter.LayerType.CONVOLUTION);
conv1.name = "conv1";
conv1.bottom.Add("data");
conv1.top.Add("conv1");
conv1.parameters.Add(new ParamSpec(1, 2));
conv1.convolution_param.num_output = 20;
conv1.convolution_param.kernel_size.Add(5);
conv1.convolution_param.stride.Add(1);
conv1.convolution_param.weight_filler = new FillerParameter("xavier");
conv1.convolution_param.bias_filler = new FillerParameter("constant");
net_param.layer.Add(conv1);
LayerParameter pool1 = new LayerParameter(LayerParameter.LayerType.POOLING);
pool1.name = "pool1";
pool1.bottom.Add("conv1");
pool1.top.Add("pool1");
pool1.pooling_param.pool = PoolingParameter.PoolingMethod.MAX;
pool1.pooling_param.kernel_size.Add(2);
pool1.pooling_param.stride.Add(2);
net_param.layer.Add(pool1);
LayerParameter conv2 = new LayerParameter(LayerParameter.LayerType.CONVOLUTION);
conv2.name = "conv2";
conv2.bottom.Add("pool1");
conv2.top.Add("conv2");
conv2.parameters.Add(new ParamSpec(1, 2));
conv2.convolution_param.num_output = 50;
conv2.convolution_param.kernel_size.Add(5);
conv2.convolution_param.stride.Add(1);
conv2.convolution_param.weight_filler = new FillerParameter("xavier");
conv2.convolution_param.bias_filler = new FillerParameter("constant");
net_param.layer.Add(conv2);
LayerParameter pool2 = new LayerParameter(LayerParameter.LayerType.POOLING);
pool2.name = "pool2";
pool2.bottom.Add("conv2");
pool2.top.Add("pool2");
pool2.pooling_param.pool = PoolingParameter.PoolingMethod.MAX;
pool2.pooling_param.kernel_size.Add(2);
pool2.pooling_param.stride.Add(2);
net_param.layer.Add(pool2);
LayerParameter ip1 = new LayerParameter(LayerParameter.LayerType.INNERPRODUCT);
ip1.name = "ip1";
ip1.bottom.Add("pool2");
ip1.top.Add("ip1");
ip1.parameters.Add(new ParamSpec(1, 2));
ip1.inner_product_param.num_output = 500;
ip1.inner_product_param.weight_filler = new FillerParameter("xavier");
ip1.inner_product_param.bias_filler = new FillerParameter("constant");
net_param.layer.Add(ip1);
LayerParameter relu1 = new LayerParameter(LayerParameter.LayerType.RELU);
relu1.name = "relu1";
relu1.bottom.Add("ip1");
relu1.top.Add("ip1"); // inline.
net_param.layer.Add(relu1);
LayerParameter ip2 = new LayerParameter(LayerParameter.LayerType.INNERPRODUCT);
ip2.name = "ip2";
ip2.bottom.Add("ip1");
ip2.top.Add("ip2");
ip2.parameters.Add(new ParamSpec(1, 2));
ip2.inner_product_param.num_output = 10;
ip2.inner_product_param.weight_filler = new FillerParameter("xavier");
ip2.inner_product_param.bias_filler = new FillerParameter("constant");
net_param.layer.Add(ip2);
LayerParameter accuracy = new LayerParameter(LayerParameter.LayerType.ACCURACY);
accuracy.name = "accuracy";
accuracy.bottom.Add("ip2");
accuracy.bottom.Add("label");
accuracy.top.Add("accuracy");
accuracy.include.Add(new NetStateRule(Phase.TEST));
net_param.layer.Add(accuracy);
LayerParameter loss = new LayerParameter(LayerParameter.LayerType.SOFTMAXWITH_LOSS);
loss.name = "loss";
loss.bottom.Add("ip2");
loss.bottom.Add("label");
loss.top.Add("loss");
net_param.layer.Add(loss);
// Convert model to text descriptor.
RawProto proto = net_param.ToProto("root");
return(proto.ToString());
}
/// <summary>
/// The worker thread used to either train or run the models.
/// </summary>
/// <remarks>
/// When training, first the input hand-written image model is trained
/// using the LeNet model.
///
/// This input mode is then run in the onTrainingStart event to get the
/// detected hand written character representation. The outputs of layer
/// 'ip1' from the input model are then fed as input to the sequence
/// model which is then trained to encode the 'ip1' input data with one
/// lstm and then decoded with another which is then trained to detect
/// a section of the Sin curve data.
///
/// When running, the first input model is run to get its 'ip1' representation,
/// which is then fed into the sequence model to detect the section of the
/// Sin curve.
/// </remarks>
/// <param name="sender">Specifies the sender of the event (e.g. the BackgroundWorker)</param>
/// <param name="args">Specifies the event args.</param>
private void m_bw_DoWork(object sender, DoWorkEventArgs e)
{
BackgroundWorker bw = sender as BackgroundWorker;
OPERATION op = (OPERATION)e.Argument;
SettingsCaffe s = new SettingsCaffe();
s.ImageDbLoadMethod = IMAGEDB_LOAD_METHOD.LOAD_ALL;
m_operation = op;
m_mycaffe = new MyCaffeControl <float>(s, m_log, m_evtCancel);
m_mycaffeInput = new MyCaffeControl <float>(s, m_log, m_evtCancel);
m_imgDb = new MyCaffeImageDatabase2(m_log);
// Load the image database.
m_imgDb.InitializeWithDsName1(s, "MNIST");
m_ds = m_imgDb.GetDatasetByName("MNIST");
// Create the MNIST image detection model
NetParameter netParamMnist = m_model.CreateMnistModel(m_ds);
SolverParameter solverParamMnist = m_model.CreateMnistSolver();
byte[] rgWts = loadWeights("input");
m_mycaffeInput.Load(Phase.TRAIN, solverParamMnist.ToProto("root").ToString(), netParamMnist.ToProto("root").ToString(), rgWts, null, null, false, m_imgDb);
Net <float> netTrain = m_mycaffeInput.GetInternalNet(Phase.TRAIN);
Blob <float> input_ip = netTrain.FindBlob(m_strInputOutputBlobName); // input model's second to last output (includes relu)
// Run the train or run operation.
if (op == OPERATION.TRAIN)
{
// Train the MNIST model first.
m_mycaffeInput.OnTrainingIteration += m_mycaffeInput_OnTrainingIteration;
m_plotsInputLoss = new PlotCollection("Input Loss");
m_mycaffeInput.Train(2000);
saveWeights("input", m_mycaffeInput.GetWeights());
// Load the Seq2Seq training model.
NetParameter netParam = m_model.CreateModel(input_ip.channels, 10);
string strModel = netParam.ToProto("root").ToString();
SolverParameter solverParam = m_model.CreateSolver();
rgWts = loadWeights("sequence");
m_mycaffe.OnTrainingIteration += m_mycaffe_OnTrainingIteration;
m_mycaffe.LoadLite(Phase.TRAIN, solverParam.ToProto("root").ToString(), netParam.ToProto("root").ToString(), rgWts, false, false);
m_mycaffe.SetOnTrainingStartOverride(new EventHandler(onTrainingStart));
// Train the Seq2Seq model.
m_plotsSequenceLoss = new PlotCollection("Sequence Loss");
m_mycaffe.Train(m_model.Iterations);
saveWeights("sequence", m_mycaffe.GetWeights());
}
else
{
NetParameter netParam = m_model.CreateModel(input_ip.channels, 10, 1, 1);
string strModel = netParam.ToProto("root").ToString();
rgWts = loadWeights("sequence");
int nN = 1;
m_mycaffe.LoadToRun(netParam.ToProto("root").ToString(), rgWts, new BlobShape(new List <int>()
{
nN, 1, 1, 1
}), null, null, false, false);
runModel(m_mycaffe, bw);
}
// Cleanup.
m_mycaffe.Dispose();
m_mycaffe = null;
m_mycaffeInput.Dispose();
m_mycaffeInput = null;
}
