void create_model(ref CNTK.Function model, ref float[][] labels)
{
var target_image = preprocess_image(target_image_path, img_height, img_width);
var style_reference_image = preprocess_image(style_reference_image_path, img_height, img_width);
var base_model = create_base_content_and_styles_model(img_height, img_width);
labels = compute_labels(base_model, target_image, style_reference_image);
var dream_weights_init = new CNTK.NDArrayView(new int[] { img_width, img_height, 3 }, target_image, computeDevice);
var dream_weights = new CNTK.Parameter(dream_weights_init, "the_dream");
var dummy_features = CNTK.Variable.InputVariable(new int[] { 1 }, CNTK.DataType.Float, "dummy_features");
var dream_layer = CNTK.CNTKLib.ElementTimes(dream_weights, dummy_features, "the_dream_layler");
var replacements = new Dictionary <CNTK.Variable, CNTK.Variable>()
{
{ base_model.Arguments[0], dream_layer.Output }
};
model = base_model.Clone(CNTK.ParameterCloningMethod.Freeze, replacements);
var all_outputs = new List <CNTK.Variable>()
{
dream_layer
};
all_outputs.AddRange(model.Outputs);
model = CNTK.Function.Combine(all_outputs, name: "overall_model");
}
/// <summary>
/// Use the generator to create a list of fake images/
/// </summary>
/// <param name="generator">The generator to use.</param>
/// <param name="batchSize">The batch size.</param>
/// <param name="latentDimensions">The number of dimensions in the latent input vector.</param>
/// <returns>A list of images created by the generator.</returns>
public static IList <IList <float> > GenerateImages(
CNTK.Function generator,
int batchSize,
int latentDimensions)
{
// set up a Gaussian random number generator
var random = new Random();
var gaussianRandom = new GaussianRandom(random);
// set up randomized input for the generator
var random_latent_vectors = gaussianRandom.getFloatSamples(batchSize * latentDimensions);
var random_latent_vectors_nd = new CNTK.NDArrayView(new int[] { latentDimensions, 1, batchSize }, random_latent_vectors, NetUtil.CurrentDevice);
var generator_inputs = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ generator.Arguments[0], new CNTK.Value(random_latent_vectors_nd) }
};
var generator_outputs = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ generator.Output, null }
};
// run the generator and collect the images
generator.Evaluate(generator_inputs, generator_outputs, NetUtil.CurrentDevice);
return(generator_outputs[generator.Output].GetDenseData <float>(generator.Output));
}
/// <summary>
/// Adds a dream layer to a neural network.
/// </summary>
/// <param name="input">The neural network to extend.</param>
/// <param name="image">The content image.</param>
/// <param name="width">The width of the content image.</param>
/// <param name="height">The height of the content image.</param>
/// <returns>The neural network extended with a dream layer.</returns>
public static CNTK.Function DreamLayer(
this CNTK.Function input,
float[] image,
int width,
int height)
{
// set up the dream layer
var dream_weights_init = new CNTK.NDArrayView(new int[] { width, height, 3 }, image, NetUtil.CurrentDevice);
var dream_weights = new CNTK.Parameter(dream_weights_init, "the_dream");
var dummy_features = CNTK.Variable.InputVariable(new int[] { 1 }, CNTK.DataType.Float, "dummy_features");
var dream_layer = CNTK.CNTKLib.ElementTimes(dream_weights, dummy_features, "the_dream_layer");
// combine the dream layer with the content and style layers
var replacements = new Dictionary <CNTK.Variable, CNTK.Variable>()
{
{ input.Arguments[0], dream_layer.Output }
};
var model = input.Clone(CNTK.ParameterCloningMethod.Freeze, replacements);
// return the finished model
var all_outputs = new List <CNTK.Variable>()
{
dream_layer
};
all_outputs.AddRange(model.Outputs);
return(CNTK.Function.Combine(all_outputs, name: "overall_model"));
}
CNTK.Value create_x_minibatch(bool sequence_mode, CNTK.Variable x, GeneratorsInfo gi, SamplesTargets st, CNTK.DeviceDescriptor computeDevice)
{
if (sequence_mode == false)
{
return(CNTK.Value.CreateBatch(x.Shape, st.samples, computeDevice));
}
var sequence_length = gi.lookback / gi.step;
var minibatch_size = st.samples.Length / sequence_length / gi.num_records;
var x_shape = CNTK.NDShape.CreateNDShape(new int[] { gi.num_records, sequence_length, minibatch_size });
var ndArrayView = new CNTK.NDArrayView(x_shape, st.samples, computeDevice, readOnly: true);
return(new CNTK.Value(ndArrayView));
}
static CNTK.NDArrayView[] get_minibatch_data_CPU(CNTK.NDShape shape, float[][] src, int indices_begin, int indices_end)
{
var num_indices = indices_end - indices_begin;
var result = new CNTK.NDArrayView[num_indices];
var row_index = 0;
for (var index = indices_begin; index != indices_end; index++)
{
var dataBuffer = src[index];
var ndArrayView = new CNTK.NDArrayView(shape, dataBuffer, CNTK.DeviceDescriptor.CPUDevice, true);
result[row_index++] = ndArrayView;
}
return(result);
}
void generateImages()
{
if (model == null)
{
System.Diagnostics.Debug.Assert(System.IO.File.Exists(model_filename));
model = CNTK.Function.Load(model_filename, computeDevice);
}
var node_z = model.FindByName("input_z");
var decoder_start = Util.find_function_with_input(model, node_z);
var z_sample_var = CNTK.Variable.InputVariable(node_z.Output.Shape, CNTK.DataType.Float, "z_sample");
var replacements = new Dictionary <CNTK.Variable, CNTK.Variable>()
{
{ node_z, z_sample_var }
};
var decoder = model.Clone(CNTK.ParameterCloningMethod.Freeze, replacements);
var n = 15; // figure with 15x15 digits
var xy_buffer = new float[n * n * 2];
var sample_start = -2f;
var sample_interval_width = 4f;
for (int i = 0, pos = 0; i < n; i++)
{
for (int j = 0; j < n; j++)
{
xy_buffer[pos++] = sample_start + (sample_interval_width / (n - 1)) * i;
xy_buffer[pos++] = sample_start + (sample_interval_width / (n - 1)) * j;
}
}
var ndArrayView = new CNTK.NDArrayView(new int[] { 2, 1, xy_buffer.Length / 2 }, xy_buffer, computeDevice, true);
var value = new CNTK.Value(ndArrayView);
var inputs_dir = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ z_sample_var, value }
};
var outputs_dir = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ decoder.Output, null }
};
decoder.Evaluate(inputs_dir, outputs_dir, computeDevice);
var values = outputs_dir[decoder.Output].GetDenseData <float>(decoder.Output);
plotImages(values);
}
void for_debugging()
{
#if false
var shape = data[features_stream_info].data.Shape;
var numElements = shape.TotalSize;
var buffer = new float[numElements];
var buffer_cpu = new CNTK.NDArrayView(shape, buffer, CNTK.DeviceDescriptor.CPUDevice);
buffer_cpu.CopyFrom(data[features_stream_info].data.Data);
var firstImage = new float[3 * 150 * 150];
System.Array.Copy(buffer, firstImage.Length, firstImage, 0, firstImage.Length);
var wpfApp = new System.Windows.Application();
wpfApp.Run(new PlotWindow(firstImage));
var mel = new float[40];
var nd_cpu = new CNTK.NDArrayView(CNTK.NDShape.CreateNDShape(new int[] { 2, 1, (int)data[labels_stream_info].numberOfSamples }), mel, CNTK.DeviceDescriptor.CPUDevice);
nd_cpu.CopyFrom(data[labels_stream_info].data.Data);
#endif
}
/// <summary>
/// Get a batch from the given variable.
/// </summary>
/// <param name="variable">The variable to use.</param>
/// <param name="source">The variable data.</param>
/// <param name="begin">The index of the first value to use.</param>
/// <param name="end">The index of the last value to use.</param>
/// <returns>A batch of values taken from the given variable.</returns>
public static CNTK.Value GetBatch(
this CNTK.Variable variable,
float[][] source,
int begin,
int end)
{
var num_indices = end - begin;
var result = new CNTK.NDArrayView[num_indices];
var row_index = 0;
for (var index = begin; index != end; index++)
{
var dataBuffer = source[index];
var ndArrayView = new CNTK.NDArrayView(variable.Shape, dataBuffer, CNTK.DeviceDescriptor.CPUDevice, true);
result[row_index++] = ndArrayView;
}
return(CNTK.Value.Create(variable.Shape, result, NetUtil.CurrentDevice, true));
}
public static (CNTK.Value featureBatch, CNTK.Value labelBatch) GetMisleadingBatch(
CNTK.Function gan,
int batchSize,
int latentDimensions)
{
// set up a Gaussian random number generator
var random = new Random();
var gaussianRandom = new GaussianRandom(random);
// prepare a batch to fool the discriminator: we generate fake images
// but we label them as real with label=0
var random_latent_vectors = gaussianRandom.getFloatSamples(batchSize * latentDimensions);
var misleading_targets = new float[batchSize];
var random_latent_vectors_nd = new CNTK.NDArrayView(new int[] { latentDimensions, 1, batchSize }, random_latent_vectors, NetUtil.CurrentDevice);
// return results
return(
new CNTK.Value(random_latent_vectors_nd),
CNTK.Value.CreateBatch(new CNTK.NDShape(0), misleading_targets, NetUtil.CurrentDevice, true)
);
}
float[][] compute_labels(CNTK.Function model, float[] target_image, float[] style_reference_image)
{
var input_shape = model.Arguments[0].Shape.Dimensions.ToArray();
System.Diagnostics.Debug.Assert(input_shape[0] * input_shape[1] * input_shape[2] == target_image.Length);
System.Diagnostics.Debug.Assert(target_image.Length == style_reference_image.Length);
#if false
var cpuDevice = CNTK.DeviceDescriptor.CPUDevice;
var target_image_nd = new CNTK.NDArrayView(input_shape, target_image, cpuDevice, readOnly: true);
var style_reference_image_nd = new CNTK.NDArrayView(input_shape, style_reference_image, cpuDevice, readOnly: true);
var batch_nd = new CNTK.NDArrayView[] { target_image_nd, style_reference_image_nd };
var batch = CNTK.Value.Create(input_shape, batch_nd, computeDevice, readOnly: true);
#else
var batch_buffer = new float[2 * target_image.Length];
Array.Copy(target_image, 0, batch_buffer, 0, target_image.Length);
Array.Copy(style_reference_image, 0, batch_buffer, target_image.Length, target_image.Length);
var batch_nd = new CNTK.NDArrayView(new int[] { model.Arguments[0].Shape[0], model.Arguments[0].Shape[1], model.Arguments[0].Shape[2], 1, 2 }, batch_buffer, computeDevice);
var batch = new CNTK.Value(batch_nd);
#endif
var inputs = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ model.Arguments[0], batch }
};
var outputs = new Dictionary <CNTK.Variable, CNTK.Value>();
foreach (var output in model.Outputs)
{
outputs.Add(output, null);
}
model.Evaluate(inputs, outputs, computeDevice);
float[][] labels = new float[model.Outputs.Count][];
labels[0] = outputs[model.Outputs[0]].GetDenseData <float>(model.Outputs[0])[0].ToArray();
for (int i = 1; i < labels.Length; i++)
{
labels[i] = outputs[model.Outputs[i]].GetDenseData <float>(model.Outputs[i])[1].ToArray();
}
return(labels);
}
Dictionary <CNTK.Variable, CNTK.Value> create_batch(CNTK.Function model, float[][] labels)
{
var dict_inputs = new Dictionary <CNTK.Variable, CNTK.Value>();
for (int i = 0; i < model.Arguments.Count; i++)
{
var loss_input_variable = model.Arguments[i];
if (loss_input_variable.Name == "dummy_features")
{
var dummy_scalar_buffer = new float[] { 1 };
var dummy_scalar_nd = new CNTK.NDArrayView(new int[] { 1 }, dummy_scalar_buffer, computeDevice, readOnly: true);
dict_inputs[loss_input_variable] = new CNTK.Value(dummy_scalar_nd);
}
else
{
var cs_index = Int32.Parse(loss_input_variable.Name.Substring("content_and_style_".Length));
var nd = new CNTK.NDArrayView(loss_input_variable.Shape, labels[cs_index], computeDevice, readOnly: true);
dict_inputs[loss_input_variable] = new CNTK.Value(nd);
}
}
return(dict_inputs);
}
/// <summary>
/// Calculate the output labels for style transfer.
/// </summary>
/// <param name="model">The neural network to use.</param>
/// <param name="contentImage">The content image to use.</param>
/// <param name="styleImage">The style image to use.</param>
/// <returns></returns>
public static float[][] CalculateLabels(CNTK.Function model, float[] contentImage, float[] styleImage)
{
// make sure the content image dimensions match the neural network input size
// make sure the content and style images are the same size
var input_shape = model.Arguments[0].Shape.Dimensions.ToArray();
System.Diagnostics.Debug.Assert(input_shape[0] * input_shape[1] * input_shape[2] == contentImage.Length);
System.Diagnostics.Debug.Assert(contentImage.Length == styleImage.Length);
// set up a batch with the content and the style image
var batch_buffer = new float[2 * contentImage.Length];
Array.Copy(contentImage, 0, batch_buffer, 0, contentImage.Length);
Array.Copy(styleImage, 0, batch_buffer, contentImage.Length, contentImage.Length);
var batch_nd = new CNTK.NDArrayView(new int[] { model.Arguments[0].Shape[0], model.Arguments[0].Shape[1], model.Arguments[0].Shape[2], 1, 2 }, batch_buffer, NetUtil.CurrentDevice);
var batch = new CNTK.Value(batch_nd);
// let the model evaluate the batch
var inputs = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ model.Arguments[0], batch }
};
var outputs = new Dictionary <CNTK.Variable, CNTK.Value>();
foreach (var output in model.Outputs)
{
outputs.Add(output, null);
}
model.Evaluate(inputs, outputs, NetUtil.CurrentDevice);
// collect and return the model outputs
float[][] labels = new float[model.Outputs.Count][];
labels[0] = outputs[model.Outputs[0]].GetDenseData <float>(model.Outputs[0])[0].ToArray();
for (int i = 1; i < labels.Length; i++)
{
labels[i] = outputs[model.Outputs[i]].GetDenseData <float>(model.Outputs[i])[1].ToArray();
}
return(labels);
}
/// <summary>
/// Get a sequence batch from the given variable.
/// </summary>
/// <param name="variable">The variable to use.</param>
/// <param name="sequenceLength">The number of time periods in the data sequence.</param>
/// <param name="source">The variable data.</param>
/// <param name="begin">The index of the first value to use.</param>
/// <param name="end">The index of the last value to use.</param>
/// <returns>A batch of values taken from the given variable.</returns>
public static CNTK.Value GetSequenceBatch(
this CNTK.Variable variable,
int sequenceLength,
float[][] source,
int begin,
int end)
{
System.Diagnostics.Debug.Assert((variable.Shape.Dimensions.Count == 0) || ((variable.Shape.Dimensions.Count == 1) && (variable.Shape.Dimensions[0] == 1)));
System.Diagnostics.Debug.Assert(source[0].Length == sequenceLength);
var num_indices = end - begin;
var cpu_blob = new float[num_indices * sequenceLength];
var row_index = 0;
for (var index = begin; index != end; index++)
{
System.Buffer.BlockCopy(source[index], 0, cpu_blob, row_index * sequenceLength * sizeof(float), sequenceLength * sizeof(float));
row_index++;
}
var blob_shape = variable.Shape.AppendShape(new int[] { sequenceLength, end - begin });
var ndArrayView = new CNTK.NDArrayView(blob_shape, cpu_blob, NetUtil.CurrentDevice);
return(new CNTK.Value(ndArrayView));
}
void train()
{
var random = new Random();
var gaussianRandom = new FromStackOverflow.GaussianRandom(random);
create_gan();
load_data();
var iterations = 100000;
var batch_size = 20;
var save_dir = "images";
System.IO.Directory.CreateDirectory(save_dir);
var start = 0;
for (int step = 0; step < iterations; step++)
{
// use the generator to generate fake images
var random_latent_vectors = gaussianRandom.getFloatSamples(batch_size * latent_dim);
var random_latent_vectors_nd = new CNTK.NDArrayView(new int[] { latent_dim, 1, batch_size }, random_latent_vectors, computeDevice);
var generator_inputs = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ generator.Arguments[0], new CNTK.Value(random_latent_vectors_nd) }
};
var generator_outputs = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ generator.Output, null }
};
generator.Evaluate(generator_inputs, generator_outputs, computeDevice);
var generated_images = generator_outputs[generator.Output].GetDenseData <float>(generator.Output);
// train the discriminator: the first half of the mini-batch are the fake images (marked with label='1')
// whereas the second half are real images (marked with label='0')
var combined_images = new float[2 * batch_size][];
var labels = new float[2 * batch_size];
start = Math.Min(start, x_train.Length - batch_size);
for (int i = 0; i < batch_size; i++)
{
combined_images[i] = generated_images[i].ToArray();
labels[i] = (float)(1 + 0.05 * gaussianRandom.NextGaussian());
combined_images[i + batch_size] = x_train[start + i];
labels[i + batch_size] = (float)(0.05 * gaussianRandom.NextGaussian());
}
start += batch_size;
if (start >= x_train.Length)
{
start = 0;
}
var combined_images_minibatch = Util.get_tensors(new int[] { width, height, channels }, combined_images, 0, combined_images.Length, computeDevice);
var labels_minibatch = CNTK.Value.CreateBatch(new CNTK.NDShape(0), labels, computeDevice, true);
var discriminator_minibatch = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ discriminator.Arguments[0], combined_images_minibatch },
{ label_var, labels_minibatch }
};
discriminator_trainer.TrainMinibatch(discriminator_minibatch, true, computeDevice);
var d_loss = discriminator_trainer.PreviousMinibatchLossAverage();
// train the gan: the generator will try to fool the discriminator: we generate fake
// images, but we label them as "real" (with label='0')
random_latent_vectors = gaussianRandom.getFloatSamples(batch_size * latent_dim);
var misleading_targets = new float[batch_size];
random_latent_vectors_nd = new CNTK.NDArrayView(new int[] { latent_dim, 1, batch_size }, random_latent_vectors, computeDevice);
var gan_inputs = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ gan.Arguments[0], new CNTK.Value(random_latent_vectors_nd) },
{ label_var, CNTK.Value.CreateBatch(new CNTK.NDShape(0), misleading_targets, computeDevice, true) }
};
gan_trainer.TrainMinibatch(gan_inputs, true, computeDevice);
var g_loss = gan_trainer.PreviousMinibatchLossAverage();
if (step % 100 == 0)
{
Console.WriteLine($"discriminator loss at step {step}: {d_loss:F3}");
Console.WriteLine($"adversarial loss at step {step}: {g_loss:F3}");
// Save one generated image
var img = generated_images[0].ToArray();
var img_bytes = Util.convert_from_channels_first(img, scaling: 255, invertOrder: true);
var mat = new OpenCvSharp.Mat(height, width, OpenCvSharp.MatType.CV_8UC3, img_bytes, 3 * width);
var image_filename = $"generated_frog_{step}.png";
var image_path = System.IO.Path.Combine(save_dir, image_filename);
mat.SaveImage(image_path);
mat.Dispose(); mat = null;
// Save one real image for comparison
img = x_train[Math.Max(start - batch_size, 0)];
img_bytes = Util.convert_from_channels_first(img, scaling: 255, invertOrder: true);
mat = new OpenCvSharp.Mat(height, width, OpenCvSharp.MatType.CV_8UC3, img_bytes, 3 * width);
image_filename = $"real_frog_{step}.png";
image_path = System.IO.Path.Combine(save_dir, image_filename);
mat.SaveImage(image_path);
mat.Dispose(); mat = null;
}
}
}
