private void Start()
{
// Predict style bottleneck;
using (var predict = new StylePredict(predictionFileName))
{
predict.Invoke(styleImage);
styleBottleneck = predict.GetStyleBottleneck();
}
styleTransfer = new StyleTransfer(transferFileName, styleBottleneck, compute);
var webCamInput = GetComponent <WebCamInput>();
webCamInput.OnTextureUpdate.AddListener(OnTextureUpdate);
}
/// <summary>
/// Detect what's in the given image.
/// </summary>
/// <param name="imagePath">The path of the image to check.</param>
/// <returns>A tuple of probabilities indicating if the image contains a dog or a cat.</returns>
private (double cat, double dog) DetectScene(string imagePath)
{
// load the neural network
if (network == null)
{
var path = @"..\..\..\..\..\models\catdogdetector.model";
network = Function.Load(path, NetUtil.CurrentDevice);
}
// grab the image
var imageData = new float[150 * 150 * 3];
using (var mat = Cv.Cv2.ImRead(imagePath))
{
using (var mat2 = new Cv.Mat(150, 150, mat.Type()))
{
Cv.Cv2.Resize(mat, mat2, new OpenCvSharp.Size(150, 150));
imageData = StyleTransfer.FlattenByChannel(mat2, new float[] { 0, 0, 0 });
}
}
// set up a tensor to hold the image data
var imageValue = new CNTK.NDArrayView(
new int[] { 150, 150, 3 },
imageData,
NetUtil.CurrentDevice);
// set up input and output dictionaries
var inputs = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ network.Arguments[0], new CNTK.Value(imageValue) }
};
var outputs = new Dictionary <CNTK.Variable, CNTK.Value>()
{
{ network.Outputs[0], null }
};
// run the neural network
network.Evaluate(inputs, outputs, NetUtil.CurrentDevice);
// return the result
var key = network.Outputs[0];
var result = outputs[key].GetDenseData <float>(key);
return(result[0][0], result[0][1]);
}
void Start()
{
// Predict style bottleneck;
string predictionModelPath = Path.Combine(Application.streamingAssetsPath, predictionFileName);
using (var predict = new StylePredict(predictionModelPath))
{
predict.Invoke(styleImage);
styleBottleneck = predict.GetStyleBottleneck();
}
string transferModelPath = Path.Combine(Application.streamingAssetsPath, transferFileName);
styleTransfer = new StyleTransfer(transferModelPath, styleBottleneck, compute);
// Init camera
string cameraName = WebCamUtil.FindName();
webcamTexture = new WebCamTexture(cameraName, 640, 480, 30);
webcamTexture.Play();
preview.texture = webcamTexture;
}
/// <summary>
/// The main program entry point.
/// </summary>
/// <param name="args"></param>
static void Main(string[] args)
{
// check compute device
Console.WriteLine("Checking compute device...");
Console.WriteLine($" Using {NetUtil.CurrentDevice.AsString()}");
// load images
Console.WriteLine("Loading content and style images...");
var contentImage = StyleTransfer.LoadImage(contentImagePath, imageWidth, imageHeight);
var styleImage = StyleTransfer.LoadImage(styleImagePath, imageWidth, imageHeight);
// create the feature variable
var featureVariable = CNTK.Variable.InputVariable(new int[] { imageWidth, imageHeight, 3 }, CNTK.DataType.Float);
// create the neural network base (just the content and style layers)
Console.WriteLine("Creating VGG19 style transfer model...");
var model = featureVariable
.VGG19(freeze: true)
.StyleTransferBase();
// calculate the labels
Console.WriteLine("Calculating output labels...");
var labels = StyleTransfer.CalculateLabels(model, contentImage, styleImage);
// add the dream layer
model = model.DreamLayer(contentImage, imageWidth, imageHeight);
// show the model summary
Console.WriteLine(model.ToSummary());
// create the label variable
var contentAndStyle = model.GetContentAndStyleLayers();
var labelVariable = new List <CNTK.Variable>();
for (int i = 0; i < labels.Length; i++)
{
var shape = contentAndStyle[i].Shape;
var input_variable = CNTK.Variable.InputVariable(shape, CNTK.DataType.Float, "content_and_style_" + i);
labelVariable.Add(input_variable);
}
// create the loss function
var lossFunction = StyleTransfer.CreateLossFunction(model, contentAndStyle, labelVariable);
// set up an AdamLearner
var learner = model.GetAdamLearner(10, 0.95);
// get the model trainer
var trainer = model.GetTrainer(learner, lossFunction, lossFunction);
// create the batch to train on
var trainingBatch = StyleTransfer.CreateBatch(lossFunction, labels);
// train the model
Console.WriteLine("Training the model...");
var numEpochs = 300;
for (int i = 0; i < numEpochs; i++)
{
trainer.TrainMinibatch(trainingBatch, true, NetUtil.CurrentDevice);
if (i % 50 == 0)
{
Console.WriteLine($"epoch {i}, training loss = {trainer.PreviousMinibatchLossAverage()}");
}
}
// create a batch to evaluate the model on
var evaluationBatch = StyleTransfer.CreateBatch(model, labels);
// infer the image from the model
Console.WriteLine("Inferring transformed image...");
var img = model.InferImage(evaluationBatch);
// show image
var mat = new Mat(imageHeight, imageWidth, OpenCvSharp.MatType.CV_8UC3, img, 3 * imageWidth);
Cv2.ImShow("Image With Style Transfer", mat);
Cv2.WaitKey();
}
