public ListWindow()
{
InitializeComponent();
ROOT_ENTRY = new FrameEntry();
ROOT_ENTRY.RefID = 0;
}
public override void DidOutputSampleBuffer(AVCaptureOutput captureOutput, CMSampleBuffer sampleBuffer, AVCaptureConnection connection)
{
frameNumber++;
frameId++;
if (frameNumber % 30 == 0)
{
var currentTime = NSProcessInfo.ProcessInfo.SystemUptime;
var fps = Math.Round(frameNumber / (currentTime - initialTime));
action?.Invoke((int)fps);
frameNumber = 0;
initialTime = NSProcessInfo.ProcessInfo.SystemUptime;
}
var buffer = sampleBuffer.GetImageBuffer() as CVPixelBuffer;
var frameEntry = new FrameEntry
{
TimeStamp = DateTimeOffset.UtcNow,
Width = (int)buffer.Width,
Height = (int)buffer.Height,
Frame = CVPixelBufferToByte(buffer)
};
Mvx.IoCProvider.Resolve <IInternalNotificationHubService>().NewFrame(frameEntry);
TryDispose(sampleBuffer);
TryDispose(buffer);
}
public byte[] ToJpegBytes(FrameEntry frame)
{
var cvBuffer = NNHelpers.ToCVPixelBuffer(frame);
cvBuffer.ToCGImage(out CGImage image);
cvBuffer.Dispose();
return(UIImage.FromImage(image).AsJPEG().ToArray());
}
private Rectangle ScaleCarBBoxAddingMargins(FrameEntry frame, RectangleF carBB)
{
int widthMargin = (int)(carBB.Width * frame.Width * 0.05);
int heightMargin = (int)(carBB.Height * frame.Height * 0.05);
int left = Math.Max(0, (int)(carBB.Left * frame.Width - widthMargin));
int top = Math.Max(0, (int)(carBB.Top * frame.Height - heightMargin));
int right = Math.Min(frame.Width, (int)(carBB.Right * frame.Width + widthMargin));
int bottom = Math.Min(frame.Height, (int)(carBB.Bottom * frame.Height + heightMargin));
return Rectangle.FromLTRB(left, top, right, bottom);
}
public FrameEntry RotateImage(FrameEntry frame, int rotateTimes)
{
byte[] rotatedBytes = new byte[frame.Frame.Length];
NativeMethods.RotateImage(frame.Frame, frame.Width, frame.Height, 4,
rotateTimes, rotatedBytes);
return(new FrameEntry()
{
Frame = rotatedBytes,
Width = rotateTimes % 2 == 0 ? frame.Width : frame.Height,
Height = rotateTimes % 2 == 0 ? frame.Height : frame.Width
});
}
private void SetFrameValueById(string index, string value)
{
// Replace any existing frame
for (int i = 0; i < frames.Count; ++i)
{
if (((FrameEntry)frames[i]).frameId == index)
{
frames[i] = new FrameEntry(index, value);
return;
}
}
// Otherwise, put a new one in.
frames.Add(new FrameEntry(index, value));
}
public FrameEntry CropImage(FrameEntry frame, Rectangle rect)
{
int bytesPerPixel = 4;
byte[] croppedBytes = new byte[rect.Width * rect.Height * bytesPerPixel];
NativeMethods.CropImage(frame.Frame, frame.Width, frame.Height, bytesPerPixel,
rect.Left, rect.Top, rect.Right, rect.Bottom, croppedBytes);
FrameEntry croppedFrame = new FrameEntry()
{
Frame = croppedBytes,
Width = rect.Width,
Height = rect.Height
};
return(croppedFrame);
}
// you will need to explicitly dispose the returned CVPixelBuffer after using it
public static CVPixelBuffer ToCVPixelBuffer(FrameEntry frame)
{
var bytesPerPixel = 4;
var bytesPerRow = frame.Width * bytesPerPixel;
var attr = new CVPixelBufferAttributes()
{
PixelFormatType = CVPixelFormatType.CV32BGRA,
AllocateWithIOSurface = true,
MetalCompatibility = true,
OpenGLCompatibility = true,
CGImageCompatibility = true,
OpenGLESCompatibility = true,
CGBitmapContextCompatibility = true,
BytesPerRowAlignment = bytesPerRow,
Height = frame.Height,
Width = frame.Width
};
var pixelBuffer = CVPixelBuffer.Create(frame.Width, frame.Height, CVPixelFormatType.CV32BGRA, frame.Frame, bytesPerRow, attr);
return(pixelBuffer);
}
public async Task FeedFrame(FrameEntry frame, RectangleF carBB, bool isFirst)
{
Rectangle bb = ScaleCarBBoxAddingMargins(frame, carBB);
var croppedCar = _accessor.Helpers.CropImage(frame, bb);
var orientNumber = _accessor.Helpers.DeviceOrientationAsNumber();
croppedCar = _accessor.Helpers.RotateImage(croppedCar, orientNumber);
LastCarImage = croppedCar;
// _accessor.DumpService.SaveFrame(croppedCar);
var carImage = _accessor.Helpers.ToNativeImage(croppedCar);
await _accessor.CarModelClassifier.Classify(carImage);
await _accessor.CarColorClassifier.Classify(carImage);
if (carImage is IDisposable disposable)
{
disposable.Dispose();
}
if (isFirst)
{
//Array.Fill(_carModelProbSum, 0.0f);
//Array.Fill(_carColorProbSum, 0.0f);
for (int i = 0; i < _carModelProbSum.Length; i++)
_carModelProbSum[i] = 0;
for (int i = 0; i < _carColorProbSum.Length; i++)
_carColorProbSum[i] = 0;
_feedCount = 0;
}
_feedCount++;
for (int idx = 0; idx < _accessor.CarModelClassifier.OutputSize(); idx++)
_carModelProbSum[idx] += _accessor.CarModelClassifier.Probabilities[idx];
for (int idx = 0; idx < _accessor.CarColorClassifier.OutputSize(); idx++)
_carColorProbSum[idx] += _accessor.CarColorClassifier.Probabilities[idx];
// ----------------------------------------- //
// Throw events for the screen log
var top3 = TopKFeatures(3);
var colorPreTime = _accessor.CarColorClassifier.PreprocessTime.TotalMilliseconds.ToString("F0");
var colorInferTime = _accessor.CarColorClassifier.InferenceTime.TotalMilliseconds.ToString("F0");
var colorPostTime = _accessor.CarColorClassifier.PostprocessTime.TotalMilliseconds.ToString("F0");
var logText = $"Color NN: {colorPreTime}ms, {colorInferTime}ms, {colorPostTime}ms{Environment.NewLine}" +
$" - {top3[0].ColorName} - {100 * top3[0].ColorProbability:F1}%{Environment.NewLine}" +
$" - {top3[1].ColorName} - {100 * top3[1].ColorProbability:F1}%{Environment.NewLine}" +
$" - {top3[2].ColorName} - {100 * top3[2].ColorProbability:F1}%{Environment.NewLine}";
var mmPreTime = _accessor.CarModelClassifier.PreprocessTime.TotalMilliseconds.ToString("F0");
var mmInferTime = _accessor.CarModelClassifier.InferenceTime.TotalMilliseconds.ToString("F0");
var mmPostTime = _accessor.CarModelClassifier.PostprocessTime.TotalMilliseconds.ToString("F0");
logText += $"M/m NN: {mmPreTime}ms, {mmInferTime}ms, {mmPostTime}ms{Environment.NewLine}" +
$" - {top3[0].ModelName} - {100 * top3[0].ModelProbability:F1}%{Environment.NewLine}" +
$" - {top3[1].ModelName} - {100 * top3[1].ModelProbability:F1}%{Environment.NewLine}" +
$" - {top3[2].ModelName} - {100 * top3[2].ModelProbability:F1}%{Environment.NewLine}";
var logParams = new LogParams()
{
ShowIdx = 3,
Color = Color.Cyan,
Text = logText
};
LogChanged?.Invoke(this, logParams);
// ----------------------------------------- //
}
private void SetFrameValueById(string index, string value)
{
// Replace any existing frame
for (int i = 0; i < frames.Count; ++i)
{
if (((FrameEntry)frames[i]).frameId == index)
{
frames[i] = new FrameEntry(index, value);
return;
}
}
// Otherwise, put a new one in.
frames.Add(new FrameEntry(index, value));
}
public void NewFrame(FrameEntry frameEntry)
{
OnNewFrame?.Invoke(this, frameEntry);
}
public void SetParent(int index, FrameEntry entry)
{
SetParent(index, entry.ToString(), entry.RefID);
}
public object ToNativeImage(FrameEntry frame)
{
return(NNHelpers.ToCVPixelBuffer(frame));
}