/// <summary>
/// Transform this image to a greyscale version out-of-place
/// </summary>
/// <param name="source">The source image</param>
/// <param name="output">The image to write to</param>
/// <returns>The output writable lockbit image</returns>
public static WritableLockBitImage Transform(WritableLockBitImage source, WritableLockBitImage output)
{
if (source.Locked || output.Locked)
{
throw new ArgumentException("Lockbit image is locked.");
}
for (int y = 0; y < source.Height; y++)
{
for (int x = 0; x < source.Width; x++)
{
Color sourcePixel = source.GetPixel(x, y);
int greyColor =
(int)Math.Floor(sourcePixel.R * 0.299) +
(int)Math.Floor(sourcePixel.G * 0.587) +
(int)Math.Floor(sourcePixel.B * 0.114);
Color greyScale = Color.FromArgb(
greyColor,
greyColor,
greyColor
);
output.SetPixel(x, y, greyScale);
}
}
return(output);
}
public static WritableLockBitImage Transform(
WritableLockBitImage sourceImage,
int radius = 3
)
{
using (var copyOfSourceImage = new WritableLockBitImage(sourceImage))
{
WritableLockBitImage outputImage = new WritableLockBitImage(sourceImage.Width, sourceImage.Height);
int index = 0;
foreach (int boxWidth in EnumerateBoxesForGauss(radius, 3))
{
if (index % 2 == 0)
{
PerformSplitBoxBlurAcc(copyOfSourceImage, outputImage, (boxWidth - 1) / 2);
}
else
{
// Switching of outputImage and copyOfSourceImage is INTENTIONAL!
PerformSplitBoxBlurAcc(outputImage, copyOfSourceImage, (boxWidth - 1) / 2);
}
index++;
}
return(outputImage);
}
}
/// <summary>
/// Resize the image to the destination width and height
/// </summary>
public static WritableLockBitImage Transform(WritableLockBitImage sourceImage, int width, int height)
{
using (WritableLockBitImage copyOfSourceImage = new WritableLockBitImage(sourceImage))
{
copyOfSourceImage.Lock();
return(new WritableLockBitImage(Transform(copyOfSourceImage.GetImage(), width, height)));
}
}
private static WritableLockBitImage ApplySobelFilter(WritableLockBitImage sourceImage)
{
return(CalculateContour(
CalculateHorizontalSobelMatrix(sourceImage),
CalculateVerticalSobelMatrix(sourceImage),
sourceImage.Width,
sourceImage.Height
));
}
private static void CopyPixels(WritableLockBitImage sourceImage, WritableLockBitImage destImage)
{
for (int row = 0; row < sourceImage.Height; row++)
{
for (int col = 0; col < sourceImage.Width; col++)
{
destImage.SetPixel(col, row, sourceImage.GetPixel(col, row));
}
}
}
private void RunQueue()
{
int currentFrame = 0;
int frameSize = 3 * _width * _height;
byte[] frameBuffer = new byte[frameSize];
int currentIndex = 0;
try
{
foreach (byte[] rawBytes in _rawByteQueue.GetConsumingEnumerable())
{
// Raw bytes don't overflow frame
if (rawBytes.Length < frameSize - currentIndex)
{
Buffer.BlockCopy(rawBytes, 0, frameBuffer, currentIndex, rawBytes.Length);
currentIndex += rawBytes.Length;
}
// Raw bytes overflow frame
else
{
int numBytesToCopy = frameSize - currentIndex;
Buffer.BlockCopy(rawBytes, 0, frameBuffer, currentIndex, numBytesToCopy);
// Frame is now full. Create image and ship it off
WritableLockBitImage frame = new WritableLockBitImage(_width, _height, frameBuffer);
frame.Lock();
_videoIndexer.SubmitVideoFrame(frame, currentFrame);
currentFrame++;
// Write overflow stuff now
Buffer.BlockCopy(rawBytes, numBytesToCopy, frameBuffer, 0, rawBytes.Length - numBytesToCopy);
currentIndex = rawBytes.Length - numBytesToCopy;
}
// Reduce the amount of bytes
long currentMemoryLevels = Interlocked.Add(ref _currentMemoryLevel, -rawBytes.Length);
// Check capacity and set or reset the barrier
if (currentMemoryLevels < _maxCapacity)
{
_capacityBarrier.Set();
}
else
{
_capacityBarrier.Reset();
}
}
}
catch (OperationCanceledException)
{
Shutdown();
}
}
private static WritableLockBitImage PerformSplitBoxBlurAcc(
WritableLockBitImage sourceImage,
int radius
)
{
var outputImage = new WritableLockBitImage(sourceImage);
// The switching of outputImage and souceImage is INTENTIONAL!
PerformHorizontalBoxBlurAcc(outputImage, sourceImage, radius);
PerformTotalBoxBlurAcc(sourceImage, outputImage, radius);
return(outputImage);
}
private static byte[] CalculateGrayScaleThumbnail(WritableLockBitImage image)
{
byte[] buffer = new byte[image.Width * image.Height];
for (int row = 0; row < image.Height; row++)
{
for (int col = 0; col < image.Width; col++)
{
buffer[row * image.Width + col] = image.GetPixel(col, row).R;
}
}
return(ByteCompressor.Compress(buffer));
}
private static WritableLockBitImage PerformSplitBoxBlurAcc(
WritableLockBitImage sourceImage,
WritableLockBitImage outputImage,
int radius
)
{
// Must copy pixels from source to output
CopyPixels(sourceImage, outputImage);
// The switching of outputImage and souceImage is INTENTIONAL!
PerformHorizontalBoxBlurAcc(outputImage, sourceImage, radius);
PerformTotalBoxBlurAcc(sourceImage, outputImage, radius);
return(outputImage);
}
private static WritableLockBitImage GetFrameFromVideo(VideoFingerPrintWrapper video, int frameNumber)
{
using (MediaInfoProcess mediaInfoProcess = new MediaInfoProcess(video.FilePath))
{
MediaInfo mediaInfo = mediaInfoProcess.Execute();
if (mediaInfo.GetFramerate().Numerator == 0)
{
throw new Exception("Did not get valid frame rate");
}
int width = mediaInfo.GetWidth();
int height = mediaInfo.GetHeight();
var ffmpegProcessSettings = new FFMPEGProcessVideoSettings(
video.FilePath,
mediaInfo.GetFramerate().Numerator,
mediaInfo.GetFramerate().Denominator * 4,
FFMPEGMode.SeekFrame
);
ffmpegProcessSettings.TargetFrame = frameNumber;
byte[] frameBytes = new byte[width * height * 3];
int offset = 0;
using (
var ffmpegProcess = new FFMPEGProcess(
ffmpegProcessSettings,
(stdoutBytes, numBytes) =>
{
Buffer.BlockCopy(stdoutBytes, 0, frameBytes, offset, numBytes);
offset += numBytes;
},
false
)
)
{
ffmpegProcess.Execute();
if (offset != 3 * width * height)
{
throw new Exception("Did not get all bytes to produce valid frame");
}
WritableLockBitImage videoFrame = new WritableLockBitImage(width, height, frameBytes);
videoFrame.Lock();
return(videoFrame);
}
}
}
public void SubmitVideoFrame(WritableLockBitImage frame, int frameNumber)
{
if (_disposed)
{
throw new ObjectDisposedException("this");
}
// Wait until capacity is available
_capacityBarrier.Wait();
Interlocked.Add(ref _currentMemoryLevel, frame.MemorySize);
_buffer.Add(new WorkItem
{
FrameNumber = frameNumber,
Frame = frame,
});
}
/// <summary>
/// Quantisize a photo by ceiling or flooring each value depending one whether it's above or below the median value
/// </summary>
/// <param name="sourceImage">The source image</param>
/// <param name="outputImage">The image to write to</param>
/// <param name="maxValue">The value to quantize the maximum value to</param>
/// <returns>The output image, but returned for convenience</returns>
public static WritableLockBitImage Transform(WritableLockBitImage sourceImage, WritableLockBitImage outputImage, byte maxValue = byte.MaxValue)
{
Color medianColor = GetMedianColorValue(sourceImage);
for (int row = 0; row < sourceImage.Height; row++)
{
for (int col = 0; col < sourceImage.Width; col++)
{
// Quantisize the individual channels
Color currentColor = sourceImage.GetPixel(col, row);
byte outputRed, outputGreen, outputBlue;
if (currentColor.R < medianColor.R)
{
outputRed = 0;
}
else
{
outputRed = maxValue;
}
if (currentColor.G < medianColor.G)
{
outputGreen = 0;
}
else
{
outputGreen = maxValue;
}
if (currentColor.B < medianColor.B)
{
outputBlue = 0;
}
else
{
outputBlue = maxValue;
}
outputImage.SetPixel(col, row, Color.FromArgb(outputRed, outputGreen, outputBlue));
}
}
return(outputImage);
}
private static VideoFingerPrintWrapper FindMostLikelyVideo(
PhotoFingerPrintWrapper photo,
IDictionary <string, ISet <FrameMetricWrapper> > treeResults,
IDictionary <string, VideoFingerPrintWrapper> nameToVideoMap
)
{
if (treeResults.Count() == 1)
{
return(treeResults.First().Value.First().Video);
}
using (WritableLockBitImage photoAsLockBitImage = new WritableLockBitImage(Image.FromFile(photo.FilePath)))
{
// Filter on grayscale results first
List <FrameMetricWrapper> filteredOnGrayScaleResults = (from videoFrameResults in treeResults
from frameMetricWrapper in videoFrameResults.Value
where DistanceCalculator.CalculateHammingDistance(photo.EdgeGrayScaleThumb, frameMetricWrapper.Frame.EdgeGrayScaleThumb) < 2
select frameMetricWrapper).ToList();
if (filteredOnGrayScaleResults.Count == 1)
{
return(filteredOnGrayScaleResults.First().Video);
}
double currentBestSSIM = 0.0;
VideoFingerPrintWrapper currentBestVideo = null;
foreach (FrameMetricWrapper FrameMetricWrapper in filteredOnGrayScaleResults)
{
// Calculate SSIM
using (WritableLockBitImage videoFrame = GetFrameFromVideo(FrameMetricWrapper.Video, FrameMetricWrapper.Frame.FrameNumber))
{
double possibleBestSSIM = SSIMCalculator.Compute(photoAsLockBitImage, videoFrame);
// SSIM must be at least good enough for us to consider
if (possibleBestSSIM > currentBestSSIM)
{
currentBestSSIM = possibleBestSSIM;
currentBestVideo = FrameMetricWrapper.Video;
}
}
}
return(currentBestVideo);
}
}
/// <summary>
/// Calculate the DCT
/// </summary>
/// <param name="sourceImage">The source image to find the DCT of</param>
/// <returns>The DCT coefficients</returns>
public static double[,] Transform(WritableLockBitImage sourceImage)
{
if (sourceImage.Width != sourceImage.Height)
{
throw new ArgumentException("DCTs can only be calculated on square matrices");
}
byte[,] sourceMatrix = new byte[sourceImage.Height, sourceImage.Width];
for (int y = 0; y < sourceImage.Height; y++)
{
for (int x = 0; x < sourceImage.Width; x++)
{
// We only need the greyscale (luminance)
sourceMatrix[y, x] = sourceImage.GetPixel(x, y).R;
}
}
return(Calculate(sourceMatrix));
}
private static Color GetMedianColorValue(WritableLockBitImage sourceImage)
{
var setOfRedColorValues = new HashSet <int>();
var setOfGreenColorValues = new HashSet <int>();
var setOfBlueColorValues = new HashSet <int>();
for (int row = 0; row < sourceImage.Height; row++)
{
for (int col = 0; col < sourceImage.Width; col++)
{
Color color = sourceImage.GetPixel(col, row);
setOfRedColorValues.Add(color.R);
setOfGreenColorValues.Add(color.G);
setOfBlueColorValues.Add(color.B);
}
}
List <int> listOfRedColorValues = setOfRedColorValues.ToList();
List <int> listOfGreenColorValues = setOfRedColorValues.ToList();
List <int> listOfBlueColorValues = setOfRedColorValues.ToList();
listOfRedColorValues.Sort();
listOfGreenColorValues.Sort();
listOfBlueColorValues.Sort();
int count = listOfRedColorValues.Count;
int medianRed, medianGreen, medianBlue;
if (count % 2 == 0)
{
medianRed = (int)Math.Round((listOfRedColorValues[count / 2] + listOfRedColorValues[count / 2 - 1]) / 2.0);
medianGreen = (int)Math.Round((listOfGreenColorValues[count / 2] + listOfGreenColorValues[count / 2 - 1]) / 2.0);
medianBlue = (int)Math.Round((listOfBlueColorValues[count / 2] + listOfBlueColorValues[count / 2 - 1]) / 2.0);
}
else
{
medianRed = listOfRedColorValues[count / 2];
medianGreen = listOfGreenColorValues[count / 2];
medianBlue = listOfBlueColorValues[count / 2];
}
return(Color.FromArgb(medianRed, medianGreen, medianBlue));
}
private static WritableLockBitImage CalculateContour(int[] horizontalMatrix, int[] verticalMatrix, int width, int height)
{
WritableLockBitImage outputImage = new WritableLockBitImage(width, height);
for (int row = 0; row < height; row++)
{
for (int col = 0; col < width; col++)
{
int horizontalValue = horizontalMatrix[row * width + col];
int verticalValue = verticalMatrix[row * width + col];
byte pixelValue = (byte)Math.Round(Math.Sqrt(
(horizontalValue * horizontalValue) +
(verticalValue * verticalValue)
));
outputImage.SetPixel(col, row, Color.FromArgb(pixelValue, pixelValue, pixelValue));
}
}
return(outputImage);
}
private static Tuple <ulong, byte[]> CalcluateFramePerceptionHash(Image frame, bool shouldDoEdgeDetection)
{
using (WritableLockBitImage resizedImage = new WritableLockBitImage(ResizeTransformation.Transform(frame, FingerPrintWidth, FingerPrintWidth)))
using (WritableLockBitImage grayscaleImage = GreyScaleTransformation.TransformInPlace(resizedImage))
using (WritableLockBitImage blurredImage = FastGaussianBlur.Transform(grayscaleImage))
{
double[,] dctMatrix = FastDCTCalculator.Transform(blurredImage);
double medianOfDCTValue = CalculateMedianDCTValue(dctMatrix);
ulong hashCode = ConstructHashCode(dctMatrix, medianOfDCTValue);
if (shouldDoEdgeDetection)
{
using (WritableLockBitImage sobelImage = SobelFilter.TransformWithGrayScaleImage(grayscaleImage))
using (WritableLockBitImage quantisizedImage = QuantisizingFilter.TransformInPlace(sobelImage, 1))
using (WritableLockBitImage resizedQuantisizedImage = ResizeTransformation.Transform(quantisizedImage, 16, 16))
{
return(Tuple.Create(hashCode, CalculateGrayScaleThumbnail(resizedQuantisizedImage)));
}
}
return(Tuple.Create <ulong, byte[]>(hashCode, null));
}
}
private void RunConsumer()
{
try
{
foreach (WorkItem item in _buffer.GetConsumingEnumerable())
{
using (WritableLockBitImage frame = item.Frame)
{
frame.Lock();
Tuple <ulong, byte[]> fingerPrintHashTuple = FrameIndexer.IndexFrame(item.Frame.GetImage());
_fingerPrints.Add(new FrameFingerPrintWrapper
{
PHashCode = fingerPrintHashTuple.Item1,
FrameNumber = item.FrameNumber,
EdgeGrayScaleThumb = fingerPrintHashTuple.Item2,
});
// Reduce the current memory level
long currentMemoryLevels = Interlocked.Add(ref _currentMemoryLevel, -frame.MemorySize);
// Check capacity
if (currentMemoryLevels < _maxCapacity)
{
// If we have space, set event
_capacityBarrier.Set();
}
else
{
// Otherwise, reset the event
_capacityBarrier.Reset();
}
}
}
}
catch (OperationCanceledException)
{
Shutdown();
}
}
private static int[] CoreCalculateSobelMatrix(WritableLockBitImage sourceImage, int[] firstKernel, int[] secondKernel)
{
// Go through each pixel in the source image
int[] intermediateResult = new int[sourceImage.Height * sourceImage.Width];
int[] inputBuffer = new int[3];
int width = sourceImage.Width;
for (int row = 0; row < sourceImage.Height; row++)
{
for (int col = 0; col < sourceImage.Width; col++)
{
// Set the input buffer
inputBuffer[0] = col > 0 ? sourceImage.GetPixel(col - 1, row).R : 0;
inputBuffer[1] = sourceImage.GetPixel(col, row).R;
inputBuffer[2] = col < sourceImage.Width - 1 ? sourceImage.GetPixel(col + 1, row).R : 0;
// Convolute it with the vector
intermediateResult[row * width + col] = ConvoluteOneDimensionalVector(inputBuffer, firstKernel);
}
}
// Go through each result in the intermediate result
int[] finalMatrix = new int[sourceImage.Height * sourceImage.Width];
for (int row = 0; row < sourceImage.Height; row++)
{
for (int col = 0; col < sourceImage.Width; col++)
{
// Set input buffer
inputBuffer[0] = row > 0 ? intermediateResult[((row - 1) * width) + col] : 0;
inputBuffer[1] = intermediateResult[row * width + col];
inputBuffer[2] = row < sourceImage.Height - 1 ? intermediateResult[((row + 1) * width) + col] : 0;
// Convolute it with the second vector
finalMatrix[row * width + col] = ConvoluteOneDimensionalVector(inputBuffer, secondKernel);
}
}
return(finalMatrix);
}
private static void PerformHorizontalBoxBlurAcc(WritableLockBitImage sourceImage, WritableLockBitImage outputImage, int radius)
{
double iarr = 1 / ((double)radius + radius + 1);
for (int row = 0; row < sourceImage.Height; row++)
{
Color firstPixel = sourceImage.GetPixel(0, row);
Color lastPixel = sourceImage.GetPixel(sourceImage.Width - 1, row);
int cumRedValue = (radius + 1) * firstPixel.R;
int cumGreenValue = (radius + 1) * firstPixel.G;
int cumBlueValue = (radius + 1) * firstPixel.B;
int currentLastColIndex = 0; // li
int currentRadiusColIndex = radius; // ri
for (int col = 0; col < radius; col++)
{
Color chosenPixel = sourceImage.GetPixel(col, row);
cumRedValue += chosenPixel.R;
cumGreenValue += chosenPixel.G;
cumBlueValue += chosenPixel.B;
}
for (int col = 0; col <= radius; col++)
{
Color chosenPixel = sourceImage.GetPixel(currentRadiusColIndex, row);
cumRedValue += chosenPixel.R - firstPixel.R;
cumGreenValue += chosenPixel.G - firstPixel.G;
cumBlueValue += chosenPixel.B - firstPixel.B;
currentRadiusColIndex++;
outputImage.SetPixel(
col,
row,
Color.FromArgb(
(int)Math.Round(cumRedValue * iarr),
(int)Math.Round(cumGreenValue * iarr),
(int)Math.Round(cumBlueValue * iarr)
)
);
}
for (int col = radius + 1; col < sourceImage.Width - radius; col++)
{
Color chosenRadiusPixel = sourceImage.GetPixel(currentRadiusColIndex, row);
Color chosenLastPixel = sourceImage.GetPixel(currentLastColIndex, row);
cumRedValue += chosenRadiusPixel.R - chosenLastPixel.R;
cumGreenValue += chosenRadiusPixel.G - chosenLastPixel.G;
cumBlueValue += chosenRadiusPixel.B - chosenLastPixel.B;
currentRadiusColIndex++;
currentLastColIndex++;
outputImage.SetPixel(
col,
row,
Color.FromArgb(
(int)Math.Round(cumRedValue * iarr),
(int)Math.Round(cumGreenValue * iarr),
(int)Math.Round(cumBlueValue * iarr)
)
);
}
for (int col = sourceImage.Width - radius; col < sourceImage.Width; col++)
{
Color chosenLastPixel = sourceImage.GetPixel(currentLastColIndex, row);
cumRedValue += lastPixel.R - chosenLastPixel.R;
cumGreenValue += lastPixel.G - chosenLastPixel.G;
cumBlueValue += lastPixel.B - chosenLastPixel.B;
currentLastColIndex++;
outputImage.SetPixel(
col,
row,
Color.FromArgb(
(int)Math.Round(cumRedValue * iarr),
(int)Math.Round(cumGreenValue * iarr),
(int)Math.Round(cumBlueValue * iarr)
)
);
}
}
}
/// <summary>
/// Transform this image to a greyscale version in-place
/// </summary>
/// <remarks>This will transform the image INPLACE</remarks>
/// <returns>A black and white image. This reference is equal to the input</returns>
public static WritableLockBitImage TransformInPlace(WritableLockBitImage image)
{
Transform(image, image);
return(image);
}
public static WritableLockBitImage TransformInPlace(WritableLockBitImage sourceImage, byte maxValue = byte.MaxValue)
{
return(Transform(sourceImage, sourceImage, maxValue));
}
private static void PerformTotalBoxBlurAcc(WritableLockBitImage sourceImage, WritableLockBitImage outputImage, int radius)
{
double iarr = 1 / ((double)radius + radius + 1);
for (int col = 0; col < sourceImage.Width; col++)
{
Color topPixel = sourceImage.GetPixel(col, 0);
Color bottomPixel = sourceImage.GetPixel(col, sourceImage.Height - 1);
int cumRedValue = (radius + 1) * topPixel.R;
int cumGreenValue = (radius + 1) * topPixel.G;
int cumBlueValue = (radius + 1) * topPixel.B;
for (int row = 0; row < radius; row++)
{
Color chosenPixel = sourceImage.GetPixel(col, row);
cumRedValue += chosenPixel.R;
cumGreenValue += chosenPixel.G;
cumBlueValue += chosenPixel.B;
}
for (int row = 0; row <= radius; row++)
{
Color chosenPixel = sourceImage.GetPixel(col, row + radius);
cumRedValue += chosenPixel.R - topPixel.R;
cumGreenValue += chosenPixel.G - topPixel.G;
cumBlueValue += chosenPixel.B - topPixel.B;
outputImage.SetPixel(
col,
row,
Color.FromArgb(
(int)Math.Round(cumRedValue * iarr),
(int)Math.Round(cumGreenValue * iarr),
(int)Math.Round(cumBlueValue * iarr)
)
);
}
for (int row = radius + 1; row < sourceImage.Height - radius; row++)
{
Color radiusPixel = sourceImage.GetPixel(col, radius + row);
Color laggingPixel = sourceImage.GetPixel(col, row - radius - 1);
cumRedValue += radiusPixel.R - laggingPixel.R;
cumGreenValue += radiusPixel.G - laggingPixel.G;
cumBlueValue += radiusPixel.B - laggingPixel.B;
outputImage.SetPixel(
col,
row,
Color.FromArgb(
(int)Math.Round(cumRedValue * iarr),
(int)Math.Round(cumGreenValue * iarr),
(int)Math.Round(cumBlueValue * iarr)
)
);
}
for (int row = sourceImage.Height - radius; row < sourceImage.Height; row++)
{
Color laggingPixel = sourceImage.GetPixel(col, row - radius);
cumRedValue += bottomPixel.R - laggingPixel.R;
cumGreenValue += bottomPixel.G - laggingPixel.G;
cumBlueValue += bottomPixel.B - laggingPixel.B;
outputImage.SetPixel(
col,
row,
Color.FromArgb(
(int)Math.Round(cumRedValue * iarr),
(int)Math.Round(cumGreenValue * iarr),
(int)Math.Round(cumBlueValue * iarr)
)
);
}
}
}
private static int[] CalculateVerticalSobelMatrix(WritableLockBitImage sourceImage)
{
return(CoreCalculateSobelMatrix(sourceImage, VerticalMatrixStep1, VerticalMatrixStep2));
}
/// <summary>
/// Performs the Sobel Filter on the given gray scale image
/// </summary>
/// <param name="grayScaleImage"></param>
/// <returns></returns>
public static WritableLockBitImage TransformWithGrayScaleImage(WritableLockBitImage grayScaleImage)
{
return(ApplySobelFilter(grayScaleImage));
}
