public void TestVideoIndexerSerialization()
{
var frameFingerPrint = new FrameFingerPrintWrapper
{
FrameNumber = 0,
PHashCode = 0x1010,
EdgeGrayScaleThumb = new byte[] { 0, 1, 0 },
};
var videoFingerPrint = new VideoFingerPrintWrapper
{
FilePath = "test.mkv",
FingerPrints = new[] { frameFingerPrint },
};
var database = new VideoFingerPrintDatabaseWrapper
{
VideoFingerPrints = new[] { videoFingerPrint },
};
using (var memoryStream = new MemoryStream())
{
VideoFingerPrintDatabaseSaver.Save(database, memoryStream);
byte[] savedDatabase = memoryStream.ToArray();
VideoFingerPrintDatabaseWrapper reloadedDatabase = VideoFingerPrintDatabaseLoader.Load(savedDatabase);
Assert.AreEqual(database, reloadedDatabase);
}
}
private static VideoFingerPrintDatabaseMetaTableEntryWrapper CoalesceDatabases(IEnumerable <VideoFingerPrintDatabaseMetaTableEntryWrapper> coalescedEntries)
{
// Load each database
IEnumerable <VideoFingerPrintDatabaseWrapper> databases = from entry in coalescedEntries
select VideoFingerPrintDatabaseLoader.Load(entry.FileName);
// Merge fingerprints
IEnumerable <VideoFingerPrintWrapper> allVideoFingerPrints = from database in databases
from videoFingerPrint in database.VideoFingerPrints
select videoFingerPrint;
// Create a new database
var freshDatabase = new VideoFingerPrintDatabaseWrapper
{
VideoFingerPrints = allVideoFingerPrints.ToArray(),
};
// Save the database
string databaseFileName = Path.GetRandomFileName() + ".bin";
VideoFingerPrintDatabaseSaver.Save(freshDatabase, databaseFileName);
FileInfo databaseFileInfo = new FileInfo(databaseFileName);
return(new VideoFingerPrintDatabaseMetaTableEntryWrapper
{
FileName = databaseFileName,
FileSize = (ulong)databaseFileInfo.Length,
});
}
private static byte[] SaveDatabase(VideoFingerPrintDatabaseWrapper database)
{
var builder = new FlatBufferBuilder(DefaultBufferSize);
CreateVideoFingerPrintDatabase(database, builder);
return(builder.SizedByteArray());
}
/// <summary>
/// Save a fingerprint database to a file
/// </summary>
/// <param name="database">The database to save</param>
/// <param name="filePath">The file path to save it to</param>
public static void Save(VideoFingerPrintDatabaseWrapper database, string filePath)
{
byte[] rawDatabaseBytes = SaveDatabase(database);
using (BinaryWriter writer = new BinaryWriter(File.Open(filePath, FileMode.Create)))
{
writer.Write(rawDatabaseBytes);
}
}
private static void CreateVideoFingerPrintDatabase(VideoFingerPrintDatabaseWrapper database, FlatBufferBuilder builder)
{
Offset <VideoFingerPrint>[] videoFingerPrintArray = CreateVideoFingerPrintArray(database, builder);
VectorOffset videoFingerPrintDatabaseVectorOffset = VideoFingerPrintDatabase.CreateVideoFingerPrintsVector(builder, videoFingerPrintArray);
Offset <VideoFingerPrintDatabase> databaseOffset = VideoFingerPrintDatabase.CreateVideoFingerPrintDatabase(builder, videoFingerPrintDatabaseVectorOffset);
VideoFingerPrintDatabase.FinishVideoFingerPrintDatabaseBuffer(builder, databaseOffset);
}
private static void ExecuteCheckDatabase(string[] args)
{
string maxMemoryArg = GetMaxMemory(args);
string databaseMetaTablePath = GetDatabaseMetaTable(args);
if (string.IsNullOrWhiteSpace(databaseMetaTablePath))
{
PrintHelp("Database metatable path not provided");
return;
}
if (File.Exists(databaseMetaTablePath) == false)
{
PrintHelp("Database MetaTable does not exist");
return;
}
if (string.IsNullOrWhiteSpace(maxMemoryArg))
{
PrintHelp("--max-memory must be set");
return;
}
long maxMemory = 0;
if (long.TryParse(maxMemoryArg, out maxMemory) == false)
{
PrintHelp("--max-memory could not be parsed");
return;
}
VideoFingerPrintDatabaseMetaTableWrapper metaTable = VideoFingerPrintDatabaseMetaTableLoader.Load(databaseMetaTablePath);
var random = new Random();
foreach (string databasePath in metaTable.DatabaseMetaTableEntries.Select(e => e.FileName))
{
VideoFingerPrintDatabaseWrapper database = VideoFingerPrintDatabaseLoader.Load(databasePath);
int videoFingerPrintSampleCount = (int)Math.Round(database.VideoFingerPrints.Length / 3.0);
IEnumerable <VideoFingerPrintWrapper> videoFingerPrints = from fingerPrint in database.VideoFingerPrints
where random.Next() % 2 == 0
select fingerPrint;
foreach (VideoFingerPrintWrapper videoFingerPrint in videoFingerPrints.Take(videoFingerPrintSampleCount))
{
VideoFingerPrintWrapper actualVideoFingerPrint = Video.VideoIndexer.IndexVideo(videoFingerPrint.FilePath, maxMemory);
if (Equals(videoFingerPrint, actualVideoFingerPrint) == false)
{
Console.WriteLine("{0} Fingerprint does not match", Path.GetFileName(videoFingerPrint.FilePath));
}
}
}
}
private static IEnumerable <string> GetKnownDatabaseEntries(string databaseMetaTablePath)
{
if (File.Exists(databaseMetaTablePath) == false)
{
return(Enumerable.Empty <string>());
}
var knownDatabaseEntries = new HashSet <string>();
VideoFingerPrintDatabaseMetaTableWrapper metatable = VideoFingerPrintDatabaseMetaTableLoader.Load(databaseMetaTablePath);
foreach (VideoFingerPrintDatabaseMetaTableEntryWrapper entry in metatable.DatabaseMetaTableEntries)
{
VideoFingerPrintDatabaseWrapper database = VideoFingerPrintDatabaseLoader.Load(entry.FileName);
foreach (VideoFingerPrintWrapper fingerprint in database.VideoFingerPrints)
{
knownDatabaseEntries.Add(fingerprint.FilePath);
}
}
return(knownDatabaseEntries);
}
private static Offset <VideoFingerPrint>[] CreateVideoFingerPrintArray(VideoFingerPrintDatabaseWrapper database, FlatBufferBuilder builder)
{
int videoFingerPrintCounter = 0;
var videoFingerPrintArray = new Offset <VideoFingerPrint> [database.VideoFingerPrints.Length];
foreach (VideoFingerPrintWrapper videoFingerPrint in database.VideoFingerPrints)
{
// NOTE: Do not reorder these statements
Offset <FrameFingerPrint>[] frameFingerPrintArray = CreateFrameFingerPrintArray(builder, videoFingerPrint);
StringOffset videoFilePath = builder.CreateString(videoFingerPrint.FilePath);
VectorOffset frameFingerPrintVectorOffset = VideoFingerPrint.CreateFrameFingerPrintsVector(builder, frameFingerPrintArray);
VideoFingerPrint.StartVideoFingerPrint(builder);
VideoFingerPrint.AddFilePath(builder, videoFilePath);
VideoFingerPrint.AddFrameFingerPrints(builder, frameFingerPrintVectorOffset);
videoFingerPrintArray[videoFingerPrintCounter] = VideoFingerPrint.EndVideoFingerPrint(builder);
videoFingerPrintCounter++;
}
return(videoFingerPrintArray);
}
private Tuple <VideoFingerPrintDatabaseWrapper, string> CreateNewDatabaseAndAddToMetatable()
{
VideoFingerPrintDatabaseMetaTableWrapper metatable = CreateOrLoadMetatable(_metatablePath);
string emptyDatabaseFileName = Path.GetRandomFileName() + ".bin";
VideoFingerPrintDatabaseWrapper emptyDatabase = new VideoFingerPrintDatabaseWrapper();
VideoFingerPrintDatabaseSaver.Save(emptyDatabase, emptyDatabaseFileName);
// Add to the metatable
var databaseEntries = new List <VideoFingerPrintDatabaseMetaTableEntryWrapper>(metatable.DatabaseMetaTableEntries);
databaseEntries.Add(new VideoFingerPrintDatabaseMetaTableEntryWrapper
{
FileName = emptyDatabaseFileName,
FileSize = 0ul,
});
metatable.DatabaseMetaTableEntries = databaseEntries.ToArray();
VideoFingerPrintDatabaseMetaTableSaver.Save(metatable, _metatablePath);
return(Tuple.Create(emptyDatabase, emptyDatabaseFileName));
}
/// <summary>
/// Save a fingerprint database to a stream
/// </summary>
/// <param name="database">The database to save</param>
/// <param name="outStream">The stream to write to</param>
public static void Save(VideoFingerPrintDatabaseWrapper database, Stream outStream)
{
byte[] buffer = SaveDatabase(database);
outStream.Write(buffer, 0, buffer.Length);
}
private void RunQueue()
{
var fingerprintBuffer = new List <VideoFingerPrintWrapper>();
Tuple <VideoFingerPrintDatabaseWrapper, string> currentDatabaseTuple = GetNextEligibleDatabase();
bool needsFinalFlush = false;
foreach (VideoFingerPrintWrapper fingerprint in _workItems.GetConsumingEnumerable())
{
try
{
Console.WriteLine("Adding fingerprint: {0}", Path.GetFileName(fingerprint.FilePath));
fingerprintBuffer.Add(fingerprint);
if (fingerprintBuffer.Count > 5)
{
VideoFingerPrintDatabaseWrapper currentDatabase = currentDatabaseTuple.Item1;
string currentDatabasePath = currentDatabaseTuple.Item2;
Console.WriteLine("Flushing database");
// Flush the buffer
needsFinalFlush = false;
// Add entries to database
currentDatabase.VideoFingerPrints = currentDatabase.VideoFingerPrints.Concat(fingerprintBuffer).ToArray();
// Save entries to disk
VideoFingerPrintDatabaseSaver.Save(currentDatabase, currentDatabasePath);
FileInfo fileInfo = new FileInfo(currentDatabasePath);
ulong fileSize = (ulong)fileInfo.Length;
// Save metatable to disk
UpdateMetatable(currentDatabasePath, fileSize);
// Now, check if we need to update the current database
if (fileSize > MaxDatabaseSize)
{
currentDatabaseTuple = GetNextEligibleDatabase();
}
// Lastly, clear the buffer
fingerprintBuffer.Clear();
}
else
{
needsFinalFlush = true;
}
}
catch (Exception e)
{
Console.WriteLine("Could not write database. {0}", e.Message);
}
}
if (needsFinalFlush)
{
try
{
Console.WriteLine("Flushing database for the final time");
// Flush the buffer one last time
// Add entries to database
currentDatabaseTuple.Item1.VideoFingerPrints = currentDatabaseTuple.Item1.VideoFingerPrints.Concat(fingerprintBuffer).ToArray();
// Save entries to disk
VideoFingerPrintDatabaseSaver.Save(currentDatabaseTuple.Item1, currentDatabaseTuple.Item2);
}
catch (Exception e)
{
Console.WriteLine("Could not perform final flush of database. {0}", e.Message);
}
}
}