public void TestVideoIndexerSerialization() { var frameFingerPrint = new FrameFingerPrintWrapper { FrameNumber = 0, PHashCode = 0x1010, EdgeGrayScaleThumb = new byte[] { 0, 1, 0 }, }; var videoFingerPrint = new VideoFingerPrintWrapper { FilePath = "test.mkv", FingerPrints = new[] { frameFingerPrint }, }; var database = new VideoFingerPrintDatabaseWrapper { VideoFingerPrints = new[] { videoFingerPrint }, }; using (var memoryStream = new MemoryStream()) { VideoFingerPrintDatabaseSaver.Save(database, memoryStream); byte[] savedDatabase = memoryStream.ToArray(); VideoFingerPrintDatabaseWrapper reloadedDatabase = VideoFingerPrintDatabaseLoader.Load(savedDatabase); Assert.AreEqual(database, reloadedDatabase); } }
private static VideoFingerPrintDatabaseMetaTableEntryWrapper CoalesceDatabases(IEnumerable <VideoFingerPrintDatabaseMetaTableEntryWrapper> coalescedEntries) { // Load each database IEnumerable <VideoFingerPrintDatabaseWrapper> databases = from entry in coalescedEntries select VideoFingerPrintDatabaseLoader.Load(entry.FileName); // Merge fingerprints IEnumerable <VideoFingerPrintWrapper> allVideoFingerPrints = from database in databases from videoFingerPrint in database.VideoFingerPrints select videoFingerPrint; // Create a new database var freshDatabase = new VideoFingerPrintDatabaseWrapper { VideoFingerPrints = allVideoFingerPrints.ToArray(), }; // Save the database string databaseFileName = Path.GetRandomFileName() + ".bin"; VideoFingerPrintDatabaseSaver.Save(freshDatabase, databaseFileName); FileInfo databaseFileInfo = new FileInfo(databaseFileName); return(new VideoFingerPrintDatabaseMetaTableEntryWrapper { FileName = databaseFileName, FileSize = (ulong)databaseFileInfo.Length, }); }
private static byte[] SaveDatabase(VideoFingerPrintDatabaseWrapper database) { var builder = new FlatBufferBuilder(DefaultBufferSize); CreateVideoFingerPrintDatabase(database, builder); return(builder.SizedByteArray()); }
/// <summary> /// Save a fingerprint database to a file /// </summary> /// <param name="database">The database to save</param> /// <param name="filePath">The file path to save it to</param> public static void Save(VideoFingerPrintDatabaseWrapper database, string filePath) { byte[] rawDatabaseBytes = SaveDatabase(database); using (BinaryWriter writer = new BinaryWriter(File.Open(filePath, FileMode.Create))) { writer.Write(rawDatabaseBytes); } }
private static void CreateVideoFingerPrintDatabase(VideoFingerPrintDatabaseWrapper database, FlatBufferBuilder builder) { Offset <VideoFingerPrint>[] videoFingerPrintArray = CreateVideoFingerPrintArray(database, builder); VectorOffset videoFingerPrintDatabaseVectorOffset = VideoFingerPrintDatabase.CreateVideoFingerPrintsVector(builder, videoFingerPrintArray); Offset <VideoFingerPrintDatabase> databaseOffset = VideoFingerPrintDatabase.CreateVideoFingerPrintDatabase(builder, videoFingerPrintDatabaseVectorOffset); VideoFingerPrintDatabase.FinishVideoFingerPrintDatabaseBuffer(builder, databaseOffset); }
private static void ExecuteCheckDatabase(string[] args) { string maxMemoryArg = GetMaxMemory(args); string databaseMetaTablePath = GetDatabaseMetaTable(args); if (string.IsNullOrWhiteSpace(databaseMetaTablePath)) { PrintHelp("Database metatable path not provided"); return; } if (File.Exists(databaseMetaTablePath) == false) { PrintHelp("Database MetaTable does not exist"); return; } if (string.IsNullOrWhiteSpace(maxMemoryArg)) { PrintHelp("--max-memory must be set"); return; } long maxMemory = 0; if (long.TryParse(maxMemoryArg, out maxMemory) == false) { PrintHelp("--max-memory could not be parsed"); return; } VideoFingerPrintDatabaseMetaTableWrapper metaTable = VideoFingerPrintDatabaseMetaTableLoader.Load(databaseMetaTablePath); var random = new Random(); foreach (string databasePath in metaTable.DatabaseMetaTableEntries.Select(e => e.FileName)) { VideoFingerPrintDatabaseWrapper database = VideoFingerPrintDatabaseLoader.Load(databasePath); int videoFingerPrintSampleCount = (int)Math.Round(database.VideoFingerPrints.Length / 3.0); IEnumerable <VideoFingerPrintWrapper> videoFingerPrints = from fingerPrint in database.VideoFingerPrints where random.Next() % 2 == 0 select fingerPrint; foreach (VideoFingerPrintWrapper videoFingerPrint in videoFingerPrints.Take(videoFingerPrintSampleCount)) { VideoFingerPrintWrapper actualVideoFingerPrint = Video.VideoIndexer.IndexVideo(videoFingerPrint.FilePath, maxMemory); if (Equals(videoFingerPrint, actualVideoFingerPrint) == false) { Console.WriteLine("{0} Fingerprint does not match", Path.GetFileName(videoFingerPrint.FilePath)); } } } }
private static IEnumerable <string> GetKnownDatabaseEntries(string databaseMetaTablePath) { if (File.Exists(databaseMetaTablePath) == false) { return(Enumerable.Empty <string>()); } var knownDatabaseEntries = new HashSet <string>(); VideoFingerPrintDatabaseMetaTableWrapper metatable = VideoFingerPrintDatabaseMetaTableLoader.Load(databaseMetaTablePath); foreach (VideoFingerPrintDatabaseMetaTableEntryWrapper entry in metatable.DatabaseMetaTableEntries) { VideoFingerPrintDatabaseWrapper database = VideoFingerPrintDatabaseLoader.Load(entry.FileName); foreach (VideoFingerPrintWrapper fingerprint in database.VideoFingerPrints) { knownDatabaseEntries.Add(fingerprint.FilePath); } } return(knownDatabaseEntries); }
private static Offset <VideoFingerPrint>[] CreateVideoFingerPrintArray(VideoFingerPrintDatabaseWrapper database, FlatBufferBuilder builder) { int videoFingerPrintCounter = 0; var videoFingerPrintArray = new Offset <VideoFingerPrint> [database.VideoFingerPrints.Length]; foreach (VideoFingerPrintWrapper videoFingerPrint in database.VideoFingerPrints) { // NOTE: Do not reorder these statements Offset <FrameFingerPrint>[] frameFingerPrintArray = CreateFrameFingerPrintArray(builder, videoFingerPrint); StringOffset videoFilePath = builder.CreateString(videoFingerPrint.FilePath); VectorOffset frameFingerPrintVectorOffset = VideoFingerPrint.CreateFrameFingerPrintsVector(builder, frameFingerPrintArray); VideoFingerPrint.StartVideoFingerPrint(builder); VideoFingerPrint.AddFilePath(builder, videoFilePath); VideoFingerPrint.AddFrameFingerPrints(builder, frameFingerPrintVectorOffset); videoFingerPrintArray[videoFingerPrintCounter] = VideoFingerPrint.EndVideoFingerPrint(builder); videoFingerPrintCounter++; } return(videoFingerPrintArray); }
private Tuple <VideoFingerPrintDatabaseWrapper, string> CreateNewDatabaseAndAddToMetatable() { VideoFingerPrintDatabaseMetaTableWrapper metatable = CreateOrLoadMetatable(_metatablePath); string emptyDatabaseFileName = Path.GetRandomFileName() + ".bin"; VideoFingerPrintDatabaseWrapper emptyDatabase = new VideoFingerPrintDatabaseWrapper(); VideoFingerPrintDatabaseSaver.Save(emptyDatabase, emptyDatabaseFileName); // Add to the metatable var databaseEntries = new List <VideoFingerPrintDatabaseMetaTableEntryWrapper>(metatable.DatabaseMetaTableEntries); databaseEntries.Add(new VideoFingerPrintDatabaseMetaTableEntryWrapper { FileName = emptyDatabaseFileName, FileSize = 0ul, }); metatable.DatabaseMetaTableEntries = databaseEntries.ToArray(); VideoFingerPrintDatabaseMetaTableSaver.Save(metatable, _metatablePath); return(Tuple.Create(emptyDatabase, emptyDatabaseFileName)); }
/// <summary> /// Save a fingerprint database to a stream /// </summary> /// <param name="database">The database to save</param> /// <param name="outStream">The stream to write to</param> public static void Save(VideoFingerPrintDatabaseWrapper database, Stream outStream) { byte[] buffer = SaveDatabase(database); outStream.Write(buffer, 0, buffer.Length); }
private void RunQueue() { var fingerprintBuffer = new List <VideoFingerPrintWrapper>(); Tuple <VideoFingerPrintDatabaseWrapper, string> currentDatabaseTuple = GetNextEligibleDatabase(); bool needsFinalFlush = false; foreach (VideoFingerPrintWrapper fingerprint in _workItems.GetConsumingEnumerable()) { try { Console.WriteLine("Adding fingerprint: {0}", Path.GetFileName(fingerprint.FilePath)); fingerprintBuffer.Add(fingerprint); if (fingerprintBuffer.Count > 5) { VideoFingerPrintDatabaseWrapper currentDatabase = currentDatabaseTuple.Item1; string currentDatabasePath = currentDatabaseTuple.Item2; Console.WriteLine("Flushing database"); // Flush the buffer needsFinalFlush = false; // Add entries to database currentDatabase.VideoFingerPrints = currentDatabase.VideoFingerPrints.Concat(fingerprintBuffer).ToArray(); // Save entries to disk VideoFingerPrintDatabaseSaver.Save(currentDatabase, currentDatabasePath); FileInfo fileInfo = new FileInfo(currentDatabasePath); ulong fileSize = (ulong)fileInfo.Length; // Save metatable to disk UpdateMetatable(currentDatabasePath, fileSize); // Now, check if we need to update the current database if (fileSize > MaxDatabaseSize) { currentDatabaseTuple = GetNextEligibleDatabase(); } // Lastly, clear the buffer fingerprintBuffer.Clear(); } else { needsFinalFlush = true; } } catch (Exception e) { Console.WriteLine("Could not write database. {0}", e.Message); } } if (needsFinalFlush) { try { Console.WriteLine("Flushing database for the final time"); // Flush the buffer one last time // Add entries to database currentDatabaseTuple.Item1.VideoFingerPrints = currentDatabaseTuple.Item1.VideoFingerPrints.Concat(fingerprintBuffer).ToArray(); // Save entries to disk VideoFingerPrintDatabaseSaver.Save(currentDatabaseTuple.Item1, currentDatabaseTuple.Item2); } catch (Exception e) { Console.WriteLine("Could not perform final flush of database. {0}", e.Message); } } }