public void FileTokenizationTest()
        {
            var sourceFileHasher = new SourceFileHasher();
            var stopwatch        = Stopwatch.StartNew();

            sourceFileHasher.ProcessFile(@"D:\CONGEN\cnt\agent-library\AgentSkills\AgentSkills\InformationAgent.FetchnExtract\fetchnextract.cs");
            stopwatch.Stop();
            Console.WriteLine(stopwatch.Elapsed);
            Assert.Pass();
        }
        public void AlTokenizationTest()
        {
            var sourceFileHasher = new SourceFileHasher();
            var stopwatch        = Stopwatch.StartNew();

            foreach (var file in Directory.GetFiles(@"D:\CONGEN\cnt\agent-library\", "*.cs", SearchOption.AllDirectories))
            {
                sourceFileHasher.ProcessFile(file);
            }

            sourceFileHasher.SaveToFile(tempSavePath);

            stopwatch.Stop();

            var restoreFileHasher = new SourceFileHasher();

            restoreFileHasher.RestoreFromFile(tempSavePath);

            Assert.AreEqual(restoreFileHasher.Entries, sourceFileHasher.Entries);
            Assert.AreEqual(restoreFileHasher.FilesDictionary, sourceFileHasher.FilesDictionary);

            Console.WriteLine(stopwatch.Elapsed);
            Assert.Pass();
        }