static void ProcessFileNaive(string filename, List<int> partitionHash)
{
var rollingHash = new List<uint>();
var localMaximaPos = new List<int>();
var fciBC = new BlockingCollectionDataChunk<FileChunkInfo>();
var fileBytes = File.ReadAllBytes(filename);
using (var ms = new MemoryStream(fileBytes, 0, fileBytes.Length, true, true))
{
var fh = new FileHash(1024);
fh.StreamToHashValuesNaive(ms, rollingHash);
}
var lm = new LocalMaxima(4 * 1024);
lm.CalcUsingNaive(rollingHash, localMaximaPos);
var localMaximaPosBC = new BlockingCollectionDataChunk<int>();
foreach (var pos in localMaximaPos)
{
localMaximaPosBC.Add(pos);
}
localMaximaPosBC.CompleteAdding();
var ph = new BlockingCollectionDataChunk<uint>();
var mmh = new MurmurHash3_x86_32();
var fph = new FileParitionHash(mmh);
using (var fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
fph.ProcessStream(fs, localMaximaPosBC, ph, fciBC);
}
foreach (var items in ph.BlockingCollection.GetConsumingEnumerable())
{
for (var i = 0; i < items.DataSize; ++i)
{
partitionHash.Add((int)items.Data[i]);
}
}
}
static void ProcessFile(string filename, List<int> partitionHash, List<FileChunkInfo> fci)
{
var rollingHash = new BlockingCollectionDataChunk<uint>();
var localMaximaPos = new BlockingCollectionDataChunk<int>();
var ph = new BlockingCollectionDataChunk<uint>();
var fciBC = new BlockingCollectionDataChunk<FileChunkInfo>();
//var sw = new Stopwatch();
//sw.Start();
//var fLength = 0;
//using (var fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
//{
// fLength = (int)fs.Length;
//}
//var lmWindow = fLength / (512);
var lmWindow = 32 * 1024;
Task.Run(() =>
{
using (var fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
var fh = new FileHash(16);
fh.StreamToHashValues(fs, rollingHash);
}
});
Task.Run(() =>
{
var lm = new LocalMaxima(lmWindow);
lm.CalcUsingBlockAlgo(rollingHash, localMaximaPos);
});
Task.Run(() =>
{
var mmh = new MurmurHash3_x86_32();
var fph = new FileParitionHash(mmh);
using (var fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
fph.ProcessStream(fs, localMaximaPos, ph, fciBC);
}
});
var count = 0;
foreach (var items in ph.BlockingCollection.GetConsumingEnumerable())
{
count += items.DataSize;
//Console.WriteLine("File par hash: {0}", i);
for (var i = 0; i < items.DataSize; ++i)
{
partitionHash.Add((int)items.Data[i]);
}
}
foreach (var items in fciBC.BlockingCollection.GetConsumingEnumerable())
{
for (var i = 0; i < items.DataSize; ++i)
{
fci.Add(items.Data[i]);
}
}
//sw.Stop();
//Console.WriteLine("Number of partitions: {0}", count);
//Console.WriteLine("Time: {0} ms", sw.ElapsedMilliseconds);
}