public CheckBufferState(byte[] buffer, DiskFile diskFile, string fileName, int blockSize, Dictionary<uint, FileVerificationEntry> hashFull, MatchType matchType, int offset, int overlap)
{
this.buffer = buffer;
this.diskFile = diskFile;
this.fileName = fileName;
this.blockSize = blockSize;
this.hashFull = hashFull;
this.matchType = matchType;
this.offset = offset;
this.overlap = overlap;
}
public CriticalPacketEntry(CriticalPacketEntry other)
{
diskfile = other.diskfile;
offset = other.offset;
packet = other.packet;
}
public CriticalPacketEntry()
{
diskfile = null;
offset = 0;
packet = null;
}
public CriticalPacketEntry(DiskFile _diskFile, ulong _offset, CriticalPacket _packet)
{
diskfile = _diskFile;
offset = _offset;
packet = _packet;
}
public CriticalPacketEntry(CriticalPacketEntry other)
{
diskfile = other.diskfile;
offset = other.offset;
packet = other.packet;
}
public CriticalPacketEntry()
{
diskfile = null;
offset = 0;
packet = null;
}
public void SetCompleteFile(DiskFile diskfile)
{
completefile = diskfile;
}
public void SetTargetFile(DiskFile diskfile)
{
targetfile = diskfile;
}
public static void CheckFile_async(DiskFile diskFile, string filename, int blocksize, List <FileVerificationEntry> fileVerEntry, byte[] md5hash16k, byte[] md5hash, ref MatchType matchType, Dictionary <uint, FileVerificationEntry> hashfull, Dictionary <uint, FileVerificationEntry> hash, List <FileVerificationEntry> expectedList, ref int expectedIndex, bool multithreadCPU)
{
//int THRESHOLD = ((int)((50 * 1024 * 1024) / blocksize) * blocksize); // 50Mo threshold
int THRESHOLD = 5 * blocksize;
matchType = MatchType.FullMatch;
Console.WriteLine("Checking file '{0}'", Path.GetFileName(filename));
long filesize = new FileInfo(filename).Length;
if (filesize <= THRESHOLD)
{
using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
CheckBuffer(br.ReadBytes((int)filesize), diskFile, filename, blocksize, hashfull, ref matchType, 0);
}
}
else
{
if (multithreadCPU)
{
List <Task> tasks = new List <Task>();
int buffer_size = THRESHOLD;
int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks
List <ManualResetEvent> list = new List <ManualResetEvent>();
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 1024, true))
{
int offset = 0;
byte[] buffer = new byte[buffer_size];
ManualResetEvent mre = new ManualResetEvent(false);
list.Add(mre);
CheckBufferState state = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap);
fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(TaskEndReadCallback), new FileCheckerState(fs, buffer, mre, state));
//tasks.Add(Task.Factory.StartNew((arg)
// =>
//{
// try
// {
// object[] args = (object[])arg;
// byte[] b = (byte[])args[0];
// DiskFile df = (DiskFile)args[1];
// string f = (string)args[2];
// int bs = (int)args[3];
// Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4];
// MatchType mt = (MatchType)args[5];
// int o = (int)args[6];
// CheckBuffer(b, df, f, bs, hf, ref mt, o);
// }
// finally
// {
// //concurrencySemaphore.Release();
// }
//}, new object[] { buffer, diskFile, filename, blocksize, hashfull, matchType, offset }));
while (fs.Position < fs.Length)
{
Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap);
ManualResetEvent mre2 = new ManualResetEvent(false);
list.Add(mre2);
CheckBufferState state2 = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap);
fs.BeginRead(buffer, overlap, buffer.Length - overlap, new AsyncCallback(TaskEndReadCallback), new FileCheckerState(fs, buffer, mre2, state2));
int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap);
offset += buffer.Length - overlap;
//if (nbRead < buffer.Length - overlap)
// Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap));
//tasks.Add(Task.Factory.StartNew((arg)
// =>
//{
// try
// {
// object[] args = (object[])arg;
// byte[] b = (byte[])args[0];
// DiskFile df = (DiskFile)args[1];
// string f = (string)args[2];
// int bs = (int)args[3];
// Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4];
// MatchType mt = (MatchType)args[5];
// int o = (int)args[6];
// CheckBuffer(b, df, f, bs, hf, ref mt, o);
// }
// finally
// {
// //concurrencySemaphore.Release();
// }
//}, new object[] { (byte[])buffer.Clone(), diskFile, filename, blocksize, hashfull, matchType, offset }));
}
}
//long startWait = DateTime.Now.Ticks;
//Task.WaitAll(tasks.ToArray());
//long endWait = DateTime.Now.Ticks;
//double duration = ((double)(endWait - startWait)) / 10000;
//Console.WriteLine("Wait : {0}ms", duration);
WaitHandle.WaitAll(list.ToArray());
}
else
{
long startCheck = DateTime.Now.Ticks;
int buffer_size = THRESHOLD;
int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks
List <ManualResetEvent> list = new List <ManualResetEvent>();
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 2 * buffer_size, true))
{
int offset = 0;
byte[] buffer = new byte[buffer_size];
CheckBufferState state = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap);
ManualResetEvent mre = new ManualResetEvent(false);
list.Add(mre);
// Create a synchronization object that gets
// signaled when verification is complete.
fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(EndReadCallback), new FileCheckerState(fs, buffer, mre, state));
//CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset);
while (fs.Position < fs.Length)
{
Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap);
//int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap);
CheckBufferState state2 = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap);
// Create a synchronization object that gets
// signaled when verification is complete.
ManualResetEvent mre2 = new ManualResetEvent(false);
list.Add(mre2);
fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(EndReadCallback), new FileCheckerState(fs, buffer, mre2, state2));
offset += buffer_size;
//if (nbRead < buffer.Length - overlap)
// Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap));
//CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset);
}
WaitHandle.WaitAll(list.ToArray());
}
long endCheck = DateTime.Now.Ticks;
double duration = ((double)(endCheck - startCheck)) / 10000;
Console.WriteLine("Check : {0} ms", duration);
}
}
bool atLeastOne = false;
foreach (FileVerificationEntry entry in fileVerEntry)
{
if (entry.datablock.diskfile != null)
{
atLeastOne = true;
break;
}
}
if (!atLeastOne)
{
matchType = MatchType.NoMatch;
}
}
public static void CheckFile(DiskFile diskFile, string filename, int blocksize, List <FileVerificationEntry> fileVerEntry, byte[] md5hash16k, byte[] md5hash, ref MatchType matchType, Dictionary <uint, FileVerificationEntry> hashfull, Dictionary <uint, FileVerificationEntry> hash, List <FileVerificationEntry> expectedList, ref int expectedIndex, bool multithreadCPU)
{
//int THRESHOLD = ((int)((50 * 1024 * 1024) / blocksize) * blocksize); // 50Mo threshold
int THRESHOLD = 5 * blocksize;
matchType = MatchType.FullMatch;
Console.WriteLine("Checking file '{0}'", Path.GetFileName(filename));
long filesize = new FileInfo(filename).Length;
if (filesize <= THRESHOLD)
{
using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
CheckBuffer(br.ReadBytes((int)filesize), diskFile, filename, blocksize, hashfull, ref matchType, 0);
}
}
else
{
if (multithreadCPU)
{
List <Task> tasks = new List <Task>();
int buffer_size = THRESHOLD;
int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
int offset = 0;
byte[] buffer = new byte[buffer_size];
fs.Read(buffer, 0, buffer_size);
tasks.Add(Task.Factory.StartNew((arg)
=>
{
try
{
object[] args = (object[])arg;
byte[] b = (byte[])args[0];
DiskFile df = (DiskFile)args[1];
string f = (string)args[2];
int bs = (int)args[3];
Dictionary <uint, FileVerificationEntry> hf = (Dictionary <uint, FileVerificationEntry>)args[4];
MatchType mt = (MatchType)args[5];
int o = (int)args[6];
CheckBuffer(b, df, f, bs, hf, ref mt, o);
}
finally
{
//concurrencySemaphore.Release();
}
}, new object[] { buffer, diskFile, filename, blocksize, hashfull, matchType, offset }));
while (fs.Position < fs.Length)
{
Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap);
int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap);
offset += nbRead;
if (nbRead < buffer.Length - overlap)
{
Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap));
}
tasks.Add(Task.Factory.StartNew((arg)
=>
{
try
{
object[] args = (object[])arg;
byte[] b = (byte[])args[0];
DiskFile df = (DiskFile)args[1];
string f = (string)args[2];
int bs = (int)args[3];
Dictionary <uint, FileVerificationEntry> hf = (Dictionary <uint, FileVerificationEntry>)args[4];
MatchType mt = (MatchType)args[5];
int o = (int)args[6];
CheckBuffer(b, df, f, bs, hf, ref mt, o);
}
finally
{
//concurrencySemaphore.Release();
}
}, new object[] { (byte[])buffer.Clone(), diskFile, filename, blocksize, hashfull, matchType, offset }));
}
}
long startWait = DateTime.Now.Ticks;
Task.WaitAll(tasks.ToArray());
long endWait = DateTime.Now.Ticks;
double duration = ((double)(endWait - startWait)) / 10000;
Console.WriteLine("Wait : {0}ms", duration);
}
else
{
long startCheck = DateTime.Now.Ticks;
int buffer_size = THRESHOLD;
int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
int offset = 0;
byte[] buffer = new byte[buffer_size];
fs.Read(buffer, 0, buffer_size);
CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset);
while (fs.Position < fs.Length)
{
Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap);
int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap);
offset += nbRead;
if (nbRead < buffer.Length - overlap)
{
Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap));
}
CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset);
}
}
long endCheck = DateTime.Now.Ticks;
double duration = ((double)(endCheck - startCheck)) / 10000;
Console.WriteLine("Check : {0} ms", duration);
}
}
bool atLeastOne = false;
foreach (FileVerificationEntry entry in fileVerEntry)
{
if (entry.datablock.diskfile != null)
{
atLeastOne = true;
break;
}
}
if (!atLeastOne)
{
matchType = MatchType.NoMatch;
}
}
private static void CheckBuffer(byte[] buffer, DiskFile diskFile, string filename, int blocksize, Dictionary<uint, FileVerificationEntry> hashfull, ref MatchType matchType, int globalOffset)
{
uint partial_key = (uint)(Path.GetFileName(filename).GetHashCode());
MD5 md5Hasher = MD5.Create();
//FastCRC32.FastCRC32 crc32 = new FastCRC32.FastCRC32((ulong)blocksize);
FastCRC32.FastCRC32 crc32 = FastCRC32.FastCRC32.GetCRC32Instance((ulong)blocksize);
int offset = 0;
byte inch = 0;
byte outch = 0;
uint crc32Value = 0;
crc32Value = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)blocksize, buffer, (uint)offset) ^ 0xFFFFFFFF;
while (offset < (buffer.Length - blocksize))
{
uint key = crc32Value ^ partial_key;
FileVerificationEntry entry = null;
if (hashfull.ContainsKey(key))
{
entry = hashfull[key];
byte[] blockhash = md5Hasher.ComputeHash(buffer, offset, blocksize);
if (ToolKit.ToHex(blockhash) == ToolKit.ToHex(entry.hash))
{
// We found a complete match, so go to next block !
//Console.WriteLine("block found at offset {0}, crc {1}", globalOffset + offset, entry.crc);
if (entry.datablock.diskfile == null)
{
lock (entry)
{
if (entry.datablock.diskfile == null)
{
entry.SetBlock(diskFile, (int)(globalOffset + offset));
}
}
}
offset += blocksize;
crc32Value = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)(Math.Min(blocksize, buffer.Length - offset)), buffer, (uint)offset) ^ 0xFFFFFFFF;
}
else
{
if (offset + blocksize > buffer.Length)
return;
matchType = MatchType.PartialMatch;
inch = buffer[offset + blocksize];
outch = buffer[offset];
crc32Value = crc32.windowMask ^ crc32.CRCSlideChar(crc32.windowMask ^ crc32Value, inch, outch);
++offset;
}
}
else
{
if (offset + blocksize > buffer.Length)
return;
matchType = MatchType.PartialMatch;
inch = buffer[offset + blocksize];
outch = buffer[offset];
crc32Value = crc32.windowMask ^ crc32.CRCSlideChar(crc32.windowMask ^ crc32Value, inch, outch);
++offset;
}
}
}
public static void CheckFile(DiskFile diskFile, string filename, int blocksize, List<FileVerificationEntry> fileVerEntry, byte[] md5hash16k, byte[] md5hash, ref MatchType matchType, Dictionary<uint,FileVerificationEntry> hashfull, Dictionary<uint,FileVerificationEntry> hash, List<FileVerificationEntry> expectedList, ref int expectedIndex, bool multithreadCPU)
{
//int THRESHOLD = ((int)((50 * 1024 * 1024) / blocksize) * blocksize); // 50Mo threshold
int THRESHOLD = 5 * blocksize;
matchType = MatchType.FullMatch;
Console.WriteLine("Checking file '{0}'", Path.GetFileName(filename));
long filesize = new FileInfo(filename).Length;
if (filesize <= THRESHOLD)
{
using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
CheckBuffer(br.ReadBytes((int)filesize), diskFile, filename, blocksize, hashfull, ref matchType, 0);
}
}
else
{
if (multithreadCPU)
{
List<Task> tasks = new List<Task>();
int buffer_size = THRESHOLD;
int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
int offset = 0;
byte[] buffer = new byte[buffer_size];
fs.Read(buffer, 0, buffer_size);
tasks.Add(Task.Factory.StartNew((arg)
=>
{
try
{
object[] args = (object[])arg;
byte[] b = (byte[])args[0];
DiskFile df = (DiskFile)args[1];
string f = (string)args[2];
int bs = (int)args[3];
Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4];
MatchType mt = (MatchType)args[5];
int o = (int)args[6];
CheckBuffer(b, df, f, bs, hf, ref mt, o);
}
finally
{
//concurrencySemaphore.Release();
}
}, new object[] { buffer, diskFile, filename, blocksize, hashfull, matchType, offset }));
while (fs.Position < fs.Length)
{
Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap);
int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap);
offset += nbRead;
if (nbRead < buffer.Length - overlap)
Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap));
tasks.Add(Task.Factory.StartNew((arg)
=>
{
try
{
object[] args = (object[])arg;
byte[] b = (byte[])args[0];
DiskFile df = (DiskFile)args[1];
string f = (string)args[2];
int bs = (int)args[3];
Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4];
MatchType mt = (MatchType)args[5];
int o = (int)args[6];
CheckBuffer(b, df, f, bs, hf, ref mt, o);
}
finally
{
//concurrencySemaphore.Release();
}
}, new object[] { (byte[])buffer.Clone(), diskFile, filename, blocksize, hashfull, matchType, offset }));
}
}
long startWait = DateTime.Now.Ticks;
Task.WaitAll(tasks.ToArray());
long endWait = DateTime.Now.Ticks;
double duration = ((double)(endWait - startWait)) / 10000;
Console.WriteLine("Wait : {0}ms", duration);
}
else
{
long startCheck = DateTime.Now.Ticks;
int buffer_size = THRESHOLD;
int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
{
int offset = 0;
byte[] buffer = new byte[buffer_size];
fs.Read(buffer, 0, buffer_size);
CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset);
while (fs.Position < fs.Length)
{
Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap);
int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap);
offset += nbRead;
if (nbRead < buffer.Length - overlap)
Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap));
CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset);
}
}
long endCheck = DateTime.Now.Ticks;
double duration = ((double)(endCheck - startCheck)) / 10000;
Console.WriteLine("Check : {0} ms", duration);
}
}
bool atLeastOne = false;
foreach (FileVerificationEntry entry in fileVerEntry)
{
if (entry.datablock.diskfile != null)
{
atLeastOne = true;
break;
}
}
if (!atLeastOne)
matchType = MatchType.NoMatch;
}
public static void CheckFile_async(DiskFile diskFile, string filename, int blocksize, List<FileVerificationEntry> fileVerEntry, byte[] md5hash16k, byte[] md5hash, ref MatchType matchType, Dictionary<uint,FileVerificationEntry> hashfull, Dictionary<uint,FileVerificationEntry> hash, List<FileVerificationEntry> expectedList, ref int expectedIndex, bool multithreadCPU)
{
//int THRESHOLD = ((int)((50 * 1024 * 1024) / blocksize) * blocksize); // 50Mo threshold
int THRESHOLD = 5 * blocksize;
matchType = MatchType.FullMatch;
Console.WriteLine("Checking file '{0}'", Path.GetFileName(filename));
long filesize = new FileInfo(filename).Length;
if (filesize <= THRESHOLD)
{
using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
CheckBuffer(br.ReadBytes((int)filesize), diskFile, filename, blocksize, hashfull, ref matchType, 0);
}
}
else
{
if (multithreadCPU)
{
List<Task> tasks = new List<Task>();
int buffer_size = THRESHOLD;
int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks
List<ManualResetEvent> list = new List<ManualResetEvent>();
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 1024, true))
{
int offset = 0;
byte[] buffer = new byte[buffer_size];
ManualResetEvent mre = new ManualResetEvent(false);
list.Add(mre);
CheckBufferState state = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap);
fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(TaskEndReadCallback), new FileCheckerState(fs, buffer, mre, state));
//tasks.Add(Task.Factory.StartNew((arg)
// =>
//{
// try
// {
// object[] args = (object[])arg;
// byte[] b = (byte[])args[0];
// DiskFile df = (DiskFile)args[1];
// string f = (string)args[2];
// int bs = (int)args[3];
// Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4];
// MatchType mt = (MatchType)args[5];
// int o = (int)args[6];
// CheckBuffer(b, df, f, bs, hf, ref mt, o);
// }
// finally
// {
// //concurrencySemaphore.Release();
// }
//}, new object[] { buffer, diskFile, filename, blocksize, hashfull, matchType, offset }));
while (fs.Position < fs.Length)
{
Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap);
ManualResetEvent mre2 = new ManualResetEvent(false);
list.Add(mre2);
CheckBufferState state2 = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap);
fs.BeginRead(buffer, overlap, buffer.Length - overlap, new AsyncCallback(TaskEndReadCallback), new FileCheckerState(fs, buffer, mre2, state2));
int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap);
offset += buffer.Length - overlap;
//if (nbRead < buffer.Length - overlap)
// Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap));
//tasks.Add(Task.Factory.StartNew((arg)
// =>
//{
// try
// {
// object[] args = (object[])arg;
// byte[] b = (byte[])args[0];
// DiskFile df = (DiskFile)args[1];
// string f = (string)args[2];
// int bs = (int)args[3];
// Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4];
// MatchType mt = (MatchType)args[5];
// int o = (int)args[6];
// CheckBuffer(b, df, f, bs, hf, ref mt, o);
// }
// finally
// {
// //concurrencySemaphore.Release();
// }
//}, new object[] { (byte[])buffer.Clone(), diskFile, filename, blocksize, hashfull, matchType, offset }));
}
}
//long startWait = DateTime.Now.Ticks;
//Task.WaitAll(tasks.ToArray());
//long endWait = DateTime.Now.Ticks;
//double duration = ((double)(endWait - startWait)) / 10000;
//Console.WriteLine("Wait : {0}ms", duration);
WaitHandle.WaitAll(list.ToArray());
}
else
{
long startCheck = DateTime.Now.Ticks;
int buffer_size = THRESHOLD;
int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks
List<ManualResetEvent> list = new List<ManualResetEvent>();
using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 2 * buffer_size, true))
{
int offset = 0;
byte[] buffer = new byte[buffer_size];
CheckBufferState state = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap);
ManualResetEvent mre = new ManualResetEvent(false);
list.Add(mre);
// Create a synchronization object that gets
// signaled when verification is complete.
fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(EndReadCallback), new FileCheckerState(fs, buffer, mre, state));
//CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset);
while (fs.Position < fs.Length)
{
Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap);
//int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap);
CheckBufferState state2 = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap);
// Create a synchronization object that gets
// signaled when verification is complete.
ManualResetEvent mre2 = new ManualResetEvent(false);
list.Add(mre2);
fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(EndReadCallback), new FileCheckerState(fs, buffer, mre2, state2));
offset += buffer_size;
//if (nbRead < buffer.Length - overlap)
// Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap));
//CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset);
}
WaitHandle.WaitAll(list.ToArray());
}
long endCheck = DateTime.Now.Ticks;
double duration = ((double)(endCheck - startCheck)) / 10000;
Console.WriteLine("Check : {0} ms", duration);
}
}
bool atLeastOne = false;
foreach (FileVerificationEntry entry in fileVerEntry)
{
if (entry.datablock.diskfile != null)
{
atLeastOne = true;
break;
}
}
if (!atLeastOne)
matchType = MatchType.NoMatch;
}
// Open the source file, compute the MD5 Hash of the whole file and the first
// 16k of the file, and then compute the FileId and store the results
// in a file description packet and a file verification packet.
public bool Open(string filename, ulong blocksize, bool deferhashcomputation)
{
// Get the filename and filesize
targetfilename = filename;
filesize = (ulong)new FileInfo(filename).Length;
// Work out how many blocks the file will be sliced into
blockcount = (uint)((filesize + blocksize - 1) / blocksize);
// Determine what filename to record in the PAR2 files
parfilename = Path.GetFileName(filename);
// Create the Description and Verification packets
FileDescriptionPacket = Packets.FileDescriptionPacket.Create(parfilename, filesize);
FileVerificationPacket = Packets.FileVerificationPacket.Create(blockcount);
// Create the diskfile object
targetfile = new DiskFile();
// Open the source file
if (!targetfile.Open(targetfilename, filesize))
{
return(false);
}
// Do we want to defer the computation of the full file hash, and
// the block crc and hashes. This is only permitted if there
// is sufficient memory available to create all recovery blocks
// in one pass of the source files (i.e. chunksize == blocksize)
if (deferhashcomputation)
{
// Initialise a buffer to read the first 16k of the source file
uint buffersize = Math.Min((uint)filesize, 16 * 1024);
byte[] buffer = new byte[buffersize];
// Read the data from the file
if (!targetfile.Read(0, buffer, buffersize))
{
targetfile.Close();
return(false);
}
// Compute the hash of the data read from the file
// Store the hash in the descriptionpacket and compute the file id
MD5 md5Hasher = MD5.Create();
FileDescriptionPacket.hash16k = md5Hasher.ComputeHash(buffer);
// Compute the fileid and store it in the verification packet.
FileDescriptionPacket.ComputeFileId();
FileVerificationPacket.fileid = (byte[])FileDescriptionPacket.fileid.Clone();
//// Allocate an MD5 context for computing the file hash
//// during the recovery data generation phase
contextfull = MD5.Create();
//contextfull = new MD5Context;
}
else
{
// Compute 16k MD5 hash
using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
MD5 md5Hasher = MD5.Create();
byte[] buffer16k = br.ReadBytes(16 * 1024);
FileDescriptionPacket.hash16k = md5Hasher.ComputeHash(buffer16k);
}
// Compute the fileid and store it in the verification packet.
FileDescriptionPacket.ComputeFileId();
FileVerificationPacket.fileid = (byte[])FileDescriptionPacket.fileid.Clone();
// Compute full file MD5 hash & block CRC32 and MD5 hashes
//long readSize = 5 * (long)blocksize;
long readSize = (long)blocksize;
long fileSize = new FileInfo(filename).Length;
long nbSteps = fileSize / readSize;
long remaining = fileSize % readSize;
uint blocknumber = 0;
// TODO : switch to async filestream
using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
MD5 md5FullHasher = MD5.Create();
MD5 md5Hasher = MD5.Create();
//FastCRC32.FastCRC32 crc32 = new FastCRC32.FastCRC32(blocksize);
FastCRC32.FastCRC32 crc32 = FastCRC32.FastCRC32.GetCRC32Instance(blocksize);
byte[] blockHash = new byte[16];
uint blockCRC32 = 0;
for (int i = 0; i < nbSteps + 1; ++i)
{
byte[] buffer = br.ReadBytes((int)(i == nbSteps ? remaining : readSize));
// Global MD5 hash
if (i == nbSteps)
{
md5FullHasher.TransformFinalBlock(buffer, 0, buffer.Length);
}
else
{
md5FullHasher.TransformBlock(buffer, 0, buffer.Length, null, 0);
}
//for (uint j = 0; j < 5; ++j)
//{
// Block MD5 hash & CRC32
uint length = (uint)blocksize;
//if (i == nbSteps && j == 4)
//if (i == nbSteps && (buffer.Length - (uint)(j * blocksize)) < (int)blocksize)
if (i == nbSteps && remaining != 0)
{
// We need arry padding since calculation **MUST** always be done on blocksize-length buffers
byte[] smallBuffer = buffer;
buffer = new byte[blocksize];
Buffer.BlockCopy(smallBuffer, 0, buffer, 0, smallBuffer.Length);
}
blockCRC32 = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)blocksize, buffer, 0) ^ 0xFFFFFFFF;
blockHash = md5Hasher.ComputeHash(buffer, 0, (int)blocksize);
//Console.WriteLine("blocknumber:{0},hash:{1},crc32:{2}", blocknumber, blockHash, blockCRC32);
FileVerificationPacket.SetBlockHashAndCRC(blocknumber++, blockHash, blockCRC32);
//}
}
FileDescriptionPacket.hashfull = md5FullHasher.Hash;
}
}
return(true);
}
public CriticalPacketEntry(DiskFile _diskFile, ulong _offset, CriticalPacket _packet)
{
diskfile = _diskFile;
offset = _offset;
packet = _packet;
}
public void SetCompleteFile(DiskFile diskfile)
{
completefile = diskfile;
}
private static void CheckBuffer(byte[] buffer, DiskFile diskFile, string filename, int blocksize, Dictionary <uint, FileVerificationEntry> hashfull, ref MatchType matchType, int globalOffset)
{
uint partial_key = (uint)(Path.GetFileName(filename).GetHashCode());
MD5 md5Hasher = MD5.Create();
//FastCRC32.FastCRC32 crc32 = new FastCRC32.FastCRC32((ulong)blocksize);
FastCRC32.FastCRC32 crc32 = FastCRC32.FastCRC32.GetCRC32Instance((ulong)blocksize);
int offset = 0;
byte inch = 0;
byte outch = 0;
uint crc32Value = 0;
crc32Value = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)blocksize, buffer, (uint)offset) ^ 0xFFFFFFFF;
while (offset < (buffer.Length - blocksize))
{
uint key = crc32Value ^ partial_key;
FileVerificationEntry entry = null;
if (hashfull.ContainsKey(key))
{
entry = hashfull[key];
byte[] blockhash = md5Hasher.ComputeHash(buffer, offset, blocksize);
if (ToolKit.ToHex(blockhash) == ToolKit.ToHex(entry.hash))
{
// We found a complete match, so go to next block !
//Console.WriteLine("block found at offset {0}, crc {1}", globalOffset + offset, entry.crc);
if (entry.datablock.diskfile == null)
{
lock (entry)
{
if (entry.datablock.diskfile == null)
{
entry.SetBlock(diskFile, (int)(globalOffset + offset));
}
}
}
offset += blocksize;
crc32Value = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)(Math.Min(blocksize, buffer.Length - offset)), buffer, (uint)offset) ^ 0xFFFFFFFF;
}
else
{
if (offset + blocksize > buffer.Length)
{
return;
}
matchType = MatchType.PartialMatch;
inch = buffer[offset + blocksize];
outch = buffer[offset];
crc32Value = crc32.windowMask ^ crc32.CRCSlideChar(crc32.windowMask ^ crc32Value, inch, outch);
++offset;
}
}
else
{
if (offset + blocksize > buffer.Length)
{
return;
}
matchType = MatchType.PartialMatch;
inch = buffer[offset + blocksize];
outch = buffer[offset];
crc32Value = crc32.windowMask ^ crc32.CRCSlideChar(crc32.windowMask ^ crc32Value, inch, outch);
++offset;
}
}
}
// Open the source file, compute the MD5 Hash of the whole file and the first
// 16k of the file, and then compute the FileId and store the results
// in a file description packet and a file verification packet.
public bool Open(string filename, ulong blocksize, bool deferhashcomputation)
{
// Get the filename and filesize
targetfilename = filename;
filesize = (ulong)new FileInfo(filename).Length;
// Work out how many blocks the file will be sliced into
blockcount = (uint)((filesize + blocksize - 1) / blocksize);
// Determine what filename to record in the PAR2 files
parfilename = Path.GetFileName(filename);
// Create the Description and Verification packets
FileDescriptionPacket = Packets.FileDescriptionPacket.Create(parfilename, filesize);
FileVerificationPacket = Packets.FileVerificationPacket.Create(blockcount);
// Create the diskfile object
targetfile = new DiskFile();
// Open the source file
if (!targetfile.Open(targetfilename, filesize))
return false;
// Do we want to defer the computation of the full file hash, and
// the block crc and hashes. This is only permitted if there
// is sufficient memory available to create all recovery blocks
// in one pass of the source files (i.e. chunksize == blocksize)
if (deferhashcomputation)
{
// Initialise a buffer to read the first 16k of the source file
uint buffersize = Math.Min((uint)filesize, 16 * 1024);
byte[] buffer = new byte[buffersize];
// Read the data from the file
if (!targetfile.Read(0, buffer, buffersize))
{
targetfile.Close();
return false;
}
// Compute the hash of the data read from the file
// Store the hash in the descriptionpacket and compute the file id
MD5 md5Hasher = MD5.Create();
FileDescriptionPacket.hash16k = md5Hasher.ComputeHash(buffer);
// Compute the fileid and store it in the verification packet.
FileDescriptionPacket.ComputeFileId();
FileVerificationPacket.fileid = (byte[])FileDescriptionPacket.fileid.Clone();
//// Allocate an MD5 context for computing the file hash
//// during the recovery data generation phase
contextfull = MD5.Create();
//contextfull = new MD5Context;
}
else
{
// Compute 16k MD5 hash
using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
MD5 md5Hasher = MD5.Create();
byte[] buffer16k = br.ReadBytes(16 * 1024);
FileDescriptionPacket.hash16k = md5Hasher.ComputeHash(buffer16k);
}
// Compute the fileid and store it in the verification packet.
FileDescriptionPacket.ComputeFileId();
FileVerificationPacket.fileid = (byte[])FileDescriptionPacket.fileid.Clone();
// Compute full file MD5 hash & block CRC32 and MD5 hashes
//long readSize = 5 * (long)blocksize;
long readSize = (long)blocksize;
long fileSize = new FileInfo(filename).Length;
long nbSteps = fileSize / readSize;
long remaining = fileSize % readSize;
uint blocknumber = 0;
// TODO : switch to async filestream
using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)))
{
MD5 md5FullHasher = MD5.Create();
MD5 md5Hasher = MD5.Create();
//FastCRC32.FastCRC32 crc32 = new FastCRC32.FastCRC32(blocksize);
FastCRC32.FastCRC32 crc32 = FastCRC32.FastCRC32.GetCRC32Instance(blocksize);
byte[] blockHash = new byte[16];
uint blockCRC32 = 0;
for (int i = 0; i < nbSteps + 1; ++i)
{
byte[] buffer = br.ReadBytes((int)(i == nbSteps ? remaining : readSize));
// Global MD5 hash
if (i == nbSteps)
md5FullHasher.TransformFinalBlock(buffer, 0, buffer.Length);
else
md5FullHasher.TransformBlock(buffer, 0, buffer.Length, null, 0);
//for (uint j = 0; j < 5; ++j)
//{
// Block MD5 hash & CRC32
uint length = (uint)blocksize;
//if (i == nbSteps && j == 4)
//if (i == nbSteps && (buffer.Length - (uint)(j * blocksize)) < (int)blocksize)
if (i == nbSteps && remaining != 0)
{
// We need arry padding since calculation **MUST** always be done on blocksize-length buffers
byte[] smallBuffer = buffer;
buffer = new byte[blocksize];
Buffer.BlockCopy(smallBuffer, 0, buffer, 0, smallBuffer.Length);
}
blockCRC32 = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)blocksize, buffer, 0) ^ 0xFFFFFFFF;
blockHash = md5Hasher.ComputeHash(buffer, 0, (int)blocksize);
//Console.WriteLine("blocknumber:{0},hash:{1},crc32:{2}", blocknumber, blockHash, blockCRC32);
FileVerificationPacket.SetBlockHashAndCRC(blocknumber++, blockHash, blockCRC32);
//}
}
FileDescriptionPacket.hashfull = md5FullHasher.Hash;
}
}
return true;
}
public void SetTargetFile(DiskFile diskfile)
{
targetfile = diskfile;
}
