public CheckBufferState(byte[] buffer, DiskFile diskFile, string fileName, int blockSize, Dictionary<uint, FileVerificationEntry> hashFull, MatchType matchType, int offset, int overlap) { this.buffer = buffer; this.diskFile = diskFile; this.fileName = fileName; this.blockSize = blockSize; this.hashFull = hashFull; this.matchType = matchType; this.offset = offset; this.overlap = overlap; }
public CriticalPacketEntry(CriticalPacketEntry other) { diskfile = other.diskfile; offset = other.offset; packet = other.packet; }
public CriticalPacketEntry() { diskfile = null; offset = 0; packet = null; }
public CriticalPacketEntry(DiskFile _diskFile, ulong _offset, CriticalPacket _packet) { diskfile = _diskFile; offset = _offset; packet = _packet; }
public void SetCompleteFile(DiskFile diskfile) { completefile = diskfile; }
public void SetTargetFile(DiskFile diskfile) { targetfile = diskfile; }
public static void CheckFile_async(DiskFile diskFile, string filename, int blocksize, List <FileVerificationEntry> fileVerEntry, byte[] md5hash16k, byte[] md5hash, ref MatchType matchType, Dictionary <uint, FileVerificationEntry> hashfull, Dictionary <uint, FileVerificationEntry> hash, List <FileVerificationEntry> expectedList, ref int expectedIndex, bool multithreadCPU) { //int THRESHOLD = ((int)((50 * 1024 * 1024) / blocksize) * blocksize); // 50Mo threshold int THRESHOLD = 5 * blocksize; matchType = MatchType.FullMatch; Console.WriteLine("Checking file '{0}'", Path.GetFileName(filename)); long filesize = new FileInfo(filename).Length; if (filesize <= THRESHOLD) { using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) { CheckBuffer(br.ReadBytes((int)filesize), diskFile, filename, blocksize, hashfull, ref matchType, 0); } } else { if (multithreadCPU) { List <Task> tasks = new List <Task>(); int buffer_size = THRESHOLD; int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks List <ManualResetEvent> list = new List <ManualResetEvent>(); using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 1024, true)) { int offset = 0; byte[] buffer = new byte[buffer_size]; ManualResetEvent mre = new ManualResetEvent(false); list.Add(mre); CheckBufferState state = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap); fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(TaskEndReadCallback), new FileCheckerState(fs, buffer, mre, state)); //tasks.Add(Task.Factory.StartNew((arg) // => //{ // try // { // object[] args = (object[])arg; // byte[] b = (byte[])args[0]; // DiskFile df = (DiskFile)args[1]; // string f = (string)args[2]; // int bs = (int)args[3]; // Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4]; // MatchType mt = (MatchType)args[5]; // int o = (int)args[6]; // CheckBuffer(b, df, f, bs, hf, ref mt, o); // } // finally // { // //concurrencySemaphore.Release(); // } //}, new object[] { buffer, diskFile, filename, blocksize, hashfull, matchType, offset })); while (fs.Position < fs.Length) { Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap); ManualResetEvent mre2 = new ManualResetEvent(false); list.Add(mre2); CheckBufferState state2 = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap); fs.BeginRead(buffer, overlap, buffer.Length - overlap, new AsyncCallback(TaskEndReadCallback), new FileCheckerState(fs, buffer, mre2, state2)); int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap); offset += buffer.Length - overlap; //if (nbRead < buffer.Length - overlap) // Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap)); //tasks.Add(Task.Factory.StartNew((arg) // => //{ // try // { // object[] args = (object[])arg; // byte[] b = (byte[])args[0]; // DiskFile df = (DiskFile)args[1]; // string f = (string)args[2]; // int bs = (int)args[3]; // Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4]; // MatchType mt = (MatchType)args[5]; // int o = (int)args[6]; // CheckBuffer(b, df, f, bs, hf, ref mt, o); // } // finally // { // //concurrencySemaphore.Release(); // } //}, new object[] { (byte[])buffer.Clone(), diskFile, filename, blocksize, hashfull, matchType, offset })); } } //long startWait = DateTime.Now.Ticks; //Task.WaitAll(tasks.ToArray()); //long endWait = DateTime.Now.Ticks; //double duration = ((double)(endWait - startWait)) / 10000; //Console.WriteLine("Wait : {0}ms", duration); WaitHandle.WaitAll(list.ToArray()); } else { long startCheck = DateTime.Now.Ticks; int buffer_size = THRESHOLD; int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks List <ManualResetEvent> list = new List <ManualResetEvent>(); using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 2 * buffer_size, true)) { int offset = 0; byte[] buffer = new byte[buffer_size]; CheckBufferState state = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap); ManualResetEvent mre = new ManualResetEvent(false); list.Add(mre); // Create a synchronization object that gets // signaled when verification is complete. fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(EndReadCallback), new FileCheckerState(fs, buffer, mre, state)); //CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset); while (fs.Position < fs.Length) { Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap); //int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap); CheckBufferState state2 = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap); // Create a synchronization object that gets // signaled when verification is complete. ManualResetEvent mre2 = new ManualResetEvent(false); list.Add(mre2); fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(EndReadCallback), new FileCheckerState(fs, buffer, mre2, state2)); offset += buffer_size; //if (nbRead < buffer.Length - overlap) // Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap)); //CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset); } WaitHandle.WaitAll(list.ToArray()); } long endCheck = DateTime.Now.Ticks; double duration = ((double)(endCheck - startCheck)) / 10000; Console.WriteLine("Check : {0} ms", duration); } } bool atLeastOne = false; foreach (FileVerificationEntry entry in fileVerEntry) { if (entry.datablock.diskfile != null) { atLeastOne = true; break; } } if (!atLeastOne) { matchType = MatchType.NoMatch; } }
public static void CheckFile(DiskFile diskFile, string filename, int blocksize, List <FileVerificationEntry> fileVerEntry, byte[] md5hash16k, byte[] md5hash, ref MatchType matchType, Dictionary <uint, FileVerificationEntry> hashfull, Dictionary <uint, FileVerificationEntry> hash, List <FileVerificationEntry> expectedList, ref int expectedIndex, bool multithreadCPU) { //int THRESHOLD = ((int)((50 * 1024 * 1024) / blocksize) * blocksize); // 50Mo threshold int THRESHOLD = 5 * blocksize; matchType = MatchType.FullMatch; Console.WriteLine("Checking file '{0}'", Path.GetFileName(filename)); long filesize = new FileInfo(filename).Length; if (filesize <= THRESHOLD) { using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) { CheckBuffer(br.ReadBytes((int)filesize), diskFile, filename, blocksize, hashfull, ref matchType, 0); } } else { if (multithreadCPU) { List <Task> tasks = new List <Task>(); int buffer_size = THRESHOLD; int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { int offset = 0; byte[] buffer = new byte[buffer_size]; fs.Read(buffer, 0, buffer_size); tasks.Add(Task.Factory.StartNew((arg) => { try { object[] args = (object[])arg; byte[] b = (byte[])args[0]; DiskFile df = (DiskFile)args[1]; string f = (string)args[2]; int bs = (int)args[3]; Dictionary <uint, FileVerificationEntry> hf = (Dictionary <uint, FileVerificationEntry>)args[4]; MatchType mt = (MatchType)args[5]; int o = (int)args[6]; CheckBuffer(b, df, f, bs, hf, ref mt, o); } finally { //concurrencySemaphore.Release(); } }, new object[] { buffer, diskFile, filename, blocksize, hashfull, matchType, offset })); while (fs.Position < fs.Length) { Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap); int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap); offset += nbRead; if (nbRead < buffer.Length - overlap) { Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap)); } tasks.Add(Task.Factory.StartNew((arg) => { try { object[] args = (object[])arg; byte[] b = (byte[])args[0]; DiskFile df = (DiskFile)args[1]; string f = (string)args[2]; int bs = (int)args[3]; Dictionary <uint, FileVerificationEntry> hf = (Dictionary <uint, FileVerificationEntry>)args[4]; MatchType mt = (MatchType)args[5]; int o = (int)args[6]; CheckBuffer(b, df, f, bs, hf, ref mt, o); } finally { //concurrencySemaphore.Release(); } }, new object[] { (byte[])buffer.Clone(), diskFile, filename, blocksize, hashfull, matchType, offset })); } } long startWait = DateTime.Now.Ticks; Task.WaitAll(tasks.ToArray()); long endWait = DateTime.Now.Ticks; double duration = ((double)(endWait - startWait)) / 10000; Console.WriteLine("Wait : {0}ms", duration); } else { long startCheck = DateTime.Now.Ticks; int buffer_size = THRESHOLD; int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { int offset = 0; byte[] buffer = new byte[buffer_size]; fs.Read(buffer, 0, buffer_size); CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset); while (fs.Position < fs.Length) { Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap); int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap); offset += nbRead; if (nbRead < buffer.Length - overlap) { Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap)); } CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset); } } long endCheck = DateTime.Now.Ticks; double duration = ((double)(endCheck - startCheck)) / 10000; Console.WriteLine("Check : {0} ms", duration); } } bool atLeastOne = false; foreach (FileVerificationEntry entry in fileVerEntry) { if (entry.datablock.diskfile != null) { atLeastOne = true; break; } } if (!atLeastOne) { matchType = MatchType.NoMatch; } }
private static void CheckBuffer(byte[] buffer, DiskFile diskFile, string filename, int blocksize, Dictionary<uint, FileVerificationEntry> hashfull, ref MatchType matchType, int globalOffset) { uint partial_key = (uint)(Path.GetFileName(filename).GetHashCode()); MD5 md5Hasher = MD5.Create(); //FastCRC32.FastCRC32 crc32 = new FastCRC32.FastCRC32((ulong)blocksize); FastCRC32.FastCRC32 crc32 = FastCRC32.FastCRC32.GetCRC32Instance((ulong)blocksize); int offset = 0; byte inch = 0; byte outch = 0; uint crc32Value = 0; crc32Value = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)blocksize, buffer, (uint)offset) ^ 0xFFFFFFFF; while (offset < (buffer.Length - blocksize)) { uint key = crc32Value ^ partial_key; FileVerificationEntry entry = null; if (hashfull.ContainsKey(key)) { entry = hashfull[key]; byte[] blockhash = md5Hasher.ComputeHash(buffer, offset, blocksize); if (ToolKit.ToHex(blockhash) == ToolKit.ToHex(entry.hash)) { // We found a complete match, so go to next block ! //Console.WriteLine("block found at offset {0}, crc {1}", globalOffset + offset, entry.crc); if (entry.datablock.diskfile == null) { lock (entry) { if (entry.datablock.diskfile == null) { entry.SetBlock(diskFile, (int)(globalOffset + offset)); } } } offset += blocksize; crc32Value = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)(Math.Min(blocksize, buffer.Length - offset)), buffer, (uint)offset) ^ 0xFFFFFFFF; } else { if (offset + blocksize > buffer.Length) return; matchType = MatchType.PartialMatch; inch = buffer[offset + blocksize]; outch = buffer[offset]; crc32Value = crc32.windowMask ^ crc32.CRCSlideChar(crc32.windowMask ^ crc32Value, inch, outch); ++offset; } } else { if (offset + blocksize > buffer.Length) return; matchType = MatchType.PartialMatch; inch = buffer[offset + blocksize]; outch = buffer[offset]; crc32Value = crc32.windowMask ^ crc32.CRCSlideChar(crc32.windowMask ^ crc32Value, inch, outch); ++offset; } } }
public static void CheckFile(DiskFile diskFile, string filename, int blocksize, List<FileVerificationEntry> fileVerEntry, byte[] md5hash16k, byte[] md5hash, ref MatchType matchType, Dictionary<uint,FileVerificationEntry> hashfull, Dictionary<uint,FileVerificationEntry> hash, List<FileVerificationEntry> expectedList, ref int expectedIndex, bool multithreadCPU) { //int THRESHOLD = ((int)((50 * 1024 * 1024) / blocksize) * blocksize); // 50Mo threshold int THRESHOLD = 5 * blocksize; matchType = MatchType.FullMatch; Console.WriteLine("Checking file '{0}'", Path.GetFileName(filename)); long filesize = new FileInfo(filename).Length; if (filesize <= THRESHOLD) { using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) { CheckBuffer(br.ReadBytes((int)filesize), diskFile, filename, blocksize, hashfull, ref matchType, 0); } } else { if (multithreadCPU) { List<Task> tasks = new List<Task>(); int buffer_size = THRESHOLD; int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { int offset = 0; byte[] buffer = new byte[buffer_size]; fs.Read(buffer, 0, buffer_size); tasks.Add(Task.Factory.StartNew((arg) => { try { object[] args = (object[])arg; byte[] b = (byte[])args[0]; DiskFile df = (DiskFile)args[1]; string f = (string)args[2]; int bs = (int)args[3]; Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4]; MatchType mt = (MatchType)args[5]; int o = (int)args[6]; CheckBuffer(b, df, f, bs, hf, ref mt, o); } finally { //concurrencySemaphore.Release(); } }, new object[] { buffer, diskFile, filename, blocksize, hashfull, matchType, offset })); while (fs.Position < fs.Length) { Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap); int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap); offset += nbRead; if (nbRead < buffer.Length - overlap) Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap)); tasks.Add(Task.Factory.StartNew((arg) => { try { object[] args = (object[])arg; byte[] b = (byte[])args[0]; DiskFile df = (DiskFile)args[1]; string f = (string)args[2]; int bs = (int)args[3]; Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4]; MatchType mt = (MatchType)args[5]; int o = (int)args[6]; CheckBuffer(b, df, f, bs, hf, ref mt, o); } finally { //concurrencySemaphore.Release(); } }, new object[] { (byte[])buffer.Clone(), diskFile, filename, blocksize, hashfull, matchType, offset })); } } long startWait = DateTime.Now.Ticks; Task.WaitAll(tasks.ToArray()); long endWait = DateTime.Now.Ticks; double duration = ((double)(endWait - startWait)) / 10000; Console.WriteLine("Wait : {0}ms", duration); } else { long startCheck = DateTime.Now.Ticks; int buffer_size = THRESHOLD; int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { int offset = 0; byte[] buffer = new byte[buffer_size]; fs.Read(buffer, 0, buffer_size); CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset); while (fs.Position < fs.Length) { Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap); int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap); offset += nbRead; if (nbRead < buffer.Length - overlap) Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap)); CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset); } } long endCheck = DateTime.Now.Ticks; double duration = ((double)(endCheck - startCheck)) / 10000; Console.WriteLine("Check : {0} ms", duration); } } bool atLeastOne = false; foreach (FileVerificationEntry entry in fileVerEntry) { if (entry.datablock.diskfile != null) { atLeastOne = true; break; } } if (!atLeastOne) matchType = MatchType.NoMatch; }
public static void CheckFile_async(DiskFile diskFile, string filename, int blocksize, List<FileVerificationEntry> fileVerEntry, byte[] md5hash16k, byte[] md5hash, ref MatchType matchType, Dictionary<uint,FileVerificationEntry> hashfull, Dictionary<uint,FileVerificationEntry> hash, List<FileVerificationEntry> expectedList, ref int expectedIndex, bool multithreadCPU) { //int THRESHOLD = ((int)((50 * 1024 * 1024) / blocksize) * blocksize); // 50Mo threshold int THRESHOLD = 5 * blocksize; matchType = MatchType.FullMatch; Console.WriteLine("Checking file '{0}'", Path.GetFileName(filename)); long filesize = new FileInfo(filename).Length; if (filesize <= THRESHOLD) { using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) { CheckBuffer(br.ReadBytes((int)filesize), diskFile, filename, blocksize, hashfull, ref matchType, 0); } } else { if (multithreadCPU) { List<Task> tasks = new List<Task>(); int buffer_size = THRESHOLD; int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks List<ManualResetEvent> list = new List<ManualResetEvent>(); using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 1024, true)) { int offset = 0; byte[] buffer = new byte[buffer_size]; ManualResetEvent mre = new ManualResetEvent(false); list.Add(mre); CheckBufferState state = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap); fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(TaskEndReadCallback), new FileCheckerState(fs, buffer, mre, state)); //tasks.Add(Task.Factory.StartNew((arg) // => //{ // try // { // object[] args = (object[])arg; // byte[] b = (byte[])args[0]; // DiskFile df = (DiskFile)args[1]; // string f = (string)args[2]; // int bs = (int)args[3]; // Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4]; // MatchType mt = (MatchType)args[5]; // int o = (int)args[6]; // CheckBuffer(b, df, f, bs, hf, ref mt, o); // } // finally // { // //concurrencySemaphore.Release(); // } //}, new object[] { buffer, diskFile, filename, blocksize, hashfull, matchType, offset })); while (fs.Position < fs.Length) { Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap); ManualResetEvent mre2 = new ManualResetEvent(false); list.Add(mre2); CheckBufferState state2 = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap); fs.BeginRead(buffer, overlap, buffer.Length - overlap, new AsyncCallback(TaskEndReadCallback), new FileCheckerState(fs, buffer, mre2, state2)); int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap); offset += buffer.Length - overlap; //if (nbRead < buffer.Length - overlap) // Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap)); //tasks.Add(Task.Factory.StartNew((arg) // => //{ // try // { // object[] args = (object[])arg; // byte[] b = (byte[])args[0]; // DiskFile df = (DiskFile)args[1]; // string f = (string)args[2]; // int bs = (int)args[3]; // Dictionary<uint, FileVerificationEntry> hf = (Dictionary<uint, FileVerificationEntry>)args[4]; // MatchType mt = (MatchType)args[5]; // int o = (int)args[6]; // CheckBuffer(b, df, f, bs, hf, ref mt, o); // } // finally // { // //concurrencySemaphore.Release(); // } //}, new object[] { (byte[])buffer.Clone(), diskFile, filename, blocksize, hashfull, matchType, offset })); } } //long startWait = DateTime.Now.Ticks; //Task.WaitAll(tasks.ToArray()); //long endWait = DateTime.Now.Ticks; //double duration = ((double)(endWait - startWait)) / 10000; //Console.WriteLine("Wait : {0}ms", duration); WaitHandle.WaitAll(list.ToArray()); } else { long startCheck = DateTime.Now.Ticks; int buffer_size = THRESHOLD; int overlap = 1 * blocksize; // part which will be check in double to avoid missing moving blocks List<ManualResetEvent> list = new List<ManualResetEvent>(); using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 2 * buffer_size, true)) { int offset = 0; byte[] buffer = new byte[buffer_size]; CheckBufferState state = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap); ManualResetEvent mre = new ManualResetEvent(false); list.Add(mre); // Create a synchronization object that gets // signaled when verification is complete. fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(EndReadCallback), new FileCheckerState(fs, buffer, mre, state)); //CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset); while (fs.Position < fs.Length) { Buffer.BlockCopy(buffer, buffer.Length - overlap, buffer, 0, overlap); //int nbRead = fs.Read(buffer, overlap, buffer.Length - overlap); CheckBufferState state2 = new CheckBufferState(buffer, diskFile, filename, blocksize, hashfull, matchType, offset, overlap); // Create a synchronization object that gets // signaled when verification is complete. ManualResetEvent mre2 = new ManualResetEvent(false); list.Add(mre2); fs.BeginRead(buffer, 0, buffer_size, new AsyncCallback(EndReadCallback), new FileCheckerState(fs, buffer, mre2, state2)); offset += buffer_size; //if (nbRead < buffer.Length - overlap) // Array.Clear(buffer, (nbRead + overlap), buffer_size - (nbRead + overlap)); //CheckBuffer(buffer, diskFile, filename, blocksize, hashfull, ref matchType, offset); } WaitHandle.WaitAll(list.ToArray()); } long endCheck = DateTime.Now.Ticks; double duration = ((double)(endCheck - startCheck)) / 10000; Console.WriteLine("Check : {0} ms", duration); } } bool atLeastOne = false; foreach (FileVerificationEntry entry in fileVerEntry) { if (entry.datablock.diskfile != null) { atLeastOne = true; break; } } if (!atLeastOne) matchType = MatchType.NoMatch; }
// Open the source file, compute the MD5 Hash of the whole file and the first // 16k of the file, and then compute the FileId and store the results // in a file description packet and a file verification packet. public bool Open(string filename, ulong blocksize, bool deferhashcomputation) { // Get the filename and filesize targetfilename = filename; filesize = (ulong)new FileInfo(filename).Length; // Work out how many blocks the file will be sliced into blockcount = (uint)((filesize + blocksize - 1) / blocksize); // Determine what filename to record in the PAR2 files parfilename = Path.GetFileName(filename); // Create the Description and Verification packets FileDescriptionPacket = Packets.FileDescriptionPacket.Create(parfilename, filesize); FileVerificationPacket = Packets.FileVerificationPacket.Create(blockcount); // Create the diskfile object targetfile = new DiskFile(); // Open the source file if (!targetfile.Open(targetfilename, filesize)) { return(false); } // Do we want to defer the computation of the full file hash, and // the block crc and hashes. This is only permitted if there // is sufficient memory available to create all recovery blocks // in one pass of the source files (i.e. chunksize == blocksize) if (deferhashcomputation) { // Initialise a buffer to read the first 16k of the source file uint buffersize = Math.Min((uint)filesize, 16 * 1024); byte[] buffer = new byte[buffersize]; // Read the data from the file if (!targetfile.Read(0, buffer, buffersize)) { targetfile.Close(); return(false); } // Compute the hash of the data read from the file // Store the hash in the descriptionpacket and compute the file id MD5 md5Hasher = MD5.Create(); FileDescriptionPacket.hash16k = md5Hasher.ComputeHash(buffer); // Compute the fileid and store it in the verification packet. FileDescriptionPacket.ComputeFileId(); FileVerificationPacket.fileid = (byte[])FileDescriptionPacket.fileid.Clone(); //// Allocate an MD5 context for computing the file hash //// during the recovery data generation phase contextfull = MD5.Create(); //contextfull = new MD5Context; } else { // Compute 16k MD5 hash using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) { MD5 md5Hasher = MD5.Create(); byte[] buffer16k = br.ReadBytes(16 * 1024); FileDescriptionPacket.hash16k = md5Hasher.ComputeHash(buffer16k); } // Compute the fileid and store it in the verification packet. FileDescriptionPacket.ComputeFileId(); FileVerificationPacket.fileid = (byte[])FileDescriptionPacket.fileid.Clone(); // Compute full file MD5 hash & block CRC32 and MD5 hashes //long readSize = 5 * (long)blocksize; long readSize = (long)blocksize; long fileSize = new FileInfo(filename).Length; long nbSteps = fileSize / readSize; long remaining = fileSize % readSize; uint blocknumber = 0; // TODO : switch to async filestream using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) { MD5 md5FullHasher = MD5.Create(); MD5 md5Hasher = MD5.Create(); //FastCRC32.FastCRC32 crc32 = new FastCRC32.FastCRC32(blocksize); FastCRC32.FastCRC32 crc32 = FastCRC32.FastCRC32.GetCRC32Instance(blocksize); byte[] blockHash = new byte[16]; uint blockCRC32 = 0; for (int i = 0; i < nbSteps + 1; ++i) { byte[] buffer = br.ReadBytes((int)(i == nbSteps ? remaining : readSize)); // Global MD5 hash if (i == nbSteps) { md5FullHasher.TransformFinalBlock(buffer, 0, buffer.Length); } else { md5FullHasher.TransformBlock(buffer, 0, buffer.Length, null, 0); } //for (uint j = 0; j < 5; ++j) //{ // Block MD5 hash & CRC32 uint length = (uint)blocksize; //if (i == nbSteps && j == 4) //if (i == nbSteps && (buffer.Length - (uint)(j * blocksize)) < (int)blocksize) if (i == nbSteps && remaining != 0) { // We need arry padding since calculation **MUST** always be done on blocksize-length buffers byte[] smallBuffer = buffer; buffer = new byte[blocksize]; Buffer.BlockCopy(smallBuffer, 0, buffer, 0, smallBuffer.Length); } blockCRC32 = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)blocksize, buffer, 0) ^ 0xFFFFFFFF; blockHash = md5Hasher.ComputeHash(buffer, 0, (int)blocksize); //Console.WriteLine("blocknumber:{0},hash:{1},crc32:{2}", blocknumber, blockHash, blockCRC32); FileVerificationPacket.SetBlockHashAndCRC(blocknumber++, blockHash, blockCRC32); //} } FileDescriptionPacket.hashfull = md5FullHasher.Hash; } } return(true); }
private static void CheckBuffer(byte[] buffer, DiskFile diskFile, string filename, int blocksize, Dictionary <uint, FileVerificationEntry> hashfull, ref MatchType matchType, int globalOffset) { uint partial_key = (uint)(Path.GetFileName(filename).GetHashCode()); MD5 md5Hasher = MD5.Create(); //FastCRC32.FastCRC32 crc32 = new FastCRC32.FastCRC32((ulong)blocksize); FastCRC32.FastCRC32 crc32 = FastCRC32.FastCRC32.GetCRC32Instance((ulong)blocksize); int offset = 0; byte inch = 0; byte outch = 0; uint crc32Value = 0; crc32Value = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)blocksize, buffer, (uint)offset) ^ 0xFFFFFFFF; while (offset < (buffer.Length - blocksize)) { uint key = crc32Value ^ partial_key; FileVerificationEntry entry = null; if (hashfull.ContainsKey(key)) { entry = hashfull[key]; byte[] blockhash = md5Hasher.ComputeHash(buffer, offset, blocksize); if (ToolKit.ToHex(blockhash) == ToolKit.ToHex(entry.hash)) { // We found a complete match, so go to next block ! //Console.WriteLine("block found at offset {0}, crc {1}", globalOffset + offset, entry.crc); if (entry.datablock.diskfile == null) { lock (entry) { if (entry.datablock.diskfile == null) { entry.SetBlock(diskFile, (int)(globalOffset + offset)); } } } offset += blocksize; crc32Value = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)(Math.Min(blocksize, buffer.Length - offset)), buffer, (uint)offset) ^ 0xFFFFFFFF; } else { if (offset + blocksize > buffer.Length) { return; } matchType = MatchType.PartialMatch; inch = buffer[offset + blocksize]; outch = buffer[offset]; crc32Value = crc32.windowMask ^ crc32.CRCSlideChar(crc32.windowMask ^ crc32Value, inch, outch); ++offset; } } else { if (offset + blocksize > buffer.Length) { return; } matchType = MatchType.PartialMatch; inch = buffer[offset + blocksize]; outch = buffer[offset]; crc32Value = crc32.windowMask ^ crc32.CRCSlideChar(crc32.windowMask ^ crc32Value, inch, outch); ++offset; } } }
// Open the source file, compute the MD5 Hash of the whole file and the first // 16k of the file, and then compute the FileId and store the results // in a file description packet and a file verification packet. public bool Open(string filename, ulong blocksize, bool deferhashcomputation) { // Get the filename and filesize targetfilename = filename; filesize = (ulong)new FileInfo(filename).Length; // Work out how many blocks the file will be sliced into blockcount = (uint)((filesize + blocksize - 1) / blocksize); // Determine what filename to record in the PAR2 files parfilename = Path.GetFileName(filename); // Create the Description and Verification packets FileDescriptionPacket = Packets.FileDescriptionPacket.Create(parfilename, filesize); FileVerificationPacket = Packets.FileVerificationPacket.Create(blockcount); // Create the diskfile object targetfile = new DiskFile(); // Open the source file if (!targetfile.Open(targetfilename, filesize)) return false; // Do we want to defer the computation of the full file hash, and // the block crc and hashes. This is only permitted if there // is sufficient memory available to create all recovery blocks // in one pass of the source files (i.e. chunksize == blocksize) if (deferhashcomputation) { // Initialise a buffer to read the first 16k of the source file uint buffersize = Math.Min((uint)filesize, 16 * 1024); byte[] buffer = new byte[buffersize]; // Read the data from the file if (!targetfile.Read(0, buffer, buffersize)) { targetfile.Close(); return false; } // Compute the hash of the data read from the file // Store the hash in the descriptionpacket and compute the file id MD5 md5Hasher = MD5.Create(); FileDescriptionPacket.hash16k = md5Hasher.ComputeHash(buffer); // Compute the fileid and store it in the verification packet. FileDescriptionPacket.ComputeFileId(); FileVerificationPacket.fileid = (byte[])FileDescriptionPacket.fileid.Clone(); //// Allocate an MD5 context for computing the file hash //// during the recovery data generation phase contextfull = MD5.Create(); //contextfull = new MD5Context; } else { // Compute 16k MD5 hash using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) { MD5 md5Hasher = MD5.Create(); byte[] buffer16k = br.ReadBytes(16 * 1024); FileDescriptionPacket.hash16k = md5Hasher.ComputeHash(buffer16k); } // Compute the fileid and store it in the verification packet. FileDescriptionPacket.ComputeFileId(); FileVerificationPacket.fileid = (byte[])FileDescriptionPacket.fileid.Clone(); // Compute full file MD5 hash & block CRC32 and MD5 hashes //long readSize = 5 * (long)blocksize; long readSize = (long)blocksize; long fileSize = new FileInfo(filename).Length; long nbSteps = fileSize / readSize; long remaining = fileSize % readSize; uint blocknumber = 0; // TODO : switch to async filestream using (BinaryReader br = new BinaryReader(new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read))) { MD5 md5FullHasher = MD5.Create(); MD5 md5Hasher = MD5.Create(); //FastCRC32.FastCRC32 crc32 = new FastCRC32.FastCRC32(blocksize); FastCRC32.FastCRC32 crc32 = FastCRC32.FastCRC32.GetCRC32Instance(blocksize); byte[] blockHash = new byte[16]; uint blockCRC32 = 0; for (int i = 0; i < nbSteps + 1; ++i) { byte[] buffer = br.ReadBytes((int)(i == nbSteps ? remaining : readSize)); // Global MD5 hash if (i == nbSteps) md5FullHasher.TransformFinalBlock(buffer, 0, buffer.Length); else md5FullHasher.TransformBlock(buffer, 0, buffer.Length, null, 0); //for (uint j = 0; j < 5; ++j) //{ // Block MD5 hash & CRC32 uint length = (uint)blocksize; //if (i == nbSteps && j == 4) //if (i == nbSteps && (buffer.Length - (uint)(j * blocksize)) < (int)blocksize) if (i == nbSteps && remaining != 0) { // We need arry padding since calculation **MUST** always be done on blocksize-length buffers byte[] smallBuffer = buffer; buffer = new byte[blocksize]; Buffer.BlockCopy(smallBuffer, 0, buffer, 0, smallBuffer.Length); } blockCRC32 = crc32.CRCUpdateBlock(0xFFFFFFFF, (uint)blocksize, buffer, 0) ^ 0xFFFFFFFF; blockHash = md5Hasher.ComputeHash(buffer, 0, (int)blocksize); //Console.WriteLine("blocknumber:{0},hash:{1},crc32:{2}", blocknumber, blockHash, blockCRC32); FileVerificationPacket.SetBlockHashAndCRC(blocknumber++, blockHash, blockCRC32); //} } FileDescriptionPacket.hashfull = md5FullHasher.Hash; } } return true; }