private int CheckSumRead(Stream inStream, FileResults fileResults, ulong totalSize, bool testcrc, bool testDeep)
{
if (_buffer0 == null)
{
_buffer0 = new byte[Buffersize];
_buffer1 = new byte[Buffersize];
}
fileResults.MD5 = null;
fileResults.SHA1 = null;
fileResults.CRC = null;
ThreadReadBuffer lbuffer = null;
ThreadCRC tcrc32 = null;
ThreadMD5 tmd5 = null;
ThreadSHA1 tsha1 = null;
ThreadCRC altCrc32 = null;
ThreadMD5 altMd5 = null;
ThreadSHA1 altSha1 = null;
try
{
int maxHeaderSize = 128;
long sizetogo = (long)totalSize;
int sizenow = maxHeaderSize < sizetogo ? maxHeaderSize : (int)sizetogo;
if (sizenow > 0)
{
inStream.Read(_buffer0, 0, sizenow);
}
fileResults.HeaderFileType = FileHeaderReader.GetType(_buffer0, sizenow, out int actualHeaderSize);
// if the file has no header then just use the main hash checkers.
if (fileResults.HeaderFileType == HeaderFileType.Nothing || actualHeaderSize == 0)
{
// no header found & not reading hashes.
if (!(testcrc || testDeep))
{
return(0);
}
// no header found so just push the initial buffer read into the hash checkers.
// and then continue with the rest of the file.
tcrc32 = new ThreadCRC();
if (testDeep)
{
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
}
tcrc32.Trigger(_buffer0, sizenow);
tmd5?.Trigger(_buffer0, sizenow);
tsha1?.Trigger(_buffer0, sizenow);
tcrc32.Wait();
tmd5?.Wait();
tsha1?.Wait();
sizetogo -= sizenow;
}
else
{
// header found
fileResults.AltSize = (ulong)((long)totalSize - actualHeaderSize);
//setup main hash checkers
if (testcrc || testDeep)
{
tcrc32 = new ThreadCRC();
}
altCrc32 = new ThreadCRC();
if (testDeep)
{
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
altMd5 = new ThreadMD5();
altSha1 = new ThreadSHA1();
}
if (sizenow > actualHeaderSize)
{
// Already read more than the header, so we need to split what we read into the 2 hash checkers
// first scan the header part from what we have already read.
// scan what we read so far with just the main hashers
tcrc32?.Trigger(_buffer0, actualHeaderSize);
tmd5?.Trigger(_buffer0, actualHeaderSize);
tsha1?.Trigger(_buffer0, actualHeaderSize);
tcrc32?.Wait();
tmd5?.Wait();
tsha1?.Wait();
// put the rest of what we read into the second buffer, and scan with all hashers
int restSize = sizenow - actualHeaderSize;
for (int i = 0; i < restSize; i++)
{
_buffer1[i] = _buffer0[actualHeaderSize + i];
}
tcrc32?.Trigger(_buffer1, restSize);
tmd5?.Trigger(_buffer1, restSize);
tsha1?.Trigger(_buffer1, restSize);
altCrc32.Trigger(_buffer1, restSize);
altMd5?.Trigger(_buffer1, restSize);
altSha1?.Trigger(_buffer1, restSize);
tcrc32?.Wait();
tmd5?.Wait();
tsha1?.Wait();
altCrc32.Wait();
altMd5?.Wait();
altSha1?.Wait();
sizetogo -= sizenow;
}
else
{
// Read less than the length of the header so read the rest of the header.
// then continue to reader the full rest of the file.
// scan what we read so far
tcrc32?.Trigger(_buffer0, sizenow);
tmd5?.Trigger(_buffer0, sizenow);
tsha1?.Trigger(_buffer0, sizenow);
tcrc32?.Wait();
tmd5?.Wait();
tsha1?.Wait();
sizetogo -= sizenow;
// now read the rest of the header.
sizenow = actualHeaderSize - sizenow;
inStream.Read(_buffer0, 0, sizenow);
// scan the rest of the header
tcrc32?.Trigger(_buffer0, sizenow);
tmd5?.Trigger(_buffer0, sizenow);
tsha1?.Trigger(_buffer0, sizenow);
tcrc32?.Wait();
tmd5?.Wait();
tsha1?.Wait();
sizetogo -= sizenow;
}
}
lbuffer = new ThreadReadBuffer(inStream);
// Pre load the first buffer0
int sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
if (sizeNext > 0)
{
inStream.Read(_buffer0, 0, sizeNext);
}
int sizebuffer = sizeNext;
sizetogo -= sizeNext;
bool whichBuffer = true;
while (sizebuffer > 0 && !lbuffer.errorState)
{
sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
if (sizeNext > 0)
{
lbuffer.Trigger(whichBuffer ? _buffer1 : _buffer0, sizeNext);
}
byte[] buffer = whichBuffer ? _buffer0 : _buffer1;
tcrc32?.Trigger(buffer, sizebuffer);
tmd5?.Trigger(buffer, sizebuffer);
tsha1?.Trigger(buffer, sizebuffer);
altCrc32?.Trigger(buffer, sizebuffer);
altMd5?.Trigger(buffer, sizebuffer);
altSha1?.Trigger(buffer, sizebuffer);
if (sizeNext > 0)
{
lbuffer.Wait();
}
tcrc32?.Wait();
tmd5?.Wait();
tsha1?.Wait();
altCrc32?.Wait();
altMd5?.Wait();
altSha1?.Wait();
sizebuffer = sizeNext;
sizetogo -= sizeNext;
whichBuffer = !whichBuffer;
}
lbuffer.Finish();
tcrc32?.Finish();
tmd5?.Finish();
tsha1?.Finish();
altCrc32?.Finish();
altMd5?.Finish();
altSha1?.Finish();
}
catch
{
lbuffer?.Dispose();
tcrc32?.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
altCrc32?.Dispose();
altMd5?.Dispose();
altSha1?.Dispose();
return(0x17); // need to remember what this number is for
}
if (lbuffer.errorState)
{
lbuffer.Dispose();
tcrc32?.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
altCrc32?.Dispose();
altMd5?.Dispose();
altSha1?.Dispose();
return(0x17); // need to remember what this number is for
}
fileResults.CRC = tcrc32?.Hash;
fileResults.SHA1 = tsha1?.Hash;
fileResults.MD5 = tmd5?.Hash;
fileResults.AltCRC = altCrc32?.Hash;
fileResults.AltSHA1 = altSha1?.Hash;
fileResults.AltMD5 = altMd5?.Hash;
lbuffer.Dispose();
tcrc32?.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
altCrc32?.Dispose();
altMd5?.Dispose();
altSha1?.Dispose();
return(0);
}
public static ReturnCode DecompressSource7ZipFile(RvFile zZipFileIn, bool includeGood, out string error)
{
byte[] buffer = new byte[BufferSize];
RvFile cacheDir = DB.RvFileCache();
string fileNameIn = zZipFileIn.FullName;
SevenZ zipFileIn = new SevenZ();
ZipReturn zr1 = zipFileIn.ZipFileOpen(fileNameIn, zZipFileIn.TimeStamp, true);
if (zr1 != ZipReturn.ZipGood)
{
error = "Error opening 7zip file for caching";
return(ReturnCode.RescanNeeded);
}
RvFile outDir = new RvFile(FileType.Dir)
{
Name = zZipFileIn.Name + ".cache",
Parent = cacheDir,
DatStatus = DatStatus.InToSort,
GotStatus = GotStatus.Got
};
int nameDirIndex = 0;
while (cacheDir.ChildNameSearch(outDir, out int index) == 0)
{
nameDirIndex++;
outDir.Name = zZipFileIn.Name + ".cache (" + nameDirIndex + ")";
}
cacheDir.ChildAdd(outDir);
Directory.CreateDirectory(outDir.FullName);
for (int i = 0; i < zipFileIn.LocalFilesCount(); i++)
{
if (zZipFileIn.Child(i).IsDir)
{
continue;
}
RvFile thisFile = null;
for (int j = 0; j < zZipFileIn.ChildCount; j++)
{
if (zZipFileIn.Child(j).ZipFileIndex != i)
{
continue;
}
thisFile = zZipFileIn.Child(j);
break;
}
if (thisFile == null)
{
error = "Error opening 7zip file for caching";
return(ReturnCode.RescanNeeded);
}
bool extract = true;
// first check to see if we have a file version of this compressed file somewhere else.
foreach (RvFile f in thisFile.FileGroup.Files)
{
if (f.FileType == FileType.File && f.GotStatus == GotStatus.Got)
{
extract = false;
}
}
if (!extract)
{
continue;
}
extract = false;
if (includeGood)
{
// if this is the file we are fixing then pull out the correct files.
if (thisFile.RepStatus == RepStatus.Correct)
{
extract = true;
}
}
// next check to see if we need this extracted to fix another file
foreach (RvFile f in thisFile.FileGroup.Files)
{
if (f.RepStatus == RepStatus.CanBeFixed)
{
extract = true;
break;
}
}
if (!extract)
{
continue;
}
string cleanedName = thisFile.Name;
cleanedName = cleanedName.Replace("/", "-");
cleanedName = cleanedName.Replace("\\", "-");
RvFile outFile = new RvFile(FileType.File)
{
Name = cleanedName,
Size = thisFile.Size,
CRC = thisFile.CRC,
SHA1 = thisFile.SHA1,
MD5 = thisFile.MD5,
HeaderFileType = thisFile.HeaderFileType,
AltSize = thisFile.AltSize,
AltCRC = thisFile.AltCRC,
AltSHA1 = thisFile.AltSHA1,
AltMD5 = thisFile.AltMD5,
FileGroup = thisFile.FileGroup
};
outFile.SetStatus(DatStatus.InToSort, GotStatus.Got);
outFile.FileStatusSet(
FileStatus.HeaderFileTypeFromHeader |
FileStatus.SizeFromHeader | FileStatus.SizeVerified |
FileStatus.CRCFromHeader | FileStatus.CRCVerified |
FileStatus.SHA1FromHeader | FileStatus.SHA1Verified |
FileStatus.MD5FromHeader | FileStatus.MD5Verified |
FileStatus.AltSizeFromHeader | FileStatus.AltSizeVerified |
FileStatus.AltCRCFromHeader | FileStatus.AltCRCVerified |
FileStatus.AltSHA1FromHeader | FileStatus.AltSHA1Verified |
FileStatus.AltMD5FromHeader | FileStatus.AltMD5Verified
, thisFile);
outFile.RepStatus = RepStatus.NeededForFix;
zipFileIn.ZipFileOpenReadStream(i, out Stream readStream, out ulong unCompressedSize);
string filenameOut = Path.Combine(outDir.FullName, outFile.Name);
ThreadMD5 tmd5 = null;
ThreadSHA1 tsha1 = null;
ThreadCRC tcrc32 = new ThreadCRC();
if (Settings.rvSettings.FixLevel != EFixLevel.Level1 && Settings.rvSettings.FixLevel != EFixLevel.TrrntZipLevel1)
{
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
}
int errorCode = FileStream.OpenFileWrite(filenameOut, out Stream writeStream);
ulong sizetogo = unCompressedSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > BufferSize ? BufferSize : (int)sizetogo;
try
{
readStream.Read(buffer, 0, sizenow);
}
catch (Exception ex)
{
if (ex is ZlibException || ex is DataErrorException)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.ZipFilename;
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
writeStream.Flush();
writeStream.Close();
if (filenameOut != null)
{
File.Delete(filenameOut);
}
thisFile.GotStatus = GotStatus.Corrupt;
error = "Unexpected corrupt archive file found:\n" + zZipFileIn.FullName +
"\nRun Find Fixes, and Fix to continue fixing correctly.";
return(ReturnCode.SourceDataStreamCorrupt);
}
error = "Error reading Source File " + ex.Message;
return(ReturnCode.FileSystemError);
}
tcrc32.Trigger(buffer, sizenow);
tmd5?.Trigger(buffer, sizenow);
tsha1?.Trigger(buffer, sizenow);
tcrc32.Wait();
tmd5?.Wait();
tsha1?.Wait();
try
{
writeStream.Write(buffer, 0, sizenow);
}
catch (Exception e)
{
error = "Error writing out file. " + Environment.NewLine + e.Message;
return(ReturnCode.FileSystemError);
}
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
writeStream.Close();
writeStream.Dispose();
tcrc32.Finish();
tmd5?.Finish();
tsha1?.Finish();
byte[] bCRC = tcrc32.Hash;
byte[] bMD5 = tmd5?.Hash;
byte[] bSHA1 = tsha1?.Hash;
tcrc32.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
FileInfo fi = new FileInfo(filenameOut);
outFile.TimeStamp = fi.LastWriteTime;
if (bCRC != null && thisFile.CRC != null && !ArrByte.BCompare(bCRC, thisFile.CRC))
{
// error in file.
}
if (bMD5 != null && thisFile.MD5 != null && !ArrByte.BCompare(bMD5, thisFile.MD5))
{
// error in file.
}
if (bSHA1 != null && thisFile.SHA1 != null && !ArrByte.BCompare(bSHA1, thisFile.SHA1))
{
// error in file.
}
thisFile.FileGroup.Files.Add(outFile);
outDir.ChildAdd(outFile);
}
zipFileIn.ZipFileClose();
error = "";
return(ReturnCode.Good);
}
private ZipReturn ReadBody(bool deepScan)
{
using (BinaryReader zipBr = new BinaryReader(_zipFs, Encoding.UTF8, true))
{
byte ID1 = zipBr.ReadByte();
byte ID2 = zipBr.ReadByte();
if ((ID1 != 0x1f) || (ID2 != 0x8b))
{
_zipFs.Close();
return(ZipReturn.ZipSignatureError);
}
byte CM = zipBr.ReadByte();
if (CM != 8)
{
_zipFs.Close();
return(ZipReturn.ZipUnsupportedCompression);
}
byte FLG = zipBr.ReadByte();
uint MTime = zipBr.ReadUInt32();
byte XFL = zipBr.ReadByte();
byte OS = zipBr.ReadByte();
//if FLG.FEXTRA set
if ((FLG & 0x4) == 0x4)
{
int XLen = zipBr.ReadInt16();
byte[] bytes = zipBr.ReadBytes(XLen);
if (XLen == 28)
{
md5Hash = new byte[16];
Array.Copy(bytes, 0, md5Hash, 0, 16);
crc = new byte[4];
Array.Copy(bytes, 16, crc, 0, 4);
uncompressedSize = BitConverter.ToUInt64(bytes, 20);
}
if (XLen == 77)
{
md5Hash = new byte[16];
Array.Copy(bytes, 0, md5Hash, 0, 16);
crc = new byte[4];
Array.Copy(bytes, 16, crc, 0, 4);
uncompressedSize = BitConverter.ToUInt64(bytes, 20);
altType = (HeaderFileType)bytes[28];
altmd5Hash = new byte[16];
Array.Copy(bytes, 29, altmd5Hash, 0, 16);
altsha1Hash = new byte[20];
Array.Copy(bytes, 45, altsha1Hash, 0, 20);
altcrc = new byte[4];
Array.Copy(bytes, 65, altcrc, 0, 4);
uncompressedAltSize = BitConverter.ToUInt64(bytes, 69);
}
}
//if FLG.FNAME set
if ((FLG & 0x8) == 0x8)
{
int XLen = zipBr.ReadInt16();
byte[] bytes = zipBr.ReadBytes(XLen);
}
//if FLG.FComment set
if ((FLG & 0x10) == 0x10)
{
int XLen = zipBr.ReadInt16();
byte[] bytes = zipBr.ReadBytes(XLen);
}
//if FLG.FHCRC set
if ((FLG & 0x2) == 0x2)
{
uint crc16 = zipBr.ReadUInt16();
}
}
compressedSize = (ulong)(_zipFs.Length - _zipFs.Position) - 8;
datapos = _zipFs.Position;
if (deepScan)
{
if (Buffer0 == null)
{
Buffer0 = new byte[Buffersize];
}
if (Buffer1 == null)
{
Buffer1 = new byte[Buffersize];
}
Stream ds = new ZlibBaseStream(_zipFs, CompressionMode.Decompress, CompressionLevel.Default, ZlibStreamFlavor.DEFLATE, true);
ThreadLoadBuffer lbuffer = new ThreadLoadBuffer(ds);
ThreadCRC crc32 = new ThreadCRC();
ThreadMD5 md5 = new ThreadMD5();
ThreadSHA1 sha1 = new ThreadSHA1();
ulong uncompressedRead = 0;
int bufferRead = ds.Read(Buffer0, 0, Buffersize);
uncompressedRead += (ulong)bufferRead;
bool whichBuffer = true;
while (bufferRead > 0)
{
// trigger the buffer loading worker
lbuffer.Trigger(whichBuffer ? Buffer1 : Buffer0, Buffersize);
byte[] buffer = whichBuffer ? Buffer0 : Buffer1;
// trigger the hashing workers
crc32.Trigger(buffer, bufferRead);
md5.Trigger(buffer, bufferRead);
sha1.Trigger(buffer, bufferRead);
lbuffer.Wait();
crc32.Wait();
md5.Wait();
sha1.Wait();
// setup next loop around
bufferRead = lbuffer.SizeRead;
uncompressedRead += (ulong)bufferRead;
whichBuffer = !whichBuffer;
}
// tell all the workers we are finished
lbuffer.Finish();
crc32.Finish();
md5.Finish();
sha1.Finish();
// get the results
byte[] testcrc = crc32.Hash;
byte[] testmd5 = md5.Hash;
byte[] testsha1 = sha1.Hash;
// cleanup
lbuffer.Dispose();
crc32.Dispose();
md5.Dispose();
sha1.Dispose();
ds.Close();
ds.Dispose();
if (uncompressedSize != 0)
{
if (uncompressedSize != uncompressedRead)
{
_zipFs.Close();
return(ZipReturn.ZipDecodeError);
}
}
else
{
uncompressedSize = uncompressedRead;
}
if (crc != null)
{
for (int i = 0; i < 4; i++)
{
if (crc[i] == testcrc[i])
{
continue;
}
_zipFs.Close();
return(ZipReturn.ZipDecodeError);
}
}
else
{
crc = testcrc;
}
if (md5Hash != null)
{
for (int i = 0; i < 16; i++)
{
if (md5Hash[i] == testmd5[i])
{
continue;
}
_zipFs.Close();
return(ZipReturn.ZipDecodeError);
}
}
else
{
md5Hash = testmd5;
}
if (sha1Hash != null)
{
for (int i = 0; i < 20; i++)
{
if (sha1Hash[i] == testsha1[i])
{
continue;
}
_zipFs.Close();
return(ZipReturn.ZipDecodeError);
}
}
else
{
sha1Hash = testsha1;
}
}
_zipFs.Position = _zipFs.Length - 8;
byte[] gzcrc;
uint gzLength;
using (BinaryReader zipBr = new BinaryReader(_zipFs, Encoding.UTF8, true))
{
gzcrc = zipBr.ReadBytes(4);
gzLength = zipBr.ReadUInt32();
}
if (crc != null)
{
for (int i = 0; i < 4; i++)
{
if (gzcrc[3 - i] == crc[i])
{
continue;
}
_zipFs.Close();
return(ZipReturn.ZipDecodeError);
}
}
else
{
crc = new[] { gzcrc[3], gzcrc[2], gzcrc[1], gzcrc[0] };
}
if (uncompressedSize != 0)
{
if (gzLength != (uncompressedSize & 0xffffffff))
{
_zipFs.Close();
return(ZipReturn.ZipDecodeError);
}
}
return(ZipReturn.ZipGood);
}
// This Function returns:
// Good : Everything Worked Correctly
// RescanNeeded : Something unexpectedly changed in the files, so Stop fixing and prompt user to rescan.
// LogicError : This Should never happen and is a logic problem in the code.
// FileSystemError : Something is wrong with the files, like it was locked and could not be opened.
// SourceDataStreamCorrupt : This happens when either zlib returns ZlibException, or the CRC does not match the extracted zip.
// SourceCheckSumMismatch : If the extracted files does not match its expected SHA1 or MD5
// DestinationCheckSumMismatch : If the extracted files does not match the file to be fixed expected CRC,SHA1 or MD5.
/// <summary>
/// Performs the RomVault File Copy, with the source and destination being files or zipped files
/// </summary>
/// <param name="fileIn">This is the file being copied, it may be a zipped file or a regular file system file</param>
/// <param name="zipFileOut">This is the zip file that is being written to.</param>
/// <param name="filenameOut">This is the name of the file to be written to if we are just making a file</param>
/// <param name="fileOut">This is the actual output filename</param>
/// <param name="forceRaw">if true then we will do a raw copy, this is so that we can copy corrupt zips</param>
/// <param name="error">This is the returned error message if this copy fails</param>
/// <returns>ReturnCode.Good is the valid return code otherwise we have an error</returns>
public static ReturnCode CopyFile(RvFile fileIn, ICompress zipFileOut, string filenameOut, RvFile fileOut, bool forceRaw, out string error)
{
if (zipFileOut == null && filenameOut == null)
{
throw new Exception("Error in CopyFile: Both Outputs are null");
}
if (zipFileOut != null && filenameOut != null)
{
throw new Exception("Error in CopyFile: Both Outputs are set");
}
ICompress zipFileIn = null;
Stream readStream = null;
ThreadCRC tcrc32 = null;
ThreadMD5 tmd5 = null;
ThreadSHA1 tsha1 = null;
ReturnCode retC;
error = "";
if (_buffer == null)
{
_buffer = new byte[BufferSize];
}
bool rawCopy = TestRawCopy(fileIn, fileOut, forceRaw);
ulong streamSize = 0;
ushort compressionMethod = 8;
bool sourceTrrntzip;
bool isZeroLengthFile = DBHelper.IsZeroLengthFile(fileOut);
if (!isZeroLengthFile)
{
//check that the in and out file match
retC = CheckInputAndOutputFile(fileIn, fileOut, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
//Find and Check/Open Input Files
retC = OpenInputStream(fileIn, rawCopy, out zipFileIn, out sourceTrrntzip, out readStream, out streamSize, out compressionMethod, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
}
else
{
sourceTrrntzip = true;
}
if (!rawCopy)
{
compressionMethod = 8;
}
//Find and Check/Open Output Files
retC = OpenOutputStream(fileOut, fileIn, zipFileOut, filenameOut, compressionMethod, rawCopy, sourceTrrntzip, null, out Stream writeStream, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
byte[] bCRC;
byte[] bMD5 = null;
byte[] bSHA1 = null;
if (!isZeroLengthFile)
{
#region Do Data Tranfer
if (!rawCopy)
{
tcrc32 = new ThreadCRC();
if (Settings.rvSettings.FixLevel != EFixLevel.Level1)
{
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
}
}
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > BufferSize ? BufferSize : (int)sizetogo;
try
{
readStream.Read(_buffer, 0, sizenow);
}
catch (Exception ex)
{
if (ex is ZlibException || ex is DataErrorException)
{
if ((fileIn.FileType == FileType.ZipFile || fileIn.FileType == FileType.SevenZipFile) && zipFileIn != null)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.ZipFilename;
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
}
writeStream.Flush();
writeStream.Close();
if (filenameOut != null)
{
File.Delete(filenameOut);
}
error = "Unexpected corrupt archive file found:\n" + fileIn.FullName +
"\nRun Find Fixes, and Fix to continue fixing correctly.";
return(ReturnCode.SourceDataStreamCorrupt);
}
error = "Error reading Source File " + ex.Message;
return(ReturnCode.FileSystemError);
}
tcrc32?.Trigger(_buffer, sizenow);
tmd5?.Trigger(_buffer, sizenow);
tsha1?.Trigger(_buffer, sizenow);
try
{
writeStream.Write(_buffer, 0, sizenow);
}
catch (Exception e)
{
error = "Error writing out file. " + Environment.NewLine + e.Message;
return(ReturnCode.FileSystemError);
}
tcrc32?.Wait();
tmd5?.Wait();
tsha1?.Wait();
sizetogo = sizetogo - (ulong)sizenow;
if (Report.CancellationPending())
{
tcrc32?.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
if ((fileIn.FileType == FileType.ZipFile || fileIn.FileType == FileType.SevenZipFile) && zipFileIn != null)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.ZipFilename;
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
}
if (fileOut.FileType == FileType.ZipFile || fileOut.FileType == FileType.SevenZipFile)
{
zipFileOut.ZipFileCloseFailed();
}
else
{
writeStream.Flush();
writeStream.Close();
}
return(ReturnCode.Cancel);
}
}
writeStream.Flush();
#endregion
#region Collect Checksums
// if we did a full copy then we just calculated all the checksums while doing the copy
if (!rawCopy)
{
tcrc32.Finish();
tmd5?.Finish();
tsha1?.Finish();
bCRC = tcrc32.Hash;
bMD5 = tmd5?.Hash;
bSHA1 = tsha1?.Hash;
tcrc32.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
}
// if we raw copied and the source file has been FileChecked then we can trust the checksums in the source file
else
{
bCRC = fileIn.CRC.Copy();
if (fileIn.FileStatusIs(FileStatus.MD5Verified))
{
bMD5 = fileIn.MD5.Copy();
}
if (fileIn.FileStatusIs(FileStatus.SHA1Verified))
{
bSHA1 = fileIn.SHA1.Copy();
}
}
#endregion
#region close the ReadStream
if ((fileIn.FileType == FileType.ZipFile || fileIn.FileType == FileType.SevenZipFile) && zipFileIn != null)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.ZipFilename;
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
}
#endregion
}
else
{
CopyZeroLengthFile(fileOut, zipFileOut, out bCRC, out bMD5, out bSHA1);
}
#region close the WriteStream
if (fileOut.FileType == FileType.ZipFile || fileOut.FileType == FileType.SevenZipFile)
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(bCRC);
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
fileOut.ZipFileIndex = zipFileOut.LocalFilesCount() - 1;
fileOut.ZipFileHeaderPosition = zipFileOut.GetLocalFile(fileOut.ZipFileIndex).LocalHead;
fileOut.FileModTimeStamp = 629870671200000000;
}
else
{
writeStream.Flush();
writeStream.Close();
FileInfo fi = new FileInfo(filenameOut);
fileOut.FileModTimeStamp = fi.LastWriteTime;
}
#endregion
if (!isZeroLengthFile)
{
if (!rawCopy)
{
retC = ValidateFileIn(fileIn, bCRC, bSHA1, bMD5, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
}
}
retC = ValidateFileOut(fileIn, fileOut, rawCopy, bCRC, bSHA1, bMD5, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
return(ReturnCode.Good);
}
public static int CheckSumRead(string filename, bool testDeep, out byte[] crc, out byte[] bMD5, out byte[] bSHA1)
{
bMD5 = null;
bSHA1 = null;
crc = null;
Stream ds = null;
ThreadLoadBuffer lbuffer = null;
ThreadCRC tcrc32 = null;
ThreadMD5 tmd5 = null;
ThreadSHA1 tsha1 = null;
try
{
int errorCode = FileStream.OpenFileRead(filename, out ds);
if (errorCode != 0)
{
return(errorCode);
}
lbuffer = new ThreadLoadBuffer(ds);
tcrc32 = new ThreadCRC();
if (testDeep)
{
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
}
long sizetogo = ds.Length;
// Pre load the first buffer0
int sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
ds.Read(Buffer0, 0, sizeNext);
int sizebuffer = sizeNext;
sizetogo -= sizeNext;
bool whichBuffer = true;
while ((sizebuffer > 0) && !lbuffer.errorState)
{
sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
if (sizeNext > 0)
{
lbuffer.Trigger(whichBuffer ? Buffer1 : Buffer0, sizeNext);
}
byte[] buffer = whichBuffer ? Buffer0 : Buffer1;
tcrc32.Trigger(buffer, sizebuffer);
tmd5?.Trigger(buffer, sizebuffer);
tsha1?.Trigger(buffer, sizebuffer);
if (sizeNext > 0)
{
lbuffer.Wait();
}
tcrc32.Wait();
tmd5?.Wait();
tsha1?.Wait();
sizebuffer = sizeNext;
sizetogo -= sizeNext;
whichBuffer = !whichBuffer;
}
lbuffer.Finish();
tcrc32.Finish();
tmd5?.Finish();
tsha1?.Finish();
ds.Close();
}
catch
{
ds?.Close();
lbuffer?.Dispose();
tcrc32?.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
return(0x17); // need to remember what this number is for
}
if (lbuffer.errorState)
{
ds?.Close();
lbuffer?.Dispose();
tcrc32?.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
return(0x17); // need to remember what this number is for
}
crc = tcrc32.Hash;
if (testDeep)
{
bMD5 = tmd5.Hash;
bSHA1 = tsha1.Hash;
}
lbuffer.Dispose();
tcrc32.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
return(0);
}
// This Function returns:
// Good : Everything Worked Correctly
// RescanNeeded : Something unexpectidly changed in the files, so Stop prompt user to rescan.
// LogicError : This Should never happen and is a logic problem in the code
// FileSystemError : Something is wrong with the files
/// <summary>
/// Performs the RomVault File Copy, with the source and destination being files or zipped files
/// </summary>
/// <param name="fileIn">This is the file being copied, it may be a zipped file or a regular file system file</param>
/// <param name="zipFileOut">This is the zip file that is being writen to, if it is null a new zip file will be made if we are coping to a zip</param>
/// <param name="zipFilenameOut">This is the name of the .zip file to be made that will be created in zipFileOut</param>
/// <param name="fileOut">This is the actual output filename</param>
/// <param name="forceRaw">if true then we will do a raw copy, this is so that we can copy corrupt zips</param>
/// <param name="error">This is the returned error message if this copy fails</param>
/// <param name="foundFile">If we are SHA1/MD5 checking the source file for the first time, and it is different from what we expected the correct values for this file are returned in foundFile</param>
/// <returns>ReturnCode.Good is the valid return code otherwire we have an error</returns>
public static ReturnCode CopyFile(RvFile fileIn, ref ZipFile zipFileOut, string zipFilenameOut, RvFile fileOut, bool forceRaw, out string error, out RvFile foundFile)
{
foundFile = null;
error = "";
if (_buffer == null)
{
_buffer = new byte[BufferSize];
}
bool rawCopy = RawCopy(fileIn, fileOut, forceRaw);
ulong streamSize = 0;
ushort compressionMethod = 8;
bool sourceTrrntzip = false;
ZipFile zipFileIn = null;
System.IO.Stream readStream = null;
bool isZeroLengthFile = DBHelper.IsZeroLengthFile(fileOut);
if (!isZeroLengthFile)
{
#region check that the in and out file match
if (fileOut.FileStatusIs(FileStatus.SizeFromDAT) && fileOut.Size != null && fileIn.Size != fileOut.Size)
{
error = "Source and destination Size does not match. Logic Error.";
return(ReturnCode.LogicError);
}
if (fileOut.FileStatusIs(FileStatus.CRCFromDAT) && fileOut.CRC != null && !ArrByte.bCompare(fileIn.CRC, fileOut.CRC))
{
error = "Source and destination CRC does not match. Logic Error.";
return(ReturnCode.LogicError);
}
if (fileOut.FileStatusIs(FileStatus.SHA1FromDAT) && fileIn.FileStatusIs(FileStatus.SHA1Verified))
{
if (fileIn.SHA1 != null && fileOut.SHA1 != null && !ArrByte.bCompare(fileIn.SHA1, fileOut.SHA1))
{
error = "Source and destination SHA1 does not match. Logic Error.";
return(ReturnCode.LogicError);
}
}
if (fileOut.FileStatusIs(FileStatus.MD5CHDFromDAT) && fileIn.FileStatusIs(FileStatus.MD5Verified))
{
if (fileIn.MD5 != null && fileOut.MD5 != null && !ArrByte.bCompare(fileIn.MD5, fileOut.MD5))
{
error = "Source and destination SHA1 does not match. Logic Error.";
return(ReturnCode.LogicError);
}
}
#endregion
#region Find and Check/Open Input Files
if (fileIn.FileType == FileType.ZipFile) // Input is a ZipFile
{
RvDir zZipFileIn = fileIn.Parent;
if (zZipFileIn.FileType != FileType.Zip)
{
error = "Zip File Open but Source File is not a zip, Logic Error.";
return(ReturnCode.LogicError);
}
string fileNameIn = zZipFileIn.FullName;
sourceTrrntzip = (zZipFileIn.ZipStatus & ZipStatus.TrrntZip) == ZipStatus.TrrntZip;
//if (zZipFileIn.ZipFileType == RvZip.ZipType.Zip)
//{
zipFileIn = new ZipFile();
ZipReturn zr1;
if (fileIn.ZipFileHeaderPosition != null)
{
zr1 = zipFileIn.ZipFileOpen(fileNameIn, zZipFileIn.TimeStamp, false);
}
else
{
zr1 = zipFileIn.ZipFileOpen(fileNameIn, zZipFileIn.TimeStamp, true);
}
switch (zr1)
{
case ZipReturn.ZipGood:
break;
case ZipReturn.ZipErrorFileNotFound:
error = "File not found, Rescan required before fixing " + fileIn.Name;
return(ReturnCode.FileSystemError);
case ZipReturn.ZipErrorTimeStamp:
error = "File has changed, Rescan required before fixing " + fileIn.Name;
return(ReturnCode.FileSystemError);
default:
error = "Error Open Zip" + zr1 + ", with File " + fileIn.DatFullName;
return(ReturnCode.FileSystemError);
}
if (fileIn.ZipFileHeaderPosition != null)
{
zipFileIn.ZipFileOpenReadStreamQuick((ulong)fileIn.ZipFileHeaderPosition, rawCopy, out readStream, out streamSize, out compressionMethod);
}
else
{
zipFileIn.ZipFileOpenReadStream(fileIn.ZipFileIndex, rawCopy, out readStream, out streamSize, out compressionMethod);
}
}
else // Input is a regular file
{
string fileNameIn = fileIn.FullName;
if (!IO.File.Exists(fileNameIn))
{
error = "Rescan needed, File Changed :" + fileNameIn;
return(ReturnCode.RescanNeeded);
}
IO.FileInfo fileInInfo = new IO.FileInfo(fileNameIn);
if (fileInInfo.LastWriteTime != fileIn.TimeStamp)
{
error = "Rescan needed, File Changed :" + fileNameIn;
return(ReturnCode.RescanNeeded);
}
int errorCode = IO.FileStream.OpenFileRead(fileNameIn, out readStream);
if (errorCode != 0)
{
error = new Win32Exception(errorCode).Message + ". " + fileNameIn;
return(ReturnCode.FileSystemError);
}
if (fileIn.Size == null)
{
error = "Null File Size found in Fixing File :" + fileNameIn;
return(ReturnCode.LogicError);
}
streamSize = (ulong)fileIn.Size;
if ((ulong)readStream.Length != streamSize)
{
error = "Rescan needed, File Length Changed :" + fileNameIn;
return(ReturnCode.RescanNeeded);
}
}
#endregion
}
else
{
sourceTrrntzip = true;
}
if (!rawCopy && (Program.rvSettings.FixLevel == eFixLevel.TrrntZipLevel1 || Program.rvSettings.FixLevel == eFixLevel.TrrntZipLevel2 || Program.rvSettings.FixLevel == eFixLevel.TrrntZipLevel3))
{
compressionMethod = 8;
}
#region Find and Check/Open Output Files
System.IO.Stream writeStream;
if (fileOut.FileType == FileType.ZipFile)
{
// if ZipFileOut == null then we have not open the output zip yet so open it from writing.
if (zipFileOut == null)
{
if (IO.Path.GetFileName(zipFilenameOut) == "__RomVault.tmp")
{
if (IO.File.Exists(zipFilenameOut))
{
IO.File.Delete(zipFilenameOut);
}
}
else if (IO.File.Exists(zipFilenameOut))
{
error = "Rescan needed, File Changed :" + zipFilenameOut;
return(ReturnCode.RescanNeeded);
}
zipFileOut = new ZipFile();
ZipReturn zrf = zipFileOut.ZipFileCreate(zipFilenameOut);
if (zrf != ZipReturn.ZipGood)
{
error = "Error Opening Write Stream " + zrf;
return(ReturnCode.FileSystemError);
}
}
else
{
if (zipFileOut.ZipOpen != ZipOpenType.OpenWrite)
{
error = "Output Zip File is not set to OpenWrite, Logic Error.";
return(ReturnCode.LogicError);
}
if (zipFileOut.ZipFilename != (new IO.FileInfo(zipFilenameOut).FullName))
{
error = "Output Zip file has changed name from " + zipFileOut.ZipFilename + " to " + zipFilenameOut + ". Logic Error";
return(ReturnCode.LogicError);
}
}
if (fileIn.Size == null)
{
error = "Null File Size found in Fixing File :" + fileIn.FullName;
return(ReturnCode.LogicError);
}
ZipReturn zr = zipFileOut.ZipFileOpenWriteStream(rawCopy, sourceTrrntzip, fileOut.Name, (ulong)fileIn.Size, compressionMethod, out writeStream);
if (zr != ZipReturn.ZipGood)
{
error = "Error Opening Write Stream " + zr;
return(ReturnCode.FileSystemError);
}
}
else
{
if (IO.File.Exists(zipFilenameOut) && fileOut.GotStatus != GotStatus.Corrupt)
{
error = "Rescan needed, File Changed :" + zipFilenameOut;
return(ReturnCode.RescanNeeded);
}
int errorCode = IO.FileStream.OpenFileWrite(zipFilenameOut, out writeStream);
if (errorCode != 0)
{
error = new Win32Exception(errorCode).Message + ". " + zipFilenameOut;
return(ReturnCode.FileSystemError);
}
}
#endregion
byte[] bCRC;
byte[] bMD5 = null;
byte[] bSHA1 = null;
if (!isZeroLengthFile)
{
#region Do Data Tranfer
ThreadCRC tcrc32 = null;
ThreadMD5 tmd5 = null;
ThreadSHA1 tsha1 = null;
if (!rawCopy)
{
tcrc32 = new ThreadCRC();
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
}
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > BufferSize ? BufferSize : (int)sizetogo;
try
{
readStream.Read(_buffer, 0, sizenow);
}
catch (ZlibException)
{
if (fileIn.FileType == FileType.ZipFile && zipFileIn != null)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.Filename(fileIn.ReportIndex);
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
}
if (fileOut.FileType == FileType.ZipFile)
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(new byte[] { 0, 0, 0, 0 });
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
zipFileOut.ZipFileRollBack();
}
else
{
writeStream.Flush();
writeStream.Close();
IO.File.Delete(zipFilenameOut);
}
error = "Error in Data Stream";
return(ReturnCode.SourceCRCCheckSumError);
}
catch (Exception e)
{
error = "Error reading Source File " + e.Message;
return(ReturnCode.FileSystemError);
}
if (!rawCopy)
{
tcrc32.Trigger(_buffer, sizenow);
tmd5.Trigger(_buffer, sizenow);
tsha1.Trigger(_buffer, sizenow);
tcrc32.Wait();
tmd5.Wait();
tsha1.Wait();
}
try
{
writeStream.Write(_buffer, 0, sizenow);
}
catch (Exception e)
{
Debug.WriteLine(e.Message);
error = "Error writing out file. " + Environment.NewLine + e.Message;
return(ReturnCode.FileSystemError);
}
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
#endregion
#region Collect Checksums
// if we did a full copy then we just calculated all the checksums while doing the copy
if (!rawCopy)
{
tcrc32.Finish();
tmd5.Finish();
tsha1.Finish();
bCRC = tcrc32.Hash;
bMD5 = tmd5.Hash;
bSHA1 = tsha1.Hash;
tcrc32.Dispose();
tmd5.Dispose();
tsha1.Dispose();
}
// if we raw copied and the source file has been FileChecked then we can trust the checksums in the source file
else
{
bCRC = ArrByte.Copy(fileIn.CRC);
if (fileIn.FileStatusIs(FileStatus.MD5Verified))
{
bMD5 = ArrByte.Copy(fileIn.MD5);
}
if (fileIn.FileStatusIs(FileStatus.SHA1Verified))
{
bSHA1 = ArrByte.Copy(fileIn.SHA1);
}
}
#endregion
#region close the ReadStream
if (fileIn.FileType == FileType.ZipFile && zipFileIn != null)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.Filename(fileIn.ReportIndex);
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
//if (IO.File.Exists(tmpFilename))
// IO.File.Delete(tmpFilename);
}
#endregion
}
else
{
// Zero Length File (Directory in a Zip)
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileAddDirectory();
}
bCRC = VarFix.CleanMD5SHA1("00000000", 8);
bMD5 = VarFix.CleanMD5SHA1("d41d8cd98f00b204e9800998ecf8427e", 32);
bSHA1 = VarFix.CleanMD5SHA1("da39a3ee5e6b4b0d3255bfef95601890afd80709", 40);
}
#region close the WriteStream
if (fileOut.FileType == FileType.ZipFile)
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(bCRC);
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
fileOut.ZipFileIndex = zipFileOut.LocalFilesCount() - 1;
fileOut.ZipFileHeaderPosition = zipFileOut.LocalHeader(fileOut.ZipFileIndex);
}
else
{
writeStream.Flush();
writeStream.Close();
IO.FileInfo fi = new IO.FileInfo(zipFilenameOut);
fileOut.TimeStamp = fi.LastWriteTime;
}
#endregion
if (!isZeroLengthFile)
{
if (!rawCopy)
{
if (!ArrByte.bCompare(bCRC, fileIn.CRC))
{
fileIn.GotStatus = GotStatus.Corrupt;
error = "Source CRC does not match Source Data stream, corrupt Zip";
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.SourceCRCCheckSumError);
}
fileIn.FileStatusSet(FileStatus.CRCVerified | FileStatus.SizeVerified);
bool sourceFailed = false;
// check to see if we have a MD5 from the DAT file
if (fileIn.FileStatusIs(FileStatus.MD5FromDAT))
{
if (fileIn.MD5 == null)
{
error = "Should have an filein MD5 from Dat but not found. Logic Error.";
return(ReturnCode.LogicError);
}
if (!ArrByte.bCompare(fileIn.MD5, bMD5))
{
sourceFailed = true;
}
else
{
fileIn.FileStatusSet(FileStatus.MD5Verified);
}
}
// check to see if we have an MD5 (not from the DAT) so must be from previously scanning this file.
else if (fileIn.MD5 != null)
{
if (!ArrByte.bCompare(fileIn.MD5, bMD5))
{
// if we had an MD5 from a preview scan and it now does not match, something has gone really bad.
error = "The MD5 found does not match a previously scanned MD5, this should not happen, unless something got corrupt.";
return(ReturnCode.LogicError);
}
}
else // (FileIn.MD5 == null)
{
fileIn.MD5 = bMD5;
fileIn.FileStatusSet(FileStatus.MD5Verified);
}
// check to see if we have a SHA1 from the DAT file
if (fileIn.FileStatusIs(FileStatus.SHA1FromDAT))
{
if (fileIn.SHA1 == null)
{
error = "Should have an filein SHA1 from Dat but not found. Logic Error.";
return(ReturnCode.LogicError);
}
if (!ArrByte.bCompare(fileIn.SHA1, bSHA1))
{
sourceFailed = true;
}
else
{
fileIn.FileStatusSet(FileStatus.SHA1Verified);
}
}
// check to see if we have an SHA1 (not from the DAT) so must be from previously scanning this file.
else if (fileIn.SHA1 != null)
{
if (!ArrByte.bCompare(fileIn.SHA1, bSHA1))
{
// if we had an SHA1 from a preview scan and it now does not match, something has gone really bad.
error = "The SHA1 found does not match a previously scanned SHA1, this should not happen, unless something got corrupt.";
return(ReturnCode.LogicError);
}
}
else // (FileIn.SHA1 == null)
{
fileIn.SHA1 = bSHA1;
fileIn.FileStatusSet(FileStatus.SHA1Verified);
}
if (sourceFailed)
{
if (fileIn.FileType == FileType.ZipFile)
{
RvFile tZFile = new RvFile(FileType.ZipFile);
foundFile = tZFile;
tZFile.ZipFileIndex = fileIn.ZipFileIndex;
tZFile.ZipFileHeaderPosition = fileIn.ZipFileHeaderPosition;
}
else
{
foundFile = new RvFile(fileIn.FileType);
}
foundFile.Name = fileIn.Name;
foundFile.Size = fileIn.Size;
foundFile.CRC = bCRC;
foundFile.MD5 = bMD5;
foundFile.SHA1 = bSHA1;
foundFile.TimeStamp = fileIn.TimeStamp;
foundFile.SetStatus(DatStatus.NotInDat, GotStatus.Got);
foundFile.FileStatusSet(FileStatus.SizeVerified | FileStatus.CRCVerified | FileStatus.MD5Verified | FileStatus.SHA1Verified);
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.SourceCheckSumError);
}
}
}
if (fileOut.FileType == FileType.ZipFile)
{
fileOut.FileStatusSet(FileStatus.SizeFromHeader | FileStatus.CRCFromHeader);
}
if (fileOut.FileStatusIs(FileStatus.CRCFromDAT) && fileOut.CRC != null && !ArrByte.bCompare(fileOut.CRC, bCRC))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.CRC = bCRC;
if (!rawCopy || fileIn.FileStatusIs(FileStatus.CRCVerified))
{
fileOut.FileStatusSet(FileStatus.CRCVerified);
}
if (bSHA1 != null)
{
if (fileOut.FileStatusIs(FileStatus.SHA1FromDAT) && fileOut.SHA1 != null && !ArrByte.bCompare(fileOut.SHA1, bSHA1))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.SHA1 = bSHA1;
fileOut.FileStatusSet(FileStatus.SHA1Verified);
}
if (bMD5 != null)
{
if (fileOut.FileStatusIs(FileStatus.MD5FromDAT) && fileOut.MD5 != null && !ArrByte.bCompare(fileOut.MD5, bMD5))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.MD5 = bMD5;
fileOut.FileStatusSet(FileStatus.MD5Verified);
}
if (fileIn.Size != null)
{
fileOut.Size = fileIn.Size;
fileOut.FileStatusSet(FileStatus.SizeVerified);
}
if (fileIn.GotStatus == GotStatus.Corrupt)
{
fileOut.GotStatus = GotStatus.Corrupt;
}
else
{
fileOut.GotStatus = GotStatus.Got; // Changes RepStatus to Correct
}
fileOut.FileStatusSet(FileStatus.SizeVerified);
if (fileOut.SHA1CHD == null && fileIn.SHA1CHD != null)
{
fileOut.SHA1CHD = fileIn.SHA1CHD;
}
if (fileOut.MD5CHD == null && fileIn.MD5CHD != null)
{
fileOut.MD5CHD = fileIn.MD5CHD;
}
fileOut.CHDVersion = fileIn.CHDVersion;
fileOut.FileStatusSet(FileStatus.SHA1CHDFromHeader | FileStatus.MD5CHDFromHeader | FileStatus.SHA1CHDVerified | FileStatus.MD5CHDVerified, fileIn);
return(ReturnCode.Good);
}
public void DeepScan()
{
if (_buffer0 == null)
{
_buffer0 = new byte[Buffersize];
_buffer1 = new byte[Buffersize];
}
for (int index = 0; index < _localFiles.Count; index++)
{
if (_localFiles[index].isDirectory || (_localFiles[index].UncompressedSize == 0))
{
_localFiles[index].md5 = new byte[]
{ 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e };
_localFiles[index].sha1 = new byte[]
{
0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
0xaf, 0xd8, 0x07, 0x09
};
_localFiles[index].FileStatus = ZipReturn.ZipGood;
continue;
}
ulong sizetogo;
Stream inStream;
ZipReturn zr = ZipFileOpenReadStream(index, out inStream, out sizetogo);
if (zr != ZipReturn.ZipGood)
{
continue;
}
if (inStream == null)
{
continue;
}
ThreadLoadBuffer lbuffer = new ThreadLoadBuffer(inStream);
ThreadCRC tcrc32 = new ThreadCRC();
ThreadMD5 tmd5 = new ThreadMD5();
ThreadSHA1 tsha1 = new ThreadSHA1();
// Pre load the first buffer0
int sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
inStream.Read(_buffer0, 0, sizeNext);
int sizebuffer = sizeNext;
sizetogo -= (ulong)sizeNext;
bool whichBuffer = true;
while ((sizebuffer > 0) && !lbuffer.errorState)
{
sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
if (sizeNext > 0)
{
lbuffer.Trigger(whichBuffer ? _buffer1 : _buffer0, sizeNext);
}
byte[] buffer = whichBuffer ? _buffer0 : _buffer1;
tcrc32.Trigger(buffer, sizebuffer);
tmd5.Trigger(buffer, sizebuffer);
tsha1.Trigger(buffer, sizebuffer);
if (sizeNext > 0)
{
lbuffer.Wait();
}
tcrc32.Wait();
tmd5.Wait();
tsha1.Wait();
sizebuffer = sizeNext;
sizetogo -= (ulong)sizeNext;
whichBuffer = !whichBuffer;
}
if (lbuffer.errorState)
{
lbuffer.Dispose();
tcrc32.Dispose();
tmd5.Dispose();
tsha1.Dispose();
_localFiles[index].FileStatus = ZipReturn.ZipDecodeError;
continue;
}
lbuffer.Finish();
tcrc32.Finish();
tmd5.Finish();
tsha1.Finish();
byte[] testcrc = tcrc32.Hash;
_localFiles[index].md5 = tmd5.Hash;
_localFiles[index].sha1 = tsha1.Hash;
lbuffer.Dispose();
tcrc32.Dispose();
tmd5.Dispose();
tsha1.Dispose();
_localFiles[index].FileStatus = Util.ByteArrCompare(_localFiles[index].crc, testcrc)
? ZipReturn.ZipGood
: ZipReturn.ZipCRCDecodeError;
}
}
// This Function returns:
// Good : Everything Worked Correctly
// RescanNeeded : Something unexpectidly changed in the files, so Stop prompt user to rescan.
// LogicError : This Should never happen and is a logic problem in the code
// FileSystemError : Something is wrong with the files
/// <summary>
/// Performs the RomVault File Copy, with the source and destination being files or zipped files
/// </summary>
/// <param name="fileIn">This is the file being copied, it may be a zipped file or a regular file system file</param>
/// <param name="zipFileOut">
/// This is the zip file that is being writen to, if it is null a new zip file will be made if we
/// are coping to a zip
/// </param>
/// <param name="zipFilenameOut">This is the name of the .zip file to be made that will be created in zipFileOut</param>
/// <param name="fileOut">This is the actual output filename</param>
/// <param name="forceRaw">if true then we will do a raw copy, this is so that we can copy corrupt zips</param>
/// <param name="error">This is the returned error message if this copy fails</param>
/// <param name="foundFile">
/// If we are SHA1/MD5 checking the source file for the first time, and it is different from what
/// we expected the correct values for this file are returned in foundFile
/// </param>
/// <returns>ReturnCode.Good is the valid return code otherwire we have an error</returns>
public static ReturnCode CopyFile(RvFile fileIn, ref ICompress zipFileOut, string zipFilenameOut, RvFile fileOut, bool forceRaw, out string error, out RvFile foundFile)
{
foundFile = null;
error = "";
if (_buffer == null)
{
_buffer = new byte[BufferSize];
}
bool rawCopy = RawCopy(fileIn, fileOut, forceRaw);
ulong streamSize = 0;
ushort compressionMethod = 8;
bool sourceTrrntzip = false;
ICompress zipFileIn = null;
Stream readStream = null;
ReturnCode retC;
bool isZeroLengthFile = DBHelper.IsZeroLengthFile(fileOut);
if (!isZeroLengthFile)
{
//check that the in and out file match
retC = CheckInputAndOutputFile(fileIn, fileOut, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
//Find and Check/Open Input Files
retC = OpenInputStream(fileIn, rawCopy, out zipFileIn, out sourceTrrntzip, out readStream, out streamSize, out compressionMethod, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
}
else
{
sourceTrrntzip = true;
}
if (!rawCopy && ((Program.rvSettings.FixLevel == eFixLevel.TrrntZipLevel1) || (Program.rvSettings.FixLevel == eFixLevel.TrrntZipLevel2) || (Program.rvSettings.FixLevel == eFixLevel.TrrntZipLevel3)))
{
compressionMethod = 8;
}
//Find and Check/Open Output Files
Stream writeStream;
retC = OpenOutputStream(fileOut, fileIn, ref zipFileOut, zipFilenameOut, compressionMethod, rawCopy, sourceTrrntzip, out writeStream, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
byte[] bCRC;
byte[] bMD5 = null;
byte[] bSHA1 = null;
if (!isZeroLengthFile)
{
#region Do Data Tranfer
ThreadCRC tcrc32 = null;
ThreadMD5 tmd5 = null;
ThreadSHA1 tsha1 = null;
if (!rawCopy)
{
tcrc32 = new ThreadCRC();
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
}
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > BufferSize ? BufferSize : (int)sizetogo;
try
{
readStream.Read(_buffer, 0, sizenow);
}
catch (ZlibException)
{
if ((fileIn.FileType == FileType.ZipFile) && (zipFileIn != null))
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.Filename(fileIn.ReportIndex);
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
}
if (fileOut.FileType == FileType.ZipFile)
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(new byte[] { 0, 0, 0, 0 });
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
zipFileOut.ZipFileRollBack();
}
else
{
writeStream.Flush();
writeStream.Close();
File.Delete(zipFilenameOut);
}
error = "Error in Data Stream";
return(ReturnCode.SourceCRCCheckSumError);
}
catch (Exception e)
{
error = "Error reading Source File " + e.Message;
return(ReturnCode.FileSystemError);
}
if (!rawCopy)
{
tcrc32.Trigger(_buffer, sizenow);
tmd5.Trigger(_buffer, sizenow);
tsha1.Trigger(_buffer, sizenow);
tcrc32.Wait();
tmd5.Wait();
tsha1.Wait();
}
try
{
writeStream.Write(_buffer, 0, sizenow);
}
catch (Exception e)
{
Debug.WriteLine(e.Message);
error = "Error writing out file. " + Environment.NewLine + e.Message;
return(ReturnCode.FileSystemError);
}
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
#endregion
#region Collect Checksums
// if we did a full copy then we just calculated all the checksums while doing the copy
if (!rawCopy)
{
tcrc32.Finish();
tmd5.Finish();
tsha1.Finish();
bCRC = tcrc32.Hash;
bMD5 = tmd5.Hash;
bSHA1 = tsha1.Hash;
tcrc32.Dispose();
tmd5.Dispose();
tsha1.Dispose();
}
// if we raw copied and the source file has been FileChecked then we can trust the checksums in the source file
else
{
bCRC = ArrByte.Copy(fileIn.CRC);
if (fileIn.FileStatusIs(FileStatus.MD5Verified))
{
bMD5 = ArrByte.Copy(fileIn.MD5);
}
if (fileIn.FileStatusIs(FileStatus.SHA1Verified))
{
bSHA1 = ArrByte.Copy(fileIn.SHA1);
}
}
#endregion
#region close the ReadStream
if (((fileIn.FileType == FileType.ZipFile) || (fileIn.FileType == FileType.SevenZipFile)) && (zipFileIn != null))
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.Filename(fileIn.ReportIndex);
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
//if (RVIO.File.Exists(tmpFilename))
// RVIO.File.Delete(tmpFilename);
}
#endregion
}
else
{
// Zero Length File (Directory in a Zip)
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileAddDirectory();
}
bCRC = VarFix.CleanMD5SHA1("00000000", 8);
bMD5 = VarFix.CleanMD5SHA1("d41d8cd98f00b204e9800998ecf8427e", 32);
bSHA1 = VarFix.CleanMD5SHA1("da39a3ee5e6b4b0d3255bfef95601890afd80709", 40);
}
#region close the WriteStream
if ((fileOut.FileType == FileType.ZipFile) || (fileOut.FileType == FileType.SevenZipFile))
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(bCRC);
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
fileOut.ZipFileIndex = zipFileOut.LocalFilesCount() - 1;
fileOut.ZipFileHeaderPosition = zipFileOut.LocalHeader(fileOut.ZipFileIndex);
}
else
{
writeStream.Flush();
writeStream.Close();
FileInfo fi = new FileInfo(zipFilenameOut);
fileOut.TimeStamp = fi.LastWriteTime;
}
#endregion
if (!isZeroLengthFile)
{
if (!rawCopy)
{
if (!ArrByte.bCompare(bCRC, fileIn.CRC))
{
fileIn.GotStatus = GotStatus.Corrupt;
error = "Source CRC does not match Source Data stream, corrupt Zip";
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.SourceCRCCheckSumError);
}
fileIn.FileStatusSet(FileStatus.CRCVerified | FileStatus.SizeVerified);
bool sourceFailed = false;
// check to see if we have a MD5 from the DAT file
if (fileIn.FileStatusIs(FileStatus.MD5FromDAT))
{
if (fileIn.MD5 == null)
{
error = "Should have an filein MD5 from Dat but not found. Logic Error.";
return(ReturnCode.LogicError);
}
if (!ArrByte.bCompare(fileIn.MD5, bMD5))
{
sourceFailed = true;
}
else
{
fileIn.FileStatusSet(FileStatus.MD5Verified);
}
}
// check to see if we have an MD5 (not from the DAT) so must be from previously scanning this file.
else if (fileIn.MD5 != null)
{
if (!ArrByte.bCompare(fileIn.MD5, bMD5))
{
// if we had an MD5 from a preview scan and it now does not match, something has gone really bad.
error = "The MD5 found does not match a previously scanned MD5, this should not happen, unless something got corrupt.";
return(ReturnCode.LogicError);
}
}
else // (FileIn.MD5 == null)
{
fileIn.MD5 = bMD5;
fileIn.FileStatusSet(FileStatus.MD5Verified);
}
// check to see if we have a SHA1 from the DAT file
if (fileIn.FileStatusIs(FileStatus.SHA1FromDAT))
{
if (fileIn.SHA1 == null)
{
error = "Should have an filein SHA1 from Dat but not found. Logic Error.";
return(ReturnCode.LogicError);
}
if (!ArrByte.bCompare(fileIn.SHA1, bSHA1))
{
sourceFailed = true;
}
else
{
fileIn.FileStatusSet(FileStatus.SHA1Verified);
}
}
// check to see if we have an SHA1 (not from the DAT) so must be from previously scanning this file.
else if (fileIn.SHA1 != null)
{
if (!ArrByte.bCompare(fileIn.SHA1, bSHA1))
{
// if we had an SHA1 from a preview scan and it now does not match, something has gone really bad.
error = "The SHA1 found does not match a previously scanned SHA1, this should not happen, unless something got corrupt.";
return(ReturnCode.LogicError);
}
}
else // (FileIn.SHA1 == null)
{
fileIn.SHA1 = bSHA1;
fileIn.FileStatusSet(FileStatus.SHA1Verified);
}
if (sourceFailed)
{
if ((fileIn.FileType == FileType.ZipFile) || (fileIn.FileType == FileType.SevenZipFile))
{
RvFile tZFile = new RvFile(FileType.ZipFile);
foundFile = tZFile;
tZFile.ZipFileIndex = fileIn.ZipFileIndex;
tZFile.ZipFileHeaderPosition = fileIn.ZipFileHeaderPosition;
}
else
{
foundFile = new RvFile(fileIn.FileType);
}
foundFile.Name = fileIn.Name;
foundFile.Size = fileIn.Size;
foundFile.CRC = bCRC;
foundFile.MD5 = bMD5;
foundFile.SHA1 = bSHA1;
foundFile.TimeStamp = fileIn.TimeStamp;
foundFile.SetStatus(DatStatus.NotInDat, GotStatus.Got);
foundFile.FileStatusSet(FileStatus.SizeVerified | FileStatus.CRCVerified | FileStatus.MD5Verified | FileStatus.SHA1Verified);
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.SourceCheckSumError);
}
}
}
if ((fileOut.FileType == FileType.ZipFile) || (fileOut.FileType == FileType.SevenZipFile))
{
fileOut.FileStatusSet(FileStatus.SizeFromHeader | FileStatus.CRCFromHeader);
}
if (fileOut.FileStatusIs(FileStatus.CRCFromDAT) && (fileOut.CRC != null) && !ArrByte.bCompare(fileOut.CRC, bCRC))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.CRC = bCRC;
if (!rawCopy || fileIn.FileStatusIs(FileStatus.CRCVerified))
{
fileOut.FileStatusSet(FileStatus.CRCVerified);
}
if (bSHA1 != null)
{
if (fileOut.FileStatusIs(FileStatus.SHA1FromDAT) && (fileOut.SHA1 != null) && !ArrByte.bCompare(fileOut.SHA1, bSHA1))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.SHA1 = bSHA1;
fileOut.FileStatusSet(FileStatus.SHA1Verified);
}
if (bMD5 != null)
{
if (fileOut.FileStatusIs(FileStatus.MD5FromDAT) && (fileOut.MD5 != null) && !ArrByte.bCompare(fileOut.MD5, bMD5))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.MD5 = bMD5;
fileOut.FileStatusSet(FileStatus.MD5Verified);
}
if (fileIn.Size != null)
{
fileOut.Size = fileIn.Size;
fileOut.FileStatusSet(FileStatus.SizeVerified);
}
if (fileIn.GotStatus == GotStatus.Corrupt)
{
fileOut.GotStatus = GotStatus.Corrupt;
}
else
{
fileOut.GotStatus = GotStatus.Got; // Changes RepStatus to Correct
}
fileOut.FileStatusSet(FileStatus.SizeVerified);
if ((fileOut.SHA1CHD == null) && (fileIn.SHA1CHD != null))
{
fileOut.SHA1CHD = fileIn.SHA1CHD;
}
if ((fileOut.MD5CHD == null) && (fileIn.MD5CHD != null))
{
fileOut.MD5CHD = fileIn.MD5CHD;
}
fileOut.CHDVersion = fileIn.CHDVersion;
fileOut.FileStatusSet(FileStatus.SHA1CHDFromHeader | FileStatus.MD5CHDFromHeader | FileStatus.SHA1CHDVerified | FileStatus.MD5CHDVerified, fileIn);
return(ReturnCode.Good);
}
