public void LocalFileCheck()
{
if (FileStatus != ZipReturn.ZipUntested) return;
try
{
Stream sInput = null;
_zipFs.Seek((long)_dataLocation, SeekOrigin.Begin);
switch (_compressionMethod)
{
case 8:
sInput = new DeflateStream(_zipFs, CompressionMode.Decompress, true);
break;
case 0:
sInput = _zipFs;
break;
}
if (sInput == null)
{
FileStatus = ZipReturn.ZipErrorGettingDataStream;
return;
}
CRC32Hash crc32 = new CRC32Hash();
MD5 lmd5 = System.Security.Cryptography.MD5.Create();
SHA1 lsha1 = System.Security.Cryptography.SHA1.Create();
ulong sizetogo = UncompressedSize;
if (_buffer == null)
_buffer = new byte[Buffersize];
while (sizetogo > 0)
{
int sizenow = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
sInput.Read(_buffer, 0, sizenow);
crc32.TransformBlock(_buffer, 0, sizenow, null, 0);
lmd5.TransformBlock(_buffer, 0, sizenow, null, 0);
lsha1.TransformBlock(_buffer, 0, sizenow, null, 0);
sizetogo = sizetogo - (ulong)sizenow;
}
crc32.TransformFinalBlock(_buffer, 0, 0);
lmd5.TransformFinalBlock(_buffer, 0, 0);
lsha1.TransformFinalBlock(_buffer, 0, 0);
byte[] testcrc = crc32.Hash;
md5 = lmd5.Hash;
sha1 = lsha1.Hash;
if (_compressionMethod == 8)
{
sInput.Close();
sInput.Dispose();
}
FileStatus = ByteArrCompare(CRC, testcrc) ? ZipReturn.ZipGood : ZipReturn.ZipCRCDecodeError;
}
catch
{
FileStatus = ZipReturn.ZipDecodeError;
}
}
/// <summary>
/// Attempt to extract a file as an archive
/// </summary>
/// <param name="outDir">Output directory for archive extraction</param>
/// <returns>True if the extraction was a success, false otherwise</returns>
public override bool CopyAll(string outDir)
{
bool encounteredErrors = true;
try
{
// Create the temp directory
Directory.CreateDirectory(outDir);
// Extract all files to the temp directory
ZipFile zf = new ZipFile();
ZipReturn zr = zf.Open(this.Filename, new FileInfo(this.Filename).LastWriteTime.Ticks, true);
if (zr != ZipReturn.ZipGood)
{
throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
for (int i = 0; i < zf.EntriesCount && zr == ZipReturn.ZipGood; i++)
{
// Open the read stream
zr = zf.OpenReadStream(i, false, out Stream readStream, out ulong streamsize, out SabreTools.Library.Data.CompressionMethod cm, out uint lastMod);
// Create the rest of the path, if needed
if (!String.IsNullOrWhiteSpace(Path.GetDirectoryName(zf.Entries[i].FileName)))
{
Directory.CreateDirectory(Path.Combine(outDir, Path.GetDirectoryName(zf.Entries[i].FileName)));
}
// If the entry ends with a directory separator, continue to the next item, if any
if (zf.Entries[i].FileName.EndsWith(Path.DirectorySeparatorChar.ToString()) ||
zf.Entries[i].FileName.EndsWith(Path.AltDirectorySeparatorChar.ToString()) ||
zf.Entries[i].FileName.EndsWith(Path.PathSeparator.ToString()))
{
continue;
}
FileStream writeStream = Utilities.TryCreate(Path.Combine(outDir, zf.Entries[i].FileName));
// If the stream is smaller than the buffer, just run one loop through to avoid issues
if (streamsize < _bufferSize)
{
byte[] ibuffer = new byte[streamsize];
int ilen = readStream.Read(ibuffer, 0, (int)streamsize);
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
// Otherwise, we do the normal loop
else
{
int realBufferSize = (streamsize < _bufferSize ? (int)streamsize : _bufferSize);
byte[] ibuffer = new byte[realBufferSize];
int ilen;
while ((ilen = readStream.Read(ibuffer, 0, realBufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
}
zr = zf.CloseReadStream();
writeStream.Dispose();
}
zf.Close();
encounteredErrors = false;
}
catch (EndOfStreamException)
{
// Catch this but don't count it as an error because SharpCompress is unsafe
}
catch (InvalidOperationException)
{
encounteredErrors = true;
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
encounteredErrors = true;
}
return(encounteredErrors);
}
private static void ScanRomRoot(string directory)
{
_bgw.ReportProgress(0, new bgwText("Scanning Dir : " + directory));
DirectoryInfo di = new DirectoryInfo(directory);
FileInfo[] fi = di.GetFiles();
_bgw.ReportProgress(0, new bgwRange2Visible(true));
_bgw.ReportProgress(0, new bgwSetRange2(fi.Count()));
for (int j = 0; j < fi.Count(); j++)
{
if (_bgw.CancellationPending)
{
return;
}
FileInfo f = fi[j];
_bgw.ReportProgress(0, new bgwValue2(j));
_bgw.ReportProgress(0, new bgwText2(f.Name));
string ext = Path.GetExtension(f.Name);
if (ext.ToLower() == ".gz")
{
GZip gZipTest = new GZip();
ZipReturn errorcode = gZipTest.ReadGZip(f.FullName, true);
gZipTest.sha1Hash = VarFix.CleanMD5SHA1(Path.GetFileNameWithoutExtension(f.Name), 40);
if (errorcode != ZipReturn.ZipGood)
{
_bgw.ReportProgress(0, new bgwShowError(f.FullName, "gz File corrupt"));
continue;
}
RvFile tFile = new RvFile();
tFile.CRC = gZipTest.crc;
tFile.MD5 = gZipTest.md5Hash;
tFile.SHA1 = gZipTest.sha1Hash;
tFile.Size = gZipTest.uncompressedSize;
tFile.CompressedSize = gZipTest.compressedSize;
FindStatus res = fileneededTest(tFile);
if (res != FindStatus.FoundFileInArchive)
{
tFile.DBWrite();
}
}
if (_bgw.CancellationPending)
{
return;
}
}
DirectoryInfo[] childdi = di.GetDirectories();
foreach (DirectoryInfo d in childdi)
{
if (_bgw.CancellationPending)
{
return;
}
ScanRomRoot(d.FullName);
}
}
public static ReturnCode DecompressSource7ZipFile(RvFile zZipFileIn, bool includeGood, out string error)
{
byte[] buffer = new byte[BufferSize];
RvFile cacheDir = DB.RvFileCache();
string fileNameIn = zZipFileIn.FullName;
SevenZ zipFileIn = new SevenZ();
ZipReturn zr1 = zipFileIn.ZipFileOpen(fileNameIn, zZipFileIn.TimeStamp, true);
if (zr1 != ZipReturn.ZipGood)
{
error = "Error opening 7zip file for caching";
return(ReturnCode.RescanNeeded);
}
RvFile outDir = new RvFile(FileType.Dir)
{
Name = zZipFileIn.Name + ".cache",
Parent = cacheDir,
DatStatus = DatStatus.InToSort,
GotStatus = GotStatus.Got
};
int nameDirIndex = 0;
while (cacheDir.ChildNameSearch(outDir, out int index) == 0)
{
nameDirIndex++;
outDir.Name = zZipFileIn.Name + ".cache (" + nameDirIndex + ")";
}
cacheDir.ChildAdd(outDir);
Directory.CreateDirectory(outDir.FullName);
for (int i = 0; i < zipFileIn.LocalFilesCount(); i++)
{
if (zipFileIn.IsDirectory(i))
{
continue;
}
RvFile thisFile = null;
for (int j = 0; j < zZipFileIn.ChildCount; j++)
{
if (zZipFileIn.Child(j).ZipFileIndex != i)
{
continue;
}
thisFile = zZipFileIn.Child(j);
break;
}
if (thisFile == null)
{
error = "Error opening 7zip file for caching";
return(ReturnCode.RescanNeeded);
}
bool extract = true;
// first check to see if we have a file version of this compressed file somewhere else.
foreach (RvFile f in thisFile.FileGroup.Files)
{
if (f.FileType == FileType.File && f.GotStatus == GotStatus.Got)
{
extract = false;
}
}
if (!extract)
{
continue;
}
extract = false;
if (includeGood)
{
// if this is the file we are fixing then pull out the correct files.
if (thisFile.RepStatus == RepStatus.Correct)
{
extract = true;
}
}
// next check to see if we need this extracted to fix another file
foreach (RvFile f in thisFile.FileGroup.Files)
{
if (f.RepStatus == RepStatus.CanBeFixed)
{
extract = true;
break;
}
}
if (!extract)
{
continue;
}
string cleanedName = thisFile.Name;
cleanedName = cleanedName.Replace("/", "-");
cleanedName = cleanedName.Replace("\\", "-");
RvFile outFile = new RvFile(FileType.File)
{
Name = cleanedName,
Size = thisFile.Size,
CRC = thisFile.CRC,
SHA1 = thisFile.SHA1,
MD5 = thisFile.MD5,
HeaderFileType = thisFile.HeaderFileType,
AltSize = thisFile.AltSize,
AltCRC = thisFile.AltCRC,
AltSHA1 = thisFile.AltSHA1,
AltMD5 = thisFile.AltMD5,
FileGroup = thisFile.FileGroup
};
outFile.SetStatus(DatStatus.InToSort, GotStatus.Got);
outFile.FileStatusSet(
FileStatus.HeaderFileTypeFromHeader |
FileStatus.SizeFromHeader | FileStatus.SizeVerified |
FileStatus.CRCFromHeader | FileStatus.CRCVerified |
FileStatus.SHA1FromHeader | FileStatus.SHA1Verified |
FileStatus.MD5FromHeader | FileStatus.MD5Verified |
FileStatus.AltSizeFromHeader | FileStatus.AltSizeVerified |
FileStatus.AltCRCFromHeader | FileStatus.AltCRCVerified |
FileStatus.AltSHA1FromHeader | FileStatus.AltSHA1Verified |
FileStatus.AltMD5FromHeader | FileStatus.AltMD5Verified
, thisFile);
outFile.RepStatus = RepStatus.NeededForFix;
zipFileIn.ZipFileOpenReadStream(i, out Stream readStream, out ulong unCompressedSize);
string filenameOut = Path.Combine(outDir.FullName, outFile.Name);
if (Settings.rvSettings.DetailedFixReporting)
{
string fixZipFullName = zZipFileIn.TreeFullName;
Report.ReportProgress(new bgwShowFix(Path.GetDirectoryName(fixZipFullName), Path.GetFileName(fixZipFullName), thisFile.Name, thisFile.Size, "-->", outDir.FullName, "", outFile.Name));
}
ThreadMD5 tmd5 = null;
ThreadSHA1 tsha1 = null;
ThreadCRC tcrc32 = new ThreadCRC();
if (Settings.rvSettings.FixLevel != EFixLevel.Level1 && Settings.rvSettings.FixLevel != EFixLevel.TrrntZipLevel1)
{
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
}
int errorCode = FileStream.OpenFileWrite(filenameOut, out Stream writeStream);
ulong sizetogo = unCompressedSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > BufferSize ? BufferSize : (int)sizetogo;
try
{
readStream.Read(buffer, 0, sizenow);
}
catch (Exception ex)
{
if (ex is ZlibException || ex is DataErrorException)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.ZipFilename;
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
writeStream.Flush();
writeStream.Close();
if (filenameOut != null)
{
File.Delete(filenameOut);
}
thisFile.GotStatus = GotStatus.Corrupt;
error = "Unexpected corrupt archive file found:\n" + zZipFileIn.FullName +
"\nRun Find Fixes, and Fix to continue fixing correctly.";
return(ReturnCode.SourceDataStreamCorrupt);
}
error = "Error reading Source File " + ex.Message;
return(ReturnCode.FileSystemError);
}
tcrc32.Trigger(buffer, sizenow);
tmd5?.Trigger(buffer, sizenow);
tsha1?.Trigger(buffer, sizenow);
tcrc32.Wait();
tmd5?.Wait();
tsha1?.Wait();
try
{
writeStream.Write(buffer, 0, sizenow);
}
catch (Exception e)
{
error = "Error writing out file. " + Environment.NewLine + e.Message;
return(ReturnCode.FileSystemError);
}
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
writeStream.Close();
writeStream.Dispose();
tcrc32.Finish();
tmd5?.Finish();
tsha1?.Finish();
byte[] bCRC = tcrc32.Hash;
byte[] bMD5 = tmd5?.Hash;
byte[] bSHA1 = tsha1?.Hash;
tcrc32.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
FileInfo fi = new FileInfo(filenameOut);
outFile.TimeStamp = fi.LastWriteTime;
if (bCRC != null && thisFile.CRC != null && !ArrByte.BCompare(bCRC, thisFile.CRC))
{
// error in file.
}
if (bMD5 != null && thisFile.MD5 != null && !ArrByte.BCompare(bMD5, thisFile.MD5))
{
// error in file.
}
if (bSHA1 != null && thisFile.SHA1 != null && !ArrByte.BCompare(bSHA1, thisFile.SHA1))
{
// error in file.
}
thisFile.FileGroup.Files.Add(outFile);
outDir.ChildAdd(outFile);
}
zipFileIn.ZipFileClose();
error = "";
return(ReturnCode.Good);
}
/// <summary>
/// Attempt to extract a stream from an archive
/// </summary>
/// <param name="entryName">Name of the entry to be extracted</param>
/// <param name="realEntry">Output representing the entry name that was found</param>
/// <returns>MemoryStream representing the entry, null on error</returns>
public override (MemoryStream, string) CopyToStream(string entryName)
{
MemoryStream ms = new MemoryStream();
string realEntry = null;
try
{
ZipFile zf = new ZipFile();
ZipReturn zr = zf.Open(this.Filename, new FileInfo(this.Filename).LastWriteTime.Ticks, true);
if (zr != ZipReturn.ZipGood)
{
throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
for (int i = 0; i < zf.EntriesCount && zr == ZipReturn.ZipGood; i++)
{
if (zf.Entries[i].FileName.Contains(entryName))
{
// Open the read stream
realEntry = zf.Entries[i].FileName;
zr = zf.OpenReadStream(i, false, out Stream readStream, out ulong streamsize, out SabreTools.Library.Data.CompressionMethod cm, out uint lastMod);
// If the stream is smaller than the buffer, just run one loop through to avoid issues
if (streamsize < _bufferSize)
{
byte[] ibuffer = new byte[streamsize];
int ilen = readStream.Read(ibuffer, 0, (int)streamsize);
ms.Write(ibuffer, 0, ilen);
ms.Flush();
}
// Otherwise, we do the normal loop
else
{
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while (streamsize > _bufferSize)
{
ilen = readStream.Read(ibuffer, 0, _bufferSize);
ms.Write(ibuffer, 0, ilen);
ms.Flush();
streamsize -= _bufferSize;
}
ilen = readStream.Read(ibuffer, 0, (int)streamsize);
ms.Write(ibuffer, 0, ilen);
ms.Flush();
}
zr = zf.CloseReadStream();
}
}
zf.Dispose();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
ms = null;
realEntry = null;
}
return(ms, realEntry);
}
public void DeepScan()
{
if (_buffer0 == null)
{
_buffer0 = new byte[Buffersize];
_buffer1 = new byte[Buffersize];
}
for (int index = 0; index < _localFiles.Count; index++)
{
if (_localFiles[index].isDirectory || (_localFiles[index].UncompressedSize == 0))
{
_localFiles[index].md5 = new byte[]
{ 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e };
_localFiles[index].sha1 = new byte[]
{
0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
0xaf, 0xd8, 0x07, 0x09
};
_localFiles[index].FileStatus = ZipReturn.ZipGood;
continue;
}
ulong sizetogo;
Stream inStream;
ZipReturn zr = ZipFileOpenReadStream(index, out inStream, out sizetogo);
if (zr != ZipReturn.ZipGood)
{
continue;
}
if (inStream == null)
{
continue;
}
ThreadLoadBuffer lbuffer = new ThreadLoadBuffer(inStream);
ThreadCRC tcrc32 = new ThreadCRC();
ThreadMD5 tmd5 = new ThreadMD5();
ThreadSHA1 tsha1 = new ThreadSHA1();
// Pre load the first buffer0
int sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
inStream.Read(_buffer0, 0, sizeNext);
int sizebuffer = sizeNext;
sizetogo -= (ulong)sizeNext;
bool whichBuffer = true;
while ((sizebuffer > 0) && !lbuffer.errorState)
{
sizeNext = sizetogo > Buffersize ? Buffersize : (int)sizetogo;
if (sizeNext > 0)
{
lbuffer.Trigger(whichBuffer ? _buffer1 : _buffer0, sizeNext);
}
byte[] buffer = whichBuffer ? _buffer0 : _buffer1;
tcrc32.Trigger(buffer, sizebuffer);
tmd5.Trigger(buffer, sizebuffer);
tsha1.Trigger(buffer, sizebuffer);
if (sizeNext > 0)
{
lbuffer.Wait();
}
tcrc32.Wait();
tmd5.Wait();
tsha1.Wait();
sizebuffer = sizeNext;
sizetogo -= (ulong)sizeNext;
whichBuffer = !whichBuffer;
}
if (lbuffer.errorState)
{
lbuffer.Dispose();
tcrc32.Dispose();
tmd5.Dispose();
tsha1.Dispose();
_localFiles[index].FileStatus = ZipReturn.ZipDecodeError;
continue;
}
lbuffer.Finish();
tcrc32.Finish();
tmd5.Finish();
tsha1.Finish();
byte[] testcrc = tcrc32.Hash;
_localFiles[index].md5 = tmd5.Hash;
_localFiles[index].sha1 = tsha1.Hash;
lbuffer.Dispose();
tcrc32.Dispose();
tmd5.Dispose();
tsha1.Dispose();
_localFiles[index].FileStatus = Util.ByteArrCompare(_localFiles[index].crc, testcrc)
? ZipReturn.ZipGood
: ZipReturn.ZipCRCDecodeError;
}
}
private static bool ScanAFile(string realFilename, Stream memzip, string displayFilename)
{
Stream fStream;
if (string.IsNullOrEmpty(realFilename) && memzip != null)
{
fStream = memzip;
}
else
{
int errorCode = FileStream.OpenFileRead(realFilename, out fStream);
if (errorCode != 0)
{
return(false);
}
}
bool ret = false;
HeaderFileType foundFileType = FileHeaderReader.FileHeaderReader.GetType(fStream, out int offset);
fStream.Position = 0;
RvFile tFile = UnCompFiles.CheckSumRead(fStream, offset);
tFile.AltType = foundFileType;
// CHDs are handled like regular files
/*
* if (foundFileType == HeaderFileType.CHD)
* {
* // read altheader values from CHD file.
* }
*/
// test if needed.
FindStatus res = RvRomFileMatchup.FileneededTest(tFile);
if (res == FindStatus.FileNeededInArchive)
{
_bgw?.ReportProgress(0, new bgwShowError(displayFilename, "found"));
Debug.WriteLine("Reading file as " + tFile.SHA1);
GZip gz = new GZip(tFile);
string outfile = RomRootDir.GetFilename(tFile.SHA1, false);
fStream.Position = 0;
gz.WriteGZip(outfile, fStream, false);
tFile.CompressedSize = gz.compressedSize;
tFile.DBWrite();
ret = true;
}
else if (res == FindStatus.FoundFileInArchive)
{
ret = true;
}
if (foundFileType == HeaderFileType.ZIP || foundFileType == HeaderFileType.SevenZip || foundFileType == HeaderFileType.GZ)
{
ICompress fz;
switch (foundFileType)
{
case HeaderFileType.SevenZip:
fz = new SevenZ();
break;
case HeaderFileType.GZ:
fz = new gZip();
break;
//case HeaderFileType.ZIP:
default:
fz = new ZipFile();
break;
}
fStream.Position = 0;
ZipReturn zp;
if (string.IsNullOrEmpty(realFilename) && memzip != null)
{
zp = fz.ZipFileOpen(memzip);
}
else
{
zp = fz.ZipFileOpen(realFilename);
}
if (zp == ZipReturn.ZipGood)
{
bool allZipFound = true;
for (int i = 0; i < fz.LocalFilesCount(); i++)
{
ZipReturn openFile = fz.ZipFileOpenReadStream(i, out Stream stream, out ulong streamSize);
if (streamSize <= _inMemorySize)
{
if (openFile == ZipReturn.ZipTryingToAccessADirectory)
{
continue;
}
byte[] tmpFile = new byte[streamSize];
stream.Read(tmpFile, 0, (int)streamSize);
using (Stream memStream = new MemoryStream(tmpFile, false))
{
allZipFound &= ScanAFile(null, memStream, fz.Filename(i));
}
}
else
{
string file = Path.Combine(_tmpDir, Guid.NewGuid().ToString());
FileStream.OpenFileWrite(file, out Stream fs);
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > (ulong)Buffersize ? Buffersize : (int)sizetogo;
stream.Read(Buffer, 0, sizenow);
fs.Write(Buffer, 0, sizenow);
sizetogo -= (ulong)sizenow;
}
fs.Close();
allZipFound &= ScanAFile(file, null, fz.Filename(i));
File.Delete(file);
}
//fz.ZipFileCloseReadStream();
}
fz.ZipFileClose();
ret |= allZipFound;
}
else
{
ret = false;
}
}
if (!string.IsNullOrEmpty(realFilename) || memzip == null)
{
fStream.Close();
fStream.Dispose();
}
return(ret);
}
/// <summary>
/// Write an input stream to a torrentzip archive
/// </summary>
/// <param name="inputStream">Input filename to be moved</param>
/// <param name="outDir">Output directory to build to</param>
/// <param name="rom">DatItem representing the new information</param>
/// <returns>True if the archive was written properly, false otherwise</returns>
public override bool Write(Stream inputStream, string outDir, Rom rom)
{
bool success = false;
string tempFile = Path.Combine(outDir, $"tmp{Guid.NewGuid()}");
// If either input is null or empty, return
if (inputStream == null || rom == null || rom.Name == null)
{
return(success);
}
// If the stream is not readable, return
if (!inputStream.CanRead)
{
return(success);
}
// Seek to the beginning of the stream
inputStream.Seek(0, SeekOrigin.Begin);
// Get the output archive name from the first rebuild rom
string archiveFileName = Path.Combine(outDir, Sanitizer.RemovePathUnsafeCharacters(rom.Machine.Name) + (rom.Machine.Name.EndsWith(".zip") ? string.Empty : ".zip"));
// Set internal variables
Stream writeStream = null;
ZipFile oldZipFile = new ZipFile();
ZipFile zipFile = new ZipFile();
ZipReturn zipReturn = ZipReturn.ZipGood;
try
{
// If the full output path doesn't exist, create it
if (!Directory.Exists(Path.GetDirectoryName(archiveFileName)))
{
Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName));
}
// If the archive doesn't exist, create it and put the single file
if (!File.Exists(archiveFileName))
{
inputStream.Seek(0, SeekOrigin.Begin);
zipReturn = zipFile.ZipFileCreate(tempFile);
// Open the input file for reading
ulong istreamSize = (ulong)(inputStream.Length);
DateTime dt = DateTime.Now;
if (UseDates && !string.IsNullOrWhiteSpace(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out dt))
{
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.ZipFileOpenWriteStream(false, false, rom.Name.Replace('\\', '/'), istreamSize, (ushort)CompressionMethod.Deflated, msDosDateTime, out writeStream);
}
else
{
zipFile.ZipFileOpenWriteStream(false, true, rom.Name.Replace('\\', '/'), istreamSize, (ushort)CompressionMethod.Deflated, null, out writeStream);
}
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
zipFile.ZipFileCloseWriteStream(Utilities.StringToByteArray(rom.CRC));
}
// Otherwise, sort the input files and write out in the correct order
else
{
// Open the old archive for reading
oldZipFile.ZipFileOpen(archiveFileName, -1, true);
// Map all inputs to index
Dictionary <string, int> inputIndexMap = new Dictionary <string, int>();
var oldZipFileContents = new List <string>();
for (int i = 0; i < oldZipFile.LocalFilesCount(); i++)
{
oldZipFileContents.Add(oldZipFile.Filename(i));
}
// If the old one doesn't contain the new file, then add it
if (!oldZipFileContents.Contains(rom.Name.Replace('\\', '/')))
{
inputIndexMap.Add(rom.Name.Replace('\\', '/'), -1);
}
// Then add all of the old entries to it too
for (int i = 0; i < oldZipFile.LocalFilesCount(); i++)
{
inputIndexMap.Add(oldZipFile.Filename(i), i);
}
// If the number of entries is the same as the old archive, skip out
if (inputIndexMap.Keys.Count <= oldZipFile.LocalFilesCount())
{
success = true;
return(success);
}
// Otherwise, process the old zipfile
zipFile.ZipFileCreate(tempFile);
// Get the order for the entries with the new file
List <string> keys = inputIndexMap.Keys.ToList();
keys.Sort(ZipFile.TrrntZipStringCompare);
// Copy over all files to the new archive
foreach (string key in keys)
{
// Get the index mapped to the key
int index = inputIndexMap[key];
// If we have the input file, add it now
if (index < 0)
{
// Open the input file for reading
ulong istreamSize = (ulong)(inputStream.Length);
DateTime dt = DateTime.Now;
if (UseDates && !string.IsNullOrWhiteSpace(rom.Date) && DateTime.TryParse(rom.Date.Replace('\\', '/'), out dt))
{
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.ZipFileOpenWriteStream(false, false, rom.Name.Replace('\\', '/'), istreamSize, (ushort)CompressionMethod.Deflated, msDosDateTime, out writeStream);
}
else
{
zipFile.ZipFileOpenWriteStream(false, true, rom.Name.Replace('\\', '/'), istreamSize, (ushort)CompressionMethod.Deflated, null, out writeStream);
}
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = inputStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
zipFile.ZipFileCloseWriteStream(Utilities.StringToByteArray(rom.CRC));
}
// Otherwise, copy the file from the old archive
else
{
// Instantiate the streams
oldZipFile.ZipFileOpenReadStream(index, false, out Stream zreadStream, out ulong istreamSize, out ushort icompressionMethod);
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(oldZipFile.LastModified(index));
zipFile.ZipFileOpenWriteStream(false, true, oldZipFile.Filename(index), istreamSize, (ushort)CompressionMethod.Deflated, msDosDateTime, out writeStream);
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = zreadStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
oldZipFile.ZipFileCloseReadStream();
zipFile.ZipFileCloseWriteStream(oldZipFile.CRC32(index));
}
}
}
// Close the output zip file
zipFile.ZipFileClose();
oldZipFile.ZipFileClose();
success = true;
}
catch (Exception ex)
{
logger.Error(ex);
success = false;
}
finally
{
}
// If the old file exists, delete it and replace
if (File.Exists(archiveFileName))
{
FileExtensions.TryDelete(archiveFileName);
}
File.Move(tempFile, archiveFileName);
return(true);
}
private static bool ScanAFile(string realFilename, Stream memzip, string displayFilename)
{
Compress.File.File fStream = new Compress.File.File();
if (string.IsNullOrEmpty(realFilename) && memzip != null)
{
fStream.ZipFileOpen(memzip);
}
else
{
ZipReturn zRet = fStream.ZipFileOpen(realFilename, -1, true);
if (zRet != ZipReturn.ZipGood)
{
return(false);
}
}
bool ret = false;
FileScan fScan = new FileScan();
List <FileScan.FileResults> resScan = fScan.Scan(fStream, true, true);
HeaderFileType foundFileType = resScan[0].HeaderFileType;
if (foundFileType == HeaderFileType.CHD)
{
// read altheader values from CHD file.
}
RvFile tFile = new RvFile
{
Size = resScan[0].Size,
CRC = resScan[0].CRC,
MD5 = resScan[0].MD5,
SHA1 = resScan[0].SHA1,
AltType = resScan[0].HeaderFileType,
AltSize = resScan[0].AltSize,
AltCRC = resScan[0].AltCRC,
AltMD5 = resScan[0].AltMD5,
AltSHA1 = resScan[0].AltSHA1
};
// test if needed.
FindStatus res = RvRomFileMatchup.FileneededTest(tFile);
if (res == FindStatus.FileNeededInArchive)
{
_bgw?.ReportProgress(0, new bgwShowError(displayFilename, "found"));
Debug.WriteLine("Reading file as " + tFile.SHA1);
string outfile = RomRootDir.Getfilename(tFile.SHA1);
gZip gz1 = new gZip();
gz1.ZipFileCreate(outfile);
gz1.ExtraData = gZipExtraData.SetExtraData(tFile);
gz1.ZipFileOpenWriteStream(false, true, "", tFile.Size, 8, out Stream write, null);
fStream.ZipFileOpenReadStream(0, out Stream s, out ulong _);
// do copy
StreamCopier.StreamCopy(s, write, tFile.Size);
fStream.ZipFileCloseReadStream();
fStream.ZipFileClose();
gz1.ZipFileCloseWriteStream(tFile.CRC);
tFile.CompressedSize = gz1.CompressedSize;
gz1.ZipFileClose();
tFile.DBWrite();
ret = true;
}
else if (res == FindStatus.FoundFileInArchive)
{
ret = true;
}
fStream.ZipFileClose();
if (foundFileType == HeaderFileType.ZIP || foundFileType == HeaderFileType.SevenZip || foundFileType == HeaderFileType.GZ)
{
ICompress fz;
switch (foundFileType)
{
case HeaderFileType.SevenZip:
fz = new SevenZ();
break;
case HeaderFileType.GZ:
fz = new gZip();
break;
//case HeaderFileType.ZIP:
default:
fz = new Zip();
break;
}
ZipReturn zp;
if (string.IsNullOrEmpty(realFilename) && memzip != null)
{
memzip.Position = 0;
zp = fz.ZipFileOpen(memzip);
}
else
{
zp = fz.ZipFileOpen(realFilename);
}
if (zp == ZipReturn.ZipGood)
{
bool allZipFound = true;
for (int i = 0; i < fz.LocalFilesCount(); i++)
{
LocalFile lf = fz.GetLocalFile(i);
ZipReturn openFile = fz.ZipFileOpenReadStream(i, out Stream stream, out ulong streamSize);
if (streamSize <= _inMemorySize)
{
if (openFile == ZipReturn.ZipTryingToAccessADirectory)
{
continue;
}
byte[] tmpFile = new byte[streamSize];
stream.Read(tmpFile, 0, (int)streamSize);
using (Stream memStream = new MemoryStream(tmpFile, false))
{
allZipFound &= ScanAFile(null, memStream, lf.Filename);
}
}
else
{
string file = Path.Combine(_tmpDir, Guid.NewGuid().ToString());
FileStream.OpenFileWrite(file, out Stream fs);
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > (ulong)Buffersize ? Buffersize : (int)sizetogo;
stream.Read(Buffer, 0, sizenow);
fs.Write(Buffer, 0, sizenow);
sizetogo -= (ulong)sizenow;
}
fs.Close();
allZipFound &= ScanAFile(file, null, lf.Filename);
File.Delete(file);
}
//fz.ZipFileCloseReadStream();
}
fz.ZipFileClose();
ret |= allZipFound;
}
else
{
ret = false;
}
}
return(ret);
}
/// <summary>
/// Attempt to extract a stream from an archive
/// </summary>
/// <param name="entryName">Name of the entry to be extracted</param>
/// <param name="realEntry">Output representing the entry name that was found</param>
/// <returns>MemoryStream representing the entry, null on error</returns>
public override (MemoryStream, string) CopyToStream(string entryName)
{
MemoryStream ms = new MemoryStream();
string realEntry = null;
try
{
ZipFile zf = new ZipFile();
ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
if (zr != ZipReturn.ZipGood)
{
throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
for (int i = 0; i < zf.LocalFilesCount() && zr == ZipReturn.ZipGood; i++)
{
if (zf.Filename(i).Contains(entryName))
{
// Open the read stream
realEntry = zf.Filename(i);
zr = zf.ZipFileOpenReadStream(i, false, out Stream readStream, out ulong streamsize, out ushort cm);
// If the stream is smaller than the buffer, just run one loop through to avoid issues
if (streamsize < _bufferSize)
{
byte[] ibuffer = new byte[streamsize];
int ilen = readStream.Read(ibuffer, 0, (int)streamsize);
ms.Write(ibuffer, 0, ilen);
ms.Flush();
}
// Otherwise, we do the normal loop
else
{
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while (streamsize > _bufferSize)
{
ilen = readStream.Read(ibuffer, 0, _bufferSize);
ms.Write(ibuffer, 0, ilen);
ms.Flush();
streamsize -= _bufferSize;
}
ilen = readStream.Read(ibuffer, 0, (int)streamsize);
ms.Write(ibuffer, 0, ilen);
ms.Flush();
}
zr = zf.ZipFileCloseReadStream();
}
}
zf.ZipFileClose();
}
catch (Exception ex)
{
logger.Error(ex);
ms = null;
realEntry = null;
}
return(ms, realEntry);
}
/// <summary>
/// Generate a list of DatItem objects from the header values in an archive
/// </summary>
/// <returns>List of DatItem objects representing the found data</returns>
public override List <BaseFile> GetChildren()
{
List <BaseFile> found = new List <BaseFile>();
string gamename = Path.GetFileNameWithoutExtension(this.Filename);
try
{
ZipFile zf = new ZipFile();
ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
if (zr != ZipReturn.ZipGood)
{
throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
for (int i = 0; i < zf.LocalFilesCount(); i++)
{
// If the entry is a directory (or looks like a directory), we don't want to open it
if (zf.IsDirectory(i) ||
zf.Filename(i).EndsWith(Path.DirectorySeparatorChar.ToString()) ||
zf.Filename(i).EndsWith(Path.AltDirectorySeparatorChar.ToString()) ||
zf.Filename(i).EndsWith(Path.PathSeparator.ToString()))
{
continue;
}
// Open the read stream
zr = zf.ZipFileOpenReadStream(i, false, out Stream readStream, out ulong streamsize, out ushort cm);
// If we get a read error, log it and continue
if (zr != ZipReturn.ZipGood)
{
logger.Warning($"An error occurred while reading archive {this.Filename}: Zip Error - {zr}");
zr = zf.ZipFileCloseReadStream();
continue;
}
// Create a blank item for the entry
BaseFile zipEntryRom = new BaseFile();
// Perform a quickscan, if flagged to
if (this.AvailableHashes == Hash.CRC)
{
zipEntryRom.Size = (long)zf.UncompressedSize(i);
zipEntryRom.CRC = zf.CRC32(i);
}
// Otherwise, use the stream directly
else
{
zipEntryRom = readStream.GetInfo(size: (long)zf.UncompressedSize(i), hashes: this.AvailableHashes, keepReadOpen: true);
}
// Fill in comon details and add to the list
zipEntryRom.Filename = zf.Filename(i);
zipEntryRom.Parent = gamename;
zipEntryRom.Date = zf.LastModified(i).ToString("yyyy/MM/dd hh:mm:ss");
found.Add(zipEntryRom);
}
// Dispose of the archive
zr = zf.ZipFileCloseReadStream();
zf.ZipFileClose();
}
catch (Exception ex)
{
logger.Error(ex);
return(null);
}
return(found);
}
public bool FullExtract(string filename, string outDir)
{
MessageCallBack?.Invoke($"Processing file: {filename}");
if (!string.IsNullOrEmpty(outDir))
{
MessageCallBack?.Invoke($"Output dir: {outDir}");
}
string ext = Path.GetExtension(filename);
ICompress z = null;
switch (ext.ToLower())
{
case ".zip":
z = new Zip();
break;
case ".7z":
z = new SevenZ();
break;
}
if (z == null)
{
MessageCallBack?.Invoke($"Unknown file type {ext}");
return(false);
}
ZipReturn zRet = z.ZipFileOpen(filename);
if (zRet != ZipReturn.ZipGood)
{
MessageCallBack?.Invoke($"Error opening archive {zRet}");
return(false);
}
ulong buflen = 409600;
byte[] buffer = new byte[buflen];
for (int i = 0; i < z.LocalFilesCount(); i++)
{
LocalFile lf = z.GetLocalFile(i);
byte[] cread = null;
string filenameOut = lf.Filename;
if (lf.IsDirectory)
{
string outFullDir = Path.Combine(outDir, filenameOut.Substring(0, filenameOut.Length - 1).Replace('/', '\\'));
Directory.CreateDirectory(outFullDir);
continue;
}
else
{
MessageCallBack?.Invoke($"Extracting {filenameOut}");
string fOut = Path.Combine(outDir, filenameOut.Replace('/', '\\'));
string dOut = Path.GetDirectoryName(fOut);
if (!string.IsNullOrWhiteSpace(dOut) && !Directory.Exists(dOut))
{
Directory.CreateDirectory(dOut);
}
int errorCode = FileStream.OpenFileWrite(fOut, out Stream sWrite);
if (errorCode != 0)
{
MessageCallBack?.Invoke($"Error opening outputfile {fOut}");
}
z.ZipFileOpenReadStream(i, out Stream sRead, out _);
CRC crc = new();
ulong sizeToGo = lf.UncompressedSize;
while (sizeToGo > 0)
{
ulong sizeNow = sizeToGo > buflen ? buflen : sizeToGo;
int sizeRead = sRead.Read(buffer, 0, (int)sizeNow);
crc.SlurpBlock(buffer, 0, sizeRead);
sWrite.Write(buffer, 0, sizeRead);
sizeToGo -= (ulong)sizeRead;
}
sWrite.Close();
sWrite.Dispose();
cread = crc.Crc32ResultB;
}
byte[] fread = lf.CRC;
if (cread[0] != fread[0] || cread[1] != fread[1] || cread[2] != fread[2] || cread[3] != fread[3])
{
MessageCallBack?.Invoke($"CRC error. Expected {fread.ToHex()} found {cread.ToHex()}");
return(false);
}
}
return(true);
}
private static void AddZip(FileInfo f, DatDir thisDir)
{
ZipFile zf1 = new ZipFile();
ZipReturn result = zf1.ZipFileOpen(f.FullName, -1, true);
if (result != ZipReturn.ZipGood)
{
return;
}
//zf1.ZipStatus = ZipStatus.TrrntZip;
DatDir ZipDir = new DatDir(zf1.ZipStatus == ZipStatus.TrrntZip ? DatFileType.DirTorrentZip : DatFileType.DirRVZip)
{
Name = Path.GetFileNameWithoutExtension(f.Name),
DGame = new DatGame()
};
ZipDir.DGame.Description = ZipDir.Name;
thisDir.ChildAdd(ZipDir);
FileScan fs = new FileScan();
List <FileScan.FileResults> fr = fs.Scan(zf1, true, true);
bool isTorrentZipDate = true;
for (int i = 0; i < fr.Count; i++)
{
if (fr[i].FileStatus != ZipReturn.ZipGood)
{
Console.WriteLine("File Error :" + zf1.Filename(i) + " : " + fr[i].FileStatus);
continue;
}
DatFile df = new DatFile(DatFileType.FileTorrentZip)
{
Name = zf1.Filename(i),
Size = fr[i].Size,
CRC = fr[i].CRC,
SHA1 = fr[i].SHA1,
Date = zf1.LastModified(i).ToString("yyyy/MM/dd HH:mm:ss")
//df.MD5 = zf.MD5(i)
};
if (zf1.LastModified(i).Ticks != 629870671200000000)
{
isTorrentZipDate = false;
}
ZipDir.ChildAdd(df);
}
zf1.ZipFileClose();
if (isTorrentZipDate && ZipDir.DatFileType == DatFileType.DirRVZip)
{
ZipDir.DatFileType = DatFileType.DirTorrentZip;
}
if (ZipDir.DatFileType == DatFileType.DirTorrentZip)
{
DatSetCompressionType.SetZip(ZipDir);
DatClean.RemoveUnNeededDirectoriesFromZip(ZipDir);
}
}
/*
* raw is true if we are just going to copy the raw data stream from the source to the destination zip file
* trrntzip is thue if the source zip is a valid trrntzip file
* compressionMethod must be set to 8 to make a valid trrntzip file.
*
* if raw is false then compressionMthod must be 0,8 or 93 (zstd)
*/
public ZipReturn ZipFileOpenWriteStream(bool raw, bool trrntzip, string filename, ulong uncompressedSize, ushort compressionMethod, out Stream stream, TimeStamps timeStamp = null)
{
stream = null;
if (ZipOpen != ZipOpenType.OpenWrite)
{
return(ZipReturn.ZipWritingToInputFile);
}
ZipReturn validTrrntzip = ZipReturn.ZipGood;
//invalid torrentZip Input If:
if (compressionMethod != 8)
{
validTrrntzip = ZipReturn.ZipTrrntzipIncorrectCompressionUsed;
}
if (raw && !trrntzip)
{
validTrrntzip = ZipReturn.ZipTrrntZipIncorrectDataStream;
}
int localFilesCount = _localFiles.Count;
if (localFilesCount > 0)
{
// check that filenames are in trrntzip order
string lastFilename = _localFiles[localFilesCount - 1].Filename;
if (CompressUtils.TrrntZipStringCompare(lastFilename, filename) > 0)
{
validTrrntzip = ZipReturn.ZipTrrntzipIncorrectFileOrder;
}
// check that no un-needed directory entries are added
if (_localFiles[localFilesCount - 1].IsDirectory && filename.Length > lastFilename.Length)
{
if (CompressUtils.TrrntZipStringCompare(lastFilename, filename.Substring(0, lastFilename.Length)) == 0)
{
validTrrntzip = ZipReturn.ZipTrrntzipIncorrectDirectoryAddedToZip;
}
}
}
// if we are requirering a trrrntzp file and it is not a trrntzip formated supplied stream then error out
if (writeZipType == OutputZipType.TrrntZip)
{
if (validTrrntzip != ZipReturn.ZipGood)
{
return(validTrrntzip);
}
}
ZipLocalFile lf = new ZipLocalFile(filename, timeStamp);
lf.SetStatus(LocalFileStatus.TrrntZip, validTrrntzip == ZipReturn.ZipGood);
ZipReturn retVal = lf.LocalFileOpenWriteStream(_zipFs, raw, uncompressedSize, compressionMethod, out stream);
if (retVal != ZipReturn.ZipGood)
{
return(retVal);
}
if (filename.Length > 0)
{
lf.IsDirectory = (filename.Substring(filename.Length - 1, 1) == "/");
}
_compressionStream = stream;
_localFiles.Add(lf);
return(ZipReturn.ZipGood);
}
/// <summary>
/// Write a set of input files to a torrent7z archive (assuming the same output archive name)
/// </summary>
/// <param name="inputFiles">Input files to be moved</param>
/// <param name="outDir">Output directory to build to</param>
/// <param name="rom">DatItem representing the new information</param>
/// <returns>True if the archive was written properly, false otherwise</returns>
public override bool Write(List <string> inputFiles, string outDir, List <Rom> roms)
{
bool success = false;
string tempFile = Path.Combine(outDir, $"tmp{Guid.NewGuid()}");
// If either list of roms is null or empty, return
if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0)
{
return(success);
}
// If the number of inputs is less than the number of available roms, return
if (inputFiles.Count < roms.Count)
{
return(success);
}
// If one of the files doesn't exist, return
foreach (string file in inputFiles)
{
if (!File.Exists(file))
{
return(success);
}
}
// Get the output archive name from the first rebuild rom
string archiveFileName = Path.Combine(outDir, Sanitizer.RemovePathUnsafeCharacters(roms[0].Machine.Name) + (roms[0].Machine.Name.EndsWith(".7z") ? string.Empty : ".7z"));
// Set internal variables
Stream writeStream = null;
SevenZ oldZipFile = new SevenZ();
SevenZ zipFile = new SevenZ();
ZipReturn zipReturn = ZipReturn.ZipGood;
try
{
// If the full output path doesn't exist, create it
if (!Directory.Exists(Path.GetDirectoryName(archiveFileName)))
{
Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName));
}
// If the archive doesn't exist, create it and put the single file
if (!File.Exists(archiveFileName))
{
zipReturn = zipFile.ZipFileCreate(tempFile);
// Map all inputs to index
Dictionary <string, int> inputIndexMap = new Dictionary <string, int>();
for (int i = 0; i < inputFiles.Count; i++)
{
inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i);
}
// Sort the keys in TZIP order
List <string> keys = inputIndexMap.Keys.ToList();
keys.Sort(ZipFile.TrrntZipStringCompare);
// Now add all of the files in order
foreach (string key in keys)
{
// Get the index mapped to the key
int index = inputIndexMap[key];
// Open the input file for reading
Stream freadStream = FileExtensions.TryOpenRead(inputFiles[index]);
ulong istreamSize = (ulong)(new FileInfo(inputFiles[index]).Length);
DateTime dt = DateTime.Now;
if (UseDates && !string.IsNullOrWhiteSpace(roms[index].Date) && DateTime.TryParse(roms[index].Date.Replace('\\', '/'), out dt))
{
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.ZipFileOpenWriteStream(false, false, roms[index].Name.Replace('\\', '/'), istreamSize, 0, msDosDateTime, out writeStream);
}
else
{
zipFile.ZipFileOpenWriteStream(false, true, roms[index].Name.Replace('\\', '/'), istreamSize, 0, null, out writeStream);
}
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
freadStream.Dispose();
zipFile.ZipFileCloseWriteStream(Utilities.StringToByteArray(roms[index].CRC));
}
}
// Otherwise, sort the input files and write out in the correct order
else
{
// Open the old archive for reading
oldZipFile.ZipFileOpen(archiveFileName, -1, true);
// Map all inputs to index
Dictionary <string, int> inputIndexMap = new Dictionary <string, int>();
for (int i = 0; i < inputFiles.Count; i++)
{
var oldZipFileContents = new List <string>();
for (int j = 0; j < oldZipFile.LocalFilesCount(); j++)
{
oldZipFileContents.Add(oldZipFile.Filename(j));
}
// If the old one contains the new file, then just skip out
if (oldZipFileContents.Contains(roms[i].Name.Replace('\\', '/')))
{
continue;
}
inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1));
}
// Then add all of the old entries to it too
for (int i = 0; i < oldZipFile.LocalFilesCount(); i++)
{
inputIndexMap.Add(oldZipFile.Filename(i), i);
}
// If the number of entries is the same as the old archive, skip out
if (inputIndexMap.Keys.Count <= oldZipFile.LocalFilesCount())
{
success = true;
return(success);
}
// Otherwise, process the old zipfile
zipFile.ZipFileCreate(tempFile);
// Get the order for the entries with the new file
List <string> keys = inputIndexMap.Keys.ToList();
keys.Sort(ZipFile.TrrntZipStringCompare);
// Copy over all files to the new archive
foreach (string key in keys)
{
// Get the index mapped to the key
int index = inputIndexMap[key];
// If we have the input file, add it now
if (index < 0)
{
// Open the input file for reading
Stream freadStream = FileExtensions.TryOpenRead(inputFiles[-index - 1]);
ulong istreamSize = (ulong)(new FileInfo(inputFiles[-index - 1]).Length);
DateTime dt = DateTime.Now;
if (UseDates && !string.IsNullOrWhiteSpace(roms[-index - 1].Date) && DateTime.TryParse(roms[-index - 1].Date.Replace('\\', '/'), out dt))
{
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.ZipFileOpenWriteStream(false, false, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, 0, msDosDateTime, out writeStream);
}
else
{
zipFile.ZipFileOpenWriteStream(false, true, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, 0, null, out writeStream);
}
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
freadStream.Dispose();
zipFile.ZipFileCloseWriteStream(Utilities.StringToByteArray(roms[-index - 1].CRC));
}
// Otherwise, copy the file from the old archive
else
{
// Instantiate the streams
oldZipFile.ZipFileOpenReadStream(index, out Stream zreadStream, out ulong istreamSize);
zipFile.ZipFileOpenWriteStream(false, true, oldZipFile.Filename(index), istreamSize, 0, null, out writeStream);
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = zreadStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
zipFile.ZipFileCloseWriteStream(oldZipFile.CRC32(index));
}
}
}
// Close the output zip file
zipFile.ZipFileClose();
oldZipFile.ZipFileClose();
success = true;
}
catch (Exception ex)
{
logger.Error(ex);
success = false;
}
// If the old file exists, delete it and replace
if (File.Exists(archiveFileName))
{
FileExtensions.TryDelete(archiveFileName);
}
File.Move(tempFile, archiveFileName);
return(true);
}
public static TrrntZipStatus ReZipFiles(List <ZippedFile> zippedFiles, ICompress originalZipFile, byte[] buffer, StatusCallback statusCallBack, LogCallback logCallback, int threadId)
{
zipType inputType;
switch (originalZipFile)
{
case Zip _:
inputType = zipType.zip;
break;
case SevenZ _:
inputType = zipType.sevenzip;
break;
case Compress.File.File _:
inputType = zipType.iso;
break;
default:
return(TrrntZipStatus.Unknown);
}
zipType outputType = Program.OutZip == zipType.both ? inputType : Program.OutZip;
if (outputType == zipType.iso)
{
outputType = zipType.zip;
}
int bufferSize = buffer.Length;
string filename = originalZipFile.ZipFilename;
string tmpFilename = Path.Combine(Path.GetDirectoryName(filename), Path.GetFileNameWithoutExtension(filename) + ".tmp");
string outExt = outputType == zipType.zip ? ".zip" : ".7z";
string outfilename = Path.Combine(Path.GetDirectoryName(filename), Path.GetFileNameWithoutExtension(filename) + outExt);
if (inputType != outputType)
{
if (File.Exists(outfilename))
{
logCallback?.Invoke(threadId, "Error output " + outExt + " file already exists");
return(TrrntZipStatus.RepeatFilesFound);
}
}
if (File.Exists(tmpFilename))
{
File.Delete(tmpFilename);
}
ICompress zipFileOut = outputType == zipType.zip ? new Zip() : (ICompress) new SevenZ();
try
{
zipFileOut.ZipFileCreate(tmpFilename);
ulong fileSizeTotal = 0;
ulong fileSizeProgress = 0;
int filePercentReported = 20;
foreach (ZippedFile f in zippedFiles)
{
fileSizeTotal += f.Size;
}
// by now the zippedFiles have been sorted so just loop over them
foreach (ZippedFile t in zippedFiles)
{
if (Program.VerboseLogging)
{
logCallback?.Invoke(threadId, $"{t.Size,15} {t.StringCRC} {t.Name}");
}
Stream readStream = null;
ulong streamSize = 0;
ZipReturn zrInput = ZipReturn.ZipUntested;
if (t.Size > 0)
{
switch (originalZipFile)
{
case Zip z:
zrInput = z.ZipFileOpenReadStream(t.Index, false, out readStream, out streamSize, out ushort _);
break;
case SevenZ z7:
zrInput = z7.ZipFileOpenReadStream(t.Index, out readStream, out streamSize);
break;
case Compress.File.File zf:
zrInput = zf.ZipFileOpenReadStream(t.Index, out readStream, out streamSize);
break;
}
}
else
{
// do nothing for a zero size file, the stream will not be used.
zrInput = ZipReturn.ZipGood;
}
ZipReturn zrOutput = zipFileOut.ZipFileOpenWriteStream(false, true, t.Name, streamSize, 8, out Stream writeStream);
if ((zrInput != ZipReturn.ZipGood) || (zrOutput != ZipReturn.ZipGood))
{
//Error writing local File.
zipFileOut.ZipFileCloseFailed();
originalZipFile.ZipFileClose();
File.Delete(tmpFilename);
return(TrrntZipStatus.CorruptZip);
}
Stream crcCs = new CrcCalculatorStream(readStream, true);
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > (ulong)bufferSize ? bufferSize : (int)sizetogo;
fileSizeProgress += (ulong)sizenow;
int filePercent = (int)((double)fileSizeProgress / fileSizeTotal * 20);
if (filePercent != filePercentReported)
{
statusCallBack?.Invoke(threadId, filePercent * 5);
filePercentReported = filePercent;
}
crcCs.Read(buffer, 0, sizenow);
writeStream.Write(buffer, 0, sizenow);
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
crcCs.Close();
if (inputType != zipType.sevenzip)
{
originalZipFile.ZipFileCloseReadStream();
}
uint crc = (uint)((CrcCalculatorStream)crcCs).Crc;
if (t.CRC == null)
{
t.CRC = crc;
}
if (crc != t.CRC)
{
return(TrrntZipStatus.CorruptZip);
}
zipFileOut.ZipFileCloseWriteStream(t.ByteCRC);
}
statusCallBack?.Invoke(threadId, 100);
zipFileOut.ZipFileClose();
originalZipFile.ZipFileClose();
File.Delete(filename);
File.Move(tmpFilename, outfilename);
return(TrrntZipStatus.Trrntzipped);
}
catch (Exception)
{
zipFileOut?.ZipFileCloseFailed();
originalZipFile?.ZipFileClose();
return(TrrntZipStatus.CorruptZip);
}
}
public bgwShowCorrupt(ZipReturn zr, string filename)
{
this.zr = zr;
this.filename = filename;
}
/// <summary>
/// Get the data from the current file, if not already checked
/// </summary>
public void Check()
{
// If the file has been tested or has an error, return
if (_fileStatus != ZipReturn.ZipUntested)
{
return;
}
try
{
Stream stream = null;
_zipstream.Seek((long)_dataLocation, SeekOrigin.Begin);
switch (_compressionMethod)
{
case CompressionMethod.Deflated:
stream = new DeflateStream(_zipstream, CompressionMode.Decompress, true);
break;
case CompressionMethod.Stored:
stream = _zipstream;
break;
}
if (stream == null)
{
_fileStatus = ZipReturn.ZipErrorGettingDataStream;
return;
}
// Create the hashers
uint tempCrc;
OptimizedCRC crc = new OptimizedCRC();
MD5 md5 = System.Security.Cryptography.MD5.Create();
SHA1 sha1 = System.Security.Cryptography.SHA1.Create();
// Now get the hash of the stream
BinaryReader fs = new BinaryReader(stream);
byte[] buffer = new byte[1024];
int read;
while ((read = fs.Read(buffer, 0, buffer.Length)) > 0)
{
crc.Update(buffer, 0, read);
md5.TransformBlock(buffer, 0, read, buffer, 0);
sha1.TransformBlock(buffer, 0, read, buffer, 0);
}
crc.Update(buffer, 0, 0);
md5.TransformFinalBlock(buffer, 0, 0);
sha1.TransformFinalBlock(buffer, 0, 0);
tempCrc = crc.UnsignedValue;
_md5 = md5.Hash;
_sha1 = sha1.Hash;
// Dispose of the hashers
crc.Dispose();
md5.Dispose();
sha1.Dispose();
if (_compressionMethod == CompressionMethod.Deflated)
{
stream.Close();
stream.Dispose();
}
_fileStatus = (_crc == tempCrc ? ZipReturn.ZipGood : ZipReturn.ZipCRCDecodeError);
}
catch
{
_fileStatus = ZipReturn.ZipDecodeError;
}
}
private static void AddZip(FileInfo f, DatDir thisDir)
{
Zip zf1 = new Zip();
ZipReturn result = zf1.ZipFileOpen(f.FullName, -1, true);
if (result != ZipReturn.ZipGood)
{
return;
}
zCount += 1;
if ((zf1.ZipStatus & ZipStatus.TrrntZip) == ZipStatus.TrrntZip)
{
tCount += 1;
Console.WriteLine($"{zCount} {tCount} {cCount}");
}
else if (zf1.Filecomment != null && zf1.Filecomment.Length > 0)
{
string comments = ZipUtils.GetString(zf1.Filecomment);
if (comments.Length > 13 && comments.Substring(0, 13) == "TORRENTZIPPED")
{
tCount += 1;
}
else
{
cCount += 1;
Console.WriteLine(f.FullName + " " + zCount);
Console.WriteLine("------------------------");
Console.WriteLine(comments);
}
Console.WriteLine($"{zCount} {tCount} {cCount}");
}
//zf1.ZipStatus = ZipStatus.TrrntZip;
//DatDir ZipDir = new DatDir(zf1.ZipStatus == ZipStatus.TrrntZip ? DatFileType.DirTorrentZip : DatFileType.DirRVZip)
DatDir ZipDir = new DatDir(DatFileType.UnSet)
{
Name = Path.GetFileNameWithoutExtension(f.Name),
DGame = new DatGame()
};
ZipDir.DGame.Description = ZipDir.Name;
thisDir.ChildAdd(ZipDir);
FileScan fs = new FileScan();
List <FileScan.FileResults> fr = fs.Scan(zf1, !quick, !quick);
bool isTorrentZipDate = true;
for (int i = 0; i < fr.Count; i++)
{
if (fr[i].FileStatus != ZipReturn.ZipGood)
{
Console.WriteLine("File Error :" + zf1.Filename(i) + " : " + fr[i].FileStatus);
continue;
}
DatFile df = new DatFile(DatFileType.UnSet)
{
Name = zf1.Filename(i),
Size = fr[i].Size,
CRC = fr[i].CRC,
SHA1 = fr[i].SHA1,
DateModified = new DateTime(zf1.LastModified(i)).ToString("yyyy/MM/dd HH:mm:ss"),
DateCreated = zf1.Created(i) == null ? null : new DateTime((long)zf1.Created(i)).ToString("yyyy/MM/dd HH:mm:ss"),
DateAccessed = zf1.Accessed(i) == null ? null : new DateTime((long)zf1.Accessed(i)).ToString("yyyy/MM/dd HH:mm:ss")
//df.MD5 = zf.MD5(i)
};
if (zf1.LastModified(i) != 629870671200000000)
{
isTorrentZipDate = false;
}
ZipDir.ChildAdd(df);
}
zf1.ZipFileClose();
if (isTorrentZipDate && ZipDir.DatFileType == DatFileType.DirRVZip)
{
ZipDir.DatFileType = DatFileType.DirTorrentZip;
}
if (ZipDir.DatFileType == DatFileType.DirTorrentZip)
{
DatSetCompressionType.SetZip(ZipDir);
DatClean.RemoveUnNeededDirectoriesFromZip(ZipDir);
}
}
private static void CheckADir(RvDir dbDir, bool report)
{
if (_cacheSaveTimer.Elapsed.TotalMinutes > Settings.CacheSaveTimePeriod)
{
_bgw.ReportProgress(0, "Saving Cache");
DB.Write();
_bgw.ReportProgress(0, "Saving Cache Complete");
_cacheSaveTimer.Reset();
_cacheSaveTimer.Start();
}
string fullDir = dbDir.FullName;
if (report)
{
_bgw.ReportProgress(0, new bgwText2(fullDir));
}
DatStatus chechingDatStatus = dbDir.IsInUnsorted ? DatStatus.InUnsorted : DatStatus.NotInDat;
// this is a temporary rvDir structure to store the data about the actual directory/files we are scanning
// we will first populate this variable with the real file data from the directory, and then compare it
// with the data in dbDir.
RvDir fileDir = null;
Debug.WriteLine(fullDir);
FileType ft = dbDir.FileType;
#region "Populate fileDir"
// if we are scanning a ZIP file then populate scanDir from the ZIP file
switch (ft)
{
case FileType.Zip:
{
fileDir = new RvDir(ft);
// open the zip file
ZipFile checkZ = new ZipFile();
ZipReturn zr = checkZ.ZipFileOpen(fullDir, dbDir.TimeStamp, true);
if (zr == ZipReturn.ZipGood)
{
dbDir.ZipStatus = checkZ.ZipStatus;
// to be Scanning a ZIP file means it is either new or has changed.
// as the code below only calls back here if that is true.
//
// Level1: Only use header CRC's
// Just get the CRC for the ZIP headers.
//
// Level2: Fully checksum changed only files
// We know this file has been changed to do a full checksum scan.
//
// Level3: Fully checksum everything
// So do a full checksum scan.
if (EScanLevel == eScanLevel.Level2 || EScanLevel == eScanLevel.Level3)
{
checkZ.DeepScan();
}
// add all of the file information from the zip file into scanDir
for (int i = 0; i < checkZ.LocalFilesCount(); i++)
{
RvFile tFile = new RvFile(DBTypeGet.FileFromDir(ft))
{
Name = checkZ.Filename(i),
ZipFileIndex = i,
ZipFileHeaderPosition = checkZ.LocalHeader(i),
Size = checkZ.UncompressedSize(i),
CRC = checkZ.CRC32(i)
};
// all 3 levels read the CRC from the ZIP header
tFile.SetStatus(chechingDatStatus, GotStatus.Got);
tFile.FileStatusSet(FileStatus.SizeFromHeader | FileStatus.CRCFromHeader);
// if we are in level 2 or level 3 then do a full CheckSum Scan.
if (EScanLevel == eScanLevel.Level2 || EScanLevel == eScanLevel.Level3)
{
// DeepScan will return ZipReturn.ZipCRCDecodeError if the headers CRC and
// the actual CRC do not match.
// So we just need to set the MD5 and SHA1 from the ZIP file.
zr = checkZ.FileStatus(i);
if (zr == ZipReturn.ZipUntested)
{
_bgw.ReportProgress(0, new bgwShowCorrupt(zr, fullDir + " : " + checkZ.Filename(i)));
}
else if (zr != ZipReturn.ZipGood)
{
_bgw.ReportProgress(0, new bgwShowCorrupt(zr, fullDir + " : " + checkZ.Filename(i)));
tFile.GotStatus = GotStatus.Corrupt;
}
else
{
tFile.MD5 = checkZ.MD5(i);
tFile.SHA1 = checkZ.SHA1(i);
tFile.FileStatusSet(FileStatus.SizeVerified | FileStatus.CRCVerified | FileStatus.SHA1Verified | FileStatus.MD5Verified);
}
}
fileDir.ChildAdd(tFile);
}
}
else if (zr == ZipReturn.ZipFileLocked)
{
_bgw.ReportProgress(0, new bgwShowError(fullDir, "Zip File Locked"));
dbDir.TimeStamp = 0;
dbDir.GotStatus = GotStatus.FileLocked;
}
else
{
_bgw.ReportProgress(0, new bgwShowCorrupt(zr, fullDir));
dbDir.GotStatus = GotStatus.Corrupt;
}
checkZ.ZipFileClose();
}
break;
case FileType.Dir:
{
fileDir = new RvDir(FileType.Dir);
DirectoryInfo oDir = new DirectoryInfo(fullDir);
DirectoryInfo[] oDirs = oDir.GetDirectories();
FileInfo[] oFiles = oDir.GetFiles();
// add all the subdirectories into scanDir
foreach (DirectoryInfo dir in oDirs)
{
RvBase tDir = new RvDir(FileType.Dir)
{
Name = dir.Name,
TimeStamp = dir.LastWriteTime,
};
tDir.SetStatus(chechingDatStatus, GotStatus.Got);
fileDir.ChildAdd(tDir);
}
// add all the files into scanDir
foreach (FileInfo oFile in oFiles)
{
// if we find any zip files add them as zip files.
string fExt = Path.GetExtension(oFile.Name);
switch (fExt.ToLower())
{
case ".zip":
{
RvDir tGame = new RvDir(FileType.Zip)
{
Name = Path.GetFileNameWithoutExtension(oFile.Name),
TimeStamp = oFile.LastWriteTime,
};
tGame.SetStatus(chechingDatStatus, GotStatus.Got);
fileDir.ChildAdd(tGame);
}
break;
default:
{
string fName = oFile.Name;
if (fName == "__ROMVault.tmp")
{
File.Delete(oFile.FullName);
continue;
}
// Scanning a file
//
// Level1 & 2 : (are the same for files) Fully checksum changed only files
// Here we are just getting the TimeStamp of the File, and later
// if the TimeStamp was not matched we will have to read the files CRC, MD5 & SHA1
//
// Level3: Fully checksum everything
// Get everything about the file right here so
// read CRC, MD5 & SHA1
// add all the files in the sub-directory to scanDir
RvFile tFile = new RvFile(FileType.File)
{
Name = fName,
Size = (ulong)oFile.Length,
TimeStamp = oFile.LastWriteTime
};
tFile.FileStatusSet(FileStatus.SizeVerified);
int errorCode = CHD.CheckFile(oFile, out tFile.SHA1CHD, out tFile.MD5CHD, out tFile.CHDVersion);
if (errorCode == 0)
{
if (tFile.SHA1CHD != null)
{
tFile.FileStatusSet(FileStatus.SHA1CHDFromHeader);
}
if (tFile.MD5CHD != null)
{
tFile.FileStatusSet(FileStatus.MD5CHDFromHeader);
}
tFile.SetStatus(chechingDatStatus, GotStatus.Got);
// if we are scanning at Level3 then we get all the info here
if (EScanLevel == eScanLevel.Level3)
{
DeepScanFile(fullDir, tFile);
ChdManCheck(fullDir, tFile);
}
}
else if (errorCode == 32)
{
tFile.GotStatus = GotStatus.FileLocked;
_bgw.ReportProgress(0, new bgwShowError(fullDir, "File Locked"));
}
else
{
string filename = Path.Combine(fullDir, tFile.Name);
ReportError.Show("File: " + filename + " Error: " + new Win32Exception(errorCode).Message + ". Scan Aborted.");
_fileErrorAbort = true;
return;
}
fileDir.ChildAdd(tFile);
}
break;
}
}
}
break;
default:
ReportError.SendAndShow("Un supported file type in CheckADir " + ft);
break;
}
#endregion
if (fileDir == null)
{
ReportError.SendAndShow("Unknown Reading File Type in Dir Scanner");
return;
}
if (report)
{
_bgw.ReportProgress(0, new bgwSetRange2(fileDir.ChildCount - 1));
_bgw.ReportProgress(0, new bgwRange2Visible(true));
}
if (!DBTypeGet.isCompressedDir(ft) && _bgw.CancellationPending)
{
return;
}
Compare(dbDir, fileDir, report, true);
}
// This Function returns:
// Good : Everything Worked Correctly
// RescanNeeded : Something unexpectedly changed in the files, so Stop fixing and prompt user to rescan.
// LogicError : This Should never happen and is a logic problem in the code.
// FileSystemError : Something is wrong with the files, like it was locked and could not be opened.
// SourceDataStreamCorrupt : This happens when either zlib returns ZlibException, or the CRC does not match the extracted zip.
// SourceCheckSumMismatch : If the extracted files does not match its expected SHA1 or MD5
// DestinationCheckSumMismatch : If the extracted files does not match the file to be fixed expected CRC,SHA1 or MD5.
/// <summary>
/// Performs the RomVault File Copy, with the source and destination being files or zipped files
/// </summary>
/// <param name="fileIn">This is the file being copied, it may be a zipped file or a regular file system file</param>
/// <param name="zipFileOut">This is the zip file that is being written to.</param>
/// <param name="filenameOut">This is the name of the file to be written to if we are just making a file</param>
/// <param name="fileOut">This is the actual output filename</param>
/// <param name="forceRaw">if true then we will do a raw copy, this is so that we can copy corrupt zips</param>
/// <param name="error">This is the returned error message if this copy fails</param>
/// <returns>ReturnCode.Good is the valid return code otherwise we have an error</returns>
public static ReturnCode CopyFile(RvFile fileIn, ICompress zipFileOut, string filenameOut, RvFile fileOut, bool forceRaw, out string error)
{
if (zipFileOut == null && filenameOut == null)
{
throw new Exception("Error in CopyFile: Both Outputs are null");
}
if (zipFileOut != null && filenameOut != null)
{
throw new Exception("Error in CopyFile: Both Outputs are set");
}
ICompress zipFileIn = null;
Stream readStream = null;
ThreadCRC tcrc32 = null;
ThreadMD5 tmd5 = null;
ThreadSHA1 tsha1 = null;
ReturnCode retC;
error = "";
if (_buffer == null)
{
_buffer = new byte[BufferSize];
}
bool rawCopy = TestRawCopy(fileIn, fileOut, forceRaw);
ulong streamSize = 0;
ushort compressionMethod = 8;
bool sourceTrrntzip;
bool isZeroLengthFile = DBHelper.IsZeroLengthFile(fileOut);
if (!isZeroLengthFile)
{
//check that the in and out file match
retC = CheckInputAndOutputFile(fileIn, fileOut, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
//Find and Check/Open Input Files
retC = OpenInputStream(fileIn, rawCopy, out zipFileIn, out sourceTrrntzip, out readStream, out streamSize, out compressionMethod, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
}
else
{
sourceTrrntzip = true;
}
if (!rawCopy && (Settings.rvSettings.FixLevel == EFixLevel.TrrntZipLevel1 || Settings.rvSettings.FixLevel == EFixLevel.TrrntZipLevel2 || Settings.rvSettings.FixLevel == EFixLevel.TrrntZipLevel3))
{
compressionMethod = 8;
}
//Find and Check/Open Output Files
retC = OpenOutputStream(fileOut, fileIn, zipFileOut, filenameOut, compressionMethod, rawCopy, sourceTrrntzip, null, out Stream writeStream, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
byte[] bCRC;
byte[] bMD5 = null;
byte[] bSHA1 = null;
if (!isZeroLengthFile)
{
#region Do Data Tranfer
if (!rawCopy)
{
tcrc32 = new ThreadCRC();
if (Settings.rvSettings.FixLevel != EFixLevel.Level1 && Settings.rvSettings.FixLevel != EFixLevel.TrrntZipLevel1)
{
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
}
}
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > BufferSize ? BufferSize : (int)sizetogo;
try
{
readStream.Read(_buffer, 0, sizenow);
}
catch (Exception ex)
{
if (ex is ZlibException || ex is DataErrorException)
{
if ((fileIn.FileType == FileType.ZipFile || fileIn.FileType == FileType.SevenZipFile) && zipFileIn != null)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.ZipFilename;
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
}
writeStream.Flush();
writeStream.Close();
if (filenameOut != null)
{
File.Delete(filenameOut);
}
error = "Unexpected corrupt archive file found:\n" + fileIn.FullName +
"\nRun Find Fixes, and Fix to continue fixing correctly.";
return(ReturnCode.SourceDataStreamCorrupt);
}
error = "Error reading Source File " + ex.Message;
return(ReturnCode.FileSystemError);
}
tcrc32?.Trigger(_buffer, sizenow);
tmd5?.Trigger(_buffer, sizenow);
tsha1?.Trigger(_buffer, sizenow);
try
{
writeStream.Write(_buffer, 0, sizenow);
}
catch (Exception e)
{
error = "Error writing out file. " + Environment.NewLine + e.Message;
return(ReturnCode.FileSystemError);
}
tcrc32?.Wait();
tmd5?.Wait();
tsha1?.Wait();
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
#endregion
#region Collect Checksums
// if we did a full copy then we just calculated all the checksums while doing the copy
if (!rawCopy)
{
tcrc32.Finish();
tmd5?.Finish();
tsha1?.Finish();
bCRC = tcrc32.Hash;
bMD5 = tmd5?.Hash;
bSHA1 = tsha1?.Hash;
tcrc32.Dispose();
tmd5?.Dispose();
tsha1?.Dispose();
}
// if we raw copied and the source file has been FileChecked then we can trust the checksums in the source file
else
{
bCRC = fileIn.CRC.Copy();
if (fileIn.FileStatusIs(FileStatus.MD5Verified))
{
bMD5 = fileIn.MD5.Copy();
}
if (fileIn.FileStatusIs(FileStatus.SHA1Verified))
{
bSHA1 = fileIn.SHA1.Copy();
}
}
#endregion
#region close the ReadStream
if ((fileIn.FileType == FileType.ZipFile || fileIn.FileType == FileType.SevenZipFile) && zipFileIn != null)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.ZipFilename;
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
}
#endregion
}
else
{
CopyZeroLengthFile(fileOut, zipFileOut, out bCRC, out bMD5, out bSHA1);
}
#region close the WriteStream
if (fileOut.FileType == FileType.ZipFile || fileOut.FileType == FileType.SevenZipFile)
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(bCRC);
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
fileOut.ZipFileIndex = zipFileOut.LocalFilesCount() - 1;
fileOut.ZipFileHeaderPosition = zipFileOut.LocalHeader(fileOut.ZipFileIndex);
fileOut.FileModTimeStamp = 629870671200000000;
}
else
{
writeStream.Flush();
writeStream.Close();
FileInfo fi = new FileInfo(filenameOut);
fileOut.FileModTimeStamp = fi.LastWriteTime;
}
#endregion
if (!isZeroLengthFile)
{
if (!rawCopy)
{
retC = ValidateFileIn(fileIn, bCRC, bSHA1, bMD5, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
}
}
retC = ValidateFileOut(fileIn, fileOut, rawCopy, bCRC, bSHA1, bMD5, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
return(ReturnCode.Good);
}
public static RvFile FromAZipFile(RvFile dbDir, EScanLevel eScanLevel, ThreadWorker thWrk)
{
RvFile fileDir = new RvFile(dbDir.FileType);
DatStatus chechingDatStatus = dbDir.IsInToSort ? DatStatus.InToSort : DatStatus.NotInDat;
string filename = dbDir.FullNameCase;
ICompress checkZ = dbDir.FileType == FileType.Zip ? new Zip() : (ICompress) new SevenZ();
ZipReturn zr = checkZ.ZipFileOpen(filename, dbDir.FileModTimeStamp);
if (zr == ZipReturn.ZipGood)
{
dbDir.ZipStatus = checkZ.ZipStatus;
// to be Scanning a ZIP file means it is either new or has changed.
// as the code below only calls back here if that is true.
//
// Level1: Only use header CRC's
// Just get the CRC for the ZIP headers.
//
// Level2: Fully checksum changed only files
// We know this file has been changed to do a full checksum scan.
//
// Level3: Fully checksum everything
// So do a full checksum scan.
if (_fs == null)
{
_fs = new FileScan();
}
List <FileScan.FileResults> fr = _fs.Scan(checkZ, false, eScanLevel == EScanLevel.Level2 || eScanLevel == EScanLevel.Level3);
// add all of the file information from the zip file into scanDir
for (int i = 0; i < checkZ.LocalFilesCount(); i++)
{
LocalFile lf = checkZ.GetLocalFile(i);
RvFile tFile = new RvFile(DBTypeGet.FileFromDir(dbDir.FileType))
{
Name = lf.Filename,
ZipFileIndex = i,
ZipFileHeaderPosition = lf.LocalHead,
Size = lf.UncompressedSize,
CRC = lf.CRC,
FileModTimeStamp = lf.LastModified
};
// all levels read the CRC from the ZIP header
tFile.SetStatus(chechingDatStatus, GotStatus.Got);
tFile.FileStatusSet(FileStatus.SizeFromHeader | FileStatus.CRCFromHeader);
if (fr[i].FileStatus != ZipReturn.ZipGood)
{
thWrk.Report(new bgwShowCorrupt(fr[i].FileStatus, filename + " : " + lf.Filename));
tFile.GotStatus = GotStatus.Corrupt;
}
else
{
tFile.HeaderFileType = fr[i].HeaderFileType;
tFile.SHA1 = fr[i].SHA1;
tFile.MD5 = fr[i].MD5;
tFile.AltSize = fr[i].AltSize;
tFile.AltCRC = fr[i].AltCRC;
tFile.AltSHA1 = fr[i].AltSHA1;
tFile.AltMD5 = fr[i].AltMD5;
if (fr[i].CRC != null)
{
tFile.FileStatusSet(FileStatus.CRCFromHeader);
if (eScanLevel == EScanLevel.Level2 || eScanLevel == EScanLevel.Level3)
{
tFile.FileStatusSet(FileStatus.CRCVerified);
}
}
tFile.FileStatusSet(
FileStatus.SizeVerified |
(fr[i].HeaderFileType != HeaderFileType.Nothing ? FileStatus.HeaderFileTypeFromHeader : 0) |
(fr[i].SHA1 != null ? FileStatus.SHA1Verified : 0) |
(fr[i].MD5 != null ? FileStatus.MD5Verified : 0) |
(fr[i].AltSize != null ? FileStatus.AltSizeVerified : 0) |
(fr[i].AltCRC != null ? FileStatus.AltCRCVerified : 0) |
(fr[i].AltSHA1 != null ? FileStatus.AltSHA1Verified : 0) |
(fr[i].AltMD5 != null ? FileStatus.AltMD5Verified : 0)
);
}
fileDir.ChildAdd(tFile);
}
}
else if (zr == ZipReturn.ZipFileLocked)
{
thWrk.Report(new bgwShowError(filename, "Zip File Locked"));
dbDir.FileModTimeStamp = 0;
dbDir.GotStatus = GotStatus.FileLocked;
}
else if (zr == ZipReturn.ZipErrorOpeningFile)
{
thWrk.Report(new bgwShowError(filename, "Zip Error Opening File"));
dbDir.FileModTimeStamp = 0;
dbDir.GotStatus = GotStatus.FileLocked;
}
else
{
thWrk.Report(new bgwShowCorrupt(zr, filename));
dbDir.GotStatus = GotStatus.Corrupt;
}
checkZ.ZipFileClose();
return(fileDir);
}
// This Function returns:
// Good : Everything Worked Correctly
// RescanNeeded : Something unexpectidly changed in the files, so Stop prompt user to rescan.
// LogicError : This Should never happen and is a logic problem in the code
// FileSystemError : Something is wrong with the files
/// <summary>
/// Performs the ROMVault File Copy, with the source and destination being files or zipped files
/// </summary>
/// <param name="fileIn">This is the file being copied, it may be a zipped file or a regular file system file</param>
/// <param name="zipFileOut">This is the zip file that is being writen to, if it is null a new zip file will be made if we are coping to a zip</param>
/// <param name="zipFilenameOut">This is the name of the .zip file to be made that will be created in zipFileOut</param>
/// <param name="fileOut">This is the actual output filename</param>
/// <param name="forceRaw">if true then we will do a raw copy, this is so that we can copy corrupt zips</param>
/// <param name="error">This is the returned error message if this copy fails</param>
/// <param name="foundFile">If we are SHA1/MD5 checking the source file for the first time, and it is different from what we expected the correct values for this file are returned in foundFile</param>
/// <returns>ReturnCode.Good is the valid return code otherwire we have an error</returns>
public static ReturnCode CopyFile(RvFile fileIn, ref ZipFile zipFileOut, string zipFilenameOut, RvFile fileOut, bool forceRaw, out string error, out RvFile foundFile)
{
foundFile = null;
error = "";
if (_buffer == null)
{
_buffer = new byte[BufferSize];
}
bool rawCopy = RawCopy(fileIn, fileOut, forceRaw);
ulong streamSize = 0;
ushort compressionMethod = 8;
bool sourceTrrntzip = false;
ZipFile zipFileIn = null;
System.IO.Stream readStream = null;
bool isZeroLengthFile = DBHelper.IsZeroLengthFile(fileOut);
if (!isZeroLengthFile)
{
#region check that the in and out file match
if (fileOut.FileStatusIs(FileStatus.SizeFromDAT) && fileOut.Size != null && fileIn.Size != fileOut.Size)
{
error = "Source and destination Size does not match. Logic Error.";
return(ReturnCode.LogicError);
}
if (fileOut.FileStatusIs(FileStatus.CRCFromDAT) && fileOut.CRC != null && !ArrByte.bCompare(fileIn.CRC, fileOut.CRC))
{
error = "Source and destination CRC does not match. Logic Error.";
return(ReturnCode.LogicError);
}
if (fileOut.FileStatusIs(FileStatus.SHA1FromDAT) && fileIn.FileStatusIs(FileStatus.SHA1Verified))
{
if (fileIn.SHA1 != null && fileOut.SHA1 != null && !ArrByte.bCompare(fileIn.SHA1, fileOut.SHA1))
{
error = "Source and destination SHA1 does not match. Logic Error.";
return(ReturnCode.LogicError);
}
}
if (fileOut.FileStatusIs(FileStatus.MD5CHDFromDAT) && fileIn.FileStatusIs(FileStatus.MD5Verified))
{
if (fileIn.MD5 != null && fileOut.MD5 != null && !ArrByte.bCompare(fileIn.MD5, fileOut.MD5))
{
error = "Source and destination SHA1 does not match. Logic Error.";
return(ReturnCode.LogicError);
}
}
#endregion
#region Find and Check/Open Input Files
if (fileIn.FileType == FileType.ZipFile) // Input is a ZipFile
{
RvDir zZipFileIn = fileIn.Parent;
if (zZipFileIn.FileType != FileType.Zip)
{
error = "Zip File Open but Source File is not a zip, Logic Error.";
return(ReturnCode.LogicError);
}
string fileNameIn = zZipFileIn.FullName;
sourceTrrntzip = (zZipFileIn.ZipStatus & ZipStatus.TrrntZip) == ZipStatus.TrrntZip;
//if (zZipFileIn.ZipFileType == RvZip.ZipType.Zip)
//{
zipFileIn = new ZipFile();
ZipReturn zr1;
if (fileIn.ZipFileHeaderPosition != null)
{
zr1 = zipFileIn.ZipFileOpen(fileNameIn, zZipFileIn.TimeStamp, false);
}
else
{
zr1 = zipFileIn.ZipFileOpen(fileNameIn, zZipFileIn.TimeStamp, true);
}
switch (zr1)
{
case ZipReturn.ZipGood:
break;
case ZipReturn.ZipErrorFileNotFound:
error = "File not found, Rescan required before fixing " + fileIn.Name;
return(ReturnCode.FileSystemError);
case ZipReturn.ZipErrorTimeStamp:
error = "File has changed, Rescan required before fixing " + fileIn.Name;
return(ReturnCode.FileSystemError);
default:
error = "Error Open Zip" + zr1 + ", with File " + fileIn.DatFullName;
return(ReturnCode.FileSystemError);
}
if (fileIn.ZipFileHeaderPosition != null)
{
zipFileIn.ZipFileOpenReadStreamQuick((ulong)fileIn.ZipFileHeaderPosition, rawCopy, out readStream, out streamSize, out compressionMethod);
}
else
{
zipFileIn.ZipFileOpenReadStream(fileIn.ZipFileIndex, rawCopy, out readStream, out streamSize, out compressionMethod);
}
}
else // Input is a regular file
{
string fileNameIn = fileIn.FullName;
if (!IO.File.Exists(fileNameIn))
{
error = "Rescan needed, File Changed :" + fileNameIn;
return(ReturnCode.RescanNeeded);
}
IO.FileInfo fileInInfo = new IO.FileInfo(fileNameIn);
if (fileInInfo.LastWriteTime != fileIn.TimeStamp)
{
error = "Rescan needed, File Changed :" + fileNameIn;
return(ReturnCode.RescanNeeded);
}
int errorCode = IO.FileStream.OpenFileRead(fileNameIn, out readStream);
if (errorCode != 0)
{
error = new Win32Exception(errorCode).Message + ". " + fileNameIn;
return(ReturnCode.FileSystemError);
}
if (fileIn.Size == null)
{
error = "Null File Size found in Fixing File :" + fileNameIn;
return(ReturnCode.LogicError);
}
streamSize = (ulong)fileIn.Size;
if ((ulong)readStream.Length != streamSize)
{
error = "Rescan needed, File Length Changed :" + fileNameIn;
return(ReturnCode.RescanNeeded);
}
}
#endregion
}
else
{
sourceTrrntzip = true;
}
if (!rawCopy && (Settings.FixLevel == eFixLevel.TrrntZipLevel1 || Settings.FixLevel == eFixLevel.TrrntZipLevel2 || Settings.FixLevel == eFixLevel.TrrntZipLevel3))
{
compressionMethod = 8;
}
#region Find and Check/Open Output Files
System.IO.Stream writeStream;
if (fileOut.FileType == FileType.ZipFile)
{
// if ZipFileOut == null then we have not open the output zip yet so open it from writing.
if (zipFileOut == null)
{
if (IO.Path.GetFileName(zipFilenameOut) == "__ROMVault.tmp")
{
if (IO.File.Exists(zipFilenameOut))
{
IO.File.Delete(zipFilenameOut);
}
}
else if (IO.File.Exists(zipFilenameOut))
{
error = "Rescan needed, File Changed :" + zipFilenameOut;
return(ReturnCode.RescanNeeded);
}
zipFileOut = new ZipFile();
ZipReturn zrf = zipFileOut.ZipFileCreate(zipFilenameOut);
if (zrf != ZipReturn.ZipGood)
{
error = "Error Opening Write Stream " + zrf;
return(ReturnCode.FileSystemError);
}
}
else
{
if (zipFileOut.ZipOpen != ZipOpenType.OpenWrite)
{
error = "Output Zip File is not set to OpenWrite, Logic Error.";
return(ReturnCode.LogicError);
}
if (zipFileOut.ZipFilename != (new IO.FileInfo(zipFilenameOut).FullName))
{
error = "Output Zip file has changed name from " + zipFileOut.ZipFilename + " to " + zipFilenameOut + ". Logic Error";
return(ReturnCode.LogicError);
}
}
if (fileIn.Size == null)
{
error = "Null File Size found in Fixing File :" + fileIn.FullName;
return(ReturnCode.LogicError);
}
ZipReturn zr = zipFileOut.ZipFileOpenWriteStream(rawCopy, sourceTrrntzip, fileOut.Name, (ulong)fileIn.Size, compressionMethod, out writeStream);
if (zr != ZipReturn.ZipGood)
{
error = "Error Opening Write Stream " + zr;
return(ReturnCode.FileSystemError);
}
}
else
{
if (IO.File.Exists(zipFilenameOut) && fileOut.GotStatus != GotStatus.Corrupt)
{
error = "Rescan needed, File Changed :" + zipFilenameOut;
return(ReturnCode.RescanNeeded);
}
int errorCode = IO.FileStream.OpenFileWrite(zipFilenameOut, out writeStream);
if (errorCode != 0)
{
error = new Win32Exception(errorCode).Message + ". " + zipFilenameOut;
return(ReturnCode.FileSystemError);
}
}
#endregion
byte[] bCRC;
byte[] bMD5 = null;
byte[] bSHA1 = null;
if (!isZeroLengthFile)
{
#region Do Data Tranfer
CRC32Hash crc32 = null;
MD5 md5 = null;
SHA1 sha1 = null;
if (!rawCopy)
{
crc32 = new CRC32Hash();
md5 = MD5.Create();
sha1 = SHA1.Create();
}
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > BufferSize ? BufferSize : (int)sizetogo;
try
{
readStream.Read(_buffer, 0, sizenow);
}
catch (ZlibException)
{
if (fileIn.FileType == FileType.ZipFile && zipFileIn != null)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.Filename(fileIn.ReportIndex);
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
}
if (fileOut.FileType == FileType.ZipFile)
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(new byte[] { 0, 0, 0, 0 });
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
zipFileOut.ZipFileRollBack();
}
else
{
writeStream.Flush();
writeStream.Close();
IO.File.Delete(zipFilenameOut);
}
error = "Error in Data Stream";
return(ReturnCode.SourceCRCCheckSumError);
}
catch (Exception e)
{
error = "Error reading Source File " + e.Message;
return(ReturnCode.FileSystemError);
}
if (!rawCopy)
{
crc32.TransformBlock(_buffer, 0, sizenow, null, 0);
md5.TransformBlock(_buffer, 0, sizenow, null, 0);
sha1.TransformBlock(_buffer, 0, sizenow, null, 0);
}
try
{
writeStream.Write(_buffer, 0, sizenow);
}
catch (Exception e)
{
Debug.WriteLine(e.Message);
error = "Error writing out file. " + Environment.NewLine + e.Message;
return(ReturnCode.FileSystemError);
}
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
#endregion
#region Collect Checksums
// if we did a full copy then we just calculated all the checksums while doing the copy
if (!rawCopy)
{
crc32.TransformFinalBlock(_buffer, 0, 0);
md5.TransformFinalBlock(_buffer, 0, 0);
sha1.TransformFinalBlock(_buffer, 0, 0);
bCRC = crc32.Hash;
bMD5 = md5.Hash;
bSHA1 = sha1.Hash;
}
// if we raw copied and the source file has been FileChecked then we can trust the checksums in the source file
else
{
bCRC = ArrByte.Copy(fileIn.CRC);
if (fileIn.FileStatusIs(FileStatus.MD5Verified))
{
bMD5 = ArrByte.Copy(fileIn.MD5);
}
if (fileIn.FileStatusIs(FileStatus.SHA1Verified))
{
bSHA1 = ArrByte.Copy(fileIn.SHA1);
}
}
#endregion
#region close the ReadStream
if (fileIn.FileType == FileType.ZipFile && zipFileIn != null)
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.Filename(fileIn.ReportIndex);
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
//if (IO.File.Exists(tmpFilename))
// IO.File.Delete(tmpFilename);
}
#endregion
}
else
{
// Zero Length File (Directory in a Zip)
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileAddDirectory();
}
bCRC = VarFix.CleanMD5SHA1("00000000", 8);
bMD5 = VarFix.CleanMD5SHA1("d41d8cd98f00b204e9800998ecf8427e", 32);
bSHA1 = VarFix.CleanMD5SHA1("da39a3ee5e6b4b0d3255bfef95601890afd80709", 40);
}
#region close the WriteStream
if (fileOut.FileType == FileType.ZipFile)
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(bCRC);
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
fileOut.ZipFileIndex = zipFileOut.LocalFilesCount() - 1;
fileOut.ZipFileHeaderPosition = zipFileOut.LocalHeader(fileOut.ZipFileIndex);
}
else
{
writeStream.Flush();
writeStream.Close();
IO.FileInfo fi = new IO.FileInfo(zipFilenameOut);
fileOut.TimeStamp = fi.LastWriteTime;
}
#endregion
if (!isZeroLengthFile)
{
if (!rawCopy)
{
if (!ArrByte.bCompare(bCRC, fileIn.CRC))
{
fileIn.GotStatus = GotStatus.Corrupt;
error = "Source CRC does not match Source Data stream, corrupt Zip";
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.SourceCRCCheckSumError);
}
fileIn.FileStatusSet(FileStatus.CRCVerified | FileStatus.SizeVerified);
bool sourceFailed = false;
// check to see if we have a MD5 from the DAT file
if (fileIn.FileStatusIs(FileStatus.MD5FromDAT))
{
if (fileIn.MD5 == null)
{
error = "Should have an filein MD5 from Dat but not found. Logic Error.";
return(ReturnCode.LogicError);
}
if (!ArrByte.bCompare(fileIn.MD5, bMD5))
{
sourceFailed = true;
}
else
{
fileIn.FileStatusSet(FileStatus.MD5Verified);
}
}
// check to see if we have an MD5 (not from the DAT) so must be from previously scanning this file.
else if (fileIn.MD5 != null)
{
if (!ArrByte.bCompare(fileIn.MD5, bMD5))
{
// if we had an MD5 from a preview scan and it now does not match, something has gone really bad.
error = "The MD5 found does not match a previously scanned MD5, this should not happen, unless something got corrupt.";
return(ReturnCode.LogicError);
}
}
else // (FileIn.MD5 == null)
{
fileIn.MD5 = bMD5;
fileIn.FileStatusSet(FileStatus.MD5Verified);
}
// check to see if we have a SHA1 from the DAT file
if (fileIn.FileStatusIs(FileStatus.SHA1FromDAT))
{
if (fileIn.SHA1 == null)
{
error = "Should have an filein SHA1 from Dat but not found. Logic Error.";
return(ReturnCode.LogicError);
}
if (!ArrByte.bCompare(fileIn.SHA1, bSHA1))
{
sourceFailed = true;
}
else
{
fileIn.FileStatusSet(FileStatus.SHA1Verified);
}
}
// check to see if we have an SHA1 (not from the DAT) so must be from previously scanning this file.
else if (fileIn.SHA1 != null)
{
if (!ArrByte.bCompare(fileIn.SHA1, bSHA1))
{
// if we had an SHA1 from a preview scan and it now does not match, something has gone really bad.
error = "The SHA1 found does not match a previously scanned SHA1, this should not happen, unless something got corrupt.";
return(ReturnCode.LogicError);
}
}
else // (FileIn.SHA1 == null)
{
fileIn.SHA1 = bSHA1;
fileIn.FileStatusSet(FileStatus.SHA1Verified);
}
if (sourceFailed)
{
if (fileIn.FileType == FileType.ZipFile)
{
RvFile tZFile = new RvFile(FileType.ZipFile);
foundFile = tZFile;
tZFile.ZipFileIndex = fileIn.ZipFileIndex;
tZFile.ZipFileHeaderPosition = fileIn.ZipFileHeaderPosition;
}
else
{
foundFile = new RvFile(fileIn.FileType);
}
foundFile.Name = fileIn.Name;
foundFile.Size = fileIn.Size;
foundFile.CRC = bCRC;
foundFile.MD5 = bMD5;
foundFile.SHA1 = bSHA1;
foundFile.TimeStamp = fileIn.TimeStamp;
foundFile.SetStatus(DatStatus.NotInDat, GotStatus.Got);
foundFile.FileStatusSet(FileStatus.SizeVerified | FileStatus.CRCVerified | FileStatus.MD5Verified | FileStatus.SHA1Verified);
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.SourceCheckSumError);
}
}
}
if (fileOut.FileType == FileType.ZipFile)
{
fileOut.FileStatusSet(FileStatus.SizeFromHeader | FileStatus.CRCFromHeader);
}
if (fileOut.FileStatusIs(FileStatus.CRCFromDAT) && fileOut.CRC != null && !ArrByte.bCompare(fileOut.CRC, bCRC))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.CRC = bCRC;
if (!rawCopy || fileIn.FileStatusIs(FileStatus.CRCVerified))
{
fileOut.FileStatusSet(FileStatus.CRCVerified);
}
if (bSHA1 != null)
{
if (fileOut.FileStatusIs(FileStatus.SHA1FromDAT) && fileOut.SHA1 != null && !ArrByte.bCompare(fileOut.SHA1, bSHA1))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.SHA1 = bSHA1;
fileOut.FileStatusSet(FileStatus.SHA1Verified);
}
if (bMD5 != null)
{
if (fileOut.FileStatusIs(FileStatus.MD5FromDAT) && fileOut.MD5 != null && !ArrByte.bCompare(fileOut.MD5, bMD5))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
IO.File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.MD5 = bMD5;
fileOut.FileStatusSet(FileStatus.MD5Verified);
}
if (fileIn.Size != null)
{
fileOut.Size = fileIn.Size;
fileOut.FileStatusSet(FileStatus.SizeVerified);
}
if (fileIn.GotStatus == GotStatus.Corrupt)
{
fileOut.GotStatus = GotStatus.Corrupt;
}
else
{
fileOut.GotStatus = GotStatus.Got; // Changes RepStatus to Correct
}
fileOut.FileStatusSet(FileStatus.SizeVerified);
if (fileOut.SHA1CHD == null && fileIn.SHA1CHD != null)
{
fileOut.SHA1CHD = fileIn.SHA1CHD;
}
if (fileOut.MD5CHD == null && fileIn.MD5CHD != null)
{
fileOut.MD5CHD = fileIn.MD5CHD;
}
fileOut.CHDVersion = fileIn.CHDVersion;
fileOut.FileStatusSet(FileStatus.SHA1CHDFromHeader | FileStatus.MD5CHDFromHeader | FileStatus.SHA1CHDVerified | FileStatus.MD5CHDVerified, fileIn);
return(ReturnCode.Good);
}
public static void FromAFile(RvFile file, string directory, EScanLevel eScanLevel, ThreadWorker bgw, ref bool fileErrorAbort)
{
_bgw = bgw;
string filename = Path.Combine(directory, string.IsNullOrWhiteSpace(file.FileName) ? file.Name : file.FileName);
ICompress fileToScan = new Compress.File.File();
ZipReturn zr = fileToScan.ZipFileOpen(filename, file.FileModTimeStamp);
if (zr == ZipReturn.ZipFileLocked)
{
file.GotStatus = GotStatus.FileLocked;
return;
}
if (zr != ZipReturn.ZipGood)
{
string error = zr.ToString();
if (error.ToLower().StartsWith("zip"))
{
error = error.Substring(3);
}
ReportError.Show($"File: {filename} Error: {error}. Scan Aborted.");
file.GotStatus = GotStatus.FileLocked;
fileErrorAbort = true;
return;
}
if (_fs == null)
{
_fs = new FileScan();
}
List <FileScan.FileResults> fr = _fs.Scan(fileToScan, true, eScanLevel == EScanLevel.Level2 || eScanLevel == EScanLevel.Level3);
file.HeaderFileType = fr[0].HeaderFileType;
file.Size = fr[0].Size;
file.CRC = fr[0].CRC;
file.SHA1 = fr[0].SHA1;
file.MD5 = fr[0].MD5;
file.AltSize = fr[0].AltSize;
file.AltCRC = fr[0].AltCRC;
file.AltSHA1 = fr[0].AltSHA1;
file.AltMD5 = fr[0].AltMD5;
file.FileStatusSet(
FileStatus.SizeVerified |
(file.HeaderFileType != HeaderFileType.Nothing ? FileStatus.HeaderFileTypeFromHeader : 0) |
(file.CRC != null ? FileStatus.CRCVerified : 0) |
(file.SHA1 != null ? FileStatus.SHA1Verified : 0) |
(file.MD5 != null ? FileStatus.MD5Verified : 0) |
(file.AltSize != null ? FileStatus.AltSizeVerified : 0) |
(file.AltCRC != null ? FileStatus.AltCRCVerified : 0) |
(file.AltSHA1 != null ? FileStatus.AltSHA1Verified : 0) |
(file.AltMD5 != null ? FileStatus.AltMD5Verified : 0)
);
if (fr[0].HeaderFileType == HeaderFileType.CHD)
{
bool deepCheck = (eScanLevel == EScanLevel.Level2 || eScanLevel == EScanLevel.Level3);
CHD.fileProcess = FileProcess;
CHD.fileProgress = FileProgress;
CHD.fileSystemError = FileSystemError;
CHD.fileError = FileError;
CHD.generalError = GeneralError;
hdErr result = CHD.CheckFile(file.Name, directory, Settings.isLinux, ref deepCheck, out uint?chdVersion, out byte[] chdSHA1, out byte[] chdMD5, ref fileErrorAbort);
switch (result)
{
case hdErr.HDERR_NONE:
file.CHDVersion = chdVersion;
if (chdSHA1 != null)
{
file.AltSHA1 = chdSHA1;
file.FileStatusSet(FileStatus.AltSHA1FromHeader);
if (deepCheck)
{
file.FileStatusSet(FileStatus.AltSHA1Verified);
}
}
if (chdMD5 != null)
{
file.AltMD5 = chdMD5;
file.FileStatusSet(FileStatus.AltMD5FromHeader);
if (deepCheck)
{
file.FileStatusSet(FileStatus.AltMD5Verified);
}
}
break;
case hdErr.HDERR_OUT_OF_MEMORY:
case hdErr.HDERR_INVALID_FILE:
case hdErr.HDERR_INVALID_DATA:
case hdErr.HDERR_READ_ERROR:
case hdErr.HDERR_DECOMPRESSION_ERROR:
case hdErr.HDERR_CANT_VERIFY:
file.GotStatus = GotStatus.Corrupt;
break;
default:
ReportError.UnhandledExceptionHandler(result.ToString());
break;
}
}
fileToScan.ZipFileClose();
}
/// <summary>
/// Generate a list of DatItem objects from the header values in an archive
/// </summary>
/// <param name="omitFromScan">Hash representing the hashes that should be skipped</param>
/// <param name="date">True if entry dates should be included, false otherwise (default)</param>
/// <returns>List of DatItem objects representing the found data</returns>
/// <remarks>TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually</remarks>
public override List <BaseFile> GetChildren(Hash omitFromScan = Hash.DeepHashes, bool date = false)
{
List <BaseFile> found = new List <BaseFile>();
string gamename = Path.GetFileNameWithoutExtension(this.Filename);
try
{
ZipFile zf = new ZipFile();
ZipReturn zr = zf.Open(this.Filename, new FileInfo(this.Filename).LastWriteTime.Ticks, true);
if (zr != ZipReturn.ZipGood)
{
throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
for (int i = 0; i < zf.EntriesCount; i++)
{
// Open the read stream
zr = zf.OpenReadStream(i, false, out Stream readStream, out ulong streamsize, out SabreTools.Library.Data.CompressionMethod cm, out uint lastMod);
// If we get a read error, log it and continue
if (zr != ZipReturn.ZipGood)
{
Globals.Logger.Warning("An error occurred while reading archive {0}: Zip Error - {1}", this.Filename, zr);
continue;
}
// If the entry ends with a directory separator, continue to the next item, if any
if (zf.Entries[i].FileName.EndsWith(Path.DirectorySeparatorChar.ToString()) ||
zf.Entries[i].FileName.EndsWith(Path.AltDirectorySeparatorChar.ToString()) ||
zf.Entries[i].FileName.EndsWith(Path.PathSeparator.ToString()))
{
continue;
}
// If secure hashes are disabled, do a quickscan
if (omitFromScan == Hash.SecureHashes)
{
string newname = zf.Entries[i].FileName;
long newsize = (long)zf.Entries[i].UncompressedSize;
byte[] newcrc = zf.Entries[i].CRC.Reverse().ToArray();
string convertedDate = Utilities.ConvertMsDosTimeFormatToDateTime(zf.Entries[i].LastMod).ToString("yyyy/MM/dd hh:mm:ss");
found.Add(new BaseFile
{
Filename = newname,
Size = newsize,
CRC = newcrc,
Date = (date ? convertedDate : null),
Parent = gamename,
});
}
// Otherwise, use the stream directly
else
{
BaseFile zipEntryRom = Utilities.GetStreamInfo(readStream, (long)zf.Entries[i].UncompressedSize, omitFromScan: omitFromScan, keepReadOpen: true);
zipEntryRom.Filename = zf.Entries[i].FileName;
zipEntryRom.Parent = gamename;
string convertedDate = Utilities.ConvertMsDosTimeFormatToDateTime(zf.Entries[i].LastMod).ToString("yyyy/MM/dd hh:mm:ss");
zipEntryRom.Date = (date ? convertedDate : null);
found.Add(zipEntryRom);
}
}
// Dispose of the archive
zr = zf.CloseReadStream();
zf.Close();
}
catch (Exception ex)
{
Globals.Logger.Error(ex.ToString());
return(null);
}
return(found);
}
private ZipReturn ZipFileReadHeaders()
{
try
{
ZipReturn zRet = FindEndOfCentralDirSignature();
if (zRet != ZipReturn.ZipGood)
{
ZipFileClose();
return(zRet);
}
long endOfCentralDir = _zipFs.Position;
zRet = EndOfCentralDirRead();
if (zRet != ZipReturn.ZipGood)
{
ZipFileClose();
return(zRet);
}
// check if this is a ZIP64 zip and if it is read the Zip64 End Of Central Dir Info
if (_centerDirStart == 0xffffffff || _centerDirSize == 0xffffffff || _localFilesCount == 0xffff)
{
_zip64 = true;
_zipFs.Position = endOfCentralDir - 20;
zRet = Zip64EndOfCentralDirectoryLocatorRead();
if (zRet != ZipReturn.ZipGood)
{
ZipFileClose();
return(zRet);
}
_zipFs.Position = (long)_endOfCenterDir64;
zRet = Zip64EndOfCentralDirRead();
if (zRet != ZipReturn.ZipGood)
{
ZipFileClose();
return(zRet);
}
}
bool trrntzip = false;
// check if the ZIP has a valid TorrentZip file comment
if (_fileComment.Length == 22)
{
if (ZipUtils.GetString(_fileComment).Substring(0, 14) == "TORRENTZIPPED-")
{
CrcCalculatorStream crcCs = new CrcCalculatorStream(_zipFs, true);
byte[] buffer = new byte[_centerDirSize];
_zipFs.Position = (long)_centerDirStart;
crcCs.Read(buffer, 0, (int)_centerDirSize);
crcCs.Flush();
crcCs.Close();
uint r = (uint)crcCs.Crc;
crcCs.Dispose();
string tcrc = ZipUtils.GetString(_fileComment).Substring(14, 8);
string zcrc = r.ToString("X8");
if (string.Compare(tcrc, zcrc, StringComparison.Ordinal) == 0)
{
trrntzip = true;
}
}
}
// now read the central directory
_zipFs.Position = (long)_centerDirStart;
_localFiles.Clear();
_localFiles.Capacity = (int)_localFilesCount;
for (int i = 0; i < _localFilesCount; i++)
{
LocalFile lc = new LocalFile();
zRet = lc.CenteralDirectoryRead(_zipFs);
if (zRet != ZipReturn.ZipGood)
{
ZipFileClose();
return(zRet);
}
_zip64 |= lc.Zip64;
_localFiles.Add(lc);
}
for (int i = 0; i < _localFilesCount; i++)
{
zRet = _localFiles[i].LocalFileHeaderRead(_zipFs);
if (zRet != ZipReturn.ZipGood)
{
ZipFileClose();
return(zRet);
}
trrntzip &= _localFiles[i].TrrntZip;
}
// check trrntzip file order
if (trrntzip)
{
for (int i = 0; i < _localFilesCount - 1; i++)
{
if (ZipUtils.TrrntZipStringCompare(_localFiles[i].FileName, _localFiles[i + 1].FileName) < 0)
{
continue;
}
trrntzip = false;
break;
}
}
// check trrntzip directories
if (trrntzip)
{
for (int i = 0; i < _localFilesCount - 1; i++)
{
// see if we found a directory
string filename0 = _localFiles[i].FileName;
if (filename0.Substring(filename0.Length - 1, 1) != "/")
{
continue;
}
// see if the next file is in that directory
string filename1 = _localFiles[i + 1].FileName;
if (filename1.Length <= filename0.Length)
{
continue;
}
if (ZipUtils.TrrntZipStringCompare(filename0, filename1.Substring(0, filename0.Length)) != 0)
{
continue;
}
// if we found a file in the directory then we do not need the directory entry
trrntzip = false;
break;
}
}
if (trrntzip)
{
ZipStatus |= ZipStatus.TrrntZip;
}
return(ZipReturn.ZipGood);
}
catch
{
ZipFileClose();
return(ZipReturn.ZipErrorReadingFile);
}
}
/// <summary>
/// Write a set of input files to a torrentzip archive (assuming the same output archive name)
/// </summary>
/// <param name="inputFile">Input filenames to be moved</param>
/// <param name="outDir">Output directory to build to</param>
/// <param name="rom">List of Rom representing the new information</param>
/// <param name="date">True if the date from the DAT should be used if available, false otherwise (default)</param>
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
/// <returns>True if the archive was written properly, false otherwise</returns>
public override bool Write(List <string> inputFiles, string outDir, List <Rom> roms, bool date = false, bool romba = false)
{
bool success = false;
string tempFile = Path.Combine(outDir, "tmp" + Guid.NewGuid().ToString());
// If either list of roms is null or empty, return
if (inputFiles == null || roms == null || inputFiles.Count == 0 || roms.Count == 0)
{
return(success);
}
// If the number of inputs is less than the number of available roms, return
if (inputFiles.Count < roms.Count)
{
return(success);
}
// If one of the files doesn't exist, return
foreach (string file in inputFiles)
{
if (!File.Exists(file))
{
return(success);
}
}
// Get the output archive name from the first rebuild rom
string archiveFileName = Path.Combine(outDir, Utilities.RemovePathUnsafeCharacters(roms[0].MachineName) + (roms[0].MachineName.EndsWith(".zip") ? "" : ".zip"));
// Set internal variables
Stream writeStream = null;
ZipFile oldZipFile = new ZipFile();
ZipFile zipFile = new ZipFile();
ZipReturn zipReturn = ZipReturn.ZipGood;
try
{
// If the full output path doesn't exist, create it
if (!Directory.Exists(Path.GetDirectoryName(archiveFileName)))
{
Directory.CreateDirectory(Path.GetDirectoryName(archiveFileName));
}
// If the archive doesn't exist, create it and put the single file
if (!File.Exists(archiveFileName))
{
zipReturn = zipFile.Create(tempFile);
// Map all inputs to index
Dictionary <string, int> inputIndexMap = new Dictionary <string, int>();
for (int i = 0; i < inputFiles.Count; i++)
{
inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), i);
}
// Sort the keys in TZIP order
List <string> keys = inputIndexMap.Keys.ToList();
keys.Sort(ZipFile.TorrentZipStringCompare);
// Now add all of the files in order
foreach (string key in keys)
{
// Get the index mapped to the key
int index = inputIndexMap[key];
// Open the input file for reading
Stream freadStream = Utilities.TryOpenRead(inputFiles[index]);
ulong istreamSize = (ulong)(new FileInfo(inputFiles[index]).Length);
DateTime dt = DateTime.Now;
if (date && !String.IsNullOrWhiteSpace(roms[index].Date) && DateTime.TryParse(roms[index].Date.Replace('\\', '/'), out dt))
{
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.OpenWriteStream(false, false, roms[index].Name.Replace('\\', '/'), istreamSize,
SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime);
}
else
{
zipFile.OpenWriteStream(false, true, roms[index].Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream);
}
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
freadStream.Dispose();
zipFile.CloseWriteStream(Convert.ToUInt32(roms[index].CRC, 16));
}
}
// Otherwise, sort the input files and write out in the correct order
else
{
// Open the old archive for reading
oldZipFile.Open(archiveFileName, new FileInfo(archiveFileName).LastWriteTime.Ticks, true);
// Map all inputs to index
Dictionary <string, int> inputIndexMap = new Dictionary <string, int>();
for (int i = 0; i < inputFiles.Count; i++)
{
// If the old one contains the new file, then just skip out
if (oldZipFile.Contains(roms[i].Name.Replace('\\', '/')))
{
continue;
}
inputIndexMap.Add(roms[i].Name.Replace('\\', '/'), -(i + 1));
}
// Then add all of the old entries to it too
for (int i = 0; i < oldZipFile.EntriesCount; i++)
{
inputIndexMap.Add(oldZipFile.Filename(i), i);
}
// If the number of entries is the same as the old archive, skip out
if (inputIndexMap.Keys.Count <= oldZipFile.EntriesCount)
{
success = true;
return(success);
}
// Otherwise, process the old zipfile
zipFile.Create(tempFile);
// Get the order for the entries with the new file
List <string> keys = inputIndexMap.Keys.ToList();
keys.Sort(ZipFile.TorrentZipStringCompare);
// Copy over all files to the new archive
foreach (string key in keys)
{
// Get the index mapped to the key
int index = inputIndexMap[key];
// If we have the input file, add it now
if (index < 0)
{
// Open the input file for reading
Stream freadStream = Utilities.TryOpenRead(inputFiles[-index - 1]);
ulong istreamSize = (ulong)(new FileInfo(inputFiles[-index - 1]).Length);
DateTime dt = DateTime.Now;
if (date && !String.IsNullOrWhiteSpace(roms[-index - 1].Date) && DateTime.TryParse(roms[-index - 1].Date.Replace('\\', '/'), out dt))
{
uint msDosDateTime = Utilities.ConvertDateTimeToMsDosTimeFormat(dt);
zipFile.OpenWriteStream(false, false, roms[-index - 1].Name.Replace('\\', '/'), istreamSize,
SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: msDosDateTime);
}
else
{
zipFile.OpenWriteStream(false, true, roms[-index - 1].Name.Replace('\\', '/'), istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream);
}
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = freadStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
freadStream.Dispose();
zipFile.CloseWriteStream(Convert.ToUInt32(roms[-index - 1].CRC, 16));
}
// Otherwise, copy the file from the old archive
else
{
// Instantiate the streams
oldZipFile.OpenReadStream(index, false, out Stream zreadStream, out ulong istreamSize, out SabreTools.Library.Data.CompressionMethod icompressionMethod, out uint lastMod);
zipFile.OpenWriteStream(false, lastMod == Constants.TorrentZipFileDateTime, oldZipFile.Filename(index),
istreamSize, SabreTools.Library.Data.CompressionMethod.Deflated, out writeStream, lastMod: lastMod);
// Copy the input stream to the output
byte[] ibuffer = new byte[_bufferSize];
int ilen;
while ((ilen = zreadStream.Read(ibuffer, 0, _bufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
zipFile.CloseWriteStream(BitConverter.ToUInt32(oldZipFile.CRC32(index), 0));
}
}
}
// Close the output zip file
zipFile.Close();
success = true;
}
catch (Exception ex)
{
Console.WriteLine(ex);
success = false;
}
finally
{
zipFile.Dispose();
oldZipFile.Dispose();
}
// If the old file exists, delete it and replace
if (File.Exists(archiveFileName))
{
Utilities.TryDeleteFile(archiveFileName);
}
File.Move(tempFile, archiveFileName);
return(true);
}
// This Function returns:
// Good : Everything Worked Correctly
// RescanNeeded : Something unexpectidly changed in the files, so Stop prompt user to rescan.
// LogicError : This Should never happen and is a logic problem in the code
// FileSystemError : Something is wrong with the files
/// <summary>
/// Performs the RomVault File Copy, with the source and destination being files or zipped files
/// </summary>
/// <param name="fileIn">This is the file being copied, it may be a zipped file or a regular file system file</param>
/// <param name="zipFileOut">
/// This is the zip file that is being writen to, if it is null a new zip file will be made if we
/// are coping to a zip
/// </param>
/// <param name="zipFilenameOut">This is the name of the .zip file to be made that will be created in zipFileOut</param>
/// <param name="fileOut">This is the actual output filename</param>
/// <param name="forceRaw">if true then we will do a raw copy, this is so that we can copy corrupt zips</param>
/// <param name="error">This is the returned error message if this copy fails</param>
/// <param name="foundFile">
/// If we are SHA1/MD5 checking the source file for the first time, and it is different from what
/// we expected the correct values for this file are returned in foundFile
/// </param>
/// <returns>ReturnCode.Good is the valid return code otherwire we have an error</returns>
public static ReturnCode CopyFile(RvFile fileIn, ref ICompress zipFileOut, string zipFilenameOut, RvFile fileOut, bool forceRaw, out string error, out RvFile foundFile)
{
foundFile = null;
error = "";
if (_buffer == null)
{
_buffer = new byte[BufferSize];
}
bool rawCopy = RawCopy(fileIn, fileOut, forceRaw);
ulong streamSize = 0;
ushort compressionMethod = 8;
bool sourceTrrntzip = false;
ICompress zipFileIn = null;
Stream readStream = null;
ReturnCode retC;
bool isZeroLengthFile = DBHelper.IsZeroLengthFile(fileOut);
if (!isZeroLengthFile)
{
//check that the in and out file match
retC = CheckInputAndOutputFile(fileIn, fileOut, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
//Find and Check/Open Input Files
retC = OpenInputStream(fileIn, rawCopy, out zipFileIn, out sourceTrrntzip, out readStream, out streamSize, out compressionMethod, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
}
else
{
sourceTrrntzip = true;
}
if (!rawCopy && ((Program.rvSettings.FixLevel == eFixLevel.TrrntZipLevel1) || (Program.rvSettings.FixLevel == eFixLevel.TrrntZipLevel2) || (Program.rvSettings.FixLevel == eFixLevel.TrrntZipLevel3)))
{
compressionMethod = 8;
}
//Find and Check/Open Output Files
Stream writeStream;
retC = OpenOutputStream(fileOut, fileIn, ref zipFileOut, zipFilenameOut, compressionMethod, rawCopy, sourceTrrntzip, out writeStream, out error);
if (retC != ReturnCode.Good)
{
return(retC);
}
byte[] bCRC;
byte[] bMD5 = null;
byte[] bSHA1 = null;
if (!isZeroLengthFile)
{
#region Do Data Tranfer
ThreadCRC tcrc32 = null;
ThreadMD5 tmd5 = null;
ThreadSHA1 tsha1 = null;
if (!rawCopy)
{
tcrc32 = new ThreadCRC();
tmd5 = new ThreadMD5();
tsha1 = new ThreadSHA1();
}
ulong sizetogo = streamSize;
while (sizetogo > 0)
{
int sizenow = sizetogo > BufferSize ? BufferSize : (int)sizetogo;
try
{
readStream.Read(_buffer, 0, sizenow);
}
catch (ZlibException)
{
if ((fileIn.FileType == FileType.ZipFile) && (zipFileIn != null))
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.Filename(fileIn.ReportIndex);
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
}
if (fileOut.FileType == FileType.ZipFile)
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(new byte[] { 0, 0, 0, 0 });
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
zipFileOut.ZipFileRollBack();
}
else
{
writeStream.Flush();
writeStream.Close();
File.Delete(zipFilenameOut);
}
error = "Error in Data Stream";
return(ReturnCode.SourceCRCCheckSumError);
}
catch (Exception e)
{
error = "Error reading Source File " + e.Message;
return(ReturnCode.FileSystemError);
}
if (!rawCopy)
{
tcrc32.Trigger(_buffer, sizenow);
tmd5.Trigger(_buffer, sizenow);
tsha1.Trigger(_buffer, sizenow);
tcrc32.Wait();
tmd5.Wait();
tsha1.Wait();
}
try
{
writeStream.Write(_buffer, 0, sizenow);
}
catch (Exception e)
{
Debug.WriteLine(e.Message);
error = "Error writing out file. " + Environment.NewLine + e.Message;
return(ReturnCode.FileSystemError);
}
sizetogo = sizetogo - (ulong)sizenow;
}
writeStream.Flush();
#endregion
#region Collect Checksums
// if we did a full copy then we just calculated all the checksums while doing the copy
if (!rawCopy)
{
tcrc32.Finish();
tmd5.Finish();
tsha1.Finish();
bCRC = tcrc32.Hash;
bMD5 = tmd5.Hash;
bSHA1 = tsha1.Hash;
tcrc32.Dispose();
tmd5.Dispose();
tsha1.Dispose();
}
// if we raw copied and the source file has been FileChecked then we can trust the checksums in the source file
else
{
bCRC = ArrByte.Copy(fileIn.CRC);
if (fileIn.FileStatusIs(FileStatus.MD5Verified))
{
bMD5 = ArrByte.Copy(fileIn.MD5);
}
if (fileIn.FileStatusIs(FileStatus.SHA1Verified))
{
bSHA1 = ArrByte.Copy(fileIn.SHA1);
}
}
#endregion
#region close the ReadStream
if (((fileIn.FileType == FileType.ZipFile) || (fileIn.FileType == FileType.SevenZipFile)) && (zipFileIn != null))
{
ZipReturn zr = zipFileIn.ZipFileCloseReadStream();
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing " + zr + " Stream :" + zipFileIn.Filename(fileIn.ReportIndex);
return(ReturnCode.FileSystemError);
}
zipFileIn.ZipFileClose();
}
else
{
readStream.Close();
//if (RVIO.File.Exists(tmpFilename))
// RVIO.File.Delete(tmpFilename);
}
#endregion
}
else
{
// Zero Length File (Directory in a Zip)
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileAddDirectory();
}
bCRC = VarFix.CleanMD5SHA1("00000000", 8);
bMD5 = VarFix.CleanMD5SHA1("d41d8cd98f00b204e9800998ecf8427e", 32);
bSHA1 = VarFix.CleanMD5SHA1("da39a3ee5e6b4b0d3255bfef95601890afd80709", 40);
}
#region close the WriteStream
if ((fileOut.FileType == FileType.ZipFile) || (fileOut.FileType == FileType.SevenZipFile))
{
ZipReturn zr = zipFileOut.ZipFileCloseWriteStream(bCRC);
if (zr != ZipReturn.ZipGood)
{
error = "Error Closing Stream " + zr;
return(ReturnCode.FileSystemError);
}
fileOut.ZipFileIndex = zipFileOut.LocalFilesCount() - 1;
fileOut.ZipFileHeaderPosition = zipFileOut.LocalHeader(fileOut.ZipFileIndex);
}
else
{
writeStream.Flush();
writeStream.Close();
FileInfo fi = new FileInfo(zipFilenameOut);
fileOut.TimeStamp = fi.LastWriteTime;
}
#endregion
if (!isZeroLengthFile)
{
if (!rawCopy)
{
if (!ArrByte.bCompare(bCRC, fileIn.CRC))
{
fileIn.GotStatus = GotStatus.Corrupt;
error = "Source CRC does not match Source Data stream, corrupt Zip";
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.SourceCRCCheckSumError);
}
fileIn.FileStatusSet(FileStatus.CRCVerified | FileStatus.SizeVerified);
bool sourceFailed = false;
// check to see if we have a MD5 from the DAT file
if (fileIn.FileStatusIs(FileStatus.MD5FromDAT))
{
if (fileIn.MD5 == null)
{
error = "Should have an filein MD5 from Dat but not found. Logic Error.";
return(ReturnCode.LogicError);
}
if (!ArrByte.bCompare(fileIn.MD5, bMD5))
{
sourceFailed = true;
}
else
{
fileIn.FileStatusSet(FileStatus.MD5Verified);
}
}
// check to see if we have an MD5 (not from the DAT) so must be from previously scanning this file.
else if (fileIn.MD5 != null)
{
if (!ArrByte.bCompare(fileIn.MD5, bMD5))
{
// if we had an MD5 from a preview scan and it now does not match, something has gone really bad.
error = "The MD5 found does not match a previously scanned MD5, this should not happen, unless something got corrupt.";
return(ReturnCode.LogicError);
}
}
else // (FileIn.MD5 == null)
{
fileIn.MD5 = bMD5;
fileIn.FileStatusSet(FileStatus.MD5Verified);
}
// check to see if we have a SHA1 from the DAT file
if (fileIn.FileStatusIs(FileStatus.SHA1FromDAT))
{
if (fileIn.SHA1 == null)
{
error = "Should have an filein SHA1 from Dat but not found. Logic Error.";
return(ReturnCode.LogicError);
}
if (!ArrByte.bCompare(fileIn.SHA1, bSHA1))
{
sourceFailed = true;
}
else
{
fileIn.FileStatusSet(FileStatus.SHA1Verified);
}
}
// check to see if we have an SHA1 (not from the DAT) so must be from previously scanning this file.
else if (fileIn.SHA1 != null)
{
if (!ArrByte.bCompare(fileIn.SHA1, bSHA1))
{
// if we had an SHA1 from a preview scan and it now does not match, something has gone really bad.
error = "The SHA1 found does not match a previously scanned SHA1, this should not happen, unless something got corrupt.";
return(ReturnCode.LogicError);
}
}
else // (FileIn.SHA1 == null)
{
fileIn.SHA1 = bSHA1;
fileIn.FileStatusSet(FileStatus.SHA1Verified);
}
if (sourceFailed)
{
if ((fileIn.FileType == FileType.ZipFile) || (fileIn.FileType == FileType.SevenZipFile))
{
RvFile tZFile = new RvFile(FileType.ZipFile);
foundFile = tZFile;
tZFile.ZipFileIndex = fileIn.ZipFileIndex;
tZFile.ZipFileHeaderPosition = fileIn.ZipFileHeaderPosition;
}
else
{
foundFile = new RvFile(fileIn.FileType);
}
foundFile.Name = fileIn.Name;
foundFile.Size = fileIn.Size;
foundFile.CRC = bCRC;
foundFile.MD5 = bMD5;
foundFile.SHA1 = bSHA1;
foundFile.TimeStamp = fileIn.TimeStamp;
foundFile.SetStatus(DatStatus.NotInDat, GotStatus.Got);
foundFile.FileStatusSet(FileStatus.SizeVerified | FileStatus.CRCVerified | FileStatus.MD5Verified | FileStatus.SHA1Verified);
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.SourceCheckSumError);
}
}
}
if ((fileOut.FileType == FileType.ZipFile) || (fileOut.FileType == FileType.SevenZipFile))
{
fileOut.FileStatusSet(FileStatus.SizeFromHeader | FileStatus.CRCFromHeader);
}
if (fileOut.FileStatusIs(FileStatus.CRCFromDAT) && (fileOut.CRC != null) && !ArrByte.bCompare(fileOut.CRC, bCRC))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.CRC = bCRC;
if (!rawCopy || fileIn.FileStatusIs(FileStatus.CRCVerified))
{
fileOut.FileStatusSet(FileStatus.CRCVerified);
}
if (bSHA1 != null)
{
if (fileOut.FileStatusIs(FileStatus.SHA1FromDAT) && (fileOut.SHA1 != null) && !ArrByte.bCompare(fileOut.SHA1, bSHA1))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.SHA1 = bSHA1;
fileOut.FileStatusSet(FileStatus.SHA1Verified);
}
if (bMD5 != null)
{
if (fileOut.FileStatusIs(FileStatus.MD5FromDAT) && (fileOut.MD5 != null) && !ArrByte.bCompare(fileOut.MD5, bMD5))
{
//Rollback the file
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut.ZipFileRollBack();
}
else
{
File.Delete(zipFilenameOut);
}
return(ReturnCode.DestinationCheckSumError);
}
fileOut.MD5 = bMD5;
fileOut.FileStatusSet(FileStatus.MD5Verified);
}
if (fileIn.Size != null)
{
fileOut.Size = fileIn.Size;
fileOut.FileStatusSet(FileStatus.SizeVerified);
}
if (fileIn.GotStatus == GotStatus.Corrupt)
{
fileOut.GotStatus = GotStatus.Corrupt;
}
else
{
fileOut.GotStatus = GotStatus.Got; // Changes RepStatus to Correct
}
fileOut.FileStatusSet(FileStatus.SizeVerified);
if ((fileOut.SHA1CHD == null) && (fileIn.SHA1CHD != null))
{
fileOut.SHA1CHD = fileIn.SHA1CHD;
}
if ((fileOut.MD5CHD == null) && (fileIn.MD5CHD != null))
{
fileOut.MD5CHD = fileIn.MD5CHD;
}
fileOut.CHDVersion = fileIn.CHDVersion;
fileOut.FileStatusSet(FileStatus.SHA1CHDFromHeader | FileStatus.MD5CHDFromHeader | FileStatus.SHA1CHDVerified | FileStatus.MD5CHDVerified, fileIn);
return(ReturnCode.Good);
}
public bgwShowCorrupt(ZipReturn zr, string filename)
{
this.zr = zr;
this.filename = filename;
}
private static ReturnCode OpenOutputStream(RvFile fileOut, RvFile fileIn, ref ICompress zipFileOut, string zipFilenameOut, ushort compressionMethod, bool rawCopy, bool sourceTrrntzip, out Stream writeStream, out string error)
{
writeStream = null;
if ((fileOut.FileType == FileType.ZipFile) || (fileOut.FileType == FileType.SevenZipFile))
{
// if ZipFileOut == null then we have not open the output zip yet so open it from writing.
if (zipFileOut == null)
{
if (Path.GetFileName(zipFilenameOut) == "__RomVault.tmp")
{
if (File.Exists(zipFilenameOut))
{
File.Delete(zipFilenameOut);
}
}
else if (File.Exists(zipFilenameOut))
{
error = "Rescan needed, File Changed :" + zipFilenameOut;
return(ReturnCode.RescanNeeded);
}
if (fileOut.FileType == FileType.ZipFile)
{
zipFileOut = new ZipFile();
}
else
{
zipFileOut = new SevenZ();
}
ZipReturn zrf = zipFileOut.ZipFileCreate(zipFilenameOut);
if (zrf != ZipReturn.ZipGood)
{
error = "Error Opening Write Stream " + zrf;
return(ReturnCode.FileSystemError);
}
}
else
{
if (zipFileOut.ZipOpen != ZipOpenType.OpenWrite)
{
error = "Output Zip File is not set to OpenWrite, Logic Error.";
return(ReturnCode.LogicError);
}
if (zipFileOut.ZipFilename != new FileInfo(zipFilenameOut).FullName)
{
error = "Output Zip file has changed name from " + zipFileOut.ZipFilename + " to " + zipFilenameOut + ". Logic Error";
return(ReturnCode.LogicError);
}
}
if (fileIn.Size == null)
{
error = "Null File Size found in Fixing File :" + fileIn.FullName;
return(ReturnCode.LogicError);
}
ZipReturn zr = zipFileOut.ZipFileOpenWriteStream(rawCopy, sourceTrrntzip, fileOut.Name, (ulong)fileIn.Size, compressionMethod, out writeStream);
if (zr != ZipReturn.ZipGood)
{
error = "Error Opening Write Stream " + zr;
return(ReturnCode.FileSystemError);
}
}
else
{
if (File.Exists(zipFilenameOut) && (fileOut.GotStatus != GotStatus.Corrupt))
{
error = "Rescan needed, File Changed :" + zipFilenameOut;
return(ReturnCode.RescanNeeded);
}
int errorCode = FileStream.OpenFileWrite(zipFilenameOut, out writeStream);
if (errorCode != 0)
{
error = new Win32Exception(errorCode).Message + ". " + zipFilenameOut;
return(ReturnCode.FileSystemError);
}
}
error = "";
return(ReturnCode.Good);
}
public static string ZipErrorMessageText(ZipReturn zS)
{
string ret = "Unknown";
switch (zS)
{
case ZipReturn.ZipGood:
ret = "";
break;
case ZipReturn.ZipFileCountError:
ret = "The number of file in the Zip does not mach the number of files in the Zips Centeral Directory";
break;
case ZipReturn.ZipSignatureError:
ret = "An unknown Signature Block was found in the Zip";
break;
case ZipReturn.ZipExtraDataOnEndOfZip:
ret = "Extra Data was found on the end of the Zip";
break;
case ZipReturn.ZipUnsupportedCompression:
ret = "An unsupported Compression method was found in the Zip, if you recompress this zip it will be usable";
break;
case ZipReturn.ZipLocalFileHeaderError:
ret = "Error reading a zipped file header information";
break;
case ZipReturn.ZipCenteralDirError:
ret = "There is an error in the Zip Centeral Directory";
break;
case ZipReturn.ZipReadingFromOutputFile:
ret = "Trying to write to a Zip file open for output only";
break;
case ZipReturn.ZipWritingToInputFile:
ret = "Tring to read from a Zip file open for input only";
break;
case ZipReturn.ZipErrorGettingDataStream:
ret = "Error creating Data Stream";
break;
case ZipReturn.ZipCRCDecodeError:
ret = "CRC error";
break;
case ZipReturn.ZipDecodeError:
ret = "Error unzipping a file";
break;
}
return ret;
}
/// <summary>
/// Attempt to extract a file as an archive
/// </summary>
/// <param name="outDir">Output directory for archive extraction</param>
/// <returns>True if the extraction was a success, false otherwise</returns>
public override bool CopyAll(string outDir)
{
bool encounteredErrors = true;
try
{
// Create the temp directory
Directory.CreateDirectory(outDir);
// Extract all files to the temp directory
SevenZ zf = new SevenZ();
ZipReturn zr = zf.ZipFileOpen(this.Filename, -1, true);
if (zr != ZipReturn.ZipGood)
{
throw new Exception(ZipFile.ZipErrorMessageText(zr));
}
for (int i = 0; i < zf.LocalFilesCount() && zr == ZipReturn.ZipGood; i++)
{
// Open the read stream
zr = zf.ZipFileOpenReadStream(i, out Stream readStream, out ulong streamsize);
// Create the rest of the path, if needed
if (!string.IsNullOrWhiteSpace(Path.GetDirectoryName(zf.Filename(i))))
{
Directory.CreateDirectory(Path.Combine(outDir, Path.GetDirectoryName(zf.Filename(i))));
}
// If the entry ends with a directory separator, continue to the next item, if any
if (zf.Filename(i).EndsWith(Path.DirectorySeparatorChar.ToString()) ||
zf.Filename(i).EndsWith(Path.AltDirectorySeparatorChar.ToString()) ||
zf.Filename(i).EndsWith(Path.PathSeparator.ToString()))
{
zf.ZipFileCloseReadStream();
continue;
}
FileStream writeStream = FileExtensions.TryCreate(Path.Combine(outDir, zf.Filename(i)));
// If the stream is smaller than the buffer, just run one loop through to avoid issues
if (streamsize < _bufferSize)
{
byte[] ibuffer = new byte[streamsize];
int ilen = readStream.Read(ibuffer, 0, (int)streamsize);
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
// Otherwise, we do the normal loop
else
{
int realBufferSize = (streamsize < _bufferSize ? (int)streamsize : _bufferSize);
byte[] ibuffer = new byte[realBufferSize];
int ilen;
while ((ilen = readStream.Read(ibuffer, 0, realBufferSize)) > 0)
{
writeStream.Write(ibuffer, 0, ilen);
writeStream.Flush();
}
}
zr = zf.ZipFileCloseReadStream();
writeStream.Dispose();
}
zf.ZipFileClose();
encounteredErrors = false;
}
catch (EndOfStreamException ex)
{
// Catch this but don't count it as an error because SharpCompress is unsafe
logger.Verbose(ex);
}
catch (InvalidOperationException ex)
{
logger.Warning(ex);
encounteredErrors = true;
}
catch (Exception ex)
{
logger.Error(ex);
encounteredErrors = true;
}
return(encounteredErrors);
}
