// we are adding got files so we can assume a few things:
// these got files may be level 1 or level 2 scanned.
//
// If they are just level 1 then there may or may not be SHA1 / MD5 info, which is unvalidated.
// So: We will always have CRC & Size info at a minimum and may also have SHA1 / MD5
//
// Next: Due to the possilibity of CRC hash collisions
// we could find matching CRC that have different SHA1 & MD5
// so we should seach for one or more matching CRC/Size sets.
// then check to see if we have anything else that matches and then either:
// add the rom to an existing set or make a new set.
private static void MergeGotFiles(RvFile[] gotFilesSortedByCRC, out FileGroup[] fileGroups)
{
List <FileGroup> listFileGroupsOut = new List <FileGroup>();
// insert a zero byte file.
RvFile fileZero = MakeFileZero();
FileGroup newFileGroup = new FileGroup(fileZero);
List <FileGroup> lstFileWithSameCRC = new List <FileGroup>();
byte[] crc = fileZero.CRC;
lstFileWithSameCRC.Add(newFileGroup);
listFileGroupsOut.Add(newFileGroup);
foreach (RvFile file in gotFilesSortedByCRC)
{
if (file.CRC == null)
{
continue;
}
if (crc != null && ArrByte.ICompare(crc, file.CRC) == 0)
{
bool found = false;
foreach (FileGroup fileGroup in lstFileWithSameCRC)
{
if (!fileGroup.FindExactMatch(file))
{
continue;
}
fileGroup.MergeFileIntoGroup(file);
found = true;
break;
}
if (found)
{
continue;
}
// new File with the same CRC but different sha1/md5/size
newFileGroup = new FileGroup(file);
lstFileWithSameCRC.Add(newFileGroup);
listFileGroupsOut.Add(newFileGroup);
continue;
}
crc = file.CRC;
lstFileWithSameCRC.Clear();
newFileGroup = new FileGroup(file);
lstFileWithSameCRC.Add(newFileGroup);
listFileGroupsOut.Add(newFileGroup);
}
fileGroups = listFileGroupsOut.ToArray();
}
private static int FamilySortAltMD5(FileGroup fileGroup1, FileGroup fileGroup2)
{
return(ArrByte.ICompare(fileGroup1.AltMD5, fileGroup2.AltMD5));
}
private static int FamilySortSHA1(FileGroup fileGroup1, FileGroup fileGroup2)
{
return(ArrByte.ICompare(fileGroup1.SHA1, fileGroup2.SHA1));
}
private static bool FindAltMD5(FileGroup fileGroup)
{
return(fileGroup.AltSHA1 != null);
}
private static bool FindAltCRC(FileGroup fileGroup)
{
return(fileGroup.AltCRC != null);
}
private static bool FindMD5(FileGroup fileGroup)
{
return(fileGroup.MD5 != null);
}
private static bool FindSHA1(FileGroup fileGroup)
{
return(fileGroup.SHA1 != null);
}
private static int CompareAltMD5(RvFile file, FileGroup fileGroup)
{
return(ArrByte.ICompare(file.MD5, fileGroup.AltMD5));
}
private static int CompareAltSHA1(RvFile file, FileGroup fileGroup)
{
return(ArrByte.ICompare(file.SHA1, fileGroup.AltSHA1));
}
private static int CompareAltCRC(RvFile file, FileGroup fileGroup)
{
return(ArrByte.ICompare(file.CRC, fileGroup.AltCRC));
}
private static bool FindMatch(FileGroup[] fileGroups, RvFile file, Compare comp, ExactMatch match, out List <int> listIndex)
{
int intBottom = 0;
int intTop = fileGroups.Length;
int intMid = 0;
int intRes = -1;
//Binary chop to find the closest match
while ((intBottom < intTop) && (intRes != 0))
{
intMid = (intBottom + intTop) / 2;
FileGroup ff = fileGroups[intMid];
intRes = comp(file, ff);
if (intRes < 0)
{
intTop = intMid;
}
else if (intRes > 0)
{
intBottom = intMid + 1;
}
}
int index = intMid;
listIndex = new List <int>();
// if match was found check up the list for the first match
if (intRes == 0)
{
int intRes1 = 0;
while (index > 0 && intRes1 == 0)
{
FileGroup ff = fileGroups[index - 1];
intRes1 = comp(file, ff);
if (intRes1 != 0)
{
continue;
}
index--;
}
int indexFirst = index;
intTop = fileGroups.Length;
intRes1 = 0;
while (index < intTop && intRes1 == 0)
{
FileGroup ff = fileGroups[index];
intRes1 = comp(file, ff);
if (intRes1 != 0)
{
continue;
}
if (match(ff, file))
{
listIndex.Add(index);
}
index++;
}
if (listIndex.Count == 0)
{
listIndex.Add(indexFirst);
intRes = -1;
}
}
// if the search is greater than the closest match move one up the list
else
{
if (intRes > 0)
{
index++;
}
listIndex.Add(index);
}
return(intRes == 0);
}
private static void SortFamily(int intBase, int intTop, FileGroup[] arrFamily, SortOn sortFunction, int depth)
{
int sortSize = intTop - intBase;
if (sortSize <= 1)
{
return;
}
// if just 2 tests
if (sortSize == 2)
{
// compare the 2 files
FileGroup t0 = arrFamily[intBase];
FileGroup t1 = arrFamily[intBase + 1];
if (sortFunction(t0, t1) < 1)
{
return;
}
// swap them
arrFamily[intBase] = t1;
arrFamily[intBase + 1] = t0;
return;
}
int intMiddle = (intTop + intBase) / 2;
if (depth < 2)
{
Thread t0 = new Thread(() => SortFamily(intBase, intMiddle, arrFamily, sortFunction, depth + 1));
Thread t1 = new Thread(() => SortFamily(intMiddle, intTop, arrFamily, sortFunction, depth + 1));
t0.Start();
t1.Start();
t0.Join();
t1.Join();
}
else
{
SortFamily(intBase, intMiddle, arrFamily, sortFunction, depth + 1);
SortFamily(intMiddle, intTop, arrFamily, sortFunction, depth + 1);
}
int intBottomSize = intMiddle - intBase;
int intTopSize = intTop - intMiddle;
FileGroup[] arrBottom = new FileGroup[intBottomSize];
FileGroup[] arrTop = new FileGroup[intTopSize];
if (depth == 0)
{
Thread t0 = new Thread(() => Array.Copy(arrFamily, intBase, arrBottom, 0, intBottomSize));
Thread t1 = new Thread(() => Array.Copy(arrFamily, intMiddle, arrTop, 0, intTopSize));
t0.Start();
t1.Start();
t0.Join();
t1.Join();
}
else
{
Array.Copy(arrFamily, intBase, arrBottom, 0, intBottomSize);
Array.Copy(arrFamily, intMiddle, arrTop, 0, intTopSize);
}
int intBottomCount = 0;
int intTopCount = 0;
int intCount = intBase;
while (intBottomCount < intBottomSize && intTopCount < intTopSize)
{
if (sortFunction(arrBottom[intBottomCount], arrTop[intTopCount]) < 1)
{
arrFamily[intCount++] = arrBottom[intBottomCount++];
}
else
{
arrFamily[intCount++] = arrTop[intTopCount++];
}
}
while (intBottomCount < intBottomSize)
{
arrFamily[intCount++] = arrBottom[intBottomCount++];
}
while (intTopCount < intTopSize)
{
arrFamily[intCount++] = arrTop[intTopCount++];
}
}
public static bool FindAltExactMatch(FileGroup fGroup, RvFile file)
{
return(fGroup.FindAltExactMatch(file));
}
private static void ListCheck(FileGroup family)
{
List <RvFile> files = family.Files;
List <RvFile> missingFiles = new List <RvFile>(); // files we dont have that we need
List <RvFile> correctFiles = new List <RvFile>(); // files we have that are in the correct place
List <RvFile> unNeededFiles = new List <RvFile>(); // files we have that are not in the correct place
List <RvFile> inToSortFiles = new List <RvFile>(); // files we have that are in tosort
List <RvFile> allGotFiles = new List <RvFile>(); // all files we have
List <RvFile> corruptFiles = new List <RvFile>(); // corrupt files that we do not need, a corrupt file is missing if it is needed
// set the found status of this file
foreach (RvFile tFile in files)
{
switch (tFile.RepStatus)
{
case RepStatus.UnScanned:
break;
case RepStatus.Missing:
missingFiles.Add(tFile); // these are checked in step 1 to fixes from the allGotFiles List.
break;
case RepStatus.Correct:
correctFiles.Add(tFile);
break;
case RepStatus.Corrupt:
if (tFile.DatStatus == DatStatus.InDatCollect)
{
missingFiles.Add(tFile); // corrupt files that are also InDatcollect are treated as missing files, and a fix should be found.
}
else
{
corruptFiles.Add(tFile); // all other corrupt files should be deleted or moved to tosort/corrupt
}
break;
case RepStatus.UnNeeded:
case RepStatus.Unknown:
unNeededFiles.Add(tFile);
break;
case RepStatus.NotCollected:
break;
case RepStatus.InToSort:
inToSortFiles.Add(tFile);
break;
case RepStatus.Ignore:
break; // Ignore File
default:
ReportError.SendAndShow("Unknown test status " + tFile.FullName + "," + tFile.DatStatus + "," + tFile.RepStatus);
break;
}
}
allGotFiles.AddRange(correctFiles);
allGotFiles.AddRange(unNeededFiles);
allGotFiles.AddRange(inToSortFiles);
#region Step 1 Check the Missing files from the allGotFiles List.
// check to see if we can find any of the missing files in the gotFiles list.
// if we find them mark them as CanBeFixed,
// or if they are missing corrupt files set then as corruptCanBefixed
foreach (RvFile missingFile in missingFiles)
{
if (treeType(missingFile) == RvTreeRow.TreeSelect.Locked)
{
continue;
}
/*
* if (DBHelper.IsZeroLengthFile(missingFile))
* {
* missingFile.RepStatus = missingFile.RepStatus == RepStatus.Corrupt ? RepStatus.CorruptCanBeFixed : RepStatus.CanBeFixed;
* continue;
* }
*/
foreach (RvFile gotFile in allGotFiles)
{
if (!DBHelper.CheckIfMissingFileCanBeFixedByGotFile(missingFile, gotFile))
{
continue;
}
missingFile.RepStatus = missingFile.RepStatus == RepStatus.Corrupt ? RepStatus.CorruptCanBeFixed : RepStatus.CanBeFixed;
break;
}
if (missingFile.RepStatus == RepStatus.Corrupt)
{
missingFile.RepStatus = RepStatus.MoveToCorrupt;
}
}
#endregion
#region Step 2 Check all corrupt files.
// if we have a correct version of the corrupt file then the corrput file can just be deleted,
// otherwise if the corrupt file is not already in ToSort it should be moved out to ToSort.
// we can only check corrupt files using the CRC from the ZIP header, as it is corrupt so we cannot get a correct SHA1 / MD5 to check with
foreach (RvFile corruptFile in corruptFiles)
{
if (treeType(corruptFile) == RvTreeRow.TreeSelect.Locked)
{
continue;
}
if (allGotFiles.Count > 0)
{
corruptFile.RepStatus = RepStatus.Delete;
}
if (corruptFile.RepStatus == RepStatus.Corrupt && corruptFile.DatStatus != DatStatus.InToSort)
{
corruptFile.RepStatus = RepStatus.MoveToCorrupt;
}
}
#endregion
#region Step 3 Check if unNeeded files are needed for a fix, otherwise delete them or move them to tosort
foreach (RvFile unNeededFile in unNeededFiles)
{
/*
* // check if we have a confirmed SHA1 / MD5 match of this file, and if we do we just mark this file to be deleted.
* foreach (RvFile correctFile in correctFiles)
* {
* if (!FindSHA1MD5MatchingFiles(unNeededFile, correctFile)) continue;
* unNeededFile.RepStatus = RepStatus.Delete;
* break;
* }
* if (unNeededFile.RepStatus == RepStatus.Delete) continue;
*/
/*
* if (DBHelper.IsZeroLengthFile(unNeededFile))
* {
* if (treeType(unNeededFile) == RvTreeRow.TreeSelect.Locked)
* continue;
*
* unNeededFile.RepStatus = RepStatus.Delete;
* continue;
* }
*/
// check if the unNeededFile is needed to fix a missing file
foreach (RvFile missingFile in missingFiles)
{
if (!DBHelper.CheckIfMissingFileCanBeFixedByGotFile(missingFile, unNeededFile))
{
continue;
}
unNeededFile.RepStatus = RepStatus.NeededForFix;
break;
}
if (unNeededFile.RepStatus == RepStatus.NeededForFix)
{
continue;
}
if (treeType(unNeededFile) == RvTreeRow.TreeSelect.Locked)
{
continue;
}
// now that we know this file is not needed for a fix do a CRC only find against correct files to see if this file can be deleted.
foreach (RvFile correctFile in correctFiles)
{
if (!DBHelper.CheckIfGotfileAndMatchingFileAreFullMatches(unNeededFile, correctFile))
{
continue;
}
unNeededFile.RepStatus = RepStatus.Delete;
break;
}
if (unNeededFile.RepStatus == RepStatus.Delete)
{
continue;
}
// and finally see if the file is already in ToSort, and if it is deleted.
foreach (RvFile inToSortFile in inToSortFiles)
{
if (!DBHelper.CheckIfGotfileAndMatchingFileAreFullMatches(unNeededFile, inToSortFile))
{
continue;
}
unNeededFile.RepStatus = RepStatus.Delete;
break;
}
if (unNeededFile.RepStatus == RepStatus.Delete)
{
continue;
}
// otherwise move the file out to ToSort
unNeededFile.RepStatus = RepStatus.MoveToSort;
}
#endregion
#region Step 4 Check if ToSort files are needed for a fix, otherwise delete them or leave them in tosort
foreach (RvFile inToSortFile in inToSortFiles)
{
/*
* // check if we have a confirmed SHA1 / MD5 match of this file, and if we do we just mark this file to be deleted.
* foreach (RvFile correctFile in correctFiles)
* {
* if (!FindSHA1MD5MatchingFiles(inToSortFile, correctFile)) continue;
* inToSortFile.RepStatus = RepStatus.Delete;
* break;
* }
* if (inToSortFile.RepStatus == RepStatus.Delete) continue;
*/
// check if the ToSortFile is needed to fix a missing file
foreach (RvFile missingFile in missingFiles)
{
if (treeType(missingFile) == RvTreeRow.TreeSelect.Locked)
{
continue;
}
if (!DBHelper.CheckIfMissingFileCanBeFixedByGotFile(missingFile, inToSortFile))
{
continue;
}
inToSortFile.RepStatus = RepStatus.NeededForFix;
break;
}
if (inToSortFile.RepStatus == RepStatus.NeededForFix)
{
continue;
}
if (treeType(inToSortFile) == RvTreeRow.TreeSelect.Locked)
{
continue;
}
// now that we know this file is not needed for a fix do a CRC only find against correct files to see if this file can be deleted.
foreach (RvFile correctFile in correctFiles)
{
if (!DBHelper.CheckIfGotfileAndMatchingFileAreFullMatches(inToSortFile, correctFile))
{
continue;
}
inToSortFile.RepStatus = RepStatus.Delete;
break;
}
// otherwise leave the file in ToSort
}
#endregion
//need to check here for roms that just need renamed inside the one ZIP
//this prevents Zips from self deadlocking
foreach (RvFile fOutLoop in files)
{
if (fOutLoop.RepStatus != RepStatus.NeededForFix)
{
continue;
}
foreach (RvFile fInLoop in files)
{
if (fInLoop.RepStatus != RepStatus.CanBeFixed)
{
continue;
}
if (!DBHelper.CheckIfMissingFileCanBeFixedByGotFile(fInLoop, fOutLoop))
{
continue;
}
if (DBHelper.RomFromSameGame(fOutLoop, fInLoop))
{
fOutLoop.RepStatus = RepStatus.Rename;
}
}
}
}