Esempio n. 1
0
        // we are adding got files so we can assume a few things:
        //  these got files may be level 1 or level 2 scanned.
        //
        //  If they are just level 1 then there may or may not be SHA1 / MD5 info, which is unvalidated.
        //  So: We will always have CRC & Size info at a minimum and may also have SHA1 / MD5
        //
        //  Next: Due to the possilibity of CRC hash collisions
        //  we could find matching CRC that have different SHA1 & MD5
        //  so we should seach for one or more matching CRC/Size sets.
        //  then check to see if we have anything else that matches and then either:
        //  add the rom to an existing set or make a new set.


        private static void MergeGotFiles(RvFile[] gotFilesSortedByCRC, out FileGroup[] fileGroups)
        {
            List <FileGroup> listFileGroupsOut = new List <FileGroup>();


            // insert a zero byte file.
            RvFile    fileZero     = MakeFileZero();
            FileGroup newFileGroup = new FileGroup(fileZero);

            List <FileGroup> lstFileWithSameCRC = new List <FileGroup>();

            byte[] crc = fileZero.CRC;


            lstFileWithSameCRC.Add(newFileGroup);
            listFileGroupsOut.Add(newFileGroup);

            foreach (RvFile file in gotFilesSortedByCRC)
            {
                if (file.CRC == null)
                {
                    continue;
                }

                if (crc != null && ArrByte.ICompare(crc, file.CRC) == 0)
                {
                    bool found = false;
                    foreach (FileGroup fileGroup in lstFileWithSameCRC)
                    {
                        if (!fileGroup.FindExactMatch(file))
                        {
                            continue;
                        }

                        fileGroup.MergeFileIntoGroup(file);
                        found = true;
                        break;
                    }

                    if (found)
                    {
                        continue;
                    }

                    // new File with the same CRC but different sha1/md5/size
                    newFileGroup = new FileGroup(file);
                    lstFileWithSameCRC.Add(newFileGroup);
                    listFileGroupsOut.Add(newFileGroup);
                    continue;
                }

                crc = file.CRC;
                lstFileWithSameCRC.Clear();
                newFileGroup = new FileGroup(file);
                lstFileWithSameCRC.Add(newFileGroup);
                listFileGroupsOut.Add(newFileGroup);
            }

            fileGroups = listFileGroupsOut.ToArray();
        }
Esempio n. 2
0
 private static int FamilySortAltMD5(FileGroup fileGroup1, FileGroup fileGroup2)
 {
     return(ArrByte.ICompare(fileGroup1.AltMD5, fileGroup2.AltMD5));
 }
Esempio n. 3
0
 private static int FamilySortSHA1(FileGroup fileGroup1, FileGroup fileGroup2)
 {
     return(ArrByte.ICompare(fileGroup1.SHA1, fileGroup2.SHA1));
 }
Esempio n. 4
0
 private static bool FindAltMD5(FileGroup fileGroup)
 {
     return(fileGroup.AltSHA1 != null);
 }
Esempio n. 5
0
 private static bool FindAltCRC(FileGroup fileGroup)
 {
     return(fileGroup.AltCRC != null);
 }
Esempio n. 6
0
 private static bool FindMD5(FileGroup fileGroup)
 {
     return(fileGroup.MD5 != null);
 }
Esempio n. 7
0
 private static bool FindSHA1(FileGroup fileGroup)
 {
     return(fileGroup.SHA1 != null);
 }
Esempio n. 8
0
 private static int CompareAltMD5(RvFile file, FileGroup fileGroup)
 {
     return(ArrByte.ICompare(file.MD5, fileGroup.AltMD5));
 }
Esempio n. 9
0
 private static int CompareAltSHA1(RvFile file, FileGroup fileGroup)
 {
     return(ArrByte.ICompare(file.SHA1, fileGroup.AltSHA1));
 }
Esempio n. 10
0
 private static int CompareAltCRC(RvFile file, FileGroup fileGroup)
 {
     return(ArrByte.ICompare(file.CRC, fileGroup.AltCRC));
 }
Esempio n. 11
0
        private static bool FindMatch(FileGroup[] fileGroups, RvFile file, Compare comp, ExactMatch match, out List <int> listIndex)
        {
            int intBottom = 0;
            int intTop    = fileGroups.Length;
            int intMid    = 0;
            int intRes    = -1;

            //Binary chop to find the closest match
            while ((intBottom < intTop) && (intRes != 0))
            {
                intMid = (intBottom + intTop) / 2;

                FileGroup ff = fileGroups[intMid];
                intRes = comp(file, ff);
                if (intRes < 0)
                {
                    intTop = intMid;
                }
                else if (intRes > 0)
                {
                    intBottom = intMid + 1;
                }
            }
            int index = intMid;

            listIndex = new List <int>();

            // if match was found check up the list for the first match
            if (intRes == 0)
            {
                int intRes1 = 0;
                while (index > 0 && intRes1 == 0)
                {
                    FileGroup ff = fileGroups[index - 1];
                    intRes1 = comp(file, ff);

                    if (intRes1 != 0)
                    {
                        continue;
                    }
                    index--;
                }

                int indexFirst = index;

                intTop  = fileGroups.Length;
                intRes1 = 0;
                while (index < intTop && intRes1 == 0)
                {
                    FileGroup ff = fileGroups[index];
                    intRes1 = comp(file, ff);
                    if (intRes1 != 0)
                    {
                        continue;
                    }
                    if (match(ff, file))
                    {
                        listIndex.Add(index);
                    }
                    index++;
                }

                if (listIndex.Count == 0)
                {
                    listIndex.Add(indexFirst);
                    intRes = -1;
                }
            }
            // if the search is greater than the closest match move one up the list
            else
            {
                if (intRes > 0)
                {
                    index++;
                }
                listIndex.Add(index);
            }

            return(intRes == 0);
        }
Esempio n. 12
0
        private static void SortFamily(int intBase, int intTop, FileGroup[] arrFamily, SortOn sortFunction, int depth)
        {
            int sortSize = intTop - intBase;

            if (sortSize <= 1)
            {
                return;
            }

            // if just 2 tests
            if (sortSize == 2)
            {
                // compare the 2 files
                FileGroup t0 = arrFamily[intBase];
                FileGroup t1 = arrFamily[intBase + 1];
                if (sortFunction(t0, t1) < 1)
                {
                    return;
                }
                // swap them
                arrFamily[intBase]     = t1;
                arrFamily[intBase + 1] = t0;
                return;
            }

            int intMiddle = (intTop + intBase) / 2;

            if (depth < 2)
            {
                Thread t0 = new Thread(() => SortFamily(intBase, intMiddle, arrFamily, sortFunction, depth + 1));
                Thread t1 = new Thread(() => SortFamily(intMiddle, intTop, arrFamily, sortFunction, depth + 1));
                t0.Start();
                t1.Start();
                t0.Join();
                t1.Join();
            }
            else
            {
                SortFamily(intBase, intMiddle, arrFamily, sortFunction, depth + 1);
                SortFamily(intMiddle, intTop, arrFamily, sortFunction, depth + 1);
            }

            int intBottomSize = intMiddle - intBase;
            int intTopSize    = intTop - intMiddle;

            FileGroup[] arrBottom = new FileGroup[intBottomSize];
            FileGroup[] arrTop    = new FileGroup[intTopSize];

            if (depth == 0)
            {
                Thread t0 = new Thread(() => Array.Copy(arrFamily, intBase, arrBottom, 0, intBottomSize));
                Thread t1 = new Thread(() => Array.Copy(arrFamily, intMiddle, arrTop, 0, intTopSize));
                t0.Start();
                t1.Start();
                t0.Join();
                t1.Join();
            }
            else
            {
                Array.Copy(arrFamily, intBase, arrBottom, 0, intBottomSize);
                Array.Copy(arrFamily, intMiddle, arrTop, 0, intTopSize);
            }

            int intBottomCount = 0;
            int intTopCount    = 0;
            int intCount       = intBase;

            while (intBottomCount < intBottomSize && intTopCount < intTopSize)
            {
                if (sortFunction(arrBottom[intBottomCount], arrTop[intTopCount]) < 1)
                {
                    arrFamily[intCount++] = arrBottom[intBottomCount++];
                }
                else
                {
                    arrFamily[intCount++] = arrTop[intTopCount++];
                }
            }

            while (intBottomCount < intBottomSize)
            {
                arrFamily[intCount++] = arrBottom[intBottomCount++];
            }

            while (intTopCount < intTopSize)
            {
                arrFamily[intCount++] = arrTop[intTopCount++];
            }
        }
Esempio n. 13
0
 public static bool FindAltExactMatch(FileGroup fGroup, RvFile file)
 {
     return(fGroup.FindAltExactMatch(file));
 }
Esempio n. 14
0
        private static void ListCheck(FileGroup family)
        {
            List <RvFile> files = family.Files;

            List <RvFile> missingFiles = new List <RvFile>();  // files we dont have that we need

            List <RvFile> correctFiles  = new List <RvFile>(); // files we have that are in the correct place
            List <RvFile> unNeededFiles = new List <RvFile>(); // files we have that are not in the correct place
            List <RvFile> inToSortFiles = new List <RvFile>(); // files we have that are in tosort
            List <RvFile> allGotFiles   = new List <RvFile>(); // all files we have

            List <RvFile> corruptFiles = new List <RvFile>();  // corrupt files that we do not need, a corrupt file is missing if it is needed


            // set the found status of this file
            foreach (RvFile tFile in files)
            {
                switch (tFile.RepStatus)
                {
                case RepStatus.UnScanned:
                    break;

                case RepStatus.Missing:
                    missingFiles.Add(tFile);     // these are checked in step 1 to fixes from the allGotFiles List.
                    break;

                case RepStatus.Correct:
                    correctFiles.Add(tFile);
                    break;

                case RepStatus.Corrupt:
                    if (tFile.DatStatus == DatStatus.InDatCollect)
                    {
                        missingFiles.Add(tFile);     // corrupt files that are also InDatcollect are treated as missing files, and a fix should be found.
                    }
                    else
                    {
                        corruptFiles.Add(tFile);     // all other corrupt files should be deleted or moved to tosort/corrupt
                    }
                    break;

                case RepStatus.UnNeeded:
                case RepStatus.Unknown:
                    unNeededFiles.Add(tFile);
                    break;

                case RepStatus.NotCollected:
                    break;

                case RepStatus.InToSort:
                    inToSortFiles.Add(tFile);
                    break;

                case RepStatus.Ignore:
                    break;     // Ignore File

                default:
                    ReportError.SendAndShow("Unknown test status " + tFile.FullName + "," + tFile.DatStatus + "," + tFile.RepStatus);
                    break;
                }
            }
            allGotFiles.AddRange(correctFiles);
            allGotFiles.AddRange(unNeededFiles);
            allGotFiles.AddRange(inToSortFiles);

            #region Step 1 Check the Missing files from the allGotFiles List.
            // check to see if we can find any of the missing files in the gotFiles list.
            // if we find them mark them as CanBeFixed,
            // or if they are missing corrupt files set then as corruptCanBefixed

            foreach (RvFile missingFile in missingFiles)
            {
                if (treeType(missingFile) == RvTreeRow.TreeSelect.Locked)
                {
                    continue;
                }

                /*
                 * if (DBHelper.IsZeroLengthFile(missingFile))
                 * {
                 * missingFile.RepStatus = missingFile.RepStatus == RepStatus.Corrupt ? RepStatus.CorruptCanBeFixed : RepStatus.CanBeFixed;
                 * continue;
                 * }
                 */

                foreach (RvFile gotFile in allGotFiles)
                {
                    if (!DBHelper.CheckIfMissingFileCanBeFixedByGotFile(missingFile, gotFile))
                    {
                        continue;
                    }
                    missingFile.RepStatus = missingFile.RepStatus == RepStatus.Corrupt ? RepStatus.CorruptCanBeFixed : RepStatus.CanBeFixed;
                    break;
                }
                if (missingFile.RepStatus == RepStatus.Corrupt)
                {
                    missingFile.RepStatus = RepStatus.MoveToCorrupt;
                }
            }
            #endregion

            #region Step 2 Check all corrupt files.
            // if we have a correct version of the corrupt file then the corrput file can just be deleted,
            // otherwise if the corrupt file is not already in ToSort it should be moved out to ToSort.

            // we can only check corrupt files using the CRC from the ZIP header, as it is corrupt so we cannot get a correct SHA1 / MD5 to check with

            foreach (RvFile corruptFile in corruptFiles)
            {
                if (treeType(corruptFile) == RvTreeRow.TreeSelect.Locked)
                {
                    continue;
                }

                if (allGotFiles.Count > 0)
                {
                    corruptFile.RepStatus = RepStatus.Delete;
                }

                if (corruptFile.RepStatus == RepStatus.Corrupt && corruptFile.DatStatus != DatStatus.InToSort)
                {
                    corruptFile.RepStatus = RepStatus.MoveToCorrupt;
                }
            }
            #endregion

            #region Step 3 Check if unNeeded files are needed for a fix, otherwise delete them or move them to tosort
            foreach (RvFile unNeededFile in unNeededFiles)
            {
                /*
                 * // check if we have a confirmed SHA1 / MD5 match of this file, and if we do we just mark this file to be deleted.
                 * foreach (RvFile correctFile in correctFiles)
                 * {
                 *  if (!FindSHA1MD5MatchingFiles(unNeededFile, correctFile)) continue;
                 *  unNeededFile.RepStatus = RepStatus.Delete;
                 *  break;
                 * }
                 * if (unNeededFile.RepStatus == RepStatus.Delete) continue;
                 */

                /*
                 * if (DBHelper.IsZeroLengthFile(unNeededFile))
                 * {
                 *  if (treeType(unNeededFile) == RvTreeRow.TreeSelect.Locked)
                 *      continue;
                 *
                 *  unNeededFile.RepStatus = RepStatus.Delete;
                 *  continue;
                 * }
                 */

                // check if the unNeededFile is needed to fix a missing file
                foreach (RvFile missingFile in missingFiles)
                {
                    if (!DBHelper.CheckIfMissingFileCanBeFixedByGotFile(missingFile, unNeededFile))
                    {
                        continue;
                    }
                    unNeededFile.RepStatus = RepStatus.NeededForFix;
                    break;
                }
                if (unNeededFile.RepStatus == RepStatus.NeededForFix)
                {
                    continue;
                }

                if (treeType(unNeededFile) == RvTreeRow.TreeSelect.Locked)
                {
                    continue;
                }

                // now that we know this file is not needed for a fix do a CRC only find against correct files to see if this file can be deleted.
                foreach (RvFile correctFile in correctFiles)
                {
                    if (!DBHelper.CheckIfGotfileAndMatchingFileAreFullMatches(unNeededFile, correctFile))
                    {
                        continue;
                    }
                    unNeededFile.RepStatus = RepStatus.Delete;
                    break;
                }
                if (unNeededFile.RepStatus == RepStatus.Delete)
                {
                    continue;
                }

                // and finally see if the file is already in ToSort, and if it is deleted.
                foreach (RvFile inToSortFile in inToSortFiles)
                {
                    if (!DBHelper.CheckIfGotfileAndMatchingFileAreFullMatches(unNeededFile, inToSortFile))
                    {
                        continue;
                    }
                    unNeededFile.RepStatus = RepStatus.Delete;
                    break;
                }
                if (unNeededFile.RepStatus == RepStatus.Delete)
                {
                    continue;
                }

                // otherwise move the file out to ToSort
                unNeededFile.RepStatus = RepStatus.MoveToSort;
            }
            #endregion

            #region Step 4 Check if ToSort files are needed for a fix, otherwise delete them or leave them in tosort
            foreach (RvFile inToSortFile in inToSortFiles)
            {
                /*
                 * // check if we have a confirmed SHA1 / MD5 match of this file, and if we do we just mark this file to be deleted.
                 * foreach (RvFile correctFile in correctFiles)
                 * {
                 *  if (!FindSHA1MD5MatchingFiles(inToSortFile, correctFile)) continue;
                 *  inToSortFile.RepStatus = RepStatus.Delete;
                 *  break;
                 * }
                 * if (inToSortFile.RepStatus == RepStatus.Delete) continue;
                 */

                // check if the ToSortFile is needed to fix a missing file
                foreach (RvFile missingFile in missingFiles)
                {
                    if (treeType(missingFile) == RvTreeRow.TreeSelect.Locked)
                    {
                        continue;
                    }

                    if (!DBHelper.CheckIfMissingFileCanBeFixedByGotFile(missingFile, inToSortFile))
                    {
                        continue;
                    }
                    inToSortFile.RepStatus = RepStatus.NeededForFix;
                    break;
                }
                if (inToSortFile.RepStatus == RepStatus.NeededForFix)
                {
                    continue;
                }

                if (treeType(inToSortFile) == RvTreeRow.TreeSelect.Locked)
                {
                    continue;
                }


                // now that we know this file is not needed for a fix do a CRC only find against correct files to see if this file can be deleted.
                foreach (RvFile correctFile in correctFiles)
                {
                    if (!DBHelper.CheckIfGotfileAndMatchingFileAreFullMatches(inToSortFile, correctFile))
                    {
                        continue;
                    }
                    inToSortFile.RepStatus = RepStatus.Delete;
                    break;
                }

                // otherwise leave the file in ToSort
            }
            #endregion


            //need to check here for roms that just need renamed inside the one ZIP
            //this prevents Zips from self deadlocking
            foreach (RvFile fOutLoop in files)
            {
                if (fOutLoop.RepStatus != RepStatus.NeededForFix)
                {
                    continue;
                }
                foreach (RvFile fInLoop in files)
                {
                    if (fInLoop.RepStatus != RepStatus.CanBeFixed)
                    {
                        continue;
                    }

                    if (!DBHelper.CheckIfMissingFileCanBeFixedByGotFile(fInLoop, fOutLoop))
                    {
                        continue;
                    }

                    if (DBHelper.RomFromSameGame(fOutLoop, fInLoop))
                    {
                        fOutLoop.RepStatus = RepStatus.Rename;
                    }
                }
            }
        }