Ejemplo n.º 1
0
        /// <summary>
        /// Create or check the digest file
        /// </summary>
        /// <param name="DigestMode">Digest mode</param>
        /// <param name="DataFile_">Data file name</param>
        /// <param name="MapFile_">Map file name</param>
        /// <param name="DigestFile_">Digest file name</param>
        /// <param name="SegmentSize_">Segment size</param>
        public void Proc(int DigestMode, string DataFile_, string MapFile_, string DigestFile_, int SegmentSize_, bool PromptConfirm)
        {
            long DigestFileSize    = 0;
            int  DigestSegmentSize = 0;

            int SegmentSize = SegmentSize_;

            DigestSegmentSize = SegmentSize;

            MailFile MF = new MailFile();

            if (MF.Open(false, true, DataFile_, MapFile_))
            {
                MF.SetSegmentSize(SegmentSize);
                MF.CalcSegmentCount();

                long FileSize = MF.GetDataSize();
                DigestFileSize = FileSize;
                int SegmentCount__ = MF.GetSegmentCount();
                MailSegment.ConsoleLineToLog    = true;
                MailSegment.ConsoleLineToLogSum = true;
                MailSegment.Console_WriteLine("");
                MailSegment.Console_WriteLine("Data file size: " + FileSize);
                MailSegment.Console_WriteLine("Data segment count: " + SegmentCount__);
                MailSegment.ConsoleLineToLogSum = false;
                MailSegment.ConsoleLineToLog    = false;
                MailSegment.Console_WriteLine("");

                if (DigestMode != 0)
                {
                    MailFile MD = new MailFile();
                    if (MD.Open(true, DigestMode != 0, DigestFile_, null))
                    {
                        MD.SetSegmentSize(SegmentSize);
                        if (DigestFile_ != null)
                        {
                            MD.CalcSegmentCount();
                            DigestSegmentSize = MD.DigestSegmentSize;
                            DigestFileSize    = MD.DigestFileSize;
                        }
                        else
                        {
                            DigestSegmentSize = 0;
                            DigestFileSize    = 0;
                        }
                        MailSegment.Console_WriteLine("Data file size from digest file: " + DigestFileSize);
                        MailSegment.Console_WriteLine("Segment size from digest file: " + DigestSegmentSize);
                        MailSegment.Console_WriteLine("");
                        MD.Close();
                    }
                    else
                    {
                        MailSegment.Console_WriteLine("Digest file open error");
                    }
                }

                if (Program.PromptConfirm(PromptConfirm))
                {
                    if ((DigestMode == 1) && (DigestMode == 3))
                    {
                        MF.MapChange(1, 0);
                        MF.MapChange(2, 0);
                    }

                    if (DigestMode == 0)
                    {
                        try
                        {
                            if (File.Exists(DigestFile_))
                            {
                                File.Delete(DigestFile_);
                            }
                        }
                        catch
                        {
                            MailSegment.ConsoleLineToLog    = true;
                            MailSegment.ConsoleLineToLogSum = true;
                            MailSegment.Console_WriteLine("Digest file creation error");
                            MailSegment.ConsoleLineToLogSum = false;
                            MailSegment.ConsoleLineToLog    = false;
                            return;
                        }
                    }

                    MailFile MD = new MailFile();
                    if (MD.Open(true, DigestMode != 0, DigestFile_, null))
                    {
                        Stopwatch_ TSW = new Stopwatch_();
                        MailSegment.Log();
                        MailSegment.LogReset();
                        MailSegment.Log("Time stamp", "Processed segments since previous entry", "Totally processed segments", "All segments", "Processed bytes since previous entry", "Totally processed bytes", "All bytes");

                        MD.SetSegmentSize(SegmentSize);
                        if (DigestMode != 0)
                        {
                            if (DigestFile_ != null)
                            {
                                MD.CalcSegmentCount();
                                DigestSegmentSize = MD.DigestSegmentSize;
                                DigestFileSize    = MD.DigestFileSize;
                            }
                            else
                            {
                                DigestSegmentSize = 0;
                                DigestFileSize    = 0;
                            }
                        }
                        if ((DigestMode == 2) || (DigestMode == 3))
                        {
                            if (DigestSegmentSize == SegmentSize)
                            {
                                if (FileSize != DigestFileSize)
                                {
                                    MailSegment.Console_WriteLine("Data file size correction started");
                                    MF.ResizeData(DigestFileSize);
                                    MF.CalcSegmentCount();
                                    FileSize = MF.GetDataSize();
                                    MailSegment.Console_WriteLine("Data file size correction finished");
                                    MailSegment.Console_WriteLine("");
                                }
                                else
                                {
                                    MailSegment.Console_WriteLine("Data file size is correct");
                                    MailSegment.Console_WriteLine("");
                                }
                            }
                        }
                        if ((DigestSegmentSize == SegmentSize) && (FileSize == DigestFileSize))
                        {
                            if (DigestMode != 2)
                            {
                                Stopwatch_ SWProgress = new Stopwatch_();
                                long       SWWorkTime = 0;
                                SWProgress.Reset();

                                int DigestG = 0;
                                int DigestB = 0;

                                long ToLogSize         = 0;
                                bool PrintLastProgress = true;

                                for (int i = 0; i < SegmentCount__; i++)
                                {
                                    ToLogSize += MF.DataGetSize(i);
                                    if (DigestMode == 0)
                                    {
                                        if (DigestFile_ != null)
                                        {
                                            byte[] Temp = MF.DataGet(i);
                                            MD.DataSet(i, Temp, Temp.Length);
                                        }
                                        MF.MapSet(i, 1);
                                        DigestG++;
                                    }
                                    else
                                    {
                                        if ((DigestFile_ != null) && (MailSegment.BinToStr(MD.DataGetDigest(i)) == MailSegment.BinToStr(MF.DataGetDigest(i))))
                                        {
                                            MF.MapSet(i, 1);
                                            DigestG++;
                                        }
                                        else
                                        {
                                            MF.MapSet(i, 0);
                                            DigestB++;
                                        }
                                    }

                                    if (SWWorkTime < SWProgress.Elapsed())
                                    {
                                        while (SWWorkTime < SWProgress.Elapsed())
                                        {
                                            SWWorkTime += 1000L;
                                        }
                                        MailSegment.Console_WriteLine("Segment " + (i + 1) + "/" + SegmentCount__ + " (" + ((i + 1) * 100 / SegmentCount__) + "%)");
                                        MailSegment.Log(TSW.Elapsed().ToString(), MailSegment.LogDiffS(i + 1).ToString(), (i + 1).ToString(), SegmentCount__.ToString(), MailSegment.LogDiffB(ToLogSize).ToString(), ToLogSize.ToString(), FileSize.ToString());
                                        if ((i + 1) == SegmentCount__)
                                        {
                                            PrintLastProgress = false;
                                        }
                                    }
                                }
                                MF.ResizeMap();

                                if (PrintLastProgress)
                                {
                                    MailSegment.Console_WriteLine("Segment " + SegmentCount__ + "/" + SegmentCount__ + " (100%)");
                                    MailSegment.Log(TSW.Elapsed().ToString(), MailSegment.LogDiffS(SegmentCount__).ToString(), SegmentCount__.ToString(), SegmentCount__.ToString(), MailSegment.LogDiffB(ToLogSize).ToString(), ToLogSize.ToString(), FileSize.ToString());
                                }

                                MailSegment.ConsoleLineToLog    = true;
                                MailSegment.ConsoleLineToLogSum = true;
                                MailSegment.Console_WriteLine("");
                                MailSegment.Console_WriteLine("Total segments: " + (DigestG + DigestB));
                                if (DigestMode != 0)
                                {
                                    MailSegment.Console_WriteLine("Good segments: " + DigestG);
                                    MailSegment.Console_WriteLine("Bad segments: " + DigestB);
                                }
                                MailSegment.Console_WriteLine("Total time: " + MailSegment.TimeHMSM(TSW.Elapsed()));
                                MailSegment.ConsoleLineToLogSum = false;
                                MailSegment.ConsoleLineToLog    = false;
                            }
                            else
                            {
                                MailSegment.ConsoleLineToLog    = true;
                                MailSegment.ConsoleLineToLogSum = true;
                                MailSegment.Console_WriteLine("Data file contents are not checked");
                                MailSegment.Console_WriteLine("Total time: " + MailSegment.TimeHMSM(TSW.Elapsed()));
                                MailSegment.ConsoleLineToLogSum = false;
                                MailSegment.ConsoleLineToLog    = false;
                            }
                        }
                        else
                        {
                            MailSegment.ConsoleLineToLog    = true;
                            MailSegment.ConsoleLineToLogSum = true;
                            if (FileSize != DigestFileSize)
                            {
                                MailSegment.Console_WriteLine("Data file size mismatch");
                            }
                            if (DigestSegmentSize != SegmentSize)
                            {
                                MailSegment.Console_WriteLine("Segment size mismatch");
                            }
                            MailSegment.Console_WriteLine("Total time: " + MailSegment.TimeHMSM(TSW.Elapsed()));
                            MailSegment.ConsoleLineToLogSum = false;
                            MailSegment.ConsoleLineToLog    = false;
                        }

                        MD.Close();
                    }
                    else
                    {
                        MailSegment.ConsoleLineToLog    = true;
                        MailSegment.ConsoleLineToLogSum = true;
                        if (DigestMode == 0)
                        {
                            MailSegment.Console_WriteLine("Digest file creation error");
                        }
                        else
                        {
                            MailSegment.Console_WriteLine("Digest file open error");
                        }
                        MailSegment.ConsoleLineToLogSum = false;
                        MailSegment.ConsoleLineToLog    = false;
                    }
                }

                MF.Close();
            }
            else
            {
                MailSegment.ConsoleLineToLog    = true;
                MailSegment.ConsoleLineToLogSum = true;
                MailSegment.Console_WriteLine("Data file open error");
                MailSegment.ConsoleLineToLogSum = false;
                MailSegment.ConsoleLineToLog    = false;
            }
        }
        public void CreateFile(string FileNameSrc, string FileNameDst, int SegmentSize, int CreateStats, int CreatePeriod)
        {
            string FileNameSta = FileNameDst;

            MailFile FileSrc = new MailFile();
            MailFile FileDst = new MailFile();

            if (FileSrc.Open(false, true, FileNameSrc, null))
            {
                Stopwatch_ TSW = new Stopwatch_();

                FileSrc.SetSegmentSize(SegmentSize);
                FileSrc.CalcSegmentCount();
                long FileSize    = FileSrc.GetDataSize();
                bool DstFileGood = false;
                if (FileNameDst != null)
                {
                    if (FileDst.Open(false, false, FileNameDst, null))
                    {
                        DstFileGood = true;
                    }
                    else
                    {
                        FileNameSta = FileNameSrc;
                        MailSegment.ConsoleLineToLog    = true;
                        MailSegment.ConsoleLineToLogSum = true;
                        MailSegment.Console_WriteLine("");
                        MailSegment.Console_WriteLine("Destination file create error: " + FileDst.OpenError);
                        MailSegment.ConsoleLineToLogSum = false;
                        MailSegment.ConsoleLineToLog    = false;
                    }
                }


                if (DstFileGood || (CreateStats > 0))
                {
                    ulong[] Stats = new ulong[256];
                    for (int i = 0; i < 256; i++)
                    {
                        Stats[i] = 0;
                    }

                    MailSegment.Console_WriteLine("");
                    Stopwatch_ SW = new Stopwatch_();
                    MailSegment.Log();
                    MailSegment.LogReset();
                    MailSegment.Log("Time stamp", "Processed segments since previous entry", "Totally processed segments", "All segments", "Processed bytes since previous entry", "Totally processed bytes", "All bytes");

                    long SWWorkTime = 0;
                    if (DstFileGood)
                    {
                        FileDst.SetSegmentSize(SegmentSize);
                        FileDst.CalcSegmentCount();
                    }
                    long ToLogSize = 0;


                    int  SegmentCount__    = FileSrc.GetSegmentCount();
                    bool PrintLastProgress = true;
                    for (int i = 0; i < SegmentCount__; i++)
                    {
                        byte[] Temp = FileSrc.DataGet(i);
                        if (DstFileGood)
                        {
                            FileDst.DataSet(i, Temp, Temp.Length);
                        }
                        for (int ii = Temp.Length - 1; ii >= 0; ii--)
                        {
                            Stats[Temp[ii]]++;
                        }
                        ToLogSize += Temp.LongLength;

                        if (SWWorkTime < SW.Elapsed())
                        {
                            while (SWWorkTime < SW.Elapsed())
                            {
                                SWWorkTime += 1000L;
                            }
                            MailSegment.Console_WriteLine("Segment " + (i + 1) + "/" + SegmentCount__ + " (" + ((i + 1) * 100 / SegmentCount__) + "%)");
                            MailSegment.Log(SW.Elapsed().ToString(), MailSegment.LogDiffS(i + 1).ToString(), (i + 1).ToString(), SegmentCount__.ToString(), MailSegment.LogDiffB(ToLogSize).ToString(), ToLogSize.ToString(), FileSize.ToString());
                            if ((i + 1) == SegmentCount__)
                            {
                                PrintLastProgress = false;
                            }
                        }
                    }
                    if (PrintLastProgress)
                    {
                        MailSegment.Console_WriteLine("Segment " + SegmentCount__ + "/" + SegmentCount__ + " (100%)");
                        MailSegment.Log(SW.Elapsed().ToString(), MailSegment.LogDiffS(SegmentCount__).ToString(), SegmentCount__.ToString(), SegmentCount__.ToString(), MailSegment.LogDiffB(ToLogSize).ToString(), ToLogSize.ToString(), FileSize.ToString());
                    }

                    if (DstFileGood)
                    {
                        FileDst.Close();
                    }

                    MailSegment.ConsoleLineToLog    = true;
                    MailSegment.ConsoleLineToLogSum = true;
                    MailSegment.Console_WriteLine("");
                    MailSegment.Console_WriteLine("File created in time: " + MailSegment.TimeHMSM(SW.Elapsed()));
                    if (CreateStats > 0)
                    {
                        MailSegment.Console_WriteLine("");
                        DisplayStats("File distribution", Stats, CreateStats);
                    }
                    MailSegment.ConsoleLineToLogSum = false;
                    MailSegment.ConsoleLineToLog    = false;
                }

                FileSrc.Close();

                if (CreatePeriod > 0)
                {
                    MailSegment.Console_WriteLine("");
                    MailSegment.Console_WriteLine("Searching for sequence period");

                    MailFile FileSta = new MailFile();
                    if (FileSta.Open(false, true, FileNameSta, null))
                    {
                        FileSta.DataValueFileOpen();

                        long   PeriodBufferSize = SegmentSize;
                        byte[] PeriodArray0     = new byte[PeriodBufferSize];
                        byte[] PeriodArray1     = new byte[PeriodBufferSize];

                        if (PeriodBufferSize > FileSize)
                        {
                            PeriodBufferSize = (int)FileSize;
                        }

                        long PeriodSize = 0;
                        int  PeriodChunks;
                        int  PeriodChunkOffset;
                        long PeriodChunkSize;
                        long PeriodChunkSize0;
                        long PeriodFilePos;

                        Stopwatch_ SW__ = new Stopwatch_();
                        SW__.Reset();

                        ulong[] PeriodStats = new ulong[256];
                        for (int i = 0; i < 256; i++)
                        {
                            PeriodStats[i] = 0;
                        }

                        long WorkTime = 0;

                        MailSegment.Log();
                        MailSegment.LogReset();
                        MailSegment.Log("Time stamp", "Current period length", "File length", "Current period occurence", "All period occurences");

                        for (long i = 1; i < FileSize; i++)
                        {
                            FileSta.DataGetBytes(i - 1, 1, PeriodArray0);
                            PeriodStats[PeriodArray0[0]]++;

                            bool IsPeriodical    = true;
                            bool PeriodicalPrint = true;

                            PeriodChunkSize  = PeriodBufferSize;
                            PeriodChunkSize0 = i;
                            PeriodChunks     = (int)(i / PeriodBufferSize);
                            if ((i % PeriodBufferSize) > 0)
                            {
                                PeriodChunks++;
                            }

                            long PeriodCount = (FileSize / i);
                            if ((FileSize % i) > 0)
                            {
                                PeriodCount++;
                            }

                            PeriodChunkOffset = 0;
                            for (int ii = 0; ii < PeriodChunks; ii++)
                            {
                                PeriodFilePos = PeriodChunkOffset;

                                if (PeriodChunkSize > PeriodChunkSize0)
                                {
                                    PeriodChunkSize = PeriodChunkSize0;
                                }

                                // Read the first period occurence and treat as pattern
                                FileSta.DataGetBytes(PeriodFilePos, PeriodChunkSize, PeriodArray0);

                                int PeriodChunkSize__ = 0;
                                for (long iii = (PeriodCount - 2); iii >= 0; iii--)
                                {
                                    PeriodFilePos    += i;
                                    PeriodChunkSize__ = (int)PeriodChunkSize;
                                    if (iii == 0)
                                    {
                                        int FileRemain = (int)(FileSize - PeriodFilePos);
                                        if (PeriodChunkSize__ > FileRemain)
                                        {
                                            PeriodChunkSize__ = FileRemain;
                                        }
                                    }

                                    // Read the period occurence other than first and compare with pattern,
                                    // if doest match the pattern, the reading and comparing will be broken
                                    if (PeriodChunkSize__ > 0)
                                    {
                                        FileSta.DataGetBytes(PeriodFilePos, PeriodChunkSize, PeriodArray1);

                                        for (int iiii = (PeriodChunkSize__ - 1); iiii >= 0; iiii--)
                                        {
                                            if (PeriodArray0[iiii] != PeriodArray1[iiii])
                                            {
                                                IsPeriodical = false;

                                                // Break all check iteration if data has no period by length given by i
                                                ii  = PeriodChunks;
                                                iii = (-1);
                                                break;
                                            }
                                        }
                                    }

                                    if (WorkTime < SW__.Elapsed())
                                    {
                                        while (WorkTime < SW__.Elapsed())
                                        {
                                            WorkTime += 1000L;
                                        }
                                        MailSegment.Console_WriteLine("Period " + i + "/" + FileSize + " (" + (FileSize > 0 ? (i * 100 / FileSize) : 0) + "%); occurence " + (PeriodCount - iii - 1) + "/" + PeriodCount + " (" + (PeriodCount > 0 ? (((PeriodCount - iii - 1) * 100) / PeriodCount) : 0) + "%)");
                                        MailSegment.Log(SW__.Elapsed().ToString(), i.ToString(), FileSize.ToString(), (PeriodCount - iii - 1).ToString(), PeriodCount.ToString());
                                        if (((PeriodCount - iii - 1) == PeriodCount))
                                        {
                                            PeriodicalPrint = false;
                                        }
                                    }
                                }
                                PeriodChunkOffset += (int)PeriodChunkSize;

                                PeriodChunkSize0 -= PeriodChunkSize;
                            }

                            if (IsPeriodical && PeriodicalPrint)
                            {
                                MailSegment.Console_WriteLine("Period " + i + "/" + FileSize + " (" + (FileSize > 0 ? (i * 100 / FileSize) : 0) + "%); occurence " + PeriodCount + "/" + PeriodCount + " (100%)");
                                MailSegment.Log(SW__.Elapsed().ToString(), i.ToString(), FileSize.ToString(), PeriodCount.ToString(), PeriodCount.ToString());
                                PeriodSize = i;
                                break;
                            }
                        }
                        if (PeriodSize == 0)
                        {
                            MailSegment.Console_WriteLine("Period " + FileSize + "/" + FileSize + " (100%); occurence " + "1/1" + " (100%)");
                            MailSegment.Log(SW__.Elapsed().ToString(), FileSize.ToString(), FileSize.ToString(), "1", "1");
                        }

                        FileSta.DataValueFileClose();
                        FileSta.Close();

                        MailSegment.ConsoleLineToLog    = true;
                        MailSegment.ConsoleLineToLogSum = true;
                        MailSegment.Console_WriteLine("");
                        if (PeriodSize > 0)
                        {
                            MailSegment.Console_WriteLine("Sequence period length: " + PeriodSize);
                            MailSegment.Console_WriteLine("Period search time: " + MailSegment.TimeHMSM(SW__.Elapsed()));
                            MailSegment.Console_WriteLine("");
                            DisplayStats("Sequence period distribution", PeriodStats, CreatePeriod);
                        }
                        else
                        {
                            MailSegment.Console_WriteLine("Sequence has no period");
                            MailSegment.Console_WriteLine("Period search time: " + MailSegment.TimeHMSM(SW__.Elapsed()));
                        }
                        MailSegment.ConsoleLineToLogSum = false;
                        MailSegment.ConsoleLineToLog    = false;
                    }
                    else
                    {
                        MailSegment.ConsoleLineToLog    = true;
                        MailSegment.ConsoleLineToLogSum = true;
                        MailSegment.Console_WriteLine("Period search error: " + FileSta.OpenError);
                        MailSegment.ConsoleLineToLogSum = false;
                        MailSegment.ConsoleLineToLog    = false;
                    }
                }

                MailSegment.ConsoleLineToLog    = true;
                MailSegment.ConsoleLineToLogSum = true;
                MailSegment.Console_WriteLine("");
                MailSegment.Console_WriteLine("Total time: " + MailSegment.TimeHMSM(TSW.Elapsed()));
                MailSegment.ConsoleLineToLogSum = false;
                MailSegment.ConsoleLineToLog    = false;
            }
            else
            {
                MailSegment.ConsoleLineToLog    = true;
                MailSegment.ConsoleLineToLogSum = true;
                MailSegment.Console_WriteLine("");
                MailSegment.Console_WriteLine("Source file open error: " + FileSrc.OpenError);
                MailSegment.ConsoleLineToLogSum = false;
                MailSegment.ConsoleLineToLog    = false;
            }
        }