public bool DoDedupeBatch(fingerprintDMSG[] fplist)
        {
            int fsid  = -1;
            int inode = -1;

            for (int i = 0; i < 1024; i++)
            {
                if (fplist[i] != null)
                {
                    //if ((fsid != -1 || fsid != fplist[i].fsid) ||
                    //    (inode != -1 || inode != fplist[i].inode))
                    //{
                    //    DEFS.DEBUG("BATCH", "Error detected in fplist");
                    //    return false;
                    //}
                    fsid  = fplist[i].fsid;
                    inode = fplist[i].inode;
                }
            }

            int[] list = prepare_Inheritance_list(fsid, inode);

            //check fsid's
            for (int i = 0; i < list.Length; i++)
            {
                FileAttributes fa    = FileAttributes.NotContentIndexed;
                string         rpath = Load_Inode(list[i], inode, ref fa);
                if (rpath == null)
                {
                    continue;
                }

                DEFS.ASSERT(fa == FileAttributes.Normal, "Only normal files can be deduped 2");
                REDDY.FSIDList[list[i]].rootdir.do_dedupe_tlock_batch(rpath, fplist);
            }

            return(true);
        }
Example #2
0
        private void insert_item_output(Item i, bool is_last)
        {
            DEFS.ASSERT(_internal_cache_op != null, "Cache memory is empty");
            DEFS.ASSERT(outputF != null, "output file cannot be null in insert-item-output function");

            if (i != null)
            {
                i.get_bytes(_internal_cache_op, _internal_cnt_op * _item.get_size());
                _internal_cnt_op++;
            }

            if (is_last || _internal_cnt_op == 1024)
            {
                outputF.Write(_internal_cache_op, 0, _internal_cnt_op * _item.get_size());
                _internal_cnt_op = 0;
                outputF.Flush();
            }

            if (is_last)
            {
                DEFS.ASSERT(_internal_cnt_op == 0, "_internal_cnt should be zero bcoz we just flushed");
            }
        }
Example #3
0
        void insert_item(Item i, bool is_last)
        {
            DEFS.ASSERT(_internal_cache != null, "Cache memory is empty");
            DEFS.ASSERT(inputF != null, "input file cannot be null in insert-item function");

            i.get_bytes(_internal_cache, _internal_cnt * _item.get_size());
            _internal_cnt++;

            if (is_last || _internal_cnt == 1024)
            {
                //print_contents(_internal_cache, _internal_cnt);
                inputF.Write(_internal_cache, 0, _internal_cnt * _item.get_size());
                _internal_cnt = 0;
                inputF.Flush();
            }

            if (is_last)
            {
                DEFS.ASSERT(_internal_cnt == 0, "_internal_cnt should be zero bcoz we just flushed");
                DEFS.DEBUG("SORT", "Finised inserting all test_insert_items");
                inputF.Seek(0, SeekOrigin.Begin);
            }
        }
        public void sync()
        {
            if (!initialized)
            {
                return;
            }

            lock (mfile)
            {
                for (int i = 0; i < 1024; i++)
                {
                    if (mbufs[i] != null)
                    {
                        if (mbufs[i].is_dirty)
                        {
                            mfile.Seek((long)i * (256 * 1024), SeekOrigin.Begin);
                            mfile.Write(mbufs[i].data, 0, (256 * 1024));
                            mbufs[i].is_dirty = false;
                        }

                        if (mbufs[i].get_buf_age() > 120000)
                        {
                            mbufs[i] = null;
                        }
                    }
                }

                xfile.SetLength(0);
                xfile.Seek(0, SeekOrigin.Begin);
                byte[] buf = new byte[4];
                OPS.set_dbn(buf, 0, USED_BLK_COUNT);
                xfile.Write(buf, 0, 4);
                xfile.Flush();
                DEFS.DEBUG("REF", "SAVED used block cnt = " + USED_BLK_COUNT);
            } //lock
        }
Example #5
0
        /*
         * Below three functions are exposed for public.
         * MP Safe function, can take long if there
         * are too many updates to be made.
         */
        public void touch_refcount(Red_Buffer wb, bool isinodefilel0)
        {
            DEFS.ASSERT(wb != null, "touch refcount needs the wb");

            if (wb.get_touchrefcnt_needed() == false)// || wb.get_ondisk_dbn() == 0)
            {
                return;
            }
            else
            {
                //DEFS.DEBUG("-REF-", "CTH refcount for dbn = " + wb.get_ondisk_dbn() + " inofile = " + isinodefilel0);
                wb.set_touchrefcnt_needed(false);
            }

            if (wb.get_level() == 0 && isinodefilel0)
            {
                m_wrloader.mod_refcount(0, wb.get_ondisk_dbn(), REFCNT_OP.TOUCH_REFCOUNT, wb, true);
            }
            else
            {
                DEFS.ASSERT(wb.get_level() > 0, "touch_refcount is only for indirects only, except for ino-L0!");
                m_wrloader.mod_refcount(0, wb.get_ondisk_dbn(), REFCNT_OP.TOUCH_REFCOUNT, wb, false);
            }
        }
Example #6
0
        /*
         * this will periodically check if any dbns have become free and free
         * it in the bitmap appropriately.
         */
        private void tServiceThread()
        {
            while (m_shutdown == false)
            {
                for (int i = 0; i < 10; i++)
                {
                    if (m_shutdown)
                    {
                        m_dbnbitmap.sync();
                        return;
                    }
                    System.Threading.Thread.Sleep(1000);
                }

                lock (GLOBALQ.m_deletelog2)
                {
                    int count = GLOBALQ.m_deletelog2.Count;
                    try
                    {
                        for (int i = 0; i < count; i++)
                        {
                            int dbn = (int)(GLOBALQ.m_deletelog2.ElementAt(0));
                            GLOBALQ.m_deletelog2.RemoveAt(0);
                            m_dbnbitmap.free_bit(dbn);
                        }
                        DEFS.DEBUGCLR("FREE", "Deallocated " + count + " dbns in this iteration");
                    }
                    catch (Exception e)
                    {
                        DEFS.DEBUGYELLOW("EXEPTION", "Caught in dellog : cnt = " + count + " and size = " +
                                         GLOBALQ.m_deletelog2.Count + " e.msg = " + e.Message);
                    }
                }
                m_dbnbitmap.sync();
            }
        }
        /*
         * Remove duplicate ino, fbn entries. keep the lastest one always
         */
        private void clean_clog_file_P1(string input, string output)
        {
            DEFS.DEBUG("DEDUPE", "Starting clog P1");

            Item fp       = new fingerprintCLOG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
            int  last_ino = 0;
            int  last_fbn = 0;

            byte[] buffer = new byte[fp.get_size()];

            FileStream fsrc  = new FileStream(input, FileMode.Open);
            FileStream fdest = new FileStream(output, FileMode.Create);

            long count = fsrc.Length / fp.get_size();
            int  final = 0;

            for (int i = 0; i < count; i++)
            {
                fsrc.Read(buffer, 0, fp.get_size());
                fp.parse_bytes(buffer, 0);

                fingerprintCLOG fpt = (fingerprintCLOG)fp;
                if (fpt.inode == last_ino && fpt.fbn == last_fbn)
                {
                    continue;
                }
                fdest.Write(buffer, 0, fp.get_size());
                last_ino = fpt.inode;
                last_fbn = fpt.fbn;
                final++;
            }
            fsrc.Close();
            fdest.Flush();
            fdest.Close();
            DEFS.DEBUG("DEDUPE", "Finished clog P1, count = " + count + " to " + final);
        }
 public int UnlockFile(String filename, long offset, long length, DokanFileInfo info)
 {
     DEFS.DEBUGCLR("DOKAN", "Unlockfile = " + filename);
     return(0);
 }
 public int MoveFile(String filename, String newname, bool replace, DokanFileInfo info)
 {
     DEFS.DEBUG("DOKAN", "Move file request for " + filename + " to " + newname);
     DEFS.ASSERT(false, "no move possible");
     return(0);// _fs.move_file(filename, newname);
 }
 public int SetFileAttributes(String filename, FileAttributes attr, DokanFileInfo info)
 {
     DEFS.DEBUGCLR("DOKAN", "Trying to set file attributes " + attr + "," + filename);
     return(-1);
 }
        /*
         * The crux.
         */
        private void GenDedupeScript(string fpdbdata, string clogdata, string mergedfile, string scriptfilepath)
        {
            int existing_dbn_counter = 0;

            DEFS.DEBUG("DEDUPE", "starting dedupe op genscript");
            FileStream fsrcfpdb = null;

            if (File.Exists(fpdbdata) == true)
            {
                fsrcfpdb = new FileStream(fpdbdata, FileMode.Open);
            }
            FileStream fsrcclog   = new FileStream(clogdata, FileMode.Open);
            FileStream fdest      = new FileStream(mergedfile, FileMode.Create);
            FileStream scriptfile = new FileStream(scriptfilepath, FileMode.Create);

            fingerprintCLOG fpclog = new fingerprintCLOG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
            fingerprintFPDB fpfpdb = new fingerprintFPDB(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
            fingerprintDMSG fpmsg  = new fingerprintDMSG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);

            int clogrecsize  = ((Item)fpclog).get_size();
            int fpdbrecsize  = ((Item)fpfpdb).get_size();
            int fpmsgrecsize = ((Item)fpmsg).get_size();

            int clogcnt = (int)(fsrcclog.Length / clogrecsize);

            byte[] currfp  = new byte[16];
            int    currdbn = -1;

            byte[] buffer1 = new byte[clogrecsize];
            byte[] buffer2 = new byte[fpdbrecsize];
            byte[] buffer3 = new byte[fpmsgrecsize];

            while (clogcnt-- > 0)
            {
                fsrcclog.Read(buffer1, 0, clogrecsize);
                ((Item)fpclog).parse_bytes(buffer1, 0);

                if (compare_fp(currfp, fpclog.fp) != 0)
                {
                    for (int i = 0; i < 16; i++)
                    {
                        currfp[i] = fpclog.fp[i];
                    }
                    currdbn = fpclog.dbn;

                    //check if this fp is thre in fpdb, if yes get the dbn.
                    bool foundinfpdb = false;

                    if (fsrcfpdb != null && fsrcfpdb.Position < fsrcfpdb.Length)
                    {
                        do
                        {
                            fsrcfpdb.Read(buffer2, 0, fpdbrecsize);
                            ((Item)fpfpdb).parse_bytes(buffer2, 0);
                            fdest.Write(buffer2, 0, fpdbrecsize);
                            //Console.WriteLine("Read FPDB file : " + fsrcfpdb.Position + " : " + ((Item)fpfpdb).get_string_rep() + " curr=" + currdbn);

                            if (compare_fp(currfp, fpfpdb.fp) == 0)
                            {
                                currdbn     = fpfpdb.dbn; //let dedupe to old block preferably
                                foundinfpdb = true;
                                existing_dbn_counter++;
                                break;
                            }
                            else if ((compare_fp(currfp, fpfpdb.fp) > 0))
                            {
                                fsrcfpdb.Position -= fpdbrecsize;
                                break;
                            }
                        } while (fsrcfpdb.Position < fsrcfpdb.Length);
                    }

                    if (foundinfpdb == false)
                    {
                        //write to new fpdb, which was encounted from newly written data.
                        fpfpdb.dbn = currdbn;
                        for (int i = 0; i < 16; i++)
                        {
                            fpfpdb.fp[i] = currfp[i];
                        }
                        ((Item)fpfpdb).get_bytes(buffer2, 0);
                        fdest.Write(buffer2, 0, fpdbrecsize);
                    }
                }

                //dont have to copy the same duplicates. i.e the first dbn which we saw from some file
                //need not be deduped to the same file right?

                if (currdbn != fpclog.dbn)
                {
                    //push this to the messagequeue scriptfile
                    fpmsg.fsid           = fpclog.fsid;
                    fpmsg.inode          = fpclog.inode;
                    fpmsg.fbn            = fpclog.fbn;
                    fpmsg.sourcedbn      = fpclog.dbn;
                    fpmsg.destinationdbn = currdbn;
                    for (int i = 0; i < 16; i++)
                    {
                        fpmsg.fp[i] = fpclog.fp[i];
                    }

                    ((Item)fpmsg).get_bytes(buffer3, 0);
                    scriptfile.Write(buffer3, 0, fpmsgrecsize);
                }
            }

            if (fsrcfpdb != null)
            {
                fsrcfpdb.Close();
            }
            fsrcclog.Close();
            fdest.Flush();
            fdest.Close();
            scriptfile.Flush();
            scriptfile.Close();

            DEFS.DEBUG("DEDUPE", "finishing dedupe op Genscript : EXISTING : " + existing_dbn_counter);
        }
 private void remove_file(string file)
 {
     File.Delete(file);
     DEFS.DEBUG("DEDUPE", "Removing file " + file);
 }
        public BackupJobDetailsUI(string name, List <backup_pair> pairs, List <job_item> joblist)
        {
            InitializeComponent();
            textBox3.Text = name;

            if (pairs != null)
            {
                for (int i = 0; i < pairs.Count; i++)
                {
                    backup_pair bp = pairs.ElementAt(i);
                    dgv_fldlist.Rows.Add(new string[] { bp.SourcePath, bp.DestinationPath, (bp.IsFile)? "Y": "N" });
                }
                dgv_fldlist.Invalidate();
            }

            if (joblist != null)
            {
                for (int i = 0; i < joblist.Count; i++)
                {
                    job_item ji = joblist.ElementAt(i);
                    dgv_joblist.Rows.Add(new string[] { ji.JobID.ToString(), ji.JobName, ji.StartTime, DEFS.getDataInStringRep(ji.NewCopiedData) });
                }
                dgv_joblist.Invalidate();
            }
        }
        int IComparer.Compare(object obj1, object obj2)
        {
            switch (((Item)obj1).get_itemtype())
            {
            case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
            {
                fingerprintCLOG c1 = (fingerprintCLOG)obj1;
                fingerprintCLOG c2 = (fingerprintCLOG)obj2;

                if (c1.fsid < c2.fsid)
                {
                    return(-1);
                }
                else if (c1.fsid > c2.fsid)
                {
                    return(1);
                }
                else
                {
                    if (c1.inode < c2.inode)
                    {
                        return(-1);
                    }
                    else if (c1.inode > c2.inode)
                    {
                        return(1);
                    }
                    else
                    {
                        if (c1.fbn < c2.fbn)
                        {
                            return(-1);
                        }
                        else if (c1.fbn > c2.fbn)
                        {
                            return(1);
                        }
                        else
                        {
                            if (c1.cnt > c2.cnt)
                            {
                                return(-1);
                            }
                            else if (c1.cnt < c2.cnt)
                            {
                                return(1);
                            }
                            else
                            {
                                return(0);         //can actually assert!
                            }
                        }
                    }
                }
            }

            //break; unreachable
            case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
            {
                fingerprintDMSG c1 = (fingerprintDMSG)obj1;
                fingerprintDMSG c2 = (fingerprintDMSG)obj2;

                if (c1.fsid < c2.fsid)
                {
                    return(-1);
                }
                else if (c1.fsid > c2.fsid)
                {
                    return(1);
                }
                else
                {
                    if (c1.inode < c2.inode)
                    {
                        return(-1);
                    }
                    else if (c1.inode > c2.inode)
                    {
                        return(1);
                    }
                    else
                    {
                        if (c1.fbn < c2.fbn)
                        {
                            return(-1);
                        }
                        else if (c1.fbn > c2.fbn)
                        {
                            return(1);
                        }
                        else
                        {
                            return(0);
                        }
                    }
                }
            }
                //break; unreachable
            }
            DEFS.ASSERT(false, "Shouldnt have come here 34234a23");
            return(0);
        }
        /*
         * Just get the list of all files and their properties. If a new lun is created
         * then the list must be reloaded. After the list is reloaded, then it can be displayed
         * on the UI.
         */
        public void load_lun_list(bool freshload)
        {
            Inode_Info[] inodes = REDDY.ptrIFSDMux.FindFilesInternalAPI(1, "\\");
            DEFS.DEBUG("lun", "Found " + inodes.Length + " luns in load_lun_list");
            lock (m_lunlist)
            {
                if (freshload)
                {
                    DEFS.ASSERT(m_lunlist.Count == 0, "some Lun_Items cannot already exist");
                }
                for (int i = 0; i < inodes.Length; i++)
                {
                    DEFS.DEBUG("lun", inodes[i].name);
                    bool exists = false;
                    if (freshload)
                    {
                        exists = false;
                    }
                    else
                    {
                        //find out..
                        for (int j = 0; j < m_lunlist.Count; j++)
                        {
                            try
                            {
                                Lun_Item li  = m_lunlist.ElementAt(j);
                                int      did = Int32.Parse(inodes[i].name);

                                if (li.drive_id == did)
                                {
                                    exists = true;
                                    break;
                                }
                            }
                            catch (Exception extp)
                            {
                                DEFS.DEBUG("lun", "Error in filename from LUNdisk");
                                DEFS.DEBUG("lun", "Exception:" + extp.Message);
                                exists = true; //just to skip adding this.
                            }
                        }
                    }

                    if (!exists)
                    {
                        //create item and insert.
                        try
                        {
                            Lun_Item li = new Lun_Item();
                            li.drive_id   = Int32.Parse(inodes[i].name);
                            li.drive_size = inodes[i].size;
                            li.ctime      = inodes[i].CreationTime.Ticks;
                            m_lunlist.AddFirst(li);
                        }
                        catch (Exception ep)
                        {
                            DEFS.DEBUG("lun", ep.Message);
                        }
                    }
                }
            }//lock
        }
 public int Unmount(DokanFileInfo info)
 {
     DEFS.ASSERT(false, "This is not called in C# dokan");
     unmountcalled2 = true;
     return(0);
 }
Example #17
0
        public void mod_refcount(int fsid, int dbn, REFCNT_OP optype, Red_Buffer wb, bool isinodefilel0)
        {
            DEFS.ASSERT(optype == REFCNT_OP.INCREMENT_REFCOUNT || /*optype == REFCNT_OP.DECREMENT_REFCOUNT ||*/
                        optype == REFCNT_OP.TOUCH_REFCOUNT ||     /*optype == REFCNT_OP.DO_LOAD || */
                        optype == REFCNT_OP.INCREMENT_REFCOUNT_ALLOC ||
                        optype == REFCNT_OP.DECREMENT_REFCOUNT_ONDEALLOC, "Wrong param in mod_refcount");

            DEFS.ASSERT(isinodefilel0 || (wb == null || wb.get_level() > 0), "wrong type to mod_refcount " + isinodefilel0 + (wb == null));

            UpdateReqI r = new UpdateReqI();

            r.optype = optype;
            r.dbn    = dbn;
            r.fsid   = fsid;

            switch (optype)
            {
            case REFCNT_OP.INCREMENT_REFCOUNT:
            case REFCNT_OP.INCREMENT_REFCOUNT_ALLOC:
                r.value = 1;
                break;

            //case REFCNT_OP.DECREMENT_REFCOUNT:
            case REFCNT_OP.DECREMENT_REFCOUNT_ONDEALLOC:
                r.value = -1;
                break;

            case REFCNT_OP.TOUCH_REFCOUNT:
                //case REFCNT_OP.DO_LOAD:
                r.value = 0;
                break;
            }

            r.blktype = (wb != null) ? ((isinodefilel0) ? BLK_TYPE.PUBLIC_INODE_FILE_L0 : wb.get_blk_type()) :
                        ((optype == REFCNT_OP.INCREMENT_REFCOUNT_ALLOC || optype == REFCNT_OP.DECREMENT_REFCOUNT_ONDEALLOC)?
                         BLK_TYPE.IGNORE : BLK_TYPE.REGULAR_FILE_L0);

            if (wb != null && (wb.get_level() > 0 || BLK_TYPE.PUBLIC_INODE_FILE_L0 == r.blktype))
            {
                lock (tfile0)
                {
                    CONFIG.Encrypt_Data_ForWrite(tmpiodatatfileW, wb.buf_to_data());
                    tfile0.Seek((long)tfilefbn * 4096, SeekOrigin.Begin);
                    tfile0.Write(tmpiodatatfileW, 0, 4096);
                    //DEFS.DEBUG("ENCY", "Wrote : " + OPS.ChecksumPageWRLoader(wb.buf_to_data()));
                    r.tfbn = tfilefbn;
                    tfilefbn++;
                }
            }
            else
            {
                r.tfbn = -1;
            }

            if (optype != REFCNT_OP.INCREMENT_REFCOUNT_ALLOC && optype != REFCNT_OP.DECREMENT_REFCOUNT &&
                optype != REFCNT_OP.DECREMENT_REFCOUNT_ONDEALLOC && optype != REFCNT_OP.TOUCH_REFCOUNT)
            {
                DEFS.DEBUG("REFCNT", "Queued update for " + r.blktype + ", dbn = " +
                           r.dbn + ", and operation = " + r.optype + ", transaction offset : " + r.tfbn);
            }

            GLOBALQ.m_reqi_queue.Add(r);
        }
Example #18
0
        /*
         * Will block on GLOBALQ.m_reqi_queue and take it to
         * its logical conclusion.
         */
        public void tServiceThread()
        {
            //long protected_blkdiff_counter = 0;
            long[] protected_blkdiff_counter = new long[1024];

            while (true)
            {
                UpdateReqI cu = (UpdateReqI)GLOBALQ.m_reqi_queue.Take();

                if (cu.optype == REFCNT_OP.SHUT_DOWN)
                {
                    internal_sync_and_flush_cache_advanced();
                    DEFS.ASSERT(GLOBALQ.m_reqi_queue.Count == 0, "There cannot be any pending updates when shutting down");
                    DEFS.DEBUGYELLOW("REF", "Bailing out now!!");
                    //dont take a lock here.

                    for (int i = 0; i < 1024; i++)
                    {
                        if (REDDY.FSIDList[i] == null || protected_blkdiff_counter[i] == 0)
                        {
                            continue;
                        }

                        REDDY.FSIDList[i].diff_upadate_logical_data(protected_blkdiff_counter[i]);
                        REDDY.FSIDList[i].set_dirty(true);
                        protected_blkdiff_counter[i] = 0;
                    }

                    cu.processed  = true;
                    m_initialized = false;
                    break;
                }

                if (cu.optype == REFCNT_OP.DO_SYNC)
                {
                    internal_sync_and_flush_cache_advanced();

                    //dont take a lock here.
                    for (int i = 0; i < 1024; i++)
                    {
                        if (REDDY.FSIDList[i] == null || protected_blkdiff_counter[i] == 0)
                        {
                            continue;
                        }

                        REDDY.FSIDList[i].diff_upadate_logical_data(protected_blkdiff_counter[i]);
                        REDDY.FSIDList[i].set_dirty(true);
                        protected_blkdiff_counter[i] = 0;
                    }
                    cu.processed = true;
                    tfile0.Flush();
                    mfile1.Flush();
                    dfile1.Flush();
                    continue;
                }

                if (cu.optype == REFCNT_OP.TAKE_DISK_SNAPSHOT ||
                    cu.optype == REFCNT_OP.UNDO_DISK_SNAPSHOT)
                {
                    int rbn_update = cu.tfbn; //overloaded since its just file offset.
                    load_wrbufx(rbn_update);  //will dowork
                    DEFS.ASSERT(cu.dbn == 0, "This should not be set");
                    DEFS.ASSERT(cu.optype == GLOBALQ.disk_snapshot_optype, "this must also match");
                    //DoSnapshotWork(rbn_update);
                    counter++;
                    total_ops++;
                    printspeed();
                    continue;
                }

                if (cu.dbn != 0)
                {
                    if (cu.optype == REFCNT_OP.DECREMENT_REFCOUNT_ONDEALLOC)
                    {
                        protected_blkdiff_counter[cu.fsid] -= 4096;
                    }
                    else if (cu.optype == REFCNT_OP.INCREMENT_REFCOUNT_ALLOC)
                    {
                        protected_blkdiff_counter[cu.fsid] += 4096;
                    }
                    //all other ops you can ignore.
                }

                int rbn = REFDEF.dbn_to_rbn(cu.dbn);
                total_ops++;
                counter++;

                /*
                 * Now if this has a child update pending, then we must clean it up.
                 * For each entry, i.e dbn, load the upto 1024, into memory and update
                 * the refcount. Essentially when we access this buffer - it must not
                 * have any pending update to itself or its children.
                 *
                 * How the children are updated depends on the blk_type, thats why so many
                 * cases.
                 */
                load_wrbufx(rbn);

                if (cu.optype == REFCNT_OP.GET_REFANDCHD_INFO)
                {
                    cu.processed = true;
                    continue;
                }

                int childcnt = GLOBALQ.WRObj[rbn].incoretbuf.get_childcount(cu.dbn);

                if (childcnt > 0)
                {
                    DEFS.DEBUG("CNTr", "Encountered child update for " + cu.dbn + " = " +
                               GLOBALQ.WRObj[rbn].incoretbuf.get_refcount(cu.dbn) + "," + childcnt);

                    if (cu.blktype == BLK_TYPE.REGULAR_FILE_L0)// || cu.blktype == BLK_TYPE.DIRFILE_L0)
                    {
                        /* Normal handling*/
                        //DEFS.ASSERT(cu.blktype == GLOBALQ.WRObj[rbn].incoretbuf.get_blk_type(cu.dbn), "Block mismatch");
                        DEFS.ASSERT(cu.tfbn == -1, "tfbn cannot be set for a level 0 block generally");
                        DEFS.ASSERT(false, "How can there be a childcnt update for a level zero block?");
                    }
                    else if (cu.blktype == BLK_TYPE.REGULAR_FILE_L1 || cu.blktype == BLK_TYPE.REGULAR_FILE_L2) /* ||
                                                                                                                * cu.blktype == BLK_TYPE.DIRFILE_L1 || cu.blktype == BLK_TYPE.DIRFILE_L2 ||
                                                                                                                * cu.blktype == BLK_TYPE.PUBLIC_INODE_FILE_L2 || cu.blktype == BLK_TYPE.PUBLIC_INODE_FILE_L1)*/
                    {
                        //DEFS.ASSERT(false, "Not yet implimented chdcnt in wrloader : " + REFDEF.get_string_rep(cu));
                        DEFS.ASSERT(cu.tfbn != -1, "Tfbn should've been set here.");
                        do_regular_dirORfile_work(cu, childcnt);
                        GLOBALQ.WRObj[rbn].incoretbuf.set_childcount(cu.dbn, 0);
                    }
                    else if (cu.blktype == BLK_TYPE.PUBLIC_INODE_FILE_L0)
                    {
                        DEFS.DEBUGCLR("------", "Do ino-L0 update work," + cu.optype + " , chdcnt = " + childcnt +
                                      " curr_refcnt = " + GLOBALQ.WRObj[rbn].incoretbuf.get_refcount(cu.dbn));
                        do_inode_refupdate_work(cu, childcnt);
                        GLOBALQ.WRObj[rbn].incoretbuf.set_childcount(cu.dbn, 0);
                    }
                    else
                    {
                        DEFS.ASSERT(false, "passed type = " + cu.blktype + "dbn = " + cu.dbn + " chdcnt = " + childcnt);
                    }
                }

                if (cu.optype != REFCNT_OP.TOUCH_REFCOUNT)
                {
                    /*
                     * Now that pending updates are propogated ,apply the queued update to this refcount.
                     * If it becomes free, notify that.
                     */
                    load_wrbufx(rbn);

                    apply_update_internal(cu.dbn, cu.blktype, cu.value, cu.optype, (cu.optype == REFCNT_OP.INCREMENT_REFCOUNT));

                    checkset_if_blockfree(cu.dbn, childcnt);
                }

                /* After the load, see if we have to clean up */
                if (cachesize > 15 * 1024)
                {
                    internal_sync_and_flush_cache_advanced();
                }
                printspeed();
            }

            tfile0.Flush();
            tfile0.Close();
            dfile1.Flush();
            dfile1.Close();
            mfile1.Flush();
            mfile1.Close();
        }
Example #19
0
        private bool backup_file(bool firstjob, string oldchecksumfile, string newchecksumfile, ref int curroffset, string sourcefile, string destfile)
        {
            DEFS.DEBUG("BACKUP", "Entering backup_file ( " + firstjob + "," + oldchecksumfile + "," +
                       newchecksumfile + "," + curroffset + "," + sourcefile + "," + destfile);

            MD5 md5 = System.Security.Cryptography.MD5.Create();
            fingerprintBACKUP fptemp1 = new fingerprintBACKUP();
            fingerprintBACKUP fptemp2 = new fingerprintBACKUP();

            if (firstjob)
            {
                FileInfo srcfi = new FileInfo(sourcefile);
                if (srcfi.Exists == false)
                {
                    REDDY.ptrIFSDMux.DeleteFile(2, destfile, null);
                    return(false);
                }
                else
                {
                    if (REDDY.ptrIFSDMux.CreateFile(2, destfile, FileAccess.ReadWrite, FileShare.ReadWrite, FileMode.Create, FileOptions.None, null) != 0)
                    {
                        MessageBox.Show("failed to create file 1");
                        return(false);
                    }

                    if (REDDY.ptrIFSDMux.SetEndOfFile(2, destfile, srcfi.Length, null) != 0)
                    {
                        MessageBox.Show("failed to seteof 1");
                        return(false);
                    }

                    Inode_Info di = REDDY.ptrIFSDMux.GetFileInfoInternalAPI(2, destfile);
                    REDDY.ptrIFSDMux.SetInternalFlag(2, destfile, 0, curroffset);

                    if (di == null)
                    {
                        MessageBox.Show("failed to get a valid di 1");
                        return(false);
                    }
                    int ino = di.ino;

                    byte[] buffer = new byte[4096];
                    byte[] tmpbuf = new byte[((Item)fptemp1).get_size()];
                    uint   wrote  = 0;

                    int bcount = OPS.NUML0(srcfi.Length);

                    FileStream fs = new FileStream(sourcefile, FileMode.Open);

                    long   outfileoffset = 0;
                    byte[] lastchunkbuf  = null;

                    for (int i = 0; i < bcount; i++)
                    {
                        int size = fs.Read(buffer, 0, 4096);
                        if (size < 4096)
                        {
                            lastchunkbuf = new byte[size];
                            for (int kx = size; kx < 4096; kx++)
                            {
                                buffer[kx] = 0;
                            }
                            for (int kx2 = 0; kx2 < size; kx2++)
                            {
                                lastchunkbuf[kx2] = buffer[kx2];
                            }
                        }
                        byte[] hash = md5.ComputeHash(buffer, 0, 4096);

                        fptemp1.inode = ino;
                        fptemp1.fbn   = i;
                        for (int k = 0; k < 16; k++)
                        {
                            fptemp1.fp[k] = hash[k];
                        }
                        ((Item)fptemp1).get_bytes(tmpbuf, 0);

                        if (REDDY.ptrIFSDMux.WriteFile(2, newchecksumfile, tmpbuf, ref wrote, curroffset, null) != 0)
                        {
                            MessageBox.Show("write failed, wrote = " + wrote);
                            return(false);
                        }
                        if (size > 0)
                        {
                            if (size == 4096)
                            {
                                if (REDDY.ptrIFSDMux.WriteFile(2, destfile, buffer, ref wrote, outfileoffset, null) != 0)
                                {
                                    MessageBox.Show("write failed ee, wrote = " + wrote);
                                    return(false);
                                }
                            }
                            else
                            {
                                if (REDDY.ptrIFSDMux.WriteFile(2, destfile, lastchunkbuf, ref wrote, outfileoffset, null) != 0)
                                {
                                    MessageBox.Show("write failed ee2, wrote = " + wrote);
                                    return(false);
                                }
                            }
                        }
                        newdatacopied += size;
                        outfileoffset += size;

                        curroffset += ((Item)fptemp1).get_size();
                    }
                    //if (REDDY.ptrIFSDMux.SetEndOfFile(2, destfile, srcfi.Length, null) != 0)
                    //{
                    //    MessageBox.Show("failed to seteof 1a");
                    //    return false;
                    //}
                    fs.Close();
                    REDDY.FSIDList[2].set_dirty(true);
                    return(true);
                }
            }
            else
            {
                DEFS.ASSERT(oldchecksumfile != null, "You must pass the oldchecksumfile path");

                FileInfo srcfi = new FileInfo(sourcefile);
                if (srcfi.Exists == false)
                {
                    REDDY.ptrIFSDMux.DeleteFile(2, destfile, null);
                    return(false);
                }
                else
                {
                    if (REDDY.ptrIFSDMux.CreateFile(2, destfile, FileAccess.ReadWrite, FileShare.ReadWrite, FileMode.CreateNew, FileOptions.None, null) != 0)
                    {
                        MessageBox.Show("Createfile has failed");
                        return(false);
                    }
                    if (REDDY.ptrIFSDMux.SetEndOfFile(2, destfile, srcfi.Length, null) != 0)
                    {
                        MessageBox.Show("Set eof has failed!");
                        return(false);
                    }

                    Inode_Info di         = REDDY.ptrIFSDMux.GetFileInfoInternalAPI(2, destfile);
                    int        localoffet = di.backupoffset;

                    REDDY.ptrIFSDMux.SetInternalFlag(2, destfile, 0, curroffset);

                    int ino = di.ino;

                    byte[] buffer = new byte[4096];
                    byte[] tmpbuf = new byte[((Item)fptemp1).get_size()];
                    uint   wrote  = 0;

                    int bcount = OPS.NUML0(srcfi.Length);

                    FileStream fs = new FileStream(sourcefile, FileMode.Open);

                    long   outfileoffset = 0;
                    byte[] lastchunkbuf  = null;

                    DEFS.DEBUG("--------", bcount + ", ArrangeStartingPosition LOOP ");
                    for (int i = 0; i < bcount; i++)
                    {
                        int size = fs.Read(buffer, 0, 4096);
                        if (size < 4096)
                        {
                            lastchunkbuf = new byte[size];
                            for (int kx = size; kx < 4096; kx++)
                            {
                                buffer[kx] = 0;
                            }
                            for (int kx2 = 0; kx2 < size; kx2++)
                            {
                                lastchunkbuf[kx2] = buffer[kx2];
                            }
                        }
                        byte[] hash = md5.ComputeHash(buffer, 0, 4096);
                        fptemp1.inode = ino;
                        fptemp1.fbn   = i;
                        for (int k = 0; k < 16; k++)
                        {
                            fptemp1.fp[k] = hash[k];
                        }

                        byte[] existinghash = new byte[24];
                        uint   readsize     = 0;

                        if (REDDY.ptrIFSDMux.ReadFile(2, oldchecksumfile, existinghash, ref readsize, localoffet, null) != 0)
                        {
                            MessageBox.Show("read failed, " + readsize + ",path = " + oldchecksumfile);
                            return(false);
                        }
                        ((Item)fptemp2).parse_bytes(existinghash, 0);

                        if (!(/* fptemp1.inode == fptemp2.inode &&*/ fptemp1.fbn == fptemp2.fbn &&
                              is_equal(fptemp1.fp, fptemp2.fp)))
                        {
                            if (size > 0)
                            {
                                if (size == 4096)
                                {
                                    if (REDDY.ptrIFSDMux.WriteFile(2, destfile, buffer, ref wrote, outfileoffset, null) != 0)
                                    {
                                        MessageBox.Show("write failed ee, wrote = " + wrote);
                                        return(false);
                                    }
                                }
                                else
                                {
                                    if (REDDY.ptrIFSDMux.WriteFile(2, destfile, lastchunkbuf, ref wrote, outfileoffset, null) != 0)
                                    {
                                        MessageBox.Show("write failed ee2, wrote = " + wrote);
                                        return(false);
                                    }
                                }
                            }
                            newdatacopied += size;
                        }

                        ((Item)fptemp1).get_bytes(tmpbuf, 0);
                        if (REDDY.ptrIFSDMux.WriteFile(2, newchecksumfile, tmpbuf, ref wrote, curroffset, null) != 0)
                        {
                            MessageBox.Show("write failed 22, wrote = " + wrote);
                            return(false);
                        }

                        curroffset    += ((Item)fptemp1).get_size();
                        localoffet    += ((Item)fptemp1).get_size();
                        outfileoffset += size;
                        //DEFS.DEBUG("---", bcount + "," + fs.Position);
                    }

                    fs.Close();

                    if (REDDY.ptrIFSDMux.SetEndOfFile(2, destfile, srcfi.Length, null) != 0)
                    {
                        MessageBox.Show("Set eof has failed! 2");
                        return(false);
                    }

                    DEFS.DEBUG("--------", bcount + ", ArrangeStartingPosition LOOP sdfsfda");
                    REDDY.FSIDList[2].set_dirty(true);
                    return(true);
                }
            }
        }
Example #20
0
        private void worker_thread()
        {
            /*
             * First check the folder for the last job exists/
             */
            update_ui(1, 0, "Working..");
            string oldjobpath = null;

            for (int id = (m_bt.JobCount - 1); id >= 0; id--)
            {
                string jobname = m_bt.fetch_job_name(id);
                if (jobname == null)
                {
                    continue;
                }
                else
                {
                    oldjobpath = "\\" + m_bt.TaskName + "\\" + jobname;
                    break;
                }
            }

            string newjobpath = "\\" + m_bt.TaskName + "\\" + newjobname;

            DEFS.DEBUG("BACKUP", "Found current job path = " + oldjobpath);

            //MessageBox.Show("Backup folders " + oldjobpath + "->" + newjobpath);
            if (oldjobpath != null)
            {
                if (REDDY.ptrIFSDMux.CloneDirectoryTlock(2, oldjobpath, 2, newjobpath) != true)
                {
                    MessageBox.Show("failed to create new job dir " + oldjobpath + "->" + newjobpath);
                    thread_stopped = true;
                    update_ui(0, 0, null);
                    return;
                }
            }
            else
            {
                if (REDDY.ptrIFSDMux.CreateDirectory(2, newjobpath, null) != 0)
                {
                    MessageBox.Show("failed to create new job dir");
                    thread_stopped = true;
                    update_ui(0, 0, null);
                    return;
                }
            }
            update_ui(1, 0, "[DONE]");

            update_ui(2, 0, "Working..");

            long totaldatasetsize = fetch_total_data_to_scan();

            DEFS.DEBUG("BACKUP", "Inititing target directory, " + newjobpath);

            /*
             * Now create a new checksum file, and start writing new checksums there.
             */
            string oldchecksumfilepath = "\\" + m_bt.TaskName + "\\" + newjobname + "\\checksumfile";
            string newchecksumfilepath = "\\" + m_bt.TaskName + "\\" + newjobname + "\\checksumfile.new";

            REDDY.ptrIFSDMux.CreateFile(2, newchecksumfilepath, FileAccess.ReadWrite, FileShare.ReadWrite, FileMode.CreateNew, FileOptions.None, null);

            bool isfirstjob = (oldjobpath == null);

            if (stop_thread == true)
            {
                thread_stopped = true;
                update_ui(0, 0, null);
                return;
            }

            int curroffset = 0;

            update_ui(2, 0, "[DONE] " + DEFS.getDataInStringRep(totaldatasetsize));


            for (int i = 0; i < m_bt.BackupPaths.Count; i++)
            {
                backup_pair bp = m_bt.BackupPaths.ElementAt(i);
                if (bp.IsFile)
                {
                    string destfile = "\\" + m_bt.TaskName + "\\" + newjobname + "\\" + bp.DestinationPath;
                    bool   ret      = backup_file(isfirstjob, oldchecksumfilepath, newchecksumfilepath, ref curroffset, bp.SourcePath, destfile);
                    if (ret == false)
                    {
                        DEFS.DEBUG("BACKUP", "Backup of file " + bp.SourcePath + " failed!");
                        break;
                    }
                }
                else
                {
                    string destfolder = "\\" + m_bt.TaskName + "\\" + newjobname + "\\" + bp.DestinationPath;
                    backup_folder(isfirstjob, oldchecksumfilepath, newchecksumfilepath, ref curroffset, bp.SourcePath, destfolder);
                }
            }
            REDDY.ptrIFSDMux.DeleteFile(2, oldchecksumfilepath, null);
            REDDY.ptrIFSDMux.RenameInode2a(2, newchecksumfilepath, "checksumfile");
            update_ui(5, 0, "[OKAY]");
            thread_stopped = true;
            update_ui(0, 0, null);
        }
        int IComparer.Compare(object obj1, object obj2)
        {
            switch (((Item)obj1).get_itemtype())
            {
            case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
            {
                fingerprintCLOG c1 = (fingerprintCLOG)obj1;
                fingerprintCLOG c2 = (fingerprintCLOG)obj2;

                for (int i = 0; i < 16; i++)
                {
                    if (c1.fp[i] < c2.fp[i])
                    {
                        return(-1);
                    }
                    else if (c1.fp[i] > c2.fp[i])
                    {
                        return(1);
                    }
                }
                return(0);
            }

            //break; unreachable
            case RECORD_TYPE.FINGERPRINT_RECORD_FPDB:
            {
                fingerprintFPDB c1 = (fingerprintFPDB)obj1;
                fingerprintFPDB c2 = (fingerprintFPDB)obj2;

                for (int i = 0; i < 16; i++)
                {
                    if (c1.fp[i] < c2.fp[i])
                    {
                        return(-1);
                    }
                    else if (c1.fp[i] > c2.fp[i])
                    {
                        return(1);
                    }
                }
                return(0);
            }

            //break; unreachable
            case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
            {
                fingerprintDMSG c1 = (fingerprintDMSG)obj1;
                fingerprintDMSG c2 = (fingerprintDMSG)obj2;

                for (int i = 0; i < 16; i++)
                {
                    if (c1.fp[i] < c2.fp[i])
                    {
                        return(-1);
                    }
                    else if (c1.fp[i] > c2.fp[i])
                    {
                        return(1);
                    }
                }
                return(0);
            }
                //break; unreachable
            }
            DEFS.ASSERT(false, "Shouldnt have come here wewrwr2");
            return(0);
        }
Example #22
0
 public void verify_savings()
 {
     DEFS.ASSERT(outputF != null, "Output file cannot be null in verifcation phase");
     long num_items = outputF.Length / _item.get_size();
 }
Example #23
0
        /*
         * We dont expect a write to come before opening because, cdirectory would
         * call a open_file() before inserting into the DIR CACHE. We shouldnt call
         * this with cfile-lock held.
         */
        public bool open_file(bool justcreated)
        {
            if (m_state == FILE_STATE.FILE_DELETED)
            {
                return(false);
            }
            else if (m_state == FILE_STATE.FILE_IN_DOKAN_IO)
            {
                DEFS.ASSERT(_mywip != null, "My wip cannot be null when dokan_io flag is set in open_file");
                return(true);
            }

            touch();
            if (_mywip == null)
            {
                lock (this)
                {
                    if (_mywip != null)
                    {
                        /*
                         * It could be the case that someone already opend it, maybe previous call
                         * that was locked in open_file(), just bail out.
                         */
                        DEFS.ASSERT(m_state != FILE_STATE.FILE_IN_DOKAN_IO, "Suddendly cannot be in dokan io when it was just null");
                        return(true);
                    }

                    lock (REDDY.FSIDList[m_associated_fsid])
                    {
                        _mywip = new RedFS_Inode(WIP_TYPE.REGULAR_FILE, m_inode, m_parent_inode);
                        long oldsize = _mywip.get_filesize();

                        RedFS_Inode inowip = REDDY.FSIDList[m_associated_fsid].get_inode_file_wip("OF:" + m_name);
                        DEFS.DEBUG("F(_mywip)", "Loaded ino= " + m_inode + "wip from disk, size = " + _mywip.get_filesize());

                        bool ret = OPS.Checkout_Wip2(inowip, _mywip, m_inode);

                        if (ret)
                        {
                            DEFS.DEBUG("FILE", "Loaded ino= " + m_inode + "wip from disk, size = " + _mywip.get_filesize());
                        }
                        else
                        {
                            DEFS.DEBUG("FILE", "Loaded ino = " + m_inode + " (new) size = " + _mywip.get_filesize());
                            _mywip.set_ino(m_parent_inode, m_inode);
                        }

                        DEFS.ASSERT(m_size == _mywip.get_filesize(), "File size should match, irrespecitive of weather its " +
                                    " from disk, (=0) then, or inserted from an existing dir load, >= 0 in that case, msize:" + m_size +
                                    " _mywip.size:" + _mywip.get_filesize() + " fname =" + m_name + " ino=" + m_inode + " beforeread size = " +
                                    oldsize + " contents : " + _mywip.get_string_rep2() + " ret = " + ret);

                        if (justcreated)
                        {
                            DEFS.ASSERT(ret == false, "This should be a new file " + _mywip.get_filesize() + " fname =" + m_name +
                                        " ino=" + m_inode + " beforeread size = " + oldsize + " contents : " + _mywip.get_string_rep2());
                            _mywip.setfilefsid_on_dirty(m_associated_fsid);
                            _mywip.is_dirty = true; //this must make it to disk.
                        }
                        REDDY.FSIDList[m_associated_fsid].sync_internal();
                        m_state = FILE_STATE.FILE_DEFAULT;
                    }
                }
            }
            return(true);
        }
        private void start_operation()
        {
            string input_clog       = CONFIG.GetBasePath() + "clog1";
            string clog_inofbn      = CONFIG.GetBasePath() + "clog.inofbn";
            string clog_inofbnclean = CONFIG.GetBasePath() + "clog.inofbn.clean";
            string clog_dbn         = CONFIG.GetBasePath() + "clog.dbn";
            string clog_dbnclean    = CONFIG.GetBasePath() + "clog.dbn.clean";
            string clog_fporder     = CONFIG.GetBasePath() + "clog.fporder";

            string fpdb_input = CONFIG.GetBasePath() + "fpdb1";

            string fpdb_dbncleaned = CONFIG.GetBasePath() + "fpdb.dbn.clean";
            string fpdb_fporder    = CONFIG.GetBasePath() + "fpdb.fporder";

            string fpdb_temp = CONFIG.GetBasePath() + "fpdb.temp";

            string dedupeScript_a = CONFIG.GetBasePath() + "script.dedupeop.temp";
            string dedupeScript   = CONFIG.GetBasePath() + "script.dedupeop";

            REDDY.ptrRedFS.dedupe_swap_changelog();

            if (File.Exists(input_clog) == false)
            {
                DEFS.DEBUG("DE", "Input clog does not exist, so no dedupe op now!");
                return;
            }

            /*
             * Prepare the changelog.
             */
            print_file(RECORD_TYPE.FINGERPRINT_RECORD_CLOG, DEDUP_SORT_ORDER.DBN_BASED, input_clog, input_clog + ".txt");
            m_stage = DEDUP_ENGINE_STAGE.SORT_CLOG_INOFBN;
            sort_dedupe_file(RECORD_TYPE.FINGERPRINT_RECORD_CLOG, input_clog, clog_inofbn, DEDUP_SORT_ORDER.INO_FBN_BASED);
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.SORT_CLOG_INOFBN, 100);

            print_file(RECORD_TYPE.FINGERPRINT_RECORD_CLOG, DEDUP_SORT_ORDER.INO_FBN_BASED, clog_inofbn, clog_inofbn + ".txt");
            m_stage = DEDUP_ENGINE_STAGE.CLEAN_CLOG_INOFBNSTALES;
            clean_clog_file_P1(clog_inofbn, clog_inofbnclean);
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.CLEAN_CLOG_INOFBNSTALES, 100);

            print_file(RECORD_TYPE.FINGERPRINT_RECORD_CLOG, DEDUP_SORT_ORDER.INO_FBN_BASED, clog_inofbnclean, clog_inofbnclean + ".txt");
            m_stage = DEDUP_ENGINE_STAGE.SORT_CLOG_DBN;
            sort_dedupe_file(RECORD_TYPE.FINGERPRINT_RECORD_CLOG, clog_inofbnclean, clog_dbn, DEDUP_SORT_ORDER.DBN_BASED);
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.SORT_CLOG_DBN, 100);

            //commenting this off for now.
            print_file(RECORD_TYPE.FINGERPRINT_RECORD_CLOG, DEDUP_SORT_ORDER.DBN_BASED, clog_dbn, clog_dbn + ".txt");
            m_stage = DEDUP_ENGINE_STAGE.CLEAN_CLOG_DBNSTALES;
            clean_clog_file_P2(clog_dbn, clog_dbnclean);
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.CLEAN_CLOG_DBNSTALES, 100);

            print_file(RECORD_TYPE.FINGERPRINT_RECORD_CLOG, DEDUP_SORT_ORDER.DBN_BASED, clog_dbnclean, clog_dbnclean + ".txt");
            m_stage = DEDUP_ENGINE_STAGE.SORT_CLOG_FPORDER;
            sort_dedupe_file(RECORD_TYPE.FINGERPRINT_RECORD_CLOG, clog_dbnclean, clog_fporder, DEDUP_SORT_ORDER.FINGERPRINT_BASED);
            print_file(RECORD_TYPE.FINGERPRINT_RECORD_CLOG, DEDUP_SORT_ORDER.FINGERPRINT_BASED, clog_fporder, clog_fporder + ".txt");
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.SORT_CLOG_FPORDER, 100);

            remove_file(input_clog);
            remove_file(clog_inofbn);
            remove_file(clog_inofbnclean);
            remove_file(clog_dbn);
            remove_file(clog_dbnclean);

            /*
             * Prepare the fpdb. we have 'clog_fporder' from above steps
             * We must have a delete log that should be applied to clean the FPDB,
             */
            if (File.Exists(fpdb_input))
            {
                print_file(RECORD_TYPE.FINGERPRINT_RECORD_FPDB, DEDUP_SORT_ORDER.DBN_BASED, fpdb_input, fpdb_input + ".old.txt");

                m_stage = DEDUP_ENGINE_STAGE.CLEAN_FPDB_DBNSTALES;
                clean_fpdb_P1(fpdb_input, fpdb_dbncleaned);
                remove_file(fpdb_input);
                ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.CLEAN_FPDB_DBNSTALES, 100);
                print_file(RECORD_TYPE.FINGERPRINT_RECORD_FPDB, DEDUP_SORT_ORDER.DBN_BASED, fpdb_dbncleaned, fpdb_dbncleaned + ".txt");

                m_stage = DEDUP_ENGINE_STAGE.SORT_FPDB_FPORDER;
                sort_dedupe_file(RECORD_TYPE.FINGERPRINT_RECORD_FPDB, fpdb_dbncleaned, fpdb_fporder, DEDUP_SORT_ORDER.FINGERPRINT_BASED);
                remove_file(fpdb_dbncleaned);
                print_file(RECORD_TYPE.FINGERPRINT_RECORD_FPDB, DEDUP_SORT_ORDER.FINGERPRINT_BASED, fpdb_fporder, fpdb_fporder + ".txt");
                ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.SORT_FPDB_FPORDER, 100);
            }

            /*
             * Do the dedupe operation. we have 'fpdb_fporder' from above steps.
             */
            m_stage = DEDUP_ENGINE_STAGE.GENERATE_DEDUPE_SCRIPT;
            GenDedupeScript(fpdb_fporder, clog_fporder, fpdb_temp, dedupeScript_a);
            print_file(RECORD_TYPE.FINGERPRINT_RECORD_MSG, DEDUP_SORT_ORDER.FINGERPRINT_BASED, dedupeScript_a, dedupeScript_a + ".txt");
            print_file(RECORD_TYPE.FINGERPRINT_RECORD_FPDB, DEDUP_SORT_ORDER.FINGERPRINT_BASED, fpdb_temp, fpdb_temp + ".txt");
            remove_file(fpdb_fporder);
            remove_file(clog_fporder);
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.GENERATE_DEDUPE_SCRIPT, 100);

            /*
             * Create the new fpdb.
             */
            m_stage = DEDUP_ENGINE_STAGE.SORT_FPDBT_DBN;
            sort_dedupe_file(RECORD_TYPE.FINGERPRINT_RECORD_FPDB, fpdb_temp, fpdb_input, DEDUP_SORT_ORDER.DBN_BASED);
            print_file(RECORD_TYPE.FINGERPRINT_RECORD_FPDB, DEDUP_SORT_ORDER.DBN_BASED, fpdb_input, fpdb_input + ".txt");
            remove_file(fpdb_temp);
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.SORT_FPDBT_DBN, 100);

            m_stage = DEDUP_ENGINE_STAGE.SORT_SCRIPT_INOFBN;
            sort_dedupe_file(RECORD_TYPE.FINGERPRINT_RECORD_MSG, dedupeScript_a, dedupeScript, DEDUP_SORT_ORDER.INO_FBN_BASED);
            print_file(RECORD_TYPE.FINGERPRINT_RECORD_MSG, DEDUP_SORT_ORDER.INO_FBN_BASED, dedupeScript, dedupeScript + ".txt");
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.SORT_SCRIPT_INOFBN, 100);


            //rename the file and get over with it.
            m_stage = DEDUP_ENGINE_STAGE.DEDUPE_MESSAGES;
            ACTUAL_DEDUPE(dedupeScript);
            remove_file(dedupeScript);
            remove_file(dedupeScript_a);
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.DEDUPE_MESSAGES, 100);
            m_stage = DEDUP_ENGINE_STAGE.DEDUPE_DONE;

            System.Threading.Thread.Sleep(2000);
            ui.Update_DedupeUI(DEDUP_ENGINE_STAGE.DEDUPE_DONE, 100);
        }
 public IFSD_Mux()
 {
     DEFS.DEBUG("IFSDMux", "Entering constructor if IFSDMux()");
     init_if_necessary();
     DEFS.DEBUG("IFSDMux", "Leaving constructor if IFSDMux()");
 }