private void sort_dedupe_file(RECORD_TYPE type, string input, string output, DEDUP_SORT_ORDER order)
        {
            Item ix = null;

            switch (type)
            {
            case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
                ix = new fingerprintCLOG(order);
                break;

            case RECORD_TYPE.FINGERPRINT_RECORD_FPDB:
                ix = new fingerprintFPDB(order);
                break;

            case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
                ix = new fingerprintDMSG(order);
                break;
            }

            SortAPI sapi = new SortAPI(input, output, ix);

            sapi.do_chunk_sort();
            sapi.do_merge_work();
            sapi.close_streams();
        }
        private void MSG_AGGREGATE(fingerprintDMSG msg)
        {
            if (msg == null)
            {
                if (counter > 0)
                {
                    REDDY.ptrIFSDMux.DoDedupeBatch(msglist);
                }

                DEFS.DEBUGYELLOW("BATCH", "DoDedupeBatch, counter = " + counter);
                for (int i = 0; i < 1024; i++)
                {
                    msglist[i] = null;
                }
                counter = 0;
                top     = -1;
                return;
            }
            else if ((top == -1) || (msg.fsid == msglist[top].fsid && msg.inode == msglist[top].inode &&
                                     OPS.SomeFBNToStartFBN(1, msg.fbn) == OPS.SomeFBNToStartFBN(1, msglist[top].fbn)))
            {
                int idx = (int)(msg.fbn % 1024);
                DEFS.ASSERT(msglist[idx] == null, "This cannot be populated already");
                msglist[idx] = DUP(msg);
                top          = idx;
                counter++;
            }
            else
            {
                //send msg here.
                MSG_AGGREGATE(null);
                MSG_AGGREGATE(msg);
                return;
            }
        }
        int IComparer.Compare(object obj1, object obj2)
        {
            switch (((Item)obj1).get_itemtype())
            {
            case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
            {
                fingerprintCLOG c1 = (fingerprintCLOG)obj1;
                fingerprintCLOG c2 = (fingerprintCLOG)obj2;

                if (c1.dbn < c2.dbn)
                {
                    return(-1);
                }
                else if (c1.dbn > c2.dbn)
                {
                    return(1);
                }
                return(0);
            }

            //break; unreachable
            case RECORD_TYPE.FINGERPRINT_RECORD_FPDB:
            {
                fingerprintFPDB c1 = (fingerprintFPDB)obj1;
                fingerprintFPDB c2 = (fingerprintFPDB)obj2;

                if (c1.dbn < c2.dbn)
                {
                    return(-1);
                }
                else if (c1.dbn > c2.dbn)
                {
                    return(1);
                }
                return(0);
            }

            case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
            {
                fingerprintDMSG c1 = (fingerprintDMSG)obj1;
                fingerprintDMSG c2 = (fingerprintDMSG)obj2;

                if (c1.sourcedbn < c2.sourcedbn)
                {
                    return(-1);
                }
                else if (c1.sourcedbn > c2.sourcedbn)
                {
                    return(1);
                }
                return(0);
            }
                //break;
            }
            DEFS.ASSERT(false, "Shouldnt have come here 3423423");
            return(0);
        }
        private fingerprintDMSG DUP(fingerprintDMSG msg2)
        {
            fingerprintDMSG obj = new fingerprintDMSG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);

            obj.fsid           = msg2.fsid;
            obj.inode          = msg2.inode;
            obj.fbn            = msg2.fbn;
            obj.sourcedbn      = msg2.sourcedbn;
            obj.destinationdbn = msg2.destinationdbn;
            //obj.fp
            return(obj);
        }
        private void print_file(RECORD_TYPE type, DEDUP_SORT_ORDER order, string fpath, string txtfile)
        {
            Item record = null;

            switch (type)
            {
            case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
                record = new fingerprintCLOG(order);
                break;

            case RECORD_TYPE.FINGERPRINT_RECORD_FPDB:
                record = new fingerprintFPDB(order);
                break;

            case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
                record = new fingerprintDMSG(order);
                break;
            }

            FileStream   fsrc  = new FileStream(fpath, FileMode.Open);
            FileStream   fdest = new FileStream(txtfile, FileMode.Create);
            StreamWriter log   = new StreamWriter(fdest);

            int count = (int)(fsrc.Length / record.get_size());

            byte[] buffer = new byte[record.get_size()];

            for (int i = 0; i < count; i++)
            {
                fsrc.Read(buffer, 0, record.get_size());
                record.parse_bytes(buffer, 0);
                log.WriteLine(record.get_string_rep());
            }
            log.Flush();
            log.Close();
            fsrc.Close();
        }
        /*
         * Send the messages and get work done.
         */
        private void ACTUAL_DEDUPE(string script)
        {
            FileStream fsrc   = new FileStream(script, FileMode.Open);
            Item       record = new fingerprintDMSG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);

            int count = (int)(fsrc.Length / record.get_size());

            byte[] buffer = new byte[record.get_size()];

            for (int i = 0; i < count; i++)
            {
                fsrc.Read(buffer, 0, record.get_size());
                record.parse_bytes(buffer, 0);

                fingerprintDMSG msg = (fingerprintDMSG)record;
                //REDDY.ptrIFSDMux.DoDedupe(msg.fsid, msg.inode, msg.fbn, msg.sourcedbn, msg.destinationdbn);
                MSG_AGGREGATE(msg);

                int progress = (i * 100) / count;
                ui.Update_DedupeUI(m_stage, progress);
            }
            MSG_AGGREGATE(null);
            fsrc.Close();
        }
        /*
         * The crux.
         */
        private void GenDedupeScript(string fpdbdata, string clogdata, string mergedfile, string scriptfilepath)
        {
            int existing_dbn_counter = 0;

            DEFS.DEBUG("DEDUPE", "starting dedupe op genscript");
            FileStream fsrcfpdb = null;

            if (File.Exists(fpdbdata) == true)
            {
                fsrcfpdb = new FileStream(fpdbdata, FileMode.Open);
            }
            FileStream fsrcclog   = new FileStream(clogdata, FileMode.Open);
            FileStream fdest      = new FileStream(mergedfile, FileMode.Create);
            FileStream scriptfile = new FileStream(scriptfilepath, FileMode.Create);

            fingerprintCLOG fpclog = new fingerprintCLOG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
            fingerprintFPDB fpfpdb = new fingerprintFPDB(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
            fingerprintDMSG fpmsg  = new fingerprintDMSG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);

            int clogrecsize  = ((Item)fpclog).get_size();
            int fpdbrecsize  = ((Item)fpfpdb).get_size();
            int fpmsgrecsize = ((Item)fpmsg).get_size();

            int clogcnt = (int)(fsrcclog.Length / clogrecsize);

            byte[] currfp  = new byte[16];
            int    currdbn = -1;

            byte[] buffer1 = new byte[clogrecsize];
            byte[] buffer2 = new byte[fpdbrecsize];
            byte[] buffer3 = new byte[fpmsgrecsize];

            while (clogcnt-- > 0)
            {
                fsrcclog.Read(buffer1, 0, clogrecsize);
                ((Item)fpclog).parse_bytes(buffer1, 0);

                if (compare_fp(currfp, fpclog.fp) != 0)
                {
                    for (int i = 0; i < 16; i++)
                    {
                        currfp[i] = fpclog.fp[i];
                    }
                    currdbn = fpclog.dbn;

                    //check if this fp is thre in fpdb, if yes get the dbn.
                    bool foundinfpdb = false;

                    if (fsrcfpdb != null && fsrcfpdb.Position < fsrcfpdb.Length)
                    {
                        do
                        {
                            fsrcfpdb.Read(buffer2, 0, fpdbrecsize);
                            ((Item)fpfpdb).parse_bytes(buffer2, 0);
                            fdest.Write(buffer2, 0, fpdbrecsize);
                            //Console.WriteLine("Read FPDB file : " + fsrcfpdb.Position + " : " + ((Item)fpfpdb).get_string_rep() + " curr=" + currdbn);

                            if (compare_fp(currfp, fpfpdb.fp) == 0)
                            {
                                currdbn     = fpfpdb.dbn; //let dedupe to old block preferably
                                foundinfpdb = true;
                                existing_dbn_counter++;
                                break;
                            }
                            else if ((compare_fp(currfp, fpfpdb.fp) > 0))
                            {
                                fsrcfpdb.Position -= fpdbrecsize;
                                break;
                            }
                        } while (fsrcfpdb.Position < fsrcfpdb.Length);
                    }

                    if (foundinfpdb == false)
                    {
                        //write to new fpdb, which was encounted from newly written data.
                        fpfpdb.dbn = currdbn;
                        for (int i = 0; i < 16; i++)
                        {
                            fpfpdb.fp[i] = currfp[i];
                        }
                        ((Item)fpfpdb).get_bytes(buffer2, 0);
                        fdest.Write(buffer2, 0, fpdbrecsize);
                    }
                }

                //dont have to copy the same duplicates. i.e the first dbn which we saw from some file
                //need not be deduped to the same file right?

                if (currdbn != fpclog.dbn)
                {
                    //push this to the messagequeue scriptfile
                    fpmsg.fsid           = fpclog.fsid;
                    fpmsg.inode          = fpclog.inode;
                    fpmsg.fbn            = fpclog.fbn;
                    fpmsg.sourcedbn      = fpclog.dbn;
                    fpmsg.destinationdbn = currdbn;
                    for (int i = 0; i < 16; i++)
                    {
                        fpmsg.fp[i] = fpclog.fp[i];
                    }

                    ((Item)fpmsg).get_bytes(buffer3, 0);
                    scriptfile.Write(buffer3, 0, fpmsgrecsize);
                }
            }

            if (fsrcfpdb != null)
            {
                fsrcfpdb.Close();
            }
            fsrcclog.Close();
            fdest.Flush();
            fdest.Close();
            scriptfile.Flush();
            scriptfile.Close();

            DEFS.DEBUG("DEDUPE", "finishing dedupe op Genscript : EXISTING : " + existing_dbn_counter);
        }
        int IComparer.Compare(object obj1, object obj2)
        {
            switch (((Item)obj1).get_itemtype())
            {
            case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
            {
                fingerprintCLOG c1 = (fingerprintCLOG)obj1;
                fingerprintCLOG c2 = (fingerprintCLOG)obj2;

                for (int i = 0; i < 16; i++)
                {
                    if (c1.fp[i] < c2.fp[i])
                    {
                        return(-1);
                    }
                    else if (c1.fp[i] > c2.fp[i])
                    {
                        return(1);
                    }
                }
                return(0);
            }

            //break; unreachable
            case RECORD_TYPE.FINGERPRINT_RECORD_FPDB:
            {
                fingerprintFPDB c1 = (fingerprintFPDB)obj1;
                fingerprintFPDB c2 = (fingerprintFPDB)obj2;

                for (int i = 0; i < 16; i++)
                {
                    if (c1.fp[i] < c2.fp[i])
                    {
                        return(-1);
                    }
                    else if (c1.fp[i] > c2.fp[i])
                    {
                        return(1);
                    }
                }
                return(0);
            }

            //break; unreachable
            case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
            {
                fingerprintDMSG c1 = (fingerprintDMSG)obj1;
                fingerprintDMSG c2 = (fingerprintDMSG)obj2;

                for (int i = 0; i < 16; i++)
                {
                    if (c1.fp[i] < c2.fp[i])
                    {
                        return(-1);
                    }
                    else if (c1.fp[i] > c2.fp[i])
                    {
                        return(1);
                    }
                }
                return(0);
            }
                //break; unreachable
            }
            DEFS.ASSERT(false, "Shouldnt have come here wewrwr2");
            return(0);
        }
        int IComparer.Compare(object obj1, object obj2)
        {
            switch (((Item)obj1).get_itemtype())
            {
            case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
            {
                fingerprintCLOG c1 = (fingerprintCLOG)obj1;
                fingerprintCLOG c2 = (fingerprintCLOG)obj2;

                if (c1.fsid < c2.fsid)
                {
                    return(-1);
                }
                else if (c1.fsid > c2.fsid)
                {
                    return(1);
                }
                else
                {
                    if (c1.inode < c2.inode)
                    {
                        return(-1);
                    }
                    else if (c1.inode > c2.inode)
                    {
                        return(1);
                    }
                    else
                    {
                        if (c1.fbn < c2.fbn)
                        {
                            return(-1);
                        }
                        else if (c1.fbn > c2.fbn)
                        {
                            return(1);
                        }
                        else
                        {
                            if (c1.cnt > c2.cnt)
                            {
                                return(-1);
                            }
                            else if (c1.cnt < c2.cnt)
                            {
                                return(1);
                            }
                            else
                            {
                                return(0);         //can actually assert!
                            }
                        }
                    }
                }
            }

            //break; unreachable
            case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
            {
                fingerprintDMSG c1 = (fingerprintDMSG)obj1;
                fingerprintDMSG c2 = (fingerprintDMSG)obj2;

                if (c1.fsid < c2.fsid)
                {
                    return(-1);
                }
                else if (c1.fsid > c2.fsid)
                {
                    return(1);
                }
                else
                {
                    if (c1.inode < c2.inode)
                    {
                        return(-1);
                    }
                    else if (c1.inode > c2.inode)
                    {
                        return(1);
                    }
                    else
                    {
                        if (c1.fbn < c2.fbn)
                        {
                            return(-1);
                        }
                        else if (c1.fbn > c2.fbn)
                        {
                            return(1);
                        }
                        else
                        {
                            return(0);
                        }
                    }
                }
            }
                //break; unreachable
            }
            DEFS.ASSERT(false, "Shouldnt have come here 34234a23");
            return(0);
        }