private void CheckSumBuf(RedFS_Inode wip, int fbn, int dbn, byte[] buffer, int offset) { lock (fpcache_buf) { if (fpcache_cnt == 1024) { fpcache_cnt = 0; clogfile.Write(fpcache_buf, 0, fpcache_buf.Length); clogfile.Flush(); } if (wip.get_wiptype() == WIP_TYPE.REGULAR_FILE) { fingerprintCLOG fpt = (fingerprintCLOG)fptemp; fpt.fsid = wip.get_filefsid(); fpt.inode = wip.get_ino(); fpt.fbn = fbn; fpt.dbn = dbn; fpt.cnt = (int)clogfile.Position; byte[] hash = md5.ComputeHash(buffer, offset, 4096); for (int i = 0; i < 16; i++) { fpt.fp[i] = hash[i]; } fptemp.get_bytes(fpcache_buf, fpcache_cnt * fptemp.get_size()); fpcache_cnt++; } } }
/* * Need to look up refcount file/bitmap file. */ private void clean_clog_file_P2(string input, string output) { DEFS.DEBUG("DEDUPE", "Starting clog P2"); Item fp = new fingerprintCLOG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER); byte[] buffer = new byte[fp.get_size()]; FileStream fsrc = new FileStream(input, FileMode.Open); FileStream fdest = new FileStream(output, FileMode.Create); long count = fsrc.Length / fp.get_size(); int final = 0; for (int i = 0; i < count; i++) { fsrc.Read(buffer, 0, fp.get_size()); fp.parse_bytes(buffer, 0); fingerprintCLOG fpt = (fingerprintCLOG)fp; if (REDDY.ptrRedFS != null && REDDY.ptrRedFS.is_block_free(fpt.dbn)) { continue; } fdest.Write(buffer, 0, fp.get_size()); final++; } fsrc.Close(); fdest.Flush(); fdest.Close(); DEFS.DEBUG("DEDUPE", "Finished clog P2, count = " + count + " to " + final); }
private void CheckSumBuf(RedFS_Inode wip, Red_Buffer wb) { lock (fpcache_buf) { if (fpcache_cnt == 1024) { fpcache_cnt = 0; clogfile.Write(fpcache_buf, 0, fpcache_buf.Length); clogfile.Flush(); } if (wb.get_level() == 0 && wip.get_wiptype() == WIP_TYPE.REGULAR_FILE) { fingerprintCLOG fpt = (fingerprintCLOG)fptemp; fpt.fsid = wip.get_filefsid(); fpt.inode = wip.get_ino(); fpt.fbn = (int)wb.get_start_fbn(); fpt.dbn = wb.get_ondisk_dbn(); fpt.cnt = (int)clogfile.Position; byte[] hash = md5.ComputeHash(wb.buf_to_data()); for (int i = 0; i < 16; i++) { fpt.fp[i] = hash[i]; } fptemp.get_bytes(fpcache_buf, fpcache_cnt * fptemp.get_size()); fpcache_cnt++; } } }
private void sort_dedupe_file(RECORD_TYPE type, string input, string output, DEDUP_SORT_ORDER order) { Item ix = null; switch (type) { case RECORD_TYPE.FINGERPRINT_RECORD_CLOG: ix = new fingerprintCLOG(order); break; case RECORD_TYPE.FINGERPRINT_RECORD_FPDB: ix = new fingerprintFPDB(order); break; case RECORD_TYPE.FINGERPRINT_RECORD_MSG: ix = new fingerprintDMSG(order); break; } SortAPI sapi = new SortAPI(input, output, ix); sapi.do_chunk_sort(); sapi.do_merge_work(); sapi.close_streams(); }
int IComparer.Compare(object obj1, object obj2) { switch (((Item)obj1).get_itemtype()) { case RECORD_TYPE.FINGERPRINT_RECORD_CLOG: { fingerprintCLOG c1 = (fingerprintCLOG)obj1; fingerprintCLOG c2 = (fingerprintCLOG)obj2; if (c1.dbn < c2.dbn) { return(-1); } else if (c1.dbn > c2.dbn) { return(1); } return(0); } //break; unreachable case RECORD_TYPE.FINGERPRINT_RECORD_FPDB: { fingerprintFPDB c1 = (fingerprintFPDB)obj1; fingerprintFPDB c2 = (fingerprintFPDB)obj2; if (c1.dbn < c2.dbn) { return(-1); } else if (c1.dbn > c2.dbn) { return(1); } return(0); } case RECORD_TYPE.FINGERPRINT_RECORD_MSG: { fingerprintDMSG c1 = (fingerprintDMSG)obj1; fingerprintDMSG c2 = (fingerprintDMSG)obj2; if (c1.sourcedbn < c2.sourcedbn) { return(-1); } else if (c1.sourcedbn > c2.sourcedbn) { return(1); } return(0); } //break; } DEFS.ASSERT(false, "Shouldnt have come here 3423423"); return(0); }
private void print_file(RECORD_TYPE type, DEDUP_SORT_ORDER order, string fpath, string txtfile) { Item record = null; switch (type) { case RECORD_TYPE.FINGERPRINT_RECORD_CLOG: record = new fingerprintCLOG(order); break; case RECORD_TYPE.FINGERPRINT_RECORD_FPDB: record = new fingerprintFPDB(order); break; case RECORD_TYPE.FINGERPRINT_RECORD_MSG: record = new fingerprintDMSG(order); break; } FileStream fsrc = new FileStream(fpath, FileMode.Open); FileStream fdest = new FileStream(txtfile, FileMode.Create); StreamWriter log = new StreamWriter(fdest); int count = (int)(fsrc.Length / record.get_size()); byte[] buffer = new byte[record.get_size()]; for (int i = 0; i < count; i++) { fsrc.Read(buffer, 0, record.get_size()); record.parse_bytes(buffer, 0); log.WriteLine(record.get_string_rep()); } log.Flush(); log.Close(); fsrc.Close(); }
/* * Remove duplicate ino, fbn entries. keep the lastest one always */ private void clean_clog_file_P1(string input, string output) { DEFS.DEBUG("DEDUPE", "Starting clog P1"); Item fp = new fingerprintCLOG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER); int last_ino = 0; int last_fbn = 0; byte[] buffer = new byte[fp.get_size()]; FileStream fsrc = new FileStream(input, FileMode.Open); FileStream fdest = new FileStream(output, FileMode.Create); long count = fsrc.Length / fp.get_size(); int final = 0; for (int i = 0; i < count; i++) { fsrc.Read(buffer, 0, fp.get_size()); fp.parse_bytes(buffer, 0); fingerprintCLOG fpt = (fingerprintCLOG)fp; if (fpt.inode == last_ino && fpt.fbn == last_fbn) { continue; } fdest.Write(buffer, 0, fp.get_size()); last_ino = fpt.inode; last_fbn = fpt.fbn; final++; } fsrc.Close(); fdest.Flush(); fdest.Close(); DEFS.DEBUG("DEDUPE", "Finished clog P1, count = " + count + " to " + final); }
/* * The crux. */ private void GenDedupeScript(string fpdbdata, string clogdata, string mergedfile, string scriptfilepath) { int existing_dbn_counter = 0; DEFS.DEBUG("DEDUPE", "starting dedupe op genscript"); FileStream fsrcfpdb = null; if (File.Exists(fpdbdata) == true) { fsrcfpdb = new FileStream(fpdbdata, FileMode.Open); } FileStream fsrcclog = new FileStream(clogdata, FileMode.Open); FileStream fdest = new FileStream(mergedfile, FileMode.Create); FileStream scriptfile = new FileStream(scriptfilepath, FileMode.Create); fingerprintCLOG fpclog = new fingerprintCLOG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER); fingerprintFPDB fpfpdb = new fingerprintFPDB(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER); fingerprintDMSG fpmsg = new fingerprintDMSG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER); int clogrecsize = ((Item)fpclog).get_size(); int fpdbrecsize = ((Item)fpfpdb).get_size(); int fpmsgrecsize = ((Item)fpmsg).get_size(); int clogcnt = (int)(fsrcclog.Length / clogrecsize); byte[] currfp = new byte[16]; int currdbn = -1; byte[] buffer1 = new byte[clogrecsize]; byte[] buffer2 = new byte[fpdbrecsize]; byte[] buffer3 = new byte[fpmsgrecsize]; while (clogcnt-- > 0) { fsrcclog.Read(buffer1, 0, clogrecsize); ((Item)fpclog).parse_bytes(buffer1, 0); if (compare_fp(currfp, fpclog.fp) != 0) { for (int i = 0; i < 16; i++) { currfp[i] = fpclog.fp[i]; } currdbn = fpclog.dbn; //check if this fp is thre in fpdb, if yes get the dbn. bool foundinfpdb = false; if (fsrcfpdb != null && fsrcfpdb.Position < fsrcfpdb.Length) { do { fsrcfpdb.Read(buffer2, 0, fpdbrecsize); ((Item)fpfpdb).parse_bytes(buffer2, 0); fdest.Write(buffer2, 0, fpdbrecsize); //Console.WriteLine("Read FPDB file : " + fsrcfpdb.Position + " : " + ((Item)fpfpdb).get_string_rep() + " curr=" + currdbn); if (compare_fp(currfp, fpfpdb.fp) == 0) { currdbn = fpfpdb.dbn; //let dedupe to old block preferably foundinfpdb = true; existing_dbn_counter++; break; } else if ((compare_fp(currfp, fpfpdb.fp) > 0)) { fsrcfpdb.Position -= fpdbrecsize; break; } } while (fsrcfpdb.Position < fsrcfpdb.Length); } if (foundinfpdb == false) { //write to new fpdb, which was encounted from newly written data. fpfpdb.dbn = currdbn; for (int i = 0; i < 16; i++) { fpfpdb.fp[i] = currfp[i]; } ((Item)fpfpdb).get_bytes(buffer2, 0); fdest.Write(buffer2, 0, fpdbrecsize); } } //dont have to copy the same duplicates. i.e the first dbn which we saw from some file //need not be deduped to the same file right? if (currdbn != fpclog.dbn) { //push this to the messagequeue scriptfile fpmsg.fsid = fpclog.fsid; fpmsg.inode = fpclog.inode; fpmsg.fbn = fpclog.fbn; fpmsg.sourcedbn = fpclog.dbn; fpmsg.destinationdbn = currdbn; for (int i = 0; i < 16; i++) { fpmsg.fp[i] = fpclog.fp[i]; } ((Item)fpmsg).get_bytes(buffer3, 0); scriptfile.Write(buffer3, 0, fpmsgrecsize); } } if (fsrcfpdb != null) { fsrcfpdb.Close(); } fsrcclog.Close(); fdest.Flush(); fdest.Close(); scriptfile.Flush(); scriptfile.Close(); DEFS.DEBUG("DEDUPE", "finishing dedupe op Genscript : EXISTING : " + existing_dbn_counter); }
int IComparer.Compare(object obj1, object obj2) { switch (((Item)obj1).get_itemtype()) { case RECORD_TYPE.FINGERPRINT_RECORD_CLOG: { fingerprintCLOG c1 = (fingerprintCLOG)obj1; fingerprintCLOG c2 = (fingerprintCLOG)obj2; for (int i = 0; i < 16; i++) { if (c1.fp[i] < c2.fp[i]) { return(-1); } else if (c1.fp[i] > c2.fp[i]) { return(1); } } return(0); } //break; unreachable case RECORD_TYPE.FINGERPRINT_RECORD_FPDB: { fingerprintFPDB c1 = (fingerprintFPDB)obj1; fingerprintFPDB c2 = (fingerprintFPDB)obj2; for (int i = 0; i < 16; i++) { if (c1.fp[i] < c2.fp[i]) { return(-1); } else if (c1.fp[i] > c2.fp[i]) { return(1); } } return(0); } //break; unreachable case RECORD_TYPE.FINGERPRINT_RECORD_MSG: { fingerprintDMSG c1 = (fingerprintDMSG)obj1; fingerprintDMSG c2 = (fingerprintDMSG)obj2; for (int i = 0; i < 16; i++) { if (c1.fp[i] < c2.fp[i]) { return(-1); } else if (c1.fp[i] > c2.fp[i]) { return(1); } } return(0); } //break; unreachable } DEFS.ASSERT(false, "Shouldnt have come here wewrwr2"); return(0); }
int IComparer.Compare(object obj1, object obj2) { switch (((Item)obj1).get_itemtype()) { case RECORD_TYPE.FINGERPRINT_RECORD_CLOG: { fingerprintCLOG c1 = (fingerprintCLOG)obj1; fingerprintCLOG c2 = (fingerprintCLOG)obj2; if (c1.fsid < c2.fsid) { return(-1); } else if (c1.fsid > c2.fsid) { return(1); } else { if (c1.inode < c2.inode) { return(-1); } else if (c1.inode > c2.inode) { return(1); } else { if (c1.fbn < c2.fbn) { return(-1); } else if (c1.fbn > c2.fbn) { return(1); } else { if (c1.cnt > c2.cnt) { return(-1); } else if (c1.cnt < c2.cnt) { return(1); } else { return(0); //can actually assert! } } } } } //break; unreachable case RECORD_TYPE.FINGERPRINT_RECORD_MSG: { fingerprintDMSG c1 = (fingerprintDMSG)obj1; fingerprintDMSG c2 = (fingerprintDMSG)obj2; if (c1.fsid < c2.fsid) { return(-1); } else if (c1.fsid > c2.fsid) { return(1); } else { if (c1.inode < c2.inode) { return(-1); } else if (c1.inode > c2.inode) { return(1); } else { if (c1.fbn < c2.fbn) { return(-1); } else if (c1.fbn > c2.fbn) { return(1); } else { return(0); } } } } //break; unreachable } DEFS.ASSERT(false, "Shouldnt have come here 34234a23"); return(0); }