private void CheckSumBuf(RedFS_Inode wip, int fbn, int dbn, byte[] buffer, int offset)
{
lock (fpcache_buf)
{
if (fpcache_cnt == 1024)
{
fpcache_cnt = 0;
clogfile.Write(fpcache_buf, 0, fpcache_buf.Length);
clogfile.Flush();
}
if (wip.get_wiptype() == WIP_TYPE.REGULAR_FILE)
{
fingerprintCLOG fpt = (fingerprintCLOG)fptemp;
fpt.fsid = wip.get_filefsid();
fpt.inode = wip.get_ino();
fpt.fbn = fbn;
fpt.dbn = dbn;
fpt.cnt = (int)clogfile.Position;
byte[] hash = md5.ComputeHash(buffer, offset, 4096);
for (int i = 0; i < 16; i++)
{
fpt.fp[i] = hash[i];
}
fptemp.get_bytes(fpcache_buf, fpcache_cnt * fptemp.get_size());
fpcache_cnt++;
}
}
}
/*
* Need to look up refcount file/bitmap file.
*/
private void clean_clog_file_P2(string input, string output)
{
DEFS.DEBUG("DEDUPE", "Starting clog P2");
Item fp = new fingerprintCLOG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
byte[] buffer = new byte[fp.get_size()];
FileStream fsrc = new FileStream(input, FileMode.Open);
FileStream fdest = new FileStream(output, FileMode.Create);
long count = fsrc.Length / fp.get_size();
int final = 0;
for (int i = 0; i < count; i++)
{
fsrc.Read(buffer, 0, fp.get_size());
fp.parse_bytes(buffer, 0);
fingerprintCLOG fpt = (fingerprintCLOG)fp;
if (REDDY.ptrRedFS != null && REDDY.ptrRedFS.is_block_free(fpt.dbn))
{
continue;
}
fdest.Write(buffer, 0, fp.get_size());
final++;
}
fsrc.Close();
fdest.Flush();
fdest.Close();
DEFS.DEBUG("DEDUPE", "Finished clog P2, count = " + count + " to " + final);
}
private void CheckSumBuf(RedFS_Inode wip, Red_Buffer wb)
{
lock (fpcache_buf)
{
if (fpcache_cnt == 1024)
{
fpcache_cnt = 0;
clogfile.Write(fpcache_buf, 0, fpcache_buf.Length);
clogfile.Flush();
}
if (wb.get_level() == 0 && wip.get_wiptype() == WIP_TYPE.REGULAR_FILE)
{
fingerprintCLOG fpt = (fingerprintCLOG)fptemp;
fpt.fsid = wip.get_filefsid();
fpt.inode = wip.get_ino();
fpt.fbn = (int)wb.get_start_fbn();
fpt.dbn = wb.get_ondisk_dbn();
fpt.cnt = (int)clogfile.Position;
byte[] hash = md5.ComputeHash(wb.buf_to_data());
for (int i = 0; i < 16; i++)
{
fpt.fp[i] = hash[i];
}
fptemp.get_bytes(fpcache_buf, fpcache_cnt * fptemp.get_size());
fpcache_cnt++;
}
}
}
private void sort_dedupe_file(RECORD_TYPE type, string input, string output, DEDUP_SORT_ORDER order)
{
Item ix = null;
switch (type)
{
case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
ix = new fingerprintCLOG(order);
break;
case RECORD_TYPE.FINGERPRINT_RECORD_FPDB:
ix = new fingerprintFPDB(order);
break;
case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
ix = new fingerprintDMSG(order);
break;
}
SortAPI sapi = new SortAPI(input, output, ix);
sapi.do_chunk_sort();
sapi.do_merge_work();
sapi.close_streams();
}
int IComparer.Compare(object obj1, object obj2)
{
switch (((Item)obj1).get_itemtype())
{
case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
{
fingerprintCLOG c1 = (fingerprintCLOG)obj1;
fingerprintCLOG c2 = (fingerprintCLOG)obj2;
if (c1.dbn < c2.dbn)
{
return(-1);
}
else if (c1.dbn > c2.dbn)
{
return(1);
}
return(0);
}
//break; unreachable
case RECORD_TYPE.FINGERPRINT_RECORD_FPDB:
{
fingerprintFPDB c1 = (fingerprintFPDB)obj1;
fingerprintFPDB c2 = (fingerprintFPDB)obj2;
if (c1.dbn < c2.dbn)
{
return(-1);
}
else if (c1.dbn > c2.dbn)
{
return(1);
}
return(0);
}
case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
{
fingerprintDMSG c1 = (fingerprintDMSG)obj1;
fingerprintDMSG c2 = (fingerprintDMSG)obj2;
if (c1.sourcedbn < c2.sourcedbn)
{
return(-1);
}
else if (c1.sourcedbn > c2.sourcedbn)
{
return(1);
}
return(0);
}
//break;
}
DEFS.ASSERT(false, "Shouldnt have come here 3423423");
return(0);
}
private void print_file(RECORD_TYPE type, DEDUP_SORT_ORDER order, string fpath, string txtfile)
{
Item record = null;
switch (type)
{
case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
record = new fingerprintCLOG(order);
break;
case RECORD_TYPE.FINGERPRINT_RECORD_FPDB:
record = new fingerprintFPDB(order);
break;
case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
record = new fingerprintDMSG(order);
break;
}
FileStream fsrc = new FileStream(fpath, FileMode.Open);
FileStream fdest = new FileStream(txtfile, FileMode.Create);
StreamWriter log = new StreamWriter(fdest);
int count = (int)(fsrc.Length / record.get_size());
byte[] buffer = new byte[record.get_size()];
for (int i = 0; i < count; i++)
{
fsrc.Read(buffer, 0, record.get_size());
record.parse_bytes(buffer, 0);
log.WriteLine(record.get_string_rep());
}
log.Flush();
log.Close();
fsrc.Close();
}
/*
* Remove duplicate ino, fbn entries. keep the lastest one always
*/
private void clean_clog_file_P1(string input, string output)
{
DEFS.DEBUG("DEDUPE", "Starting clog P1");
Item fp = new fingerprintCLOG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
int last_ino = 0;
int last_fbn = 0;
byte[] buffer = new byte[fp.get_size()];
FileStream fsrc = new FileStream(input, FileMode.Open);
FileStream fdest = new FileStream(output, FileMode.Create);
long count = fsrc.Length / fp.get_size();
int final = 0;
for (int i = 0; i < count; i++)
{
fsrc.Read(buffer, 0, fp.get_size());
fp.parse_bytes(buffer, 0);
fingerprintCLOG fpt = (fingerprintCLOG)fp;
if (fpt.inode == last_ino && fpt.fbn == last_fbn)
{
continue;
}
fdest.Write(buffer, 0, fp.get_size());
last_ino = fpt.inode;
last_fbn = fpt.fbn;
final++;
}
fsrc.Close();
fdest.Flush();
fdest.Close();
DEFS.DEBUG("DEDUPE", "Finished clog P1, count = " + count + " to " + final);
}
/*
* The crux.
*/
private void GenDedupeScript(string fpdbdata, string clogdata, string mergedfile, string scriptfilepath)
{
int existing_dbn_counter = 0;
DEFS.DEBUG("DEDUPE", "starting dedupe op genscript");
FileStream fsrcfpdb = null;
if (File.Exists(fpdbdata) == true)
{
fsrcfpdb = new FileStream(fpdbdata, FileMode.Open);
}
FileStream fsrcclog = new FileStream(clogdata, FileMode.Open);
FileStream fdest = new FileStream(mergedfile, FileMode.Create);
FileStream scriptfile = new FileStream(scriptfilepath, FileMode.Create);
fingerprintCLOG fpclog = new fingerprintCLOG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
fingerprintFPDB fpfpdb = new fingerprintFPDB(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
fingerprintDMSG fpmsg = new fingerprintDMSG(DEDUP_SORT_ORDER.UNDEFINED_PLACEHOLDER);
int clogrecsize = ((Item)fpclog).get_size();
int fpdbrecsize = ((Item)fpfpdb).get_size();
int fpmsgrecsize = ((Item)fpmsg).get_size();
int clogcnt = (int)(fsrcclog.Length / clogrecsize);
byte[] currfp = new byte[16];
int currdbn = -1;
byte[] buffer1 = new byte[clogrecsize];
byte[] buffer2 = new byte[fpdbrecsize];
byte[] buffer3 = new byte[fpmsgrecsize];
while (clogcnt-- > 0)
{
fsrcclog.Read(buffer1, 0, clogrecsize);
((Item)fpclog).parse_bytes(buffer1, 0);
if (compare_fp(currfp, fpclog.fp) != 0)
{
for (int i = 0; i < 16; i++)
{
currfp[i] = fpclog.fp[i];
}
currdbn = fpclog.dbn;
//check if this fp is thre in fpdb, if yes get the dbn.
bool foundinfpdb = false;
if (fsrcfpdb != null && fsrcfpdb.Position < fsrcfpdb.Length)
{
do
{
fsrcfpdb.Read(buffer2, 0, fpdbrecsize);
((Item)fpfpdb).parse_bytes(buffer2, 0);
fdest.Write(buffer2, 0, fpdbrecsize);
//Console.WriteLine("Read FPDB file : " + fsrcfpdb.Position + " : " + ((Item)fpfpdb).get_string_rep() + " curr=" + currdbn);
if (compare_fp(currfp, fpfpdb.fp) == 0)
{
currdbn = fpfpdb.dbn; //let dedupe to old block preferably
foundinfpdb = true;
existing_dbn_counter++;
break;
}
else if ((compare_fp(currfp, fpfpdb.fp) > 0))
{
fsrcfpdb.Position -= fpdbrecsize;
break;
}
} while (fsrcfpdb.Position < fsrcfpdb.Length);
}
if (foundinfpdb == false)
{
//write to new fpdb, which was encounted from newly written data.
fpfpdb.dbn = currdbn;
for (int i = 0; i < 16; i++)
{
fpfpdb.fp[i] = currfp[i];
}
((Item)fpfpdb).get_bytes(buffer2, 0);
fdest.Write(buffer2, 0, fpdbrecsize);
}
}
//dont have to copy the same duplicates. i.e the first dbn which we saw from some file
//need not be deduped to the same file right?
if (currdbn != fpclog.dbn)
{
//push this to the messagequeue scriptfile
fpmsg.fsid = fpclog.fsid;
fpmsg.inode = fpclog.inode;
fpmsg.fbn = fpclog.fbn;
fpmsg.sourcedbn = fpclog.dbn;
fpmsg.destinationdbn = currdbn;
for (int i = 0; i < 16; i++)
{
fpmsg.fp[i] = fpclog.fp[i];
}
((Item)fpmsg).get_bytes(buffer3, 0);
scriptfile.Write(buffer3, 0, fpmsgrecsize);
}
}
if (fsrcfpdb != null)
{
fsrcfpdb.Close();
}
fsrcclog.Close();
fdest.Flush();
fdest.Close();
scriptfile.Flush();
scriptfile.Close();
DEFS.DEBUG("DEDUPE", "finishing dedupe op Genscript : EXISTING : " + existing_dbn_counter);
}
int IComparer.Compare(object obj1, object obj2)
{
switch (((Item)obj1).get_itemtype())
{
case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
{
fingerprintCLOG c1 = (fingerprintCLOG)obj1;
fingerprintCLOG c2 = (fingerprintCLOG)obj2;
for (int i = 0; i < 16; i++)
{
if (c1.fp[i] < c2.fp[i])
{
return(-1);
}
else if (c1.fp[i] > c2.fp[i])
{
return(1);
}
}
return(0);
}
//break; unreachable
case RECORD_TYPE.FINGERPRINT_RECORD_FPDB:
{
fingerprintFPDB c1 = (fingerprintFPDB)obj1;
fingerprintFPDB c2 = (fingerprintFPDB)obj2;
for (int i = 0; i < 16; i++)
{
if (c1.fp[i] < c2.fp[i])
{
return(-1);
}
else if (c1.fp[i] > c2.fp[i])
{
return(1);
}
}
return(0);
}
//break; unreachable
case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
{
fingerprintDMSG c1 = (fingerprintDMSG)obj1;
fingerprintDMSG c2 = (fingerprintDMSG)obj2;
for (int i = 0; i < 16; i++)
{
if (c1.fp[i] < c2.fp[i])
{
return(-1);
}
else if (c1.fp[i] > c2.fp[i])
{
return(1);
}
}
return(0);
}
//break; unreachable
}
DEFS.ASSERT(false, "Shouldnt have come here wewrwr2");
return(0);
}
int IComparer.Compare(object obj1, object obj2)
{
switch (((Item)obj1).get_itemtype())
{
case RECORD_TYPE.FINGERPRINT_RECORD_CLOG:
{
fingerprintCLOG c1 = (fingerprintCLOG)obj1;
fingerprintCLOG c2 = (fingerprintCLOG)obj2;
if (c1.fsid < c2.fsid)
{
return(-1);
}
else if (c1.fsid > c2.fsid)
{
return(1);
}
else
{
if (c1.inode < c2.inode)
{
return(-1);
}
else if (c1.inode > c2.inode)
{
return(1);
}
else
{
if (c1.fbn < c2.fbn)
{
return(-1);
}
else if (c1.fbn > c2.fbn)
{
return(1);
}
else
{
if (c1.cnt > c2.cnt)
{
return(-1);
}
else if (c1.cnt < c2.cnt)
{
return(1);
}
else
{
return(0); //can actually assert!
}
}
}
}
}
//break; unreachable
case RECORD_TYPE.FINGERPRINT_RECORD_MSG:
{
fingerprintDMSG c1 = (fingerprintDMSG)obj1;
fingerprintDMSG c2 = (fingerprintDMSG)obj2;
if (c1.fsid < c2.fsid)
{
return(-1);
}
else if (c1.fsid > c2.fsid)
{
return(1);
}
else
{
if (c1.inode < c2.inode)
{
return(-1);
}
else if (c1.inode > c2.inode)
{
return(1);
}
else
{
if (c1.fbn < c2.fbn)
{
return(-1);
}
else if (c1.fbn > c2.fbn)
{
return(1);
}
else
{
return(0);
}
}
}
}
//break; unreachable
}
DEFS.ASSERT(false, "Shouldnt have come here 34234a23");
return(0);
}
