static void GenCPFile(string input, string bfFile, string cpFile, int additionalXValues)
{
BloomFilter bf;
using (var file = File.OpenRead(bfFile))
{
bf = Serializer.Deserialize<BloomFilter>(file);
}
bf.SetHashFunctions(BloomFilter.DefaultHashFuncs());
var setOld = new List<int>();
var fciOld = new List<FileChunkInfo>();
ProcessFile(input, setOld, fciOld);
for (var i = 0; i < setOld.Count; ++i)
{
setOld[i] = setOld[i] & HashBitMask;
}
var n0 = 0;
foreach (var item in setOld)
{
var byteArr = BitConverter.GetBytes(item);
if (bf.Contains(byteArr))
{
n0++;
}
}
var d0 = (int)(Helper.EstimateD0(bf.Count, setOld.Count, n0, bf) + 3);
var _cp = new CharacteristicPolynomial(FieldOrder);
var xVal = GenXValues(FieldOrder, d0 + additionalXValues + VerificationNum);
var cpb = _cp.Calc(setOld, xVal);
var cpdata = new CPData
{
CPValues = cpb,
SetCount = setOld.Count,
};
using (var file = File.Create(cpFile))
{
Serializer.Serialize(file, cpdata);
}
}
static void Sync(string oldFileName, string newFileName, string outputFileName)
{
var setNew = new List<int>();
var fciNew = new List<FileChunkInfo>();
ProcessFile(newFileName, setNew, fciNew);
for (var i = 0; i < setNew.Count; ++i)
{
setNew[i] = setNew[i] & HashBitMask;
}
var setOld = new List<int>();
var fciOld = new List<FileChunkInfo>();
ProcessFile(oldFileName, setOld, fciOld);
for (var i = 0; i < setOld.Count; ++i)
{
setOld[i] = setOld[i] & HashBitMask;
}
// On device A.
var bf = GenerateBF(setNew);
// Send this bloom filter to device B, in device B
var n0 = 0;
foreach (var item in setOld)
{
var byteArr = BitConverter.GetBytes(item);
if (bf.Contains(byteArr))
{
n0++;
}
}
var d0 = (int)(Helper.EstimateD0(bf.Count, setOld.Count, n0, bf) + 3);
// Debug infor
var snmso = setNew.Except(setOld).ToList();
var somsn = setOld.Except(setNew).ToList();
var diff = snmso.Count + somsn.Count;
Debug.Assert(diff <= d0);
// 46337 is the last prime number which ^2 < (2^31 - 1)
// 1048583 is the smallest prime > 2^20
var _cp = new CharacteristicPolynomial(2147483647);
var xVal = new List<int>(d0);
for (var i = 0; i < d0; ++i)
{
xVal.Add(i);
}
var cpb = _cp.Calc(setOld, xVal);
// Send cpb to device A, in A:
var cpa = _cp.Calc(setNew, xVal);
var cpaocpb = _cp.Div(cpa, cpb);
List<int> p;
List<int> q;
_cp.Interpolate(cpaocpb, xVal,
setNew.Count - setOld.Count,
out p, out q);
// Verification.
var xValVer = new List<int>(VerificationNum);
for (var i = 0; i < VerificationNum; ++i)
{
xValVer.Add(d0 + 1 + i);
}
var cpaVer = _cp.Calc(setNew, xValVer);
var cpbVer = _cp.Calc(setOld, xValVer);
var cpaVeroCpbVer = _cp.Div(cpaVer, cpbVer);
for (var i = 0; i < VerificationNum; ++i)
{
var pval = _cp.CalcCoeff(p, xValVer[i]);
var qVal = _cp.CalcCoeff(q, xValVer[i]);
var verNum = _cp.DivGF(pval, qVal);
if (verNum != cpaVeroCpbVer[i])
{
logger.Debug("Verification failed, need to increase d0");
throw new InvalidOperationException();
}
}
var missingFromOldList = _cp.Factoring(p);
var checkList = missingFromOldList.Except(snmso).Count();
Debug.Assert(checkList == 0);
Debug.Assert(missingFromOldList.Count == snmso.Count);
var missingSet = new HashSet<int>();
foreach (var item in missingFromOldList)
{
missingSet.Add(item);
}
// Genereate patch file
var patchFile = new List<PatchData>();
using (var fs = new FileStream(newFileName, FileMode.Open, FileAccess.Read, FileShare.Read))
{
for (var i = 0; i < setNew.Count; ++i)
{
var currH = setNew[i];
var currPatch = new PatchData()
{
HashValue = currH
};
if (missingSet.Contains(currH))
{
// Need to include the actual data.
var currFileChunkInfo = fciNew[i];
fs.Position = currFileChunkInfo.Pos;
var fcData = new byte[currFileChunkInfo.Length];
var bRead = fs.Read(fcData, 0, currFileChunkInfo.Length);
if (bRead != currFileChunkInfo.Length)
{
throw new InvalidDataException();
}
currPatch.Data = fcData;
}
patchFile.Add(currPatch);
}
}
// Send patch to device B to reconstruct the file.
var existingSet = new Dictionary<int, int>();
for (var i = 0; i < setOld.Count; ++i)
{
existingSet.Add(setOld[i], i);
}
using (var fsout = new FileStream(outputFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
using (var fsOld = new FileStream(oldFileName, FileMode.Open, FileAccess.Read, FileShare.Read))
{
for (var i = 0; i < patchFile.Count; ++i)
{
var currPatch = patchFile[i];
if (currPatch.Data == null)
{
// Existing data.
var idx = existingSet[currPatch.HashValue];
var currFileChunkInfo = fciOld[idx];
fsOld.Position = currFileChunkInfo.Pos;
var fcData = new byte[currFileChunkInfo.Length];
var bRead = fsOld.Read(fcData, 0, currFileChunkInfo.Length);
if (bRead != currFileChunkInfo.Length)
{
throw new InvalidDataException();
}
fsout.Write(fcData, 0, fcData.Length);
}
else
{
// New data.
fsout.Write(currPatch.Data, 0, currPatch.Data.Length);
}
}
}
}
var bfSize = Helper.SizeOfBF(bf);
var cpbSize = Helper.SizeCPB(cpb);
var pfSize = Helper.SizeOfPatchFile(patchFile);
var totalBandwidth = bfSize + cpbSize + pfSize;
Console.WriteLine("Total: {0} bytes", totalBandwidth);
}
static void TestReconciliation()
{
var setA = new List<int> { 1, 3, 5 };
var setB = new List<int> { 3, 5 };
// In device A
var bf = GenerateBF(setA);
// Send this bloom filter to device B, in device B
var n0 = 0;
foreach (var item in setB)
{
var byteArr = BitConverter.GetBytes(item);
if (bf.Contains(byteArr))
{
n0++;
}
}
var d0 = Helper.EstimateD0(bf.Count, setB.Count, n0, bf);
var _cp = new CharacteristicPolynomial(97);
var xVal = new List<int>(d0);
for (var i = 0; i < d0; ++i)
{
xVal.Add(i);
}
var cpb = _cp.Calc(setB, xVal);
// Send cpb to device A, in A:
var cpa = _cp.Calc(setA, xVal);
var cpaocpb = _cp.Div(cpa, cpb);
List<int> p;
List<int> q;
_cp.Interpolate(cpaocpb, xVal,
setA.Count - setB.Count,
out p, out q);
var samsb = _cp.Factoring(p);
var sbmsa = _cp.Factoring(q);
}
static bool GenDeltaFile(string input, string cpFile, string deltaFile)
{
CPData cpd;
using (var file = File.OpenRead(cpFile))
{
cpd = Serializer.Deserialize<CPData>(file);
}
var cpbVer = new List<int>();
for (var i = 0; i < VerificationNum; ++i)
{
cpbVer.Add(cpd.CPValues[cpd.CPValues.Count - VerificationNum + i]);
}
cpd.CPValues.RemoveRange(cpd.CPValues.Count - VerificationNum, VerificationNum);
var cpb = cpd.CPValues;
var setNew = new List<int>();
var fciNew = new List<FileChunkInfo>();
ProcessFile(input, setNew, fciNew);
for (var i = 0; i < setNew.Count; ++i)
{
setNew[i] = setNew[i] & HashBitMask;
}
var _cp = new CharacteristicPolynomial(FieldOrder);
var d0 = cpb.Count;
var xVal = GenXValues(FieldOrder, d0);
var cpa = _cp.Calc(setNew, xVal);
var cpaocpb = _cp.Div(cpa, cpb);
List<int> p;
List<int> q;
_cp.Interpolate(cpaocpb, xVal,
setNew.Count - cpd.SetCount,
out p, out q);
// TODO: verification.
// If verification failed => return false;
var xValVer = GenXValues(FieldOrder, d0 + VerificationNum);
xValVer.RemoveRange(0, d0);
var cpaVer = _cp.Calc(setNew, xValVer);
var cpaVeroCpbVer = _cp.Div(cpaVer, cpbVer);
for (var i = 0; i < VerificationNum; ++i)
{
var pval = _cp.CalcCoeff(p, xValVer[i]);
var qVal = _cp.CalcCoeff(q, xValVer[i]);
var verNum = _cp.DivGF(pval, qVal);
if (verNum != cpaVeroCpbVer[i])
{
logger.Debug("Verification failed, need to increase d0");
return false;
}
}
var missingFromOldList = _cp.Factoring(p);
var missingSet = new HashSet<int>();
foreach (var item in missingFromOldList)
{
missingSet.Add(item);
}
// Genereate delta file.
var deltaData = new List<PatchData>();
using (var fs = new FileStream(input, FileMode.Open, FileAccess.Read, FileShare.Read))
{
for (var i = 0; i < setNew.Count; ++i)
{
var currH = setNew[i];
var currPatch = new PatchData()
{
HashValue = currH
};
if (missingSet.Contains(currH))
{
// Need to include the actual data.
var currFileChunkInfo = fciNew[i];
fs.Position = currFileChunkInfo.Pos;
var fcData = new byte[currFileChunkInfo.Length];
var bRead = fs.Read(fcData, 0, currFileChunkInfo.Length);
if (bRead != currFileChunkInfo.Length)
{
throw new InvalidDataException();
}
currPatch.Data = fcData;
}
deltaData.Add(currPatch);
}
}
using (var file = File.Create(deltaFile))
{
Serializer.Serialize(file, deltaData);
}
return true;
}