Exemple #1
0
        static void GenCPFile(string input, string bfFile, string cpFile, int additionalXValues)
        {
            BloomFilter bf;
            using (var file = File.OpenRead(bfFile))
            {
                bf = Serializer.Deserialize<BloomFilter>(file);
            }
            bf.SetHashFunctions(BloomFilter.DefaultHashFuncs());

            var setOld = new List<int>();
            var fciOld = new List<FileChunkInfo>();
            ProcessFile(input, setOld, fciOld);
            for (var i = 0; i < setOld.Count; ++i)
            {
                setOld[i] = setOld[i] & HashBitMask;
            }

            var n0 = 0;
            foreach (var item in setOld)
            {
                var byteArr = BitConverter.GetBytes(item);
                if (bf.Contains(byteArr))
                {
                    n0++;
                }
            }
            var d0 = (int)(Helper.EstimateD0(bf.Count, setOld.Count, n0, bf) + 3);

            var _cp = new CharacteristicPolynomial(FieldOrder);
            var xVal = GenXValues(FieldOrder, d0 + additionalXValues + VerificationNum);

            var cpb = _cp.Calc(setOld, xVal);

            var cpdata = new CPData
            {
                CPValues = cpb,
                SetCount = setOld.Count,
            };

            using (var file = File.Create(cpFile))
            {
                Serializer.Serialize(file, cpdata);
            }
        }
Exemple #2
0
        static void Sync(string oldFileName, string newFileName, string outputFileName)
        {
            var setNew = new List<int>();
            var fciNew = new List<FileChunkInfo>();
            ProcessFile(newFileName, setNew, fciNew);
            for (var i = 0; i < setNew.Count; ++i)
            {
                setNew[i] = setNew[i] & HashBitMask;
            }
            var setOld = new List<int>();
            var fciOld = new List<FileChunkInfo>();
            ProcessFile(oldFileName, setOld, fciOld);
            for (var i = 0; i < setOld.Count; ++i)
            {
                setOld[i] = setOld[i] & HashBitMask;
            }

            // On device A.
            var bf = GenerateBF(setNew);

            // Send this bloom filter to device B, in device B
            var n0 = 0;
            foreach (var item in setOld)
            {
                var byteArr = BitConverter.GetBytes(item);
                if (bf.Contains(byteArr))
                {
                    n0++;
                }
            }
            var d0 = (int)(Helper.EstimateD0(bf.Count, setOld.Count, n0, bf) + 3);

            // Debug infor
            var snmso = setNew.Except(setOld).ToList();
            var somsn = setOld.Except(setNew).ToList();
            var diff = snmso.Count + somsn.Count;

            Debug.Assert(diff <= d0);

            // 46337 is the last prime number which ^2 < (2^31 - 1)
            // 1048583 is the smallest prime > 2^20
            var _cp = new CharacteristicPolynomial(2147483647);
            var xVal = new List<int>(d0);
            for (var i = 0; i < d0; ++i)
            {
                xVal.Add(i);
            }
            var cpb = _cp.Calc(setOld, xVal);

            // Send cpb to device A, in A:
            var cpa = _cp.Calc(setNew, xVal);
            var cpaocpb = _cp.Div(cpa, cpb);

            List<int> p;
            List<int> q;
            _cp.Interpolate(cpaocpb, xVal,
                setNew.Count - setOld.Count,
                out p, out q);

            // Verification.
            var xValVer = new List<int>(VerificationNum);
            for (var i = 0; i < VerificationNum; ++i)
            {
                xValVer.Add(d0 + 1 + i);
            }
            var cpaVer = _cp.Calc(setNew, xValVer);
            var cpbVer = _cp.Calc(setOld, xValVer);
            var cpaVeroCpbVer = _cp.Div(cpaVer, cpbVer);

            for (var i = 0; i < VerificationNum; ++i)
            {
                var pval = _cp.CalcCoeff(p, xValVer[i]);
                var qVal = _cp.CalcCoeff(q, xValVer[i]);
                var verNum = _cp.DivGF(pval, qVal);
                if (verNum != cpaVeroCpbVer[i])
                {
                    logger.Debug("Verification failed, need to increase d0");
                    throw new InvalidOperationException();
                }
            }

            var missingFromOldList = _cp.Factoring(p);

            var checkList = missingFromOldList.Except(snmso).Count();
            Debug.Assert(checkList == 0);
            Debug.Assert(missingFromOldList.Count == snmso.Count);

            var missingSet = new HashSet<int>();
            foreach (var item in missingFromOldList)
            {
                missingSet.Add(item);
            }

            // Genereate patch file
            var patchFile = new List<PatchData>();
            using (var fs = new FileStream(newFileName, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                for (var i = 0; i < setNew.Count; ++i)
                {
                    var currH = setNew[i];
                    var currPatch = new PatchData()
                    {
                        HashValue = currH
                    };
                    if (missingSet.Contains(currH))
                    {
                        // Need to include the actual data.
                        var currFileChunkInfo = fciNew[i];
                        fs.Position = currFileChunkInfo.Pos;
                        var fcData = new byte[currFileChunkInfo.Length];
                        var bRead = fs.Read(fcData, 0, currFileChunkInfo.Length);
                        if (bRead != currFileChunkInfo.Length)
                        {
                            throw new InvalidDataException();
                        }
                        currPatch.Data = fcData;
                    }
                    patchFile.Add(currPatch);
                }
            }

            // Send patch to device B to reconstruct the file.
            var existingSet = new Dictionary<int, int>();
            for (var i = 0; i < setOld.Count; ++i)
            {
                existingSet.Add(setOld[i], i);
            }

            using (var fsout = new FileStream(outputFileName, FileMode.Create, FileAccess.Write, FileShare.None))
            {
                using (var fsOld = new FileStream(oldFileName, FileMode.Open, FileAccess.Read, FileShare.Read))
                {
                    for (var i = 0; i < patchFile.Count; ++i)
                    {
                        var currPatch = patchFile[i];
                        if (currPatch.Data == null)
                        {
                            // Existing data.
                            var idx = existingSet[currPatch.HashValue];
                            var currFileChunkInfo = fciOld[idx];
                            fsOld.Position = currFileChunkInfo.Pos;

                            var fcData = new byte[currFileChunkInfo.Length];
                            var bRead = fsOld.Read(fcData, 0, currFileChunkInfo.Length);
                            if (bRead != currFileChunkInfo.Length)
                            {
                                throw new InvalidDataException();
                            }
                            fsout.Write(fcData, 0, fcData.Length);
                        }
                        else
                        {
                            // New data.
                            fsout.Write(currPatch.Data, 0, currPatch.Data.Length);
                        }
                    }
                }
            }

            var bfSize = Helper.SizeOfBF(bf);
            var cpbSize = Helper.SizeCPB(cpb);
            var pfSize = Helper.SizeOfPatchFile(patchFile);

            var totalBandwidth = bfSize + cpbSize + pfSize;

            Console.WriteLine("Total: {0} bytes", totalBandwidth);
        }
Exemple #3
0
        static void TestReconciliation()
        {
            var setA = new List<int> { 1, 3, 5 };
            var setB = new List<int> { 3, 5 };

            // In device A
            var bf = GenerateBF(setA);

            // Send this bloom filter to device B, in device B
            var n0 = 0;
            foreach (var item in setB)
            {
                var byteArr = BitConverter.GetBytes(item);
                if (bf.Contains(byteArr))
                {
                    n0++;
                }
            }
            var d0 = Helper.EstimateD0(bf.Count, setB.Count, n0, bf);

            var _cp = new CharacteristicPolynomial(97);
            var xVal = new List<int>(d0);
            for (var i = 0; i < d0; ++i)
            {
                xVal.Add(i);
            }
            var cpb = _cp.Calc(setB, xVal);

            // Send cpb to device A, in A:
            var cpa = _cp.Calc(setA, xVal);
            var cpaocpb = _cp.Div(cpa, cpb);

            List<int> p;
            List<int> q;
            _cp.Interpolate(cpaocpb, xVal,
                setA.Count - setB.Count,
                out p, out q);

            var samsb = _cp.Factoring(p);
            var sbmsa = _cp.Factoring(q);
        }
Exemple #4
0
        static bool GenDeltaFile(string input, string cpFile, string deltaFile)
        {
            CPData cpd;
            using (var file = File.OpenRead(cpFile))
            {
                cpd = Serializer.Deserialize<CPData>(file);
            }

            var cpbVer = new List<int>();
            for (var i = 0; i < VerificationNum; ++i)
            {
                cpbVer.Add(cpd.CPValues[cpd.CPValues.Count - VerificationNum + i]);
            }
            cpd.CPValues.RemoveRange(cpd.CPValues.Count - VerificationNum, VerificationNum);

            var cpb = cpd.CPValues;

            var setNew = new List<int>();
            var fciNew = new List<FileChunkInfo>();
            ProcessFile(input, setNew, fciNew);
            for (var i = 0; i < setNew.Count; ++i)
            {
                setNew[i] = setNew[i] & HashBitMask;
            }

            var _cp = new CharacteristicPolynomial(FieldOrder);
            var d0 = cpb.Count;
            var xVal = GenXValues(FieldOrder, d0);

            var cpa = _cp.Calc(setNew, xVal);
            var cpaocpb = _cp.Div(cpa, cpb);

            List<int> p;
            List<int> q;
            _cp.Interpolate(cpaocpb, xVal,
                setNew.Count - cpd.SetCount,
                out p, out q);

            // TODO: verification.
            // If verification failed => return false;
            var xValVer = GenXValues(FieldOrder, d0 + VerificationNum);
            xValVer.RemoveRange(0, d0);
            var cpaVer = _cp.Calc(setNew, xValVer);
            var cpaVeroCpbVer = _cp.Div(cpaVer, cpbVer);

            for (var i = 0; i < VerificationNum; ++i)
            {
                var pval = _cp.CalcCoeff(p, xValVer[i]);
                var qVal = _cp.CalcCoeff(q, xValVer[i]);
                var verNum = _cp.DivGF(pval, qVal);
                if (verNum != cpaVeroCpbVer[i])
                {
                    logger.Debug("Verification failed, need to increase d0");
                    return false;
                }
            }

            var missingFromOldList = _cp.Factoring(p);

            var missingSet = new HashSet<int>();
            foreach (var item in missingFromOldList)
            {
                missingSet.Add(item);
            }

            // Genereate delta file.
            var deltaData = new List<PatchData>();
            using (var fs = new FileStream(input, FileMode.Open, FileAccess.Read, FileShare.Read))
            {
                for (var i = 0; i < setNew.Count; ++i)
                {
                    var currH = setNew[i];
                    var currPatch = new PatchData()
                    {
                        HashValue = currH
                    };
                    if (missingSet.Contains(currH))
                    {
                        // Need to include the actual data.
                        var currFileChunkInfo = fciNew[i];
                        fs.Position = currFileChunkInfo.Pos;
                        var fcData = new byte[currFileChunkInfo.Length];
                        var bRead = fs.Read(fcData, 0, currFileChunkInfo.Length);
                        if (bRead != currFileChunkInfo.Length)
                        {
                            throw new InvalidDataException();
                        }
                        currPatch.Data = fcData;
                    }
                    deltaData.Add(currPatch);
                }
            }

            using (var file = File.Create(deltaFile))
            {
                Serializer.Serialize(file, deltaData);
            }
            return true;
        }