Ejemplo n.º 1
0
        public static Tuple <INetwork, IFactory> SmallLoLa(string FileName, bool Encrypt)
        {
            Console.WriteLine("Small LoLa mode");
            Console.Write("Generating keys in ");
            var start   = DateTime.Now;
            var Factory = (Encrypt) ? (IFactory) new EncryptedSealBfvFactory(new ulong[] { 2277377, 2424833 }, 8192, DecompositionBitCount: 40, GaloisDecompositionBitCount: 40, SmallModulusCount: 3)
                : new RawFactory(8192);
            var end = DateTime.Now;

            Console.WriteLine("{0} seconds", (end - start).TotalSeconds);

            int weightscale = 64; // with weightscale of 64 the accuracy is 96.92% and the maximal value is 534491448976

            var readerLayer = new LLConvReader()
            {
                FileName            = FileName,
                SparseFormat        = true,
                NormalizationFactor = 1.0 / 256.0,
                Scale        = 16.0,
                InputShape   = new int[] { 28, 28 },
                KernelShape  = new int[] { 5, 5 },
                Upperpadding = new int[] { 1, 1 },
                Stride       = new int[] { 2, 2 },
            };
            var encryptLayer = new EncryptLayer()
            {
                Source = readerLayer, Factory = Factory
            };
            var ConvLayer1 = new LLPoolLayer()
            {
                Source       = encryptLayer,
                InputShape   = new int[] { 28, 28 },
                KernelShape  = new int[] { 5, 5 },
                Upperpadding = new int[] { 1, 1 },
                Stride       = new int[] { 2, 2 },
                MapCount     = new int[] { 5, 1 },
                WeightsScale = weightscale,
                Weights      = SmallModel.Weights_0
            };
            var VectorizeLayer2 = new LLVectorizeLayer()
            {
                Source = ConvLayer1
            };

            var ActivationLayer3 = new SquareActivation()
            {
                Source = VectorizeLayer2
            };

            var DenseLayer4 = new LLDenseLayer()
            {
                Source       = ActivationLayer3,
                Bias         = SmallModel.Biases_1,
                Weights      = SmallModel.Weights_1,
                WeightsScale = weightscale,
                InputFormat  = EVectorFormat.dense
            };

            return(new Tuple <INetwork, IFactory>(DenseLayer4, Factory));
        }
Ejemplo n.º 2
0
        public static Tuple<INetwork, IFactory> LoLa(string FileName, bool Encrypt)
        {
            Console.WriteLine("LoLa mode");
            Console.Write("Generating keys in ");
            var start = DateTime.Now;
            var Factory = Encrypt ? (IFactory)new EncryptedSealBfvFactory(new ulong[] { 557057, 638977, 737281, 786433 }, 8192) : new RawFactory(8192);
            var end = DateTime.Now;
            Console.WriteLine("{0} seconds", (end - start).TotalSeconds);

            int weightscale = 32;
            
            var readerLayer = new LLConvReader()
            {
                FileName = FileName,
                SparseFormat = true,
                NormalizationFactor = 1.0 / 256.0,
                Scale = 16.0,
                InputShape = new int[] { 28, 28 },
                KernelShape = new int[] { 5, 5 },
                Upperpadding = new int[] { 1, 1 },
                Stride = new int[] { 2, 2 },

            };
            var encryptLayer = new EncryptLayer() { Source = readerLayer, Factory = Factory };
            var ConvLayer1 = new LLPoolLayer()
            {
                Source = encryptLayer,
                InputShape = new int[] { 28, 28 },
                KernelShape = new int[] { 5, 5 },
                Upperpadding = new int[] { 1, 1 },
                Stride = new int[] { 2, 2 },
                MapCount = new int[] { 5, 1 },
                WeightsScale = weightscale,
                Weights = Weights.Weights_0
            };
            var VectorizeLayer2 = new LLVectorizeLayer() { Source = ConvLayer1 };

            //var ActivationLayer3 = new SquareActivation() { Source = VectorizeLayer2 };
            var ActivationLayer3 = new AppxReLUActivation() { Source = VectorizeLayer2 };
            //var ActivationLayer3 = new LeakyReLUActivation() { Source = VectorizeLayer2 };
            //var ActivationLayer3 = new ReLUActivation() { Source = VectorizeLayer2 };

            var DuplicateLayer4 = new LLDuplicateLayer() { Source = ActivationLayer3, Count = 8 };


            var DenseLayer5 = new LLPackedDenseLayer()
            {
                Source = DuplicateLayer4,
                Weights = Transpose(Weights.Weights_1, 5 * 13 * 13, 100),
                Bias = Weights.Biases_2,
                WeightsScale = weightscale * weightscale,
                PackingCount = DuplicateLayer4.Count,
                PackingShift = 1024
            };




            var InterleaveLayer6 = new LLInterleaveLayer()
            {
                Source = DenseLayer5,
                Shift = -1,
                SelectedIndices = Enumerable.Range(0, (int)DuplicateLayer4.Count).Select(i => 1023 + i * 1024).ToList()
            };
            //var ActivationLayer7 = new SquareActivation() { Source = InterleaveLayer6 };
            var ActivationLayer7 = new AppxReLUActivation() { Source = InterleaveLayer6 };
            //var ActivationLayer7 = new LeakyReLUActivation() { Source = InterleaveLayer6 };
            //var ActivationLayer7 = new ReLUActivation() { Source = InterleaveLayer6 };

            var DenseLayer8 = new LLInterleavedDenseLayer()
            {
                Source = ActivationLayer7,
                Weights = Weights.Weights_3,
                Bias = Weights.Biases_3,
                WeightsScale = weightscale,
                Shift = -1,
                SelectedIndices = Enumerable.Range(0, (int)DuplicateLayer4.Count).Select(i => 1023 + i * 1024).ToList()

            };

            return new Tuple<INetwork, IFactory>(DenseLayer8, Factory);
        }
Ejemplo n.º 3
0
        public static Tuple<INetwork, IFactory> LoLaDense(string FileName, bool Encrypt)
        {
            Console.WriteLine("LoLa-Dense mode");
            Console.Write("Generating keys in ");
            var start = DateTime.Now;
            var Factory = (Encrypt) ? (IFactory)new EncryptedSealBfvFactory(new ulong[] { 34359771137, 34360754177 }, 16384, DecompositionBitCount: 60, GaloisDecompositionBitCount: 60, SmallModulusCount: 7)
                : (IFactory)new RawFactory(16384);
            var end = DateTime.Now;
            Console.WriteLine("{0} seconds", (end - start).TotalSeconds);

            int weightscale = 32;

            var readerLayer = new LLSingleLineReader()
            {
                FileName = FileName,
                SparseFormat = true,
                NormalizationFactor = 1.0 / 256.0,
                Scale = 16.0,
            };
            var encryptLayer = new EncryptLayer() { Source = readerLayer, Factory = Factory };
            var preConvLayer1 = new LLPreConvLayer()
            {
                Source = encryptLayer,
                InputShape = new int[] { 28, 28 },
                KernelShape = new int[] { 5, 5 },
                Upperpadding = new int[] { 1, 1 },
                Stride = new int[] { 2, 2 },
                UseAxisForBlocks = new bool[] { true, true }
            };
            var ConvLayer2 = new LLPoolLayer()
            {
                Source = preConvLayer1,
                InputShape = new int[] { 28, 28 },
                KernelShape = new int[] { 5, 5 },
                Upperpadding = new int[] { 1, 1 },
                Stride = new int[] { 2, 2 },
                MapCount = new int[] { 5, 1 },
                WeightsScale = weightscale,
                Weights = Weights.Weights_0,
                HotIndices = preConvLayer1.HotIndices
            };

            var VectorizeLayer3 = new LLVectorizeLayer() { Source = ConvLayer2 };

            //var ActivationLayer4 = new SquareActivation() { Source = VectorizeLayer3 };
            var ActivationLayer4 = new AppxReLUActivation() { Source = VectorizeLayer3 };
            //var ActivationLayer4 = new LeakyReLUActivation() { Source = VectorizeLayer3 };
            //var ActivationLayer4 = new ReLUActivation() { Source = VectorizeLayer3 };

            var DuplicateLayer5 = new LLDuplicateLayer() { Source = ActivationLayer4, Count = 16};


            var DenseLayer6 = new LLPackedDenseLayer()
            {
                Source = DuplicateLayer5,
                Weights = preConvLayer1.RearrangeWeights(Transpose(Weights.Weights_1, 5 * 13 * 13, 100)),
                Bias = Weights.Biases_2,
                WeightsScale = weightscale * weightscale,
                PackingCount = DuplicateLayer5.Count,
                PackingShift = 1024,
            };

            //var ActivationLayer7 = new SquareActivation() { Source = DenseLayer6 };
            var ActivationLayer7 = new AppxReLUActivation() { Source = DenseLayer6 };
            //var ActivationLayer7 = new LeakyReLUActivation() { Source = DenseLayer6 };
            //var ActivationLayer7 = new ReLUActivation() { Source = DenseLayer6 };

            var InterleaveLayer8 = new LLInterleaveLayer()
            {
                Source = ActivationLayer7,
                Shift = -1,
                SelectedIndices = Enumerable.Range(0, (int)DuplicateLayer5.Count).Select(i => 1023 + i * 1024).ToList()
            };


            var DenseLayer8 = new LLInterleavedDenseLayer()
            {
                Source = InterleaveLayer8,
                Weights = Weights.Weights_3,
                Bias = Weights.Biases_3,
                WeightsScale = weightscale,
                Shift = -1,
                SelectedIndices = Enumerable.Range(0, (int)DuplicateLayer5.Count).Select(i => 1023 + i * 1024).ToList()

            };

            var network = DenseLayer8;

            return new Tuple<INetwork, IFactory>(network, Factory);
        }
Ejemplo n.º 4
0
        public static void Main(string[] args)
        {
            WeightsReader wr = new WeightsReader("CifarWeight.csv", "CifarBias.csv");

            Console.WriteLine("Generating encryption keys {0}", DateTime.Now);
            var factory = new EncryptedSealBfvFactory(new ulong[] { 2148728833, 2148794369, 2149810177 }, 16384, DecompositionBitCount: 60, GaloisDecompositionBitCount: 60, SmallModulusCount: 7);

            Console.WriteLine("Encryption keys ready {0}", DateTime.Now);


            string fileName    = "cifar-test.tsv";
            var    readerLayer = new LLConvReader
            {
                FileName            = fileName,
                SparseFormat        = false,
                InputShape          = new int[] { 3, 32, 32 },
                KernelShape         = new int[] { 3, 8, 8 },
                Upperpadding        = new int[] { 0, 1, 1 },
                Lowerpadding        = new int[] { 0, 1, 1 },
                Stride              = new int[] { 1000, 2, 2 },
                NormalizationFactor = 1.0,
                Scale = 128.0
            };

            var encryptLayer = new EncryptLayer()
            {
                Source = readerLayer, Factory = factory
            };

            var convLayer1 = new LLPoolLayer()
            {
                Source       = encryptLayer,
                InputShape   = new int[] { 3, 32, 32 },
                KernelShape  = new int[] { 3, 8, 8 },
                Upperpadding = new int[] { 0, 1, 1 },
                Lowerpadding = new int[] { 0, 1, 1 },
                Stride       = new int[] { 1000, 2, 2 },
                MapCount     = new int[] { 83, 1, 1 },
                WeightsScale = 256.0,
                Weights      = (double[])wr.Weights[0],
                Bias         = (double[])wr.Biases[0]
            };

            var VectorizeLayer2 = new LLVectorizeLayer()
            {
                Source = convLayer1
            };

            var activationLayer3 = new SquareActivation()
            {
                Source = VectorizeLayer2
            };

            var convEngine = new ConvolutionEngine()
            {
                InputShape  = new int[] { 83, 14, 14 },
                KernelShape = new int[] { 83, 6, 6 },
                Padding     = new bool[] { false, false, false },
                Stride      = new int[] { 83, 2, 2 },
                MapCount    = new int[] { 163, 1, 1 }
            };

            var denseLayer4 = new LLDenseLayer
            {
                Source           = activationLayer3,
                WeightsScale     = 512.0,
                Weights          = convEngine.GetDenseWeights((double[])wr.Weights[1]),
                Bias             = convEngine.GetDenseBias((double[])wr.Biases[1]),
                InputFormat      = EVectorFormat.dense,
                ForceDenseFormat = true
            };


            var activationLayer5 = new SquareActivation()
            {
                Source = denseLayer4
            };

            var denseLayer6 = new LLDenseLayer()
            {
                Source       = activationLayer5,
                Weights      = (double[])wr.Weights[2],
                Bias         = (double[])wr.Biases[2],
                WeightsScale = 1024.0,
                InputFormat  = EVectorFormat.dense
            };

            var network = denseLayer6;

            Console.WriteLine("Preparing");
            // Visualize layer construction
            for (var p = (INetwork)network; p != null; p = p.Source)
            {
                if (p is BaseLayer b)
                {
                    b.Verbose = true;
                }
            }

            network.PrepareNetwork();

            var m = network.GetNext();

            Utils.Show(m, factory, readerLayer.Labels);
            Console.WriteLine("Max computed value {0} ({1})", RawMatrix.Max, Math.Log(RawMatrix.Max) / Math.Log(2));
        }
Ejemplo n.º 5
0
        public static void Main(string[] args)
        {
            var options = new Options();
            var parsed  = Parser.Default.ParseArguments <Options>(args).WithParsed(x => options = x);

            if (parsed.Tag == ParserResultType.NotParsed)
            {
                Environment.Exit(-2);
            }

            WeightsReader wr = new WeightsReader("CifarWeight.csv", "CifarBias.csv");

            // Model has accuracy of 76.5
            // Current parameters (scale) provide accuracy of 76.31% and uses 78.55 + 1 bits in message length
            // It has a latency of 740 seconds on the reference machine (Azure B8ms server at rest)

            Console.WriteLine("Generating encryption keys {0}", DateTime.Now);
            IFactory factory = null;

            if (options.Encrypt)
            {
                factory = new EncryptedSealBfvFactory(new ulong[] { 957181001729, 957181034497 }, 16384, DecompositionBitCount: 60, GaloisDecompositionBitCount: 60, SmallModulusCount: 8);
            }
            else
            {
                factory = new RawFactory(16 * 1024);
            }
            Console.WriteLine("Encryption keys ready {0}", DateTime.Now);
            int  numberOfRecords = 10000;
            bool verbose         = options.Verbose;

            string fileName    = "cifar-test.tsv";
            var    readerLayer = new LLConvReader
            {
                FileName            = fileName,
                SparseFormat        = false,
                InputShape          = new int[] { 3, 32, 32 },
                KernelShape         = new int[] { 3, 8, 8 },
                Upperpadding        = new int[] { 0, 1, 1 },
                Lowerpadding        = new int[] { 0, 1, 1 },
                Stride              = new int[] { 1000, 2, 2 },
                NormalizationFactor = 1.0 / 256.0,
                Scale   = 8,
                Verbose = verbose
            };


            var EncryptLayer = new EncryptLayer()
            {
                Source = readerLayer, Factory = factory
            };
            var StartTimingLayer = new TimingLayer()
            {
                Source = EncryptLayer, StartCounters = new string[] { "Inference-Time" }
            };


            var ConvLayer1 = new LLPoolLayer()
            {
                Source       = StartTimingLayer,
                InputShape   = new int[] { 3, 32, 32 },
                KernelShape  = new int[] { 3, 8, 8 },
                Upperpadding = new int[] { 0, 1, 1 },
                Lowerpadding = new int[] { 0, 1, 1 },
                Stride       = new int[] { 1000, 2, 2 },
                MapCount     = new int[] { 83, 1, 1 },
                WeightsScale = 256.0,
                Weights      = (double[])wr.Weights[0],
                Bias         = (double[])wr.Biases[0],
                Verbose      = verbose
            };

            var VectorizeLayer2 = new LLVectorizeLayer()
            {
                Source  = ConvLayer1,
                Verbose = verbose
            };

            var ActivationLayer3 = new SquareActivation()
            {
                Source  = VectorizeLayer2,
                Verbose = verbose
            };



            var ConvEngine = new ConvolutionEngine()
            {
                InputShape   = new int[] { 83, 14, 14 },
                KernelShape  = new int[] { 83, 10, 10 },
                Upperpadding = new int[] { 0, 4, 4 },
                Lowerpadding = new int[] { 0, 4, 4 },
                Stride       = new int[] { 83, 2, 2 },
                MapCount     = new int[] { 112, 1, 1 }
            };

            var DenseLayer4 = new LLDenseLayer
            {
                Source           = ActivationLayer3,
                WeightsScale     = 512.0,
                Weights          = ConvEngine.GetDenseWeights((double[])wr.Weights[1]),
                Bias             = ConvEngine.GetDenseBias((double[])wr.Biases[1]),
                InputFormat      = EVectorFormat.dense,
                ForceDenseFormat = true,
                Verbose          = verbose
            };


            var ActivationLayer5 = new SquareActivation()
            {
                Source  = DenseLayer4,
                Verbose = verbose
            };

            var DenseLayer6 = new LLDenseLayer()
            {
                Source       = ActivationLayer5,
                Weights      = (double[])wr.Weights[2],
                Bias         = (double[])wr.Biases[2],
                WeightsScale = 512.0,
                InputFormat  = EVectorFormat.dense,
                Verbose      = verbose
            };

            var StopTimingLayer = new TimingLayer()
            {
                Source = DenseLayer6, StopCounters = new string[] { "Inference-Time" }
            };

            var network = StopTimingLayer;

            Console.WriteLine("Preparing");
            network.PrepareNetwork();
            int count     = 0;
            int errs      = 0;
            int batchSize = 1;

            while (count < numberOfRecords)
            {
                using (var m = network.GetNext())
                {
                    Utils.ProcessInEnv(env =>
                    {
                        var decrypted = m.Decrypt(env);
                        int pred      = 0;
                        for (int j = 1; j < decrypted.RowCount; j++)
                        {
                            if (decrypted[j, 0] > decrypted[pred, 0])
                            {
                                pred = j;
                            }
                        }
                        if (pred != readerLayer.Labels[0])
                        {
                            errs++;
                        }
                        count++;
                        if (count % batchSize == 0)
                        {
                            Console.Write("errs {0}/{1} accuracy {2:0.000}% prediction {3} label {4} {5}ms", errs, count, 100 - (100.0 * errs / (count)), pred, readerLayer.Labels[0], TimingLayer.GetStats());
                            if (options.Encrypt)
                            {
                                Console.WriteLine();
                            }
                            else
                            {
                                Console.WriteLine(" 2^{0} largest-value", Math.Log(RawMatrix.Max) / Math.Log(2));
                            }
                        }
                    }, factory);
                }
            }
            Console.WriteLine("errs {0}/{1} accuracy {2:0.000}%", errs, count, 100 - (100.0 * errs / (count)));
            network.DisposeNetwork();
            if (!options.Encrypt)
            {
                Console.WriteLine("Max computed value 2^{1}", Math.Log(RawMatrix.Max) / Math.Log(2));
            }
        }