public static Tuple<INetwork, IFactory> SmallLoLa(string FileName, bool Encrypt)
{
Console.WriteLine("Small LoLa mode");
Console.Write("Generating keys in ");
var start = DateTime.Now;
var Factory = (Encrypt) ? (IFactory)new EncryptedSealBfvFactory(new ulong[] { 2277377, 2424833 }, 8192, DecompositionBitCount: 40, GaloisDecompositionBitCount: 40, SmallModulusCount: 3)
: new RawFactory(8192);
var end = DateTime.Now;
Console.WriteLine("{0} seconds", (end - start).TotalSeconds);
int weightscale = 64; // with weightscale of 64 the accuracy is 96.92% and the maximal value is 534491448976
var readerLayer = new LLConvReader()
{
FileName = FileName,
SparseFormat = true,
NormalizationFactor = 1.0 / 256.0,
Scale = 16.0,
InputShape = new int[] { 28, 28 },
KernelShape = new int[] { 5, 5 },
Upperpadding = new int[] { 1, 1 },
Stride = new int[] { 2, 2 },
};
var encryptLayer = new EncryptLayer() { Source = readerLayer, Factory = Factory };
var ConvLayer1 = new LLPoolLayer()
{
Source = encryptLayer,
InputShape = new int[] { 28, 28 },
KernelShape = new int[] { 5, 5 },
Upperpadding = new int[] { 1, 1 },
Stride = new int[] { 2, 2 },
MapCount = new int[] { 5, 1 },
WeightsScale = weightscale,
Weights = SmallModel.Weights_0
};
var VectorizeLayer2 = new LLVectorizeLayer() { Source = ConvLayer1 };
//var ActivationLayer3 = new SquareActivation() { Source = VectorizeLayer2 };
var ActivationLayer3 = new AppxReLUActivation() { Source = VectorizeLayer2 };
//var ActivationLayer3 = new LeakyReLUActivation() { Source = VectorizeLayer2 };
//var ActivationLayer3 = new ReLUActivation() { Source = VectorizeLayer2 };
var DenseLayer4 = new LLDenseLayer()
{
Source = ActivationLayer3,
Bias = SmallModel.Biases_1,
Weights = SmallModel.Weights_1,
WeightsScale = weightscale,
InputFormat = EVectorFormat.dense
};
return new Tuple<INetwork, IFactory>(DenseLayer4, Factory);
}
public static Tuple<INetwork, IFactory> LargeLoLa(string FileName, bool Encrypt)
{
Console.WriteLine("Large LoLa mode");
WeightsReader wr = new WeightsReader("MnistLargeWeight.csv", "MnistLargeBias.csv");
Console.Write("Generating keys in ");
var start = DateTime.Now;
var Factory = Encrypt ? (IFactory)new EncryptedSealBfvFactory(new ulong[] { 2148728833, 2148794369, 2149810177 }, 16384, DecompositionBitCount: 60, GaloisDecompositionBitCount: 60, SmallModulusCount: 7)
: new RawFactory(16384);
var end = DateTime.Now;
Console.WriteLine("{0} seconds", (end - start).TotalSeconds);
var readerLayer = new LLConvReader
{
FileName = FileName,
SparseFormat = true,
InputShape = new int[] { 1, 28, 28 },
KernelShape = new int[] { 1, 8, 8 },
Upperpadding = new int[] { 0, 1, 1 },
Lowerpadding = new int[] { 0, 1, 1 },
Stride = new int[] { 1000, 2, 2 },
NormalizationFactor = 1.0,
Scale = 16.0
};
var encryptLayer = new EncryptLayer() { Source = readerLayer, Factory = Factory };
var convLayer1 = new LLPoolLayer()
{
Source = encryptLayer,
InputShape = new int[] { 1, 28, 28 },
KernelShape = new int[] { 1, 8, 8 },
Upperpadding = new int[] { 0, 1, 1 },
Lowerpadding = new int[] { 0, 1, 1 },
Stride = new int[] { 1000, 2, 2 },
MapCount = new int[] { 83, 1, 1 },
WeightsScale = 4096,
Weights = ((double[])wr.Weights[0]).Select(x => x/256).ToArray(),
Bias = (double[])wr.Biases[0]
};
var VectorizeLayer2 = new LLVectorizeLayer() { Source = convLayer1 };
//var activationLayer3 = new SquareActivation() { Source = VectorizeLayer2 };
var activationLayer3 = new AppxReLUActivation() { Source = VectorizeLayer2 };
//var activationLayer3 = new LeakyReLUActivation() { Source = VectorizeLayer2 };
//var activationLayer3 = new ReLUActivation() { Source = VectorizeLayer2 };
var convEngine = new ConvolutionEngine()
{
InputShape = new int[] { 83, 12, 12 },
KernelShape = new int[] { 83, 6, 6 },
Padding = new bool[] { false, false, false },
Stride = new int[] { 83, 2, 2 },
MapCount = new int[] { 163, 1, 1 }
};
var denseLayer4 = new LLDenseLayer
{
Source = activationLayer3,
WeightsScale = 64,
Weights = convEngine.GetDenseWeights((double[])wr.Weights[1]),
Bias = convEngine.GetDenseBias((double[])wr.Biases[1]),
InputFormat = EVectorFormat.dense,
ForceDenseFormat = true
};
//var activationLayer5 = new SquareActivation() { Source = denseLayer4 };
var activationLayer5 = new AppxReLUActivation() { Source = denseLayer4 };
//var activationLayer5 = new LeakyReLUActivation() { Source = denseLayer4 };
//var activationLayer5 = new ReLUActivation() { Source = denseLayer4 };
var denseLayer6 = new LLDenseLayer()
{
Source = activationLayer5,
Weights = (double[])wr.Weights[2],
Bias = (double[])wr.Biases[2],
WeightsScale = 512,
InputFormat = EVectorFormat.dense
};
return new Tuple<INetwork, IFactory>(denseLayer6, Factory);
}
public static Tuple<INetwork, IFactory> LoLa(string FileName, bool Encrypt)
{
Console.WriteLine("LoLa mode");
Console.Write("Generating keys in ");
var start = DateTime.Now;
var Factory = Encrypt ? (IFactory)new EncryptedSealBfvFactory(new ulong[] { 557057, 638977, 737281, 786433 }, 8192) : new RawFactory(8192);
var end = DateTime.Now;
Console.WriteLine("{0} seconds", (end - start).TotalSeconds);
int weightscale = 32;
var readerLayer = new LLConvReader()
{
FileName = FileName,
SparseFormat = true,
NormalizationFactor = 1.0 / 256.0,
Scale = 16.0,
InputShape = new int[] { 28, 28 },
KernelShape = new int[] { 5, 5 },
Upperpadding = new int[] { 1, 1 },
Stride = new int[] { 2, 2 },
};
var encryptLayer = new EncryptLayer() { Source = readerLayer, Factory = Factory };
var ConvLayer1 = new LLPoolLayer()
{
Source = encryptLayer,
InputShape = new int[] { 28, 28 },
KernelShape = new int[] { 5, 5 },
Upperpadding = new int[] { 1, 1 },
Stride = new int[] { 2, 2 },
MapCount = new int[] { 5, 1 },
WeightsScale = weightscale,
Weights = Weights.Weights_0
};
var VectorizeLayer2 = new LLVectorizeLayer() { Source = ConvLayer1 };
//var ActivationLayer3 = new SquareActivation() { Source = VectorizeLayer2 };
var ActivationLayer3 = new AppxReLUActivation() { Source = VectorizeLayer2 };
//var ActivationLayer3 = new LeakyReLUActivation() { Source = VectorizeLayer2 };
//var ActivationLayer3 = new ReLUActivation() { Source = VectorizeLayer2 };
var DuplicateLayer4 = new LLDuplicateLayer() { Source = ActivationLayer3, Count = 8 };
var DenseLayer5 = new LLPackedDenseLayer()
{
Source = DuplicateLayer4,
Weights = Transpose(Weights.Weights_1, 5 * 13 * 13, 100),
Bias = Weights.Biases_2,
WeightsScale = weightscale * weightscale,
PackingCount = DuplicateLayer4.Count,
PackingShift = 1024
};
var InterleaveLayer6 = new LLInterleaveLayer()
{
Source = DenseLayer5,
Shift = -1,
SelectedIndices = Enumerable.Range(0, (int)DuplicateLayer4.Count).Select(i => 1023 + i * 1024).ToList()
};
//var ActivationLayer7 = new SquareActivation() { Source = InterleaveLayer6 };
var ActivationLayer7 = new AppxReLUActivation() { Source = InterleaveLayer6 };
//var ActivationLayer7 = new LeakyReLUActivation() { Source = InterleaveLayer6 };
//var ActivationLayer7 = new ReLUActivation() { Source = InterleaveLayer6 };
var DenseLayer8 = new LLInterleavedDenseLayer()
{
Source = ActivationLayer7,
Weights = Weights.Weights_3,
Bias = Weights.Biases_3,
WeightsScale = weightscale,
Shift = -1,
SelectedIndices = Enumerable.Range(0, (int)DuplicateLayer4.Count).Select(i => 1023 + i * 1024).ToList()
};
return new Tuple<INetwork, IFactory>(DenseLayer8, Factory);
}
public static Tuple<INetwork, IFactory> LoLaDense(string FileName, bool Encrypt)
{
Console.WriteLine("LoLa-Dense mode");
Console.Write("Generating keys in ");
var start = DateTime.Now;
var Factory = (Encrypt) ? (IFactory)new EncryptedSealBfvFactory(new ulong[] { 34359771137, 34360754177 }, 16384, DecompositionBitCount: 60, GaloisDecompositionBitCount: 60, SmallModulusCount: 7)
: (IFactory)new RawFactory(16384);
var end = DateTime.Now;
Console.WriteLine("{0} seconds", (end - start).TotalSeconds);
int weightscale = 32;
var readerLayer = new LLSingleLineReader()
{
FileName = FileName,
SparseFormat = true,
NormalizationFactor = 1.0 / 256.0,
Scale = 16.0,
};
var encryptLayer = new EncryptLayer() { Source = readerLayer, Factory = Factory };
var preConvLayer1 = new LLPreConvLayer()
{
Source = encryptLayer,
InputShape = new int[] { 28, 28 },
KernelShape = new int[] { 5, 5 },
Upperpadding = new int[] { 1, 1 },
Stride = new int[] { 2, 2 },
UseAxisForBlocks = new bool[] { true, true }
};
var ConvLayer2 = new LLPoolLayer()
{
Source = preConvLayer1,
InputShape = new int[] { 28, 28 },
KernelShape = new int[] { 5, 5 },
Upperpadding = new int[] { 1, 1 },
Stride = new int[] { 2, 2 },
MapCount = new int[] { 5, 1 },
WeightsScale = weightscale,
Weights = Weights.Weights_0,
HotIndices = preConvLayer1.HotIndices
};
var VectorizeLayer3 = new LLVectorizeLayer() { Source = ConvLayer2 };
//var ActivationLayer4 = new SquareActivation() { Source = VectorizeLayer3 };
var ActivationLayer4 = new AppxReLUActivation() { Source = VectorizeLayer3 };
//var ActivationLayer4 = new LeakyReLUActivation() { Source = VectorizeLayer3 };
//var ActivationLayer4 = new ReLUActivation() { Source = VectorizeLayer3 };
var DuplicateLayer5 = new LLDuplicateLayer() { Source = ActivationLayer4, Count = 16};
var DenseLayer6 = new LLPackedDenseLayer()
{
Source = DuplicateLayer5,
Weights = preConvLayer1.RearrangeWeights(Transpose(Weights.Weights_1, 5 * 13 * 13, 100)),
Bias = Weights.Biases_2,
WeightsScale = weightscale * weightscale,
PackingCount = DuplicateLayer5.Count,
PackingShift = 1024,
};
//var ActivationLayer7 = new SquareActivation() { Source = DenseLayer6 };
var ActivationLayer7 = new AppxReLUActivation() { Source = DenseLayer6 };
//var ActivationLayer7 = new LeakyReLUActivation() { Source = DenseLayer6 };
//var ActivationLayer7 = new ReLUActivation() { Source = DenseLayer6 };
var InterleaveLayer8 = new LLInterleaveLayer()
{
Source = ActivationLayer7,
Shift = -1,
SelectedIndices = Enumerable.Range(0, (int)DuplicateLayer5.Count).Select(i => 1023 + i * 1024).ToList()
};
var DenseLayer8 = new LLInterleavedDenseLayer()
{
Source = InterleaveLayer8,
Weights = Weights.Weights_3,
Bias = Weights.Biases_3,
WeightsScale = weightscale,
Shift = -1,
SelectedIndices = Enumerable.Range(0, (int)DuplicateLayer5.Count).Select(i => 1023 + i * 1024).ToList()
};
var network = DenseLayer8;
return new Tuple<INetwork, IFactory>(network, Factory);
}
static void Main(string[] args)
{
string fileName = "MNIST-28x28-test.txt";
int batchSize = 8192;
int numberOfRecords = 10000;
var Factory = new EncryptedSealBfvFactory(new ulong[] { 2148728833, 2148794369, 2149810177 }, 16384, DecompositionBitCount: 60, GaloisDecompositionBitCount: 60, SmallModulusCount: 7);
int weightscale = 32;
var ReaderLayer = new BatchReader
{
FileName = fileName,
SparseFormat = true,
MaxSlots = batchSize,
NormalizationFactor = 1.0 / 256.0,
Scale = 16.0
};
var EncryptedLayer = new EncryptLayer()
{
Source = ReaderLayer, Factory = Factory
};
var StartTimingLayer = new TimingLayer()
{
Source = EncryptedLayer, StartCounters = new string[] { "Batch-Time" }
};
var ConvLayer1 = new PoolLayer()
{
Source = StartTimingLayer,
InputShape = new int[] { 28, 28 },
KernelShape = new int[] { 5, 5 },
Upperpadding = new int[] { 1, 1 },
Stride = new int[] { 2, 2 },
MapCount = new int[] { 5, 1 },
WeightsScale = weightscale,
Weights = Weights.Weights_0
};
//var ActivationLayer2 = new SquareActivation() { Source = ConvLayer1 };
var ActivationLayer2 = new AppxReLUActivation()
{
Source = ConvLayer1
};
var DenseLayer3 = new PoolLayer()
{
Source = ActivationLayer2,
InputShape = new int[] { 5 * 13 * 13 },
KernelShape = new int[] { 5 * 13 * 13 },
Stride = new int[] { 1000 },
MapCount = new int[] { 100 },
Weights = Transpose(Weights.Weights_1, 5 * 13 * 13, 100),
Bias = Weights.Biases_2,
WeightsScale = weightscale * weightscale
};
//var ActivationLayer4 = new SquareActivation() { Source = DenseLayer3 };
var ActivationLayer4 = new AppxReLUActivation()
{
Source = DenseLayer3
};
var DenseLayer5 = new PoolLayer()
{
Source = ActivationLayer4,
InputShape = new int[] { 100 },
KernelShape = new int[] { 100 },
Stride = new int[] { 1000 },
MapCount = new int[] { 10 },
Weights = Weights.Weights_3,
Bias = Weights.Biases_3,
WeightsScale = weightscale
};
var StopTimingLayer = new TimingLayer()
{
Source = DenseLayer5, StopCounters = new string[] { "Batch-Time" }
};
var network = StopTimingLayer;
OperationsCount.Reset();
Console.WriteLine("Preparing");
network.PrepareNetwork();
OperationsCount.Print();
OperationsCount.Reset();
for (var p = (INetwork)network; p != null; p = p.Source)
{
if (p is BaseLayer b)
{
b.Verbose = true;
}
}
int count = 0;
int errs = 0;
while (count < numberOfRecords)
{
using (var m = network.GetNext())
Utils.ProcessInEnv(env =>
{
var decrypted = m.Decrypt(env);
for (int i = 0; i < decrypted.RowCount; i++)
{
int pred = 0;
for (int j = 1; j < decrypted.ColumnCount; j++)
{
if (decrypted[i, j] > decrypted[i, pred])
{
pred = j;
}
}
if (pred != ReaderLayer.Labels[i])
{
errs++;
}
count++;
if (count % 100 == 0)
{
Console.WriteLine("errs {0}/{1} accuracy {2:0.000}% prediction {3} label {4}", errs, count, 100 - (100.0 * errs / (count)), pred, ReaderLayer.Labels[i]);
}
}
Console.WriteLine("Batch size {0} {1}", batchSize, TimingLayer.GetStats());
}, Factory);
}
Console.WriteLine("errs {0}/{1} accuracy {2:0.000}%", errs, count, 100 - (100.0 * errs / (count)));
network.DisposeNetwork();
}
public static void Main(string[] args)
{
WeightsReader wr = new WeightsReader("CifarWeight.csv", "CifarBias.csv");
Console.WriteLine("Generating encryption keys {0}", DateTime.Now);
var factory = new EncryptedSealBfvFactory(new ulong[] { 2148728833, 2148794369, 2149810177 }, 16384, DecompositionBitCount: 60, GaloisDecompositionBitCount: 60, SmallModulusCount: 7);
Console.WriteLine("Encryption keys ready {0}", DateTime.Now);
string fileName = "cifar-test.tsv";
var readerLayer = new LLConvReader
{
FileName = fileName,
SparseFormat = false,
InputShape = new int[] { 3, 32, 32 },
KernelShape = new int[] { 3, 8, 8 },
Upperpadding = new int[] { 0, 1, 1 },
Lowerpadding = new int[] { 0, 1, 1 },
Stride = new int[] { 1000, 2, 2 },
NormalizationFactor = 1.0,
Scale = 128.0
};
var encryptLayer = new EncryptLayer()
{
Source = readerLayer, Factory = factory
};
var convLayer1 = new LLPoolLayer()
{
Source = encryptLayer,
InputShape = new int[] { 3, 32, 32 },
KernelShape = new int[] { 3, 8, 8 },
Upperpadding = new int[] { 0, 1, 1 },
Lowerpadding = new int[] { 0, 1, 1 },
Stride = new int[] { 1000, 2, 2 },
MapCount = new int[] { 83, 1, 1 },
WeightsScale = 256.0,
Weights = (double[])wr.Weights[0],
Bias = (double[])wr.Biases[0]
};
var VectorizeLayer2 = new LLVectorizeLayer()
{
Source = convLayer1
};
//var activationLayer3 = new SquareActivation() { Source = VectorizeLayer2 };
var activationLayer3 = new AppxReLUActivation()
{
Source = VectorizeLayer2
};
//var activationLayer3 = new LeakyReLUActivation() { Source = VectorizeLayer2 };
//var activationLayer3 = new ReLUActivation() { Source = VectorizeLayer2 };
var convEngine = new ConvolutionEngine()
{
InputShape = new int[] { 83, 14, 14 },
KernelShape = new int[] { 83, 6, 6 },
Padding = new bool[] { false, false, false },
Stride = new int[] { 83, 2, 2 },
MapCount = new int[] { 163, 1, 1 }
};
var denseLayer4 = new LLDenseLayer
{
Source = activationLayer3,
WeightsScale = 512.0,
Weights = convEngine.GetDenseWeights((double[])wr.Weights[1]),
Bias = convEngine.GetDenseBias((double[])wr.Biases[1]),
InputFormat = EVectorFormat.dense,
ForceDenseFormat = true
};
//var activationLayer5 = new SquareActivation() { Source = denseLayer4 };
var activationLayer5 = new AppxReLUActivation()
{
Source = denseLayer4
};
//var activationLayer5 = new LeakyReLUActivation() { Source = denseLayer4 };
//var activationLayer5 = new ReLUActivation() { Source = denseLayer4 };
var denseLayer6 = new LLDenseLayer()
{
Source = activationLayer5,
Weights = (double[])wr.Weights[2],
Bias = (double[])wr.Biases[2],
WeightsScale = 1024.0,
InputFormat = EVectorFormat.dense
};
var network = denseLayer6;
Console.WriteLine("Preparing");
network.PrepareNetwork();
var m = network.GetNext();
Utils.Show(m, factory);
Console.WriteLine("Max computed value {0} ({1})", RawMatrix.Max, Math.Log(RawMatrix.Max) / Math.Log(2));
}