public static void FitMnist()
{
var model = new Sequential();
model.Add(new Conv2D(32, kernelSize: new int[] { 3, 3 }, inputShape: new int[] { 28, 28, 1 }, activation: "relu"));
model.Add(new Conv2D(64, kernelSize: new int[] { 3, 3 }, activation: "relu"));
// model.Add(new MaxPooling1D(poolSize: 2));
model.Add(new MaxPooling2D(poolSize: new int[] { 2, 2 }));
model.Add(new Dropout(0.25));
model.Add(new Flatten());
model.Add(new Dense(128, activation: "relu"));
model.Add(new Dropout(0.5));
model.Add(new Dense(10, activation: "softmax"));
var optimizer = new SGD(lr: 0.01);
model.Compile("categorical_crossentropy", optimizer, new string[] { "accuracy" });
var xtrain = TensorUtils.Deserialize(new FileStream(GetDataPath("datasets/nda_mnist/mnist_xtrain.nda"), FileMode.Open));
var ytrain = TensorUtils.Deserialize(new FileStream(GetDataPath("datasets/nda_mnist/mnist_ytrain.nda"), FileMode.Open));
xtrain = xtrain.Cast(DType.Float32);
xtrain = Ops.Div(null, xtrain, 255f);
ytrain = ytrain.Cast(DType.Float32);
model.Fit(xtrain, ytrain, batchSize: 128, epochs: 12);
var stream = new FileStream("c:/ttt/mnist.model", FileMode.OpenOrCreate, FileAccess.Write);
stream.SetLength(0);
model.Save(stream);
}
protected override string ProcessMessageAsLayerOrLoss(Command msgObj, SyftController ctrl)
{
switch (msgObj.functionCall)
{
case "prepare_to_fit":
{
FloatTensor input = ctrl.floatTensorFactory.Get(int.Parse(msgObj.tensorIndexParams[0]));
FloatTensor target = ctrl.floatTensorFactory.Get(int.Parse(msgObj.tensorIndexParams[1]));
Loss.Loss criterion = ctrl.getLoss(int.Parse(msgObj.tensorIndexParams[2]));
SGD optim = ctrl.getOptimizer(int.Parse(msgObj.tensorIndexParams[3]));
int batch_size = int.Parse(msgObj.tensorIndexParams[4]);
return(PrepareToFit(input, target, criterion, optim, batch_size).ToString());
}
case "fit":
{
int start_batch_id = int.Parse(msgObj.tensorIndexParams[0]);
int end_batch_id = int.Parse(msgObj.tensorIndexParams[1]);
int iters = int.Parse(msgObj.tensorIndexParams[2]);
return(Fit(start_batch_id, end_batch_id, iters));
}
}
return(ProcessMessageAsLayerObject(msgObj, ctrl));
}
public void Should_compare_with_null_instance(decimal value)
{
var instance = new SGD(value);
Assert.IsFalse(instance.Equals(null), "Equals");
Assert.AreEqual(1, instance.CompareTo(null), "CompareTo");
}
public static void FitMnistSimple()
{
var model = new Sequential();
model.Add(new Dense(512, activation: "relu", inputShape: new int[] { 784 }));
model.Add(new Dropout(0.2));
model.Add(new Dense(512, activation: "relu"));
model.Add(new Dropout(0.2));
model.Add(new Dense(10, activation: "softmax"));
var optimizer = new SGD(lr: 0.01);
model.Compile("categorical_crossentropy", optimizer, new string[] { "accuracy" });
var xtrain = TensorUtils.Deserialize(new FileStream(GetDataPath("datasets/nda_mnist/mnist_xtrain.nda"), FileMode.Open));
var ytrain = TensorUtils.Deserialize(new FileStream(GetDataPath("datasets/nda_mnist/mnist_ytrain.nda"), FileMode.Open));
xtrain = xtrain.Cast(DType.Float32);
xtrain = Ops.Div(null, xtrain, 255f);
ytrain = ytrain.Cast(DType.Float32);
model.Fit(xtrain, ytrain, batchSize: 128, epochs: 20);
var stream = new FileStream("c:/ttt/mnist-simple.model", FileMode.OpenOrCreate, FileAccess.Write);
stream.SetLength(0);
model.Save(stream);
}
public static void Run()
{
//目標とするフィルタを作成（実践であればココは不明な値となる）
Deconvolution2D decon_core = new Deconvolution2D(1, 1, 15, 1, 7, gpuEnable: true)
{
Weight = { Data = MakeOneCore() }
};
Deconvolution2D model = new Deconvolution2D(1, 1, 15, 1, 7, gpuEnable: true);
SGD optimizer = new SGD(learningRate: 0.00005); //大きいと発散する
model.SetOptimizer(optimizer);
MeanSquaredError meanSquaredError = new MeanSquaredError();
//移植元では同じ教育画像で教育しているが、より実践に近い学習に変更
for (int i = 0; i < 11; i++)
{
//ランダムに点が打たれた画像を生成
NdArray img_p = getRandomImage();
//目標とするフィルタで学習用の画像を出力
NdArray[] img_core = decon_core.Forward(img_p);
//未学習のフィルタで画像を出力
NdArray[] img_y = model.Forward(img_p);
Real loss = meanSquaredError.Evaluate(img_y, img_core);
model.Backward(img_y);
model.Update();
Console.WriteLine("epoch" + i + " : " + loss);
}
}
public static bool SetPageLanguage(ZebraPrinter printer)
{
bool set = false;
string setLang = "zpl";
if (PrinterLanguage.ZPL != printer.PrinterControlLanguage)
{
setLang = "line_print";
}
try
{
VerifyConnection(printer);
SGD.SET("device.languages", setLang, printer.Connection);
string getLang = SGD.GET("device.languages", printer.Connection);
if (getLang.Contains(setLang))
{
set = true;
}
else
{
Console.WriteLine($"This is not a {setLang} printer.");
}
}
catch (ConnectionException e)
{
Console.WriteLine($"Unable to set print language: {e.Message}");
}
return(set);
}
public void sequential_guide_mlp_multiclass()
{
// Generate dummy data
double[,] x_train = Accord.Math.Matrix.Random(1000, 20);
int[] y_train = Accord.Math.Vector.Random(1000, min: 0, max: 10);
double[,] x_test = Accord.Math.Matrix.Random(1000, 20);
int[] y_test = Accord.Math.Vector.Random(1000, min: 0, max: 10);
var model = new Sequential();
// Dense(64) is a fully-connected layer with 64 hidden units.
// in the first layer, you must specify the expected input data shape:
// here, 20-dimensional vectors.
model.Add(new Dense(64, activation: "relu", input_dim: 20));
model.Add(new Dropout(0.5));
model.Add(new Dense(64, activation: "relu"));
model.Add(new Dropout(0.5));
model.Add(new Dense(10, activation: "softmax"));
var sgd = new SGD(lr: 0.01, decay: 1e-6, momentum: 0.9, nesterov: true);
model.Compile(loss: "categorical_crossentropy",
optimizer: sgd,
metrics: new[] { "accuracy" });
model.fit(x_train, y_train,
epochs: 20,
batch_size: 128);
var score = model.evaluate(x_test, y_test, batch_size: 128);
}
public void Should_initialize_instance(decimal value)
{
var actual = new SGD(value);
Assert.IsAssignableFrom <SGD>(actual);
Assert.AreEqual(value, actual.Value, nameof(actual.Value));
}
public static void Run()
{
// Create a target filter (In case of practice, here is the unknown value)
Deconvolution2D decon_core = new Deconvolution2D(true, 1, 1, 15, 1, 7, gpuEnable: true)
{
Weight = { Data = MakeOneCore() }
};
Deconvolution2D model = new Deconvolution2D(true, 1, 1, 15, 1, 7, gpuEnable: true);
SGD optimizer = new SGD(learningRate: 0.00005); // diverge if big
model.SetOptimizer(optimizer);
MeanSquaredError meanSquaredError = new MeanSquaredError();
// I am educating with the same educational image at the transplanting source, but changing to learning closer to practice
for (int i = 0; i < 11; i++)
{
// Generate an image with randomly struck points
NdArray img_p = getRandomImage();
// Output a learning image with a target filter
NdArray[] img_core = decon_core.Forward(true, img_p);
// Output an image with an unlearned filter
NdArray[] img_y = model.Forward(true, img_p);
Real loss = meanSquaredError.Evaluate(img_y, img_core);
model.Backward(true, img_y);
model.Update();
RILogManager.Default?.SendDebug("epoch" + i + " : " + loss);
}
}
public static void Run()
{
//Create a target filter (If it is practical, here is an unknown value)
Deconvolution2D decon_core = new Deconvolution2D(1, 1, 15, 1, 7, gpuEnable: true)
{
Weight = { Data = MakeOneCore() }
};
Deconvolution2D model = new Deconvolution2D(1, 1, 15, 1, 7, gpuEnable: true);
SGD optimizer = new SGD(learningRate: 0.00005); //When it is big, it diverges.
model.SetOptimizer(optimizer);
MeanSquaredError meanSquaredError = new MeanSquaredError();
//At the transplant source, we are educating with the same educational image, but changing to learning closer to practice
for (int i = 0; i < 11; i++)
{
//Generate random dotted images
NdArray img_p = getRandomImage();
//Output a learning image with a target filter
NdArray[] img_core = decon_core.Forward(img_p);
//Output an image with an unlearned filter
NdArray[] img_y = model.Forward(img_p);
Real loss = meanSquaredError.Evaluate(img_y, img_core);
model.Backward(img_y);
model.Update();
Console.WriteLine("epoch" + i + " : " + loss);
}
}
public void sequential_guide_convnet()
{
// Generate dummy data
double[,,,] x_train = (double[, , , ])Accord.Math.Matrix.Zeros <double>(new int[] { 100, 100, 100, 3 }); // TODO: Add a better overload in Accord
int[] y_train = Accord.Math.Vector.Random(100, min: 0, max: 10);
double[,,,] x_test = (double[, , , ])Accord.Math.Matrix.Zeros <double>(new int[] { 20, 100, 100, 3 }); // TODO: Add a better overload in Accord
int[] y_test = Accord.Math.Vector.Random(100, min: 0, max: 10);
var model = new Sequential();
// input: 100x100 images with 3 channels -> (100, 100, 3) tensors.
// this applies 32 convolution filters of size 3x3 each.
model.Add(new Conv2D(32, new[] { 3, 3 }, activation: "relu", input_shape: new int?[] { 100, 100, 3 }));
model.Add(new Conv2D(32, new[] { 3, 3 }, activation: "relu"));
model.Add(new MaxPooling2D(pool_size: new[] { 2, 2 }));
model.Add(new Dropout(0.25));
model.Add(new Conv2D(64, new[] { 3, 3 }, activation: "relu"));
model.Add(new Conv2D(64, new[] { 3, 3 }, activation: "relu"));
model.Add(new MaxPooling2D(pool_size: new[] { 2, 2 }));
model.Add(new Dropout(0.25));
model.Add(new Flatten());
model.Add(new Dense(256, activation: "relu"));
model.Add(new Dropout(0.5));
model.Add(new Dense(10, activation: "softmax"));
var sgd = new SGD(lr: 0.01, decay: 1e-6, momentum: 0.9, nesterov: true);
model.Compile(loss: "categorical_crossentropy", optimizer: sgd);
model.fit(x_train, y_train, batch_size: 32, epochs: 10);
var score = model.evaluate(x_test, y_test, batch_size: 32);
}
public void LinearRegresionTrainingMomentumNesterovWithTimeDecayTest()
{
Console.WriteLine("Linear regression");
// Parameters
var learning_rate = 0.01f;
var training_epochs = 2;
// Training data
var train_x = new float[] {
3.3f, 4.4f, 5.5f, 6.71f, 6.93f, 4.168f, 9.779f, 6.182f, 7.59f, 2.167f,
7.042f, 10.791f, 5.313f, 7.997f, 5.654f, 9.27f, 3.1f
};
var train_y = new float[] {
1.7f, 2.76f, 2.09f, 3.19f, 1.694f, 1.573f, 3.366f, 2.596f, 2.53f, 1.221f,
2.827f, 3.465f, 1.65f, 2.904f, 2.42f, 2.94f, 1.3f
};
var n_samples = train_x.Length;
using (var graph = new TFGraph())
{
var rng = new Random(0);
// tf Graph Input
var X = graph.Placeholder(TFDataType.Float, TFShape.Scalar);
var Y = graph.Placeholder(TFDataType.Float, TFShape.Scalar);
var W = graph.Variable(graph.Const(0.1f), operName: "weight");
var b = graph.Variable(graph.Const(0.1f), operName: "bias");
var pred = graph.Add(graph.Mul(X, W.Read, "x_w"), b.Read);
var cost = graph.Div(graph.ReduceSum(graph.Pow(graph.Sub(pred, Y), graph.Const(2f))), graph.Mul(graph.Const(2f), graph.Const((float)n_samples), "2_n_samples"));
var sgd = new SGD(graph, learning_rate, 0.9f, 0.5f, nesterov: true);
var updateOps = sgd.Minimize(cost);
var readIter = sgd.Iterations.ReadAfter(updateOps);
var readW = W.ReadAfter(updateOps);
var readb = b.ReadAfter(updateOps);
using (var sesssion = new TFSession(graph))
{
sesssion.GetRunner().AddTarget(graph.GetGlobalVariablesInitializer()).Run();
var expectedLines = File.ReadAllLines(Path.Combine(_testDataPath, "MomentumNesterovTimeDecay", "expected.txt"));
for (int i = 0; i < training_epochs; i++)
{
for (int j = 0; j < n_samples; j++)
{
var tensors = sesssion.GetRunner()
.AddInput(X, new TFTensor(train_x[j]))
.AddInput(Y, new TFTensor(train_y[j]))
.AddTarget(updateOps)
.Fetch(readIter, cost, readW, readb, sgd.LearningRate).Run();
var output = Invariant($"step: {tensors[0].GetValue():D}, loss: {tensors[1].GetValue():F4}, W: {tensors[2].GetValue():F4}, b: {tensors[3].GetValue():F4}, lr: {tensors[4].GetValue():F8}");
Assert.Equal(expectedLines[i * n_samples + j], output);
}
}
}
}
}
public string TrainModel(IpfsJob job)
{
var tmpInput = Ipfs.Get <JToken>(job.input);
var tmpTarget = Ipfs.Get <JToken>(job.target);
var seq = CreateSequential(job.Model);
var inputData = tmpInput.SelectToken("data").ToObject <float[]>();
var inputShape = tmpInput.SelectToken("shape").ToObject <int[]>();
var inputTensor = controller.floatTensorFactory.Create(_data: inputData, _shape: inputShape, _autograd: true);
var targetData = tmpTarget.SelectToken("data").ToObject <float[]>();
var targetShape = tmpTarget.SelectToken("shape").ToObject <int[]>();
var targetTensor = controller.floatTensorFactory.Create(_data: targetData, _shape: targetShape, _autograd: true);
var grad = controller.floatTensorFactory.Create(_data: new float[] { 1, 1, 1, 1 },
_shape: new int[] { 4, 1 });
Loss loss;
switch (job.config.criterion)
{
case "mseloss":
loss = new MSELoss(this.controller);
break;
case "categorical_crossentropy":
loss = new CategoricalCrossEntropyLoss(this.controller);
break;
case "cross_entropy_loss":
loss = new CrossEntropyLoss(this.controller, 1); // TODO -- real value
break;
case "nll_loss":
loss = new NLLLoss(this.controller);
break;
default:
loss = new MSELoss(this.controller);
break;
}
var optimizer = new SGD(this.controller, seq.getParameters(), job.config.lr, 0, 0);
for (var i = 0; i < job.config.iters; ++i)
{
var pred = seq.Forward(inputTensor);
var l = loss.Forward(pred, targetTensor);
l.Backward();
// TODO -- better batch size
optimizer.Step(100, i);
}
var resultJob = new Ipfs();
var response = resultJob.Write(new IpfsJob(job.input, job.target, seq.GetConfig(), job.config));
return(response.Hash);
}
public static void XorNeuroTry()
{
var trainData = new double[] { 0.0, 1.0 };
var n_samples = trainData.Length;
using (var g = new TFGraph())
{
var s = new TFSession(g);
var rng = new Random(0);
// tf Graph Input
var X1 = g.Placeholder(TFDataType.Double);
var X2 = g.Placeholder(TFDataType.Double);
//расчетов начальных весов
var W = g.Variable(g.Const(rng.NextDouble()), operName: "weight");
//не уверен, что рассчет смещения рандомным образом - хорошая идея.
var b = g.Variable(g.Const(rng.NextDouble()), operName: "bias");
//вход умноженный на весовой коэффициент плюс смещение = операция которая вычисляет взвешенную сумма весов.
var predX1 = g.Add(g.Mul(X1, W.Read, "x1_w"), b.Read);
var predX2 = g.Add(g.Mul(X2, W.Read, "x2_w"), b.Read);
var pred = g.Add(predX1, predX2);
var cost = g.Sigmoid(pred);
var learning_rate = 0.001f;
var training_epochs = 100;
var sgd = new SGD(g, learning_rate);
var updateOps = sgd.Minimize(cost);
using (var sesssion = new TFSession(g))
{
sesssion.GetRunner().AddTarget(g.GetGlobalVariablesInitializer()).Run();
for (int i = 0; i < training_epochs; i++)
{
double avgLoss = 0;
for (int j = 0; j < n_samples; j++)
{
var tensors = sesssion.GetRunner()
.AddInput(X1, new TFTensor(trainData[j]))
.AddInput(X2, new TFTensor(trainData[j]))
.AddTarget(updateOps).Fetch(sgd.Iterations.Read, cost, W.Read, b.Read, sgd.LearningRate).Run();
avgLoss += (double)tensors[1].GetValue();
}
var tensors2 = sesssion.GetRunner()
.Fetch(W.Read, b.Read).Run();
var output = $"Epoch: {i + 1:D}, loss: {avgLoss / n_samples:F4}, W: {tensors2[0].GetValue():F4}, b: {tensors2[1].GetValue():F4}";
Console.WriteLine(output);
}
}
}
}
public void OptimizeSum()
{
Monomial mon = new Monomial(2, 1);
AbsoluteValue abs = new AbsoluteValue();
var sum = new FAddition <Real, Real, Real>(mon, abs);
SGD.MinimizeFunctionValue(sum, 3, 1);
}
public void Should_throw_exception_on_division_by_zero(decimal value)
{
var instance = new SGD(value);
Assert.Throws <DivideByZeroException>(() => {
var unused = instance / 0;
});
}
public void Should_cast_to_decimal(decimal value)
{
var instance = new SGD(value);
var actual = (decimal)instance;
Assert.AreEqual(value, actual);
}
public void Should_cast_from_decimal(decimal value)
{
var expected = new SGD(value);
var actual = (SGD)value;
Assert.AreEqual(expected, actual);
}
public void Should_compare_with_another_type_of_instance(decimal value)
{
var instance1 = new SGD(value);
object instance2 = value;
Assert.IsFalse(instance1.Equals(instance2), "Equals");
Assert.Throws <ArgumentException>(() => instance1.CompareTo(instance2), "CompareTo");
}
public void Should_divide_instance_by_decimal(double leftValue, double rightValue, double expectedValue)
{
var expected = new SGD((decimal)expectedValue);
var instance = new SGD((decimal)leftValue);
var actual = instance / (decimal)rightValue;
Assert.AreEqual(expected, actual);
}
public void Should_convert_to_string(decimal value)
{
var expected = $"$$ {value:0.00}";
var instance = new SGD(value);
var actual = instance.ToString();
Assert.AreEqual(expected, actual);
}
public void Should_floor_value(double value, double expectedValue)
{
var expected = new SGD((decimal)expectedValue);
var instance = new SGD((decimal)value);
var actual = instance.Floor();
Assert.AreEqual(expected, actual);
}
public void Should_format_string(string format, string mask)
{
var expected = string.Format(Consts.CultureEnUS, mask, 1.7578m);
var instance = new SGD(1.7578m);
var actual = instance.ToString(format, Consts.CultureEnUS);
Assert.AreEqual(expected, actual);
}
private async Task ResetPrinterLanguageToLinePrintAsync(Connection connection)
{
await Task.Factory.StartNew(() => {
try {
connection?.Open();
SGD.SET(DeviceLanguagesSgd, "line_print", connection);
} catch (ConnectionException) { }
});
}
public void Should_own_a_HashCode(decimal value)
{
var expected = value.GetHashCode();
var instance = new SGD(value);
var actual = instance.GetHashCode();
Assert.AreEqual(expected, actual);
}
public void Should_round_value_withDigit(double value, double expectedValue)
{
var expected = new SGD((decimal)expectedValue);
var instance = new SGD((decimal)value);
var actual = instance.Round(1);
Assert.AreEqual(expected, actual);
}
public void Should_roundvalue_withMode(MidpointRounding mode, double value, double expectedValue)
{
var expected = new SGD((decimal)expectedValue);
var instance = new SGD((decimal)value);
var actual = instance.Round(mode);
Assert.AreEqual(expected, actual);
}
public void Should_subtract_two_instances(double leftValue, double rightValue, double expectedValue)
{
var expected = new SGD((decimal)expectedValue);
var leftInstance = new SGD((decimal)leftValue);
var rightInstance = new SGD((decimal)rightValue);
var actual = leftInstance - rightInstance;
Assert.AreEqual(expected, actual);
}
public void Should_multiply_decimal_by_instance(double leftValue, double rightValue, double expectedValue)
{
var expected = new SGD((decimal)expectedValue);
var instance = new SGD((decimal)rightValue);
var actual = (decimal)leftValue * instance;
Assert.AreEqual(expected, actual);
}
public void TestFit()
{
var model = new Sequential();
// model.Add(new Dense(64, inputShape: new Shape(784), activation: "relu"));
model.Add(new Dense(64, inputShape: new int[] { 784 }));
// model.Add(new Dense(128));
var sgd = new SGD(lr: 0.003125);
model.Compile("categorical_crossentropy", sgd, new string[] { "accuracy" });
}
