public void Should_construct_serializer_that_uses_requested_encoding()
{
var settings = new SettingsHolder();
var extensions = new SerializationExtensions<JsonSerializer>(settings);
extensions.Encoding(Encoding.UTF7);
var serializer = (NServiceBus.JsonMessageSerializer)new JsonSerializer().Configure(settings)(new MessageMapper());
Assert.AreSame(Encoding.UTF7, serializer.Encoding);
}
public void Dropout()
{
var cns = new ConvNetSharp <double>();
var x = cns.Const(1.0, "x");
var dropoutProbability = 0.5;
var op = cns.Dropout(x, dropoutProbability);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Dropout <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(dropoutProbability, ((Const <double>)deserialized.DropoutProbability).Value);
}
public void Reshape1()
{
var x = new Const <double>(1.0, "x");
var op = new Reshape <double>(x, new Shape(1, 2, 3, 4));
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Reshape <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(1, deserialized.Parents.Count);
Assert.AreEqual("x", (deserialized.Parents[0] as Const <double>).Name);
Assert.AreEqual(op.OutputShape, deserialized.OutputShape);
}
public void Softmax()
{
var cns = new ConvNetSharp <double>();
var x = cns.Const(1.0, "x");
var op = cns.Softmax(x);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Softmax <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(1, deserialized.Parents.Count);
Assert.AreEqual("x", (deserialized.Parents[0] as Const <double>).Name);
}
public void Activation()
{
var x = new Const <double>(1.0, "x");
var op = new Activation <double>(x, ActivationType.Relu);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Activation <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(1, deserialized.Parents.Count);
Assert.AreEqual("x", (deserialized.Parents[0] as Const <double>).Name);
Assert.AreEqual(ActivationType.Relu, deserialized.Type);
}
public void Log()
{
var cns = new ConvNetSharp <double>();
var a = cns.Const(1.0, "one");
var op = cns.Log(a);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Log <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(1, deserialized.Parents.Count);
Assert.AreEqual("one", (deserialized.Parents[0] as Const <double>).Name);
}
public void SoftmaxCrossEntropy()
{
var softmax = new Const <double>(1.0, "softmax");
var y = new Const <double>(1.0, "y");
var op = new SoftmaxCrossEntropy <double>(softmax, y);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as SoftmaxCrossEntropy <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(2, deserialized.Parents.Count);
Assert.AreEqual("softmax", (deserialized.Parents[0] as Const <double>).Name);
Assert.AreEqual("y", (deserialized.Parents[1] as Const <double>).Name);
}
public void Div()
{
var a = new Const <double>(1.0, "one");
var b = new Const <double>(2.0, "two");
var op = new Div <double>(a, b);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Div <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(2, deserialized.Parents.Count);
Assert.AreEqual("one", (deserialized.Parents[0] as Const <double>).Name);
Assert.AreEqual("two", (deserialized.Parents[1] as Const <double>).Name);
}
public static INet LoadNet(string filename)
{
INet result = null;
if (File.Exists(filename))
{
using (var fs = new FileStream(filename, FileMode.Open))
{
result = SerializationExtensions.LoadBinary(fs);
}
}
return(result);
}
/// <summary>
/// Get a base64 encoded encrypted xml serialization string representing the specified target object
/// </summary>
/// <param name="target">The object to serialize</param>
/// <param name="key">The key used to encrypt and decrypt the resulting string</param>
/// <returns>string</returns>
public static string ToBase64EncodedEncryptedXml(this object target, out AesKeyVectorPair key)
{
string xml = SerializationExtensions.ToXml(target);
AesManaged rm = new AesManaged();
rm.GenerateIV();
rm.GenerateKey();
key = new AesKeyVectorPair
{
Key = Convert.ToBase64String(rm.Key),
IV = Convert.ToBase64String(rm.IV)
};
return(Encrypt(xml, rm.CreateEncryptor()));
}
public void Reshape2()
{
var x = new Const <double>(1.0, "x");
var shape = new Const <double>(new[] { 1.0, 2.0, 3.0, 4.0 }, "shape");
var op = new Reshape <double>(x, shape);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Reshape <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(2, deserialized.Parents.Count);
Assert.AreEqual("x", (deserialized.Parents[0] as Const <double>).Name);
Assert.AreEqual("shape", (deserialized.Parents[1] as Const <double>).Name);
}
private void openButton_Click(object sender, EventArgs e)
{
try
{
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
var jsonText = File.ReadAllText(openFileDialog1.FileName);
_net = SerializationExtensions.FromJson <double>(jsonText);
}
}
catch (Exception ex)
{
MessageBox.Show("Invalid file.");
}
}
public void Add()
{
var cns = new ConvNetSharp <double>();
var a = cns.Const(1.0, "one");
var b = cns.Const(2.0, "two");
var op = a + b;
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Add <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(2, deserialized.Parents.Count);
Assert.AreEqual("one", (deserialized.Parents[0] as Const <double>).Name);
Assert.AreEqual("two", (deserialized.Parents[1] as Const <double>).Name);
}
/// <summary>
/// This sample shows how to serialize and deserialize a ConvNetSharp.Core network
/// 1) Network creation
/// 2) Dummy Training (only use a single data point)
/// 3) Serialization
/// 4) Deserialization
/// </summary>
private static void Main()
{
// 1) Network creation
var net = new Net <double>();
net.AddLayer(new InputLayer(1, 1, 2));
net.AddLayer(new FullyConnLayer(20));
net.AddLayer(new ReluLayer());
net.AddLayer(new FullyConnLayer(10));
net.AddLayer(new SoftmaxLayer(10));
// 2) Dummy Training (only use a single data point)
var x = BuilderInstance.Volume.From(new[] { 0.3, -0.5 }, new Shape(2));
var y = BuilderInstance.Volume.From(new[] { 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 }, new Shape(10));
var count = 0;
var trainer = new SgdTrainer(net)
{
LearningRate = 0.01
};
do
{
trainer.Train(x, y); // train the network, specifying that x is class zero
Console.WriteLine($"Loss: {trainer.Loss}");
count++;
} while (trainer.Loss > 1e-2);
Console.WriteLine($"{count}");
// Forward pass with original network
var prob1 = net.Forward(x);
Console.WriteLine("probability that x is class 0: " + prob1.Get(0));
// 3) Serialization
var json = net.ToJson();
// 4) Deserialization
var deserialized = SerializationExtensions.FromJson <double>(json);
// Forward pass with deserialized network
var prob2 = deserialized.Forward(x);
Console.WriteLine("probability that x is class 0: " + prob2.Get(0)); // This should give exactly the same result as previous network evaluation
Console.ReadLine();
}
public void JoinAccountsAndContactsWithMetadata()
{
Guid g = StopWatch.Start();
SqlBuilder builder = SqlBuilder.Select()
.From("Account")
.AllColumns()
.WithMetadata().InnerJoin("Contact", null)
.AllColumns()
.Builder();
Console.WriteLine(builder.ToSql());
ResultTable result = builder.Execute(30, false);
Console.WriteLine("{0} rows selected in {1}ms", result.Count, StopWatch.Stop(g, StopWatch.WatchTypes.Milliseconds));
Console.WriteLine(SerializationExtensions.ToJson <dynamic>(result.First(), true));
}
/// <summary>
/// オブジェクトの内容をXMLに書き込む
/// </summary>
public void WriteJson()
{
var rootPath = Path.GetDirectoryName(this.FilePath) + "\\";
foreach (var b in this.Books)
{
b.FilePath = b.FilePath.Replace(rootPath, string.Empty);
}
SerializationExtensions.WriteJson(this, this.FilePath);
foreach (var b in this.Books)
{
b.FilePath = rootPath + b.FilePath;
}
}
protected async override Task <Message[]> GetInternalAsync(Stream stream)
{
var buffer = new byte[_length];
await stream.ReadBytesAsync(buffer, "message data", false).ConfigureAwait(false);
try
{
var messages = SerializationExtensions.ToMessages(buffer);
_logger.Debug("Deserialized {0} messages from {1}", messages.Length, _endpoint);
return(messages);
}
catch (Exception ex)
{
throw new SerializationException("Failed to deserialize message", ex);
}
}
private void Step(Item sample)
{
var x = sample.Volume;
var y = sample.Label; // pego o rótulo e a imagem
if (sample.IsValidation) // vejo se é de validação ou treino
{
// a estimação do erro é feita com os dados de entrada 'x' e a previsão boa ou ruim é 'valAcc'
net.Forward(x);
var yhat = net.GetPrediction();
var valAcc = yhat == y ? 1.0 : 0.0;
valAccWindow.Add(valAcc);
return;
}
trainer.Train(x, y);
var lossx = trainer.CostLoss;
var lossw = trainer.L2DecayLoss;
// erro e a função perda
var prediction = net.GetPrediction();
var trainAcc = prediction == y ? 1.0 : 0.0;
xLossWindow.Add(lossx);
wLossWindow.Add(lossw);
trainAccWindow.Add(trainAcc);
if (stepCount % 200 == 0)
{
if (xLossWindow.Count == xLossWindow.Capacity)
{
var xa = xLossWindow.Items.Average();
var xw = wLossWindow.Items.Average();
var loss = xa + xw;
Console.WriteLine("Perda: {0} precisão no treiono: {1}% precisão do teste: {2}%", loss, Math.Round(trainAccWindow.Items.Average() * 100.0, 2), Math.Round(valAccWindow.Items.Average() * 100.0, 2));
Console.WriteLine("{0} exemplos vistos. tempo pro fwd: {1}ms tempo pro bckw: {2}ms", stepCount, Math.Round(trainer.ForwardTime.TotalMilliseconds, 2), Math.Round(trainer.BackwardTime.TotalMilliseconds, 2));
// AQUI, salvo A REDE ATUAl, POSSO ESCOLHER MAIS TARDE qual usar
// a melhor rede atual está salva com o nome de 'best' na pasta debug
File.WriteAllText("rede" + stepCount.ToString(), SerializationExtensions.ToJSON(net));
}
}
if (stepCount % 1000 == 0)
{
TestPredict();
}
stepCount++;
}
public Form1()
{
InitializeComponent();
this.openFileDialog1.InitialDirectory = Directory.GetCurrentDirectory();
if (!string.IsNullOrEmpty(Settings.Default.LastFilename))
{
SetFilename(Settings.Default.LastFilename);
}
BuilderInstance <float> .Volume = new VolumeBuilder();
; // Needed for GPU, must be done on AI thread
// Must be done after setting BuilderInstance<float>.Volume
this._yhat = SerializationExtensions.Load <float>("FaceDetection", false)[0];
}
public void Convolution()
{
var x = new Const <double>(1.0, "x");
var op = new Convolution <double>(x, 5, 5, 16, 2, 1);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Convolution <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(2, deserialized.Parents.Count); // input and filters
Assert.AreEqual("x", (deserialized.Parents[0] as Const <double>).Name);
Assert.AreEqual(5, deserialized.Width);
Assert.AreEqual(5, deserialized.Height);
Assert.AreEqual(16, deserialized.FilterCount);
Assert.AreEqual(2, deserialized.Stride);
Assert.AreEqual(1, deserialized.Pad);
}
public void SelectAndSerializeDataRows()
{
SqlBuilder builder = SqlBuilder.Select(100)
.From("Account")
.AllColumns(false)
//.WhereNotExists("Contact").And("AccountID", SqlOperators.Equal, "AccountID")
//.EndExists().Builder.BaseTable()
.SubSelect("Contact", "AccountID", "AccountID", "c")
.AllColumns(false)
.Builder();
Console.WriteLine(builder.ToSql());
ResultTable result = builder.Execute();
string prefix = DateTime.Now.Ticks.ToString();
string path = Path.GetTempPath();
for (int i = 0; i < result.Count; i++)
{
SerializationExtensions.ToFile <RowData>(result[i], Path.Combine(path, string.Format("Row{0}@{1}.json", i + 1, prefix)));
}
ResultTable result2 = new ResultTable();
for (int i = 0; i < result.Count; i++)
{
result2.Add(SerializationExtensions.FromFile <RowData>(Path.Combine(path, string.Format("Row{0}@{1}.json", i + 1, prefix))));
}
Assert.IsTrue(result.Count == result2.Count, "Number of rows does not match");
result = new ResultTable(builder, 60, false);
string prefix2 = DateTime.Now.Ticks.ToString();
for (int i = 0; i < result.Count; i++)
{
SerializationExtensions.ToFile <RowData>(result[i], Path.Combine(path, string.Format("Row{0}@{1}.json", i + 1, prefix2)));
}
string[] files = Directory.EnumerateFiles(path, "Row*.json").ToArray();
foreach (string file in files)
{
File.Delete(file);
}
}
public void Pool()
{
var x = new Const <double>(1.0, "x");
var op = new Pool <double>(x, 3, 4, 1, 2, 1, 2);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Pool <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(1, deserialized.Parents.Count);
Assert.AreEqual("x", (deserialized.Parents[0] as Const <double>).Name);
Assert.AreEqual(3, deserialized.Width);
Assert.AreEqual(4, deserialized.Height);
Assert.AreEqual(1, deserialized.HorizontalPad);
Assert.AreEqual(2, deserialized.VerticalPad);
Assert.AreEqual(1, deserialized.HorizontalStride);
Assert.AreEqual(2, deserialized.VerticalStride);
}
public org.iringtools.library.Configuration GetConfig(string scope, string application)
{
org.iringtools.library.Configuration config = null;
try
{
WebHttpClient client = CreateWebClient(_adapterServiceUri);
XElement element = client.Get <XElement>(string.Format("/{0}/{1}/config", scope, application), true);
config = SerializationExtensions.ToObject <org.iringtools.library.Configuration>(element);
}
catch (Exception ex)
{
_logger.Error(ex.ToString());
throw ex;
}
return(config);
}
public void ExecuteAndSerialize()
{
Guid g = StopWatch.Start();
SqlBuilder sb = SqlBuilder.Select(5)
.From("Account")
.AllColumns(false)
.Builder;
Console.WriteLine(sb.ToSql());
ResultTable result = sb.Execute();
string s = SerializationExtensions.ToJson <ResultTable>(result, true);
Console.WriteLine("{0} rows executed and serialized in {1}ms", result.Count, StopWatch.Stop(g, StopWatch.WatchTypes.Milliseconds));
g = StopWatch.Start();
result = SerializationExtensions.FromJson <ResultTable>(s);
Console.WriteLine("{0} rows de-serialized in {1}ms", result.Count, StopWatch.Stop(g, StopWatch.WatchTypes.Milliseconds));
Console.WriteLine(s);
}
public void Dense()
{
var cns = new ConvNetSharp <double>();
var x = cns.Const(1.0, "x");
var op = new Dense <double>(cns, x, 16);
var xml = op.ToXml();
var deserialized = SerializationExtensions.FromXml <double>(xml) as Dense <double>;
Assert.IsNotNull(deserialized);
Assert.AreEqual(2, deserialized.Parents.Count); // input and filters
Assert.AreEqual("x", (deserialized.Parents[0] as Const <double>).Name);
Assert.AreEqual(1, deserialized.Width);
Assert.AreEqual(1, deserialized.Height);
Assert.AreEqual(16, deserialized.FilterCount);
Assert.AreEqual(1, deserialized.Stride);
Assert.AreEqual(0, deserialized.Pad);
}
public void ExecuteDeepAndSerialize()
{
ResultTable result = ExecuteDeepInternal();
string file = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".json");
Guid g = StopWatch.Start();
File.WriteAllText(file, TinySql.Serialization.SerializationExtensions.ToJson <ResultTable>(result));
Console.WriteLine(StopWatch.Stop(g, StopWatch.WatchTypes.Milliseconds, "Results serialized in {0}ms"));
g = StopWatch.Start();
ResultTable deserialized = SerializationExtensions.FromJson <ResultTable>(File.ReadAllText(file));
Console.WriteLine(StopWatch.Stop(g, StopWatch.WatchTypes.Milliseconds, "Results deserialized in {0}ms"));
FileInfo fi = new FileInfo(file);
Console.WriteLine("The File is {0:0.00}MB in size", (double)fi.Length / (double)(1024 * 1024));
fi.Delete();
Assert.IsFalse(File.Exists(file));
Assert.IsTrue(result.Count == deserialized.Count);
}
private MetadataDatabase FromCache()
{
if (UseCache == false)
{
return(null);
}
MetadataDatabase mdb = CacheMetadata(MetadataKey);
if (mdb != null)
{
return(mdb);
}
if (!string.IsNullOrEmpty(FileName))
{
mdb = SerializationExtensions.FromFile(FileName);
CacheMetadata(MetadataKey, mdb);
}
return(mdb);
}
public void FluentNetSerialization()
{
// Fluent version
var net = FluentNet <double> .Create(24, 24, 1)
.Conv(5, 5, 8).Stride(1).Pad(2)
.Relu()
.Pool(2, 2).Stride(2)
.Conv(5, 5, 16).Stride(1).Pad(2)
.Relu()
.Pool(3, 3).Stride(3)
.FullyConn(10)
.Softmax(10)
.Build();
var json = net.ToJson();
var deserialized = SerializationExtensions.FromJson <double>(json);
Assert.AreEqual(9, deserialized.Layers.Count);
}
public int LoadCNN(string path)
{
if ((path != "") && (File.Exists(path)))
{
loadedJson = "";
loadedJson = File.ReadAllText(path);
var deserialized = SerializationExtensions.FromJson <double>(loadedJson);
this.net = deserialized;
isNetLearned = true;
this.path = path;
classes = net.Layers[GetLayersCount() - 1].OutputDepth;
return(net.Layers.Count);
}
else
{
return(-1);
}
}
private static void MemberDictionaryClone(
Dictionary <SignatureKey, object> source,
Dictionary <SignatureKey, object> target)
{
foreach (var item in source)
{
if (item.Value is Delegate)
{
var del = item.Value as Delegate;
var copy = del.Clone();
target.Add(item.Key, copy);
}
else
{
var copy = SerializationExtensions.FromBinary(item.Value.ToBinary());
target.Add(item.Key, copy);
}
}
}
public void ThreadPoolCallback(object threadContext)
{
int threadIndex = (int)threadContext;
if (_identifiers != null && _identifiers.Count > 0)
{
List <IDataObject> dataObjects = _dataLayerGateway.Get(_dataObject, _identifiers);
if (dataObjects != null)
{
XDocument dtoDoc = _projectionLayer.ToXml(_graphMap.name, ref dataObjects);
if (dtoDoc != null && dtoDoc.Root != null)
{
_dataTransferObjects = SerializationExtensions.ToObject <DataTransferObjects>(dtoDoc.Root);
}
}
}
_doneEvent.Set();
}
