private async Task<bool> ListKeys(DataBatch batch, TcpResponse tcpResponse)
{
var writer = tcpResponse.CreateTextResponse();
var keys = await _blobStore.ListKeys<MultilayerNetwork>();
foreach (var key in keys)
{
writer.WriteLine(Util.ConvertProtocol(new Uri(_server.BaseEndpoint, key), tcpResponse.Header.TransportProtocol).ToString());
}
return false;
}
private async Task EndRequest(DataBatch endBatch)
{
var fe = endBatch.ForwardingEndpoint;
if (fe != null)
{
DebugOutput.Log("Forwarding to {0}", fe);
using (var client = new VectorTransferClient(_serverId, fe.Port, fe.Host))
{
client.CompressUsing(_compression);
var tx = await client.BeginTransfer(fe.PathAndQuery);
await tx.Send(endBatch);
await tx.End(endBatch.Properties);
}
}
}
private DataBatch ParseRequest(IOwinContext context)
{
var doc = new DataBatch();
{
if (context.Request.Header.ContentLength > 0)
{
doc.Load(context.Request.Content);
((OwinContext)context).Path = doc.Path;
}
else
{
doc.Id = Util.GenerateId();
doc.BatchNum = 1;
doc.Path = context.Request.Header.Path;
doc.Verb = context.Request.Header.Verb;
}
}
return doc;
}
private async Task<bool> Restore(DataBatch batch, TcpResponse tcpResponse)
{
var response = tcpResponse.Content;
var key = batch.Properties["key"];
await _blobStore.Transfer<MultilayerNetwork>(key, response);
return true;
}
private bool ServerStatus(DataBatch batch, TcpResponse tcpResponse)
{
tcpResponse.CreateTextResponse().Write(_server.Status.ToString());
return false;
}
private async Task<bool> Process(DataBatch batch, TcpResponse tcpResponse)
{
IRawClassifierTrainingContext<NetworkParameters> ctx = await GetOrCreateContext(batch);
if (batch.Vectors.Any())
{
var iterations = batch.PropertyOrDefault("Iterations", 1);
foreach (var i in Enumerable.Range(0, iterations))
{
foreach (var inputOutputPair in batch
.Vectors
.Select(v => v.Split(ctx.Parameters.InputVectorSize))
.RandomOrder()
)
{
ctx.Train(inputOutputPair[1], inputOutputPair[0]);
}
}
}
if (!batch.KeepAlive)
{
if (batch.PropertyOrDefault("SaveOutput", false))
{
var network = (MultilayerNetwork)ctx.Output;
foreach (var prop in batch.Properties)
{
network.Properties[prop.Key] = prop.Value;
}
var name = batch.PropertyOrDefault("Name", batch.Id);
await _blobStore.StoreAsync(name, network);
}
ctx.Output.Save(tcpResponse.Content);
_trainingContexts.Remove(batch.Id);
}
else
{
if (batch.SendResponse)
{
WriteSummaryResponse(ctx, tcpResponse.Content);
}
}
return true;
}
private async Task<IRawClassifierTrainingContext<NetworkParameters>> GetOrCreateContext(DataBatch batch)
{
IRawClassifierTrainingContext<NetworkParameters> ctx;
//lock (_trainingContexts)
{
if (!_trainingContexts.TryGetValue(batch.Id, out ctx))
{
var name = batch.PropertyOrDefault("Name", batch.Id);
var layerSizes = batch.Properties["LayerSizes"].Split(',').Select(s => int.Parse(s)).ToArray();
var networkParams = new NetworkParameters(layerSizes);
try
{
var network = new MultilayerNetwork(networkParams);
await _blobStore.RestoreAsync(name, network);
_trainingContexts[batch.Id] = ctx = _trainingContextFactory.Create(network);
}
catch
{
_trainingContexts[batch.Id] = ctx = _trainingContextFactory.Create(networkParams);
}
}
}
return ctx;
}
private bool Delete(DataBatch batch, TcpResponse tcpResponse)
{
var response = tcpResponse.Content;
var key = batch.Properties["key"];
var deleted = _blobStore.Delete<MultilayerNetwork>(key);
batch.Properties["Deleted"] = deleted.ToString();
batch.Save(response);
return deleted;
}
public async Task<Stream> End(IDictionary<string, string> parameters = null, Uri forwardResponseTo = null)
{
var doc = new DataBatch();
if (parameters != null)
{
foreach (var kv in parameters) doc.Properties[kv.Key] = kv.Value;
}
if (forwardResponseTo != null)
{
doc.ForwardingEndpoint = forwardResponseTo;
}
var response = await SendBatch(doc, true);
DebugOutput.Log("Shutting down client socket");
ClientSocket.Shutdown(SocketShutdown.Both);
return response;
}
private static async Task Send(TransferHandle state, DataBatch transferDoc, ICompressionProvider compression)
{
var buffer = new byte[state.BufferSize];
using (var ms = new MemoryStream())
using (var cs = compression.CompressTo(ms))
{
transferDoc.Save(cs);
cs.Flush();
cs.Dispose();
ms.Flush();
ms.Position = 0;
var sockWriter = new AsyncSocketWriterReader(state.ClientSocket, state.BufferSize);
var sent = await sockWriter.WriteAsync(ms);
DebugOutput.LogVerbose("Sent {0} bytes", sent);
}
}
private async Task<Stream> SendBatch(BinaryVectorDocument doc, bool isLast, bool sendResponse = false)
{
{
var transferDoc = new DataBatch();
foreach (var prop in doc.Properties)
{
transferDoc.Properties[prop.Key] = prop.Value;
}
foreach (var vec in doc.Vectors)
{
transferDoc.Vectors.Add(vec);
}
foreach (var child in doc.Children)
{
transferDoc.Children.Add(child);
}
transferDoc.Id = Id;
transferDoc.ClientId = ClientId;
transferDoc.BatchNum = (_batchIndex++);
transferDoc.KeepAlive = !isLast;
transferDoc.SendResponse = sendResponse;
transferDoc.Path = Path;
transferDoc.Verb = Verb;
DebugOutput.Log("Sending batch {0}/{1}", Id, transferDoc.BatchNum);
await Send(this, transferDoc, _compression);
}
return await ReceiveData(this, _compression);
}