public IVertex CreateVertex(string vertexDefinition, CloudBlobContainer container)
{
if (_dynamicLoadingEnabled)
{
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/binaries");
Stream blobStream = blockBlob.OpenReadAsync().GetAwaiter().GetResult();
try
{
AssemblyUtils.LoadAssembliesFromStream(blobStream);
}
catch (FileLoadException e)
{
Debug.WriteLine("Ignoring exception from assembly loading: " + e.Message);
Debug.WriteLine("If vertex creation fails, the caller will need to sideload the vertex.");
}
blobStream.Close();
}
else
{
Debug.WriteLine("Dynamic assembly loading is disabled. The caller will need to sideload the vertex.");
}
var row = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition);
// CREATE THE VERTEX
IVertex vertex = null;
try
{
vertex = row.GetVertexCreateAction()();
}
catch (Exception e)
{
Debug.WriteLine("Vertex creation failed: " + e.Message);
Debug.WriteLine("The caller will need to sideload the vertex.");
}
return(vertex);
}
internal CRAErrorCode InstantiateVertex(string instanceName, string vertexName, string vertexDefinition, object vertexParameter, bool sharded)
{
var procDefRow = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition);
string blobName = vertexName + "-" + instanceName;
// Serialize and write the vertex parameters to a blob
CloudBlobContainer container = _blobClient.GetContainerReference("cra");
container.CreateIfNotExistsAsync().Wait();
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/" + blobName);
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
byte[] parameterBytes = Encoding.UTF8.GetBytes(
SerializationHelper.SerializeObject(vertexParameter));
blobStream.WriteByteArray(parameterBytes);
blobStream.Close();
// Add metadata
var newRow = new VertexTable(instanceName, vertexName, vertexDefinition, "", 0,
procDefRow.VertexCreateAction,
blobName,
false, sharded);
TableOperation insertOperation = TableOperation.InsertOrReplace(newRow);
_vertexTable.ExecuteAsync(insertOperation).Wait();
CRAErrorCode result = CRAErrorCode.Success;
// Send request to CRA instance
VertexTable instanceRow;
try
{
instanceRow = VertexTable.GetRowForInstance(_vertexTable, instanceName);
// Get a stream connection from the pool if available
Stream stream;
if (!TryGetSenderStreamFromPool(instanceRow.Address, instanceRow.Port.ToString(), out stream))
{
TcpClient client = new TcpClient(instanceRow.Address, instanceRow.Port);
client.NoDelay = true;
stream = client.GetStream();
if (SecureStreamConnectionDescriptor != null)
{
stream = SecureStreamConnectionDescriptor.CreateSecureClient(stream, instanceName);
}
}
stream.WriteInt32((int)CRATaskMessageType.LOAD_VERTEX);
stream.WriteByteArray(Encoding.UTF8.GetBytes(vertexName));
stream.WriteByteArray(Encoding.UTF8.GetBytes(vertexDefinition));
stream.WriteByteArray(Encoding.UTF8.GetBytes(newRow.VertexParameter));
result = (CRAErrorCode)stream.ReadInt32();
if (result != 0)
{
Console.WriteLine("Vertex was logically loaded. However, we received an error code from the hosting CRA instance: " + result);
}
// Add/Return stream connection to the pool
TryAddSenderStreamToPool(instanceRow.Address, instanceRow.Port.ToString(), stream);
}
catch
{
Console.WriteLine("The CRA instance appears to be down. Restart it and this vertex will be instantiated automatically");
}
return(result);
}
/// <summary>
/// Load a vertex on the local instance
/// </summary>
/// <param name="vertexName"></param>
/// <param name="vertexDefinition"></param>
/// <param name="vertexParameter"></param>
/// <param name="instanceName"></param>
/// <param name="table"></param>
/// <returns></returns>
public async Task <IVertex> LoadVertexAsync(string vertexName, string vertexDefinition, string vertexParameter, string instanceName, ConcurrentDictionary <string, IVertex> table)
{
// Deactivate vertex
_vertexTableManager.DeactivateVertexOnInstance(vertexName, instanceName);
CloudBlobContainer container = _blobClient.GetContainerReference("cra");
container.CreateIfNotExistsAsync().Wait();
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/binaries");
Stream blobStream = blockBlob.OpenReadAsync().GetAwaiter().GetResult();
AssemblyUtils.LoadAssembliesFromStream(blobStream);
blobStream.Close();
var row = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition);
// CREATE THE VERTEX
var vertex = row.GetVertexCreateAction()();
// INITIALIZE
if ((VertexBase)vertex != null)
{
((VertexBase)vertex).VertexName = vertexName;
((VertexBase)vertex).ClientLibrary = this;
}
// LATCH CALLBACKS TO POPULATE ENDPOINT TABLE
vertex.OnAddInputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, true, false));
vertex.OnAddOutputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, false, false));
vertex.OnAddAsyncInputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, true, true));
vertex.OnAddAsyncOutputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, false, true));
//ADD TO TABLE
if (table != null)
{
table.AddOrUpdate(vertexName, vertex, (procName, oldProc) => { oldProc.Dispose(); return(vertex); });
vertex.OnDispose(() =>
{
// Delete all endpoints of the vertex
foreach (var key in vertex.InputEndpoints)
{
_endpointTableManager.DeleteEndpoint(vertexName, key.Key);
}
foreach (var key in vertex.AsyncInputEndpoints)
{
_endpointTableManager.DeleteEndpoint(vertexName, key.Key);
}
foreach (var key in vertex.OutputEndpoints)
{
_endpointTableManager.DeleteEndpoint(vertexName, key.Key);
}
foreach (var key in vertex.AsyncOutputEndpoints)
{
_endpointTableManager.DeleteEndpoint(vertexName, key.Key);
}
IVertex old;
if (!table.TryRemove(vertexName, out old))
{
Console.WriteLine("Unable to remove vertex on disposal");
}
var entity = new DynamicTableEntity(instanceName, vertexName);
entity.ETag = "*";
TableOperation deleteOperation = TableOperation.Delete(entity);
_vertexTable.ExecuteAsync(deleteOperation).Wait();
});
}
string blobName = vertexName + "-" + instanceName;
var parametersBlob = container.GetBlockBlobReference(vertexDefinition + "/" + blobName);
Stream parametersStream = parametersBlob.OpenReadAsync().GetAwaiter().GetResult();
byte[] parametersBytes = parametersStream.ReadByteArray();
string parameterString = Encoding.UTF8.GetString(parametersBytes);
parametersStream.Close();
var par = SerializationHelper.DeserializeObject(parameterString);
vertex.Initialize(par);
await vertex.InitializeAsync(par);
// Activate vertex
ActivateVertex(vertexName, instanceName);
return(vertex);
}
