internal VertexTable GetRowForActiveVertex(string vertexName)
{
return(VertexTable.GetAll(_vertexTable)
.Where(gn => vertexName == gn.VertexName && !string.IsNullOrEmpty(gn.InstanceName))
.Where(gn => gn.IsActive)
.First());
}
/// <summary>
/// Connect a set of vertices in one sharded CRA vertex to another with a full mesh topology, via pre-defined endpoints. We contact the "from" vertex
/// to initiate the creation of the link.
/// </summary>
/// <param name="fromVertexName"></param>
/// <param name="fromEndpoints"></param>
/// <param name="toVertexName"></param>
/// <param name="toEndpoints"></param>
/// <returns></returns>
public CRAErrorCode ConnectShardedVerticesWithFullMesh(string fromVertexName, string[] fromEndpoints, string toVertexName, string[] toEndpoints)
{
var fromVerticesRows = VertexTable.GetRowsForShardedVertex(_vertexTable, fromVertexName);
string[] fromVerticesNames = new string[fromVerticesRows.Count()];
int i = 0;
foreach (var fromVertexRow in fromVerticesRows)
{
fromVerticesNames[i] = fromVertexRow.VertexName;
i++;
}
var toVerticesRows = VertexTable.GetRowsForShardedVertex(_vertexTable, toVertexName);
string[] toVerticesNames = new string[toVerticesRows.Count()];
i = 0;
foreach (var toVertexRow in toVerticesRows)
{
toVerticesNames[i] = toVertexRow.VertexName;
i++;
}
return(ConnectShardedVerticesWithFullMesh(fromVerticesNames, fromEndpoints, toVerticesNames, toEndpoints, ConnectionInitiator.FromSide));
}
/// <summary>
/// Disconnect a set of vertices in one sharded CRA vertex from another, via pre-defined endpoints.
/// </summary>
/// <param name="fromVertexName"></param>
/// <param name="fromEndpoints"></param>
/// <param name="toVertexName"></param>
/// <param name="toEndpoints"></param>
/// <returns></returns>
public void DisconnectShardedVertices(string fromVertexName, string[] fromEndpoints, string toVertexName, string[] toEndpoints)
{
var fromVerticesRows = VertexTable.GetRowsForShardedVertex(_vertexTable, fromVertexName);
string[] fromVerticesNames = new string[fromVerticesRows.Count()];
int i = 0;
foreach (var fromVertexRow in fromVerticesRows)
{
fromVerticesNames[i] = fromVertexRow.VertexName;
i++;
}
var toVerticesRows = VertexTable.GetRowsForShardedVertex(_vertexTable, toVertexName);
string[] toVerticesNames = new string[toVerticesRows.Count()];
i = 0;
foreach (var toVertexRow in toVerticesRows)
{
toVerticesNames[i] = toVertexRow.VertexName;
i++;
}
DisconnectShardedVertices(fromVerticesNames, fromEndpoints, toVerticesNames, toEndpoints);
}
internal void DeleteShardedVertex(string vertexName)
{
foreach (var row in VertexTable.GetRowsForShardedVertex(_vertexTable, vertexName))
{
TableOperation deleteOperation = TableOperation.Delete(row);
_vertexTable.ExecuteAsync(deleteOperation).Wait();
}
}
internal void DeactivateVertexOnInstance(string vertexName, string instanceName)
{
var newActiveVertex = VertexTable.GetAll(_vertexTable)
.Where(gn => instanceName == gn.InstanceName && vertexName == gn.VertexName)
.First();
newActiveVertex.IsActive = false;
TableOperation insertOperation = TableOperation.InsertOrReplace(newActiveVertex);
_vertexTable.ExecuteAsync(insertOperation).Wait();
}
private void RestoreVerticesAndConnections()
{
var rows = VertexTable.GetAllRowsForInstance(_workerInstanceTable, _workerinstanceName);
foreach (var _row in rows)
{
if (string.IsNullOrEmpty(_row.VertexName))
{
continue;
}
RestoreVertexAndConnections(_row);
}
}
private void RestoreConnections(VertexTable _row)
{
// Decide what to do if connection creation fails
var outRows = ConnectionTable.GetAllConnectionsFromVertex(_connectionTable, _row.VertexName).ToList();
foreach (var row in outRows)
{
Task.Run(() => RetryRestoreConnection(row.FromVertex, row.FromEndpoint, row.ToVertex, row.ToEndpoint, false));
}
var inRows = ConnectionTable.GetAllConnectionsToVertex(_connectionTable, _row.VertexName).ToList();
foreach (var row in inRows)
{
Task.Run(() => RetryRestoreConnection(row.FromVertex, row.FromEndpoint, row.ToVertex, row.ToEndpoint, true));
}
}
/// <summary>
/// Load all vertices for the given instance name, returns only when all
/// vertices have been initialized and activated.
/// </summary>
/// <param name="thisInstanceName"></param>
/// <returns></returns>
public ConcurrentDictionary <string, IVertex> LoadAllVertices(string thisInstanceName)
{
ConcurrentDictionary <string, IVertex> result = new ConcurrentDictionary <string, IVertex>();
var rows = VertexTable.GetAllRowsForInstance(_vertexTable, thisInstanceName);
List <Task> t = new List <Task>();
foreach (var row in rows)
{
if (row.VertexName == "")
{
continue;
}
t.Add(LoadVertexAsync(row.VertexName, row.VertexDefinition, row.VertexParameter, thisInstanceName, result));
}
Task.WaitAll(t.ToArray());
return(result);
}
/// <summary>
/// Define a sharded vertex type and register with CRA.
/// </summary>
/// <param name="vertexDefinition">Name of the vertex type</param>
/// <param name="creator">Lambda that describes how to instantiate the vertex, taking in an object as parameter</param>
public CRAErrorCode DefineVertex(string vertexDefinition, Expression <Func <IShardedVertex> > creator)
{
CloudBlobContainer container = _blobClient.GetContainerReference("cra");
container.CreateIfNotExistsAsync().Wait();
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/binaries");
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
AssemblyUtils.WriteAssembliesToStream(blobStream);
blobStream.Close();
// Add metadata
var newRow = new VertexTable("", vertexDefinition, vertexDefinition, "", 0, creator, null, true);
TableOperation insertOperation = TableOperation.InsertOrReplace(newRow);
_vertexTable.ExecuteAsync(insertOperation).Wait();
return(CRAErrorCode.Success);
}
public IVertex CreateVertex(string vertexDefinition, CloudBlobContainer container)
{
if (_dynamicLoadingEnabled)
{
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/binaries");
Stream blobStream = blockBlob.OpenReadAsync().GetAwaiter().GetResult();
try
{
AssemblyUtils.LoadAssembliesFromStream(blobStream);
}
catch (FileLoadException e)
{
Debug.WriteLine("Ignoring exception from assembly loading: " + e.Message);
Debug.WriteLine("If vertex creation fails, the caller will need to sideload the vertex.");
}
blobStream.Close();
}
else
{
Debug.WriteLine("Dynamic assembly loading is disabled. The caller will need to sideload the vertex.");
}
var row = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition);
// CREATE THE VERTEX
IVertex vertex = null;
try
{
vertex = row.GetVertexCreateAction()();
}
catch (Exception e)
{
Debug.WriteLine("Vertex creation failed: " + e.Message);
Debug.WriteLine("The caller will need to sideload the vertex.");
}
return(vertex);
}
internal void ActivateVertexOnInstance(string vertexName, string instanceName)
{
var newActiveVertex = VertexTable.GetAll(_vertexTable)
.Where(gn => instanceName == gn.InstanceName && vertexName == gn.VertexName)
.First();
newActiveVertex.IsActive = true;
TableOperation insertOperation = TableOperation.InsertOrReplace(newActiveVertex);
_vertexTable.ExecuteAsync(insertOperation).Wait();
var procs = VertexTable.GetAll(_vertexTable)
.Where(gn => vertexName == gn.VertexName && instanceName != gn.InstanceName);
foreach (var proc in procs)
{
if (proc.IsActive)
{
proc.IsActive = false;
TableOperation _insertOperation = TableOperation.InsertOrReplace(proc);
_vertexTable.ExecuteAsync(_insertOperation).Wait();
}
}
}
internal CRAErrorCode Connect_InitiatorSide(string fromVertexName, string fromVertexOutput, string toVertexName, string toVertexInput, bool reverse, bool killIfExists = true, bool killRemote = true)
{
VertexTable row;
try
{
// Need to get the latest address & port
row = reverse ? VertexTable.GetRowForVertex(_workerInstanceTable, fromVertexName)
: VertexTable.GetRowForVertex(_workerInstanceTable, toVertexName);
}
catch
{
return(CRAErrorCode.ActiveVertexNotFound);
}
// If from and to vertices are on the same (this) instance,
// we can convert a "reverse" connection into a normal connection
if (reverse && (row.InstanceName == InstanceName))
{
reverse = false;
}
CancellationTokenSource oldSource;
var conn = reverse ? inConnections : outConnections;
if (conn.TryGetValue(fromVertexName + ":" + fromVertexOutput + ":" + toVertexName + ":" + toVertexInput,
out oldSource))
{
if (killIfExists)
{
Debug.WriteLine("Deleting prior connection - it will automatically reconnect");
oldSource.Cancel();
}
return(CRAErrorCode.Success);
}
if (TryFusedConnect(row.InstanceName, fromVertexName, fromVertexOutput, toVertexName, toVertexInput))
{
return(CRAErrorCode.Success);
}
// Re-check the connection table as someone may have successfully
// created a fused connection
if (conn.TryGetValue(fromVertexName + ":" + fromVertexOutput + ":" + toVertexName + ":" + toVertexInput,
out oldSource))
{
if (killIfExists)
{
Debug.WriteLine("Deleting prior connection - it will automatically reconnect");
oldSource.Cancel();
}
return(CRAErrorCode.Success);
}
// Send request to CRA instance
Stream ns = null;
var _row = VertexTable.GetRowForInstanceVertex(_workerInstanceTable, row.InstanceName, "");
try
{
// Get a stream connection from the pool if available
if (!_craClient.TryGetSenderStreamFromPool(_row.Address, _row.Port.ToString(), out ns))
{
TcpClient client = new TcpClient(_row.Address, _row.Port);
client.NoDelay = true;
ns = _craClient.SecureStreamConnectionDescriptor
.CreateSecureClient(client.GetStream(), row.InstanceName);
}
}
catch
{
return(CRAErrorCode.ConnectionEstablishFailed);
}
if (!reverse)
{
ns.WriteInt32((int)CRATaskMessageType.CONNECT_VERTEX_RECEIVER);
}
else
{
ns.WriteInt32((int)CRATaskMessageType.CONNECT_VERTEX_RECEIVER_REVERSE);
}
ns.WriteByteArray(Encoding.UTF8.GetBytes(fromVertexName));
ns.WriteByteArray(Encoding.UTF8.GetBytes(fromVertexOutput));
ns.WriteByteArray(Encoding.UTF8.GetBytes(toVertexName));
ns.WriteByteArray(Encoding.UTF8.GetBytes(toVertexInput));
ns.WriteInt32(killRemote ? 1 : 0);
CRAErrorCode result = (CRAErrorCode)ns.ReadInt32();
if (result != 0)
{
Debug.WriteLine("Error occurs while establishing the connection!!");
return(result);
}
else
{
CancellationTokenSource source = new CancellationTokenSource();
if (!reverse)
{
if (outConnections.TryAdd(fromVertexName + ":" + fromVertexOutput + ":" + toVertexName + ":" + toVertexInput, source))
{
Task.Run(() =>
EgressToStream(fromVertexName, fromVertexOutput, toVertexName, toVertexInput, reverse, ns, source, _row.Address, _row.Port));
return(CRAErrorCode.Success);
}
else
{
source.Dispose();
ns.Close();
Console.WriteLine("Race adding connection - deleting outgoing stream");
return(CRAErrorCode.ConnectionAdditionRace);
}
}
else
{
if (inConnections.TryAdd(fromVertexName + ":" + fromVertexOutput + ":" + toVertexName + ":" + toVertexInput, source))
{
Task.Run(() =>
IngressFromStream(fromVertexName, fromVertexOutput, toVertexName, toVertexInput, reverse, ns, source, _row.Address, _row.Port));
return(CRAErrorCode.Success);
}
else
{
source.Dispose();
ns.Close();
Debug.WriteLine("Race adding connection - deleting outgoing stream");
return(CRAErrorCode.ConnectionAdditionRace);
}
}
}
}
/// <summary>
/// Equals
/// </summary>
/// <param name="obj"></param>
/// <returns></returns>
public override bool Equals(object obj)
{
VertexTable other = obj as VertexTable;
return(this.PartitionKey.Equals(other.PartitionKey) && this.RowKey.Equals(other.RowKey));
}
/// <summary>
/// Connect one CRA vertex to another, via pre-defined endpoints. We contact the "from" vertex
/// to initiate the creation of the link.
/// </summary>
/// <param name="fromVertexName">Name of the vertex from which connection is being made</param>
/// <param name="fromEndpoint">Name of the endpoint on the fromVertex, from which connection is being made</param>
/// <param name="toVertexName">Name of the vertex to which connection is being made</param>
/// <param name="toEndpoint">Name of the endpoint on the toVertex, to which connection is being made</param>
/// <param name="direction">Which vertex initiates the connection</param>
/// <returns>Status of the Connect operation</returns>
public CRAErrorCode Connect(string fromVertexName, string fromEndpoint, string toVertexName, string toEndpoint, ConnectionInitiator direction)
{
// Tell from vertex to establish connection
// Send request to CRA instance
// Check that vertex and endpoints are valid and existing
if (!_vertexTableManager.ExistsVertex(fromVertexName) || !_vertexTableManager.ExistsVertex(toVertexName))
{
// Check for sharded vertices
List <int> fromVertexShards, toVertexShards;
if (!_vertexTableManager.ExistsShardedVertex(fromVertexName, out fromVertexShards))
{
return(CRAErrorCode.VertexNotFound);
}
if (!_vertexTableManager.ExistsShardedVertex(toVertexName, out toVertexShards))
{
return(CRAErrorCode.VertexNotFound);
}
return(ConnectSharded(fromVertexName, fromVertexShards, fromEndpoint, toVertexName, toVertexShards, toEndpoint, direction));
}
// Make the connection information stable
_connectionTableManager.AddConnection(fromVertexName, fromEndpoint, toVertexName, toEndpoint);
// We now try best-effort to tell the CRA instance of this connection
CRAErrorCode result = CRAErrorCode.Success;
VertexTable _row;
try
{
// Get instance for source vertex
_row = direction == ConnectionInitiator.FromSide ?
VertexTable.GetRowForVertex(_vertexTable, fromVertexName) :
VertexTable.GetRowForVertex(_vertexTable, toVertexName);
}
catch
{
Console.WriteLine("Unable to find active instance with vertex. On vertex activation, the connection should be completed automatically.");
return(result);
}
try
{
if (_localWorker != null)
{
if (_localWorker.InstanceName == _row.InstanceName)
{
return(_localWorker.Connect_InitiatorSide(fromVertexName, fromEndpoint,
toVertexName, toEndpoint, direction == ConnectionInitiator.ToSide));
}
}
// Send request to CRA instance
TcpClient client = null;
// Get address and port for instance, using row with vertex = ""
var row = VertexTable.GetRowForInstance(_vertexTable, _row.InstanceName);
// Get a stream connection from the pool if available
Stream stream;
if (!TryGetSenderStreamFromPool(row.Address, row.Port.ToString(), out stream))
{
client = new TcpClient(row.Address, row.Port);
client.NoDelay = true;
stream = client.GetStream();
if (SecureStreamConnectionDescriptor != null)
{
stream = SecureStreamConnectionDescriptor.CreateSecureClient(stream, _row.InstanceName);
}
}
if (direction == ConnectionInitiator.FromSide)
{
stream.WriteInt32((int)CRATaskMessageType.CONNECT_VERTEX_INITIATOR);
}
else
{
stream.WriteInt32((int)CRATaskMessageType.CONNECT_VERTEX_INITIATOR_REVERSE);
}
stream.WriteByteArray(Encoding.UTF8.GetBytes(fromVertexName));
stream.WriteByteArray(Encoding.UTF8.GetBytes(fromEndpoint));
stream.WriteByteArray(Encoding.UTF8.GetBytes(toVertexName));
stream.WriteByteArray(Encoding.UTF8.GetBytes(toEndpoint));
result = (CRAErrorCode)stream.ReadInt32();
if (result != 0)
{
Console.WriteLine("Connection was logically established. However, the client received an error code from the connection-initiating CRA instance: " + result);
}
else
{
// Add/Return a stream connection to the pool
TryAddSenderStreamToPool(row.Address, row.Port.ToString(), stream);
}
}
catch
{
Console.WriteLine("The connection-initiating CRA instance appears to be down or could not be found. Restart it and this connection will be completed automatically");
}
return(result);
}
private async void RestoreVertexAndConnections(VertexTable _row)
{
await _craClient.LoadVertexAsync(_row.VertexName, _row.VertexDefinition, _row.VertexParameter, _workerinstanceName, _localVertexTable);
RestoreConnections(_row);
}
internal VertexTable GetRowForInstanceVertex(string instanceName, string vertexName)
{
return(VertexTable.GetAll(_vertexTable).Where(gn => instanceName == gn.InstanceName && vertexName == gn.VertexName).First());
}
internal VertexTable GetRowForDefaultInstance()
{
return(VertexTable.GetAll(_vertexTable).Where(gn => string.IsNullOrEmpty(gn.VertexName)).First());
}
internal CRAErrorCode InstantiateVertex(string instanceName, string vertexName, string vertexDefinition, object vertexParameter, bool sharded)
{
var procDefRow = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition);
string blobName = vertexName + "-" + instanceName;
// Serialize and write the vertex parameters to a blob
CloudBlobContainer container = _blobClient.GetContainerReference("cra");
container.CreateIfNotExistsAsync().Wait();
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/" + blobName);
CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult();
byte[] parameterBytes = Encoding.UTF8.GetBytes(
SerializationHelper.SerializeObject(vertexParameter));
blobStream.WriteByteArray(parameterBytes);
blobStream.Close();
// Add metadata
var newRow = new VertexTable(instanceName, vertexName, vertexDefinition, "", 0,
procDefRow.VertexCreateAction,
blobName,
false, sharded);
TableOperation insertOperation = TableOperation.InsertOrReplace(newRow);
_vertexTable.ExecuteAsync(insertOperation).Wait();
CRAErrorCode result = CRAErrorCode.Success;
// Send request to CRA instance
VertexTable instanceRow;
try
{
instanceRow = VertexTable.GetRowForInstance(_vertexTable, instanceName);
// Get a stream connection from the pool if available
Stream stream;
if (!TryGetSenderStreamFromPool(instanceRow.Address, instanceRow.Port.ToString(), out stream))
{
TcpClient client = new TcpClient(instanceRow.Address, instanceRow.Port);
client.NoDelay = true;
stream = client.GetStream();
if (SecureStreamConnectionDescriptor != null)
{
stream = SecureStreamConnectionDescriptor.CreateSecureClient(stream, instanceName);
}
}
stream.WriteInt32((int)CRATaskMessageType.LOAD_VERTEX);
stream.WriteByteArray(Encoding.UTF8.GetBytes(vertexName));
stream.WriteByteArray(Encoding.UTF8.GetBytes(vertexDefinition));
stream.WriteByteArray(Encoding.UTF8.GetBytes(newRow.VertexParameter));
result = (CRAErrorCode)stream.ReadInt32();
if (result != 0)
{
Console.WriteLine("Vertex was logically loaded. However, we received an error code from the hosting CRA instance: " + result);
}
// Add/Return stream connection to the pool
TryAddSenderStreamToPool(instanceRow.Address, instanceRow.Port.ToString(), stream);
}
catch
{
Console.WriteLine("The CRA instance appears to be down. Restart it and this vertex will be instantiated automatically");
}
return(result);
}
/// <summary>
/// Load a vertex on the local instance
/// </summary>
/// <param name="vertexName"></param>
/// <param name="vertexDefinition"></param>
/// <param name="vertexParameter"></param>
/// <param name="instanceName"></param>
/// <param name="table"></param>
/// <returns></returns>
public async Task <IVertex> LoadVertexAsync(string vertexName, string vertexDefinition, string vertexParameter, string instanceName, ConcurrentDictionary <string, IVertex> table)
{
// Deactivate vertex
_vertexTableManager.DeactivateVertexOnInstance(vertexName, instanceName);
CloudBlobContainer container = _blobClient.GetContainerReference("cra");
container.CreateIfNotExistsAsync().Wait();
var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/binaries");
Stream blobStream = blockBlob.OpenReadAsync().GetAwaiter().GetResult();
AssemblyUtils.LoadAssembliesFromStream(blobStream);
blobStream.Close();
var row = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition);
// CREATE THE VERTEX
var vertex = row.GetVertexCreateAction()();
// INITIALIZE
if ((VertexBase)vertex != null)
{
((VertexBase)vertex).VertexName = vertexName;
((VertexBase)vertex).ClientLibrary = this;
}
// LATCH CALLBACKS TO POPULATE ENDPOINT TABLE
vertex.OnAddInputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, true, false));
vertex.OnAddOutputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, false, false));
vertex.OnAddAsyncInputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, true, true));
vertex.OnAddAsyncOutputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, false, true));
//ADD TO TABLE
if (table != null)
{
table.AddOrUpdate(vertexName, vertex, (procName, oldProc) => { oldProc.Dispose(); return(vertex); });
vertex.OnDispose(() =>
{
// Delete all endpoints of the vertex
foreach (var key in vertex.InputEndpoints)
{
_endpointTableManager.DeleteEndpoint(vertexName, key.Key);
}
foreach (var key in vertex.AsyncInputEndpoints)
{
_endpointTableManager.DeleteEndpoint(vertexName, key.Key);
}
foreach (var key in vertex.OutputEndpoints)
{
_endpointTableManager.DeleteEndpoint(vertexName, key.Key);
}
foreach (var key in vertex.AsyncOutputEndpoints)
{
_endpointTableManager.DeleteEndpoint(vertexName, key.Key);
}
IVertex old;
if (!table.TryRemove(vertexName, out old))
{
Console.WriteLine("Unable to remove vertex on disposal");
}
var entity = new DynamicTableEntity(instanceName, vertexName);
entity.ETag = "*";
TableOperation deleteOperation = TableOperation.Delete(entity);
_vertexTable.ExecuteAsync(deleteOperation).Wait();
});
}
string blobName = vertexName + "-" + instanceName;
var parametersBlob = container.GetBlockBlobReference(vertexDefinition + "/" + blobName);
Stream parametersStream = parametersBlob.OpenReadAsync().GetAwaiter().GetResult();
byte[] parametersBytes = parametersStream.ReadByteArray();
string parameterString = Encoding.UTF8.GetString(parametersBytes);
parametersStream.Close();
var par = SerializationHelper.DeserializeObject(parameterString);
vertex.Initialize(par);
await vertex.InitializeAsync(par);
// Activate vertex
ActivateVertex(vertexName, instanceName);
return(vertex);
}
internal static bool ContainsRow(CloudTable instanceTable, VertexTable entity)
{
var temp = GetAll(instanceTable);
return(temp.Where(gn => entity.Equals(gn)).Count() > 0);
}
