public IVertex CreateVertex(string vertexDefinition, CloudBlobContainer container) { if (_dynamicLoadingEnabled) { var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/binaries"); Stream blobStream = blockBlob.OpenReadAsync().GetAwaiter().GetResult(); try { AssemblyUtils.LoadAssembliesFromStream(blobStream); } catch (FileLoadException e) { Debug.WriteLine("Ignoring exception from assembly loading: " + e.Message); Debug.WriteLine("If vertex creation fails, the caller will need to sideload the vertex."); } blobStream.Close(); } else { Debug.WriteLine("Dynamic assembly loading is disabled. The caller will need to sideload the vertex."); } var row = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition); // CREATE THE VERTEX IVertex vertex = null; try { vertex = row.GetVertexCreateAction()(); } catch (Exception e) { Debug.WriteLine("Vertex creation failed: " + e.Message); Debug.WriteLine("The caller will need to sideload the vertex."); } return(vertex); }
internal CRAErrorCode InstantiateVertex(string instanceName, string vertexName, string vertexDefinition, object vertexParameter, bool sharded) { var procDefRow = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition); string blobName = vertexName + "-" + instanceName; // Serialize and write the vertex parameters to a blob CloudBlobContainer container = _blobClient.GetContainerReference("cra"); container.CreateIfNotExistsAsync().Wait(); var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/" + blobName); CloudBlobStream blobStream = blockBlob.OpenWriteAsync().GetAwaiter().GetResult(); byte[] parameterBytes = Encoding.UTF8.GetBytes( SerializationHelper.SerializeObject(vertexParameter)); blobStream.WriteByteArray(parameterBytes); blobStream.Close(); // Add metadata var newRow = new VertexTable(instanceName, vertexName, vertexDefinition, "", 0, procDefRow.VertexCreateAction, blobName, false, sharded); TableOperation insertOperation = TableOperation.InsertOrReplace(newRow); _vertexTable.ExecuteAsync(insertOperation).Wait(); CRAErrorCode result = CRAErrorCode.Success; // Send request to CRA instance VertexTable instanceRow; try { instanceRow = VertexTable.GetRowForInstance(_vertexTable, instanceName); // Get a stream connection from the pool if available Stream stream; if (!TryGetSenderStreamFromPool(instanceRow.Address, instanceRow.Port.ToString(), out stream)) { TcpClient client = new TcpClient(instanceRow.Address, instanceRow.Port); client.NoDelay = true; stream = client.GetStream(); if (SecureStreamConnectionDescriptor != null) { stream = SecureStreamConnectionDescriptor.CreateSecureClient(stream, instanceName); } } stream.WriteInt32((int)CRATaskMessageType.LOAD_VERTEX); stream.WriteByteArray(Encoding.UTF8.GetBytes(vertexName)); stream.WriteByteArray(Encoding.UTF8.GetBytes(vertexDefinition)); stream.WriteByteArray(Encoding.UTF8.GetBytes(newRow.VertexParameter)); result = (CRAErrorCode)stream.ReadInt32(); if (result != 0) { Console.WriteLine("Vertex was logically loaded. However, we received an error code from the hosting CRA instance: " + result); } // Add/Return stream connection to the pool TryAddSenderStreamToPool(instanceRow.Address, instanceRow.Port.ToString(), stream); } catch { Console.WriteLine("The CRA instance appears to be down. Restart it and this vertex will be instantiated automatically"); } return(result); }
/// <summary> /// Load a vertex on the local instance /// </summary> /// <param name="vertexName"></param> /// <param name="vertexDefinition"></param> /// <param name="vertexParameter"></param> /// <param name="instanceName"></param> /// <param name="table"></param> /// <returns></returns> public async Task <IVertex> LoadVertexAsync(string vertexName, string vertexDefinition, string vertexParameter, string instanceName, ConcurrentDictionary <string, IVertex> table) { // Deactivate vertex _vertexTableManager.DeactivateVertexOnInstance(vertexName, instanceName); CloudBlobContainer container = _blobClient.GetContainerReference("cra"); container.CreateIfNotExistsAsync().Wait(); var blockBlob = container.GetBlockBlobReference(vertexDefinition + "/binaries"); Stream blobStream = blockBlob.OpenReadAsync().GetAwaiter().GetResult(); AssemblyUtils.LoadAssembliesFromStream(blobStream); blobStream.Close(); var row = VertexTable.GetRowForVertexDefinition(_vertexTable, vertexDefinition); // CREATE THE VERTEX var vertex = row.GetVertexCreateAction()(); // INITIALIZE if ((VertexBase)vertex != null) { ((VertexBase)vertex).VertexName = vertexName; ((VertexBase)vertex).ClientLibrary = this; } // LATCH CALLBACKS TO POPULATE ENDPOINT TABLE vertex.OnAddInputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, true, false)); vertex.OnAddOutputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, false, false)); vertex.OnAddAsyncInputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, true, true)); vertex.OnAddAsyncOutputEndpoint((name, endpt) => _endpointTableManager.AddEndpoint(vertexName, name, false, true)); //ADD TO TABLE if (table != null) { table.AddOrUpdate(vertexName, vertex, (procName, oldProc) => { oldProc.Dispose(); return(vertex); }); vertex.OnDispose(() => { // Delete all endpoints of the vertex foreach (var key in vertex.InputEndpoints) { _endpointTableManager.DeleteEndpoint(vertexName, key.Key); } foreach (var key in vertex.AsyncInputEndpoints) { _endpointTableManager.DeleteEndpoint(vertexName, key.Key); } foreach (var key in vertex.OutputEndpoints) { _endpointTableManager.DeleteEndpoint(vertexName, key.Key); } foreach (var key in vertex.AsyncOutputEndpoints) { _endpointTableManager.DeleteEndpoint(vertexName, key.Key); } IVertex old; if (!table.TryRemove(vertexName, out old)) { Console.WriteLine("Unable to remove vertex on disposal"); } var entity = new DynamicTableEntity(instanceName, vertexName); entity.ETag = "*"; TableOperation deleteOperation = TableOperation.Delete(entity); _vertexTable.ExecuteAsync(deleteOperation).Wait(); }); } string blobName = vertexName + "-" + instanceName; var parametersBlob = container.GetBlockBlobReference(vertexDefinition + "/" + blobName); Stream parametersStream = parametersBlob.OpenReadAsync().GetAwaiter().GetResult(); byte[] parametersBytes = parametersStream.ReadByteArray(); string parameterString = Encoding.UTF8.GetString(parametersBytes); parametersStream.Close(); var par = SerializationHelper.DeserializeObject(parameterString); vertex.Initialize(par); await vertex.InitializeAsync(par); // Activate vertex ActivateVertex(vertexName, instanceName); return(vertex); }