public static async Task <bool> DeploySubscribeTask(CRAClientLibrary client, SubscribeTask task, OperatorsToplogy topology)
{
try
{
if (!_isSubscribeOperatorDefined)
{
await client.DefineVertexAsync(typeof(ShardedSubscribeOperator).Name.ToLower(), () => new ShardedSubscribeOperator());
_isSubscribeOperatorDefined = true;
}
var status = await client.InstantiateVertexAsync(CreateInstancesNames(task.DeployDescriptor.InstancesMap()), task.OutputId, typeof(ShardedSubscribeOperator).Name.ToLower(), task, 1);
if (status == CRAErrorCode.Success)
{
foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys)
{
var fromToConnection = task.VerticesConnectionsMap[fromInputId + task.OutputId][0];
await client.ConnectAsync(fromToConnection.FromVertex, fromToConnection.FromEndpoint, fromToConnection.ToVertex, fromToConnection.ToEndpoint);
}
return(true);
}
else
{
return(false);
}
}
catch (Exception e)
{
Console.WriteLine("Error in deploying a sharded CRA Subscribe task. Please, double check your task configurations: " + e.ToString());
return(false);
}
}
static void Main(string[] args)
{
var client = new CRAClientLibrary(new FileDataProvider());
int chunkSize = 10 * 1024 * 1024;
Console.ReadLine();
client.DefineVertexAsync("bandwidthtestvertex", () => new BandwidthTestVertex()).Wait();
client.InstantiateVertexAsync("crainst01", "bwvertex1", "bandwidthtestvertex", chunkSize).Wait();
client.InstantiateVertexAsync("crainst02", "bwvertex2", "bandwidthtestvertex", chunkSize).Wait();
// client.InstantiateVertexAsync("crainst03", "bwvertex3", "bandwidthtestvertex", chunkSize).Wait();
client.ConnectAsync("bwvertex1", "output1", "bwvertex2", "input1").Wait();
client.ConnectAsync("bwvertex1", "output2", "bwvertex2", "input2").Wait();
client.ConnectAsync("bwvertex1", "output3", "bwvertex2", "input3").Wait();
client.ConnectAsync("bwvertex1", "output4", "bwvertex2", "input4").Wait();
client.ConnectAsync("bwvertex1", "output5", "bwvertex2", "input5").Wait();
client.ConnectAsync("bwvertex1", "output6", "bwvertex2", "input6").Wait();
client.ConnectAsync("bwvertex1", "output7", "bwvertex2", "input7").Wait();
client.ConnectAsync("bwvertex1", "output8", "bwvertex2", "input8").Wait();
client.ConnectAsync("bwvertex2", "output9", "bwvertex1", "input9").Wait();
client.ConnectAsync("bwvertex2", "output10", "bwvertex1", "input10").Wait();
client.ConnectAsync("bwvertex2", "output11", "bwvertex1", "input11").Wait();
client.ConnectAsync("bwvertex2", "output12", "bwvertex1", "input12").Wait();
client.ConnectAsync("bwvertex2", "output13", "bwvertex1", "input13").Wait();
client.ConnectAsync("bwvertex2", "output14", "bwvertex1", "input14").Wait();
client.ConnectAsync("bwvertex2", "output15", "bwvertex1", "input15").Wait();
client.ConnectAsync("bwvertex2", "output16", "bwvertex1", "input16").Wait();
Console.ReadLine();
client.ResetAsync().Wait();
}
public static bool DeployClientTerminal(CRAClientLibrary client, ClientTerminalTask task,
ref DetachedVertex clientTerminal, OperatorsToplogy topology)
{
try
{
foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys)
{
string[] inputEndpoints = OperatorUtils.PrepareInputEndpointsIdsForOperator(fromInputId, task.EndpointsDescriptor);
string[] outputEndpoints = OperatorUtils.PrepareOutputEndpointsIdsForOperator(
task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]);
int shardsCount = client.CountVertexShards(task.DeployDescriptor.InstancesMap());
for (int i = 0; i < shardsCount; i++)
{
for (int j = 0; j < inputEndpoints.Length; j++)
{
clientTerminal.FromRemoteOutputEndpointStream(inputEndpoints[j] + i, fromInputId + "$" + i, outputEndpoints[j]);
}
}
}
return(true);
}
catch (Exception e)
{
Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString());
return(false);
}
}
private static bool DeployShuffleReduceTask(CRAClientLibrary client, ShuffleTask task, OperatorsToplogy topology)
{
try
{
client.DefineVertex(typeof(ShuffleOperator).Name.ToLower(), () => new ShuffleOperator());
CRAErrorCode status = client.InstantiateShardedVertex(task.ReducerVertexName, typeof(ShuffleOperator).Name.ToLower(),
task, task.DeployDescriptor.InstancesMap());
if (status == CRAErrorCode.Success)
{
foreach (string fromInputId in task.SecondaryEndpointsDescriptor.FromInputs.Keys)
{
client.ConnectShardedVertices(task.VerticesConnectionsMap[fromInputId + task.OutputId]);
}
var fromInput = task.EndpointsDescriptor.FromInputs.First().Key;
var newFromInputId = fromInput.Substring(0, fromInput.Length - 1);
client.ConnectShardedVertices(task.VerticesConnectionsMap[newFromInputId + task.ReducerVertexName]);
return(true);
}
else
{
return(false);
}
}
catch (Exception)
{
Console.WriteLine("Error in deploying a sharded CRA shuffle mapper task. Please, double check your task configurations");
return(false);
}
}
public static bool DeploySubscribeTask(CRAClientLibrary client, SubscribeTask task, OperatorsToplogy topology)
{
try
{
client.DefineVertex(typeof(SubscribeOperator).Name.ToLower(), () => new SubscribeOperator());
CRAErrorCode status = client.InstantiateShardedVertex(task.OutputId, typeof(SubscribeOperator).Name.ToLower(),
task, task.DeployDescriptor.InstancesMap());
if (status == CRAErrorCode.Success)
{
foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys)
{
client.ConnectShardedVertices(task.VerticesConnectionsMap[fromInputId + task.OutputId]);
}
return(true);
}
else
{
return(false);
}
}
catch (Exception e)
{
Console.WriteLine("Error in deploying a sharded CRA Subscribe task. Please, double check your task configurations: " + e.ToString());
return(false);
}
}
static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("UnsafeDeregisterInstance InstanceName");
Console.WriteLine("WARNING: This is a metadata hacking tool that should NEVER be used on a real deployment");
Console.WriteLine("This tool is a convenience for developers who want to more easily test certain application modfications");
Console.WriteLine("Usage: UnsafeDeregisterInstance InstanceName");
return;
}
var client = new CRAClientLibrary(Environment.GetEnvironmentVariable("AZURE_STORAGE_CONN_STRING"));
var serviceName = args[0];
foreach (var endpt in client.GetInputEndpoints(serviceName))
{
client.DeleteEndpoint(serviceName, endpt);
}
foreach (var endpt in client.GetOutputEndpoints(serviceName))
{
client.DeleteEndpoint(serviceName, endpt);
}
foreach (var conn in client.GetConnectionsFromVertex(serviceName))
{
client.DeleteConnectionInfo(conn);
}
foreach (var conn in client.GetConnectionsToVertex(serviceName))
{
client.DeleteConnectionInfo(conn);
}
try
{
client.DeleteVertex(serviceName);
}
catch { }
}
private static ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > PrepareVerticesConnectionsMap(
CRAClientLibrary client, TaskBase[] tasks, string[] tasksIds, Dictionary <string, TaskBase> tasksDictionary, OperatorsToplogy topology)
{
ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > verticesConnectionsMap = new ConcurrentDictionary <string, List <ConnectionInfoWithLocality> >();
for (int i = 0; i < tasks.Length; i++)
{
int shardsCount = client.CountVertexShards(tasks[i].DeployDescriptor.InstancesMap());
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
foreach (string fromInputId in tasks[i].EndpointsDescriptor.FromInputs.Keys)
{
var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount,
fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(),
tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap(), false);
verticesConnectionsMap.AddOrUpdate(fromInputId + tasks[i].OutputId,
flatFromToConnections, (key, value) => flatFromToConnections);
}
foreach (string secondaryFromInputId in tasks[i].EndpointsDescriptor.SecondaryFromInputs.Keys)
{
var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount,
secondaryFromInputId, topology.OperatorsEndpointsDescriptors[secondaryFromInputId], tasksDictionary[secondaryFromInputId].DeployDescriptor.InstancesMap(),
tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap(), true);
verticesConnectionsMap.AddOrUpdate(secondaryFromInputId + tasks[i].OutputId,
flatFromToConnections, (key, value) => flatFromToConnections);
}
}
return(verticesConnectionsMap);
}
public static async Task <bool> DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
{
topology.PrepareFinalOperatorsTasks();
string[] tasksIds = topology.OperatorsIds.ToArray();
TaskBase[] tasks = topology.OperatorsTasks.ToArray();
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Move)
{
ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor();
shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
}
}
var tasksDictionary = PrepareTasksDictionary(tasks);
var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);
var tasksDeploymentStatus = new Dictionary <string, bool>();
for (int i = 0; i < tasks.Length; i++)
{
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
tasks[i].VerticesConnectionsMap = connectionsMap;
tasksDeploymentStatus.Add(tasksIds[i], false);
}
bool isSuccessful = true;
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Produce && tasks[i].EndpointsDescriptor.FromInputs.Count == 0 &&
tasks[i].EndpointsDescriptor.SecondaryFromInputs.Count == 0)
{
isSuccessful = await DeployProduceTask(client, (ProduceTask)tasks[i], topology);
if (isSuccessful)
{
tasksDeploymentStatus[tasksIds[i]] = true;
}
}
}
for (int i = 0; i < tasks.Length; i++)
{
isSuccessful = isSuccessful & await DeployTask(i, tasks, tasksIds, tasksDeploymentStatus, client, topology);
}
return(isSuccessful);
}
public static bool DeployOperators(CRAClientLibrary client, OperatorsToplogy topology)
{
topology.PrepareFinalOperatorsTasks();
string[] tasksIds = topology.OperatorsIds.ToArray();
TaskBase[] tasks = topology.OperatorsTasks.ToArray();
for (int i = 0; i < tasks.Length; i++)
{
if (tasks[i].OperationType == OperatorType.Move)
{
ShuffleTask shuffleTask = (ShuffleTask)tasks[i];
shuffleTask.SecondaryEndpointsDescriptor = new OperatorEndpointsDescriptor();
shuffleTask.SecondaryEndpointsDescriptor.FromInputs = topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs;
topology.OperatorsEndpointsDescriptors[shuffleTask.ReducerVertexName].FromInputs = new ConcurrentDictionary <string, int>();
int shardsCount = client.CountVertexShards(shuffleTask.DeployDescriptor.InstancesMap());
topology.UpdateShuffleInputs(shuffleTask.MapperVertexName, shuffleTask.ReducerVertexName, shardsCount);
topology.UpdateShuffleOperatorTask(shuffleTask.ReducerVertexName, shuffleTask);
}
}
var tasksDictionary = PrepareTasksDictionary(tasks);
var connectionsMap = PrepareVerticesConnectionsMap(client, tasks, tasksIds, tasksDictionary, topology);
bool isSuccessful = true;
for (int i = 0; i < tasks.Length; i++)
{
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
tasks[i].VerticesConnectionsMap = connectionsMap;
if (tasks[i].OperationType == OperatorType.Produce)
{
isSuccessful = DeployProduceTask(client, (ProduceTask)tasks[i]);
}
else if (tasks[i].OperationType == OperatorType.Subscribe)
{
isSuccessful = DeploySubscribeTask(client, (SubscribeTask)tasks[i], topology);
}
else if (tasks[i].OperationType == OperatorType.Move)
{
isSuccessful = DeployShuffleReduceTask(client, (ShuffleTask)tasks[i], topology);
}
if (!isSuccessful)
{
break;
}
}
return(isSuccessful);
}
static void Main(string[] args)
{
var client = new CRAClientLibrary();
client.ResetAsync().Wait();
client.DefineVertexAsync("fusableconnectionpairvertex", () => new FusableConnectionPairVertex()).Wait();
client.InstantiateVertexAsync("crainst01", "fvertex1", "fusableconnectionpairvertex", null).Wait();
client.InstantiateVertexAsync("crainst01", "fvertex2", "fusableconnectionpairvertex", null).Wait();
client.ConnectAsync("fvertex1", "output", "fvertex2", "input").Wait();
Console.ReadLine();
}
public static async Task <bool> DeployClientTerminal(
CRAClientLibrary client,
string workerName,
ClientTerminalTask task,
OperatorsToplogy topology)
{
try
{
bool result = true;
client.DisableArtifactUploading();
if (!_isSubscribeClientOperatorDefined)
{
await client.DefineVertexAsync(typeof(ShardedSubscribeClientOperator).Name.ToLower(), () => new ShardedSubscribeClientOperator());
_isSubscribeClientOperatorDefined = true;
}
var status = await client.InstantiateVertexAsync(new string[] { workerName }, task.OutputId, typeof(ShardedSubscribeClientOperator).Name.ToLower(), task, 1);
if (status == CRAErrorCode.Success)
{
foreach (string fromInputId in task.EndpointsDescriptor.FromInputs.Keys)
{
string outputEndpoint = OperatorUtils.PrepareOutputEndpointIdForOperator(
task.OutputId, topology.OperatorsEndpointsDescriptors[fromInputId]);
string inputEndpoint = OperatorUtils.PrepareInputEndpointIdForOperator(fromInputId, task.EndpointsDescriptor, false);
await client.ConnectAsync(fromInputId, outputEndpoint, task.OutputId, inputEndpoint);
}
result = true;
}
else
{
result = false;
}
client.EnableArtifactUploading();
return(result);
}
catch (Exception e)
{
Console.WriteLine("Error in deploying a CRA client terminal vertex. Please, double check your task configurations: " + e.ToString());
return(false);
}
}
public ClientSideShardedDataset(
IDataProvider dataProvider,
Expression <Func <int, TDataset> > producer)
: base(dataProvider)
{
if (producer != null)
{
_producer = new ClosureEliminator().Visit(producer) as Expression <Func <int, TDataset> >;
}
else
{
Console.WriteLine("The producer expression of the ShardedDataset should be provided !!");
}
_dataProvider = dataProvider;
_craClient = new CRAClientLibrary(_dataProvider);
}
public override IShardedDataset <TKey, TPayload, TDataset> Deploy()
{
if (!_isDeployed)
{
OperatorsToplogy operatorsTopology = OperatorsToplogy.GetInstance();
GenerateProduceTask(ref operatorsTopology);
_craClient = new CRAClientLibrary();
_isDeployed = DeploymentUtils.DeployOperators(_craClient, operatorsTopology);
if (!_isDeployed)
{
return(null);
}
}
return(this);
}
private static ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > PrepareVerticesConnectionsMap(
CRAClientLibrary client, TaskBase[] tasks, string[] tasksIds, Dictionary <string, TaskBase> tasksDictionary, OperatorsToplogy topology)
{
ConcurrentDictionary <string, List <ConnectionInfoWithLocality> > verticesConnectionsMap = new ConcurrentDictionary <string, List <ConnectionInfoWithLocality> >();
for (int i = 0; i < tasks.Length; i++)
{
int shardsCount = client.CountVertexShards(tasks[i].DeployDescriptor.InstancesMap());
tasks[i].EndpointsDescriptor = topology.OperatorsEndpointsDescriptors[tasksIds[i]];
if (tasks[i].OperationType == OperatorType.Move)
{
var shuffleTask = (ShuffleTask)tasks[i];
foreach (string fromInputId in shuffleTask.SecondaryEndpointsDescriptor.FromInputs.Keys)
{
var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount,
fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(),
shuffleTask.ReducerVertexName, shuffleTask.EndpointsDescriptor, shuffleTask.DeployDescriptor.InstancesMap());
verticesConnectionsMap.AddOrUpdate(fromInputId + shuffleTask.OutputId,
flatFromToConnections, (key, value) => flatFromToConnections);
}
var fromInput = shuffleTask.EndpointsDescriptor.FromInputs.First().Key;
var newFromInputId = fromInput.Substring(0, fromInput.Length - 1);
var shuffleFromToConnections = PrepareShuffleConnectionsMap(client, shardsCount,
newFromInputId, topology.OperatorsEndpointsDescriptors[newFromInputId], tasksDictionary[newFromInputId].DeployDescriptor.InstancesMap(),
shuffleTask.ReducerVertexName, shuffleTask.EndpointsDescriptor, shuffleTask.DeployDescriptor.InstancesMap());
verticesConnectionsMap.AddOrUpdate(newFromInputId + shuffleTask.ReducerVertexName,
shuffleFromToConnections, (key, value) => shuffleFromToConnections);
}
else
{
foreach (string fromInputId in tasks[i].EndpointsDescriptor.FromInputs.Keys)
{
var flatFromToConnections = PrepareFlatConnectionsMap(client, shardsCount,
fromInputId, topology.OperatorsEndpointsDescriptors[fromInputId], tasksDictionary[fromInputId].DeployDescriptor.InstancesMap(),
tasks[i].OutputId, tasks[i].EndpointsDescriptor, tasks[i].DeployDescriptor.InstancesMap());
verticesConnectionsMap.AddOrUpdate(fromInputId + tasks[i].OutputId,
flatFromToConnections, (key, value) => flatFromToConnections);
}
}
}
return(verticesConnectionsMap);
}
static void Main(string[] args)
{
var client = new CRAClientLibrary();
client.DefineVertex("connectionpairvertex", () => new ConnectionPairVertex());
client.InstantiateVertex("crainst01", "vertex1", "connectionpairvertex", new MyParam {
field1 = 33, field2 = "foo"
});
client.InstantiateVertex("crainst02", "vertex2", "connectionpairvertex", new MyParam {
field1 = 34
});
client.Connect("vertex1", "output", "vertex2", "input");
client.Connect("vertex2", "output", "vertex1", "input");
Console.ReadLine();
}
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount,
string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards,
string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards, bool isSecondaryInput)
{
List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>();
string output = OperatorUtils.PrepareOutputEndpointIdForOperator(toVertex, fromVertexDescriptor);
string input = OperatorUtils.PrepareInputEndpointIdForOperator(fromVertex, toVertexDescriptor, isSecondaryInput);
bool hasSameCRAInstances = true;
for (int i = 0; i < shardsCount; i++)
{
string currentFromVertex = fromVertex + "$" + i;
string currentToVertex = toVertex + "$" + i;
hasSameCRAInstances = hasSameCRAInstances & client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards);
}
fromToConnections.Add(new ConnectionInfoWithLocality(fromVertex, output, toVertex, input, hasSameCRAInstances, isSecondaryInput));
return(fromToConnections);
}
public static bool DeployProduceTask(CRAClientLibrary client, ProduceTask task)
{
try {
client.DefineVertex(typeof(ProducerOperator).Name.ToLower(), () => new ProducerOperator());
CRAErrorCode status = client.InstantiateShardedVertex(task.OutputId, typeof(ProducerOperator).Name.ToLower(),
task, task.DeployDescriptor.InstancesMap());
if (status == CRAErrorCode.Success)
{
return(true);
}
else
{
return(false);
}
}
catch (Exception e)
{
Console.WriteLine("Error in deploying a sharded CRA produce task. Please, double check your task configurations: " + e.ToString());
return(false);
}
}
static void Main(string[] args)
{
var client = new CRAClientLibrary();
client.DefineVertex("shardedconnectionpairvertex", () => new ShardedConnectionPairVertex());
client.InstantiateVertex(
new[] { "crainst01", "crainst02" },
"vertex1",
"shardedconnectionpairvertex",
null, 1, e => e % 2);
client.InstantiateVertex(
new[] { "crainst01", "crainst02" },
"vertex2",
"shardedconnectionpairvertex",
null, 1, e => e % 2);
client.Connect("vertex1", "output", "vertex2", "input");
client.Connect("vertex2", "output", "vertex1", "input");
Console.ReadLine();
}
static void Main(string[] args)
{
var client = new CRAClientLibrary();
client.DefineVertexAsync("shardedconnectionpairvertex", (System.Linq.Expressions.Expression <Func <IVertex> >)(() => new ShardedConnectionPairVertex())).Wait();
client.InstantiateVertexAsync(
new[] { "crainst01", "crainst02" },
"vertex1",
"shardedconnectionpairvertex",
null, 1, e => e % 2).Wait();
client.InstantiateVertexAsync(
new[] { "crainst01", "crainst02" },
"vertex2",
"shardedconnectionpairvertex",
null, 1, e => e % 2).Wait();
client.ConnectAsync("vertex1", "output", "vertex2", "input").Wait();
client.ConnectAsync("vertex2", "output", "vertex1", "input").Wait();
Console.ReadLine();
}
private static List <ConnectionInfoWithLocality> PrepareFlatConnectionsMap(CRAClientLibrary client, int shardsCount,
string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards,
string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards)
{
List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>();
string[] outputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator(
toVertex, fromVertexDescriptor);
string[] inputs = OperatorUtils.PrepareInputEndpointsIdsForOperator(
fromVertex, toVertexDescriptor);
for (int i = 0; i < shardsCount; i++)
{
string currentFromVertex = fromVertex + "$" + i;
string currentToVertex = toVertex + "$" + i;
bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards);
for (int j = 0; j < outputs.Length; j++)
{
var fromVertexTuple = new Tuple <string, string>(currentFromVertex, outputs[j]);
var toVertexTuple = new Tuple <string, string>(currentToVertex, inputs[j]);
fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, outputs[j], currentToVertex, inputs[j], hasSameCRAInstances));
}
}
return(fromToConnections);
}
public OperatorBase(IDataProvider dataProvider) : base()
{
_craClient = new CRAClientLibrary(dataProvider);
}
private static List <ConnectionInfoWithLocality> PrepareShuffleConnectionsMap(CRAClientLibrary client, int shardsCount,
string fromVertex, OperatorEndpointsDescriptor fromVertexDescriptor, ConcurrentDictionary <string, int> fromVertexShards,
string toVertex, OperatorEndpointsDescriptor toVertexDescriptor, ConcurrentDictionary <string, int> toVertexShards)
{
List <ConnectionInfoWithLocality> fromToConnections = new List <ConnectionInfoWithLocality>();
for (int i = 0; i < shardsCount; i++)
{
for (int j = 0; j < shardsCount; j++)
{
string currentFromVertex = fromVertex + "$" + i;
string currentToVertex = toVertex + "$" + j;
string[] currentOutputs = OperatorUtils.PrepareOutputEndpointsIdsForOperator(
toVertex + j, fromVertexDescriptor);
string[] currentInputs = OperatorUtils.PrepareInputEndpointsIdsForOperator(
fromVertex + i, toVertexDescriptor);
bool hasSameCRAInstances = client.AreTwoVerticessOnSameCRAInstance(currentFromVertex, fromVertexShards, currentToVertex, toVertexShards);
for (int k = 0; k < currentOutputs.Length; k++)
{
fromToConnections.Add(new ConnectionInfoWithLocality(currentFromVertex, currentOutputs[k], currentToVertex, currentInputs[k], hasSameCRAInstances));
}
}
}
return(fromToConnections);
}
public override IShardedDataset <TKeyO, TPayloadO, TDataSetO> Deploy()
{
if (!_isDeployed)
{
OperatorsToplogy toplogy = OperatorsToplogy.GetInstance();
TaskBase subscribeTask = new SubscribeTask();
subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
subscribeTask.IsRightOperandInput = false;
OperatorTransforms subscribeInputTransforms = new OperatorTransforms();
Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms);
subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1);
subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2);
subscribeTask.OutputId = typeof(SubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString();
subscribeTask.PrepareTaskTransformations(subscribeInputTransforms);
toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask);
toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1);
toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2);
toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId);
toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId);
if (subscribeTask.Transforms != null)
{
foreach (OperatorInputs inputs in subscribeTask.TransformsInputs)
{
toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1);
toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2);
toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId);
toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId);
}
}
ClientTerminalTask clientTerminalTask = new ClientTerminalTask();
clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId);
clientTerminalTask.OutputId = typeof(DetachedVertex).Name.ToLower() + Guid.NewGuid().ToString();
_craClient = new CRAClientLibrary();
_clientTerminal = _craClient.RegisterAsVertex(clientTerminalTask.OutputId);
toplogy.AddOperatorBase(clientTerminalTask.OutputId, clientTerminalTask);
toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId1);
toplogy.AddOperatorInput(clientTerminalTask.OutputId, clientTerminalTask.InputIds.InputId2);
toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId1, clientTerminalTask.OutputId);
toplogy.AddOperatorOutput(clientTerminalTask.InputIds.InputId2, clientTerminalTask.OutputId);
_isDeployed = DeploymentUtils.DeployOperators(_craClient, toplogy);
if (_isDeployed)
{
_isDeployed = DeploymentUtils.DeployClientTerminal(_craClient, clientTerminalTask, ref _clientTerminal, toplogy);
}
else
{
return(null);
}
}
return(this);
}
public override async Task <IShardedDataset <TKeyO, TPayloadO, TDataSetO> > Deploy()
{
if (!_isDeployed)
{
OperatorsToplogy toplogy = OperatorsToplogy.GetInstance();
TaskBase subscribeTask = new SubscribeTask();
subscribeTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
subscribeTask.IsRightOperandInput = false;
OperatorTransforms subscribeInputTransforms = new OperatorTransforms();
Deploy(ref subscribeTask, ref toplogy, ref subscribeInputTransforms);
subscribeTask.InputIds.SetInputId1(subscribeTask.NextInputIds.InputId1);
subscribeTask.InputIds.SetInputId2(subscribeTask.NextInputIds.InputId2);
subscribeTask.OutputId = typeof(ShardedSubscribeOperator).Name.ToLower() + Guid.NewGuid().ToString();
subscribeTask.PrepareTaskTransformations(subscribeInputTransforms);
toplogy.AddOperatorBase(subscribeTask.OutputId, subscribeTask);
toplogy.AddOperatorInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId1);
toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, subscribeTask.InputIds.InputId2);
toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId1, subscribeTask.OutputId);
toplogy.AddOperatorOutput(subscribeTask.InputIds.InputId2, subscribeTask.OutputId);
if (subscribeTask.Transforms != null)
{
foreach (OperatorInputs inputs in subscribeTask.TransformsInputs)
{
toplogy.AddOperatorSecondaryInput(subscribeTask.OutputId, inputs.InputId2);
toplogy.AddOperatorOutput(inputs.InputId2, subscribeTask.OutputId);
}
foreach (OperatorInputs inputs in subscribeTask.TransformsInputs)
{
if (!toplogy.ContainsSecondaryOperatorInput(subscribeTask.OutputId, inputs.InputId1))
{
toplogy.AddOperatorInput(subscribeTask.OutputId, inputs.InputId1);
toplogy.AddOperatorOutput(inputs.InputId1, subscribeTask.OutputId);
}
}
}
_clientTerminalTask = new ClientTerminalTask();
_clientTerminalTask.InputIds.SetInputId1(subscribeTask.OutputId);
_clientTerminalTask.OutputId = typeof(ShardedSubscribeClientOperator).Name.ToLower() + Guid.NewGuid().ToString();
_clientTerminalTask.OperationTypes = TransformUtils.FillBinaryTransformTypes(
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO),
typeof(TKeyO), typeof(TPayloadO), typeof(TDataSetO));
_craClient = new CRAClientLibrary(_dataProvider);
toplogy.AddOperatorBase(_clientTerminalTask.OutputId, _clientTerminalTask);
toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId1);
toplogy.AddOperatorInput(_clientTerminalTask.OutputId, _clientTerminalTask.InputIds.InputId2);
toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId1, _clientTerminalTask.OutputId);
toplogy.AddOperatorOutput(_clientTerminalTask.InputIds.InputId2, _clientTerminalTask.OutputId);
_isDeployed = await DeploymentUtils.DeployOperators(_craClient, toplogy);
if (_isDeployed)
{
string craWorkerName = typeof(ShardedSubscribeClientOperator).Name.ToLower() + "worker" + Guid.NewGuid().ToString();
_craWorker = new CRAWorker(craWorkerName, "127.0.0.1", NetworkUtils.GetAvailablePort(), _craClient.DataProvider, null, 1000);
_craWorker.DisableDynamicLoading();
_craWorker.SideloadVertex(new ShardedSubscribeClientOperator(), typeof(ShardedSubscribeClientOperator).Name.ToLower());
new Thread(() => _craWorker.Start()).Start();
Thread.Sleep(1000);
_isDeployed = await DeploymentUtils.DeployClientTerminal(_craClient, craWorkerName, _clientTerminalTask, toplogy);
}
else
{
return(null);
}
}
return(this);
}
private static async Task <bool> DeployTask(int taskIndex, TaskBase[] tasks, string[] tasksIds, Dictionary <string, bool> tasksDeploymentStatus, CRAClientLibrary client, OperatorsToplogy topology)
{
if (!tasksDeploymentStatus[tasksIds[taskIndex]])
{
bool isSuccessful = true;
foreach (var fromInput in tasks[taskIndex].EndpointsDescriptor.FromInputs.Keys)
{
int fromInputIndex = RetrieveTaskIndexOfOperator(fromInput, tasksIds);
isSuccessful = isSuccessful & await DeployTask(fromInputIndex, tasks, tasksIds, tasksDeploymentStatus, client, topology);
}
foreach (var fromSecondaryInput in tasks[taskIndex].EndpointsDescriptor.SecondaryFromInputs.Keys)
{
int fromSecondaryInputIndex = RetrieveTaskIndexOfOperator(fromSecondaryInput, tasksIds);
isSuccessful = isSuccessful & await DeployTask(fromSecondaryInputIndex, tasks, tasksIds, tasksDeploymentStatus, client, topology);
}
if (isSuccessful)
{
if (tasks[taskIndex].OperationType == OperatorType.Produce)
{
isSuccessful = isSuccessful & await DeployProduceTask(client, (ProduceTask)tasks[taskIndex], topology);
if (isSuccessful)
{
tasksDeploymentStatus[tasksIds[taskIndex]] = true;
}
}
else if (tasks[taskIndex].OperationType == OperatorType.Subscribe)
{
isSuccessful = isSuccessful & await DeploySubscribeTask(client, (SubscribeTask)tasks[taskIndex], topology);
if (isSuccessful)
{
tasksDeploymentStatus[tasksIds[taskIndex]] = true;
}
}
else if (tasks[taskIndex].OperationType == OperatorType.Move)
{
isSuccessful = isSuccessful & await DeployShuffleReduceTask(client, (ShuffleTask)tasks[taskIndex], topology);
if (isSuccessful)
{
tasksDeploymentStatus[tasksIds[taskIndex]] = true;
}
}
}
return(isSuccessful);
}
else
{
return(true);
}
}
static void Main(string[] args)
{
GenericLogsInterface.SetToGenericLogs();
ParseAndValidateOptions(args);
Trace.Listeners.Add(new TextWriterTraceListener(Console.Out));
switch (_runtimeMode)
{
case LocalAmbrosiaRuntimeModes.DebugInstance:
var myRuntime = new AmbrosiaRuntime();
myRuntime.InitializeRepro(_instanceName, _serviceLogPath, _checkpointToLoad, _currentVersion,
_isTestingUpgrade, _serviceReceiveFromPort, _serviceSendToPort);
return;
case LocalAmbrosiaRuntimeModes.AddReplica:
case LocalAmbrosiaRuntimeModes.RegisterInstance:
if (_runtimeMode == LocalAmbrosiaRuntimeModes.AddReplica)
{
_isActiveActive = true;
}
var dataProvider = new CRA.DataProvider.Azure.AzureDataProvider(Environment.GetEnvironmentVariable("AZURE_STORAGE_CONN_STRING"));
var client = new CRAClientLibrary(dataProvider);
client.DisableArtifactUploading();
var replicaName = $"{_instanceName}{_replicaNumber}";
AmbrosiaRuntimeParams param = new AmbrosiaRuntimeParams();
param.createService = _recoveryMode == AmbrosiaRecoveryModes.A
? (bool?)null
: (_recoveryMode != AmbrosiaRecoveryModes.N);
param.pauseAtStart = _isPauseAtStart;
param.persistLogs = _isPersistLogs;
param.logTriggerSizeMB = _logTriggerSizeMB;
param.activeActive = _isActiveActive;
param.upgradeToVersion = _upgradeVersion;
param.currentVersion = _currentVersion;
param.serviceReceiveFromPort = _serviceReceiveFromPort;
param.serviceSendToPort = _serviceSendToPort;
param.serviceName = _instanceName;
param.serviceLogPath = _serviceLogPath;
param.AmbrosiaBinariesLocation = _binariesLocation;
param.storageConnectionString = Environment.GetEnvironmentVariable("AZURE_STORAGE_CONN_STRING");
try
{
if (client.DefineVertexAsync(param.AmbrosiaBinariesLocation, () => new AmbrosiaRuntime()).GetAwaiter().GetResult() != CRAErrorCode.Success)
{
throw new Exception();
}
// Workaround because of limitation in parameter serialization in CRA
XmlSerializer xmlSerializer = new XmlSerializer(param.GetType());
string serializedParams;
using (StringWriter textWriter = new StringWriter())
{
xmlSerializer.Serialize(textWriter, param);
serializedParams = textWriter.ToString();
}
if (client.InstantiateVertexAsync(replicaName, param.serviceName, param.AmbrosiaBinariesLocation, serializedParams).GetAwaiter().GetResult() != CRAErrorCode.Success)
{
throw new Exception();
}
client.AddEndpointAsync(param.serviceName, AmbrosiaRuntime.AmbrosiaDataInputsName, true, true).Wait();
client.AddEndpointAsync(param.serviceName, AmbrosiaRuntime.AmbrosiaDataOutputsName, false, true).Wait();
client.AddEndpointAsync(param.serviceName, AmbrosiaRuntime.AmbrosiaControlInputsName, true, true).Wait();
client.AddEndpointAsync(param.serviceName, AmbrosiaRuntime.AmbrosiaControlOutputsName, false, true).Wait();
}
catch (Exception e)
{
Console.WriteLine("Error trying to upload service. Exception: " + e.Message);
}
return;
default:
throw new NotSupportedException($"Runtime mode: {_runtimeMode} not supported.");
}
}
public OperatorBase() : base()
{
_craClient = new CRAClientLibrary();
}
