public static void UploadToDfs(Uri dfsDirectory, string localPath)
{
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsDirectory.AbsoluteUri);
try
{
if (dfsDirectory.Scheme == "hdfs")
{
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
{
string dfsPath = dfsDirectory.AbsolutePath + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
hdfs.UploadAll(localPath, dfsPath);
}
}
else if (dfsDirectory.Scheme == "azureblob")
{
string account, key, container, blob;
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container);
string dfsPath = blob + Path.GetFileName(localPath);
DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath);
azure.PutDfsFile(localPath, dfsPath);
}
else if (dfsDirectory.Scheme == "file")
{
string dstPath = Path.Combine(dfsDirectory.AbsolutePath, Path.GetFileName(localPath));
File.Copy(localPath, dstPath);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Failed to upload query plan: " + e.ToString());
}
}
public static void UploadToDfs(Uri dfsDirectory, string dfsName, string payload)
{
byte[] payloadBytes = System.Text.Encoding.UTF8.GetBytes(payload);
DryadLogger.LogInformation("Uploading payload to " + dfsName + " at " + dfsDirectory.AbsoluteUri);
try
{
if (dfsDirectory.Scheme == "hdfs")
{
using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory))
{
string dfsPath = dfsDirectory.AbsolutePath + dfsName;
hdfs.WriteAll(dfsPath, payloadBytes);
}
}
else if (dfsDirectory.Scheme == "azureblob")
{
string account, key, container, blob;
Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob);
var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container);
string dfsPath = blob + dfsName;
azure.PutDfsFile(payloadBytes, dfsPath);
}
else if (dfsDirectory.Scheme == "file")
{
string dstPath = Path.Combine(dfsDirectory.AbsolutePath, dfsName);
File.WriteAllBytes(dstPath, payloadBytes);
}
}
catch (Exception e)
{
DryadLogger.LogWarning("Failed to upload final output: " + e.ToString());
}
}
public void BuildGraphFromQuery(DryadLINQApp app, Query query)
{
// set configurable properties
int highThreshold = app.GetMaxAggregateInputs();
int lowThreshold = 16;
UInt64 highDataThreshold = (UInt64)app.GetAggregateThreshold();
UInt64 lowDataThreshold = (3 * highDataThreshold) / 4;
UInt64 maxSingleDataThreshold = highDataThreshold / 2;
int aggFilterThreshold = app.GetMaxAggregateFilterInputs();
// use a graph stage map to store the vertices as they are created, grouped by stage.
Dictionary <int, GraphStageInfo> graphStageMap = new Dictionary <int, GraphStageInfo>();
DryadLogger.LogInformation("Building graph");
//
// Create a set of vertices for each vertex (stage) in the query plan
//
DryadLogger.LogInformation("Adding vertices");
foreach (KeyValuePair <int, Vertex> kvp in query.queryPlan)
{
Vertex v = kvp.Value;
GraphStageInfo value = null;
if (!graphStageMap.TryGetValue(v.uniqueId, out value))
{
DryadLogger.LogInformation(String.Format("Adding vertices for stage {0}", v.name));
CreateVertexSet(v, app, query, graphStageMap);
}
}
//
// Add dynamic stage managers
//
DryadLogger.LogInformation("Build Graph From Query", "Adding stage managers");
foreach (KeyValuePair <int, GraphStageInfo> kvp in graphStageMap)
{
Vertex v = kvp.Value.vertex;
//
//There are no dynamic managers
//
if (v.dynamicManager == null)
{
continue;
}
DrStageManager newManager = kvp.Value.stageManager; // newManager
DrGraphParameters parameters = app.GetGraph().GetParameters();
string stdVertexName = "MW";
string cpyVertexName = "CP";
if (v.type != Vertex.Type.INPUTTABLE && v.type != Vertex.Type.CONCAT)
{
if (v.dynamicManager.type == DynamicManager.Type.SPLITTER)
{
if (v.info.predecessors.Length == 1)
{
DrPipelineSplitManager splitter = new DrPipelineSplitManager();
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, splitter);
}
else
{
DrSemiPipelineSplitManager splitter = new DrSemiPipelineSplitManager();
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, splitter);
}
}
else if (v.dynamicManager.type == DynamicManager.Type.PARTIALAGGR)
{
DrDynamicAggregateManager dynamicMerge = new DrDynamicAggregateManager();
dynamicMerge.SetGroupingSettings(0, 0);
dynamicMerge.SetMachineGroupingSettings(2, aggFilterThreshold);
dynamicMerge.SetDataGroupingSettings(lowDataThreshold, highDataThreshold, maxSingleDataThreshold);
dynamicMerge.SetSplitAfterGrouping(true);
foreach (Predecessor p in v.info.predecessors)
{
newManager.AddDynamicConnectionManager(graphStageMap[p.uniqueId].stageManager, dynamicMerge);
}
}
else if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR ||
v.dynamicManager.type == DynamicManager.Type.HASHDISTRIBUTOR)
{
int idx = 0;
int sz = v.dynamicManager.assemblyNames == null ? 0 : v.dynamicManager.assemblyNames.Length;
DrDynamicAggregateManager dynamicMerge = new DrDynamicAggregateManager();
if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR || sz > 1)
{
dynamicMerge = new DrDynamicAggregateManager();
string name = v.dynamicManager.methodNames[idx];
DrManagerBase newStage = new DrManagerBase(app.GetGraph(), name);
DrActiveVertex mergeVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate);
mergeVertex.AddArgument(stdVertexName);
mergeVertex.AddArgument(v.dynamicManager.assemblyNames[idx]);
mergeVertex.AddArgument(v.dynamicManager.classNames[idx]);
mergeVertex.AddArgument(v.dynamicManager.methodNames[idx]);
idx++;
dynamicMerge.SetInternalVertex(mergeVertex);
dynamicMerge.SetGroupingSettings(0, 0);
dynamicMerge.SetPodGroupingSettings(lowThreshold, highThreshold);
dynamicMerge.SetDataGroupingSettings(lowDataThreshold,
highDataThreshold,
maxSingleDataThreshold);
dynamicMerge.SetMaxAggregationLevel(v.dynamicManager.aggregationLevels);
}
if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR)
{
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, dynamicMerge);
}
else
{
string name = v.dynamicManager.methodNames[idx];
DrManagerBase newStage = new DrManagerBase(app.GetGraph(), name);
DrActiveVertex distributeVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate);
distributeVertex.AddArgument(stdVertexName);
distributeVertex.AddArgument(v.dynamicManager.assemblyNames[idx]);
distributeVertex.AddArgument(v.dynamicManager.classNames[idx]);
distributeVertex.AddArgument(v.dynamicManager.methodNames[idx]);
idx++;
DrDynamicDistributionManager dynamicHashDistribute =
new DrDynamicDistributionManager(distributeVertex, dynamicMerge);
dynamicHashDistribute.SetDataPerVertex(highDataThreshold * 2); // 2GB
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, dynamicHashDistribute);
}
}
else if (v.dynamicManager.type == DynamicManager.Type.RANGEDISTRIBUTOR)
{
DrStageManager splitManager = graphStageMap[v.dynamicManager.splitVertexId].stageManager;
DrDynamicRangeDistributionManager drdm = new DrDynamicRangeDistributionManager(splitManager, v.dynamicManager.sampleRate);
drdm.SetDataPerVertex(highDataThreshold * 2); // 2GB
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, drdm);
}
else if (v.dynamicManager.type == DynamicManager.Type.BROADCAST)
{
// the copy vertex
int bcastNumber = 0;
string nameString = String.Format("CP__{0}", bcastNumber++);
DrManagerBase newStage = new DrManagerBase(app.GetGraph(), nameString);
DrActiveVertex copyVertex =
new DrActiveVertex(newStage,
parameters.m_defaultProcessTemplate,
parameters.m_defaultVertexTemplate);
copyVertex.AddArgument(cpyVertexName);
DrDynamicBroadcastManager bcast = new DrDynamicBroadcastManager(copyVertex);
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, bcast);
}
else if (v.dynamicManager.type != DynamicManager.Type.NONE)
{
DryadLogger.LogWarning(String.Format("Dynamic manager type {0} not supported yet", v.dynamicManager.type));
}
}
}
//
// Add all the edges
//
DryadLogger.LogInformation("Build Graph From Query", "Adding edges");
foreach (KeyValuePair <int, GraphStageInfo> kvp in graphStageMap)
{
AddEdges(kvp.Value, graphStageMap);
}
//
// Register the actual created vertices with the graph
//
MaterializeToManagers(graphStageMap);
}