public static void UploadToDfs(Uri dfsDirectory, string localPath) { DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsDirectory.AbsoluteUri); try { if (dfsDirectory.Scheme == "hdfs") { using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory)) { string dfsPath = dfsDirectory.AbsolutePath + Path.GetFileName(localPath); DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath); hdfs.UploadAll(localPath, dfsPath); } } else if (dfsDirectory.Scheme == "azureblob") { string account, key, container, blob; Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob); var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container); string dfsPath = blob + Path.GetFileName(localPath); DryadLogger.LogInformation("Uploading " + localPath + " to " + dfsPath); azure.PutDfsFile(localPath, dfsPath); } else if (dfsDirectory.Scheme == "file") { string dstPath = Path.Combine(dfsDirectory.AbsolutePath, Path.GetFileName(localPath)); File.Copy(localPath, dstPath); } } catch (Exception e) { DryadLogger.LogWarning("Failed to upload query plan: " + e.ToString()); } }
public static void UploadToDfs(Uri dfsDirectory, string dfsName, string payload) { byte[] payloadBytes = System.Text.Encoding.UTF8.GetBytes(payload); DryadLogger.LogInformation("Uploading payload to " + dfsName + " at " + dfsDirectory.AbsoluteUri); try { if (dfsDirectory.Scheme == "hdfs") { using (var hdfs = new Microsoft.Research.Peloponnese.Hdfs.HdfsInstance(dfsDirectory)) { string dfsPath = dfsDirectory.AbsolutePath + dfsName; hdfs.WriteAll(dfsPath, payloadBytes); } } else if (dfsDirectory.Scheme == "azureblob") { string account, key, container, blob; Microsoft.Research.Peloponnese.Storage.AzureUtils.FromAzureUri(dfsDirectory, out account, out key, out container, out blob); var azure = new Microsoft.Research.Peloponnese.Storage.AzureDfsClient(account, key, container); string dfsPath = blob + dfsName; azure.PutDfsFile(payloadBytes, dfsPath); } else if (dfsDirectory.Scheme == "file") { string dstPath = Path.Combine(dfsDirectory.AbsolutePath, dfsName); File.WriteAllBytes(dstPath, payloadBytes); } } catch (Exception e) { DryadLogger.LogWarning("Failed to upload final output: " + e.ToString()); } }
public void BuildGraphFromQuery(DryadLINQApp app, Query query) { // set configurable properties int highThreshold = app.GetMaxAggregateInputs(); int lowThreshold = 16; UInt64 highDataThreshold = (UInt64)app.GetAggregateThreshold(); UInt64 lowDataThreshold = (3 * highDataThreshold) / 4; UInt64 maxSingleDataThreshold = highDataThreshold / 2; int aggFilterThreshold = app.GetMaxAggregateFilterInputs(); // use a graph stage map to store the vertices as they are created, grouped by stage. Dictionary <int, GraphStageInfo> graphStageMap = new Dictionary <int, GraphStageInfo>(); DryadLogger.LogInformation("Building graph"); // // Create a set of vertices for each vertex (stage) in the query plan // DryadLogger.LogInformation("Adding vertices"); foreach (KeyValuePair <int, Vertex> kvp in query.queryPlan) { Vertex v = kvp.Value; GraphStageInfo value = null; if (!graphStageMap.TryGetValue(v.uniqueId, out value)) { DryadLogger.LogInformation(String.Format("Adding vertices for stage {0}", v.name)); CreateVertexSet(v, app, query, graphStageMap); } } // // Add dynamic stage managers // DryadLogger.LogInformation("Build Graph From Query", "Adding stage managers"); foreach (KeyValuePair <int, GraphStageInfo> kvp in graphStageMap) { Vertex v = kvp.Value.vertex; // //There are no dynamic managers // if (v.dynamicManager == null) { continue; } DrStageManager newManager = kvp.Value.stageManager; // newManager DrGraphParameters parameters = app.GetGraph().GetParameters(); string stdVertexName = "MW"; string cpyVertexName = "CP"; if (v.type != Vertex.Type.INPUTTABLE && v.type != Vertex.Type.CONCAT) { if (v.dynamicManager.type == DynamicManager.Type.SPLITTER) { if (v.info.predecessors.Length == 1) { DrPipelineSplitManager splitter = new DrPipelineSplitManager(); newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, splitter); } else { DrSemiPipelineSplitManager splitter = new DrSemiPipelineSplitManager(); newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, splitter); } } else if (v.dynamicManager.type == DynamicManager.Type.PARTIALAGGR) { DrDynamicAggregateManager dynamicMerge = new DrDynamicAggregateManager(); dynamicMerge.SetGroupingSettings(0, 0); dynamicMerge.SetMachineGroupingSettings(2, aggFilterThreshold); dynamicMerge.SetDataGroupingSettings(lowDataThreshold, highDataThreshold, maxSingleDataThreshold); dynamicMerge.SetSplitAfterGrouping(true); foreach (Predecessor p in v.info.predecessors) { newManager.AddDynamicConnectionManager(graphStageMap[p.uniqueId].stageManager, dynamicMerge); } } else if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR || v.dynamicManager.type == DynamicManager.Type.HASHDISTRIBUTOR) { int idx = 0; int sz = v.dynamicManager.assemblyNames == null ? 0 : v.dynamicManager.assemblyNames.Length; DrDynamicAggregateManager dynamicMerge = new DrDynamicAggregateManager(); if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR || sz > 1) { dynamicMerge = new DrDynamicAggregateManager(); string name = v.dynamicManager.methodNames[idx]; DrManagerBase newStage = new DrManagerBase(app.GetGraph(), name); DrActiveVertex mergeVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate); mergeVertex.AddArgument(stdVertexName); mergeVertex.AddArgument(v.dynamicManager.assemblyNames[idx]); mergeVertex.AddArgument(v.dynamicManager.classNames[idx]); mergeVertex.AddArgument(v.dynamicManager.methodNames[idx]); idx++; dynamicMerge.SetInternalVertex(mergeVertex); dynamicMerge.SetGroupingSettings(0, 0); dynamicMerge.SetPodGroupingSettings(lowThreshold, highThreshold); dynamicMerge.SetDataGroupingSettings(lowDataThreshold, highDataThreshold, maxSingleDataThreshold); dynamicMerge.SetMaxAggregationLevel(v.dynamicManager.aggregationLevels); } if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR) { newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, dynamicMerge); } else { string name = v.dynamicManager.methodNames[idx]; DrManagerBase newStage = new DrManagerBase(app.GetGraph(), name); DrActiveVertex distributeVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate); distributeVertex.AddArgument(stdVertexName); distributeVertex.AddArgument(v.dynamicManager.assemblyNames[idx]); distributeVertex.AddArgument(v.dynamicManager.classNames[idx]); distributeVertex.AddArgument(v.dynamicManager.methodNames[idx]); idx++; DrDynamicDistributionManager dynamicHashDistribute = new DrDynamicDistributionManager(distributeVertex, dynamicMerge); dynamicHashDistribute.SetDataPerVertex(highDataThreshold * 2); // 2GB newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, dynamicHashDistribute); } } else if (v.dynamicManager.type == DynamicManager.Type.RANGEDISTRIBUTOR) { DrStageManager splitManager = graphStageMap[v.dynamicManager.splitVertexId].stageManager; DrDynamicRangeDistributionManager drdm = new DrDynamicRangeDistributionManager(splitManager, v.dynamicManager.sampleRate); drdm.SetDataPerVertex(highDataThreshold * 2); // 2GB newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, drdm); } else if (v.dynamicManager.type == DynamicManager.Type.BROADCAST) { // the copy vertex int bcastNumber = 0; string nameString = String.Format("CP__{0}", bcastNumber++); DrManagerBase newStage = new DrManagerBase(app.GetGraph(), nameString); DrActiveVertex copyVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate); copyVertex.AddArgument(cpyVertexName); DrDynamicBroadcastManager bcast = new DrDynamicBroadcastManager(copyVertex); newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, bcast); } else if (v.dynamicManager.type != DynamicManager.Type.NONE) { DryadLogger.LogWarning(String.Format("Dynamic manager type {0} not supported yet", v.dynamicManager.type)); } } } // // Add all the edges // DryadLogger.LogInformation("Build Graph From Query", "Adding edges"); foreach (KeyValuePair <int, GraphStageInfo> kvp in graphStageMap) { AddEdges(kvp.Value, graphStageMap); } // // Register the actual created vertices with the graph // MaterializeToManagers(graphStageMap); }