public void PrintGraph(DrGraph graph)
{
using (StreamWriter sw = new StreamWriter(new FileStream("toplogy.txt", FileMode.Create)))
{
List <DrStageManager> stages = graph.GetStages();
foreach (DrStageManager s in stages)
{
List <DrVertex> vertices = s.GetVertexVector();
foreach (DrVertex v in vertices)
{
sw.Write("{0} <= ", v.GetId());
DrEdgeHolder eh = v.GetInputs();
int n = eh.GetNumberOfEdges();
if (n == 0)
{
sw.Write("DSC");
}
else
{
for (int i = 0; i < n; i++)
{
DrVertex vin = eh.GetEdge(i).m_remoteVertex;
sw.Write("{0} ", vin.GetId());
}
}
sw.WriteLine();
}
}
}
}
internal void FinalizeExecution(Query query, DrGraph graph)
{
SortedDictionary <int, Vertex> queryPlan = query.queryPlan;
foreach (KeyValuePair <int, Vertex> kvp in query.queryPlan)
{
/* used to do CSStream expiration time stuff here */
}
}
public DryadLINQApp(DrGraph graph)
{
m_graph = graph;
m_clusterAffinity = true;
m_maxAggregateInputs = 150;
m_maxAggregateFilterInputs = 32;
m_aggregateThreshold = 1024 * 1024 * 1024; // 1GB
m_startTime = graph.GetCluster().GetCurrentTimeStamp();
m_identityMapFile = null;
m_optionMap = new SortedDictionary <string, OptionDescription>();
AddOptionsToMap(s_dryadLinqOptionArray);
}
private DrVertexSet CreateVertexSet(DrGraph graph, DrVertex prototype, int copies)
{
DrVertexSet result = new DrVertexSet();
for (int i = 0; i < copies; i++)
{
DrVertex v = prototype.MakeCopy(i);
result.Add(v);
}
return(result);
}
private DrVertexSet CreateVertexSet(DrGraph graph, DrInputStreamManager inputStream)
{
DrVertexSet result = new DrVertexSet();
List <DrStorageVertex> vertices = inputStream.GetVertices();
for (int i = 0; i < vertices.Count; i++)
{
DrVertex v = vertices[i];
result.Add(v);
}
return(result);
}
private DrVertexSet CreateVertexSet(DrGraph graph, DrOutputStreamManager outputStream, int parts)
{
DrVertexSet result = new DrVertexSet();
outputStream.SetNumberOfParts(parts);
List <DrOutputVertex> vertices = outputStream.GetVertices();
for (int i = 0; i < vertices.Count; i++)
{
DrVertex v = vertices[i];
result.Add(v);
}
return(result);
}
public DryadLINQApp(DrGraph graph)
{
m_graph = graph;
m_clusterAffinity = true;
m_maxAggregateInputs = 150;
m_maxAggregateFilterInputs = 32;
m_aggregateThreshold = 1024*1024*1024; // 1GB
m_startTime = graph.GetCluster().GetCurrentTimeStamp();
m_identityMapFile = null;
m_optionMap = new SortedDictionary<string,OptionDescription>();
AddOptionsToMap(s_dryadLinqOptionArray);
}
//
// Main Dryad LINQ execution stuff
//
public int ExecLinqToDryad(string[] args)
{
//
// must be at least two arguments (program name and query XML file name)
//
if (args.Length < 2)
{
DryadLogger.LogCritical(0, null, "Must provide at least query XML file name.");
return(-1);
}
//
// break if --break is included in arguments (and eliminate it, as it is not known downstream)
//
if (ConsumeSingleArgument("--break", ref args))
{
DebugHelper.WaitForDebugger();
}
//
// parse the XML input, producing a DryadLINQ Query
//
Query query = new Query();
QueryPlanParser parser = new QueryPlanParser();
if (!parser.ParseQueryXml(args[1], query))
{
DryadLogger.LogCritical(0, null, "Invalid query plan");
return(-1);
}
//
// build internal app arguments
//
List <string> internalArgs = new List <string>();
//
// add the XmlExecHost args to the internal app arguments
//
foreach (string xmlExecHostArg in query.xmlExecHostArgs)
{
if (xmlExecHostArg == "--break")
{
DebugHelper.WaitForDebugger();
}
else
{
internalArgs.Add(xmlExecHostArg);
}
}
//
// combine internal arguments with any additional arguments received on the command line
// don't include argv[0] and argv[1] (program name and query XML file name)
//
int internalArgc = (int)internalArgs.Count;
int externalArgc = args.Length - 2; // don't include argv[0] and argv[1]
int combinedArgc = internalArgc + externalArgc;
string[] combinedArgv = new string[combinedArgc];
string msg = "";
// internal arguments first
for (int i = 0; i < internalArgc; i++)
{
combinedArgv[i] = internalArgs[i];
msg += String.Format("{0} ", combinedArgv[i]);
}
// then external arguments
for (int i = 0; i < externalArgc; i++)
{
combinedArgv[i + internalArgc] = args[i + 2]; // don't include argv[0] and argv[1]
msg += String.Format("{0} ", combinedArgv[i + internalArgc]);
}
DryadLogger.LogInformation(null, "Arguments: {0}", msg);
string jobClass = "DryadLINQ";
string dryadBinDir = Environment.GetEnvironmentVariable("DRYAD_HOME");
if (String.IsNullOrEmpty(dryadBinDir))
{
throw new ApplicationException("DryadLINQ requires the DRYAD_HOME environment variable to be set to the Dryad binary folder.");
}
string exeName = Path.Combine(dryadBinDir, "VertexHost.exe");
// create app and run it
//
DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication);
DrArtemisLegacyReporter reporter = new DrArtemisLegacyReporter();
p.m_defaultProcessTemplate.GetListenerList().Add(reporter);
p.m_defaultVertexTemplate.GetListenerList().Add(reporter);
p.m_topologyReporter = reporter;
p.m_intermediateCompressionMode = query.intermediateDataCompression;
DrGraphExecutor graphExecutor = new DrGraphExecutor();
DrGraph graph = graphExecutor.Initialize(p);
if (graph == null)
{
DryadLogger.LogCritical(0, null, "Failed to initialize Graph Executor");
return(-1);
}
DryadLINQApp app = new DryadLINQApp(graph);
// Initialize with arguments
app.SetXmlFileName(args[1]);
if (!app.ParseCommandLineFlags(combinedArgv))
{
DryadLogger.LogCritical(0, null, "Bad command-line options");
return(-1);
}
// Build graph from query plan
GraphBuilder builder = new GraphBuilder();
builder.BuildGraphFromQuery(app, query);
// Run the app
DryadLogger.LogInformation(null, "Running the app");
graphExecutor.Run();
DrError exitStatus = graphExecutor.Join();
DryadLogger.LogInformation(null, "Finished running the app");
if (exitStatus == null || exitStatus.m_code == 0)
{
FinalizeExecution(query, graph);
DryadLogger.LogInformation(null, "Application completed successfully.");
return(0);
}
else
{
DryadLogger.LogCritical(exitStatus.m_code, null, "Application failed with error code 0x{0:X8}.\n", exitStatus.m_code);
return(exitStatus.m_code);
}
}
internal void FinalizeExecution(Query query, DrGraph graph)
{
SortedDictionary<int, Vertex> queryPlan = query.queryPlan;
foreach (KeyValuePair<int, Vertex> kvp in query.queryPlan)
{
/* used to do CSStream expiration time stuff here */
}
}
//
// Main Dryad LINQ execution stuff
//
public int ExecLinqToDryad(Uri dfsDirectory, string[] args, out string errorString)
{
//
// must be at least two arguments (program name and query XML file name)
//
if (args.Length < 2)
{
errorString = "Must provide at least query XML file name and VertexHost executable.";
DryadLogger.LogCritical(errorString);
return(-1);
}
//
// break if --break is included in arguments (and eliminate it, as it is not known downstream)
//
if (ConsumeSingleArgument("--break", ref args))
{
DebugHelper.WaitForDebugger();
}
// this is where we ensure the right type of scheduler is registered
Microsoft.Research.Dryad.LocalScheduler.Registration.Ensure();
//
// parse the XML input, producing a DryadLINQ Query
//
Query query = new Query();
QueryPlanParser parser = new QueryPlanParser();
if (!parser.ParseQueryXml(args[1], query))
{
errorString = "Invalid query plan";
DryadLogger.LogCritical(errorString);
return(-1);
}
//
// upload the query plan to the job DFS directory for the benefit of debug tools
//
if (dfsDirectory != null)
{
DebugHelper.UploadToDfs(dfsDirectory, args[1]);
}
//
// build internal app arguments
//
List <string> internalArgs = new List <string>();
//
// add the XmlExecHost args to the internal app arguments
//
foreach (string xmlExecHostArg in query.xmlExecHostArgs)
{
if (xmlExecHostArg == "--break")
{
DebugHelper.WaitForDebugger();
}
else
{
internalArgs.Add(xmlExecHostArg);
}
}
//
// combine internal arguments with any additional arguments received on the command line
// don't include argv[0] and argv[1] (program name and query XML file name)
//
int internalArgc = (int)internalArgs.Count;
int externalArgc = args.Length - 2; // don't include argv[0] and argv[1]
int combinedArgc = internalArgc + externalArgc;
string[] combinedArgv = new string[combinedArgc];
string msg = "";
// internal arguments first
for (int i = 0; i < internalArgc; i++)
{
combinedArgv[i] = internalArgs[i];
msg += String.Format("{0} ", combinedArgv[i]);
}
// then external arguments
for (int i = 0; i < externalArgc; i++)
{
combinedArgv[i + internalArgc] = args[i + 2]; // don't include argv[0] and argv[1]
msg += String.Format("{0} ", combinedArgv[i + internalArgc]);
}
DryadLogger.LogInformation(String.Format("Arguments: {0}", msg));
string jobClass = "DryadLINQ";
string exeName = args[0];
// create app and run it
//
DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication);
foreach (DrIReporter reporter in DebugHelper.Reporters())
{
p.m_reporters.Add(reporter);
}
p.m_intermediateCompressionMode = query.intermediateDataCompression;
DrGraphExecutor graphExecutor = new DrGraphExecutor();
DrGraph graph = graphExecutor.Initialize(p);
if (graph == null)
{
errorString = "Failed to initialize Graph Executor";
DryadLogger.LogCritical(errorString);
return(-1);
}
DryadLINQApp app = new DryadLINQApp(graph);
// Initialize with arguments
app.SetXmlFileName(args[1]);
if (!app.ParseCommandLineFlags(combinedArgv))
{
errorString = "Bad command-line options";
DryadLogger.LogCritical(errorString);
return(-1);
}
// Build graph from query plan
GraphBuilder builder = new GraphBuilder();
builder.BuildGraphFromQuery(app, query);
// Run the app
DryadLogger.LogInformation("Running the app");
graphExecutor.Run();
DrError exitStatus = graphExecutor.Join();
DryadLogger.LogInformation("Finished running the app");
if (exitStatus == null || exitStatus.m_code == 0)
{
FinalizeExecution(query, graph);
DryadLogger.LogInformation("Application completed successfully.");
errorString = null;
return(0);
}
else
{
DryadLogger.LogCritical(String.Format("Application failed with error code 0x{0:X8}.\n", exitStatus.m_code));
errorString = exitStatus.ToFullTextNative();
return(exitStatus.m_code);
}
}
