static void ExceptionHandler(object sender, UnhandledExceptionEventArgs args)
{
int result = unchecked ((int)LinqToDryadException.E_FAIL);
Exception e = args.ExceptionObject as Exception;
if (e != null)
{
result = System.Runtime.InteropServices.Marshal.GetHRForException(e);
DryadLogger.LogCritical(0, e);
}
DebugHelper.StopLogging(result);
if (Debugger.IsAttached)
{
Debugger.Break();
}
else
{
DrLogging.WriteMiniDump();
}
// We need to Exit, since other threads in the GM
// are likely to still be running.
Environment.Exit(result);
}
public static void InitializeLogging(DateTime startTime)
{
if (!loggingInitialized)
{
lock (syncRoot)
{
if (!loggingInitialized)
{
// Initialize text-based tracing
string traceFile = Path.Combine(Directory.GetCurrentDirectory(), "GraphManagerTrace.txt");
DryadLogger.Start(traceFile);
// Initialize Graph Manager's internal logging
DrLogging.Initialize();
DebugHelper.SetLogType();
// Report start time to Artemis - must come after
// DrLogging is initialized so stdout is redirected
DrArtemisLegacyReporter.ReportStart((ulong)startTime.Ticks);
loggingInitialized = true;
}
}
}
}
public bool ParseQueryXml(string queryPlanFileName, Query query)
{
XmlNode version = null;
XmlDocument queryPlanDoc = new XmlDocument();
//
// Load query plan document
//
try
{
queryPlanDoc.Load(queryPlanFileName);
}
catch (Exception e)
{
DryadLogger.LogCritical(0, e, "Failed to load query plan: {0}", queryPlanFileName);
return(false);
}
//
// Get DryadLinqVersion - absence used to indicate Argentia query plan
//
try
{
version = queryPlanDoc.DocumentElement.SelectSingleNode("DryadLinqVersion");
}
catch (System.Xml.XPath.XPathException e)
{
DryadLogger.LogCritical(0, e, "Failed to select node DryadLinqVersion from query plan: {0}", queryPlanFileName);
return(false);
}
if (version == null)
{
DryadLogger.LogCritical(0, null, "Missing element 'DryadLinqVersion' in query plan: {0}", queryPlanFileName);
return(false);
}
//
// Parse query plan XML doc into Query
//
try
{
ParseQueryXmlLinqToDryad(queryPlanDoc, query);
}
catch (Exception e)
{
DryadLogger.LogCritical(0, e, "Failed to parse query plan: {0}", queryPlanFileName);
return(false);
}
public static void StopLogging(int retCode)
{
if (loggingInitialized)
{
lock (syncRoot)
{
if (loggingInitialized)
{
// Report stop time to Artemis
DrArtemisLegacyReporter.ReportStop(unchecked ((uint)retCode));
// Shutdown Graph Manager's internal logging
DrLogging.ShutDown(unchecked ((uint)retCode));
// Shutdown text-based tracing
DryadLogger.Stop();
loggingInitialized = false;
}
}
}
}
static int Main(string[] args)
{
Console.WriteLine("{0} starting up in {1}", Environment.CommandLine, Environment.CurrentDirectory);
bool waitForDebugger = false;
List <string> tempArgs = new List <string>();
// Record start time so we can report it once logging has been initialized
DateTime startTime = DateTime.Now.ToLocalTime();
// Set unhandled exception handler to catch anything thrown from
// Microsoft.Hpc.Query.GraphManager.dll
AppDomain currentDomain = AppDomain.CurrentDomain;
currentDomain.UnhandledException += new UnhandledExceptionEventHandler(ExceptionHandler);
//
// Add executable name to beginning of args
// to make graph manager libraries happy
//
tempArgs.Add("YarnQueryGraphManager");
foreach (string arg in args)
{
if (String.Compare(arg, "--break", StringComparison.OrdinalIgnoreCase) == 0)
{
waitForDebugger = true;
}
else
{
tempArgs.Add(arg);
}
}
args = tempArgs.ToArray();
if (waitForDebugger)
{
DebugHelper.WaitForDebugger();
}
/* Yarn removes the need for this
*
* //
* // Create job directory and copy resources
* //
* string resources = Environment.GetEnvironmentVariable("XC_RESOURCEFILES");
* if (ExecutionHelper.InitializeForJobExecution(resources) == false)
* {
* return 1;
* }
*
* //
* // Set current directory to working directory
* //
* Directory.SetCurrentDirectory(ProcessPathHelper.JobPath);
*/
//
// Configure tracing
//
DebugHelper.InitializeLogging(startTime);
// Ensure that there is a jvm.dll on the path.
string pathString = Environment.GetEnvironmentVariable("PATH");
var pathDirs = pathString.Split(';');
bool found = false;
foreach (var dir in pathDirs)
{
string targetFile = Path.Combine(dir, "jvm.dll");
if (File.Exists(targetFile))
{
found = true;
break;
}
}
if (!found)
{
string javaHome = Environment.GetEnvironmentVariable("JAVA_HOME");
if (String.IsNullOrEmpty(javaHome))
{
throw new ApplicationException("DryadLINQ requires the JAVA_HOME environment variable to be set or the jvm.dll to be on the path.");
}
var jvmPath = ";" + Path.Combine(javaHome, "jre", "bin", "server");
pathString += jvmPath;
Environment.SetEnvironmentVariable("PATH", pathString);
}
//
// Run the Graph Manager
//
int retCode = 0;
try
{
LinqToDryadJM jm = new LinqToDryadJM();
retCode = jm.Run(args);
}
catch (Exception e)
{
retCode = System.Runtime.InteropServices.Marshal.GetHRForException(e);
if (retCode == 0)
{
DryadLogger.LogCritical(0, e);
retCode = unchecked ((int)LinqToDryadException.E_FAIL);
}
else
{
DryadLogger.LogCritical(retCode, e);
}
System.Threading.Thread.Sleep(10 * 60 * 1000);
}
DebugHelper.StopLogging(retCode);
// NOTE: We don't want to log critical errors twice, so here we're assuming that
// if the GM exited "gracefully" and returned an error code instead of throwing
// an exception, that it has already logged the error.
if (retCode != 0)
{
// If the Graph Manager already started executing, we need to exit the process.
// Exiting the thread (returning from Main) will not necessarily cause the GM's
// worker threads to exit
// TODO: Consider deleting temp output stream from DSC
// requires that we have access to the URI at this point, though
Environment.Exit(retCode);
}
//
// Cleanup all vertex tasks in case any became
// unreachable.
//
return(retCode);
}
//
// Main Dryad LINQ execution stuff
//
public int ExecLinqToDryad(string[] args)
{
//
// must be at least two arguments (program name and query XML file name)
//
if (args.Length < 2)
{
DryadLogger.LogCritical(0, null, "Must provide at least query XML file name.");
return(-1);
}
//
// break if --break is included in arguments (and eliminate it, as it is not known downstream)
//
if (ConsumeSingleArgument("--break", ref args))
{
DebugHelper.WaitForDebugger();
}
//
// parse the XML input, producing a DryadLINQ Query
//
Query query = new Query();
QueryPlanParser parser = new QueryPlanParser();
if (!parser.ParseQueryXml(args[1], query))
{
DryadLogger.LogCritical(0, null, "Invalid query plan");
return(-1);
}
//
// build internal app arguments
//
List <string> internalArgs = new List <string>();
//
// add the XmlExecHost args to the internal app arguments
//
foreach (string xmlExecHostArg in query.xmlExecHostArgs)
{
if (xmlExecHostArg == "--break")
{
DebugHelper.WaitForDebugger();
}
else
{
internalArgs.Add(xmlExecHostArg);
}
}
//
// combine internal arguments with any additional arguments received on the command line
// don't include argv[0] and argv[1] (program name and query XML file name)
//
int internalArgc = (int)internalArgs.Count;
int externalArgc = args.Length - 2; // don't include argv[0] and argv[1]
int combinedArgc = internalArgc + externalArgc;
string[] combinedArgv = new string[combinedArgc];
string msg = "";
// internal arguments first
for (int i = 0; i < internalArgc; i++)
{
combinedArgv[i] = internalArgs[i];
msg += String.Format("{0} ", combinedArgv[i]);
}
// then external arguments
for (int i = 0; i < externalArgc; i++)
{
combinedArgv[i + internalArgc] = args[i + 2]; // don't include argv[0] and argv[1]
msg += String.Format("{0} ", combinedArgv[i + internalArgc]);
}
DryadLogger.LogInformation(null, "Arguments: {0}", msg);
string jobClass = "DryadLINQ";
string dryadBinDir = Environment.GetEnvironmentVariable("DRYAD_HOME");
if (String.IsNullOrEmpty(dryadBinDir))
{
throw new ApplicationException("DryadLINQ requires the DRYAD_HOME environment variable to be set to the Dryad binary folder.");
}
string exeName = Path.Combine(dryadBinDir, "VertexHost.exe");
// create app and run it
//
DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication);
DrArtemisLegacyReporter reporter = new DrArtemisLegacyReporter();
p.m_defaultProcessTemplate.GetListenerList().Add(reporter);
p.m_defaultVertexTemplate.GetListenerList().Add(reporter);
p.m_topologyReporter = reporter;
p.m_intermediateCompressionMode = query.intermediateDataCompression;
DrGraphExecutor graphExecutor = new DrGraphExecutor();
DrGraph graph = graphExecutor.Initialize(p);
if (graph == null)
{
DryadLogger.LogCritical(0, null, "Failed to initialize Graph Executor");
return(-1);
}
DryadLINQApp app = new DryadLINQApp(graph);
// Initialize with arguments
app.SetXmlFileName(args[1]);
if (!app.ParseCommandLineFlags(combinedArgv))
{
DryadLogger.LogCritical(0, null, "Bad command-line options");
return(-1);
}
// Build graph from query plan
GraphBuilder builder = new GraphBuilder();
builder.BuildGraphFromQuery(app, query);
// Run the app
DryadLogger.LogInformation(null, "Running the app");
graphExecutor.Run();
DrError exitStatus = graphExecutor.Join();
DryadLogger.LogInformation(null, "Finished running the app");
if (exitStatus == null || exitStatus.m_code == 0)
{
FinalizeExecution(query, graph);
DryadLogger.LogInformation(null, "Application completed successfully.");
return(0);
}
else
{
DryadLogger.LogCritical(exitStatus.m_code, null, "Application failed with error code 0x{0:X8}.\n", exitStatus.m_code);
return(exitStatus.m_code);
}
}
public bool ParseCommandLineFlags(string[] args)
{
bool retVal = true;
for (int index = 0; index < args.Length; index++)
{
string arg = args[index].Substring(args[index].IndexOf("-") + 1);
OptionDescription option = null;
m_optionMap.TryGetValue(arg, out option);
if (option == null)
{
retVal = false;
break;
}
int optionIndex = option.m_optionIndex;
switch (optionIndex)
{
case (int)DryadLINQAppOptions.BDJAO_AMD64:
case (int)DryadLINQAppOptions.BDJAO_I386:
case (int)DryadLINQAppOptions.BDJAO_Retail:
case (int)DryadLINQAppOptions.BDJAO_Debug:
break;
case (int)DryadLINQAppOptions.DNAO_MaxAggregateInputs:
if ((index + 1) >= args.Length)
{
DryadLogger.LogCritical(0, null, "The argument for option '{0}' was missing.\n", args[index]);
retVal = false;
}
else
{
int maxInputs;
if (!Int32.TryParse(args[index + 1], out maxInputs))
{
DryadLogger.LogCritical(0, null, "The argument '{0}' for option '{1}' could not be parsed as an integer.\n", args[index + 1], args[index]);
retVal = false;
}
else
{
m_maxAggregateInputs = maxInputs;
index++;
}
}
break;
case (int)DryadLINQAppOptions.DNAO_MaxAggregateFilterInputs:
if ((index + 1) >= args.Length)
{
DryadLogger.LogCritical(0, null, "The argument for option '{0}' was missing.\n", args[index]);
retVal = false;
}
else
{
int maxInputs;
if (!Int32.TryParse(args[index + 1], out maxInputs))
{
DryadLogger.LogCritical(0, null, "The argument '{0}' for option '{1}' could not be parsed as an integer.\n", args[index + 1], args[index]);
retVal = false;
}
else
{
m_maxAggregateFilterInputs = maxInputs;
index++;
}
}
break;
case (int)DryadLINQAppOptions.DNAO_AggregateThreshold:
if ((index + 1) >= args.Length)
{
DryadLogger.LogCritical(0, null, "The argument for option '{0}' was missing.\n", args[index]);
retVal = false;
}
else
{
UInt64 threshold;
if (!UInt64.TryParse(args[index + 1], out threshold))
{
DryadLogger.LogCritical(0, null, "The argument '{0}' for option '{1}' could not be parsed as a UIN64.\n", args[index + 1], args[index]);
retVal = false;
}
else
{
m_aggregateThreshold = threshold;
index++;
}
}
break;
case (int)DryadLINQAppOptions.DNAO_NoClusterAffinity:
m_clusterAffinity = false;
break;
default:
DryadLogger.LogCritical(0, null, "Unknown command-line option {0}\n", optionIndex);
retVal = false;
break;
}
}
if (!retVal)
{
PrintUsage(args[0]);
}
return(retVal);
}
private DrInputStreamManager CreateInputNode(DryadLINQApp app, VertexInfo info, string inputName)
{
DrInputStreamManager s;
int err = 0;
DryadLogger.LogMethodEntry(inputName);
if (info.ioType == VertexInfo.IOType.PARTITIONEDFILE)
{
DrPartitionInputStream input = new DrPartitionInputStream();
err = input.Open(app.GetUniverse(), info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read DSC input file {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
//else if ( info.ioType == VertexInfo.IOType.STREAM )
//{
// DrDscInputStream input = new DrDscInputStream();
// DryadLogger.LogInformation("Create input node", "Opening DSC input fileset");
// err = input.Open(app.GetUniverse(), info.sources[0]);
// if (!SUCCEEDED(err))
// {
// string msg = String.Format("Could not read DSC input fileset {0}", info.sources[0]);
// throw new LinqToDryadException(msg, err);
// }
// DryadLogger.LogInformation("Create input node", "Opened DSC input fileset");
// DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
// DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
// s = inputManager;
//}
else if (info.ioType == VertexInfo.IOType.HDFS_STREAM)
{
DrHdfsInputStream input = new DrHdfsInputStream();
DryadLogger.LogInformation("Create input node", "Opening HDFS input fileset");
err = input.Open(app.GetUniverse(), info.sources[0]);
if (!SUCCEEDED(err))
{
string msg = String.Format("Could not read HDFS input fileset {0}", info.sources[0]);
throw new LinqToDryadException(msg, err);
}
DryadLogger.LogInformation("Create input node", "Opened HDFS input fileset");
DrManagerBase inputStage = new DrManagerBase(app.GetGraph(), inputName);
DrInputStreamManager inputManager = new DrInputStreamManager(input, inputStage);
s = inputManager;
}
else
{
string msg = String.Format("Unknown input type {0}", info.ioType);
throw new LinqToDryadException(msg);
}
DryadLogger.LogMethodExit();
return(s);
}
public void BuildGraphFromQuery(DryadLINQApp app, Query query)
{
// set configurable properties
int highThreshold = app.GetMaxAggregateInputs();
int lowThreshold = 16;
UInt64 highDataThreshold = (UInt64)app.GetAggregateThreshold();
UInt64 lowDataThreshold = (3 * highDataThreshold) / 4;
UInt64 maxSingleDataThreshold = highDataThreshold / 2;
int aggFilterThreshold = app.GetMaxAggregateFilterInputs();
// use a graph stage map to store the vertices as they are created, grouped by stage.
Dictionary <int, GraphStageInfo> graphStageMap = new Dictionary <int, GraphStageInfo>();
DryadLogger.LogInformation("Build Graph From Query", "Building graph");
//
// Create a set of vertices for each vertex (stage) in the query plan
//
DryadLogger.LogInformation("Build Graph From Query", "Adding vertices");
foreach (KeyValuePair <int, Vertex> kvp in query.queryPlan)
{
Vertex v = kvp.Value;
GraphStageInfo value = null;
if (!graphStageMap.TryGetValue(v.uniqueId, out value))
{
DryadLogger.LogInformation("Build Graph From Query", "Adding vertices for stage {0}", v.name);
CreateVertexSet(v, app, query, graphStageMap);
}
}
//
// Add dynamic stage managers
//
DryadLogger.LogInformation("Build Graph From Query", "Adding stage managers");
foreach (KeyValuePair <int, GraphStageInfo> kvp in graphStageMap)
{
Vertex v = kvp.Value.vertex;
//
//There are no dynamic managers
//
if (v.dynamicManager == null)
{
continue;
}
DrStageManager newManager = kvp.Value.stageManager; // newManager
DrGraphParameters parameters = app.GetGraph().GetParameters();
string stdVertexName = "MW";
string cpyVertexName = "CP";
if (v.type != Vertex.Type.INPUTTABLE && v.type != Vertex.Type.CONCAT)
{
if (v.dynamicManager.type == DynamicManager.Type.SPLITTER)
{
if (v.info.predecessors.Length == 1)
{
DrPipelineSplitManager splitter = new DrPipelineSplitManager();
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, splitter);
}
else
{
DrSemiPipelineSplitManager splitter = new DrSemiPipelineSplitManager();
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, splitter);
}
}
else if (v.dynamicManager.type == DynamicManager.Type.PARTIALAGGR)
{
DrDynamicAggregateManager dynamicMerge = new DrDynamicAggregateManager();
dynamicMerge.SetGroupingSettings(0, 0);
dynamicMerge.SetMachineGroupingSettings(2, aggFilterThreshold);
dynamicMerge.SetDataGroupingSettings(lowDataThreshold, highDataThreshold, maxSingleDataThreshold);
dynamicMerge.SetSplitAfterGrouping(true);
foreach (Predecessor p in v.info.predecessors)
{
newManager.AddDynamicConnectionManager(graphStageMap[p.uniqueId].stageManager, dynamicMerge);
}
}
else if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR ||
v.dynamicManager.type == DynamicManager.Type.HASHDISTRIBUTOR)
{
int idx = 0;
int sz = v.dynamicManager.assemblyNames == null ? 0 : v.dynamicManager.assemblyNames.Length;
DrDynamicAggregateManager dynamicMerge = new DrDynamicAggregateManager();
if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR || sz > 1)
{
dynamicMerge = new DrDynamicAggregateManager();
string name = v.dynamicManager.methodNames[idx];
DrManagerBase newStage = new DrManagerBase(app.GetGraph(), name);
DrActiveVertex mergeVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate);
mergeVertex.AddArgument(stdVertexName);
mergeVertex.AddArgument(v.dynamicManager.assemblyNames[idx]);
mergeVertex.AddArgument(v.dynamicManager.classNames[idx]);
mergeVertex.AddArgument(v.dynamicManager.methodNames[idx]);
idx++;
dynamicMerge.SetInternalVertex(mergeVertex);
dynamicMerge.SetGroupingSettings(0, 0);
dynamicMerge.SetPodGroupingSettings(lowThreshold, highThreshold);
dynamicMerge.SetDataGroupingSettings(lowDataThreshold,
highDataThreshold,
maxSingleDataThreshold);
dynamicMerge.SetMaxAggregationLevel(v.dynamicManager.aggregationLevels);
}
if (v.dynamicManager.type == DynamicManager.Type.FULLAGGR)
{
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, dynamicMerge);
}
else
{
string name = v.dynamicManager.methodNames[idx];
DrManagerBase newStage = new DrManagerBase(app.GetGraph(), name);
DrActiveVertex distributeVertex = new DrActiveVertex(newStage, parameters.m_defaultProcessTemplate, parameters.m_defaultVertexTemplate);
distributeVertex.AddArgument(stdVertexName);
distributeVertex.AddArgument(v.dynamicManager.assemblyNames[idx]);
distributeVertex.AddArgument(v.dynamicManager.classNames[idx]);
distributeVertex.AddArgument(v.dynamicManager.methodNames[idx]);
idx++;
DrDynamicDistributionManager dynamicHashDistribute =
new DrDynamicDistributionManager(distributeVertex, dynamicMerge);
dynamicHashDistribute.SetDataPerVertex(highDataThreshold * 2); // 2GB
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, dynamicHashDistribute);
}
}
else if (v.dynamicManager.type == DynamicManager.Type.RANGEDISTRIBUTOR)
{
DrStageManager splitManager = graphStageMap[v.dynamicManager.splitVertexId].stageManager;
DrDynamicRangeDistributionManager drdm = new DrDynamicRangeDistributionManager(splitManager, v.dynamicManager.sampleRate);
drdm.SetDataPerVertex(highDataThreshold * 2); // 2GB
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, drdm);
}
else if (v.dynamicManager.type == DynamicManager.Type.BROADCAST)
{
// the copy vertex
int bcastNumber = 0;
string nameString = String.Format("CP__{0}", bcastNumber++);
DrManagerBase newStage = new DrManagerBase(app.GetGraph(), nameString);
DrActiveVertex copyVertex =
new DrActiveVertex(newStage,
parameters.m_defaultProcessTemplate,
parameters.m_defaultVertexTemplate);
copyVertex.AddArgument(cpyVertexName);
DrDynamicBroadcastManager bcast = new DrDynamicBroadcastManager(copyVertex);
newManager.AddDynamicConnectionManager(graphStageMap[v.info.predecessors[0].uniqueId].stageManager, bcast);
}
else if (v.dynamicManager.type != DynamicManager.Type.NONE)
{
DryadLogger.LogWarning("Build Graph From Query", "Dynamic manager type {0} not supported yet", v.dynamicManager.type);
}
}
}
//
// Add all the edges
//
DryadLogger.LogInformation("Build Graph From Query", "Adding edges");
foreach (KeyValuePair <int, GraphStageInfo> kvp in graphStageMap)
{
AddEdges(kvp.Value, graphStageMap);
}
//
// Register the actual created vertices with the graph
//
MaterializeToManagers(graphStageMap);
}
