//
// Main Dryad LINQ execution stuff
//
public int ExecLinqToDryad(Uri dfsDirectory, string[] args, out string errorString)
{
//
// must be at least two arguments (program name and query XML file name)
//
if (args.Length < 2)
{
errorString = "Must provide at least query XML file name and VertexHost executable.";
DryadLogger.LogCritical(errorString);
return -1;
}
//
// break if --break is included in arguments (and eliminate it, as it is not known downstream)
//
if (ConsumeSingleArgument("--break", ref args))
{
DebugHelper.WaitForDebugger();
}
// this is where we ensure the right type of scheduler is registered
Microsoft.Research.Dryad.LocalScheduler.Registration.Ensure();
//
// parse the XML input, producing a DryadLINQ Query
//
Query query = new Query();
QueryPlanParser parser = new QueryPlanParser();
if (!parser.ParseQueryXml(args[1], query))
{
errorString = "Invalid query plan";
DryadLogger.LogCritical(errorString);
return -1;
}
//
// upload the query plan to the job DFS directory for the benefit of debug tools
//
if (dfsDirectory != null)
{
DebugHelper.UploadToDfs(dfsDirectory, args[1]);
}
//
// build internal app arguments
//
List<string> internalArgs = new List<string>();
//
// add the XmlExecHost args to the internal app arguments
//
foreach (string xmlExecHostArg in query.xmlExecHostArgs)
{
if (xmlExecHostArg == "--break")
{
DebugHelper.WaitForDebugger();
}
else
{
internalArgs.Add(xmlExecHostArg);
}
}
//
// combine internal arguments with any additional arguments received on the command line
// don't include argv[0] and argv[1] (program name and query XML file name)
//
int internalArgc = (int)internalArgs.Count;
int externalArgc = args.Length - 2; // don't include argv[0] and argv[1]
int combinedArgc = internalArgc + externalArgc;
string[] combinedArgv = new string[combinedArgc];
string msg = "";
// internal arguments first
for (int i=0; i<internalArgc; i++)
{
combinedArgv[i] = internalArgs[i];
msg += String.Format("{0} ", combinedArgv[i]);
}
// then external arguments
for (int i = 0; i<externalArgc; i++)
{
combinedArgv[i+internalArgc] = args[i+2]; // don't include argv[0] and argv[1]
msg += String.Format("{0} ", combinedArgv[i+internalArgc]);
}
DryadLogger.LogInformation(String.Format("Arguments: {0}", msg));
string jobClass = "DryadLINQ";
string exeName = args[0];
// create app and run it
//
DrGraphParameters p = DrDefaultParameters.Make(exeName, jobClass, query.enableSpeculativeDuplication);
foreach (DrIReporter reporter in DebugHelper.Reporters())
{
p.m_reporters.Add(reporter);
}
p.m_intermediateCompressionMode = query.intermediateDataCompression;
DrGraphExecutor graphExecutor = new DrGraphExecutor();
DrGraph graph = graphExecutor.Initialize(p);
if (graph == null)
{
errorString = "Failed to initialize Graph Executor";
DryadLogger.LogCritical(errorString);
return -1;
}
DryadLINQApp app = new DryadLINQApp(graph);
// Initialize with arguments
app.SetXmlFileName(args[1]);
if (!app.ParseCommandLineFlags(combinedArgv))
{
errorString = "Bad command-line options";
DryadLogger.LogCritical(errorString);
return -1;
}
// Build graph from query plan
GraphBuilder builder = new GraphBuilder();
builder.BuildGraphFromQuery(app, query);
// Run the app
DryadLogger.LogInformation("Running the app");
graphExecutor.Run();
DrError exitStatus = graphExecutor.Join();
DryadLogger.LogInformation("Finished running the app");
if (exitStatus == null || exitStatus.m_code == 0)
{
FinalizeExecution(query, graph);
DryadLogger.LogInformation("Application completed successfully.");
errorString = null;
return 0;
}
else
{
DryadLogger.LogCritical(String.Format("Application failed with error code 0x{0:X8}.\n", exitStatus.m_code));
errorString = exitStatus.ToFullTextNative();
return exitStatus.m_code;
}
}
public bool ParseQueryXml(string queryPlanFileName, Query query)
{
XmlNode version = null;
XmlDocument queryPlanDoc = new XmlDocument();
//
// Load query plan document
//
try
{
queryPlanDoc.Load(queryPlanFileName);
}
catch (Exception e)
{
DryadLogger.LogCritical(String.Format("Failed to load query plan: {0}: {1}", queryPlanFileName, e.ToString()));
return false;
}
//
// Get DryadLinqVersion - absence used to indicate Argentia query plan
//
try
{
version = queryPlanDoc.DocumentElement.SelectSingleNode("DryadLinqVersion");
}
catch (System.Xml.XPath.XPathException e)
{
DryadLogger.LogCritical(String.Format("Failed to select node DryadLinqVersion from query plan: {0}: {1}", queryPlanFileName, e.ToString()));
return false;
}
if (version == null)
{
DryadLogger.LogCritical(String.Format("Missing element 'DryadLinqVersion' in query plan: {0}", queryPlanFileName));
return false;
}
//
// Parse query plan XML doc into Query
//
try
{
ParseQueryXmlLinqToDryad(queryPlanDoc, query);
}
catch (Exception e)
{
DryadLogger.LogCritical(String.Format("Failed to parse query plan: {0}: {1}", queryPlanFileName, e.ToString()));
return false;
}
#if REMOVE_ARGENTIA
else // If (version == null), Argentia query plan
{
// Add the namespace.
XmlNamespaceManager nsmgr = new XmlNamespaceManager(queryPlanDoc.NameTable);
nsmgr.AddNamespace("qp", "http://microsoft.com/PCP/Argentia/QueryPlan.xsd");
version = queryPlanDoc.DocumentElement.SelectSingleNode("qp:RuntimeVersion", nsmgr);
if (version != null)
{
ParseQueryXmlArgentia(queryPlanDoc, query);
}
else
{
DryadLogger.LogCritical(0, null, "Unknown query plan format.");
return false;
}
}
#endif
return true;
}
internal void FinalizeExecution(Query query, DrGraph graph)
{
SortedDictionary<int, Vertex> queryPlan = query.queryPlan;
foreach (KeyValuePair<int, Vertex> kvp in query.queryPlan)
{
/* used to do CSStream expiration time stuff here */
}
}
private void ParseQueryXmlLinqToDryad(XmlDocument queryPlanDoc, Query query)
{
XmlElement root = queryPlanDoc.DocumentElement;
//
// Query globals
//
query.queryPlan = new SortedDictionary<int, Vertex>();
query.compilerversion = root.SelectSingleNode("DryadLinqVersion").InnerText;
query.clusterName = root.SelectSingleNode("ClusterName").InnerText;
query.visualization = root.SelectSingleNode("Visualization").InnerText;
// Compression scheme for intermediate data
XmlNode compressionNode = root.SelectSingleNode("IntermediateDataCompression");
if (compressionNode != null)
{
query.intermediateDataCompression = Convert.ToInt32(compressionNode.InnerText);
}
//
// XmlExecHost arguments
//
XmlNodeList nodes = root.SelectSingleNode("XmlExecHostArgs").ChildNodes;
query.xmlExecHostArgs = new string[nodes.Count];
for (int index=0; index<nodes.Count; index++)
{
query.xmlExecHostArgs[index] = nodes[index].InnerText;
}
//
// Get Speculative duplication flag - default is enabled (true)
//
XmlNode duplicationNode = root.SelectSingleNode("EnableSpeculativeDuplication");
if (duplicationNode != null)
{
bool dupFlag;
if (bool.TryParse(duplicationNode.InnerText, out dupFlag))
{
query.enableSpeculativeDuplication = dupFlag;
}
}
nodes = root.SelectSingleNode("QueryPlan").ChildNodes;
//
// Need to remember the conection operator for use when the
// predecessors are being created.
//
string[] vertexConnectionOperator = new string[nodes.Count];
for (int index=0; index<nodes.Count; index++)
{
vertexConnectionOperator[index] = "";
}
for (int index=0; index<nodes.Count; index++)
{
Vertex vertex = new Vertex();
//
// Vertex globals
//
string uniqueId = nodes[index].SelectSingleNode("UniqueId").InnerText;
string vertexType = nodes[index].SelectSingleNode("Type").InnerText;
string name = nodes[index].SelectSingleNode("Name").InnerText;
string partitions = nodes[index].SelectSingleNode("Partitions").InnerText;
string channelType = nodes[index].SelectSingleNode("ChannelType").InnerText;
//
// Need to remember the conection operator for use when the
// predecessors are being created.
//
vertexConnectionOperator[index] = nodes[index].SelectSingleNode("ConnectionOperator").InnerText;
vertex.uniqueId = Convert.ToInt32(uniqueId);
vertex.name = name;
vertex.type = GetVertexType(vertexType);
vertex.partitions = Convert.ToInt32(partitions);
XmlNode dynamicManager = nodes[index].SelectSingleNode("DynamicManager");
string dmType = dynamicManager.SelectSingleNode("Type").InnerText;
vertex.dynamicManager = new DynamicManager();
vertex.dynamicManager.type = GetDynamicManagerType(dmType);
if (vertex.dynamicManager.type == DynamicManager.Type.FULLAGGR)
{
string levels = dynamicManager.SelectSingleNode("AggregationLevels").InnerText;
vertex.dynamicManager.aggregationLevels = Convert.ToInt32(levels);
}
if (vertex.dynamicManager.type == DynamicManager.Type.RANGEDISTRIBUTOR)
{
string sampleRate = dynamicManager.SelectSingleNode("SampleRate").InnerText;
string vertexId = dynamicManager.SelectSingleNode("VertexId").InnerText;
vertex.dynamicManager.sampleRate = Convert.ToDouble(sampleRate);
vertex.dynamicManager.splitVertexId = Convert.ToInt32(vertexId);
}
else
{
XmlNodeList entries = dynamicManager.SelectNodes("Entry");
vertex.dynamicManager.assemblyNames = new string[entries.Count];
vertex.dynamicManager.classNames = new string[entries.Count];
vertex.dynamicManager.methodNames = new string[entries.Count];
for (int entryIndex = 0; entryIndex < entries.Count; entryIndex++)
{
vertex.dynamicManager.assemblyNames[entryIndex] = entries[entryIndex].SelectSingleNode("AssemblyName").InnerText;
vertex.dynamicManager.classNames[entryIndex] = entries[entryIndex].SelectSingleNode("ClassName").InnerText;
vertex.dynamicManager.methodNames[entryIndex] = entries[entryIndex].SelectSingleNode("MethodName").InnerText;
}
}
if (vertex.type == Vertex.Type.INPUTTABLE)
{
XmlNode storageSet = nodes[index].SelectSingleNode("StorageSet");
string ioType = storageSet.SelectSingleNode("Type").InnerText;
vertex.info = new VertexInfo();
vertex.info.ioType = GetIoType(ioType);
XmlNodeList storageUris = storageSet.SelectNodes("SourceURI");
vertex.info.sources = new string[storageUris.Count];
for (int indexStorageUri=0; indexStorageUri<storageUris.Count; indexStorageUri++)
{
vertex.info.sources[indexStorageUri] = storageUris[indexStorageUri].InnerText;
}
}
else if ( vertex.type == Vertex.Type.OUTPUTTABLE )
{
XmlNode storageSet = nodes[index].SelectSingleNode("StorageSet");
string ioType = storageSet.SelectSingleNode("Type").InnerText;
vertex.info = new VertexInfo();
vertex.info.ioType = GetIoType(ioType);
string source = storageSet.SelectSingleNode("SinkURI").InnerText;
vertex.info.sources = new string[1] { source };
if (vertex.info.ioType == VertexInfo.IOType.PARTITIONEDFILE )
{
vertex.info.partitionUncPath = storageSet.SelectSingleNode("PartitionUncPath").InnerText;
}
XmlNode temporary = storageSet.SelectSingleNode("IsTemporary");
if (temporary != null)
{
if (bool.TryParse(temporary.InnerXml, out vertex.info.isTemporary) == false)
{
throw new LinqToDryadException(String.Format("Invalid value for IsTemporary: {0}", temporary.InnerXml));
}
}
else
{
vertex.info.isTemporary = false;
}
XmlNode recordType = storageSet.SelectSingleNode("RecordType");
if (recordType != null)
{
vertex.info.recordType = recordType.InnerXml;
}
XmlNode outputCompressionScheme = storageSet.SelectSingleNode("OutputCompressionScheme");
if (outputCompressionScheme != null)
{
if (int.TryParse(outputCompressionScheme.InnerXml, out vertex.info.compressionScheme) == false)
{
throw new LinqToDryadException(String.Format("Invalid value for OutputCompressionScheme: {0}", outputCompressionScheme.InnerXml));
}
}
else
{
vertex.info.compressionScheme = 0; // TODO: Change to Enum
}
}
else /* JOIN etc. */
{
XmlNode entry = nodes[index].SelectSingleNode("Entry");
vertex.info = new VertexInfo();
vertex.info.assemblyName = entry.SelectSingleNode("AssemblyName").InnerText;
vertex.info.className = entry.SelectSingleNode("ClassName").InnerText;
vertex.info.methodName = entry.SelectSingleNode("MethodName").InnerText;
}
//
// everybody except inputs have children
//
if (vertex.type != Vertex.Type.INPUTTABLE)
{
XmlNodeList children = nodes[index].SelectSingleNode("Children").ChildNodes;
vertex.info.predecessors = new Predecessor[children.Count];
for (int indexChild = 0; indexChild < children.Count; indexChild++)
{
int childId = Convert.ToInt32(children[indexChild].SelectSingleNode("UniqueId").InnerText);
string childConstraint = children[indexChild].SelectSingleNode("AffinityConstraint").InnerText;
vertex.info.predecessors[indexChild] = new Predecessor();
vertex.info.predecessors[indexChild].uniqueId = childId;
vertex.info.predecessors[indexChild].connectionOperator = GetConnectionOperator(vertexConnectionOperator[childId]);
vertex.info.predecessors[indexChild].channelType = GetChannelType(channelType);
vertex.info.predecessors[indexChild].constraint = GetAffinityConstraint(childConstraint);
}
}
//
// In this parser the only way to have an optional Tag is immediately before consuming
// a close of an outer tag. So we have to look for machines tag here.
//
if ((vertex.type != Vertex.Type.INPUTTABLE) && (vertex.type != Vertex.Type.OUTPUTTABLE))
{
XmlNode machinesRoot = nodes[index].SelectSingleNode("Machines");
if (machinesRoot != null)
{
XmlNodeList machines = machinesRoot.ChildNodes;
vertex.machines = new string[machines.Count];
for (int indexMachine = 0; indexMachine < machines.Count; indexMachine++)
{
vertex.machines[indexMachine] = machines[indexMachine].SelectSingleNode("Machine").InnerText;
}
}
}
//
// Add the vertex
//
query.queryPlan.Add(vertex.uniqueId, vertex);
}
}