public ClusterJob Submit(XDocument config, Uri errorLocation) { // upload config xml doc to blob // launch single task map only job to submit xonfig xml to yarn launcher // get job output which will be the application id Uri userDfs = this.DfsClient.Combine(this.baseUri, "user", Environment.UserName, "staging"); Guid jobGuid = Guid.NewGuid(); StreamingMapReduceJobCreateParameters jobParams = new StreamingMapReduceJobCreateParameters(); jobParams.Defines.Add("mapreduce.job.reduces", "0"); jobParams.Defines.Add("mapreduce.job.maps", "1"); jobParams.JobName = "YarnMapLauncher"; jobParams.Mapper = "Microsoft.Research.Peloponnese.YarnMapLauncher.exe"; Uri outputDir = this.DfsClient.Combine(userDfs, jobGuid.ToString()); //dfsClient removes the leading slash, but hadoop assumes job output is relative to user working directory jobParams.Output = this.dfsClient.GetClusterInternalUri(outputDir).AbsoluteUri; using (MemoryStream upload = new MemoryStream()) { XmlWriterSettings settings = new XmlWriterSettings(); settings.Encoding = new UTF8Encoding(false); using (XmlWriter xw = XmlWriter.Create(upload, settings)) { config.WriteTo(xw); } XElement elem = ConfigHelpers.MakeDfsResourceFromBuffer(jobGuid + ".xml", upload.ToArray(), this.DfsClient, userDfs); jobParams.Input = elem.Value; } Uri peloponneseDfs = this.DfsClient.Combine(this.baseUri, "staging", "peloponnese"); string[] files = { jobParams.Mapper, "Microsoft.Research.Peloponnese.YarnLauncher.jar" }; foreach (string file in files) { XElement elem = ConfigHelpers.MakeDfsResourceFromFile(Path.Combine(peloponneseHome, file), this.DfsClient, peloponneseDfs); // hash mark in resources allows a target name to be specified for the symlink jobParams.Files.Add(string.Format("{0}#{1}", elem.Value, file)); } var launchJob = JobClient.CreateStreamingJob(jobParams); return(new AzureYarnJob(this, errorLocation, outputDir, launchJob)); }