public override void Execute( Job job, Action<string> updateStatus, IRepository repositoryService, IJobManager jobManager) { foreach(var file in job.Files.OfType<CreateFile>()) { updateStatus(string.Format("Downloading file: \"{0}\"...", file.Name)); using(var stream = repositoryService.Download(file.Link.Id)) { using(var fstream = new FileStream( Path.Combine(Directory.GetCurrentDirectory(), string.Format("{0}.data", file.Name)), FileMode.Create)) { var buff = new byte[256]; while(true) { var isCompleted = stream.Read(buff, 0, buff.Length) < buff.Length; fstream.Write(buff, 0, buff.Length); if(isCompleted) break; } } } } }
public Shared.Japi.Job CreateJob() { var job = new Job { Name = "MapReduce" }; var assembly = new UploadAssembly(typeof(Map).Assembly); // map Console.WriteLine("Generating \"map\" tasks.."); var fstLayer = new List<Task>(); var n = 0; foreach (var path in pathes) { fstLayer.Add(new Task { EntryPoint = new EntryPoint { Assembly = assembly, ClassName = typeof(Map).FullName }, InputFiles = new List<File> { new UploadFile(path) }, Name = string.Format("MapReduce.Map({0})", n), Parameters = new Dictionary<string, string> { { "MapReduce.MapIndex", n.ToString() }, { "MapReduce.ItemsCount", takeTop.ToString() } }, OutputFiles = Sequence.Items(new CreateFile(string.Format("PartialResult_{0}", n))) .Cast<File>() .ToList() }); n++; } // reduce Console.WriteLine("Generating \"reduce\" tasks..."); var resultTask = new Task { EntryPoint = new EntryPoint { Assembly = assembly, ClassName = typeof(Reduce).FullName }, InputFiles = (from t in fstLayer from f in t.OutputFiles select f) .ToList(), Name = "MapReduce.Reduce()", Parameters = new Dictionary<string, string> { { "MapReduce.ItemsCount", takeTop.ToString() } }, OutputFiles = new List<File> { new CreateFile("Result") }, Dependencies = fstLayer }; job.Tasks.AddRange(fstLayer); job.Tasks.Add(resultTask); return job; }
public Job CreateJob() { var job = new Job { Name = "DF.SDK.ComputeWordFrequencies" }; var assembly = new UploadAssembly(typeof(BuildWordsList).Assembly); // 1st layer: Console.WriteLine("Generating 1st layer tasks..."); var fstLayer = new List<Task>(); var n = 0; foreach(var path in pathes) { fstLayer.Add(new Task { EntryPoint = new EntryPoint { Assembly = assembly, ClassName = typeof(BuildWordsList).FullName }, InputFiles = new List<File> { new UploadFile(path) }, Name = string.Format(typeof(BuildWordsList).FullName + "({0})", n), Parameters = new Dictionary<string, string> { { "Layer.NextSize", nodes.ToString() }, {"Node", n.ToString() } }, OutputFiles = (from target in Enumerable.Range(0, nodes) let name = string.Format("HashDistribute_{0}_to_{1})", n, target) select new CreateFile(name) { Metadata = new Dictionary<string, string> { {"Node",target.ToString()} } }) .Cast<File>() .ToList() }); n++; } // 2nd layer Console.WriteLine("Generating 2nd layer tasks..."); var sndLayer = new List<Task>(); for(n = 0; n < nodes; n++) { sndLayer.Add(new Task { EntryPoint = new EntryPoint { Assembly = assembly, ClassName = typeof(TakeTopList).FullName }, InputFiles = (from t in fstLayer from f in t.OutputFiles where int.Parse(f.Metadata["Node"]) == n select f) .ToList(), Name = string.Format(typeof(TakeTopList).FullName + "({0})", n), Parameters = new Dictionary<string, string> { { "TakeTop", takeTop.ToString() } }, OutputFiles = new List<File> { new CreateFile(string.Format("TakeTopList_{0}", n)) }, Dependencies = fstLayer }); } // 3rd layer Console.WriteLine("Generating 3rd layer tasks..."); var resultTask = new Task { EntryPoint = new EntryPoint { Assembly = assembly, ClassName = typeof(TakeTopList).FullName }, InputFiles = (from t in sndLayer from f in t.OutputFiles select f) .ToList(), Name = typeof(TakeTopList).FullName + "(last)", Parameters = new Dictionary<string, string> { { "TakeTop", takeTop.ToString() } }, OutputFiles = new List<File> { new CreateFile("Result") }, Dependencies = sndLayer }; job.Tasks.AddRange(fstLayer); job.Tasks.AddRange(sndLayer); job.Tasks.Add(resultTask); return job; }