Ejemplo n.º 1
0
 public override void Execute(
     Job job, 
     Action<string> updateStatus,
     IRepository repositoryService, 
     IJobManager jobManager)
 {
     foreach(var file in job.Files.OfType<CreateFile>())
     {
         updateStatus(string.Format("Downloading file: \"{0}\"...", file.Name));
         using(var stream = repositoryService.Download(file.Link.Id))
         {
             using(var fstream = new FileStream(
                 Path.Combine(Directory.GetCurrentDirectory(), string.Format("{0}.data", file.Name)),
                 FileMode.Create))
             {
                 var buff = new byte[256];
                 while(true)
                 {
                     var isCompleted = stream.Read(buff, 0, buff.Length) < buff.Length;
                     fstream.Write(buff, 0, buff.Length);
                     if(isCompleted)
                         break;
                 }
             }
         }
     }
 }
Ejemplo n.º 2
0
        public Shared.Japi.Job CreateJob()
        {
            var job = new Job { Name = "MapReduce" };

            var assembly = new UploadAssembly(typeof(Map).Assembly);

            // map
            Console.WriteLine("Generating \"map\" tasks..");
            var fstLayer = new List<Task>();
            var n = 0;
            foreach (var path in pathes)
            {
                fstLayer.Add(new Task
                {
                    EntryPoint = new EntryPoint
                    {
                        Assembly = assembly,
                        ClassName = typeof(Map).FullName
                    },
                    InputFiles = new List<File>
                                                      {
                                                          new UploadFile(path)
                                                      },
                    Name = string.Format("MapReduce.Map({0})", n),
                    Parameters = new Dictionary<string, string>
                    {
                        { "MapReduce.MapIndex", n.ToString() },
                        { "MapReduce.ItemsCount", takeTop.ToString() }
                    },
                    OutputFiles = Sequence.Items(new CreateFile(string.Format("PartialResult_{0}", n)))
                        .Cast<File>()
                        .ToList()
                });
                n++;
            }

            // reduce
            Console.WriteLine("Generating \"reduce\" tasks...");
            var resultTask = new Task
            {
                EntryPoint = new EntryPoint
                {
                    Assembly = assembly,
                    ClassName = typeof(Reduce).FullName
                },
                InputFiles = (from t in fstLayer
                              from f in t.OutputFiles
                              select f)
                    .ToList(),
                Name = "MapReduce.Reduce()",
                Parameters = new Dictionary<string, string> { { "MapReduce.ItemsCount", takeTop.ToString() } },
                OutputFiles = new List<File> { new CreateFile("Result") },
                Dependencies = fstLayer
            };

            job.Tasks.AddRange(fstLayer);
            job.Tasks.Add(resultTask);

            return job;
        }
Ejemplo n.º 3
0
        public Job CreateJob()
        {
            var job = new Job { Name = "DF.SDK.ComputeWordFrequencies" };

            var assembly = new UploadAssembly(typeof(BuildWordsList).Assembly);

            // 1st layer:
            Console.WriteLine("Generating 1st layer tasks...");
            var fstLayer = new List<Task>();
            var n = 0;
            foreach(var path in pathes)
            {
                fstLayer.Add(new Task
                                 {
                                     EntryPoint = new EntryPoint
                                                      {
                                                          Assembly = assembly,
                                                          ClassName = typeof(BuildWordsList).FullName
                                                      },
                                     InputFiles = new List<File>
                                                      {
                                                          new UploadFile(path)
                                                      },
                                     Name = string.Format(typeof(BuildWordsList).FullName + "({0})", n),
                                     Parameters = new Dictionary<string, string> { { "Layer.NextSize", nodes.ToString() }, {"Node", n.ToString() } },
                                     OutputFiles = (from target in Enumerable.Range(0, nodes)
                                                    let name = string.Format("HashDistribute_{0}_to_{1})", n, target)
                                                    select new CreateFile(name)
                                                               {
                                                                   Metadata = new Dictionary<string, string>
                                                                                  {
                                                                                      {"Node",target.ToString()}
                                                                                  }
                                                               })
                                         .Cast<File>()
                                         .ToList()
                                 });
                n++;
            }

            // 2nd layer
            Console.WriteLine("Generating 2nd layer tasks...");
            var sndLayer = new List<Task>();
            for(n = 0; n < nodes; n++)
            {
                sndLayer.Add(new Task
                                 {
                                     EntryPoint = new EntryPoint
                                                      {
                                                          Assembly = assembly,
                                                          ClassName = typeof(TakeTopList).FullName
                                                      },
                                     InputFiles = (from t in fstLayer
                                                   from f in t.OutputFiles
                                                   where int.Parse(f.Metadata["Node"]) == n
                                                   select f)
                                         .ToList(),
                                     Name = string.Format(typeof(TakeTopList).FullName + "({0})", n),
                                     Parameters = new Dictionary<string, string> { { "TakeTop", takeTop.ToString() } },
                                     OutputFiles =
                                         new List<File> { new CreateFile(string.Format("TakeTopList_{0}", n)) },
                                     Dependencies = fstLayer
                                 });
            }

            // 3rd layer
            Console.WriteLine("Generating 3rd layer tasks...");
            var resultTask = new Task
                                 {
                                     EntryPoint = new EntryPoint
                                                      {
                                                          Assembly = assembly,
                                                          ClassName = typeof(TakeTopList).FullName
                                                      },
                                     InputFiles = (from t in sndLayer
                                                   from f in t.OutputFiles
                                                   select f)
                                         .ToList(),
                                     Name = typeof(TakeTopList).FullName + "(last)",
                                     Parameters = new Dictionary<string, string> { { "TakeTop", takeTop.ToString() } },
                                     OutputFiles = new List<File> { new CreateFile("Result") },
                                     Dependencies = sndLayer
                                 };

            job.Tasks.AddRange(fstLayer);
            job.Tasks.AddRange(sndLayer);
            job.Tasks.Add(resultTask);

            return job;
        }