/// <returns>a list containing a single split of summation</returns> public override IList <InputSplit> GetSplits(JobContext context) { //read sigma from conf Configuration conf = context.GetConfiguration(); Summation sigma = SummationWritable.Read(typeof(DistSum), conf); //create splits IList <InputSplit> splits = new AList <InputSplit>(1); splits.AddItem(new DistSum.Machine.SummationSplit(sigma)); return(splits); }
/// <summary>Create a job</summary> /// <exception cref="System.IO.IOException"/> private Job CreateJob(string name, Summation sigma) { Job job = Job.GetInstance(GetConf(), parameters.remoteDir + "/" + name); Configuration jobconf = job.GetConfiguration(); job.SetJarByClass(typeof(DistSum)); jobconf.SetInt(NParts, parameters.nParts); SummationWritable.Write(sigma, typeof(DistSum), jobconf); // disable task timeout jobconf.SetLong(MRJobConfig.TaskTimeout, 0); // do not use speculative execution jobconf.SetBoolean(MRJobConfig.MapSpeculative, false); jobconf.SetBoolean(MRJobConfig.ReduceSpeculative, false); return(job); }
/// <summary>Partitions the summation into parts and then return them as splits</summary> public override IList <InputSplit> GetSplits(JobContext context) { //read sigma from conf Configuration conf = context.GetConfiguration(); Summation sigma = SummationWritable.Read(typeof(DistSum), conf); int nParts = conf.GetInt(NParts, 0); //create splits IList <InputSplit> splits = new AList <InputSplit>(nParts); Summation[] parts = sigma.Partition(nParts); for (int i = 0; i < parts.Length; ++i) { splits.AddItem(new DistSum.Machine.SummationSplit(parts[i])); } //LOG.info("parts[" + i + "] = " + parts[i]); return(splits); }
/// <summary> /// <inheritDoc/> /// /// </summary> /// <exception cref="System.IO.IOException"/> public void ReadFields(DataInput @in) { sigma = SummationWritable.Read(@in); }
/// <summary> /// <inheritDoc/> /// /// </summary> /// <exception cref="System.IO.IOException"/> public virtual void ReadFields(DataInput @in) { sigma = SummationWritable.Read(@in); duration = @in.ReadLong(); }
/// <summary> /// <inheritDoc/> /// /// </summary> /// <exception cref="System.IO.IOException"/> public virtual void Write(DataOutput @out) { SummationWritable.Write(sigma, @out); @out.WriteLong(duration); }