/// <summary>
/// Use the input splits to take samples of the input and generate sample
/// keys.
/// </summary>
/// <remarks>
/// Use the input splits to take samples of the input and generate sample
/// keys. By default reads 100,000 keys from 10 locations in the input, sorts
/// them and picks N-1 keys to generate N equally sized partitions.
/// </remarks>
/// <param name="job">the job to sample</param>
/// <param name="partFile">where to write the output file to</param>
/// <exception cref="System.Exception">if something goes wrong</exception>
public static void WritePartitionFile(JobContext job, Path partFile)
{
long t1 = Runtime.CurrentTimeMillis();
Configuration conf = job.GetConfiguration();
TeraInputFormat inFormat = new TeraInputFormat();
TeraInputFormat.TextSampler sampler = new TeraInputFormat.TextSampler();
int partitions = job.GetNumReduceTasks();
long sampleSize = conf.GetLong(SampleSize, 100000);
IList <InputSplit> splits = inFormat.GetSplits(job);
long t2 = Runtime.CurrentTimeMillis();
System.Console.Out.WriteLine("Computing input splits took " + (t2 - t1) + "ms");
int samples = Math.Min(conf.GetInt(NumPartitions, 10), splits.Count);
System.Console.Out.WriteLine("Sampling " + samples + " splits of " + splits.Count
);
long recordsPerSample = sampleSize / samples;
int sampleStep = splits.Count / samples;
Sharpen.Thread[] samplerReader = new Sharpen.Thread[samples];
TeraInputFormat.SamplerThreadGroup threadGroup = new TeraInputFormat.SamplerThreadGroup
("Sampler Reader Thread Group");
// take N samples from different parts of the input
for (int i = 0; i < samples; ++i)
{
int idx = i;
samplerReader[i] = new _Thread_140(job, inFormat, splits, sampleStep, idx, sampler
, recordsPerSample, threadGroup, "Sampler Reader " + idx);
samplerReader[i].Start();
}
FileSystem outFs = partFile.GetFileSystem(conf);
DataOutputStream writer = outFs.Create(partFile, true, 64 * 1024, (short)10, outFs
.GetDefaultBlockSize(partFile));
for (int i_1 = 0; i_1 < samples; i_1++)
{
try
{
samplerReader[i_1].Join();
if (threadGroup.GetThrowable() != null)
{
throw threadGroup.GetThrowable();
}
}
catch (Exception)
{
}
}
foreach (Text split in sampler.CreatePartitions(partitions))
{
split.Write(writer);
}
writer.Close();
long t3 = Runtime.CurrentTimeMillis();
System.Console.Out.WriteLine("Computing parititions took " + (t3 - t2) + "ms");
}
public _Thread_140(JobContext job, TeraInputFormat inFormat, IList <InputSplit> splits
, int sampleStep, int idx, TeraInputFormat.TextSampler sampler, long recordsPerSample
, ThreadGroup baseArg1, string baseArg2)
: base(baseArg1, baseArg2)
{
this.job = job;
this.inFormat = inFormat;
this.splits = splits;
this.sampleStep = sampleStep;
this.idx = idx;
this.sampler = sampler;
this.recordsPerSample = recordsPerSample;
{
this.SetDaemon(true);
}
}
