/// <exception cref="System.IO.IOException"/>
public static MRCaching.TestResult LaunchMRCache(string indir, string outdir, string
cacheDir, JobConf conf, string input)
{
string TestRootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).ToString
().Replace(' ', '+');
//if (TEST_ROOT_DIR.startsWith("C:")) TEST_ROOT_DIR = "/tmp";
conf.Set("test.build.data", TestRootDir);
Path inDir = new Path(indir);
Path outDir = new Path(outdir);
FileSystem fs = FileSystem.Get(conf);
fs.Delete(outDir, true);
if (!fs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
{
System.Console.Out.WriteLine("HERE:" + inDir);
DataOutputStream file = fs.Create(new Path(inDir, "part-0"));
file.WriteBytes(input);
file.Close();
}
conf.SetJobName("cachetest");
// the keys are words (strings)
conf.SetOutputKeyClass(typeof(Text));
// the values are counts (ints)
conf.SetOutputValueClass(typeof(IntWritable));
conf.SetCombinerClass(typeof(MRCaching.ReduceClass));
conf.SetReducerClass(typeof(MRCaching.ReduceClass));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(1);
conf.SetNumReduceTasks(1);
conf.SetSpeculativeExecution(false);
URI[] uris = new URI[6];
conf.SetMapperClass(typeof(MRCaching.MapClass2));
uris[0] = fs.GetUri().Resolve(cacheDir + "/test.txt");
uris[1] = fs.GetUri().Resolve(cacheDir + "/test.jar");
uris[2] = fs.GetUri().Resolve(cacheDir + "/test.zip");
uris[3] = fs.GetUri().Resolve(cacheDir + "/test.tgz");
uris[4] = fs.GetUri().Resolve(cacheDir + "/test.tar.gz");
uris[5] = fs.GetUri().Resolve(cacheDir + "/test.tar");
DistributedCache.AddCacheFile(uris[0], conf);
// Save expected file sizes
long[] fileSizes = new long[1];
fileSizes[0] = fs.GetFileStatus(new Path(uris[0].GetPath())).GetLen();
long[] archiveSizes = new long[5];
// track last 5
for (int i = 1; i < 6; i++)
{
DistributedCache.AddCacheArchive(uris[i], conf);
archiveSizes[i - 1] = fs.GetFileStatus(new Path(uris[i].GetPath())).GetLen();
}
// starting with second archive
RunningJob job = JobClient.RunJob(conf);
int count = 0;
// after the job ran check to see if the input from the localized cache
// match the real string. check if there are 3 instances or not.
Path result = new Path(TestRootDir + "/test.txt");
{
BufferedReader file = new BufferedReader(new InputStreamReader(FileSystem.GetLocal
(conf).Open(result)));
string line = file.ReadLine();
while (line != null)
{
if (!testStr.Equals(line))
{
return(new MRCaching.TestResult(job, false));
}
count++;
line = file.ReadLine();
}
file.Close();
}
if (count != 6)
{
return(new MRCaching.TestResult(job, false));
}
// Check to ensure the filesizes of files in DC were correctly saved.
// Note, the underlying job clones the original conf before determine
// various stats (timestamps etc.), so we have to getConfiguration here.
ValidateCacheFileSizes(job.GetConfiguration(), fileSizes, MRJobConfig.CacheFilesSizes
);
ValidateCacheFileSizes(job.GetConfiguration(), archiveSizes, MRJobConfig.CacheArchivesSizes
);
return(new MRCaching.TestResult(job, true));
}