public async void MapReduceStreamingJobTest()
{
MapReduceStreamingJob job = await hManager.MapReduceStreamingJob(
"/tmp/test",
"/tmp/output/1",
"/bin/cat",
"/usr/bin/wc -w",
string.Empty,
string.Empty,
string.Empty,
string.Empty,
"/tmp/status",
string.Empty,
string.Empty
);
Console.WriteLine(job.ToString());
TableProperty createProperty = await hManager.CreateTableProperty("testing", "table1", "animal", new PropertyValue()
{
value = "cat"
});
if (string.IsNullOrEmpty(createProperty.error))
{
Console.WriteLine(string.Format("Column {0}.{1}.{2} has been created", createProperty.database, createProperty.table, "animal"));
}
else
{
Console.WriteLine(((Error)createProperty).ToString());
}
}
/// <summary>
/// Create and queue a Hadoop streaming MapReduce job.
/// </summary>
/// <returns>The reduce streaming job.</returns>
/// <param name="input">Location of the input data in Hadoop.</param>
/// <param name="output">Location in which to store the output data. If not specified, WebHCat will store the output in a location that can be discovered using the queue resource.</param>
/// <param name="mapper">Location of the mapper program in Hadoop.</param>
/// <param name="reducer">Location of the reducer program in Hadoop.</param>
/// <param name="file">Add an HDFS file to the distributed cache.</param>
/// <param name="define">Set a Hadoop configuration variable using the syntax define=NAME=VALUE.</param>
/// <param name="cmdenv">Set an environment variable using the syntax cmdenv=NAME=VALUE.</param>
/// <param name="arg">Set a program argument.</param>
/// <param name="statusDir">A directory where WebHCat will write the status of the Map Reduce job. If provided, it is the caller's responsibility to remove this directory when done.</param>
/// <param name="enableLog">If statusdir is set and enablelog is "true", collect Hadoop job configuration and logs into a directory named $statusdir/logs after the job finishes. Both completed and failed attempts are logged. The layout of subdirectories in $statusdir/logs is:
/// logs/$job_id(directory for $job_id)
/// logs/$job_id/job.xml.html
/// logs/$job_id/$attempt_id(directory for $attempt_id)
/// logs/$job_id/$attempt_id/stderr
/// logs/$job_id/$attempt_id/stdout
/// logs/$job_id/$attempt_id/syslog</param>
/// <param name="callBack">Define a URL to be called upon job completion. You may embed a specific job ID into this URL using $jobId. This tag will be replaced in the callback URL with this job's job ID.</param>
public async Task <MapReduceStreamingJob> MapReduceStreamingJob(string input, string output, string mapper, string reducer, string file, string define, string cmdenv, string arg, string statusDir, string enableLog, string callBack)
{
List <KeyValuePair <string, string> > postParams = new List <KeyValuePair <string, string> >();
if (!string.IsNullOrEmpty(input))
{
postParams.Add(new KeyValuePair <string, string>("input", input));
}
if (!string.IsNullOrEmpty(output))
{
postParams.Add(new KeyValuePair <string, string>("output", output));
}
if (!string.IsNullOrEmpty(mapper))
{
postParams.Add(new KeyValuePair <string, string>("mapper", mapper));
}
if (!string.IsNullOrEmpty(reducer))
{
postParams.Add(new KeyValuePair <string, string>("reducer", reducer));
}
if (!string.IsNullOrEmpty(file))
{
postParams.Add(new KeyValuePair <string, string>("file", file));
}
if (!string.IsNullOrEmpty(define))
{
postParams.Add(new KeyValuePair <string, string>("define", define));
}
if (!string.IsNullOrEmpty(cmdenv))
{
postParams.Add(new KeyValuePair <string, string>("cmdenv", cmdenv));
}
if (!string.IsNullOrEmpty(arg))
{
postParams.Add(new KeyValuePair <string, string>("arg", arg));
}
if (!string.IsNullOrEmpty(statusDir))
{
postParams.Add(new KeyValuePair <string, string>("statusDir", statusDir));
}
if (!string.IsNullOrEmpty(enableLog))
{
postParams.Add(new KeyValuePair <string, string>("enableLog", enableLog));
}
if (!string.IsNullOrEmpty(callBack))
{
postParams.Add(new KeyValuePair <string, string>("callBack", callBack));
}
MapReduceStreamingJob tableRename = await Post <MapReduceStreamingJob>(_webHcatBaseUrl, _webHCatVersion, _webHCatUserName, requestURL.MapReduceStreamingJob(), postParams);
return(tableRename);
}