public void CanSerializeValidStreamingMapReduceJobRequest_WithFiles()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = Constants.WabsProtocolSchemeName + "input",
Output = Constants.WabsProtocolSchemeName + "output",
Mapper = Constants.WabsProtocolSchemeName + "mapper",
Reducer = Constants.WabsProtocolSchemeName + "reducer"
};
var resourceFile1 = "asv://container@hostname/myfile1";
var resourceFile2 = "asv://container@hostname/myfile2";
var payloadConverter = new PayloadConverterBase();
streamingMapReduceJob.Files.Add(resourceFile1);
streamingMapReduceJob.Files.Add(resourceFile2);
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile1)));
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile2)));
}
public void CanSerializeValidStreamingMapReduceJobRequest_Defines()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = "asv://input",
Output = "asv://output",
Mapper = "asv://mapper",
Reducer = "asv://reducer",
};
streamingMapReduceJob.Defines.Add(new KeyValuePair <string, string>("definekey1", "definevalue1"));
streamingMapReduceJob.Defines.Add(new KeyValuePair <string, string>("definekey2", "definevalue2"));
streamingMapReduceJob.Defines.Add(new KeyValuePair <string, string>("definekey3", "definevalue3"));
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
foreach (var define in streamingMapReduceJob.Defines)
{
Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", define.Key, define.Value))));
}
}
public void CanSerializeValidStreamingMapReduceJobRequest_CmdEnv()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = "asv://input",
Output = "asv://output",
Mapper = "asv://mapper",
Reducer = "asv://reducer"
};
streamingMapReduceJob.CommandEnvironment.Add("Name1=Value1");
streamingMapReduceJob.CommandEnvironment.Add("Name2=Value2");
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
foreach (var cmdEnvArgument in streamingMapReduceJob.CommandEnvironment)
{
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.CmdEnv, Uri.EscapeDataString(cmdEnvArgument))));
}
}
public void CanSerializeValidStreamingMapReduceJobRequest_WithNoReducer()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = Constants.WabsProtocolSchemeName + "input",
Output = Constants.WabsProtocolSchemeName + "output",
Mapper = Constants.WabsProtocolSchemeName + "mapper"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
}
public void CanSerializeValidStreamingMapReduceJobRequest()
{
var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
{
Input = "asv://input",
Output = "asv://output",
Mapper = "asv://mapper",
Reducer = "asv://reducer",
Combiner = "asv://combiner"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Combiner, Uri.EscapeDataString(streamingMapReduceJob.Combiner))));
}