/// <summary>Test is key-field-helper's getWordLengths.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestGetWordLengths()
{
KeyFieldHelper helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
// test getWordLengths with unspecified key-specifications
string input = "hi";
int[] result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0,
2);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 1 }));
// set the key specs
helper.SetKeyFieldSpec(1, 2);
// test getWordLengths with 3 words
input = "hi\thello there";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, input
.Length);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 2, 2, 11 }));
// test getWordLengths with 4 words but with a different separator
helper.SetKeyFieldSeparator(" ");
input = "hi hello\tthere you";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, input
.Length);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 3, 2, 11, 3 }));
// test with non zero start index
input = "hi hello there you where me there";
// .....................
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 10, 33);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 5, 4, 3, 5, 2, 3 }));
input = "hi hello there you where me ";
// ..................
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 10, input
.Length);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 5, 4, 3, 5, 2, 0 }));
input = string.Empty;
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 0);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 1, 0 }));
input = " abc";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 5);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 3, 0, 0, 3 }));
input = " abc";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 2);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 3, 0, 0, 0 }));
input = " abc ";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 2);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 2, 0, 1 }));
helper.SetKeyFieldSeparator("abcd");
input = "abc";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 3);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 1, 3 }));
}
public virtual void SetConf(Configuration conf)
{
this.conf = conf;
keyFieldHelper = new KeyFieldHelper();
string keyFieldSeparator = conf.Get(MRJobConfig.MapOutputKeyFieldSeperator, "\t");
keyFieldHelper.SetKeyFieldSeparator(keyFieldSeparator);
if (conf.Get("num.key.fields.for.partition") != null)
{
Log.Warn("Using deprecated num.key.fields.for.partition. " + "Use mapreduce.partition.keypartitioner.options instead"
);
this.numOfPartitionFields = conf.GetInt("num.key.fields.for.partition", 0);
keyFieldHelper.SetKeyFieldSpec(1, numOfPartitionFields);
}
else
{
string option = conf.Get(PartitionerOptions);
keyFieldHelper.ParseOption(option);
}
}
/// <summary>Test is key-field-helper's getStartOffset/getEndOffset.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestgetStartEndOffset()
{
KeyFieldHelper helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
// test getStartOffset with -k1,2
helper.SetKeyFieldSpec(1, 2);
string input = "hi\thello";
string expectedOutput = input;
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k1.0,0 .. should result into start = -1
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.0,0");
TestKeySpecs(input, null, helper);
// test getStartOffset with -k1,0
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1,0");
expectedOutput = input;
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k1.2,0
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.2,0");
expectedOutput = "i\thello";
TestKeySpecs(input, expectedOutput, helper);
// test getWordLengths with -k1.0,2.3
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.1,2.3");
expectedOutput = "hi\thel";
TestKeySpecs(input, expectedOutput, helper);
// test getWordLengths with -k1.2,2.3
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.2,2.3");
expectedOutput = "i\thel";
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k1.2,3.0
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.2,3.0");
expectedOutput = "i\thello";
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k2,2
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2,2");
expectedOutput = "hello";
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k3.0,4.0
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k3.1,4.0");
TestKeySpecs(input, null, helper);
// test getStartOffset with -k2.1
helper = new KeyFieldHelper();
input = "123123123123123hi\thello\thow";
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2.1");
expectedOutput = "hello\thow";
TestKeySpecs(input, expectedOutput, helper, 15, input.Length);
// test getStartOffset with -k2.1,4 with end ending on \t
helper = new KeyFieldHelper();
input = "123123123123123hi\thello\t\thow\tare";
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2.1,3");
expectedOutput = "hello\t";
TestKeySpecs(input, expectedOutput, helper, 17, input.Length);
// test getStartOffset with -k2.1 with end ending on \t
helper = new KeyFieldHelper();
input = "123123123123123hi\thello\thow\tare";
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2.1");
expectedOutput = "hello\thow\t";
TestKeySpecs(input, expectedOutput, helper, 17, 28);
// test getStartOffset with -k2.1,3 with smaller length
helper = new KeyFieldHelper();
input = "123123123123123hi\thello\thow";
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2.1,3");
expectedOutput = "hello";
TestKeySpecs(input, expectedOutput, helper, 15, 23);
}