/// <summary>Test is key-field-helper's getWordLengths.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestGetWordLengths()
{
KeyFieldHelper helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
// test getWordLengths with unspecified key-specifications
string input = "hi";
int[] result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0,
2);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 1 }));
// set the key specs
helper.SetKeyFieldSpec(1, 2);
// test getWordLengths with 3 words
input = "hi\thello there";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, input
.Length);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 2, 2, 11 }));
// test getWordLengths with 4 words but with a different separator
helper.SetKeyFieldSeparator(" ");
input = "hi hello\tthere you";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, input
.Length);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 3, 2, 11, 3 }));
// test with non zero start index
input = "hi hello there you where me there";
// .....................
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 10, 33);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 5, 4, 3, 5, 2, 3 }));
input = "hi hello there you where me ";
// ..................
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 10, input
.Length);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 5, 4, 3, 5, 2, 0 }));
input = string.Empty;
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 0);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 1, 0 }));
input = " abc";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 5);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 3, 0, 0, 3 }));
input = " abc";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 2);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 3, 0, 0, 0 }));
input = " abc ";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 2);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 2, 0, 1 }));
helper.SetKeyFieldSeparator("abcd");
input = "abc";
result = helper.GetWordLengths(Sharpen.Runtime.GetBytesForString(input), 0, 3);
NUnit.Framework.Assert.IsTrue(Equals(result, new int[] { 1, 3 }));
}
private void TestKeySpecs(string input, string expectedOutput, KeyFieldHelper helper
, int s1, int e1)
{
Log.Info("input : " + input);
string keySpecs = helper.KeySpecs()[0].ToString();
Log.Info("keyspecs : " + keySpecs);
byte[] inputBytes = Sharpen.Runtime.GetBytesForString(input);
// get the input bytes
if (e1 == -1)
{
e1 = inputBytes.Length;
}
Log.Info("length : " + e1);
// get the word lengths
int[] indices = helper.GetWordLengths(inputBytes, s1, e1);
// get the start index
int start = helper.GetStartOffset(inputBytes, s1, e1, indices, helper.KeySpecs()[
0]);
Log.Info("start : " + start);
if (expectedOutput == null)
{
NUnit.Framework.Assert.AreEqual("Expected -1 when the start index is invalid", -1
, start);
return;
}
// get the end index
int end = helper.GetEndOffset(inputBytes, s1, e1, indices, helper.KeySpecs()[0]);
Log.Info("end : " + end);
//my fix
end = (end >= inputBytes.Length) ? inputBytes.Length - 1 : end;
int length = end + 1 - start;
Log.Info("length : " + length);
byte[] outputBytes = new byte[length];
System.Array.Copy(inputBytes, start, outputBytes, 0, length);
string output = Sharpen.Runtime.GetStringForBytes(outputBytes);
Log.Info("output : " + output);
Log.Info("expected-output : " + expectedOutput);
NUnit.Framework.Assert.AreEqual(keySpecs + " failed on input '" + input + "'", expectedOutput
, output);
}
public virtual void SetConf(Configuration conf)
{
this.conf = conf;
keyFieldHelper = new KeyFieldHelper();
string keyFieldSeparator = conf.Get(MRJobConfig.MapOutputKeyFieldSeperator, "\t");
keyFieldHelper.SetKeyFieldSeparator(keyFieldSeparator);
if (conf.Get("num.key.fields.for.partition") != null)
{
Log.Warn("Using deprecated num.key.fields.for.partition. " + "Use mapreduce.partition.keypartitioner.options instead"
);
this.numOfPartitionFields = conf.GetInt("num.key.fields.for.partition", 0);
keyFieldHelper.SetKeyFieldSpec(1, numOfPartitionFields);
}
else
{
string option = conf.Get(PartitionerOptions);
keyFieldHelper.ParseOption(option);
}
}
private void TestKeySpecs(string input, string expectedOutput, KeyFieldHelper helper
)
{
TestKeySpecs(input, expectedOutput, helper, 0, -1);
}
/// <summary>Test is key-field-helper's getStartOffset/getEndOffset.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestgetStartEndOffset()
{
KeyFieldHelper helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
// test getStartOffset with -k1,2
helper.SetKeyFieldSpec(1, 2);
string input = "hi\thello";
string expectedOutput = input;
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k1.0,0 .. should result into start = -1
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.0,0");
TestKeySpecs(input, null, helper);
// test getStartOffset with -k1,0
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1,0");
expectedOutput = input;
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k1.2,0
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.2,0");
expectedOutput = "i\thello";
TestKeySpecs(input, expectedOutput, helper);
// test getWordLengths with -k1.0,2.3
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.1,2.3");
expectedOutput = "hi\thel";
TestKeySpecs(input, expectedOutput, helper);
// test getWordLengths with -k1.2,2.3
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.2,2.3");
expectedOutput = "i\thel";
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k1.2,3.0
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k1.2,3.0");
expectedOutput = "i\thello";
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k2,2
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2,2");
expectedOutput = "hello";
TestKeySpecs(input, expectedOutput, helper);
// test getStartOffset with -k3.0,4.0
helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k3.1,4.0");
TestKeySpecs(input, null, helper);
// test getStartOffset with -k2.1
helper = new KeyFieldHelper();
input = "123123123123123hi\thello\thow";
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2.1");
expectedOutput = "hello\thow";
TestKeySpecs(input, expectedOutput, helper, 15, input.Length);
// test getStartOffset with -k2.1,4 with end ending on \t
helper = new KeyFieldHelper();
input = "123123123123123hi\thello\t\thow\tare";
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2.1,3");
expectedOutput = "hello\t";
TestKeySpecs(input, expectedOutput, helper, 17, input.Length);
// test getStartOffset with -k2.1 with end ending on \t
helper = new KeyFieldHelper();
input = "123123123123123hi\thello\thow\tare";
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2.1");
expectedOutput = "hello\thow\t";
TestKeySpecs(input, expectedOutput, helper, 17, 28);
// test getStartOffset with -k2.1,3 with smaller length
helper = new KeyFieldHelper();
input = "123123123123123hi\thello\thow";
helper.SetKeyFieldSeparator("\t");
helper.ParseOption("-k2.1,3");
expectedOutput = "hello";
TestKeySpecs(input, expectedOutput, helper, 15, 23);
}
/// <summary>Test is key-field-helper's parse option.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestparseOption()
{
KeyFieldHelper helper = new KeyFieldHelper();
helper.SetKeyFieldSeparator("\t");
string keySpecs = "-k1.2,3.4";
string eKeySpecs = keySpecs;
helper.ParseOption(keySpecs);
string actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
// test -k a.b
keySpecs = "-k 1.2";
eKeySpecs = "-k1.2,0.0";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4n";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4r";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4 -k5.6,7.8n -k9.10,11.12r -k13.14,15.16nr";
//1st
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
// 2nd
eKeySpecs = "-k5.6,7.8n";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
//3rd
eKeySpecs = "-k9.10,11.12r";
actKeySpecs = helper.KeySpecs()[2].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
//4th
eKeySpecs = "-k13.14,15.16nr";
actKeySpecs = helper.KeySpecs()[3].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2n,3.4";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2r,3.4";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2nr,3.4";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4n";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4r";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4nr";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr -k1.2,3.4 -k5.6,7.8";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8nr";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-n -k1.2,3.4 -k5.6,7.8";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8n";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-r -k1.2,3.4 -k5.6,7.8";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8r";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4n -k5.6,7.8";
eKeySpecs = "-k1.2,3.4n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4r -k5.6,7.8";
eKeySpecs = "-k1.2,3.4r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-k1.2,3.4nr -k5.6,7.8";
eKeySpecs = "-k1.2,3.4nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
eKeySpecs = "-k5.6,7.8";
actKeySpecs = helper.KeySpecs()[1].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-n";
eKeySpecs = "-k1.1,0.0n";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-r";
eKeySpecs = "-k1.1,0.0r";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
keySpecs = "-nr";
eKeySpecs = "-k1.1,0.0nr";
helper = new KeyFieldHelper();
helper.ParseOption(keySpecs);
actKeySpecs = helper.KeySpecs()[0].ToString();
NUnit.Framework.Assert.AreEqual("KeyFieldHelper's parsing is garbled", eKeySpecs,
actKeySpecs);
}