IDev2Tokenizer CreateTokenizer()
{
_tokenizerValidationErrors.Clear();
var stringToSplit = VariableListString;
var splitType = SplitType;
var at = SplitToken;
if(string.IsNullOrWhiteSpace(stringToSplit))
{
return null;
}
var dtb = new Dev2TokenizerBuilder { ToTokenize = stringToSplit };
switch(splitType)
{
case "Index":
if(!string.IsNullOrEmpty(at))
{
// No need for try..parse as ValidationErrors() function checks this!
var indexNum = int.Parse(at);
dtb.AddIndexOp(indexNum);
}
break;
case "Space":
dtb.AddTokenOp(" ", false);
break;
case "Tab":
dtb.AddTokenOp("\t", false);
break;
case "New Line":
if(stringToSplit.Contains("\r\n"))
{
dtb.AddTokenOp("\r\n", false);
}
else if(stringToSplit.Contains("\n"))
{
dtb.AddTokenOp("\n", false);
}
else if(stringToSplit.Contains("\r"))
{
dtb.AddTokenOp("\r", false);
}
else
{
// Assume environment
dtb.AddTokenOp(Environment.NewLine, false);
}
break;
case "Chars":
if(!string.IsNullOrEmpty(at))
{
dtb.AddTokenOp(at, false);
}
break;
}
try
{
return dtb.Generate();
}
catch(Exception ex)
{
_tokenizerValidationErrors.Add(new ErrorInfo { ErrorType = ErrorType.Critical, Message = ex.Message });
}
return null;
}
private IDev2Tokenizer CreateSplitPattern(string stringToSplit, string splitType, string at)
{
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder { ToTokenize = stringToSplit };
switch (splitType)
{
case "Index":
if (!string.IsNullOrEmpty(at))
{
int indexNum;
if (int.TryParse(at, out indexNum) && indexNum > 0)
{
dtb.AddIndexOp(indexNum);
}
}
break;
case "Space":
dtb.AddTokenOp(" ", false);
break;
case "Tab":
dtb.AddTokenOp("\t", false);
break;
case "New Line":
if (stringToSplit.Contains("\r\n"))
{
dtb.AddTokenOp("\r\n", false);
}
else if (stringToSplit.Contains("\n"))
{
dtb.AddTokenOp("\n", false);
}
else if (stringToSplit.Contains("\r"))
{
dtb.AddTokenOp("\r", false);
}
break;
case "Chars":
if (!string.IsNullOrEmpty(at))
{
dtb.AddTokenOp(at, false);
}
break;
}
return dtb.Generate();
}
public void Token_And_Index_Op_Fwd() {
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder { ToTokenize = Search };
dtb.AddTokenOp("-", false);
dtb.AddIndexOp(3);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps()) {
result += " " + dt.NextToken();
}
Assert.AreEqual(" AB CD- DE FG- HI", result);
}
public void Token_And_Index_Op_Bwd() {
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder
{
ToTokenize = Search,
ReverseOrder = true
};
dtb.AddTokenOp("-", false);
dtb.AddIndexOp(3);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps()) {
result += "." + dt.NextToken();
}
Assert.AreEqual(".HI.-FG.DE.-CD.AB", result);
}
public void More_Then_One_Op_Fwd()
{
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder { ToTokenize = Search };
dtb.AddIndexOp(2);
dtb.AddEoFOp();
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
int cnt = 0;
while (dt.HasMoreOps())
{
result += dt.NextToken();
cnt++;
}
Assert.AreEqual("AB-CD-DE-FG-HI", result);
Assert.IsTrue(cnt == 2);
}
