public void More_Then_One_Op_Fwd()
{
var dtb = new Dev2TokenizerBuilder {
ToTokenize = Search.ToStringBuilder()
};
dtb.AddIndexOp(2);
dtb.AddEoFOp();
var dt = dtb.Generate();
var result = string.Empty;
var cnt = 0;
while (dt.HasMoreOps())
{
result += dt.NextToken();
cnt++;
}
Assert.AreEqual("AB-CD-DE-FG-HI", result);
Assert.IsTrue(cnt == 2);
}
IDev2Tokenizer CreateSplitPattern(string stringToSplit, string splitType, string at)
{
var dtb = new Dev2TokenizerBuilder {
ToTokenize = stringToSplit.ToStringBuilder()
};
switch (splitType)
{
case "Index":
if (!string.IsNullOrEmpty(at))
{
if (int.TryParse(at, out int indexNum) && indexNum > 0)
{
dtb.AddIndexOp(indexNum);
}
}
break;
case "Space":
dtb.AddTokenOp(" ", false);
break;
case "Tab":
dtb.AddTokenOp("\t", false);
break;
case "New Line":
if (stringToSplit.Contains("\r\n"))
{
dtb.AddTokenOp("\r\n", false);
}
else if (stringToSplit.Contains("\n"))
{
dtb.AddTokenOp("\n", false);
}
else
{
if (stringToSplit.Contains("\r"))
{
dtb.AddTokenOp("\r", false);
}
}
break;
case "Chars":
if (!string.IsNullOrEmpty(at))
{
dtb.AddTokenOp(at, false);
}
break;
default:
break;
}
return(dtb.Generate());
}
public void Index_Op_Fwd()
{
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder();
dtb.ToTokenize = search;
dtb.AddIndexOp(2);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps())
{
result += " " + dt.NextToken();
}
Assert.AreEqual(" AB -C D- DE -F G- HI", result);
}
public void Index_Op_Bwd()
{
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder();
dtb.ToTokenize = search;
dtb.ReverseOrder = true;
dtb.AddIndexOp(2);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps())
{
result += "." + dt.NextToken();
}
Assert.AreEqual(".HI.G-.-F.DE.D-.-C.AB", result);
}
public void Token_And_Index_Op_Fwd()
{
var dtb = new Dev2TokenizerBuilder {
ToTokenize = Search.ToStringBuilder()
};
dtb.AddTokenOp("-", false);
dtb.AddIndexOp(3);
var dt = dtb.Generate();
var result = string.Empty;
while (dt.HasMoreOps())
{
result += " " + dt.NextToken();
}
Assert.AreEqual(" AB CD- DE FG- HI", result);
}
public void Token_And_Index_Op_Bwd()
{
var dtb = new Dev2TokenizerBuilder
{
ToTokenize = Search.ToStringBuilder(),
ReverseOrder = true
};
dtb.AddTokenOp("-", false);
dtb.AddIndexOp(3);
var dt = dtb.Generate();
var result = string.Empty;
while (dt.HasMoreOps())
{
result += "." + dt.NextToken();
}
Assert.AreEqual(".HI.-FG.DE.-CD.AB", result);
}
private IDev2Tokenizer CreateSplitPattern(ref string stringToSplit, IEnumerable <DataSplitDTO> args, IExecutionEnvironment compiler, out ErrorResultTO errors, int update)
{
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder {
ToTokenize = stringToSplit, ReverseOrder = ReverseOrder
};
errors = new ErrorResultTO();
foreach (DataSplitDTO t in args)
{
t.At = t.At ?? "";
string entry;
switch (t.SplitType)
{
case "Index":
try
{
entry = compiler.EvalAsListOfStrings(t.At, update).FirstOrDefault();
if (entry == null)
{
throw new Exception("null iterator expression");
}
string index = entry;
int indexNum = Convert.ToInt32(index);
if (indexNum > 0)
{
dtb.AddIndexOp(indexNum);
}
}
catch (Exception ex)
{
errors.AddError(ex.Message);
}
break;
case "End":
dtb.AddEoFOp();
break;
case "Space":
dtb.AddTokenOp(" ", t.Include);
break;
case "Tab":
dtb.AddTokenOp("\t", t.Include);
break;
case "New Line":
if (stringToSplit.Contains("\r\n"))
{
dtb.AddTokenOp("\r\n", t.Include);
}
else if (stringToSplit.Contains("\n"))
{
dtb.AddTokenOp("\n", t.Include);
}
else if (stringToSplit.Contains("\r"))
{
dtb.AddTokenOp("\r", t.Include);
}
break;
case "Chars":
if (!string.IsNullOrEmpty(t.At))
{
entry = compiler.EvalAsListOfStrings(t.At, update).FirstOrDefault();
if (entry != null && (entry.Contains(@"\r\n") || entry.Contains(@"\n")))
{
var match = Regex.Match(stringToSplit, @"[\r\n]+");
if (match.Success && !SkipBlankRows)
{
stringToSplit = Regex.Escape(stringToSplit);
dtb.ToTokenize = stringToSplit;
}
}
string escape = t.EscapeChar;
if (!String.IsNullOrEmpty(escape))
{
escape = compiler.EvalAsListOfStrings(t.EscapeChar, update).FirstOrDefault();
}
dtb.AddTokenOp(entry, t.Include, escape);
}
break;
}
_indexCounter++;
}
return(string.IsNullOrEmpty(dtb.ToTokenize) || errors.HasErrors() ? null : dtb.Generate());
}
IDev2Tokenizer CreateTokenizer()
{
_tokenizerValidationErrors.Clear();
var stringToSplit = VariableListString;
var splitType = SplitType;
var at = SplitToken;
if (string.IsNullOrWhiteSpace(stringToSplit))
{
return(null);
}
var dtb = new Dev2TokenizerBuilder {
ToTokenize = stringToSplit
};
switch (splitType)
{
case "Index":
if (!string.IsNullOrEmpty(at))
{
// No need for try..parse as ValidationErrors() function checks this!
var indexNum = int.Parse(at);
dtb.AddIndexOp(indexNum);
}
break;
case "Space":
dtb.AddTokenOp(" ", false);
break;
case "Tab":
dtb.AddTokenOp("\t", false);
break;
case "New Line":
if (stringToSplit.Contains("\r\n"))
{
dtb.AddTokenOp("\r\n", false);
}
else if (stringToSplit.Contains("\n"))
{
dtb.AddTokenOp("\n", false);
}
else if (stringToSplit.Contains("\r"))
{
dtb.AddTokenOp("\r", false);
}
else
{
// Assume environment
dtb.AddTokenOp(Environment.NewLine, false);
}
break;
case "Chars":
if (!string.IsNullOrEmpty(at))
{
dtb.AddTokenOp(at, false);
}
break;
}
try
{
return(dtb.Generate());
}
catch (Exception ex)
{
_tokenizerValidationErrors.Add(new ErrorInfo {
ErrorType = ErrorType.Critical, Message = ex.Message
});
}
return(null);
}
private IDev2Tokenizer CreateSplitPattern(ref string stringToSplit, IEnumerable <DataSplitDTO> args, IDataListCompiler compiler, Guid dlId, out ErrorResultTO errors)
{
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder {
ToTokenize = stringToSplit, ReverseOrder = ReverseOrder
};
errors = new ErrorResultTO();
foreach (DataSplitDTO t in args)
{
var fieldName = t.OutputVariable;
t.At = t.At ?? "";
if (!string.IsNullOrEmpty(_datalistString))
{
var isValidExpr = new IsValidExpressionRule(() => fieldName, _datalistString)
{
LabelText = fieldName
};
var errorInfo = isValidExpr.Check();
if (errorInfo != null)
{
errors.AddError(errorInfo.Message);
continue;
}
}
IBinaryDataListEntry entry;
string error;
switch (t.SplitType)
{
case "Index":
try
{
entry = compiler.Evaluate(dlId, enActionType.User, t.At, false, out errors);
string index = DataListUtil.GetValueAtIndex(entry, 1, out error);
int indexNum = Convert.ToInt32(index);
if (indexNum > 0)
{
dtb.AddIndexOp(indexNum);
}
}
catch (Exception ex)
{
errors.AddError(ex.Message);
}
break;
case "End":
dtb.AddEoFOp();
break;
case "Space":
dtb.AddTokenOp(" ", t.Include);
break;
case "Tab":
dtb.AddTokenOp("\t", t.Include);
break;
case "New Line":
if (stringToSplit.Contains("\r\n"))
{
dtb.AddTokenOp("\r\n", t.Include);
}
else if (stringToSplit.Contains("\n"))
{
dtb.AddTokenOp("\n", t.Include);
}
else if (stringToSplit.Contains("\r"))
{
dtb.AddTokenOp("\r", t.Include);
}
break;
case "Chars":
if (!string.IsNullOrEmpty(t.At))
{
entry = compiler.Evaluate(dlId, enActionType.User, t.At, false, out errors);
string val = DataListUtil.GetValueAtIndex(entry, 1, out error);
string escape = t.EscapeChar;
if (!String.IsNullOrEmpty(escape))
{
entry = compiler.Evaluate(dlId, enActionType.User, t.EscapeChar, false, out errors);
escape = DataListUtil.GetValueAtIndex(entry, 1, out error);
}
dtb.AddTokenOp(val, t.Include, escape);
}
break;
}
_indexCounter++;
}
return(string.IsNullOrEmpty(dtb.ToTokenize) || errors.HasErrors() ? null : dtb.Generate());
}
