| public AddTokenOp ( string token, bool returnToken ) : void | ||
| token | string | |
| returnToken | bool | |
| return | void | |
public void Three_Token_Op_Fwd() {
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder { ToTokenize = Search2 };
dtb.AddTokenOp("AB-", false);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps()) {
result += " " + dt.NextToken();
}
Assert.AreEqual(" CD- CD", result);
}
IDev2Tokenizer CreateTokenizer()
{
_tokenizerValidationErrors.Clear();
var stringToSplit = VariableListString;
var splitType = SplitType;
var at = SplitToken;
if(string.IsNullOrWhiteSpace(stringToSplit))
{
return null;
}
var dtb = new Dev2TokenizerBuilder { ToTokenize = stringToSplit };
switch(splitType)
{
case "Index":
if(!string.IsNullOrEmpty(at))
{
// No need for try..parse as ValidationErrors() function checks this!
var indexNum = int.Parse(at);
dtb.AddIndexOp(indexNum);
}
break;
case "Space":
dtb.AddTokenOp(" ", false);
break;
case "Tab":
dtb.AddTokenOp("\t", false);
break;
case "New Line":
if(stringToSplit.Contains("\r\n"))
{
dtb.AddTokenOp("\r\n", false);
}
else if(stringToSplit.Contains("\n"))
{
dtb.AddTokenOp("\n", false);
}
else if(stringToSplit.Contains("\r"))
{
dtb.AddTokenOp("\r", false);
}
else
{
// Assume environment
dtb.AddTokenOp(Environment.NewLine, false);
}
break;
case "Chars":
if(!string.IsNullOrEmpty(at))
{
dtb.AddTokenOp(at, false);
}
break;
}
try
{
return dtb.Generate();
}
catch(Exception ex)
{
_tokenizerValidationErrors.Add(new ErrorInfo { ErrorType = ErrorType.Critical, Message = ex.Message });
}
return null;
}
private IDev2Tokenizer CreateSplitPattern(string stringToSplit, string splitType, string at)
{
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder { ToTokenize = stringToSplit };
switch (splitType)
{
case "Index":
if (!string.IsNullOrEmpty(at))
{
int indexNum;
if (int.TryParse(at, out indexNum) && indexNum > 0)
{
dtb.AddIndexOp(indexNum);
}
}
break;
case "Space":
dtb.AddTokenOp(" ", false);
break;
case "Tab":
dtb.AddTokenOp("\t", false);
break;
case "New Line":
if (stringToSplit.Contains("\r\n"))
{
dtb.AddTokenOp("\r\n", false);
}
else if (stringToSplit.Contains("\n"))
{
dtb.AddTokenOp("\n", false);
}
else if (stringToSplit.Contains("\r"))
{
dtb.AddTokenOp("\r", false);
}
break;
case "Chars":
if (!string.IsNullOrEmpty(at))
{
dtb.AddTokenOp(at, false);
}
break;
}
return dtb.Generate();
}
public void Token_Op_With_Token_Fwd() {
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder { ToTokenize = Search };
dtb.AddTokenOp("-", true);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps()) {
result += dt.NextToken();
}
Assert.AreEqual("AB-CD-DE-FG-HI", result);
}
public void Three_Token_Perfomance_Op() {
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder { ToTokenize = TestStrings.tokenizerBase };
dtb.AddTokenOp("AB-", false);
IDev2Tokenizer dt = dtb.Generate();
int opCnt = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
while (dt.HasMoreOps() && opCnt < 35000)
{
dt.NextToken();
opCnt++;
}
sw.Stop();
long exeTime = sw.ElapsedMilliseconds;
// can we do it in less then 2.5s?
// I sure hope so ;)
Console.WriteLine("Total Time : " + exeTime);
Assert.IsTrue(opCnt == 35000 && exeTime < 2500, "It took [ " + exeTime + " ]");
}
public void Token_And_Index_Op_Bwd() {
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder
{
ToTokenize = Search,
ReverseOrder = true
};
dtb.AddTokenOp("-", false);
dtb.AddIndexOp(3);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps()) {
result += "." + dt.NextToken();
}
Assert.AreEqual(".HI.-FG.DE.-CD.AB", result);
}
public void Two_Token_Op_Bwd() {
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder
{
ToTokenize = Search2,
ReverseOrder = true
};
dtb.AddTokenOp("B-", false);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps()) {
result += "." + dt.NextToken();
}
Assert.AreEqual(".CD.CD-A.A", result);
}
public void Single_Token_Op_Bwd() {
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder
{
ReverseOrder = true,
ToTokenize = Search
};
dtb.AddTokenOp("-", false);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps()) {
result += "."+ dt.NextToken();
}
Assert.AreEqual(".HI.FG.DE.CD.AB", result);
}
public void Token_And_Index_Op_Fwd() {
Dev2TokenizerBuilder dtb = new Dev2TokenizerBuilder { ToTokenize = Search };
dtb.AddTokenOp("-", false);
dtb.AddIndexOp(3);
IDev2Tokenizer dt = dtb.Generate();
string result = string.Empty;
while (dt.HasMoreOps()) {
result += " " + dt.NextToken();
}
Assert.AreEqual(" AB CD- DE FG- HI", result);
}
