/// -----------------------------------------------------------------------------------
/// <summary>
/// Initializes a new instance of the <see cref="Tokenizer"/> class.
/// </summary>
/// -----------------------------------------------------------------------------------
public Tokenizer()
{
m_stack = new Stack();
m_tri = new StringTri();
}
public void TestInlineMarkerMatch()
{
StringTri stringTri = new StringTri();
int nextIndex;
bool isInline = true;
MarkerSpec aimsAbc = MarkerSpec.CreateMarkerSpec("abc", "dummyMarker", "dummyData",
isInline, null, null, null);
MarkerSpec aimsAb = MarkerSpec.CreateMarkerSpec("ab", "dummyMarker", "dummyData",
isInline, null, null, null);
stringTri.Add(aimsAbc);
stringTri.Add(aimsAb);
// See if we can find an inline marker.
Assert.AreEqual(aimsAbc, stringTri.Match("To abc this is the day",
3, out nextIndex));
Assert.AreEqual(6, nextIndex);
// If we make ab inline we can find it.
Assert.AreEqual(aimsAb, stringTri.Match("To ab this is the day",
3, out nextIndex));
Assert.AreEqual(5, nextIndex);
// abc and ab now both match, but we want to find the longer one.
Assert.AreEqual(aimsAbc, stringTri.Match("To abc this is the day",
3, out nextIndex));
Assert.AreEqual(6, nextIndex);
// Minimal case, verifies we can match inline marker at start and end of line.
Assert.AreEqual(aimsAb, stringTri.Match("ab", 0, out nextIndex));
Assert.AreEqual(2, nextIndex);
// Make sure we can match without a following space.
Assert.AreEqual(aimsAb, stringTri.Match("To abXYZ", 3, out nextIndex));
Assert.AreEqual(5, nextIndex);
}
public void TestInlineMarkerSearch()
{
StringTri stringTri = new StringTri();
int nextIndex;
MarkerSpec target;
bool isInline = true;
MarkerSpec aimsAbc = MarkerSpec.CreateMarkerSpec("abc", "dummyMarker", "dummyData",
isInline, null, null, null);
MarkerSpec aimsAb = MarkerSpec.CreateMarkerSpec("ab", "dummyMarker", "dummyData",
isInline, null, null, null);
stringTri.Add(aimsAbc);
stringTri.Add(aimsAb);
// See if we can find an inline marker.
Assert.AreEqual(3, stringTri.Search("To abc this is the day",
0, out nextIndex, out target));
Assert.AreEqual(aimsAbc, target);
Assert.AreEqual(6, nextIndex);
// abc and ab now both match, but we want to find the longer one.
Assert.AreEqual(3, stringTri.Search("To abc this is the day",
0, out nextIndex, out target));
Assert.AreEqual(aimsAbc, target);
Assert.AreEqual(6, nextIndex);
// Minimal case, verifies we can match inline marker at start and end of line.
Assert.AreEqual(0, stringTri.Search("ab",
0, out nextIndex, out target));
Assert.AreEqual(aimsAb, target);
Assert.AreEqual(2, nextIndex);
// Make sure we can match without a following space.
Assert.AreEqual(3, stringTri.Search("To abXYZ",
0, out nextIndex, out target));
Assert.AreEqual(aimsAb, target);
Assert.AreEqual(5, nextIndex);
// Fail if we start search after target
Assert.AreEqual(-1, stringTri.Search("To abXYZ",
4, out nextIndex, out target));
Assert.IsNull(target);
Assert.AreEqual(8, nextIndex);
// Find second occurrence.
Assert.AreEqual(8, stringTri.Search("To abXYZabXYZ",
4, out nextIndex, out target));
Assert.AreEqual(aimsAb, target);
Assert.AreEqual(10, nextIndex);
// Search on an empty line.
Assert.AreEqual(-1, stringTri.Search("",
0, out nextIndex, out target));
Assert.IsNull(target);
Assert.AreEqual(0, nextIndex);
// We don't currently test cases where requiresNewlineBefore is not equal to
// requiresWhitespaceAfter, because we don't currently use it.
// This might be need for a footnote position marker, such as |fn,
// that does not need to start at the beginning of a line,
// but does need to have a space after it.
}
public void TestFieldMarkerSearch()
{
StringTri stringTri = new StringTri();
int nextIndex;
MarkerSpec target;
bool isInline = false;
MarkerSpec fmsAbc = MarkerSpec.CreateMarkerSpec("abc", "dummy1", "dummy2",
isInline, null, null, null);
MarkerSpec fmsAbd = MarkerSpec.CreateMarkerSpec("abd", "dummy1", "dummy2",
isInline, null, null, null);
MarkerSpec fmsAb = MarkerSpec.CreateMarkerSpec("ab", "dummy1", "dummy2",
isInline, null, null, null);
stringTri.Add(fmsAbc);
stringTri.Add(fmsAbd);
stringTri.Add(fmsAb);
// Should find at start.
Assert.AreEqual(0, stringTri.Search("abc this is the day",
0, out nextIndex, out target));
Assert.AreEqual(fmsAbc, target);
Assert.AreEqual(4, nextIndex);
// Should only absorb one space.
Assert.AreEqual(0, stringTri.Search("abc this is the day",
0, out nextIndex, out target));
Assert.AreEqual(fmsAbc, target);
Assert.AreEqual(4, nextIndex);
// Should match on closing newline, but no space to absorb.
Assert.AreEqual(0, stringTri.Search("abc",
0, out nextIndex, out target));
Assert.AreEqual(fmsAbc, target);
Assert.AreEqual(3, nextIndex);
// Should fail with no white space following.
Assert.AreEqual(-1, stringTri.Search("abcXYZ",
0, out nextIndex, out target));
Assert.IsNull(target);
Assert.AreEqual(6, nextIndex);
// Should be able to find abd also
Assert.AreEqual(0, stringTri.Search("abd this is the day",
0, out nextIndex, out target));
Assert.AreEqual(fmsAbd, target);
Assert.AreEqual(4, nextIndex);
// Should be able to find ab also
Assert.AreEqual(0, stringTri.Search("ab this is the day",
0, out nextIndex, out target));
Assert.AreEqual(3, nextIndex);
// Should not find except at start of line.
Assert.AreEqual(-1, stringTri.Search("To abc this",
0, out nextIndex, out target));
Assert.IsNull(target);
Assert.AreEqual(11, nextIndex);
}
