/// ----------------------------------------------------------------------------------- /// <summary> /// Initializes a new instance of the <see cref="Tokenizer"/> class. /// </summary> /// ----------------------------------------------------------------------------------- public Tokenizer() { m_stack = new Stack(); m_tri = new StringTri(); }
public void TestInlineMarkerMatch() { StringTri stringTri = new StringTri(); int nextIndex; bool isInline = true; MarkerSpec aimsAbc = MarkerSpec.CreateMarkerSpec("abc", "dummyMarker", "dummyData", isInline, null, null, null); MarkerSpec aimsAb = MarkerSpec.CreateMarkerSpec("ab", "dummyMarker", "dummyData", isInline, null, null, null); stringTri.Add(aimsAbc); stringTri.Add(aimsAb); // See if we can find an inline marker. Assert.AreEqual(aimsAbc, stringTri.Match("To abc this is the day", 3, out nextIndex)); Assert.AreEqual(6, nextIndex); // If we make ab inline we can find it. Assert.AreEqual(aimsAb, stringTri.Match("To ab this is the day", 3, out nextIndex)); Assert.AreEqual(5, nextIndex); // abc and ab now both match, but we want to find the longer one. Assert.AreEqual(aimsAbc, stringTri.Match("To abc this is the day", 3, out nextIndex)); Assert.AreEqual(6, nextIndex); // Minimal case, verifies we can match inline marker at start and end of line. Assert.AreEqual(aimsAb, stringTri.Match("ab", 0, out nextIndex)); Assert.AreEqual(2, nextIndex); // Make sure we can match without a following space. Assert.AreEqual(aimsAb, stringTri.Match("To abXYZ", 3, out nextIndex)); Assert.AreEqual(5, nextIndex); }
public void TestInlineMarkerSearch() { StringTri stringTri = new StringTri(); int nextIndex; MarkerSpec target; bool isInline = true; MarkerSpec aimsAbc = MarkerSpec.CreateMarkerSpec("abc", "dummyMarker", "dummyData", isInline, null, null, null); MarkerSpec aimsAb = MarkerSpec.CreateMarkerSpec("ab", "dummyMarker", "dummyData", isInline, null, null, null); stringTri.Add(aimsAbc); stringTri.Add(aimsAb); // See if we can find an inline marker. Assert.AreEqual(3, stringTri.Search("To abc this is the day", 0, out nextIndex, out target)); Assert.AreEqual(aimsAbc, target); Assert.AreEqual(6, nextIndex); // abc and ab now both match, but we want to find the longer one. Assert.AreEqual(3, stringTri.Search("To abc this is the day", 0, out nextIndex, out target)); Assert.AreEqual(aimsAbc, target); Assert.AreEqual(6, nextIndex); // Minimal case, verifies we can match inline marker at start and end of line. Assert.AreEqual(0, stringTri.Search("ab", 0, out nextIndex, out target)); Assert.AreEqual(aimsAb, target); Assert.AreEqual(2, nextIndex); // Make sure we can match without a following space. Assert.AreEqual(3, stringTri.Search("To abXYZ", 0, out nextIndex, out target)); Assert.AreEqual(aimsAb, target); Assert.AreEqual(5, nextIndex); // Fail if we start search after target Assert.AreEqual(-1, stringTri.Search("To abXYZ", 4, out nextIndex, out target)); Assert.IsNull(target); Assert.AreEqual(8, nextIndex); // Find second occurrence. Assert.AreEqual(8, stringTri.Search("To abXYZabXYZ", 4, out nextIndex, out target)); Assert.AreEqual(aimsAb, target); Assert.AreEqual(10, nextIndex); // Search on an empty line. Assert.AreEqual(-1, stringTri.Search("", 0, out nextIndex, out target)); Assert.IsNull(target); Assert.AreEqual(0, nextIndex); // We don't currently test cases where requiresNewlineBefore is not equal to // requiresWhitespaceAfter, because we don't currently use it. // This might be need for a footnote position marker, such as |fn, // that does not need to start at the beginning of a line, // but does need to have a space after it. }
public void TestFieldMarkerSearch() { StringTri stringTri = new StringTri(); int nextIndex; MarkerSpec target; bool isInline = false; MarkerSpec fmsAbc = MarkerSpec.CreateMarkerSpec("abc", "dummy1", "dummy2", isInline, null, null, null); MarkerSpec fmsAbd = MarkerSpec.CreateMarkerSpec("abd", "dummy1", "dummy2", isInline, null, null, null); MarkerSpec fmsAb = MarkerSpec.CreateMarkerSpec("ab", "dummy1", "dummy2", isInline, null, null, null); stringTri.Add(fmsAbc); stringTri.Add(fmsAbd); stringTri.Add(fmsAb); // Should find at start. Assert.AreEqual(0, stringTri.Search("abc this is the day", 0, out nextIndex, out target)); Assert.AreEqual(fmsAbc, target); Assert.AreEqual(4, nextIndex); // Should only absorb one space. Assert.AreEqual(0, stringTri.Search("abc this is the day", 0, out nextIndex, out target)); Assert.AreEqual(fmsAbc, target); Assert.AreEqual(4, nextIndex); // Should match on closing newline, but no space to absorb. Assert.AreEqual(0, stringTri.Search("abc", 0, out nextIndex, out target)); Assert.AreEqual(fmsAbc, target); Assert.AreEqual(3, nextIndex); // Should fail with no white space following. Assert.AreEqual(-1, stringTri.Search("abcXYZ", 0, out nextIndex, out target)); Assert.IsNull(target); Assert.AreEqual(6, nextIndex); // Should be able to find abd also Assert.AreEqual(0, stringTri.Search("abd this is the day", 0, out nextIndex, out target)); Assert.AreEqual(fmsAbd, target); Assert.AreEqual(4, nextIndex); // Should be able to find ab also Assert.AreEqual(0, stringTri.Search("ab this is the day", 0, out nextIndex, out target)); Assert.AreEqual(3, nextIndex); // Should not find except at start of line. Assert.AreEqual(-1, stringTri.Search("To abc this", 0, out nextIndex, out target)); Assert.IsNull(target); Assert.AreEqual(11, nextIndex); }