public void GetArtifacts(ITextProvider text, ArtifactCollection artifactCollection) { var reader = new TextProviderReader(text); var tokenizer = new TemplateTokenizer(reader); foreach (var token in tokenizer.GetTokens()) { if (token.Kind != TemplateTokenKind.Text) { var range = TextRange.FromBounds(token.Start, token.End + 1); var artifact = TemplateArtifact.Create(token.Kind, range, token.IsClosed); artifactCollection.Add(artifact); } } }
private void TokenizerTest(string text, bool unclosed, params TemplateTokenResult[] expected) { var tokenizer = new TemplateTokenizer(new StringReader(text)); var tokens = tokenizer.GetTokens().ToArray(); bool passed = false; try { Assert.AreEqual(expected.Length, tokens.Length); Assert.AreEqual(0, tokens[0].Start); Assert.AreEqual(text.Length - 1, tokens[tokens.Length - 1].End); for (int i = 0; i < expected.Length; i++) { var expectedToken = expected[i].Token; Assert.AreEqual(expectedToken.Kind, tokens[i].Kind); Assert.AreEqual(expectedToken.Start, tokens[i].Start); Assert.AreEqual(expectedToken.End, tokens[i].End); switch (expectedToken.Kind) { case TemplateTokenKind.Block: case TemplateTokenKind.Comment: case TemplateTokenKind.Variable: Assert.AreEqual('{', text[expectedToken.Start]); if (!unclosed) { Assert.AreEqual('}', text[expectedToken.End]); } break; } if (expected[i].Start != null) { Assert.AreEqual(expected[i].Start, text[expectedToken.Start]); } if (expected[i].End != null) { Assert.AreEqual(expected[i].End, text[expectedToken.End]); } } passed = true; } finally { if (!passed) { List<string> res = new List<string>(); for (int i = 0; i < tokens.Length; i++) { res.Add( String.Format("new TemplateToken(TemplateTokenKind.{0}, {1}, {2})", tokens[i].Kind, tokens[i].Start, tokens[i].End ) ); } Console.WriteLine(String.Join(",\r\n", res)); } } }
private void UpdateTemplateSpans(TextReader reader, int startSpan = 0, int offset = 0) { var tokenizer = new TemplateTokenizer(reader); TemplateToken? token; List <object> newProjectionSpans = new List <object>(); // spans in the projection buffer List <SpanInfo> newSpanInfos = new List <SpanInfo>(); // our SpanInfo's, with spans in the on-disk buffer while ((token = tokenizer.GetNextToken()) != null) { var curToken = token.Value; var sourceSpan = Span.FromBounds(curToken.Start + offset, curToken.End + offset + 1); switch (curToken.Kind) { case TemplateTokenKind.Block: case TemplateTokenKind.Comment: case TemplateTokenKind.Variable: // template spans are setup to not grow. We'll track the edits and update their // text if something causes them to grow. if (newSpanInfos.Count == 0 && startSpan == 0) { // insert a zero-length span which will grow if the user inserts before the template tag var emptySpan = new CustomTrackingSpan( _diskBuffer.CurrentSnapshot, new Span(0, 0), PointTrackingMode.Negative, PointTrackingMode.Positive ); newProjectionSpans.Add(emptySpan); newSpanInfos.Add(new SpanInfo(emptySpan, TemplateTokenKind.Text)); } newSpanInfos.Add( new SpanInfo( new CustomTrackingSpan( _diskBuffer.CurrentSnapshot, sourceSpan, PointTrackingMode.Positive, PointTrackingMode.Negative ), curToken.Kind, curToken.Kind == TemplateTokenKind.Block ? DjangoBlock.Parse(_diskBuffer.CurrentSnapshot.GetText(sourceSpan)) : null ) ); newProjectionSpans.Add( new CustomTrackingSpan( _templateBuffer.CurrentSnapshot, sourceSpan, PointTrackingMode.Positive, PointTrackingMode.Negative ) ); break; case TemplateTokenKind.Text: var htmlSpan = _htmlBuffer.CurrentSnapshot.Version.CreateCustomTrackingSpan( sourceSpan, TrackingFidelityMode.Forward, new LanguageSpanCustomState(sourceSpan.Start, sourceSpan.End), TrackToVersion ); var diskSpan = _diskBuffer.CurrentSnapshot.Version.CreateCustomTrackingSpan( sourceSpan, TrackingFidelityMode.Forward, new LanguageSpanCustomState(sourceSpan.Start, sourceSpan.End), TrackToVersion ); newProjectionSpans.Add(htmlSpan); newSpanInfos.Add(new SpanInfo(diskSpan, TemplateTokenKind.Text)); break; } } if (newSpanInfos.Count == 0 || newSpanInfos[newSpanInfos.Count - 1].Kind != TemplateTokenKind.Text) { // insert an empty span at the end which will receive new text. var emptySpan = _diskBuffer.CurrentSnapshot.Version.CreateCustomTrackingSpan( new Span(_diskBuffer.CurrentSnapshot.Length, 0), TrackingFidelityMode.Forward, new LanguageSpanCustomState(_diskBuffer.CurrentSnapshot.Length, _diskBuffer.CurrentSnapshot.Length), TrackToVersion ); newProjectionSpans.Add(emptySpan); newSpanInfos.Add(new SpanInfo(emptySpan, TemplateTokenKind.Text)); } var oldSpanCount = _spans.Count; _spans.RemoveRange(startSpan, oldSpanCount - startSpan); _spans.AddRange(newSpanInfos); _projBuffer.ReplaceSpans(startSpan, oldSpanCount - startSpan, newProjectionSpans, _editOptions, null); ProjectionClassifier classifier; if (newSpanInfos.Count > 0 && _projBuffer.Properties.TryGetProperty <ProjectionClassifier>(typeof(ProjectionClassifier), out classifier)) { classifier.RaiseClassificationChanged( newSpanInfos[0].DiskBufferSpan.GetStartPoint(_diskBuffer.CurrentSnapshot), newSpanInfos[newSpanInfos.Count - 1].DiskBufferSpan.GetEndPoint(_diskBuffer.CurrentSnapshot) ); } }