public void FirstTokenIsFrom()
{
string text = "from";
tokenizer = CreateTokenizer(text);
IPyCompiler.Token token = tokenizer.GetNextToken();
Assert.AreEqual(IPyCompiler.TokenKind.KeywordFrom, token.Kind);
}
void CreateTokenizer(SourceUnit source)
{
PythonCompilerSink sink = new PythonCompilerSink();
PythonCompilerOptions options = new PythonCompilerOptions();
tokenizer = new Tokenizer(sink, options);
tokenizer.Initialize(source);
}
public void FirstTokenIsFrom()
{
string text = "from";
tokenizer = CreateTokenizer(text);
IPyCompiler.Token token = tokenizer.GetNextToken();
Assert.AreEqual(IPyCompiler.TokenKind.KeywordFrom, token.Kind);
}
IPyCompiler.Token GetToken(string text, int tokenNumber)
{
tokenizer = CreateTokenizer(text);
IPyCompiler.Token token = null;
for (int i = 0; i < tokenNumber; ++i) {
token = tokenizer.GetNextToken();
}
return token;
}
IPyCompiler.Token GetToken(string text, int tokenNumber)
{
tokenizer = CreateTokenizer(text);
IPyCompiler.Token token = null;
for (int i = 0; i < tokenNumber; ++i)
{
token = tokenizer.GetNextToken();
}
return(token);
}
private Parser(Tokenizer tokenizer, ErrorSink errorSink, ParserSink parserSink, ModuleOptions languageFeatures) {
ContractUtils.RequiresNotNull(tokenizer, "tokenizer");
ContractUtils.RequiresNotNull(errorSink, "errorSink");
ContractUtils.RequiresNotNull(parserSink, "parserSink");
tokenizer.ErrorSink = new TokenizerErrorSink(this);
_tokenizer = tokenizer;
_errors = errorSink;
_sink = parserSink;
Reset(tokenizer.SourceUnit, languageFeatures);
}
public IPyCompiler.Tokenizer CreateTokenizer(string text)
{
ScriptEngine engine = Python.CreateEngine();
PythonContext context = HostingHelpers.GetLanguageContext(engine) as PythonContext;
StringTextContentProvider textProvider = new StringTextContentProvider(text);
SourceUnit source = context.CreateSourceUnit(textProvider, String.Empty, SourceCodeKind.SingleStatement);
PythonCompilerSink sink = new PythonCompilerSink();
IPyCompiler.PythonCompilerOptions options = new IPyCompiler.PythonCompilerOptions();
tokenizer = new IPyCompiler.Tokenizer(sink, options);
tokenizer.Initialize(source);
return tokenizer;
}
private Parser(CompilerContext context, Tokenizer tokenizer, ErrorSink errorSink, ParserSink parserSink, ModuleOptions languageFeatures) {
ContractUtils.RequiresNotNull(tokenizer, "tokenizer");
ContractUtils.RequiresNotNull(errorSink, "errorSink");
ContractUtils.RequiresNotNull(parserSink, "parserSink");
tokenizer.ErrorSink = new TokenizerErrorSink(this);
_tokenizer = tokenizer;
_errors = errorSink;
if (parserSink != ParserSink.Null) {
_sink = parserSink;
}
_context = context;
Reset(tokenizer.SourceUnit, languageFeatures);
}
public IPyCompiler.Tokenizer CreateTokenizer(string text)
{
ScriptEngine engine = Python.CreateEngine();
PythonContext context = HostingHelpers.GetLanguageContext(engine) as PythonContext;
StringTextContentProvider textProvider = new StringTextContentProvider(text);
SourceUnit source = context.CreateSourceUnit(textProvider, String.Empty, SourceCodeKind.SingleStatement);
PythonCompilerSink sink = new PythonCompilerSink();
IPyCompiler.PythonCompilerOptions options = new IPyCompiler.PythonCompilerOptions();
tokenizer = new IPyCompiler.Tokenizer(sink, options);
tokenizer.Initialize(source);
return(tokenizer);
}
IList<ClassificationSpan> IClassifier.GetClassificationSpans(SnapshotSpan span)
{
var classifications = new List<ClassificationSpan>();
using (var systemState = new SystemState())
{
int startIndex, endIndex;
if (span != null)
{
string spanText = span.GetText();
System.Diagnostics.Debug.WriteLine((spanText != null) ? spanText : string.Empty);
}
// Execute the IPy tokenizer
var tokenizer = new Tokenizer(span.GetText().ToCharArray(), true, systemState, new CompilerContext(string.Empty, new QuietCompilerSink()));
var token = tokenizer.Next();
// Iterate the tokens
while (token.Kind != TokenKind.EndOfFile)
{
// Determine the bounds of the classfication span
startIndex = span.Snapshot.GetLineFromLineNumber(tokenizer.StartLocation.Line - 1 + span.Start.GetContainingLine().LineNumber).Start.Position + tokenizer.StartLocation.Column;
endIndex = span.Snapshot.GetLineFromLineNumber(tokenizer.EndLocation.Line - 1 + span.Start.GetContainingLine().LineNumber).Start.Position + tokenizer.EndLocation.Column;
if (endIndex > span.Snapshot.GetText().Length)
endIndex = span.Snapshot.GetText().Length;
if (endIndex > startIndex && !span.Snapshot.TextBuffer.IsReadOnly(new Span(startIndex, endIndex - startIndex)))
{
// Add the classfication span
classifications.Add(new ClassificationSpan(new SnapshotSpan(span.Snapshot, startIndex, endIndex - startIndex), GetClassificationType(token)));
}
// Get next token
token = tokenizer.Next();
}
}
foreach (var region in span.Snapshot.TextBuffer.GetReadOnlyExtents(span))
{
// Add classfication for read only regions
classifications.Add(new ClassificationSpan(new SnapshotSpan(span.Snapshot, region), classificationRegistryService.GetClassificationType("PythonReadOnlyRegion")));
}
return classifications;
}
private static Parser CreateParserWorker(CompilerContext context, PythonOptions options, bool verbatim) {
ContractUtils.RequiresNotNull(context, "context");
ContractUtils.RequiresNotNull(options, "options");
PythonCompilerOptions compilerOptions = context.Options as PythonCompilerOptions;
if (options == null) {
throw new ValueErrorException(Resources.PythonContextRequired);
}
SourceCodeReader reader;
try {
reader = context.SourceUnit.GetReader();
if (compilerOptions.SkipFirstLine) {
reader.ReadLine();
}
} catch (IOException e) {
context.Errors.Add(context.SourceUnit, e.Message, SourceSpan.Invalid, 0, Severity.Error);
throw;
}
Tokenizer tokenizer = new Tokenizer(context.Errors, compilerOptions, verbatim);
tokenizer.Initialize(null, reader, context.SourceUnit, SourceLocation.MinValue);
tokenizer.IndentationInconsistencySeverity = options.IndentationInconsistencySeverity;
Parser result = new Parser(context, tokenizer, context.Errors, context.ParserSink, compilerOptions.Module);
result._sourceReader = reader;
return result;
}
public int Exec(ref Guid pguidCmdGroup, uint nCmdID, uint nCmdexecopt, IntPtr pvaIn, IntPtr pvaOut)
{
// Handle VS commands to support code snippets
if (pguidCmdGroup == VSConstants.VSStd2K)
{
if (nCmdID == (uint)VSConstants.VSStd2KCmdID.INSERTSNIPPET || nCmdID == (uint)VSConstants.VSStd2KCmdID.SURROUNDWITH)
{
IVsTextManager2 textManager = (IVsTextManager2)this.serviceProvider.GetService(typeof(SVsTextManager));
IVsExpansionManager expansionManager;
if (VSConstants.S_OK == textManager.GetExpansionManager(out expansionManager))
{
expansionManager.InvokeInsertionUI(
vsTextView,
this,
Constants.IronPythonLanguageServiceGuid,
null,
0,
1,
null,
0,
1,
"Insert Snippet",
string.Empty);
}
return VSConstants.S_OK;
}
if (this.expansionSession != null)
{
// Handle VS Expansion (Code Snippets) keys
if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.TAB))
{
if (expansionSession.GoToNextExpansionField(0) == VSConstants.S_OK)
return VSConstants.S_OK;
}
else if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.BACKTAB))
{
if (expansionSession.GoToPreviousExpansionField() == VSConstants.S_OK)
return VSConstants.S_OK;
}
else if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.RETURN || nCmdID == (uint)VSConstants.VSStd2KCmdID.CANCEL))
{
if (expansionSession.EndCurrentExpansion(0) == VSConstants.S_OK)
{
expansionSession = null;
return VSConstants.S_OK;
}
}
}
// Handle Edit.ListMembers or Edit.CompleteWord commands
if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.SHOWMEMBERLIST || nCmdID == (uint)VSConstants.VSStd2KCmdID.COMPLETEWORD))
{
if (activeSession != null)
{
activeSession.Dismiss();
}
ShowCompletion();
return VSConstants.S_OK;
}
// Handle Enter/Tab commit keys
if (activeSession != null && (nCmdID == (uint)VSConstants.VSStd2KCmdID.RETURN || nCmdID == (uint)VSConstants.VSStd2KCmdID.TAB))
{
if (activeSession.SelectedCompletionSet.SelectionStatus.IsSelected)
activeSession.Commit();
else
activeSession.Dismiss();
return VSConstants.S_OK;
}
// Handle Code Snippets after pressing the Tab key without completion
if (activeSession == null && (nCmdID == (uint)VSConstants.VSStd2KCmdID.TAB))
{
using (var systemState = new SystemState())
{
// Get the current line text until the cursor
var line = this.textView.GetTextViewLineContainingBufferPosition(this.textView.Caret.Position.BufferPosition);
var text = this.textView.TextSnapshot.GetText(line.Start.Position, this.textView.Caret.Position.BufferPosition - line.Start.Position);
// Create a tokenizer for the text
var tokenizer = new Tokenizer(text.ToCharArray(), true, systemState, new CompilerContext(string.Empty, new QuietCompilerSink()));
// Get the last token in the text
Token currentToken, lastToken = null;
while ((currentToken = tokenizer.Next()).Kind != TokenKind.NewLine)
{
lastToken = currentToken;
}
if (lastToken != null && lastToken.Kind != TokenKind.Constant)
{
var expansionManager = (IVsTextManager2)this.serviceProvider.GetService(typeof(SVsTextManager));
var snippetsEnumerator = new SnippetsEnumerator(expansionManager, Constants.IronPythonLanguageServiceGuid);
// Search a snippet that matched the token text
var expansion = snippetsEnumerator.FirstOrDefault(e => e.title == lastToken.Value.ToString());
if (expansion.title != null)
{
// Set the location where the snippet will be inserted
int startLine, startColumn, endLine, endColumn;
this.vsTextView.GetCaretPos(out startLine, out endColumn);
startColumn = endColumn - expansion.title.Length;
endLine = startLine;
// Insert the snippet
InsertCodeExpansion(expansion, startLine, startColumn, endLine, endColumn);
return VSConstants.S_OK;
}
}
}
}
// Hanlde other keys
if ((nCmdID == (uint)VSConstants.VSStd2KCmdID.TYPECHAR))
{
char typedChar = (char)(ushort)Marshal.GetObjectForNativeVariant(pvaIn);
if (activeSession == null)
{
// Handle trigger keys
// Check if the typed char is a trigger
if (IsTriggerKey(typedChar))
{
var result = this.nextCommandTarget.Exec(pguidCmdGroup, nCmdID, nCmdexecopt, pvaIn, pvaOut);
ShowCompletion();
return result;
}
}
else
{
// Handle commit keys
// Check if the typed char is a commit key
if (IsCommitKey(typedChar))
{
if (activeSession.SelectedCompletionSet.SelectionStatus.IsSelected)
activeSession.Commit();
else
activeSession.Dismiss();
var result = this.nextCommandTarget.Exec(pguidCmdGroup, nCmdID, nCmdexecopt, pvaIn, pvaOut);
// Check we should trigger completion after comitting the previous session (for example, after typing dot '.')
if (IsTriggerKey(typedChar))
ShowCompletion();
return result;
}
}
}
}
// we haven't handled this command so pass it onto the next target
return this.nextCommandTarget.Exec(pguidCmdGroup, nCmdID, nCmdexecopt, pvaIn, pvaOut);
}
public void SetSource(string source, int offset)
{
tokenizer = new Tokenizer(source.ToCharArray(offset, source.Length - offset), true, systemState,
new CompilerContext("", new Microsoft.Samples.VisualStudio.IronPythonInference.QuietCompilerSink()));
}
private void SearchForCodeBlocks(IVsTextLines buffer)
{
// We don't want any change in the buffer while we are parsing,
// so we have to lock it.
ErrorHandler.ThrowOnFailure(buffer.LockBufferEx((uint)BufferLockFlags.BLF_READ));
try {
// Find the total number of lines in the buffer.
int totalLines;
ErrorHandler.ThrowOnFailure(buffer.GetLineCount(out totalLines));
// Set the initial values for the variables used during the parsing.
SimpleParserState state = SimpleParserState.WaitForExternalSource;
TextSpanAndCookie blockSpan = new TextSpanAndCookie();
// Parse all the lines in the buffer
for (int line = 0; line < totalLines; ++line) {
// Get the text of the current line.
int lineLen;
ErrorHandler.ThrowOnFailure(buffer.GetLengthOfLine(line, out lineLen));
if (0 == lineLen) {
// The line is empty, no point in parsing it.
continue;
}
string lineText;
ErrorHandler.ThrowOnFailure(buffer.GetLineText(line, 0, line, lineLen, out lineText));
// Create the tokenizer.
CompilerContext context = new CompilerContext("", new QuietCompilerSink());
using (SystemState systemState = new SystemState()) {
tokenizer = new Tokenizer(lineText.ToCharArray(), true, systemState, context);
Token token = null;
string commentText;
// Read all the token looking for the code blocks inside a Snippet Statements
// nested in an External Source statement. Note that the standard IronPython
// parser does not return such statements and this is the reason for this
// parser.
while (!tokenizer.IsEndOfFile) {
token = tokenizer.Next();
// This parser is strange in that it is only interested in comments:
// an external code statement is in the form
// #ExternalSource("PathOfTheOriginalFile", originalLineNumber)
// ... (some code) ...
// #End ExternalSource
// and a snippet statement is
// # Snippet Statement
// ... (some code) ...
// #End Snippet Statement
// So if we want to find the text region inside a snippet nested
// inside an external source, we are only interested in the comment tokens.
if (TokenKind.Comment != token.Kind) {
continue;
}
// The comments are line comments, so the comment's text is everything that
// is after the beginning of the comment.
commentText = CleanCommentStart(lineText.Substring(tokenizer.StartLocation.Column));
if (string.IsNullOrEmpty(commentText)) {
continue;
}
switch (state) {
case SimpleParserState.WaitForExternalSource:
// This function returns a non zero value only if the comment text
// is a valid external source statment.
blockSpan.ulHTMLCookie = ParseExternalSource(commentText);
if (0 != blockSpan.ulHTMLCookie) {
// The CodeDOM provider is adding 1 to the line number, but in this
// case the number is actualy the HTML editor's cookie, so we have to
// restore the original value.
blockSpan.ulHTMLCookie -= 1;
state = SimpleParserState.WaitForSnippet;
}
break;
case SimpleParserState.WaitForSnippet:
// Check if this comment is the beginning of a snippet block.
if (IsBeginSnippet(commentText)) {
// This is the beginning of a snippet block, so
// the actual code will start at the beginning of the
// next line.
blockSpan.CodeSpan.iStartLine = line + 1;
// Set the default for the start index.
blockSpan.CodeSpan.iStartIndex = 0;
// Now we have to find the end of the snippet section
// to complete the span of the code.
state = SimpleParserState.WaitForEndSnippet;
} else if (IsEndExternalSource(commentText)) {
// This was and external block not related to the HTML editor.
// Reset the text span and wait for the next external source.
blockSpan = new TextSpanAndCookie();
state = SimpleParserState.WaitForExternalSource;
}
break;
case SimpleParserState.WaitForEndSnippet:
if (IsEndSnippet(commentText)) {
// The code block ends at the end of the line before
// this token.
// Update the data about the code span and add the
// block to the list of the blocks found.
blockSpan.CodeSpan.iEndLine = line - 1;
ErrorHandler.ThrowOnFailure(
buffer.GetLengthOfLine(line - 1, out blockSpan.CodeSpan.iEndIndex));
blocks.Add(blockSpan);
blockSpan = new TextSpanAndCookie();
state = SimpleParserState.WaitForEndExternal;
}
break;
case SimpleParserState.WaitForEndExternal:
// We expect only one snippet block inside the external source
// section, so here we skip everything between the end of the first
// snippet block and the end of the external code section.
if (IsEndExternalSource(commentText)) {
state = SimpleParserState.WaitForExternalSource;
}
break;
}
}
}
}
} finally {
// Make sure that the buffer is always unlocked when we exit this function.
buffer.UnlockBufferEx((uint)BufferLockFlags.BLF_READ);
}
}
private Parser(CompilerContext context, Tokenizer tokenizer)
{
this.tokenizer = tokenizer;
this.context = context;
}