internal static ExtractedClassName GetFirstClassNameFullyQualified(Stream binaryStream)
{
try
{
CSharpTokenizer tokens = new CSharpTokenizer(binaryStream, false);
return(Extract(tokens));
}
catch (DecoderFallbackException)
{
CSharpTokenizer tokenizer2 = new CSharpTokenizer(binaryStream, true);
return(Extract(tokenizer2));
}
}
/// <summary>
/// Parse a C# file and get the first class name, fully qualified with namespace.
/// </summary>
/// <param name="binaryStream"></param>
/// <returns></returns>
internal static ExtractedClassName GetFirstClassNameFullyQualified(Stream binaryStream)
{
try
{
var tokens = new CSharpTokenizer(binaryStream, /* forceANSI */ false);
return(Extract(tokens));
}
catch (DecoderFallbackException)
{
// There was no BOM and there are non UTF8 sequences. Fall back to ANSI.
var tokens = new CSharpTokenizer(binaryStream, /* forceANSI */ true);
return(Extract(tokens));
}
}
/*
* Method: AssertTokenizeStream
*
* Tokenize a string ('source') and compare it to the expected set of tokens.
* Also compare the source that is regenerated by concatenating all of the tokens
* to 'expectedSource'.
*/
static private void AssertTokenizeStream
(
Stream source,
string expectedSource,
string expectedTokenKey,
int expectedLastLineNumber
)
{
CSharpTokenizer tokens = new CSharpTokenizer
(
source,
false
);
string results = "";
string tokenKey = "";
int lastLine = 0;
bool syntaxError = false;
foreach (Token t in tokens)
{
results += t.InnerText;
lastLine = t.Line;
if (!syntaxError)
{
// Its not really a file name, but GetExtension serves the purpose of getting the class name without
// the namespace prepended.
string tokenClass = t.ToString();
int pos = tokenClass.LastIndexOfAny(new char[] { '+', '.' });
tokenKey += ".";
tokenKey += tokenClass.Substring(pos + 1);
}
if (t is SyntaxErrorToken)
{
// Stop processing after the first syntax error because
// the order of tokens after this is an implementation detail and
// shouldn't be encoded into the unit tests.
syntaxError = true;
}
}
tokenKey = tokenKey.Replace("Token", "");
Console.WriteLine(tokenKey);
Assert.AreEqual(expectedSource, results);
Assert.AreEqual(expectedTokenKey, tokenKey);
Assert.AreEqual(expectedLastLineNumber, lastLine);
}
public override IEnumerable <Chunk> GetChunks(ColorScheme style, DocumentLine line, int offset, int length)
{
// Multiline comment
if (line.StartSpan.Count != 0 && line.StartSpan.Peek().Begin.Pattern == "@*")
{
return(base.GetChunks(style, line, offset, length));
}
var tokenizer = new CSharpTokenizer(new SeekableTextReader(doc.GetTextAt(offset, length)));
chunks = new List <Chunk> ();
CSharpSymbol symbol;
CSharpSymbol prevSymbol = null;
int off = offset;
currentState = State.None;
while ((symbol = tokenizer.NextSymbol()) != null)
{
// Apostrophes in text
bool inApostrophes = false;
if (symbol.Type == CSharpSymbolType.CharacterLiteral && prevSymbol != null && Char.IsLetterOrDigit(prevSymbol.Content.Last()))
{
if (symbol.Content.Last() == '\'')
{
inApostrophes = true;
}
else
{
chunks.Add(new Chunk(off, 1, "Plain Text"));
off++;
tokenizer = new CSharpTokenizer(new SeekableTextReader(symbol.Content.Substring(1)));
symbol = tokenizer.NextSymbol();
prevSymbol = null;
}
}
string chunkStyle = inApostrophes ? "Plain Text" : GetStyleForChunk(symbol, prevSymbol, off);
chunks.Add(new Chunk(off, symbol.Content.Length, chunkStyle));
prevSymbol = symbol;
off += symbol.Content.Length;
}
return(chunks);
}
static void Main(string[] args)
{
// základní mechanismus použití (v minimální implementaci se budou lišit jen třídy tokenizeru a formátoru)
ISHTokenizer t = new CSharpTokenizer(); // vytvoříme tokenizer
StreamReader reader;
OpenFileDialog fd = new OpenFileDialog();
fd.DefaultExt = "cs";
if (fd.ShowDialog() == DialogResult.OK)
{
reader = new StreamReader(fd.OpenFile());
}
else
{
throw new Exception("File not selected");
}
t.setInput(reader); // nastavíme mu vstupní proud
ISHFormater f = new HTMLFormater(); // vytvoříme formátor
f.setSource(t); // nastavíme mu tokenizer jako vstupní zdroj tokenů
StringBuilder builder = new StringBuilder();
StringWriter writer = new StringWriter(builder);
f.setOutput(writer); // nastavíme mu výstupní proud
f.run(); // a spustíme proces zvýrazňování syntaxe
StreamWriter file = new StreamWriter(@"syntax/syntax.html");
file.Write(writer);
writer.Close();
file.Close();
}
/// <summary>
/// Extract the class name.
/// </summary>
private static ExtractedClassName Extract(CSharpTokenizer tokens)
{
var state = new ParseState();
var result = new ExtractedClassName();
foreach (Token t in tokens)
{
// Search first for the namespace keyword
if (t is KeywordToken)
{
state.Reset();
if (t.InnerText == "namespace")
{
state.ResolvingNamespace = true;
if (state.InsideConditionalDirective)
{
result.IsInsideConditionalBlock = true;
}
}
else if (t.InnerText == "class")
{
state.ResolvingClass = true;
if (state.InsideConditionalDirective)
{
result.IsInsideConditionalBlock = true;
}
}
}
else if (t is CSharpTokenizer.OpenScopeToken)
{
state.PushNamespacePart(state.Namespace);
state.Reset();
}
else if (t is CSharpTokenizer.CloseScopeToken)
{
state.Reset();
state.PopNamespacePart();
}
else if (t is OperatorOrPunctuatorToken)
{
if (state.ResolvingNamespace)
{
if (t.InnerText == ".")
{
state.Namespace += ".";
}
}
}
else if (t is IdentifierToken)
{
// If we're resolving a namespace, then this is part of the namespace.
if (state.ResolvingNamespace)
{
state.Namespace += t.InnerText;
}
// If we're resolving a class, then we're done. We found the class name.
else if (state.ResolvingClass)
{
// We're done.
result.Name = state.ComposeQualifiedClassName(t.InnerText);
return(result);
}
}
else if (t is OpenConditionalDirectiveToken)
{
state.OpenConditionalDirective();
}
else if (t is CloseConditionalDirectiveToken)
{
state.CloseConditionalDirective();
}
}
return(result);
}
private static ExtractedClassName Extract(CSharpTokenizer tokens)
{
ParseState state = new ParseState();
ExtractedClassName name = new ExtractedClassName();
foreach (Token token in tokens)
{
if (token is KeywordToken)
{
state.Reset();
if (token.InnerText == "namespace")
{
state.ResolvingNamespace = true;
if (state.InsideConditionalDirective)
{
name.IsInsideConditionalBlock = true;
}
}
else if (token.InnerText == "class")
{
state.ResolvingClass = true;
if (state.InsideConditionalDirective)
{
name.IsInsideConditionalBlock = true;
}
}
}
else if (token is CSharpTokenizer.OpenScopeToken)
{
state.PushNamespacePart(state.Namespace);
state.Reset();
}
else if (token is CSharpTokenizer.CloseScopeToken)
{
state.Reset();
state.PopNamespacePart();
}
else if (token is OperatorOrPunctuatorToken)
{
if (state.ResolvingNamespace && (token.InnerText == "."))
{
state.Namespace = state.Namespace + ".";
}
}
else if (token is IdentifierToken)
{
if (!state.ResolvingNamespace)
{
if (state.ResolvingClass)
{
name.Name = state.ComposeQualifiedClassName(token.InnerText);
return(name);
}
}
else
{
state.Namespace = state.Namespace + token.InnerText;
}
}
else if (token is OpenConditionalDirectiveToken)
{
state.OpenConditionalDirective();
}
else if (token is CloseConditionalDirectiveToken)
{
state.CloseConditionalDirective();
}
}
return(name);
}
public override IEnumerable<Chunk> GetChunks (ColorScheme style, DocumentLine line, int offset, int length)
{
// Multiline comment
if (line.StartSpan.Count != 0 && line.StartSpan.Peek ().Begin.Pattern == "@*")
return base.GetChunks (style, line, offset, length);
var tokenizer = new CSharpTokenizer (new SeekableTextReader (doc.GetTextAt (offset, length)));
chunks = new List<Chunk> ();
CSharpSymbol symbol;
CSharpSymbol prevSymbol = null;
int off = line.Offset;
currentState = State.None;
while ((symbol = tokenizer.NextSymbol ()) != null) {
// Apostrophes in text
bool inApostrophes = false;
if (symbol.Type == CSharpSymbolType.CharacterLiteral && prevSymbol != null && Char.IsLetterOrDigit (prevSymbol.Content.Last ())) {
if (symbol.Content.Last () == '\'')
inApostrophes = true;
else {
chunks.Add (new Chunk (off, 1, "Plain Text"));
off++;
tokenizer = new CSharpTokenizer (new SeekableTextReader (symbol.Content.Substring (1)));
symbol = tokenizer.NextSymbol ();
prevSymbol = null;
}
}
string chunkStyle = inApostrophes ? "Plain Text" : GetStyleForChunk (symbol, prevSymbol, off);
chunks.Add (new Chunk (off, symbol.Content.Length, chunkStyle));
prevSymbol = symbol;
off += symbol.Content.Length;
}
return chunks;
}
/// <summary>
/// Extract the class name.
/// </summary>
private static ExtractedClassName Extract(CSharpTokenizer tokens)
{
var state = new ParseState();
var result = new ExtractedClassName();
foreach (Token t in tokens)
{
// Search first for the namespace keyword
if (t is KeywordToken)
{
state.Reset();
if (t.InnerText == "namespace")
{
state.ResolvingNamespace = true;
if (state.InsideConditionalDirective)
{
result.IsInsideConditionalBlock = true;
}
}
else if (t.InnerText == "class")
{
state.ResolvingClass = true;
if (state.InsideConditionalDirective)
{
result.IsInsideConditionalBlock = true;
}
}
}
else if (t is CSharpTokenizer.OpenScopeToken)
{
state.PushNamespacePart(state.Namespace);
state.Reset();
}
else if (t is CSharpTokenizer.CloseScopeToken)
{
state.Reset();
state.PopNamespacePart();
}
else if (t is OperatorOrPunctuatorToken)
{
if (state.ResolvingNamespace)
{
// If we see a ';' while resolving a namespace, we assume it's a file-scoped namespace
// namespace foo.bar; <- At this point in code, we're at the semicolon.
// class test { ... }
// https://github.com/dotnet/csharplang/blob/088f20b6f9b714a7b68f6d792d54def0f3b3057e/proposals/csharp-10.0/file-scoped-namespaces.md
if (t.InnerText == ";")
{
state.PushNamespacePart(state.Namespace);
state.Reset();
}
else if (t.InnerText == ".")
{
state.Namespace += ".";
}
}
}
else if (t is IdentifierToken)
{
// If we're resolving a namespace, then this is part of the namespace.
if (state.ResolvingNamespace)
{
state.Namespace += t.InnerText;
}
// If we're resolving a class, then we're done. We found the class name.
else if (state.ResolvingClass)
{
// We're done.
result.Name = state.ComposeQualifiedClassName(t.InnerText);
return(result);
}
}
else if (t is OpenConditionalDirectiveToken)
{
state.OpenConditionalDirective();
}
else if (t is CloseConditionalDirectiveToken)
{
state.CloseConditionalDirective();
}
}
return(result);
}
public void TypeCast()
{
CSharpTokenizer tokenizer = new CSharpTokenizer();
Assert.IsInstanceOfType(typeof(int), _parser.EvaluateToObject("(int)5L"));
}
