public void AddResultEntity(SourceCodeFile sourceCodeFile) { if (stageExt.IsRead || IsIncludeIntermediateResult) { AddEntity(sourceCodeFiles, sourceCodeFile); } }
private static int CompileFile(string inputFilePath, string outputFilePath) { try { var file = new SourceCodeFile(inputFilePath); var ast = AnalyzeCode(file); if (ast == null) { return(1); } var byteCode = Compiler.Compile(ast); if (Context.Instance.ErrorsReported > 0) { return(1); } File.WriteAllBytes(inputFilePath ?? "out.ibc", byteCode.ToArray()); return(0); } catch (IOException e) { Console.WriteLine(e.Message); } return(1); }
private static bool RunCode(SourceCodeFile file) { var tokens = Lexer.ScanTokens(file); if (Context.Instance.ErrorsReported > 0) { return(true); } var ast = Parser.Parse(tokens); if (Context.Instance.ErrorsReported > 0) { return(true); } // TODO: Scanning on demand. tokens.Clear(); var byteCode = Compiler.Compile(ast); if (Context.Instance.ErrorsReported > 0) { return(true); } #if DEBUG Disassembler.Disassemble(byteCode.ToArray()); Console.WriteLine("----------------------------------------"); #endif VirtualMachine.Execute(byteCode); return(false); }
public SourceCodeSpan(SourceCodeFile file, int begin, int length) { File = file; Begin = begin; Length = length; (Line, Column) = File.GetLineAndColumn(begin); }
public SourceCodeFile ReadFile(string fileName) { SourceCodeFile result = null; try { var removeBeginLength = Path.Length + (Path.EndsWith("\\") ? 0 : 1); var shortFileName = System.IO.Path.GetFileName(fileName); result = new SourceCodeFile(shortFileName); int removeEndLength = shortFileName.Length + 1; result.RelativePath = removeEndLength + removeBeginLength > fileName.Length ? "" : fileName.Remove(fileName.Length - removeEndLength).Remove(0, removeBeginLength); result.Code = File.ReadAllText(fileName); return(result); } catch (Exception ex) { Logger.LogError(new ReadException(fileName, ex)); if (result == null) { result = new SourceCodeFile(fileName); } } return(result); }
public void LoadFileNotExists() { var scf = new SourceCodeFile(@"..\..\..\..\assets\TestData\FileNotExists.prg"); scf.Should().NotBeNull(); scf.SourceCode.Should().BeEmpty(); }
protected override string PreprocessText(SourceCodeFile file) { var result = base.PreprocessText(file); bool trimmed = false; var vtIndex = result.IndexOf('\v'); if (vtIndex != -1 && vtIndex > 0 && result[vtIndex - 1] == '\n' && vtIndex + 1 < result.Length && char.IsDigit(result[vtIndex + 1])) { result = result.Remove(vtIndex); trimmed = true; } // TODO: Fix Hardcode! int lastPhpInd = result.LastIndexOf("?>"); if (lastPhpInd != -1) { if (lastPhpInd + "?>".Length + 12 <= result.Length && result.Substring(lastPhpInd + "?>".Length, 12) == "\r\nChangelog:") { result = result.Remove(lastPhpInd + "?>".Length); trimmed = true; } } if (trimmed) { Logger.LogDebug($"File {Path.Combine(file.RelativePath, file.Name)} has been trimmed."); } return(result); }
/// <summary> /// Converts \r to \r\n. /// </summary> /// <param name="file"></param> /// <returns></returns> protected virtual string PreprocessText(SourceCodeFile file) { var text = file.Code; var result = new StringBuilder(text.Length); int i = 0; while (i < text.Length) { if (text[i] == '\r') { if (i + 1 >= text.Length) { result.Append('\n'); } else if (text[i + 1] != '\n') { result.Append('\n'); } else { result.Append(text[i]); } } else { result.Append(text[i]); } i++; } return(result.ToString()); }
public ParseTree Parse(SourceCodeFile sourceCodeFile) { if (MaxStackSize == 0 && MaxTimespan == 0) { return(TokenizeAndParse(sourceCodeFile)); } else { ParseTree result = null; Thread thread = new Thread(delegate() { result = TokenizeAndParse(sourceCodeFile); }, MaxStackSize); thread.Start(); bool finished = thread.Join(MaxTimespan == 0 ? int.MaxValue : MaxTimespan); if (!finished) { thread.Interrupt(); thread.Abort(); Logger.LogError($"Parsing error in \"{sourceCodeFile.Name}\": Timeout ({MaxTimespan}) expired"); } return(result); } }
protected ParserRuleContext ParseTokens(SourceCodeFile sourceCodeFile, AntlrMemoryErrorListener errorListener, BufferedTokenStream codeTokenStream, Func <ITokenStream, Parser> initParserFunc = null, Func <Parser, ParserRuleContext> parseFunc = null) { Parser parser = initParserFunc != null?initParserFunc(codeTokenStream) : InitParser(codeTokenStream); parser.RemoveErrorListeners(); Parser = parser; ParserRuleContext syntaxTree; if (UseFastParseStrategyAtFirst) { parser.Interpreter.PredictionMode = PredictionMode.Sll; parser.ErrorHandler = new BailErrorStrategy(); parser.TrimParseTree = true; parserLock.EnterReadLock(); try { syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser); } catch (ParseCanceledException) { parserLock.ExitReadLock(); parser.AddErrorListener(errorListener); codeTokenStream.Reset(); parser.Reset(); parser.Interpreter.PredictionMode = PredictionMode.Ll; parser.ErrorHandler = new DefaultErrorStrategy(); parserLock.EnterReadLock(); syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser); } finally { parserLock.ExitReadLock(); } } else { parser.AddErrorListener(errorListener); parserLock.EnterReadLock(); try { syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser); } finally { parserLock.ExitReadLock(); } } ClearParserCacheIfRequired(parser); #if DEBUG var tree = syntaxTree.ToStringTree(parser); #endif return(syntaxTree); }
static void Main(string[] args) { var fileRepository = new FileCodeRepository("code.php"); var patternsRepostiory = new FilePatternsRepository("patterns.json"); var workflow = new Workflow(fileRepository, patternsRepostiory); workflow.IsIncludeIntermediateResult = true; WorkflowResult result = workflow.Process(); var lineTokenDict = new Dictionary <int, List <string> >(); foreach (var matchingResult in result.MatchingResults) { var matchedNode = matchingResult.Nodes.Last(); UstNode[] descendants = matchedNode.GetAllDescendants(); foreach (UstNode descendant in descendants) { string text = null; if (descendant.NodeType == NodeType.InvocationExpression) { text = "Method invocation: " + ((InvocationExpression)descendant).Target; } else if (descendant.NodeType == NodeType.IndexerDeclaration) { text = "Arrays with user input: " + ((IndexerExpression)descendant).Target; } else if (descendant.NodeType == NodeType.CastExpression) { text = "Cast to type: " + ((CastExpression)descendant).Type; } if (!string.IsNullOrEmpty(text)) { SourceCodeFile sourceCodeFile = result.SourceCodeFiles.Single(file => file.FullPath == descendant.FileNode.FileName.Text); LineColumnTextSpan lineColumnTextSpan = sourceCodeFile.GetLineColumnTextSpan(descendant.TextSpan); List <string> strs; if (!lineTokenDict.TryGetValue(lineColumnTextSpan.BeginLine, out strs)) { strs = new List <string>(); lineTokenDict[lineColumnTextSpan.BeginLine] = strs; } strs.Add(text); } } } foreach (var item in lineTokenDict) { Console.WriteLine(String.Format("{0}: {1}", item.Key, string.Join(", ", item.Value))); } Console.ReadLine(); }
public ParseTree Parse(SourceCodeFile sourceCodeFile) { CSharpRoslynParseTree result = null; var filePath = Path.Combine(sourceCodeFile.RelativePath, sourceCodeFile.Name); if (sourceCodeFile.Code != null) { SyntaxNode root = null; try { SyntaxTree syntaxTree = CSharpSyntaxTree.ParseText(sourceCodeFile.Code, null, filePath); result = new CSharpRoslynParseTree(syntaxTree); root = syntaxTree.GetRoot(); result.Comments = root.DescendantTrivia().Where(node => { SyntaxKind kind = node.Kind(); return(kind == SyntaxKind.SingleLineCommentTrivia || kind == SyntaxKind.MultiLineCommentTrivia || kind == SyntaxKind.SingleLineDocumentationCommentTrivia || kind == SyntaxKind.MultiLineDocumentationCommentTrivia || kind == SyntaxKind.DocumentationCommentExteriorTrivia || kind == SyntaxKind.XmlComment); }).ToArray(); IEnumerable <Diagnostic> diagnostics = root.GetDiagnostics(); foreach (var diagnostic in diagnostics) { if (diagnostic.Severity == DiagnosticSeverity.Error && diagnostic.Id != "CS1029") { var textSpan = RoslynHelper.ConvertTextSpan(diagnostic.Location); Logger.LogError(new ParsingException(filePath, message: diagnostic.ToString()) { TextSpan = textSpan }); } } } catch (Exception ex) { Logger.LogError(new ParsingException(filePath, ex)); result = new CSharpRoslynParseTree(); } } else { result = new CSharpRoslynParseTree(); } result.FileName = filePath; result.FileData = sourceCodeFile.Code; return(result); }
protected ParseTree ReadAndParse(string fileName, TWorkflowResult workflowResult, CancellationToken cancellationToken = default(CancellationToken)) { ParseTree result = null; var stopwatch = new Stopwatch(); string file = fileName; if (stageHelper.IsContainsRead) { if (SourceCodeRepository.IsFileIgnored(fileName)) { Logger.LogInfo($"File {fileName} has not been read."); return(null); } stopwatch.Restart(); SourceCodeFile sourceCodeFile = SourceCodeRepository.ReadFile(fileName); stopwatch.Stop(); Logger.LogInfo($"File {fileName} has been read (Elapsed: {stopwatch.Elapsed})."); workflowResult.AddProcessedCharsCount(sourceCodeFile.Code.Length); workflowResult.AddProcessedLinesCount(TextHelper.GetLinesCount(sourceCodeFile.Code)); workflowResult.AddReadTime(stopwatch.ElapsedTicks); workflowResult.AddResultEntity(sourceCodeFile); cancellationToken.ThrowIfCancellationRequested(); file = sourceCodeFile.RelativePath; if (stageHelper.IsContainsParse) { stopwatch.Restart(); Language?detectedLanguage = LanguageDetector.DetectIfRequired(sourceCodeFile.Name, sourceCodeFile.Code, Languages); if (detectedLanguage == null) { Logger.LogInfo($"Input languages set is empty or {sourceCodeFile.Name} language has not been detected. File has not been converter."); return(null); } result = ParserConverterSets[(Language)detectedLanguage].Parser.Parse(sourceCodeFile); stopwatch.Stop(); Logger.LogInfo($"File {fileName} has been parsed (Elapsed: {stopwatch.Elapsed})."); workflowResult.AddParseTime(stopwatch.ElapsedTicks); var antlrParseTree = result as AntlrParseTree; if (antlrParseTree != null) { workflowResult.AddLexerTime(antlrParseTree.LexerTimeSpan.Ticks); workflowResult.AddParserTicks(antlrParseTree.ParserTimeSpan.Ticks); } cancellationToken.ThrowIfCancellationRequested(); } } return(result); }
private static List <Statement> AnalyzeCode(SourceCodeFile file) { var tokens = Lexer.ScanTokens(file); if (Context.Instance.ErrorsReported > 0) { return(null); } var ast = Parser.Parse(tokens); return(Context.Instance.ErrorsReported == 0 ? ast : null); }
public async Task AddOrUpdateSourceCodeAsync() { // Given var project = new InMemoryProject(); var code = new SourceCodeFile("Foo.cs", "public class Foo { }"); // When project.AddOrUpdate(code); var compilation = await project.CompileAsync(); // Then Assert.NotNull(compilation); Assert.True(compilation.ContainsSymbolsWithName("Foo")); }
public static List <Token> ScanTokens(SourceCodeFile file) { var lexer = new Lexer(file); while (!lexer.IsAtEnd()) { lexer._start = lexer._current; lexer.ScanToken(); } lexer._start = lexer._current; lexer.PushToken(TokenType.EndOfFile); return(lexer._tokens); }
private static int AnalyzeFile(string filePath) { try { var file = new SourceCodeFile(filePath); return(AnalyzeCode(file) != null ? 1 : 0); } catch (IOException e) { Console.WriteLine(e.Message); } return(0); }
public void resolveFileLocationsOfExtraSourceCodeFilesToCompile() { if (ExtraSourceCodeFilesToCompile.size() > 0) { // try to resolve local file references try { if (this.SourceCodeFile.isNull()) // in case this is not set { SourceCodeFile = PublicDI.CurrentScript; } for (int i = 0; i < ExtraSourceCodeFilesToCompile.size(); i++) { var fileToResolve = ExtraSourceCodeFilesToCompile[i].trim(); var resolved = false; // try using SourceCodeFile.directoryName() if (fileToResolve.fileExists().isFalse()) { if (SourceCodeFile.valid() && SourceCodeFile.isFile()) { var resolvedFile = SourceCodeFile.directoryName().pathCombine(fileToResolve); if (resolvedFile.fileExists()) { ExtraSourceCodeFilesToCompile[i] = resolvedFile; resolved = true; } } } // try using the localscripts folders var localMapping = fileToResolve.local(); if (localMapping.valid()) { ExtraSourceCodeFilesToCompile[i] = localMapping; resolved = true; } if (resolved.isFalse() && fileToResolve.fileExists().isFalse()) { ExtraSourceCodeFilesToCompile[i] = ExtraSourceCodeFilesToCompile[i].fullPath(); } } } catch (Exception ex) { ex.log("in compileExtraSourceCodeReferencesAndUpdateReferencedAssemblies while resolving ExtraSourceCodeFilesToCompile"); } } }
public void AddSourceCodeFile(SourceCodeFile sourceCodeFile, CodeInfo codeInfo) { if (sourceCodeFile == null || codeInfo == null || sourceCodeFile.FileName != codeInfo.FileName) { throw new ArgumentException(); } SourceCodeFiles.Add(codeInfo, sourceCodeFile); var buffer = new StringBuilder(); foreach (var methodInfo in codeInfo.MethodList) { void AddMethodLocations(Dictionary <uint, List <StatementLocation> > dict, int startOffset, int count) { buffer.Length = 0; for (int i = 0; i < count; i++) { buffer.Append(methodInfo.StatementList[startOffset + i].Hashcode.ToString()).Append('#'); } var statementBlockHashCode = GetStatementListHashCode(buffer); if (!dict.TryGetValue(statementBlockHashCode, out var locationList)) { locationList = new List <StatementLocation>(); dict[statementBlockHashCode] = locationList; } locationList.Add(new StatementLocation { CodeInfo = codeInfo, Method = methodInfo, Offset = startOffset }); } if (methodInfo.StatementList.Count >= Configuration.MinLineForFullMethodDuplicateCheck && methodInfo.StatementList.Count < Configuration.MinLineForDuplicate) { AddMethodLocations(FullMethods, 0, methodInfo.StatementList.Count); } if (methodInfo.StatementList.Count >= Configuration.MinLineForDuplicate) { for (int i = 0; i < methodInfo.StatementList.Count - Configuration.MinLineForDuplicate; i++) { AddMethodLocations(Locations, i, Configuration.MinLineForDuplicate); } } } }
public SourceCodeFile ReadFile(string fileName) { var result = new SourceCodeFile(fileName); try { result.RelativePath = ""; result.Code = Code; } catch (Exception ex) { Logger.LogError(new ReadException(fileName, ex)); } return(result); }
private static int RunFile(string filePath) { try { var sourceFile = new SourceCodeFile(filePath); var reportedErrors = RunCode(sourceFile); return(reportedErrors ? 1 : 0); } catch (IOException e) { Console.WriteLine(e.Message); } return(0); }
public void Parse_NewLine_CorrectLineColumn() { string fileText = File.ReadAllText(Path.Combine(TestHelper.TestsDataPath, "newLine -r-n.php")); var lineEnds = new string[] { "\r", "\n", "\r\n" }; foreach (var lineEnd in lineEnds) { var phpParser = new PhpAntlrParser(); string code = fileText.Replace("\r\n", lineEnd); var sourceCodeFile = new SourceCodeFile { Name = "newLine.php", Code = code }; var parseTree = (PhpAntlrParseTree)phpParser.Parse(sourceCodeFile); var converter = new PhpAntlrParseTreeConverter(); Ust ust = converter.Convert(parseTree); UstNode intNode = ust.Root.GetAllDescendants( node => node.NodeType == NodeType.IntLiteral && ((IntLiteral)node).Value == 42).First(); int beginLine, beginColumn, endLine, endColumn; TextHelper.ToLineColumn(intNode.TextSpan, code, out beginLine, out beginColumn, out endLine, out endColumn); Assert.AreEqual(1, beginLine); Assert.AreEqual(12, beginColumn); Assert.AreEqual(14, endColumn); UstNode heredocNode = ust.Root.GetAllDescendants( node => node.NodeType == NodeType.StringLiteral && ((StringLiteral)node).Text.StartsWith("Heredoc text")).First(); TextHelper.ToLineColumn(heredocNode.TextSpan, code, out beginLine, out beginColumn, out endLine, out endColumn); Assert.AreEqual(3, beginLine); Assert.AreEqual(6, endLine); UstNode serverAddressNode = ust.Root.GetAllDescendants( node => node.NodeType == NodeType.StringLiteral && ((StringLiteral)node).Text.Contains("http://127.0.0.1")).First(); TextHelper.ToLineColumn(serverAddressNode.TextSpan, code, out beginLine, out beginColumn, out endLine, out endColumn); Assert.AreEqual(8, beginLine); Assert.AreEqual(15, beginColumn); } }
public bool TryGetValue(SourceCodeFile key, out CodeInfo value) { value = null; var id = new BsonValue(key.FileName.ToLower()); var cacheItem = CacheItemCollection.FindById(id); if (cacheItem != null) { if (cacheItem.Data.HashCode == key.HashCode) { value = cacheItem.Data; } else { CacheItemCollection.Delete(id); } } return(value != null); }
public List <Duplicate> Execute() { ProcessUpdate?.Invoke($"Updating Cache ({Configuration.CacheFileName})"); using (var cacheDB = new CacheDB(Configuration.CacheFileName)) { var methodExtractor = new MethodExtractor(); var finder = new DuplicateFinder(Configuration); var files = Directory.EnumerateFiles(Configuration.SourceDirectory, "*.prg", SearchOption.AllDirectories).ToList(); ProcessUpdate?.Invoke($"Found {files.Count} source files"); var index = 1; foreach (var fileName in files) { var sourceCodeFile = new SourceCodeFile(fileName); ProcessUpdate?.Invoke($"[{index++} of {files.Count}] {sourceCodeFile.RelativeFileName(Configuration.SourceDirectory)}"); try { if (!cacheDB.TryGetValue(sourceCodeFile, out var codeInfo)) { codeInfo = methodExtractor.Execute(sourceCodeFile); cacheDB.Add(codeInfo); } finder.AddSourceCodeFile(sourceCodeFile, codeInfo); } catch { ProcessUpdate?.Invoke("File could not be parsed and will be skipped"); } } ProcessUpdate?.Invoke("Identifying duplicates"); var result = finder.Execute(); ProcessUpdate?.Invoke($"{(result.Count > 0 ? result.Count.ToString() : "No")} duplicates found"); return(result); } }
CodeInfo BuildDummyCodeInfo(SourceCodeFile sourceCodeFile) => new CodeInfo { FileName = sourceCodeFile.FileName, HashCode = sourceCodeFile.HashCode, MethodList = new List <MethodInfo> { new MethodInfo { Name = "test", StatementList = new List <StatementInfo> { new StatementInfo { Start = 1, End = 10, StartLine = 1, Hashcode = 999 } } } } };
public void NotFoundInsertFound() { var dbName = GetTempDBName(); using (var cache = new CacheDB(dbName)) { var sourceCodeFile = new SourceCodeFile("XXX", "test"); cache.TryGetValue(sourceCodeFile, out var codeInfo).Should().Be(false); var newCodeInfo = BuildDummyCodeInfo(sourceCodeFile); cache.Add(newCodeInfo); cache.TryGetValue(sourceCodeFile, out codeInfo).Should().Be(true); codeInfo.Should().BeEquivalentTo(newCodeInfo); } File.Delete(dbName); }
public ParseTree Parse(SourceCodeFile sourceCodeFile) { ParseTree result = null; var filePath = Path.Combine(sourceCodeFile.RelativePath, sourceCodeFile.Name); if (sourceCodeFile.Code != null) { try { var parser = new AspxParser.AspxParser(sourceCodeFile.RelativePath); var source = new AspxSource(sourceCodeFile.FullPath, sourceCodeFile.Code); AspxParseResult aspxTree = parser.Parse(source); foreach (var error in aspxTree.ParseErrors) { Logger.LogError(new ParsingException(filePath, message: error.Message) { TextSpan = error.Location.GetTextSpan() }); } result = new AspxParseTree(aspxTree.RootNode); } catch (Exception ex) { Logger.LogError(new ParsingException(filePath, ex)); result = new CSharpRoslynParseTree(); } } else { result = new CSharpRoslynParseTree(); } result.FileName = filePath; result.FileData = sourceCodeFile.Code; return(result); }
private static CudafyModule DoCudafy(CudafyModule cm, params Type[] types) { MemoryStream output = new MemoryStream(); var outputSw = new StreamWriter(output); MemoryStream structs = new MemoryStream(); var structsSw = new StreamWriter(structs); var structsPto = new PlainTextOutput(structsSw); MemoryStream declarations = new MemoryStream(); var declarationsSw = new StreamWriter(declarations); var declarationsPto = new PlainTextOutput(declarationsSw); MemoryStream code = new MemoryStream(); var codeSw = new StreamWriter(code); var codePto = new PlainTextOutput(codeSw); bool isDummy = false; eCudafyDummyBehaviour behaviour = eCudafyDummyBehaviour.Default; Dictionary <string, ModuleDefinition> modules = new Dictionary <string, ModuleDefinition>(); var compOpts = new DecompilationOptions { FullDecompilation = true }; CUDALanguage.Reset(); bool firstPass = true; if (cm == null) { cm = new CudafyModule();// #######!!! } else { firstPass = false; } // Test structs //foreach (var strct in types.Where(t => !t.IsClass)) // if (strct.GetCustomAttributes(typeof(CudafyAttribute), false).Length == 0) // throw new CudafyLanguageException(CudafyLanguageException.csCUDAFY_ATTRIBUTE_IS_MISSING_ON_X, strct.Name); IEnumerable <Type> typeList = GetWithNestedTypes(types); foreach (var type in typeList) { if (!modules.ContainsKey(type.Assembly.Location)) { modules.Add(type.Assembly.Location, ModuleDefinition.ReadModule(type.Assembly.Location)); } } // Additional loop to compile in order foreach (var requestedType in typeList) { foreach (var kvp in modules) { foreach (var td in kvp.Value.Types) { List <TypeDefinition> tdList = new List <TypeDefinition>(); tdList.Add(td); tdList.AddRange(td.NestedTypes); Type type = null; foreach (var t in tdList) { //type = typeList.Where(tt => tt.FullName.Replace("+", "") == t.FullName.Replace("/", "")).FirstOrDefault(); // Only select type if this matches the requested type (to ensure order is maintained). type = requestedType.FullName.Replace("+", "") == t.FullName.Replace("/", "") ? requestedType : null; if (type == null) { continue; } Debug.WriteLine(t.FullName); // Types var attr = t.GetCudafyType(out isDummy, out behaviour); if (attr != null) { _cl.DecompileType(t, structsPto, compOpts); if (firstPass) { cm.Types.Add(type.FullName.Replace("+", ""), new KernelTypeInfo(type, isDummy, behaviour));// #######!!! } } else if (t.Name == td.Name) { // Fields foreach (var fi in td.Fields) { attr = fi.GetCudafyType(out isDummy, out behaviour); if (attr != null) { VerifyMemberName(fi.Name); System.Reflection.FieldInfo fieldInfo = type.GetField(fi.Name, BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); if (fieldInfo == null) { throw new CudafyLanguageException(CudafyLanguageException.csX_ARE_NOT_SUPPORTED, "Non-static fields"); } int[] dims = _cl.GetFieldInfoDimensions(fieldInfo); _cl.DecompileCUDAConstantField(fi, dims, codePto, compOpts); var kci = new KernelConstantInfo(fi.Name, fieldInfo, isDummy); if (firstPass) { cm.Constants.Add(fi.Name, kci);// #######!!! } CUDALanguage.AddConstant(kci); } } #warning TODO Only Global Methods can be called from host #warning TODO For OpenCL may need to do Methods once all Constants have been handled // Methods foreach (var med in td.Methods) { attr = med.GetCudafyType(out isDummy, out behaviour); if (attr != null) { if (!med.IsStatic) { throw new CudafyLanguageException(CudafyLanguageException.csX_ARE_NOT_SUPPORTED, "Non-static methods"); } _cl.DecompileMethodDeclaration(med, declarationsPto, new DecompilationOptions { FullDecompilation = false }); _cl.DecompileMethod(med, codePto, compOpts); MethodInfo mi = type.GetMethod(med.Name, BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic); if (mi == null) { continue; } VerifyMemberName(med.Name); eKernelMethodType kmt = eKernelMethodType.Device; kmt = GetKernelMethodType(attr, mi); if (firstPass) { cm.Functions.Add(med.Name, new KernelMethodInfo(type, mi, kmt, isDummy, behaviour, cm));// #######!!! } } } } } } } } codeSw.Flush(); if (CudafyTranslator.Language == eLanguage.OpenCL) { outputSw.WriteLine("#if defined(cl_khr_fp64)"); outputSw.WriteLine("#pragma OPENCL EXTENSION cl_khr_fp64: enable"); outputSw.WriteLine("#elif defined(cl_amd_fp64)"); outputSw.WriteLine("#pragma OPENCL EXTENSION cl_amd_fp64: enable"); outputSw.WriteLine("#endif"); } foreach (var oh in CUDALanguage.OptionalHeaders) { if (oh.Used && !oh.AsResource) { outputSw.WriteLine(oh.IncludeLine); } else if (oh.Used) { outputSw.WriteLine(GetResourceString(oh.IncludeLine)); } } foreach (var oh in CUDALanguage.OptionalFunctions) { if (oh.Used) { outputSw.WriteLine(oh.Code); } } declarationsSw.WriteLine(); declarationsSw.Flush(); structsSw.WriteLine(); structsSw.Flush(); foreach (var def in cm.GetDummyDefines()) { outputSw.WriteLine(def); } foreach (var inc in cm.GetDummyStructIncludes()) { outputSw.WriteLine(inc); } foreach (var inc in cm.GetDummyIncludes()) { outputSw.WriteLine(inc); } outputSw.Flush(); output.Write(structs.GetBuffer(), 0, (int)structs.Length); output.Write(declarations.GetBuffer(), 0, (int)declarations.Length); output.Write(code.GetBuffer(), 0, (int)code.Length); outputSw.Flush(); #if DEBUG using (FileStream fs = new FileStream("output.cu", FileMode.Create)) { fs.Write(output.GetBuffer(), 0, (int)output.Length); } #endif String s = Encoding.UTF8.GetString(output.GetBuffer(), 0, (int)output.Length); //cm.SourceCode = s;// #######!!! var scf = new SourceCodeFile(s, Language, _architecture); cm.AddSourceCodeFile(scf); return(cm); }
protected virtual ParseTree TokenizeAndParse(SourceCodeFile sourceCodeFile) { AntlrParseTree result = null; var filePath = Path.Combine(sourceCodeFile.RelativePath, sourceCodeFile.Name); if (sourceCodeFile.Code != null) { var errorListener = new AntlrMemoryErrorListener(); errorListener.FileName = filePath; errorListener.FileData = sourceCodeFile.Code; errorListener.Logger = Logger; errorListener.LineOffset = sourceCodeFile.LineOffset; try { var preprocessedText = PreprocessText(sourceCodeFile); AntlrInputStream inputStream; if (Language.IsCaseInsensitive()) { inputStream = new AntlrCaseInsensitiveInputStream(preprocessedText, CaseInsensitiveType); } else { inputStream = new AntlrInputStream(preprocessedText); } inputStream.name = filePath; Lexer lexer = InitLexer(inputStream); Lexer = lexer; lexer.RemoveErrorListeners(); lexer.AddErrorListener(errorListener); var commentTokens = new List <IToken>(); var stopwatch = Stopwatch.StartNew(); IList <IToken> tokens = GetAllTokens(lexer); stopwatch.Stop(); long lexerTimeSpanTicks = stopwatch.ElapsedTicks; #if DEBUG var codeTokensStr = AntlrHelper.GetTokensString(tokens, Vocabulary, onlyDefaultChannel: false); #endif ClearLexerCacheIfRequired(lexer); foreach (var token in tokens) { if (token.Channel == CommentsChannel) { commentTokens.Add(token); } } stopwatch.Restart(); var codeTokenSource = new ListTokenSource(tokens); var codeTokenStream = new CommonTokenStream(codeTokenSource); ParserRuleContext syntaxTree = ParseTokens(sourceCodeFile, errorListener, codeTokenStream); stopwatch.Stop(); long parserTimeSpanTicks = stopwatch.ElapsedTicks; IncrementProcessedFilesCount(); result = Create(syntaxTree); result.LexerTimeSpan = new TimeSpan(lexerTimeSpanTicks); result.ParserTimeSpan = new TimeSpan(parserTimeSpanTicks); result.Tokens = tokens; result.Comments = commentTokens; } catch (Exception ex) { Logger.LogError(new ParsingException(filePath, ex)); if (result == null) { result = Create(null); } } } else { result = Create(null); } result.FileName = filePath; result.FileData = sourceCodeFile.Code; return(result); }
private Lexer(SourceCodeFile file) { _file = file; }