public void AddResultEntity(SourceCodeFile sourceCodeFile)
{
if (stageExt.IsRead || IsIncludeIntermediateResult)
{
AddEntity(sourceCodeFiles, sourceCodeFile);
}
}
private static int CompileFile(string inputFilePath, string outputFilePath)
{
try
{
var file = new SourceCodeFile(inputFilePath);
var ast = AnalyzeCode(file);
if (ast == null)
{
return(1);
}
var byteCode = Compiler.Compile(ast);
if (Context.Instance.ErrorsReported > 0)
{
return(1);
}
File.WriteAllBytes(inputFilePath ?? "out.ibc", byteCode.ToArray());
return(0);
}
catch (IOException e)
{
Console.WriteLine(e.Message);
}
return(1);
}
private static bool RunCode(SourceCodeFile file)
{
var tokens = Lexer.ScanTokens(file);
if (Context.Instance.ErrorsReported > 0)
{
return(true);
}
var ast = Parser.Parse(tokens);
if (Context.Instance.ErrorsReported > 0)
{
return(true);
}
// TODO: Scanning on demand.
tokens.Clear();
var byteCode = Compiler.Compile(ast);
if (Context.Instance.ErrorsReported > 0)
{
return(true);
}
#if DEBUG
Disassembler.Disassemble(byteCode.ToArray());
Console.WriteLine("----------------------------------------");
#endif
VirtualMachine.Execute(byteCode);
return(false);
}
public SourceCodeSpan(SourceCodeFile file, int begin, int length)
{
File = file;
Begin = begin;
Length = length;
(Line, Column) = File.GetLineAndColumn(begin);
}
public SourceCodeFile ReadFile(string fileName)
{
SourceCodeFile result = null;
try
{
var removeBeginLength = Path.Length + (Path.EndsWith("\\") ? 0 : 1);
var shortFileName = System.IO.Path.GetFileName(fileName);
result = new SourceCodeFile(shortFileName);
int removeEndLength = shortFileName.Length + 1;
result.RelativePath = removeEndLength + removeBeginLength > fileName.Length
? "" : fileName.Remove(fileName.Length - removeEndLength).Remove(0, removeBeginLength);
result.Code = File.ReadAllText(fileName);
return(result);
}
catch (Exception ex)
{
Logger.LogError(new ReadException(fileName, ex));
if (result == null)
{
result = new SourceCodeFile(fileName);
}
}
return(result);
}
public void LoadFileNotExists()
{
var scf = new SourceCodeFile(@"..\..\..\..\assets\TestData\FileNotExists.prg");
scf.Should().NotBeNull();
scf.SourceCode.Should().BeEmpty();
}
protected override string PreprocessText(SourceCodeFile file)
{
var result = base.PreprocessText(file);
bool trimmed = false;
var vtIndex = result.IndexOf('\v');
if (vtIndex != -1 && vtIndex > 0 && result[vtIndex - 1] == '\n' &&
vtIndex + 1 < result.Length && char.IsDigit(result[vtIndex + 1]))
{
result = result.Remove(vtIndex);
trimmed = true;
}
// TODO: Fix Hardcode!
int lastPhpInd = result.LastIndexOf("?>");
if (lastPhpInd != -1)
{
if (lastPhpInd + "?>".Length + 12 <= result.Length &&
result.Substring(lastPhpInd + "?>".Length, 12) == "\r\nChangelog:")
{
result = result.Remove(lastPhpInd + "?>".Length);
trimmed = true;
}
}
if (trimmed)
{
Logger.LogDebug($"File {Path.Combine(file.RelativePath, file.Name)} has been trimmed.");
}
return(result);
}
/// <summary>
/// Converts \r to \r\n.
/// </summary>
/// <param name="file"></param>
/// <returns></returns>
protected virtual string PreprocessText(SourceCodeFile file)
{
var text = file.Code;
var result = new StringBuilder(text.Length);
int i = 0;
while (i < text.Length)
{
if (text[i] == '\r')
{
if (i + 1 >= text.Length)
{
result.Append('\n');
}
else if (text[i + 1] != '\n')
{
result.Append('\n');
}
else
{
result.Append(text[i]);
}
}
else
{
result.Append(text[i]);
}
i++;
}
return(result.ToString());
}
public ParseTree Parse(SourceCodeFile sourceCodeFile)
{
if (MaxStackSize == 0 && MaxTimespan == 0)
{
return(TokenizeAndParse(sourceCodeFile));
}
else
{
ParseTree result = null;
Thread thread = new Thread(delegate()
{
result = TokenizeAndParse(sourceCodeFile);
},
MaxStackSize);
thread.Start();
bool finished = thread.Join(MaxTimespan == 0 ? int.MaxValue : MaxTimespan);
if (!finished)
{
thread.Interrupt();
thread.Abort();
Logger.LogError($"Parsing error in \"{sourceCodeFile.Name}\": Timeout ({MaxTimespan}) expired");
}
return(result);
}
}
protected ParserRuleContext ParseTokens(SourceCodeFile sourceCodeFile,
AntlrMemoryErrorListener errorListener, BufferedTokenStream codeTokenStream,
Func <ITokenStream, Parser> initParserFunc = null, Func <Parser, ParserRuleContext> parseFunc = null)
{
Parser parser = initParserFunc != null?initParserFunc(codeTokenStream) : InitParser(codeTokenStream);
parser.RemoveErrorListeners();
Parser = parser;
ParserRuleContext syntaxTree;
if (UseFastParseStrategyAtFirst)
{
parser.Interpreter.PredictionMode = PredictionMode.Sll;
parser.ErrorHandler = new BailErrorStrategy();
parser.TrimParseTree = true;
parserLock.EnterReadLock();
try
{
syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser);
}
catch (ParseCanceledException)
{
parserLock.ExitReadLock();
parser.AddErrorListener(errorListener);
codeTokenStream.Reset();
parser.Reset();
parser.Interpreter.PredictionMode = PredictionMode.Ll;
parser.ErrorHandler = new DefaultErrorStrategy();
parserLock.EnterReadLock();
syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser);
}
finally
{
parserLock.ExitReadLock();
}
}
else
{
parser.AddErrorListener(errorListener);
parserLock.EnterReadLock();
try
{
syntaxTree = parseFunc != null?parseFunc(parser) : Parse(parser);
}
finally
{
parserLock.ExitReadLock();
}
}
ClearParserCacheIfRequired(parser);
#if DEBUG
var tree = syntaxTree.ToStringTree(parser);
#endif
return(syntaxTree);
}
static void Main(string[] args)
{
var fileRepository = new FileCodeRepository("code.php");
var patternsRepostiory = new FilePatternsRepository("patterns.json");
var workflow = new Workflow(fileRepository, patternsRepostiory);
workflow.IsIncludeIntermediateResult = true;
WorkflowResult result = workflow.Process();
var lineTokenDict = new Dictionary <int, List <string> >();
foreach (var matchingResult in result.MatchingResults)
{
var matchedNode = matchingResult.Nodes.Last();
UstNode[] descendants = matchedNode.GetAllDescendants();
foreach (UstNode descendant in descendants)
{
string text = null;
if (descendant.NodeType == NodeType.InvocationExpression)
{
text = "Method invocation: " + ((InvocationExpression)descendant).Target;
}
else if (descendant.NodeType == NodeType.IndexerDeclaration)
{
text = "Arrays with user input: " + ((IndexerExpression)descendant).Target;
}
else if (descendant.NodeType == NodeType.CastExpression)
{
text = "Cast to type: " + ((CastExpression)descendant).Type;
}
if (!string.IsNullOrEmpty(text))
{
SourceCodeFile sourceCodeFile = result.SourceCodeFiles.Single(file => file.FullPath == descendant.FileNode.FileName.Text);
LineColumnTextSpan lineColumnTextSpan = sourceCodeFile.GetLineColumnTextSpan(descendant.TextSpan);
List <string> strs;
if (!lineTokenDict.TryGetValue(lineColumnTextSpan.BeginLine, out strs))
{
strs = new List <string>();
lineTokenDict[lineColumnTextSpan.BeginLine] = strs;
}
strs.Add(text);
}
}
}
foreach (var item in lineTokenDict)
{
Console.WriteLine(String.Format("{0}: {1}", item.Key, string.Join(", ", item.Value)));
}
Console.ReadLine();
}
public ParseTree Parse(SourceCodeFile sourceCodeFile)
{
CSharpRoslynParseTree result = null;
var filePath = Path.Combine(sourceCodeFile.RelativePath, sourceCodeFile.Name);
if (sourceCodeFile.Code != null)
{
SyntaxNode root = null;
try
{
SyntaxTree syntaxTree = CSharpSyntaxTree.ParseText(sourceCodeFile.Code, null, filePath);
result = new CSharpRoslynParseTree(syntaxTree);
root = syntaxTree.GetRoot();
result.Comments = root.DescendantTrivia().Where(node =>
{
SyntaxKind kind = node.Kind();
return(kind == SyntaxKind.SingleLineCommentTrivia ||
kind == SyntaxKind.MultiLineCommentTrivia ||
kind == SyntaxKind.SingleLineDocumentationCommentTrivia ||
kind == SyntaxKind.MultiLineDocumentationCommentTrivia ||
kind == SyntaxKind.DocumentationCommentExteriorTrivia ||
kind == SyntaxKind.XmlComment);
}).ToArray();
IEnumerable <Diagnostic> diagnostics = root.GetDiagnostics();
foreach (var diagnostic in diagnostics)
{
if (diagnostic.Severity == DiagnosticSeverity.Error &&
diagnostic.Id != "CS1029")
{
var textSpan = RoslynHelper.ConvertTextSpan(diagnostic.Location);
Logger.LogError(new ParsingException(filePath, message: diagnostic.ToString())
{
TextSpan = textSpan
});
}
}
}
catch (Exception ex)
{
Logger.LogError(new ParsingException(filePath, ex));
result = new CSharpRoslynParseTree();
}
}
else
{
result = new CSharpRoslynParseTree();
}
result.FileName = filePath;
result.FileData = sourceCodeFile.Code;
return(result);
}
protected ParseTree ReadAndParse(string fileName, TWorkflowResult workflowResult, CancellationToken cancellationToken = default(CancellationToken))
{
ParseTree result = null;
var stopwatch = new Stopwatch();
string file = fileName;
if (stageHelper.IsContainsRead)
{
if (SourceCodeRepository.IsFileIgnored(fileName))
{
Logger.LogInfo($"File {fileName} has not been read.");
return(null);
}
stopwatch.Restart();
SourceCodeFile sourceCodeFile = SourceCodeRepository.ReadFile(fileName);
stopwatch.Stop();
Logger.LogInfo($"File {fileName} has been read (Elapsed: {stopwatch.Elapsed}).");
workflowResult.AddProcessedCharsCount(sourceCodeFile.Code.Length);
workflowResult.AddProcessedLinesCount(TextHelper.GetLinesCount(sourceCodeFile.Code));
workflowResult.AddReadTime(stopwatch.ElapsedTicks);
workflowResult.AddResultEntity(sourceCodeFile);
cancellationToken.ThrowIfCancellationRequested();
file = sourceCodeFile.RelativePath;
if (stageHelper.IsContainsParse)
{
stopwatch.Restart();
Language?detectedLanguage = LanguageDetector.DetectIfRequired(sourceCodeFile.Name, sourceCodeFile.Code, Languages);
if (detectedLanguage == null)
{
Logger.LogInfo($"Input languages set is empty or {sourceCodeFile.Name} language has not been detected. File has not been converter.");
return(null);
}
result = ParserConverterSets[(Language)detectedLanguage].Parser.Parse(sourceCodeFile);
stopwatch.Stop();
Logger.LogInfo($"File {fileName} has been parsed (Elapsed: {stopwatch.Elapsed}).");
workflowResult.AddParseTime(stopwatch.ElapsedTicks);
var antlrParseTree = result as AntlrParseTree;
if (antlrParseTree != null)
{
workflowResult.AddLexerTime(antlrParseTree.LexerTimeSpan.Ticks);
workflowResult.AddParserTicks(antlrParseTree.ParserTimeSpan.Ticks);
}
cancellationToken.ThrowIfCancellationRequested();
}
}
return(result);
}
private static List <Statement> AnalyzeCode(SourceCodeFile file)
{
var tokens = Lexer.ScanTokens(file);
if (Context.Instance.ErrorsReported > 0)
{
return(null);
}
var ast = Parser.Parse(tokens);
return(Context.Instance.ErrorsReported == 0 ? ast : null);
}
public async Task AddOrUpdateSourceCodeAsync()
{
// Given
var project = new InMemoryProject();
var code = new SourceCodeFile("Foo.cs", "public class Foo { }");
// When
project.AddOrUpdate(code);
var compilation = await project.CompileAsync();
// Then
Assert.NotNull(compilation);
Assert.True(compilation.ContainsSymbolsWithName("Foo"));
}
public static List <Token> ScanTokens(SourceCodeFile file)
{
var lexer = new Lexer(file);
while (!lexer.IsAtEnd())
{
lexer._start = lexer._current;
lexer.ScanToken();
}
lexer._start = lexer._current;
lexer.PushToken(TokenType.EndOfFile);
return(lexer._tokens);
}
private static int AnalyzeFile(string filePath)
{
try
{
var file = new SourceCodeFile(filePath);
return(AnalyzeCode(file) != null ? 1 : 0);
}
catch (IOException e)
{
Console.WriteLine(e.Message);
}
return(0);
}
public void resolveFileLocationsOfExtraSourceCodeFilesToCompile()
{
if (ExtraSourceCodeFilesToCompile.size() > 0)
{
// try to resolve local file references
try
{
if (this.SourceCodeFile.isNull()) // in case this is not set
{
SourceCodeFile = PublicDI.CurrentScript;
}
for (int i = 0; i < ExtraSourceCodeFilesToCompile.size(); i++)
{
var fileToResolve = ExtraSourceCodeFilesToCompile[i].trim();
var resolved = false;
// try using SourceCodeFile.directoryName()
if (fileToResolve.fileExists().isFalse())
{
if (SourceCodeFile.valid() && SourceCodeFile.isFile())
{
var resolvedFile = SourceCodeFile.directoryName().pathCombine(fileToResolve);
if (resolvedFile.fileExists())
{
ExtraSourceCodeFilesToCompile[i] = resolvedFile;
resolved = true;
}
}
}
// try using the localscripts folders
var localMapping = fileToResolve.local();
if (localMapping.valid())
{
ExtraSourceCodeFilesToCompile[i] = localMapping;
resolved = true;
}
if (resolved.isFalse() && fileToResolve.fileExists().isFalse())
{
ExtraSourceCodeFilesToCompile[i] = ExtraSourceCodeFilesToCompile[i].fullPath();
}
}
}
catch (Exception ex)
{
ex.log("in compileExtraSourceCodeReferencesAndUpdateReferencedAssemblies while resolving ExtraSourceCodeFilesToCompile");
}
}
}
public void AddSourceCodeFile(SourceCodeFile sourceCodeFile, CodeInfo codeInfo)
{
if (sourceCodeFile == null || codeInfo == null || sourceCodeFile.FileName != codeInfo.FileName)
{
throw new ArgumentException();
}
SourceCodeFiles.Add(codeInfo, sourceCodeFile);
var buffer = new StringBuilder();
foreach (var methodInfo in codeInfo.MethodList)
{
void AddMethodLocations(Dictionary <uint, List <StatementLocation> > dict, int startOffset, int count)
{
buffer.Length = 0;
for (int i = 0; i < count; i++)
{
buffer.Append(methodInfo.StatementList[startOffset + i].Hashcode.ToString()).Append('#');
}
var statementBlockHashCode = GetStatementListHashCode(buffer);
if (!dict.TryGetValue(statementBlockHashCode, out var locationList))
{
locationList = new List <StatementLocation>();
dict[statementBlockHashCode] = locationList;
}
locationList.Add(new StatementLocation {
CodeInfo = codeInfo, Method = methodInfo, Offset = startOffset
});
}
if (methodInfo.StatementList.Count >= Configuration.MinLineForFullMethodDuplicateCheck && methodInfo.StatementList.Count < Configuration.MinLineForDuplicate)
{
AddMethodLocations(FullMethods, 0, methodInfo.StatementList.Count);
}
if (methodInfo.StatementList.Count >= Configuration.MinLineForDuplicate)
{
for (int i = 0; i < methodInfo.StatementList.Count - Configuration.MinLineForDuplicate; i++)
{
AddMethodLocations(Locations, i, Configuration.MinLineForDuplicate);
}
}
}
}
public SourceCodeFile ReadFile(string fileName)
{
var result = new SourceCodeFile(fileName);
try
{
result.RelativePath = "";
result.Code = Code;
}
catch (Exception ex)
{
Logger.LogError(new ReadException(fileName, ex));
}
return(result);
}
private static int RunFile(string filePath)
{
try
{
var sourceFile = new SourceCodeFile(filePath);
var reportedErrors = RunCode(sourceFile);
return(reportedErrors ? 1 : 0);
}
catch (IOException e)
{
Console.WriteLine(e.Message);
}
return(0);
}
public void Parse_NewLine_CorrectLineColumn()
{
string fileText = File.ReadAllText(Path.Combine(TestHelper.TestsDataPath, "newLine -r-n.php"));
var lineEnds = new string[] { "\r", "\n", "\r\n" };
foreach (var lineEnd in lineEnds)
{
var phpParser = new PhpAntlrParser();
string code = fileText.Replace("\r\n", lineEnd);
var sourceCodeFile = new SourceCodeFile
{
Name = "newLine.php",
Code = code
};
var parseTree = (PhpAntlrParseTree)phpParser.Parse(sourceCodeFile);
var converter = new PhpAntlrParseTreeConverter();
Ust ust = converter.Convert(parseTree);
UstNode intNode = ust.Root.GetAllDescendants(
node => node.NodeType == NodeType.IntLiteral &&
((IntLiteral)node).Value == 42).First();
int beginLine, beginColumn, endLine, endColumn;
TextHelper.ToLineColumn(intNode.TextSpan, code, out beginLine, out beginColumn, out endLine, out endColumn);
Assert.AreEqual(1, beginLine);
Assert.AreEqual(12, beginColumn);
Assert.AreEqual(14, endColumn);
UstNode heredocNode = ust.Root.GetAllDescendants(
node => node.NodeType == NodeType.StringLiteral &&
((StringLiteral)node).Text.StartsWith("Heredoc text")).First();
TextHelper.ToLineColumn(heredocNode.TextSpan, code, out beginLine, out beginColumn, out endLine, out endColumn);
Assert.AreEqual(3, beginLine);
Assert.AreEqual(6, endLine);
UstNode serverAddressNode = ust.Root.GetAllDescendants(
node => node.NodeType == NodeType.StringLiteral &&
((StringLiteral)node).Text.Contains("http://127.0.0.1")).First();
TextHelper.ToLineColumn(serverAddressNode.TextSpan, code, out beginLine, out beginColumn, out endLine, out endColumn);
Assert.AreEqual(8, beginLine);
Assert.AreEqual(15, beginColumn);
}
}
public bool TryGetValue(SourceCodeFile key, out CodeInfo value)
{
value = null;
var id = new BsonValue(key.FileName.ToLower());
var cacheItem = CacheItemCollection.FindById(id);
if (cacheItem != null)
{
if (cacheItem.Data.HashCode == key.HashCode)
{
value = cacheItem.Data;
}
else
{
CacheItemCollection.Delete(id);
}
}
return(value != null);
}
public List <Duplicate> Execute()
{
ProcessUpdate?.Invoke($"Updating Cache ({Configuration.CacheFileName})");
using (var cacheDB = new CacheDB(Configuration.CacheFileName))
{
var methodExtractor = new MethodExtractor();
var finder = new DuplicateFinder(Configuration);
var files = Directory.EnumerateFiles(Configuration.SourceDirectory, "*.prg", SearchOption.AllDirectories).ToList();
ProcessUpdate?.Invoke($"Found {files.Count} source files");
var index = 1;
foreach (var fileName in files)
{
var sourceCodeFile = new SourceCodeFile(fileName);
ProcessUpdate?.Invoke($"[{index++} of {files.Count}] {sourceCodeFile.RelativeFileName(Configuration.SourceDirectory)}");
try
{
if (!cacheDB.TryGetValue(sourceCodeFile, out var codeInfo))
{
codeInfo = methodExtractor.Execute(sourceCodeFile);
cacheDB.Add(codeInfo);
}
finder.AddSourceCodeFile(sourceCodeFile, codeInfo);
}
catch
{
ProcessUpdate?.Invoke("File could not be parsed and will be skipped");
}
}
ProcessUpdate?.Invoke("Identifying duplicates");
var result = finder.Execute();
ProcessUpdate?.Invoke($"{(result.Count > 0 ? result.Count.ToString() : "No")} duplicates found");
return(result);
}
}
CodeInfo BuildDummyCodeInfo(SourceCodeFile sourceCodeFile)
=> new CodeInfo
{
FileName = sourceCodeFile.FileName,
HashCode = sourceCodeFile.HashCode,
MethodList = new List <MethodInfo>
{
new MethodInfo
{
Name = "test", StatementList = new List <StatementInfo>
{
new StatementInfo
{
Start = 1,
End = 10,
StartLine = 1, Hashcode = 999
}
}
}
}
};
public void NotFoundInsertFound()
{
var dbName = GetTempDBName();
using (var cache = new CacheDB(dbName))
{
var sourceCodeFile = new SourceCodeFile("XXX", "test");
cache.TryGetValue(sourceCodeFile, out var codeInfo).Should().Be(false);
var newCodeInfo = BuildDummyCodeInfo(sourceCodeFile);
cache.Add(newCodeInfo);
cache.TryGetValue(sourceCodeFile, out codeInfo).Should().Be(true);
codeInfo.Should().BeEquivalentTo(newCodeInfo);
}
File.Delete(dbName);
}
public ParseTree Parse(SourceCodeFile sourceCodeFile)
{
ParseTree result = null;
var filePath = Path.Combine(sourceCodeFile.RelativePath, sourceCodeFile.Name);
if (sourceCodeFile.Code != null)
{
try
{
var parser = new AspxParser.AspxParser(sourceCodeFile.RelativePath);
var source = new AspxSource(sourceCodeFile.FullPath, sourceCodeFile.Code);
AspxParseResult aspxTree = parser.Parse(source);
foreach (var error in aspxTree.ParseErrors)
{
Logger.LogError(new ParsingException(filePath, message: error.Message)
{
TextSpan = error.Location.GetTextSpan()
});
}
result = new AspxParseTree(aspxTree.RootNode);
}
catch (Exception ex)
{
Logger.LogError(new ParsingException(filePath, ex));
result = new CSharpRoslynParseTree();
}
}
else
{
result = new CSharpRoslynParseTree();
}
result.FileName = filePath;
result.FileData = sourceCodeFile.Code;
return(result);
}
private static CudafyModule DoCudafy(CudafyModule cm, params Type[] types)
{
MemoryStream output = new MemoryStream();
var outputSw = new StreamWriter(output);
MemoryStream structs = new MemoryStream();
var structsSw = new StreamWriter(structs);
var structsPto = new PlainTextOutput(structsSw);
MemoryStream declarations = new MemoryStream();
var declarationsSw = new StreamWriter(declarations);
var declarationsPto = new PlainTextOutput(declarationsSw);
MemoryStream code = new MemoryStream();
var codeSw = new StreamWriter(code);
var codePto = new PlainTextOutput(codeSw);
bool isDummy = false;
eCudafyDummyBehaviour behaviour = eCudafyDummyBehaviour.Default;
Dictionary <string, ModuleDefinition> modules = new Dictionary <string, ModuleDefinition>();
var compOpts = new DecompilationOptions {
FullDecompilation = true
};
CUDALanguage.Reset();
bool firstPass = true;
if (cm == null)
{
cm = new CudafyModule();// #######!!!
}
else
{
firstPass = false;
}
// Test structs
//foreach (var strct in types.Where(t => !t.IsClass))
// if (strct.GetCustomAttributes(typeof(CudafyAttribute), false).Length == 0)
// throw new CudafyLanguageException(CudafyLanguageException.csCUDAFY_ATTRIBUTE_IS_MISSING_ON_X, strct.Name);
IEnumerable <Type> typeList = GetWithNestedTypes(types);
foreach (var type in typeList)
{
if (!modules.ContainsKey(type.Assembly.Location))
{
modules.Add(type.Assembly.Location, ModuleDefinition.ReadModule(type.Assembly.Location));
}
}
// Additional loop to compile in order
foreach (var requestedType in typeList)
{
foreach (var kvp in modules)
{
foreach (var td in kvp.Value.Types)
{
List <TypeDefinition> tdList = new List <TypeDefinition>();
tdList.Add(td);
tdList.AddRange(td.NestedTypes);
Type type = null;
foreach (var t in tdList)
{
//type = typeList.Where(tt => tt.FullName.Replace("+", "") == t.FullName.Replace("/", "")).FirstOrDefault();
// Only select type if this matches the requested type (to ensure order is maintained).
type = requestedType.FullName.Replace("+", "") == t.FullName.Replace("/", "") ? requestedType : null;
if (type == null)
{
continue;
}
Debug.WriteLine(t.FullName);
// Types
var attr = t.GetCudafyType(out isDummy, out behaviour);
if (attr != null)
{
_cl.DecompileType(t, structsPto, compOpts);
if (firstPass)
{
cm.Types.Add(type.FullName.Replace("+", ""), new KernelTypeInfo(type, isDummy, behaviour));// #######!!!
}
}
else if (t.Name == td.Name)
{
// Fields
foreach (var fi in td.Fields)
{
attr = fi.GetCudafyType(out isDummy, out behaviour);
if (attr != null)
{
VerifyMemberName(fi.Name);
System.Reflection.FieldInfo fieldInfo = type.GetField(fi.Name, BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic);
if (fieldInfo == null)
{
throw new CudafyLanguageException(CudafyLanguageException.csX_ARE_NOT_SUPPORTED, "Non-static fields");
}
int[] dims = _cl.GetFieldInfoDimensions(fieldInfo);
_cl.DecompileCUDAConstantField(fi, dims, codePto, compOpts);
var kci = new KernelConstantInfo(fi.Name, fieldInfo, isDummy);
if (firstPass)
{
cm.Constants.Add(fi.Name, kci);// #######!!!
}
CUDALanguage.AddConstant(kci);
}
}
#warning TODO Only Global Methods can be called from host
#warning TODO For OpenCL may need to do Methods once all Constants have been handled
// Methods
foreach (var med in td.Methods)
{
attr = med.GetCudafyType(out isDummy, out behaviour);
if (attr != null)
{
if (!med.IsStatic)
{
throw new CudafyLanguageException(CudafyLanguageException.csX_ARE_NOT_SUPPORTED, "Non-static methods");
}
_cl.DecompileMethodDeclaration(med, declarationsPto, new DecompilationOptions {
FullDecompilation = false
});
_cl.DecompileMethod(med, codePto, compOpts);
MethodInfo mi = type.GetMethod(med.Name, BindingFlags.Static | BindingFlags.Public | BindingFlags.NonPublic);
if (mi == null)
{
continue;
}
VerifyMemberName(med.Name);
eKernelMethodType kmt = eKernelMethodType.Device;
kmt = GetKernelMethodType(attr, mi);
if (firstPass)
{
cm.Functions.Add(med.Name, new KernelMethodInfo(type, mi, kmt, isDummy, behaviour, cm));// #######!!!
}
}
}
}
}
}
}
}
codeSw.Flush();
if (CudafyTranslator.Language == eLanguage.OpenCL)
{
outputSw.WriteLine("#if defined(cl_khr_fp64)");
outputSw.WriteLine("#pragma OPENCL EXTENSION cl_khr_fp64: enable");
outputSw.WriteLine("#elif defined(cl_amd_fp64)");
outputSw.WriteLine("#pragma OPENCL EXTENSION cl_amd_fp64: enable");
outputSw.WriteLine("#endif");
}
foreach (var oh in CUDALanguage.OptionalHeaders)
{
if (oh.Used && !oh.AsResource)
{
outputSw.WriteLine(oh.IncludeLine);
}
else if (oh.Used)
{
outputSw.WriteLine(GetResourceString(oh.IncludeLine));
}
}
foreach (var oh in CUDALanguage.OptionalFunctions)
{
if (oh.Used)
{
outputSw.WriteLine(oh.Code);
}
}
declarationsSw.WriteLine();
declarationsSw.Flush();
structsSw.WriteLine();
structsSw.Flush();
foreach (var def in cm.GetDummyDefines())
{
outputSw.WriteLine(def);
}
foreach (var inc in cm.GetDummyStructIncludes())
{
outputSw.WriteLine(inc);
}
foreach (var inc in cm.GetDummyIncludes())
{
outputSw.WriteLine(inc);
}
outputSw.Flush();
output.Write(structs.GetBuffer(), 0, (int)structs.Length);
output.Write(declarations.GetBuffer(), 0, (int)declarations.Length);
output.Write(code.GetBuffer(), 0, (int)code.Length);
outputSw.Flush();
#if DEBUG
using (FileStream fs = new FileStream("output.cu", FileMode.Create))
{
fs.Write(output.GetBuffer(), 0, (int)output.Length);
}
#endif
String s = Encoding.UTF8.GetString(output.GetBuffer(), 0, (int)output.Length);
//cm.SourceCode = s;// #######!!!
var scf = new SourceCodeFile(s, Language, _architecture);
cm.AddSourceCodeFile(scf);
return(cm);
}
protected virtual ParseTree TokenizeAndParse(SourceCodeFile sourceCodeFile)
{
AntlrParseTree result = null;
var filePath = Path.Combine(sourceCodeFile.RelativePath, sourceCodeFile.Name);
if (sourceCodeFile.Code != null)
{
var errorListener = new AntlrMemoryErrorListener();
errorListener.FileName = filePath;
errorListener.FileData = sourceCodeFile.Code;
errorListener.Logger = Logger;
errorListener.LineOffset = sourceCodeFile.LineOffset;
try
{
var preprocessedText = PreprocessText(sourceCodeFile);
AntlrInputStream inputStream;
if (Language.IsCaseInsensitive())
{
inputStream = new AntlrCaseInsensitiveInputStream(preprocessedText, CaseInsensitiveType);
}
else
{
inputStream = new AntlrInputStream(preprocessedText);
}
inputStream.name = filePath;
Lexer lexer = InitLexer(inputStream);
Lexer = lexer;
lexer.RemoveErrorListeners();
lexer.AddErrorListener(errorListener);
var commentTokens = new List <IToken>();
var stopwatch = Stopwatch.StartNew();
IList <IToken> tokens = GetAllTokens(lexer);
stopwatch.Stop();
long lexerTimeSpanTicks = stopwatch.ElapsedTicks;
#if DEBUG
var codeTokensStr = AntlrHelper.GetTokensString(tokens, Vocabulary, onlyDefaultChannel: false);
#endif
ClearLexerCacheIfRequired(lexer);
foreach (var token in tokens)
{
if (token.Channel == CommentsChannel)
{
commentTokens.Add(token);
}
}
stopwatch.Restart();
var codeTokenSource = new ListTokenSource(tokens);
var codeTokenStream = new CommonTokenStream(codeTokenSource);
ParserRuleContext syntaxTree = ParseTokens(sourceCodeFile, errorListener, codeTokenStream);
stopwatch.Stop();
long parserTimeSpanTicks = stopwatch.ElapsedTicks;
IncrementProcessedFilesCount();
result = Create(syntaxTree);
result.LexerTimeSpan = new TimeSpan(lexerTimeSpanTicks);
result.ParserTimeSpan = new TimeSpan(parserTimeSpanTicks);
result.Tokens = tokens;
result.Comments = commentTokens;
}
catch (Exception ex)
{
Logger.LogError(new ParsingException(filePath, ex));
if (result == null)
{
result = Create(null);
}
}
}
else
{
result = Create(null);
}
result.FileName = filePath;
result.FileData = sourceCodeFile.Code;
return(result);
}
private Lexer(SourceCodeFile file)
{
_file = file;
}
