/// <summary>
/// Create or update cache.
/// </summary>
/// <param name="destDir">Destination directory.</param>
/// <param name="scriptText">Script text.</param>
/// <param name="assemblyPath">Assembly path.</param>
/// <param name="metadata">Metadata</param>
/// <returns>Task for downloading and updating cache.</returns>
public async Task CreateOrUpdateCacheAsync(DirectoryInfo destDir, string scriptText, string assemblyPath, string metadata)
{
LogMessage($"Loading assembly : {assemblyPath}");
var asm = Assembly.LoadFrom(assemblyPath);
var newInvoker = new EntityInvoker(new EntityMetadata(scriptText, GetEntityType(), metadata));
newInvoker.InitializeEntryPoint(asm);
var cacheIdFilePath = Path.Combine(destDir.FullName, _cacheIdFileName);
// Remove the Old Invoker from Cache
var lastCacheId = await FileHelper.GetFileContentAsync(cacheIdFilePath);
if (!string.IsNullOrWhiteSpace(lastCacheId) && GetCacheService().TryRemoveValue(lastCacheId, out EntityInvoker oldInvoker))
{
LogMessage($"Removing old invoker with id : {oldInvoker.EntryPointDefinitionAttribute.Id} from Cache");
oldInvoker.Dispose();
}
// Add new invoker to Cache and update Cache Id File
if (newInvoker.EntryPointDefinitionAttribute != null)
{
LogMessage($"Updating cache with new invoker with id : {newInvoker.EntryPointDefinitionAttribute.Id}");
GetCacheService().AddOrUpdate(newInvoker.EntryPointDefinitionAttribute.Id, newInvoker);
await FileHelper.WriteToFileAsync(cacheIdFilePath, newInvoker.EntryPointDefinitionAttribute.Id);
}
else
{
LogWarning("Missing Entry Point Definition attribute. skipping cache update");
}
}
/// <summary>
/// Create or update cache.
/// </summary>
/// <param name="subDir">Directory info.</param>
/// <returns>Task for adding item to cache.</returns>
public virtual async Task CreateOrUpdateCacheAsync(DirectoryInfo subDir)
{
try
{
var cacheId = await FileHelper.GetFileContentAsync(subDir.FullName, _cacheIdFileName);
if (string.IsNullOrWhiteSpace(cacheId) || !GetCacheService().TryGetValue(cacheId, out EntityInvoker invoker))
{
LogMessage($"Folder : {subDir.FullName} missing in invoker cache.");
// Check if delete marker file exists.
var deleteMarkerFile = new FileInfo(Path.Combine(subDir.FullName, _deleteMarkerName));
if (deleteMarkerFile.Exists)
{
LogMessage("Folder marked for deletion. Skipping cache update");
return;
}
var mostRecentAssembly = GetMostRecentFileByExtension(subDir, ".dll");
var csxScriptFile = GetMostRecentFileByExtension(subDir, ".csx");
var metadataFile = Path.Combine(subDir.FullName, "metadata.json");
if (mostRecentAssembly == default(FileInfo) || csxScriptFile == default(FileInfo))
{
LogWarning("No Assembly file (.dll) or Csx File found (.csx). Skipping cache update");
return;
}
var scriptText = await FileHelper.GetFileContentAsync(csxScriptFile.FullName);
var metadata = string.Empty;
if (File.Exists(metadataFile))
{
metadata = await FileHelper.GetFileContentAsync(metadataFile);
}
LogMessage($"Loading assembly : {mostRecentAssembly.FullName}");
var asm = Assembly.LoadFrom(mostRecentAssembly.FullName);
invoker = new EntityInvoker(new EntityMetadata(scriptText, GetEntityType(), metadata));
invoker.InitializeEntryPoint(asm);
if (invoker.EntryPointDefinitionAttribute != null)
{
LogMessage($"Updating cache with new invoker with id : {invoker.EntryPointDefinitionAttribute.Id}");
GetCacheService().AddOrUpdate(invoker.EntryPointDefinitionAttribute.Id, invoker);
await FileHelper.WriteToFileAsync(subDir.FullName, _cacheIdFileName, invoker.EntryPointDefinitionAttribute.Id);
}
else
{
LogWarning("Missing Entry Point Definition attribute. skipping cache update");
}
}
}
catch (Exception ex)
{
LogException(ex.Message, ex);
}
}
private async Task AddInvokerToCacheIfNeeded(DirectoryInfo subDir)
{
try
{
string cacheId = await FileHelper.GetFileContentAsync(subDir.FullName, _cacheIdFileName);
if (string.IsNullOrWhiteSpace(cacheId) || !_invokerCache.TryGetValue(cacheId, out EntityInvoker invoker))
{
LogMessage($"Folder : {subDir.FullName} missing in invoker cache.");
FileInfo mostRecentAssembly = GetMostRecentFileByExtension(subDir, ".dll");
FileInfo csxScriptFile = GetMostRecentFileByExtension(subDir, ".csx");
FileInfo deleteMarkerFile = new FileInfo(Path.Combine(subDir.FullName, _deleteMarkerName));
if (mostRecentAssembly == default(FileInfo) || csxScriptFile == default(FileInfo))
{
LogWarning($"No Assembly file (.dll) or Csx File found (.csx). Skipping cache update");
return;
}
if (deleteMarkerFile.Exists)
{
LogMessage("Folder marked for deletion. Skipping cache update");
return;
}
string scriptText = await FileHelper.GetFileContentAsync(csxScriptFile.FullName);
LogMessage($"Loading assembly : {mostRecentAssembly.FullName}");
Assembly asm = Assembly.LoadFrom(mostRecentAssembly.FullName);
invoker = new EntityInvoker(new EntityMetadata(scriptText));
invoker.InitializeEntryPoint(asm);
if (invoker.EntryPointDefinitionAttribute != null)
{
LogMessage($"Updating cache with new invoker with id : {invoker.EntryPointDefinitionAttribute.Id}");
_invokerCache.AddOrUpdate(invoker.EntryPointDefinitionAttribute.Id, invoker);
await FileHelper.WriteToFileAsync(subDir.FullName, _cacheIdFileName, invoker.EntryPointDefinitionAttribute.Id);
}
else
{
LogWarning("Missing Entry Point Definition attribute. skipping cache update");
}
}
}
catch (Exception ex)
{
LogException(ex.Message, ex);
}
}
private async Task StartWatcherInternal()
{
try
{
LogMessage("SourceWatcher : Start");
DirectoryInfo srcDirectoryInfo = new DirectoryInfo(_localScriptsPath);
foreach (DirectoryInfo srcSubDirInfo in srcDirectoryInfo.GetDirectories())
{
LogMessage($"Scanning in folder : {srcSubDirInfo.FullName}");
var files = srcSubDirInfo.GetFiles().OrderByDescending(p => p.LastWriteTimeUtc);
var csxFile = files.FirstOrDefault(p => p.Extension.Equals(".csx", StringComparison.OrdinalIgnoreCase));
var asmFile = files.FirstOrDefault(p => p.Extension.Equals(".dll", StringComparison.OrdinalIgnoreCase));
string scriptText = string.Empty;
if (csxFile != default(FileInfo))
{
scriptText = await File.ReadAllTextAsync(csxFile.FullName);
}
EntityMetadata scriptMetadata = new EntityMetadata(scriptText);
EntityInvoker invoker = new EntityInvoker(scriptMetadata);
if (asmFile == default(FileInfo))
{
LogWarning($"No Assembly file (.dll). Skipping cache update");
continue;
}
LogMessage($"Loading assembly : {asmFile.FullName}");
Assembly asm = Assembly.LoadFrom(asmFile.FullName);
invoker.InitializeEntryPoint(asm);
LogMessage($"Updating cache with new invoker with id : {invoker.EntryPointDefinitionAttribute.Id}");
_invokerCache.AddOrUpdate(invoker.EntryPointDefinitionAttribute.Id, invoker);
}
}
catch (Exception ex)
{
LogException(ex.Message, ex);
}
finally
{
LogMessage("SourceWatcher : End");
}
}
public async void EntityInvoker_TestInitializationUsingAssembly()
{
// First Create and Save a assembly for test purposes.
Definition definitonAttribute = new Definition()
{
Id = "TestId"
};
string assemblyPath = $@"{Directory.GetCurrentDirectory()}/{Guid.NewGuid().ToString()}";
EntityMetadata metadata = ScriptTestDataHelper.GetRandomMetadata();
metadata.ScriptText = await ScriptTestDataHelper.GetDetectorScript(definitonAttribute);
using (EntityInvoker invoker = new EntityInvoker(metadata, ScriptHelper.GetFrameworkReferences(), ScriptHelper.GetFrameworkImports()))
{
await invoker.InitializeEntryPointAsync();
await invoker.SaveAssemblyToDiskAsync(assemblyPath);
Assert.True(File.Exists($"{assemblyPath}.dll"));
Assert.True(File.Exists($"{assemblyPath}.pdb"));
}
// Now test initializing Entry Point of Invoker using assembly
Assembly asm = Assembly.LoadFrom($"{assemblyPath}.dll");
using (EntityInvoker invoker = new EntityInvoker(metadata))
{
Exception ex = Record.Exception(() =>
{
invoker.InitializeEntryPoint(asm);
});
Assert.Null(ex);
Assert.True(invoker.IsCompilationSuccessful);
Assert.Equal(definitonAttribute.Id, invoker.EntryPointDefinitionAttribute.Id);
}
}
protected async Task <IActionResult> ExecuteQuery <TPostBodyResource>(TResource resource, CompilationBostBody <TPostBodyResource> jsonBody, string startTime, string endTime, string timeGrain)
{
if (jsonBody == null)
{
return(BadRequest("Missing body"));
}
if (string.IsNullOrWhiteSpace(jsonBody.Script))
{
return(BadRequest("Missing script in body"));
}
if (!DateTimeHelper.PrepareStartEndTimeWithTimeGrain(startTime, endTime, timeGrain, out DateTime startTimeUtc, out DateTime endTimeUtc, out TimeSpan timeGrainTimeSpan, out string errorMessage))
{
return(BadRequest(errorMessage));
}
await this._sourceWatcherService.Watcher.WaitForFirstCompletion();
EntityMetadata metaData = new EntityMetadata(jsonBody.Script);
var dataProviders = new DataProviders.DataProviders(_dataSourcesConfigService.Config);
QueryResponse <DiagnosticApiResponse> queryRes = new QueryResponse <DiagnosticApiResponse>
{
InvocationOutput = new DiagnosticApiResponse()
};
Assembly tempAsm = null;
this.Request.Headers.TryGetValue(HeaderConstants.RequestIdHeaderName, out StringValues requestIds);
var compilerResponse = await _compilerHostClient.GetCompilationResponse(jsonBody.Script, requestIds.FirstOrDefault() ?? string.Empty);
queryRes.CompilationOutput = compilerResponse;
if (queryRes.CompilationOutput.CompilationSucceeded)
{
byte[] asmData = Convert.FromBase64String(compilerResponse.AssemblyBytes);
byte[] pdbData = Convert.FromBase64String(compilerResponse.PdbBytes);
tempAsm = Assembly.Load(asmData, pdbData);
using (var invoker = new EntityInvoker(metaData, ScriptHelper.GetFrameworkReferences(), ScriptHelper.GetFrameworkImports()))
{
invoker.InitializeEntryPoint(tempAsm);
// Verify Detector with other detectors in the system in case of conflicts
if (!VerifyEntity(invoker, ref queryRes))
{
return(Ok(queryRes));
}
OperationContext <TResource> cxt = PrepareContext(resource, startTimeUtc, endTimeUtc);
List <DataProviderMetadata> dataProvidersMetadata = null;
try
{
var responseInput = new Response()
{
Metadata = RemovePIIFromDefinition(invoker.EntryPointDefinitionAttribute, cxt.IsInternalCall)
};
var invocationResponse = (Response)await invoker.Invoke(new object[] { dataProviders, cxt, responseInput });
invocationResponse.UpdateDetectorStatusFromInsights();
if (cxt.IsInternalCall)
{
dataProvidersMetadata = GetDataProvidersMetadata(dataProviders);
}
queryRes.RuntimeSucceeded = true;
queryRes.InvocationOutput = DiagnosticApiResponse.FromCsxResponse(invocationResponse, dataProvidersMetadata);
}
catch (Exception ex)
{
if (cxt.IsInternalCall)
{
queryRes.RuntimeSucceeded = false;
queryRes.InvocationOutput = CreateQueryExceptionResponse(ex, invoker.EntryPointDefinitionAttribute, cxt.IsInternalCall, GetDataProvidersMetadata(dataProviders));
}
else
{
throw;
}
}
}
}
return(Ok(queryRes));
}
private async Task DownloadContentAndUpdateInvokerCache(GithubEntry parentGithubEntry, DirectoryInfo destDir)
{
string assemblyName = Guid.NewGuid().ToString();
string csxFilePath = string.Empty;
string lastCacheId = string.Empty;
string cacheIdFilePath = Path.Combine(destDir.FullName, _cacheIdFileName);
Assembly asm;
HttpResponseMessage response = await _githubClient.Get(parentGithubEntry.Url);
if (!response.IsSuccessStatusCode)
{
LogException($"GET Failed. Url : {parentGithubEntry.Url}, StatusCode : {response.StatusCode.ToString()}", null);
return;
}
GithubEntry[] githubFiles = await response.Content.ReadAsAsyncCustom <GithubEntry[]>();
foreach (GithubEntry githubFile in githubFiles)
{
string fileExtension = githubFile.Name.Split(new char[] { '.' }).LastOrDefault();
if (string.IsNullOrWhiteSpace(githubFile.Download_url) || string.IsNullOrWhiteSpace(fileExtension))
{
// Skip Downloading any directory (with empty download url) or any file without an extension.
continue;
}
string downloadFilePath = Path.Combine(destDir.FullName, githubFile.Name.ToLower());
if (fileExtension.ToLower().Equals("csx"))
{
csxFilePath = downloadFilePath;
}
else
{
// Use Guids for Assembly and PDB Names to ensure uniqueness.
downloadFilePath = Path.Combine(destDir.FullName, $"{assemblyName}.{fileExtension.ToLower()}");
}
LogMessage($"Begin downloading File : {githubFile.Name.ToLower()} and saving it as : {downloadFilePath}");
await _githubClient.DownloadFile(githubFile.Download_url, downloadFilePath);
}
string scriptText = await FileHelper.GetFileContentAsync(csxFilePath);
string assemblyPath = Path.Combine(destDir.FullName, $"{assemblyName}.dll");
LogMessage($"Loading assembly : {assemblyPath}");
asm = Assembly.LoadFrom(assemblyPath);
EntityInvoker newInvoker = new EntityInvoker(new EntityMetadata(scriptText));
newInvoker.InitializeEntryPoint(asm);
// Remove the Old Invoker from Cache
lastCacheId = await FileHelper.GetFileContentAsync(cacheIdFilePath);
if (!string.IsNullOrWhiteSpace(lastCacheId) && _invokerCache.TryRemoveValue(lastCacheId, out EntityInvoker oldInvoker))
{
LogMessage($"Removing old invoker with id : {oldInvoker.EntryPointDefinitionAttribute.Id} from Cache");
oldInvoker.Dispose();
}
// Add new invoker to Cache and update Cache Id File
if (newInvoker.EntryPointDefinitionAttribute != null)
{
LogMessage($"Updating cache with new invoker with id : {newInvoker.EntryPointDefinitionAttribute.Id}");
_invokerCache.AddOrUpdate(newInvoker.EntryPointDefinitionAttribute.Id, newInvoker);
await FileHelper.WriteToFileAsync(cacheIdFilePath, newInvoker.EntryPointDefinitionAttribute.Id);
}
else
{
LogWarning("Missing Entry Point Definition attribute. skipping cache update");
}
}
private async Task StartWatcherInternal()
{
try
{
LogMessage("SourceWatcher : Start");
DirectoryInfo srcDirectoryInfo = new DirectoryInfo(_localScriptsPath);
foreach (DirectoryInfo srcSubDirInfo in srcDirectoryInfo.GetDirectories())
{
LogMessage($"Scanning in folder : {srcSubDirInfo.FullName}");
var files = srcSubDirInfo.GetFiles().OrderByDescending(p => p.LastWriteTimeUtc);
var csxFile = files.FirstOrDefault(p => p.Extension.Equals(".csx", StringComparison.OrdinalIgnoreCase));
var asmFile = files.FirstOrDefault(p => p.Extension.Equals(".dll", StringComparison.OrdinalIgnoreCase));
var packageJsonFile = files.FirstOrDefault(p => p.Name.Equals("package.json", StringComparison.CurrentCultureIgnoreCase));
var metadataFile = files.FirstOrDefault(p => p.Name.Equals("metadata.json", StringComparison.OrdinalIgnoreCase));
string scriptText = string.Empty;
if (csxFile != default(FileInfo))
{
scriptText = await File.ReadAllTextAsync(csxFile.FullName);
}
EntityType entityType = EntityType.Signal;
if (packageJsonFile != null)
{
var configFile = await FileHelper.GetFileContentAsync(packageJsonFile.FullName);
var config = JsonConvert.DeserializeObject <PackageConfig>(configFile);
entityType = string.Equals(config.Type, "gist", StringComparison.OrdinalIgnoreCase) ? EntityType.Gist : EntityType.Signal;
}
string metadata = string.Empty;
if (metadataFile != default(FileInfo))
{
metadata = await File.ReadAllTextAsync(metadataFile.FullName);
}
EntityMetadata scriptMetadata = new EntityMetadata(scriptText, entityType, metadata);
EntityInvoker invoker = new EntityInvoker(scriptMetadata);
if (asmFile == default(FileInfo))
{
LogWarning($"No Assembly file (.dll). Skipping cache update");
continue;
}
LogMessage($"Loading assembly : {asmFile.FullName}");
Assembly asm = Assembly.LoadFrom(asmFile.FullName);
if (_invokerDictionary.TryGetValue(entityType, out ICache <string, EntityInvoker> cache))
{
invoker.InitializeEntryPoint(asm);
LogMessage($"Updating cache with new invoker with id : {invoker.EntryPointDefinitionAttribute.Id}");
cache.AddOrUpdate(invoker.EntryPointDefinitionAttribute.Id, invoker);
}
else
{
LogMessage($"No invoker cache exist for {entityType}");
}
}
}
catch (Exception ex)
{
LogException(ex.Message, ex);
}
finally
{
LogMessage("SourceWatcher : End");
}
}