private async Task DownloadContentAndUpdateCache(GithubEntry parentGithubEntry, DirectoryInfo destDir, string folderName)
{
var assemblyName = Guid.NewGuid().ToString();
var csxFilePath = string.Empty;
var confFilePath = string.Empty;
var metadataFilePath = string.Empty;
var lastCacheId = string.Empty;
var cacheIdFilePath = Path.Combine(destDir.FullName, _cacheIdFileName);
bool isSearchModel = false;
var response = await _githubClient.Get(parentGithubEntry.Url);
if (!response.IsSuccessStatusCode)
{
LogException($"GET Failed. Url : {parentGithubEntry.Url}, StatusCode : {response.StatusCode.ToString()}", null);
return;
}
var githubFiles = await response.Content.ReadAsAsyncCustom <GithubEntry[]>();
foreach (GithubEntry githubFile in githubFiles)
{
var fileExtension = githubFile.Name.Split(new char[] { '.' }).LastOrDefault();
if (string.IsNullOrWhiteSpace(githubFile.Download_url) || string.IsNullOrWhiteSpace(fileExtension))
{
// Skip Downloading any directory (with empty download url) or any file without an extension.
continue;
}
var downloadFilePath = Path.Combine(destDir.FullName, githubFile.Name.ToLower());
if (fileExtension.Equals("csx", StringComparison.OrdinalIgnoreCase))
{
csxFilePath = downloadFilePath;
}
else if (githubFile.Name.Equals("package.json", StringComparison.OrdinalIgnoreCase))
{
confFilePath = downloadFilePath;
}
else if (githubFile.Name.Equals("metadata.json", StringComparison.OrdinalIgnoreCase))
{
metadataFilePath = downloadFilePath;
}
// Extensions used in search model files
else if (githubFile.Name.Split(".").Last() == "model" || githubFile.Name.Split(".").Last() == "index" || githubFile.Name.Split(".").Last() == "dict" || githubFile.Name.Split(".").Last() == "json" || githubFile.Name.Split(".").Last() == "npy")
{
isSearchModel = true;
downloadFilePath = Path.Combine(destDir.FullName, githubFile.Name);
}
else
{
// Use Guids for Assembly and PDB Names to ensure uniqueness.
downloadFilePath = Path.Combine(destDir.FullName, $"{assemblyName}.{fileExtension.ToLower()}");
}
LogMessage($"Begin downloading File : {githubFile.Name.ToLower()} and saving it as : {downloadFilePath}");
await _githubClient.DownloadFile(githubFile.Download_url, downloadFilePath);
}
if (isSearchModel)
{
HitModelRefresh(folderName);
}
var scriptText = await FileHelper.GetFileContentAsync(csxFilePath);
var assemblyPath = Path.Combine(destDir.FullName, $"{assemblyName}.dll");
var configFile = await FileHelper.GetFileContentAsync(confFilePath);
var config = JsonConvert.DeserializeObject <PackageConfig>(configFile);
var metadata = await FileHelper.GetFileContentAsync(metadataFilePath);
var workerId = string.Equals(config?.Type, "gist", StringComparison.OrdinalIgnoreCase) ? "GistWorker" : "DetectorWorker";
await FileHelper.WriteToFileAsync(destDir.FullName, _workerIdFileName, workerId);
if (GithubWorkers.ContainsKey(workerId))
{
await GithubWorkers[workerId].CreateOrUpdateCacheAsync(destDir, scriptText, assemblyPath, metadata);
}
else
{
LogWarning($"Cannot find github worker with id {workerId}. Directory: {destDir.FullName}.");
}
}
private async Task DownloadContentAndUpdateInvokerCache(GithubEntry parentGithubEntry, DirectoryInfo destDir)
{
string assemblyName = Guid.NewGuid().ToString();
string csxFilePath = string.Empty;
string lastCacheId = string.Empty;
string cacheIdFilePath = Path.Combine(destDir.FullName, _cacheIdFileName);
Assembly asm;
HttpResponseMessage response = await _githubClient.Get(parentGithubEntry.Url);
if (!response.IsSuccessStatusCode)
{
LogException($"GET Failed. Url : {parentGithubEntry.Url}, StatusCode : {response.StatusCode.ToString()}", null);
return;
}
GithubEntry[] githubFiles = await response.Content.ReadAsAsyncCustom <GithubEntry[]>();
foreach (GithubEntry githubFile in githubFiles)
{
string fileExtension = githubFile.Name.Split(new char[] { '.' }).LastOrDefault();
if (string.IsNullOrWhiteSpace(githubFile.Download_url) || string.IsNullOrWhiteSpace(fileExtension))
{
// Skip Downloading any directory (with empty download url) or any file without an extension.
continue;
}
string downloadFilePath = Path.Combine(destDir.FullName, githubFile.Name.ToLower());
if (fileExtension.ToLower().Equals("csx"))
{
csxFilePath = downloadFilePath;
}
else
{
// Use Guids for Assembly and PDB Names to ensure uniqueness.
downloadFilePath = Path.Combine(destDir.FullName, $"{assemblyName}.{fileExtension.ToLower()}");
}
LogMessage($"Begin downloading File : {githubFile.Name.ToLower()} and saving it as : {downloadFilePath}");
await _githubClient.DownloadFile(githubFile.Download_url, downloadFilePath);
}
string scriptText = await FileHelper.GetFileContentAsync(csxFilePath);
string assemblyPath = Path.Combine(destDir.FullName, $"{assemblyName}.dll");
LogMessage($"Loading assembly : {assemblyPath}");
asm = Assembly.LoadFrom(assemblyPath);
EntityInvoker newInvoker = new EntityInvoker(new EntityMetadata(scriptText));
newInvoker.InitializeEntryPoint(asm);
// Remove the Old Invoker from Cache
lastCacheId = await FileHelper.GetFileContentAsync(cacheIdFilePath);
if (!string.IsNullOrWhiteSpace(lastCacheId) && _invokerCache.TryRemoveValue(lastCacheId, out EntityInvoker oldInvoker))
{
LogMessage($"Removing old invoker with id : {oldInvoker.EntryPointDefinitionAttribute.Id} from Cache");
oldInvoker.Dispose();
}
// Add new invoker to Cache and update Cache Id File
if (newInvoker.EntryPointDefinitionAttribute != null)
{
LogMessage($"Updating cache with new invoker with id : {newInvoker.EntryPointDefinitionAttribute.Id}");
_invokerCache.AddOrUpdate(newInvoker.EntryPointDefinitionAttribute.Id, newInvoker);
await FileHelper.WriteToFileAsync(cacheIdFilePath, newInvoker.EntryPointDefinitionAttribute.Id);
}
else
{
LogWarning("Missing Entry Point Definition attribute. skipping cache update");
}
}