private async Task DownloadContentAndUpdateCache(GithubEntry parentGithubEntry, DirectoryInfo destDir, string folderName) { var assemblyName = Guid.NewGuid().ToString(); var csxFilePath = string.Empty; var confFilePath = string.Empty; var metadataFilePath = string.Empty; var lastCacheId = string.Empty; var cacheIdFilePath = Path.Combine(destDir.FullName, _cacheIdFileName); bool isSearchModel = false; var response = await _githubClient.Get(parentGithubEntry.Url); if (!response.IsSuccessStatusCode) { LogException($"GET Failed. Url : {parentGithubEntry.Url}, StatusCode : {response.StatusCode.ToString()}", null); return; } var githubFiles = await response.Content.ReadAsAsyncCustom <GithubEntry[]>(); foreach (GithubEntry githubFile in githubFiles) { var fileExtension = githubFile.Name.Split(new char[] { '.' }).LastOrDefault(); if (string.IsNullOrWhiteSpace(githubFile.Download_url) || string.IsNullOrWhiteSpace(fileExtension)) { // Skip Downloading any directory (with empty download url) or any file without an extension. continue; } var downloadFilePath = Path.Combine(destDir.FullName, githubFile.Name.ToLower()); if (fileExtension.Equals("csx", StringComparison.OrdinalIgnoreCase)) { csxFilePath = downloadFilePath; } else if (githubFile.Name.Equals("package.json", StringComparison.OrdinalIgnoreCase)) { confFilePath = downloadFilePath; } else if (githubFile.Name.Equals("metadata.json", StringComparison.OrdinalIgnoreCase)) { metadataFilePath = downloadFilePath; } // Extensions used in search model files else if (githubFile.Name.Split(".").Last() == "model" || githubFile.Name.Split(".").Last() == "index" || githubFile.Name.Split(".").Last() == "dict" || githubFile.Name.Split(".").Last() == "json" || githubFile.Name.Split(".").Last() == "npy") { isSearchModel = true; downloadFilePath = Path.Combine(destDir.FullName, githubFile.Name); } else { // Use Guids for Assembly and PDB Names to ensure uniqueness. downloadFilePath = Path.Combine(destDir.FullName, $"{assemblyName}.{fileExtension.ToLower()}"); } LogMessage($"Begin downloading File : {githubFile.Name.ToLower()} and saving it as : {downloadFilePath}"); await _githubClient.DownloadFile(githubFile.Download_url, downloadFilePath); } if (isSearchModel) { HitModelRefresh(folderName); } var scriptText = await FileHelper.GetFileContentAsync(csxFilePath); var assemblyPath = Path.Combine(destDir.FullName, $"{assemblyName}.dll"); var configFile = await FileHelper.GetFileContentAsync(confFilePath); var config = JsonConvert.DeserializeObject <PackageConfig>(configFile); var metadata = await FileHelper.GetFileContentAsync(metadataFilePath); var workerId = string.Equals(config?.Type, "gist", StringComparison.OrdinalIgnoreCase) ? "GistWorker" : "DetectorWorker"; await FileHelper.WriteToFileAsync(destDir.FullName, _workerIdFileName, workerId); if (GithubWorkers.ContainsKey(workerId)) { await GithubWorkers[workerId].CreateOrUpdateCacheAsync(destDir, scriptText, assemblyPath, metadata); } else { LogWarning($"Cannot find github worker with id {workerId}. Directory: {destDir.FullName}."); } }
private async Task DownloadContentAndUpdateInvokerCache(GithubEntry parentGithubEntry, DirectoryInfo destDir) { string assemblyName = Guid.NewGuid().ToString(); string csxFilePath = string.Empty; string lastCacheId = string.Empty; string cacheIdFilePath = Path.Combine(destDir.FullName, _cacheIdFileName); Assembly asm; HttpResponseMessage response = await _githubClient.Get(parentGithubEntry.Url); if (!response.IsSuccessStatusCode) { LogException($"GET Failed. Url : {parentGithubEntry.Url}, StatusCode : {response.StatusCode.ToString()}", null); return; } GithubEntry[] githubFiles = await response.Content.ReadAsAsyncCustom <GithubEntry[]>(); foreach (GithubEntry githubFile in githubFiles) { string fileExtension = githubFile.Name.Split(new char[] { '.' }).LastOrDefault(); if (string.IsNullOrWhiteSpace(githubFile.Download_url) || string.IsNullOrWhiteSpace(fileExtension)) { // Skip Downloading any directory (with empty download url) or any file without an extension. continue; } string downloadFilePath = Path.Combine(destDir.FullName, githubFile.Name.ToLower()); if (fileExtension.ToLower().Equals("csx")) { csxFilePath = downloadFilePath; } else { // Use Guids for Assembly and PDB Names to ensure uniqueness. downloadFilePath = Path.Combine(destDir.FullName, $"{assemblyName}.{fileExtension.ToLower()}"); } LogMessage($"Begin downloading File : {githubFile.Name.ToLower()} and saving it as : {downloadFilePath}"); await _githubClient.DownloadFile(githubFile.Download_url, downloadFilePath); } string scriptText = await FileHelper.GetFileContentAsync(csxFilePath); string assemblyPath = Path.Combine(destDir.FullName, $"{assemblyName}.dll"); LogMessage($"Loading assembly : {assemblyPath}"); asm = Assembly.LoadFrom(assemblyPath); EntityInvoker newInvoker = new EntityInvoker(new EntityMetadata(scriptText)); newInvoker.InitializeEntryPoint(asm); // Remove the Old Invoker from Cache lastCacheId = await FileHelper.GetFileContentAsync(cacheIdFilePath); if (!string.IsNullOrWhiteSpace(lastCacheId) && _invokerCache.TryRemoveValue(lastCacheId, out EntityInvoker oldInvoker)) { LogMessage($"Removing old invoker with id : {oldInvoker.EntryPointDefinitionAttribute.Id} from Cache"); oldInvoker.Dispose(); } // Add new invoker to Cache and update Cache Id File if (newInvoker.EntryPointDefinitionAttribute != null) { LogMessage($"Updating cache with new invoker with id : {newInvoker.EntryPointDefinitionAttribute.Id}"); _invokerCache.AddOrUpdate(newInvoker.EntryPointDefinitionAttribute.Id, newInvoker); await FileHelper.WriteToFileAsync(cacheIdFilePath, newInvoker.EntryPointDefinitionAttribute.Id); } else { LogWarning("Missing Entry Point Definition attribute. skipping cache update"); } }