private async void Process(List <PackageVersionPair> packageVersions, string pathToSolution) { if (!packageVersions.Any()) { _logger.LogInformation("No package version compatibilities to process."); return; } var packageVersionsGroupedByPackageId = packageVersions.ToDictionary(t => t.ToString(), t => t); var distinctPackageVersions = packageVersions.Distinct().ToList(); var exceptions = new Dictionary <PackageVersionPair, Exception>(); foreach (var compatibilityChecker in _compatibilityCheckers) { try { var compatibilityResults = compatibilityChecker.CheckAsync(distinctPackageVersions, pathToSolution); await Task.WhenAll(compatibilityResults.Select(result => { return(result.Value.ContinueWith(task => { if (task.IsCompletedSuccessfully) { packageVersionsGroupedByPackageId.Remove(result.Key.ToString()); if (_compatibilityTaskCompletionSources.TryGetValue(result.Key, out var packageVersionPairResult)) { packageVersionPairResult.TrySetResult(task.Result); } else { throw new ArgumentNullException($"Package version {result.Key} not found in compatibility tasks."); } } else { exceptions.TryAdd(result.Key, task.Exception); } })); }).ToList()); } catch (Exception ex) { _logger.LogError("Package compatibility processing failed with error: {0}", ex); } } foreach (var packageVersion in packageVersionsGroupedByPackageId.Select(packageVersionGroup => packageVersionGroup.Value)) { if (_compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var packageVersionPairResult)) { _logger.LogError($"Cound not find package {packageVersion} in all sources"); var defaultErrorMessage = $"Could not find package {packageVersion}. Compatibility task status: {packageVersionPairResult.Task.Status}."; var defaultException = new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), new PackageNotFoundException(defaultErrorMessage)); var exception = exceptions.GetValueOrDefault(packageVersion, defaultException); packageVersionPairResult.TrySetException(exception); } else { _logger.LogInformation($"Attempted to get package {packageVersion} from compatibility tasks but it was not found."); } } }
private async void ProcessCompatibility(IEnumerable <PackageVersionPair> packageVersions, Dictionary <PackageVersionPair, TaskCompletionSource <PackageDetails> > compatibilityTaskCompletionSources) { var packageVersionsFound = new HashSet <PackageVersionPair>(); var packageVersionsWithErrors = new HashSet <PackageVersionPair>(); var packageVersionsGroupedByPackageId = packageVersions .GroupBy(pv => pv.PackageId) .ToDictionary(pvGroup => pvGroup.Key, pvGroup => pvGroup.ToList()); foreach (var groupedPackageVersions in packageVersionsGroupedByPackageId) { var packageToDownload = groupedPackageVersions.Key.ToLower(); var fileToDownload = GetDownloadFilePath(CompatibilityCheckerType, packageToDownload); try { _logger.LogInformation("Downloading {0} from {1}", fileToDownload, CompatibilityCheckerType); using var stream = await _httpService.DownloadS3FileAsync(fileToDownload); using var gzipStream = new GZipStream(stream, CompressionMode.Decompress); using var streamReader = new StreamReader(gzipStream); var data = JsonConvert.DeserializeObject <PackageFromS3>(streamReader.ReadToEnd()); var packageDetails = data.Package ?? data.Namespaces; if (packageDetails.Name == null || !string.Equals(packageDetails.Name.Trim(), packageToDownload.Trim(), StringComparison.CurrentCultureIgnoreCase)) { throw new PackageDownloadMismatchException( actualPackage: packageDetails.Name, expectedPackage: packageToDownload); } foreach (var packageVersion in groupedPackageVersions.Value) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.SetResult(packageDetails); packageVersionsFound.Add(packageVersion); } } } catch (Exception ex) { if (ex.Message.Contains("404")) { _logger.LogInformation($"Encountered {ex.GetType()} while downloading and parsing {fileToDownload} " + $"from {CompatibilityCheckerType}, but it was ignored. " + $"ErrorMessage: {ex.Message}."); } else { _logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", fileToDownload, CompatibilityCheckerType, ex); } foreach (var packageVersion in groupedPackageVersions.Value) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex)); packageVersionsWithErrors.Add(packageVersion); } } } } foreach (var packageVersion in packageVersions) { if (packageVersionsFound.Contains(packageVersion) || packageVersionsWithErrors.Contains(packageVersion)) { continue; } if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { var errorMessage = $"Could not find package {packageVersion} in external source; try checking an internal source."; _logger.LogInformation(errorMessage); var innerException = new PackageNotFoundException(errorMessage); taskCompletionSource.TrySetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), innerException)); } } }
private async void GetPackageDetailsAsync(List <Task> processCompatibilityTasks, string packageId, SortedSet <string> compatibleVersions, SortedSet <string> versions, TaskCompletionSource <PackageDetails> taskCompletionSource) { await Task.WhenAll(processCompatibilityTasks.ToArray()); if (versions.Count == 0) { taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageId), null)); } var packageDetails = new PackageDetails() { Name = packageId, Versions = versions, Targets = new Dictionary <string, SortedSet <string> > { { "netcoreapp3.1", compatibleVersions } }, Api = new List <ApiDetails>().ToArray() }; taskCompletionSource.SetResult(packageDetails); }
private async void ProcessCompatibility(IEnumerable <PackageVersionPair> packageVersions, Dictionary <PackageVersionPair, TaskCompletionSource <PackageDetails> > compatibilityTaskCompletionSources, string pathToSolution, bool isIncremental, bool incrementalRefresh) { var packageVersionsFound = new HashSet <PackageVersionPair>(); var packageVersionsWithErrors = new HashSet <PackageVersionPair>(); var packageVersionsGroupedByPackageId = packageVersions .GroupBy(pv => pv.PackageId) .ToDictionary(pvGroup => pvGroup.Key, pvGroup => pvGroup.ToList()); foreach (var groupedPackageVersions in packageVersionsGroupedByPackageId) { var packageToDownload = groupedPackageVersions.Key.ToLower(); var fileToDownload = GetDownloadFilePath(CompatibilityCheckerType, packageToDownload); try { string tempDirectoryPath = GetTempDirectory(pathToSolution); PackageDetails packageDetails = null; if (isIncremental) { if (incrementalRefresh || !IsPackageInFile(fileToDownload, tempDirectoryPath)) { _logger.LogInformation("Downloading {0} from {1}", fileToDownload, CompatibilityCheckerType); packageDetails = await GetPackageDetailFromS3(fileToDownload, _httpService); _logger.LogInformation("Caching {0} from {1} to Temp", fileToDownload, CompatibilityCheckerType); CachePackageDetailsToFile(fileToDownload, packageDetails, tempDirectoryPath); } else { _logger.LogInformation("Fetching {0} from {1} from Temp", fileToDownload, CompatibilityCheckerType); packageDetails = GetPackageDetailFromFile(fileToDownload, tempDirectoryPath); } } else { packageDetails = await GetPackageDetailFromS3(fileToDownload, _httpService); } if (packageDetails.Name == null || !string.Equals(packageDetails.Name.Trim().ToLower(), packageToDownload.Trim().ToLower(), StringComparison.OrdinalIgnoreCase)) { throw new PackageDownloadMismatchException( actualPackage: packageDetails.Name, expectedPackage: packageToDownload); } foreach (var packageVersion in groupedPackageVersions.Value) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.SetResult(packageDetails); packageVersionsFound.Add(packageVersion); } } } catch (OutOfMemoryException ex) { _logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", fileToDownload, CompatibilityCheckerType, ex); MemoryUtils.LogSolutiontSize(_logger, pathToSolution); MemoryUtils.LogMemoryConsumption(_logger); } catch (Exception ex) { if (ex.Message.Contains("404")) { _logger.LogInformation($"Encountered {ex.GetType()} while downloading and parsing {fileToDownload} " + $"from {CompatibilityCheckerType}, but it was ignored. " + $"ErrorMessage: {ex.Message}."); // filter all 404 errors ex = null; } else { _logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", fileToDownload, CompatibilityCheckerType, ex); } foreach (var packageVersion in groupedPackageVersions.Value) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex)); packageVersionsWithErrors.Add(packageVersion); } } } } foreach (var packageVersion in packageVersions) { if (packageVersionsFound.Contains(packageVersion) || packageVersionsWithErrors.Contains(packageVersion)) { continue; } if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { var errorMessage = $"Could not find package {packageVersion} in external source; try checking an internal source."; _logger.LogInformation(errorMessage); var innerException = new PackageNotFoundException(errorMessage); taskCompletionSource.TrySetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), innerException)); } } }
/// <summary> /// Checks the packages in Portability Analyzer /// </summary> /// <param name="packageVersions">The package versions to check</param> /// <param name="pathToSolution">Path to the solution to check</param> /// <returns>The results of the compatibility check</returns> public Dictionary <PackageVersionPair, Task <PackageDetails> > Check( IEnumerable <PackageVersionPair> packageVersions, string pathToSolution) { var compatibilityTaskCompletionSources = new Dictionary <PackageVersionPair, TaskCompletionSource <PackageDetails> >(); try { if (_manifest == null) { var manifestTask = GetManifestAsync(); manifestTask.Wait(); _manifest = manifestTask.Result; } var foundPackages = new Dictionary <string, List <PackageVersionPair> >(); packageVersions.ToList().ForEach(p => { if (p.PackageSourceType != PackageSourceType.SDK) { return; } var value = _manifest.GetValueOrDefault(p.PackageId, null); if (value != null) { compatibilityTaskCompletionSources.Add(p, new TaskCompletionSource <PackageDetails>()); if (!foundPackages.ContainsKey(value)) { foundPackages.Add(value, new List <PackageVersionPair>()); } foundPackages[value].Add(p); } }); _logger.LogInformation("Checking Portability Analyzer source for compatibility of {0} package(s)", foundPackages.Count); if (foundPackages.Any()) { Task.Run(() => { _semaphore.Wait(); try { ProcessCompatibility(packageVersions, foundPackages, compatibilityTaskCompletionSources); } finally { _semaphore.Release(); } }); } return(compatibilityTaskCompletionSources.ToDictionary(t => t.Key, t => t.Value.Task)); } catch (Exception ex) { foreach (var packageVersion in packageVersions) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.TrySetException( new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex)); } } return(compatibilityTaskCompletionSources.ToDictionary(t => t.Key, t => t.Value.Task)); } }
/// <summary> /// Processes the package compatibility /// </summary> /// <param name="packageVersions">Collection of package versions to check</param> /// <param name="foundPackages">Collection of packages found</param> /// <param name="compatibilityTaskCompletionSources">The results of the compatibility check to process</param> private async void ProcessCompatibility(IEnumerable <PackageVersionPair> packageVersions, Dictionary <string, List <PackageVersionPair> > foundPackages, Dictionary <PackageVersionPair, TaskCompletionSource <PackageDetails> > compatibilityTaskCompletionSources) { var packageVersionsFound = new HashSet <PackageVersionPair>(); var packageVersionsWithErrors = new HashSet <PackageVersionPair>(); foreach (var url in foundPackages) { try { _logger.LogInformation("Downloading {0} from {1}", url.Key, CompatibilityCheckerType); using var stream = await _httpService.DownloadS3FileAsync(url.Key); using var gzipStream = new GZipStream(stream, CompressionMode.Decompress); using var streamReader = new StreamReader(gzipStream); var packageFromS3 = JsonConvert.DeserializeObject <PackageFromS3>(streamReader.ReadToEnd()); packageFromS3.Package.Name = url.Value.First().PackageId; foreach (var packageVersion in url.Value) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.SetResult(packageFromS3.Package); packageVersionsFound.Add(packageVersion); } } } catch (Exception ex) { if (ex.Message.Contains("404")) { _logger.LogInformation($"Encountered {ex.GetType()} while downloading and parsing {url.Key} " + $"from {CompatibilityCheckerType}, but it was ignored. " + $"ErrorMessage: {ex.Message}."); } else { _logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", url.Key, CompatibilityCheckerType, ex); } foreach (var packageVersion in url.Value) { if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex)); packageVersionsWithErrors.Add(packageVersion); } } } } foreach (var packageVersion in packageVersions) { if (packageVersionsFound.Contains(packageVersion) || packageVersionsWithErrors.Contains(packageVersion)) { continue; } if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource)) { var errorMessage = $"Could not find package {packageVersion} in external source; try checking an internal source."; _logger.LogInformation(errorMessage); var innerException = new PackageNotFoundException(errorMessage); taskCompletionSource.TrySetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), innerException)); } } }