private async void Process(List <PackageVersionPair> packageVersions, string pathToSolution)
{
if (!packageVersions.Any())
{
_logger.LogInformation("No package version compatibilities to process.");
return;
}
var packageVersionsGroupedByPackageId = packageVersions.ToDictionary(t => t.ToString(), t => t);
var distinctPackageVersions = packageVersions.Distinct().ToList();
var exceptions = new Dictionary <PackageVersionPair, Exception>();
foreach (var compatibilityChecker in _compatibilityCheckers)
{
try
{
var compatibilityResults = compatibilityChecker.CheckAsync(distinctPackageVersions, pathToSolution);
await Task.WhenAll(compatibilityResults.Select(result =>
{
return(result.Value.ContinueWith(task =>
{
if (task.IsCompletedSuccessfully)
{
packageVersionsGroupedByPackageId.Remove(result.Key.ToString());
if (_compatibilityTaskCompletionSources.TryGetValue(result.Key, out var packageVersionPairResult))
{
packageVersionPairResult.TrySetResult(task.Result);
}
else
{
throw new ArgumentNullException($"Package version {result.Key} not found in compatibility tasks.");
}
}
else
{
exceptions.TryAdd(result.Key, task.Exception);
}
}));
}).ToList());
}
catch (Exception ex)
{
_logger.LogError("Package compatibility processing failed with error: {0}", ex);
}
}
foreach (var packageVersion in packageVersionsGroupedByPackageId.Select(packageVersionGroup => packageVersionGroup.Value))
{
if (_compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var packageVersionPairResult))
{
_logger.LogError($"Cound not find package {packageVersion} in all sources");
var defaultErrorMessage = $"Could not find package {packageVersion}. Compatibility task status: {packageVersionPairResult.Task.Status}.";
var defaultException = new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), new PackageNotFoundException(defaultErrorMessage));
var exception = exceptions.GetValueOrDefault(packageVersion, defaultException);
packageVersionPairResult.TrySetException(exception);
}
else
{
_logger.LogInformation($"Attempted to get package {packageVersion} from compatibility tasks but it was not found.");
}
}
}
private async void ProcessCompatibility(IEnumerable <PackageVersionPair> packageVersions,
Dictionary <PackageVersionPair, TaskCompletionSource <PackageDetails> > compatibilityTaskCompletionSources)
{
var packageVersionsFound = new HashSet <PackageVersionPair>();
var packageVersionsWithErrors = new HashSet <PackageVersionPair>();
var packageVersionsGroupedByPackageId = packageVersions
.GroupBy(pv => pv.PackageId)
.ToDictionary(pvGroup => pvGroup.Key, pvGroup => pvGroup.ToList());
foreach (var groupedPackageVersions in packageVersionsGroupedByPackageId)
{
var packageToDownload = groupedPackageVersions.Key.ToLower();
var fileToDownload = GetDownloadFilePath(CompatibilityCheckerType, packageToDownload);
try
{
_logger.LogInformation("Downloading {0} from {1}", fileToDownload,
CompatibilityCheckerType);
using var stream = await _httpService.DownloadS3FileAsync(fileToDownload);
using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
using var streamReader = new StreamReader(gzipStream);
var data = JsonConvert.DeserializeObject <PackageFromS3>(streamReader.ReadToEnd());
var packageDetails = data.Package ?? data.Namespaces;
if (packageDetails.Name == null || !string.Equals(packageDetails.Name.Trim(),
packageToDownload.Trim(), StringComparison.CurrentCultureIgnoreCase))
{
throw new PackageDownloadMismatchException(
actualPackage: packageDetails.Name,
expectedPackage: packageToDownload);
}
foreach (var packageVersion in groupedPackageVersions.Value)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.SetResult(packageDetails);
packageVersionsFound.Add(packageVersion);
}
}
}
catch (Exception ex)
{
if (ex.Message.Contains("404"))
{
_logger.LogInformation($"Encountered {ex.GetType()} while downloading and parsing {fileToDownload} " +
$"from {CompatibilityCheckerType}, but it was ignored. " +
$"ErrorMessage: {ex.Message}.");
}
else
{
_logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", fileToDownload, CompatibilityCheckerType, ex);
}
foreach (var packageVersion in groupedPackageVersions.Value)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex));
packageVersionsWithErrors.Add(packageVersion);
}
}
}
}
foreach (var packageVersion in packageVersions)
{
if (packageVersionsFound.Contains(packageVersion) || packageVersionsWithErrors.Contains(packageVersion))
{
continue;
}
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
var errorMessage = $"Could not find package {packageVersion} in external source; try checking an internal source.";
_logger.LogInformation(errorMessage);
var innerException = new PackageNotFoundException(errorMessage);
taskCompletionSource.TrySetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), innerException));
}
}
}
private async void GetPackageDetailsAsync(List <Task> processCompatibilityTasks, string packageId,
SortedSet <string> compatibleVersions, SortedSet <string> versions,
TaskCompletionSource <PackageDetails> taskCompletionSource)
{
await Task.WhenAll(processCompatibilityTasks.ToArray());
if (versions.Count == 0)
{
taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageId), null));
}
var packageDetails = new PackageDetails()
{
Name = packageId,
Versions = versions,
Targets = new Dictionary <string, SortedSet <string> >
{
{ "netcoreapp3.1", compatibleVersions }
},
Api = new List <ApiDetails>().ToArray()
};
taskCompletionSource.SetResult(packageDetails);
}
private async void ProcessCompatibility(IEnumerable <PackageVersionPair> packageVersions,
Dictionary <PackageVersionPair, TaskCompletionSource <PackageDetails> > compatibilityTaskCompletionSources,
string pathToSolution, bool isIncremental, bool incrementalRefresh)
{
var packageVersionsFound = new HashSet <PackageVersionPair>();
var packageVersionsWithErrors = new HashSet <PackageVersionPair>();
var packageVersionsGroupedByPackageId = packageVersions
.GroupBy(pv => pv.PackageId)
.ToDictionary(pvGroup => pvGroup.Key, pvGroup => pvGroup.ToList());
foreach (var groupedPackageVersions in packageVersionsGroupedByPackageId)
{
var packageToDownload = groupedPackageVersions.Key.ToLower();
var fileToDownload = GetDownloadFilePath(CompatibilityCheckerType, packageToDownload);
try
{
string tempDirectoryPath = GetTempDirectory(pathToSolution);
PackageDetails packageDetails = null;
if (isIncremental)
{
if (incrementalRefresh || !IsPackageInFile(fileToDownload, tempDirectoryPath))
{
_logger.LogInformation("Downloading {0} from {1}", fileToDownload, CompatibilityCheckerType);
packageDetails = await GetPackageDetailFromS3(fileToDownload, _httpService);
_logger.LogInformation("Caching {0} from {1} to Temp", fileToDownload, CompatibilityCheckerType);
CachePackageDetailsToFile(fileToDownload, packageDetails, tempDirectoryPath);
}
else
{
_logger.LogInformation("Fetching {0} from {1} from Temp", fileToDownload, CompatibilityCheckerType);
packageDetails = GetPackageDetailFromFile(fileToDownload, tempDirectoryPath);
}
}
else
{
packageDetails = await GetPackageDetailFromS3(fileToDownload, _httpService);
}
if (packageDetails.Name == null || !string.Equals(packageDetails.Name.Trim().ToLower(),
packageToDownload.Trim().ToLower(), StringComparison.OrdinalIgnoreCase))
{
throw new PackageDownloadMismatchException(
actualPackage: packageDetails.Name,
expectedPackage: packageToDownload);
}
foreach (var packageVersion in groupedPackageVersions.Value)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.SetResult(packageDetails);
packageVersionsFound.Add(packageVersion);
}
}
}
catch (OutOfMemoryException ex)
{
_logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", fileToDownload, CompatibilityCheckerType, ex);
MemoryUtils.LogSolutiontSize(_logger, pathToSolution);
MemoryUtils.LogMemoryConsumption(_logger);
}
catch (Exception ex)
{
if (ex.Message.Contains("404"))
{
_logger.LogInformation($"Encountered {ex.GetType()} while downloading and parsing {fileToDownload} " +
$"from {CompatibilityCheckerType}, but it was ignored. " +
$"ErrorMessage: {ex.Message}.");
// filter all 404 errors
ex = null;
}
else
{
_logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", fileToDownload, CompatibilityCheckerType, ex);
}
foreach (var packageVersion in groupedPackageVersions.Value)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex));
packageVersionsWithErrors.Add(packageVersion);
}
}
}
}
foreach (var packageVersion in packageVersions)
{
if (packageVersionsFound.Contains(packageVersion) || packageVersionsWithErrors.Contains(packageVersion))
{
continue;
}
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
var errorMessage = $"Could not find package {packageVersion} in external source; try checking an internal source.";
_logger.LogInformation(errorMessage);
var innerException = new PackageNotFoundException(errorMessage);
taskCompletionSource.TrySetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), innerException));
}
}
}
/// <summary>
/// Checks the packages in Portability Analyzer
/// </summary>
/// <param name="packageVersions">The package versions to check</param>
/// <param name="pathToSolution">Path to the solution to check</param>
/// <returns>The results of the compatibility check</returns>
public Dictionary <PackageVersionPair, Task <PackageDetails> > Check(
IEnumerable <PackageVersionPair> packageVersions,
string pathToSolution)
{
var compatibilityTaskCompletionSources = new Dictionary <PackageVersionPair, TaskCompletionSource <PackageDetails> >();
try
{
if (_manifest == null)
{
var manifestTask = GetManifestAsync();
manifestTask.Wait();
_manifest = manifestTask.Result;
}
var foundPackages = new Dictionary <string, List <PackageVersionPair> >();
packageVersions.ToList().ForEach(p =>
{
if (p.PackageSourceType != PackageSourceType.SDK)
{
return;
}
var value = _manifest.GetValueOrDefault(p.PackageId, null);
if (value != null)
{
compatibilityTaskCompletionSources.Add(p, new TaskCompletionSource <PackageDetails>());
if (!foundPackages.ContainsKey(value))
{
foundPackages.Add(value, new List <PackageVersionPair>());
}
foundPackages[value].Add(p);
}
});
_logger.LogInformation("Checking Portability Analyzer source for compatibility of {0} package(s)", foundPackages.Count);
if (foundPackages.Any())
{
Task.Run(() =>
{
_semaphore.Wait();
try
{
ProcessCompatibility(packageVersions, foundPackages, compatibilityTaskCompletionSources);
}
finally
{
_semaphore.Release();
}
});
}
return(compatibilityTaskCompletionSources.ToDictionary(t => t.Key, t => t.Value.Task));
}
catch (Exception ex)
{
foreach (var packageVersion in packageVersions)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.TrySetException(
new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex));
}
}
return(compatibilityTaskCompletionSources.ToDictionary(t => t.Key, t => t.Value.Task));
}
}
/// <summary>
/// Processes the package compatibility
/// </summary>
/// <param name="packageVersions">Collection of package versions to check</param>
/// <param name="foundPackages">Collection of packages found</param>
/// <param name="compatibilityTaskCompletionSources">The results of the compatibility check to process</param>
private async void ProcessCompatibility(IEnumerable <PackageVersionPair> packageVersions,
Dictionary <string, List <PackageVersionPair> > foundPackages,
Dictionary <PackageVersionPair, TaskCompletionSource <PackageDetails> > compatibilityTaskCompletionSources)
{
var packageVersionsFound = new HashSet <PackageVersionPair>();
var packageVersionsWithErrors = new HashSet <PackageVersionPair>();
foreach (var url in foundPackages)
{
try
{
_logger.LogInformation("Downloading {0} from {1}", url.Key, CompatibilityCheckerType);
using var stream = await _httpService.DownloadS3FileAsync(url.Key);
using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
using var streamReader = new StreamReader(gzipStream);
var packageFromS3 = JsonConvert.DeserializeObject <PackageFromS3>(streamReader.ReadToEnd());
packageFromS3.Package.Name = url.Value.First().PackageId;
foreach (var packageVersion in url.Value)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.SetResult(packageFromS3.Package);
packageVersionsFound.Add(packageVersion);
}
}
}
catch (Exception ex)
{
if (ex.Message.Contains("404"))
{
_logger.LogInformation($"Encountered {ex.GetType()} while downloading and parsing {url.Key} " +
$"from {CompatibilityCheckerType}, but it was ignored. " +
$"ErrorMessage: {ex.Message}.");
}
else
{
_logger.LogError("Failed when downloading and parsing {0} from {1}, {2}", url.Key, CompatibilityCheckerType, ex);
}
foreach (var packageVersion in url.Value)
{
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
taskCompletionSource.SetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), ex));
packageVersionsWithErrors.Add(packageVersion);
}
}
}
}
foreach (var packageVersion in packageVersions)
{
if (packageVersionsFound.Contains(packageVersion) || packageVersionsWithErrors.Contains(packageVersion))
{
continue;
}
if (compatibilityTaskCompletionSources.TryGetValue(packageVersion, out var taskCompletionSource))
{
var errorMessage = $"Could not find package {packageVersion} in external source; try checking an internal source.";
_logger.LogInformation(errorMessage);
var innerException = new PackageNotFoundException(errorMessage);
taskCompletionSource.TrySetException(new PortingAssistantClientException(ExceptionMessage.PackageNotFound(packageVersion), innerException));
}
}
}
