/// <summary> /// Fetch the specified file and convert it to the corresponding data structure. /// </summary> protected T FetchFile <T>(string filename, string cacheKey = null, string sha256 = null) { if (string.IsNullOrEmpty(cacheKey)) { cacheKey = filename; filename = $"{baseUri}/{filename}"; } // url-encode $ signs in URLs as bad proxies choke on them var pos = filename.IndexOf('$'); if (pos != -1 && Regex.IsMatch(filename, "^https?://.*", RegexOptions.IgnoreCase)) { filename = $"{filename.Substring(0, pos)}%24{filename.Substring(pos + 1)}"; } var allowUseHttp = uriIsIntelligent; void IntelligentAdjustmentSSL() { if (allowUseHttp) { // Ssl is used by default, if it fails, we allow retry and then try again. uri = DowngradeSSL(uri); baseUri = DowngradeSSL(baseUri); filename = DowngradeSSL(filename); allowUseHttp = false; } else if (configRepository.AllowSSLDowngrade) { // undo downgrade before trying again if http seems to be hijacked or // modifying content somehow. uri = UpgradeSSL(uri); baseUri = UpgradeSSL(baseUri); filename = UpgradeSSL(filename); } } var retries = 3; while (retries-- > 0) { try { if (eventDispatcher != null) { var preFileDownloadEvent = new PreFileDownloadEventArgs(PluginEvents.PreFileDownload, transport, filename); eventDispatcher.Dispatch(this, preFileDownloadEvent); } var jsonBody = transport.GetString(filename); if (!string.IsNullOrEmpty(sha256) && sha256 != Security.Sha256(jsonBody)) { IntelligentAdjustmentSSL(); if (retries > 0) { Thread.Sleep(100); continue; } throw new RepositorySecurityException($"The contents of {filename} do not match its signature. This could indicate a man-in-the-middle attack. Please report the error immediately."); } cache.Write(cacheKey, jsonBody); return(JsonFile.Parse <T>(jsonBody)); } catch (SException ex) { if (ex is TransportException transportException) { if (transportException.HttpStatusCode == HttpStatusCode.NotFound) { throw; } // When the http status code is not recognized, this // means the server may not be able to connect. if (transportException.HttpStatusCode == 0) { IntelligentAdjustmentSSL(); } } // todo: logic exception need throw exception immediately. if (retries > 0) { Thread.Sleep(100); continue; } if (ex is RepositorySecurityException) { throw; } if (string.IsNullOrEmpty(cacheKey)) { throw; } if (!cache.TryReadSha256(cacheKey, out string content) || string.IsNullOrEmpty(content)) { throw; } if (!degradedMode) { io.WriteError($"<warning>{ex.Message}</warning>"); io.WriteError($"<warning>{uri} could not be fully loaded, package information was loaded from the local cache and may be out of date</warning>"); } degradedMode = true; return(JsonFile.Parse <T>(content)); } } throw new UnexpectedException("Unexpectedly, this code should not be executed."); }
/// <inheritdoc cref="IDownloader.Download(IPackage, string)" /> public virtual Task Download(IPackage package, string cwd, bool output) { if (string.IsNullOrEmpty(package.GetDistUri())) { throw new RuntimeException("The given package is missing url information."); } var downloadedFilePath = GetDownloadedFilePath(package, cwd); fileSystem.Delete(cwd); void DoDownload(string uri) { var processedUri = ProcessUri(package, uri); if (eventDispatcher != null) { var preFileDownloadEvent = new PreFileDownloadEventArgs(PluginEvents.PreFileDownload, transport, processedUri); eventDispatcher.Dispatch(this, preFileDownloadEvent); } try { var checksum = package.GetDistShasum(); var cacheKey = GetCacheKey(package, processedUri); void DownloadFromHttp(int retries = 3) { SException processException = null; while (retries-- > 0) { try { transport.Copy( processedUri, downloadedFilePath, new ReportDownloadProgress(io, $"{GetDownloadingPrompt(package)}: ")); break; } catch (TransportException ex) { // clean up immediately. otherwise it will lead to failure forever. fileSystem.Delete(downloadedFilePath); // if we got an http response with a proper code, then // requesting again will probably not help, abort. var serverProblems = new[] { HttpStatusCode.InternalServerError, HttpStatusCode.BadGateway, HttpStatusCode.ServiceUnavailable, HttpStatusCode.GatewayTimeout, }; if (ex.HttpStatusCode != 0 && (!Array.Exists(serverProblems, (code) => code == ex.HttpStatusCode) || retries <= 0)) { throw; } processException = ex; Thread.Sleep(500); } } if (processException != null) { ExceptionDispatchInfo.Capture(processException).Throw(); } if (cache != null && cache.CopyFrom(cacheKey, downloadedFilePath)) { lastCacheWrites[package.GetName()] = cacheKey; } } // use from cache if it is present and has a valid checksum // or we have no checksum to check against if (cache != null && cache.Sha1(cacheKey, checksum) && cache.CopyTo(cacheKey, downloadedFilePath)) { if (output) { io.WriteError($" - Loading <info>{package.GetName()}</info> (<comment>{package.GetVersionPrettyFull()}</comment>) from cache", false); } } else { if (output) { io.WriteError(GetDownloadingPrompt(package), false); } DownloadFromHttp(); } // Verify the file to ensure that the file is not damaged. if (!fileSystem.Exists(downloadedFilePath, FileSystemOptions.File)) { throw new UnexpectedException($"{uri} could not be saved to {downloadedFilePath} , make sure the directory is writable and you have internet connectivity."); } if (!string.IsNullOrEmpty(checksum)) { using (var stream = fileSystem.Read(downloadedFilePath)) { if (Security.Sha1(stream) != checksum) { throw new UnexpectedException($"The checksum verification of the file failed (downloaded from {uri})"); } } } } catch { fileSystem.Delete(downloadedFilePath); ClearLastCacheWrite(package); throw; } finally { if (output) { io.OverwriteError(string.Empty, false); } } } void Start() { var uris = package.GetDistUris(); // The performance popped from the tail will be better // than popping out from the head Array.Reverse(uris); while (uris.Length > 0) { var uri = Arr.Pop(ref uris); try { DoDownload(uri); break; } catch (SException ex) { if (io.IsDebug) { io.WriteError(string.Empty); io.WriteError($"Faild: [{ex.GetType()}] {ex.Message}"); } else if (uris.Length > 0) { io.WriteError(string.Empty); io.WriteError($" Failed, trying the next URL ({ex.Message})"); } if (uris.Length <= 0) { throw; } } } } return(Task.Run(Start)); }