public static List <LogInfo> WebGet(EngineState s, CodeCommand cmd) { // WebGet,<URL>,<DestPath>[<HashType>=<HashDigest>][,TimeOut=<Int>][,Referer=<URL>][,UserAgent=<Agent>][,NOERR] List <LogInfo> logs = new List <LogInfo>(); CodeInfo_WebGet info = cmd.Info.Cast <CodeInfo_WebGet>(); string url = StringEscaper.Preprocess(s, info.URL); string destPath = StringEscaper.Preprocess(s, info.DestPath); int timeOut = 10; if (info.TimeOut != null) { string timeOutStr = StringEscaper.Preprocess(s, info.TimeOut); if (!NumberHelper.ParseInt32(timeOutStr, out timeOut)) { return(LogInfo.LogErrorMessage(logs, $"TimeOut [{timeOutStr}] is not a valid positive integer")); } if (timeOut <= 0) { return(LogInfo.LogErrorMessage(logs, $"TimeOut [{timeOutStr}] is not a valid positive integer")); } } string refererUrl = null; if (info.Referer != null) { refererUrl = StringEscaper.Preprocess(s, info.Referer); } // Check PathSecurity in destPath if (!StringEscaper.PathSecurityCheck(destPath, out string pathErrorMsg)) { return(LogInfo.LogErrorMessage(logs, pathErrorMsg)); } Uri uri = new Uri(url); string destFile; if (Directory.Exists(destPath)) { destFile = Path.Combine(destPath, Path.GetFileName(uri.LocalPath)); } else // downloadTo is file { if (File.Exists(destPath)) { if (cmd.Type == CodeType.WebGetIfNotExist) { logs.Add(new LogInfo(LogState.Ignore, $"File [{destPath}] already exists")); return(logs); } logs.Add(new LogInfo(LogState.Overwrite, $"File [{destPath}] will be overwritten")); } else { Directory.CreateDirectory(FileHelper.GetDirNameEx(destPath)); } destFile = destPath; } string destFileExt = Path.GetExtension(destFile); s.MainViewModel.SetBuildCommandProgress("WebGet Progress"); try { // Set User-Agent to use // (1) Use Command's custom User-Agent // (2) Use EngineState's custom User-Agent // (3) Use PEBakery's default User-Agent string userAgent = null; if (info.UserAgent != null) { userAgent = info.UserAgent; } else { userAgent = s.CustomUserAgent; } if (info.HashType == HashHelper.HashType.None) { // Standard WebGet string tempPath = FileHelper.GetTempFile(destFileExt); HttpFileDownloader downloader = new HttpFileDownloader(s.MainViewModel, timeOut, userAgent, refererUrl); HttpFileDownloader.Report report; try { CancellationTokenSource ct = new CancellationTokenSource(); s.CancelWebGet = ct; Task <HttpFileDownloader.Report> task = downloader.Download(url, tempPath, ct.Token); task.Wait(ct.Token); report = task.Result; } catch (Exception e) { report = new HttpFileDownloader.Report(false, 0, Logger.LogExceptionMessage(e)); } finally { s.CancelWebGet = null; } int statusCode = report.StatusCode; if (report.Result) { FileHelper.FileReplaceEx(tempPath, destFile); logs.Add(new LogInfo(LogState.Success, $"[{destFile}] downloaded from [{url}]")); } else { LogState state = info.NoErrFlag ? LogState.Warning : LogState.Error; logs.Add(new LogInfo(state, $"Error occured while downloading [{url}]")); logs.Add(new LogInfo(LogState.Info, report.ErrorMsg)); if (statusCode == 0) { logs.Add(new LogInfo(LogState.Info, "Request failed, no response received.")); } else { logs.Add(new LogInfo(LogState.Info, $"Response returned HTTP status code [{statusCode}]")); } } // PEBakery extension -> Report exit code via #r if (!s.CompatDisableExtendedSectionParams) { s.ReturnValue = statusCode.ToString(); if (statusCode < 100) { logs.Add(new LogInfo(LogState.Success, $"Returned [{statusCode}] into [#r]")); } else { logs.Add(new LogInfo(LogState.Success, $"Returned HTTP status code [{statusCode}] to [#r]")); } } } else { // Validate downloaded file with hash Debug.Assert(info.HashDigest != null); string tempPath = FileHelper.GetTempFile(destFileExt); HttpFileDownloader downloader = new HttpFileDownloader(s.MainViewModel, timeOut, userAgent, refererUrl); HttpFileDownloader.Report report; try { CancellationTokenSource ct = new CancellationTokenSource(); s.CancelWebGet = ct; Task <HttpFileDownloader.Report> task = downloader.Download(url, tempPath, ct.Token); task.Wait(ct.Token); report = task.Result; } catch (Exception e) { report = new HttpFileDownloader.Report(false, 0, Logger.LogExceptionMessage(e)); } finally { s.CancelWebGet = null; } int statusCode = report.StatusCode; if (report.Result) { // Success -> Check hash string hashDigest = StringEscaper.Preprocess(s, info.HashDigest); if (hashDigest.Length != 2 * HashHelper.GetHashByteLen(info.HashType)) { return(LogInfo.LogErrorMessage(logs, $"Hash digest [{hashDigest}] is not [{info.HashType}]")); } string downDigest; using (FileStream fs = new FileStream(tempPath, FileMode.Open, FileAccess.Read)) { byte[] digest = HashHelper.GetHash(info.HashType, fs); downDigest = StringHelper.ToHexStr(digest); } if (hashDigest.Equals(downDigest, StringComparison.OrdinalIgnoreCase)) // Success { FileHelper.FileReplaceEx(tempPath, destFile); logs.Add(new LogInfo(LogState.Success, $"[{destFile}] downloaded from [{url}] and verified ")); } else { statusCode = 1; // 1 means hash mismatch logs.Add(new LogInfo(LogState.Error, $"Downloaded file from [{url}] was corrupted")); } } else { // Failure -> Log error message LogState state = info.NoErrFlag ? LogState.Warning : LogState.Error; logs.Add(new LogInfo(state, $"Error occured while downloading [{url}]")); logs.Add(new LogInfo(LogState.Info, report.ErrorMsg)); if (statusCode == 0) { logs.Add(new LogInfo(LogState.Info, "Request failed, no response received.")); } else { logs.Add(new LogInfo(LogState.Info, $"Response returned HTTP Status Code [{statusCode}]")); } } // PEBakery extension -> Report exit code via #r if (!s.CompatDisableExtendedSectionParams) { s.ReturnValue = statusCode.ToString(); if (statusCode < 100) { logs.Add(new LogInfo(LogState.Success, $"Returned [{statusCode}] into [#r]")); } else { logs.Add(new LogInfo(LogState.Success, $"Returned HTTP status code [{statusCode}] to [#r]")); } } } } finally { s.MainViewModel.ResetBuildCommandProgress(); } return(logs); }
public static List <LogInfo> WebGet(EngineState s, CodeCommand cmd) { // WebGet,<URL>,<DestPath>,[HashType],[HashDigest] List <LogInfo> logs = new List <LogInfo>(); Debug.Assert(cmd.Info.GetType() == typeof(CodeInfo_WebGet)); CodeInfo_WebGet info = cmd.Info as CodeInfo_WebGet; string url = StringEscaper.Preprocess(s, info.URL); string downloadTo = StringEscaper.Preprocess(s, info.DestPath); if (StringEscaper.PathSecurityCheck(downloadTo, out string errorMsg) == false) { logs.Add(new LogInfo(LogState.Error, errorMsg)); return(logs); } Uri uri = new Uri(url); string destPath; if (Directory.Exists(downloadTo)) // downloadTo is dir { destPath = Path.Combine(downloadTo, Path.GetFileName(uri.LocalPath)); } else // downloadTo is file { if (File.Exists(downloadTo)) { if (cmd.Type == CodeType.WebGetIfNotExist) { logs.Add(new LogInfo(LogState.Ignore, $"File [{downloadTo}] already exists")); return(logs); } else { logs.Add(new LogInfo(LogState.Ignore, $"File [{downloadTo}] will be overwritten")); } } else { Directory.CreateDirectory(Path.GetDirectoryName(downloadTo)); } destPath = downloadTo; } if (info.HashType != null && info.HashDigest != null) { // Calculate Hash After Downloading string tempPath = Path.GetTempFileName(); using (WebClient client = new WebClient()) { client.DownloadFile(url, tempPath); } string hashTypeStr = StringEscaper.Preprocess(s, info.HashType); string hashDigest = StringEscaper.Preprocess(s, info.HashDigest); HashType hashType; if (hashTypeStr.Equals("MD5", StringComparison.OrdinalIgnoreCase)) { hashType = HashType.MD5; } else if (hashTypeStr.Equals("SHA1", StringComparison.OrdinalIgnoreCase)) { hashType = HashType.SHA1; } else if (hashTypeStr.Equals("SHA256", StringComparison.OrdinalIgnoreCase)) { hashType = HashType.SHA256; } else if (hashTypeStr.Equals("SHA384", StringComparison.OrdinalIgnoreCase)) { hashType = HashType.SHA384; } else if (hashTypeStr.Equals("SHA512", StringComparison.OrdinalIgnoreCase)) { hashType = HashType.SHA512; } else { throw new ExecuteException($"Invalid hash type [{hashTypeStr}]"); } int byteLen = 0; switch (hashType) { case HashType.MD5: byteLen = 32; break; case HashType.SHA1: byteLen = 40; break; case HashType.SHA256: byteLen = 64; break; case HashType.SHA384: byteLen = 96; break; case HashType.SHA512: byteLen = 128; break; } if (hashDigest.Length != byteLen) { throw new ExecuteException($"Hash digest [{hashDigest}] is not [{hashTypeStr}]"); } string downDigest; using (FileStream fs = new FileStream(tempPath, FileMode.Open, FileAccess.Read)) { downDigest = HashHelper.CalcHashString(hashType, fs); } if (hashDigest.Equals(downDigest, StringComparison.OrdinalIgnoreCase)) // Success { File.Move(tempPath, destPath); logs.Add(new LogInfo(LogState.Success, $"[{url}] downloaded to [{downloadTo}] with integerity checked.")); } else { logs.Add(new LogInfo(LogState.Error, $"Downloaded [{url}], but it was corrupted")); } } else { // No Hash using (WebClient client = new WebClient()) { client.DownloadFile(url, destPath); } logs.Add(new LogInfo(LogState.Success, $"[{url}] downloaded to [{downloadTo}]")); } return(logs); }
public static List <LogInfo> WebGet(EngineState s, CodeCommand cmd) { // WebGet,<URL>,<DestPath>,[HashType],[HashDigest] List <LogInfo> logs = new List <LogInfo>(); CodeInfo_WebGet info = cmd.Info.Cast <CodeInfo_WebGet>(); string url = StringEscaper.Preprocess(s, info.URL); string destPath = StringEscaper.Preprocess(s, info.DestPath); const string destVar = @"%StatusCode%"; // Check PathSecurity in destPath if (!StringEscaper.PathSecurityCheck(destPath, out string pathErrorMsg)) { return(LogInfo.LogErrorMessage(logs, pathErrorMsg)); } Uri uri = new Uri(url); string destFile; if (Directory.Exists(destPath)) { destFile = Path.Combine(destPath, Path.GetFileName(uri.LocalPath)); } else // downloadTo is file { if (File.Exists(destPath)) { if (cmd.Type == CodeType.WebGetIfNotExist) { logs.Add(new LogInfo(LogState.Ignore, $"File [{destPath}] already exists")); return(logs); } logs.Add(new LogInfo(LogState.Overwrite, $"File [{destPath}] will be overwritten")); } else { Directory.CreateDirectory(FileHelper.GetDirNameEx(destPath)); } destFile = destPath; } s.MainViewModel.SetBuildCommandProgress("WebGet Progress"); try { if (info.HashType == HashHelper.HashType.None) { // Standard WebGet (bool result, int statusCode, string errorMsg) = DownloadFile(s, url, destFile); if (result) { logs.Add(new LogInfo(LogState.Success, $"[{destPath}] downloaded from [{url}]")); } else { LogState state = info.NoErrFlag ? LogState.Warning : LogState.Error; logs.Add(new LogInfo(state, $"Error occured while downloading [{url}]")); logs.Add(new LogInfo(LogState.Info, errorMsg)); if (statusCode == 0) { logs.Add(new LogInfo(LogState.Info, "Request failed, no response received.")); } else { logs.Add(new LogInfo(LogState.Info, $"Response returned HTTP Status Code [{statusCode}]")); } } List <LogInfo> varLogs = Variables.SetVariable(s, destVar, statusCode.ToString()); logs.AddRange(varLogs); } else { // Validate downloaded file with hash Debug.Assert(info.HashDigest != null); string tempPath = Path.GetTempFileName(); (bool result, int statusCode, string errorMsg) = DownloadFile(s, url, tempPath); if (result) { // Success -> Check hash string hashDigest = StringEscaper.Preprocess(s, info.HashDigest); if (hashDigest.Length != 2 * HashHelper.GetHashByteLen(info.HashType)) { return(LogInfo.LogErrorMessage(logs, $"Hash digest [{hashDigest}] is not [{info.HashType}]")); } string downDigest; using (FileStream fs = new FileStream(tempPath, FileMode.Open, FileAccess.Read)) { byte[] digest = HashHelper.GetHash(info.HashType, fs); downDigest = StringHelper.ToHexStr(digest); } if (hashDigest.Equals(downDigest, StringComparison.OrdinalIgnoreCase)) // Success { File.Move(tempPath, destFile); logs.Add(new LogInfo(LogState.Success, $"[{destPath}] downloaded from [{url}] and verified ")); } else { statusCode = 1; logs.Add(new LogInfo(LogState.Error, $"Downloaded file from [{url}] was corrupted")); } } else { // Failure -> Log error message LogState state = info.NoErrFlag ? LogState.Warning : LogState.Error; logs.Add(new LogInfo(state, $"Error occured while downloading [{url}]")); logs.Add(new LogInfo(LogState.Info, errorMsg)); if (statusCode == 0) { logs.Add(new LogInfo(LogState.Info, "Request failed, no response received.")); } else { logs.Add(new LogInfo(LogState.Info, $"Response returned HTTP Status Code [{statusCode}]")); } } List <LogInfo> varLogs = Variables.SetVariable(s, destVar, statusCode.ToString()); logs.AddRange(varLogs); } } finally { s.MainViewModel.ResetBuildCommandProgress(); } return(logs); }