public static List <LogInfo> WebGet(EngineState s, CodeCommand cmd)
{ // WebGet,<URL>,<DestPath>[<HashType>=<HashDigest>][,TimeOut=<Int>][,Referer=<URL>][,UserAgent=<Agent>][,NOERR]
List <LogInfo> logs = new List <LogInfo>();
CodeInfo_WebGet info = cmd.Info.Cast <CodeInfo_WebGet>();
string url = StringEscaper.Preprocess(s, info.URL);
string destPath = StringEscaper.Preprocess(s, info.DestPath);
int timeOut = 10;
if (info.TimeOut != null)
{
string timeOutStr = StringEscaper.Preprocess(s, info.TimeOut);
if (!NumberHelper.ParseInt32(timeOutStr, out timeOut))
{
return(LogInfo.LogErrorMessage(logs, $"TimeOut [{timeOutStr}] is not a valid positive integer"));
}
if (timeOut <= 0)
{
return(LogInfo.LogErrorMessage(logs, $"TimeOut [{timeOutStr}] is not a valid positive integer"));
}
}
string refererUrl = null;
if (info.Referer != null)
{
refererUrl = StringEscaper.Preprocess(s, info.Referer);
}
// Check PathSecurity in destPath
if (!StringEscaper.PathSecurityCheck(destPath, out string pathErrorMsg))
{
return(LogInfo.LogErrorMessage(logs, pathErrorMsg));
}
Uri uri = new Uri(url);
string destFile;
if (Directory.Exists(destPath))
{
destFile = Path.Combine(destPath, Path.GetFileName(uri.LocalPath));
}
else // downloadTo is file
{
if (File.Exists(destPath))
{
if (cmd.Type == CodeType.WebGetIfNotExist)
{
logs.Add(new LogInfo(LogState.Ignore, $"File [{destPath}] already exists"));
return(logs);
}
logs.Add(new LogInfo(LogState.Overwrite, $"File [{destPath}] will be overwritten"));
}
else
{
Directory.CreateDirectory(FileHelper.GetDirNameEx(destPath));
}
destFile = destPath;
}
string destFileExt = Path.GetExtension(destFile);
s.MainViewModel.SetBuildCommandProgress("WebGet Progress");
try
{
// Set User-Agent to use
// (1) Use Command's custom User-Agent
// (2) Use EngineState's custom User-Agent
// (3) Use PEBakery's default User-Agent
string userAgent = null;
if (info.UserAgent != null)
{
userAgent = info.UserAgent;
}
else
{
userAgent = s.CustomUserAgent;
}
if (info.HashType == HashHelper.HashType.None)
{ // Standard WebGet
string tempPath = FileHelper.GetTempFile(destFileExt);
HttpFileDownloader downloader = new HttpFileDownloader(s.MainViewModel, timeOut, userAgent, refererUrl);
HttpFileDownloader.Report report;
try
{
CancellationTokenSource ct = new CancellationTokenSource();
s.CancelWebGet = ct;
Task <HttpFileDownloader.Report> task = downloader.Download(url, tempPath, ct.Token);
task.Wait(ct.Token);
report = task.Result;
}
catch (Exception e)
{
report = new HttpFileDownloader.Report(false, 0, Logger.LogExceptionMessage(e));
}
finally
{
s.CancelWebGet = null;
}
int statusCode = report.StatusCode;
if (report.Result)
{
FileHelper.FileReplaceEx(tempPath, destFile);
logs.Add(new LogInfo(LogState.Success, $"[{destFile}] downloaded from [{url}]"));
}
else
{
LogState state = info.NoErrFlag ? LogState.Warning : LogState.Error;
logs.Add(new LogInfo(state, $"Error occured while downloading [{url}]"));
logs.Add(new LogInfo(LogState.Info, report.ErrorMsg));
if (statusCode == 0)
{
logs.Add(new LogInfo(LogState.Info, "Request failed, no response received."));
}
else
{
logs.Add(new LogInfo(LogState.Info, $"Response returned HTTP status code [{statusCode}]"));
}
}
// PEBakery extension -> Report exit code via #r
if (!s.CompatDisableExtendedSectionParams)
{
s.ReturnValue = statusCode.ToString();
if (statusCode < 100)
{
logs.Add(new LogInfo(LogState.Success, $"Returned [{statusCode}] into [#r]"));
}
else
{
logs.Add(new LogInfo(LogState.Success, $"Returned HTTP status code [{statusCode}] to [#r]"));
}
}
}
else
{ // Validate downloaded file with hash
Debug.Assert(info.HashDigest != null);
string tempPath = FileHelper.GetTempFile(destFileExt);
HttpFileDownloader downloader = new HttpFileDownloader(s.MainViewModel, timeOut, userAgent, refererUrl);
HttpFileDownloader.Report report;
try
{
CancellationTokenSource ct = new CancellationTokenSource();
s.CancelWebGet = ct;
Task <HttpFileDownloader.Report> task = downloader.Download(url, tempPath, ct.Token);
task.Wait(ct.Token);
report = task.Result;
}
catch (Exception e)
{
report = new HttpFileDownloader.Report(false, 0, Logger.LogExceptionMessage(e));
}
finally
{
s.CancelWebGet = null;
}
int statusCode = report.StatusCode;
if (report.Result)
{ // Success -> Check hash
string hashDigest = StringEscaper.Preprocess(s, info.HashDigest);
if (hashDigest.Length != 2 * HashHelper.GetHashByteLen(info.HashType))
{
return(LogInfo.LogErrorMessage(logs, $"Hash digest [{hashDigest}] is not [{info.HashType}]"));
}
string downDigest;
using (FileStream fs = new FileStream(tempPath, FileMode.Open, FileAccess.Read))
{
byte[] digest = HashHelper.GetHash(info.HashType, fs);
downDigest = StringHelper.ToHexStr(digest);
}
if (hashDigest.Equals(downDigest, StringComparison.OrdinalIgnoreCase)) // Success
{
FileHelper.FileReplaceEx(tempPath, destFile);
logs.Add(new LogInfo(LogState.Success, $"[{destFile}] downloaded from [{url}] and verified "));
}
else
{
statusCode = 1; // 1 means hash mismatch
logs.Add(new LogInfo(LogState.Error, $"Downloaded file from [{url}] was corrupted"));
}
}
else
{ // Failure -> Log error message
LogState state = info.NoErrFlag ? LogState.Warning : LogState.Error;
logs.Add(new LogInfo(state, $"Error occured while downloading [{url}]"));
logs.Add(new LogInfo(LogState.Info, report.ErrorMsg));
if (statusCode == 0)
{
logs.Add(new LogInfo(LogState.Info, "Request failed, no response received."));
}
else
{
logs.Add(new LogInfo(LogState.Info, $"Response returned HTTP Status Code [{statusCode}]"));
}
}
// PEBakery extension -> Report exit code via #r
if (!s.CompatDisableExtendedSectionParams)
{
s.ReturnValue = statusCode.ToString();
if (statusCode < 100)
{
logs.Add(new LogInfo(LogState.Success, $"Returned [{statusCode}] into [#r]"));
}
else
{
logs.Add(new LogInfo(LogState.Success, $"Returned HTTP status code [{statusCode}] to [#r]"));
}
}
}
}
finally
{
s.MainViewModel.ResetBuildCommandProgress();
}
return(logs);
}
public static List <LogInfo> WebGet(EngineState s, CodeCommand cmd)
{ // WebGet,<URL>,<DestPath>,[HashType],[HashDigest]
List <LogInfo> logs = new List <LogInfo>();
Debug.Assert(cmd.Info.GetType() == typeof(CodeInfo_WebGet));
CodeInfo_WebGet info = cmd.Info as CodeInfo_WebGet;
string url = StringEscaper.Preprocess(s, info.URL);
string downloadTo = StringEscaper.Preprocess(s, info.DestPath);
if (StringEscaper.PathSecurityCheck(downloadTo, out string errorMsg) == false)
{
logs.Add(new LogInfo(LogState.Error, errorMsg));
return(logs);
}
Uri uri = new Uri(url);
string destPath;
if (Directory.Exists(downloadTo)) // downloadTo is dir
{
destPath = Path.Combine(downloadTo, Path.GetFileName(uri.LocalPath));
}
else // downloadTo is file
{
if (File.Exists(downloadTo))
{
if (cmd.Type == CodeType.WebGetIfNotExist)
{
logs.Add(new LogInfo(LogState.Ignore, $"File [{downloadTo}] already exists"));
return(logs);
}
else
{
logs.Add(new LogInfo(LogState.Ignore, $"File [{downloadTo}] will be overwritten"));
}
}
else
{
Directory.CreateDirectory(Path.GetDirectoryName(downloadTo));
}
destPath = downloadTo;
}
if (info.HashType != null && info.HashDigest != null)
{ // Calculate Hash After Downloading
string tempPath = Path.GetTempFileName();
using (WebClient client = new WebClient())
{
client.DownloadFile(url, tempPath);
}
string hashTypeStr = StringEscaper.Preprocess(s, info.HashType);
string hashDigest = StringEscaper.Preprocess(s, info.HashDigest);
HashType hashType;
if (hashTypeStr.Equals("MD5", StringComparison.OrdinalIgnoreCase))
{
hashType = HashType.MD5;
}
else if (hashTypeStr.Equals("SHA1", StringComparison.OrdinalIgnoreCase))
{
hashType = HashType.SHA1;
}
else if (hashTypeStr.Equals("SHA256", StringComparison.OrdinalIgnoreCase))
{
hashType = HashType.SHA256;
}
else if (hashTypeStr.Equals("SHA384", StringComparison.OrdinalIgnoreCase))
{
hashType = HashType.SHA384;
}
else if (hashTypeStr.Equals("SHA512", StringComparison.OrdinalIgnoreCase))
{
hashType = HashType.SHA512;
}
else
{
throw new ExecuteException($"Invalid hash type [{hashTypeStr}]");
}
int byteLen = 0;
switch (hashType)
{
case HashType.MD5:
byteLen = 32;
break;
case HashType.SHA1:
byteLen = 40;
break;
case HashType.SHA256:
byteLen = 64;
break;
case HashType.SHA384:
byteLen = 96;
break;
case HashType.SHA512:
byteLen = 128;
break;
}
if (hashDigest.Length != byteLen)
{
throw new ExecuteException($"Hash digest [{hashDigest}] is not [{hashTypeStr}]");
}
string downDigest;
using (FileStream fs = new FileStream(tempPath, FileMode.Open, FileAccess.Read))
{
downDigest = HashHelper.CalcHashString(hashType, fs);
}
if (hashDigest.Equals(downDigest, StringComparison.OrdinalIgnoreCase)) // Success
{
File.Move(tempPath, destPath);
logs.Add(new LogInfo(LogState.Success, $"[{url}] downloaded to [{downloadTo}] with integerity checked."));
}
else
{
logs.Add(new LogInfo(LogState.Error, $"Downloaded [{url}], but it was corrupted"));
}
}
else
{ // No Hash
using (WebClient client = new WebClient())
{
client.DownloadFile(url, destPath);
}
logs.Add(new LogInfo(LogState.Success, $"[{url}] downloaded to [{downloadTo}]"));
}
return(logs);
}
public static List <LogInfo> WebGet(EngineState s, CodeCommand cmd)
{ // WebGet,<URL>,<DestPath>,[HashType],[HashDigest]
List <LogInfo> logs = new List <LogInfo>();
CodeInfo_WebGet info = cmd.Info.Cast <CodeInfo_WebGet>();
string url = StringEscaper.Preprocess(s, info.URL);
string destPath = StringEscaper.Preprocess(s, info.DestPath);
const string destVar = @"%StatusCode%";
// Check PathSecurity in destPath
if (!StringEscaper.PathSecurityCheck(destPath, out string pathErrorMsg))
{
return(LogInfo.LogErrorMessage(logs, pathErrorMsg));
}
Uri uri = new Uri(url);
string destFile;
if (Directory.Exists(destPath))
{
destFile = Path.Combine(destPath, Path.GetFileName(uri.LocalPath));
}
else // downloadTo is file
{
if (File.Exists(destPath))
{
if (cmd.Type == CodeType.WebGetIfNotExist)
{
logs.Add(new LogInfo(LogState.Ignore, $"File [{destPath}] already exists"));
return(logs);
}
logs.Add(new LogInfo(LogState.Overwrite, $"File [{destPath}] will be overwritten"));
}
else
{
Directory.CreateDirectory(FileHelper.GetDirNameEx(destPath));
}
destFile = destPath;
}
s.MainViewModel.SetBuildCommandProgress("WebGet Progress");
try
{
if (info.HashType == HashHelper.HashType.None)
{ // Standard WebGet
(bool result, int statusCode, string errorMsg) = DownloadFile(s, url, destFile);
if (result)
{
logs.Add(new LogInfo(LogState.Success, $"[{destPath}] downloaded from [{url}]"));
}
else
{
LogState state = info.NoErrFlag ? LogState.Warning : LogState.Error;
logs.Add(new LogInfo(state, $"Error occured while downloading [{url}]"));
logs.Add(new LogInfo(LogState.Info, errorMsg));
if (statusCode == 0)
{
logs.Add(new LogInfo(LogState.Info, "Request failed, no response received."));
}
else
{
logs.Add(new LogInfo(LogState.Info, $"Response returned HTTP Status Code [{statusCode}]"));
}
}
List <LogInfo> varLogs = Variables.SetVariable(s, destVar, statusCode.ToString());
logs.AddRange(varLogs);
}
else
{ // Validate downloaded file with hash
Debug.Assert(info.HashDigest != null);
string tempPath = Path.GetTempFileName();
(bool result, int statusCode, string errorMsg) = DownloadFile(s, url, tempPath);
if (result)
{ // Success -> Check hash
string hashDigest = StringEscaper.Preprocess(s, info.HashDigest);
if (hashDigest.Length != 2 * HashHelper.GetHashByteLen(info.HashType))
{
return(LogInfo.LogErrorMessage(logs, $"Hash digest [{hashDigest}] is not [{info.HashType}]"));
}
string downDigest;
using (FileStream fs = new FileStream(tempPath, FileMode.Open, FileAccess.Read))
{
byte[] digest = HashHelper.GetHash(info.HashType, fs);
downDigest = StringHelper.ToHexStr(digest);
}
if (hashDigest.Equals(downDigest, StringComparison.OrdinalIgnoreCase)) // Success
{
File.Move(tempPath, destFile);
logs.Add(new LogInfo(LogState.Success, $"[{destPath}] downloaded from [{url}] and verified "));
}
else
{
statusCode = 1;
logs.Add(new LogInfo(LogState.Error, $"Downloaded file from [{url}] was corrupted"));
}
}
else
{ // Failure -> Log error message
LogState state = info.NoErrFlag ? LogState.Warning : LogState.Error;
logs.Add(new LogInfo(state, $"Error occured while downloading [{url}]"));
logs.Add(new LogInfo(LogState.Info, errorMsg));
if (statusCode == 0)
{
logs.Add(new LogInfo(LogState.Info, "Request failed, no response received."));
}
else
{
logs.Add(new LogInfo(LogState.Info, $"Response returned HTTP Status Code [{statusCode}]"));
}
}
List <LogInfo> varLogs = Variables.SetVariable(s, destVar, statusCode.ToString());
logs.AddRange(varLogs);
}
}
finally
{
s.MainViewModel.ResetBuildCommandProgress();
}
return(logs);
}
