public static WebRequest CreateRequest(Uri location, ApplicationJob job, CookieContainer cookies)
{
WebRequest req = WebRequest.CreateDefault(WebClient.FixNoProtocolUri(location));
req.Timeout = Convert.ToInt32(Settings.GetValue("ConnectionTimeout", 10)) * 1000;
// 10 seconds by default
HttpWebRequest httpRequest = req as HttpWebRequest;
if (httpRequest != null)
{
httpRequest.Accept = "*/*";
// Store cookies for future requests. Some sites
// check for previously stored cookies before allowing to download.
if (httpRequest.CookieContainer == null)
{
httpRequest.CookieContainer = cookies;
}
else
{
httpRequest.CookieContainer.Add(cookies.GetCookies(httpRequest.RequestUri));
}
// If we have an HTTP request, some sites may require a correct referer
// for the download.
// If there are variables defined (from which most likely the download link
// or version is being extracted), we'll just use the first variable's URL as referer.
// The user still has the option to set a custom referer.
// Note: Some websites don't "like" certain referers
if (!NoAutoReferer.Contains(GetBaseHost(req.RequestUri)))
{
foreach (UrlVariable urlVar in job.Variables.Values)
{
httpRequest.Referer = urlVar.Url;
break;
}
}
if (!string.IsNullOrEmpty(job.HttpReferer))
{
httpRequest.Referer = job.Variables.ReplaceAllInString(job.HttpReferer);
}
LogDialog.Log(job,
"Using referer: " + (string.IsNullOrEmpty(httpRequest.Referer) ? "(none)" : httpRequest.Referer));
httpRequest.UserAgent = string.IsNullOrEmpty(job.UserAgent)
? WebClient.DefaultUserAgent
: job.Variables.ReplaceAllInString(job.UserAgent);
// PAD files may be compressed
httpRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
}
return(req);
}
/// <summary>
/// Executes the actual download from an URL. Does not handle exceptions,
/// but takes care of proper cleanup.
/// </summary>
/// <exception cref="NonBinaryFileException">This exception is thrown, if the resulting file is not of a binary type</exception>
/// <exception cref="TargetPathInvalidException">This exception is thrown, if the resulting target path of an application is not valid</exception>
/// <param name="job">The job to process</param>
/// <param name="urlToRequest">URL from which should be downloaded</param>
/// <returns>true, if a new update has been found and downloaded, false otherwise</returns>
protected Status DoDownload(ApplicationJob job, Uri urlToRequest)
{
// Determine number of segments to create
int segmentCount = Convert.ToInt32(Settings.GetValue("SegmentCount", 1));
job.Variables.ResetDownloadCount();
WebRequest req = KetarinProtocolProvider.CreateRequest(urlToRequest, job, this.m_Cookies);
AddRequestToCancel(req);
using (WebResponse response = WebClient.GetResponse(req))
{
LogDialog.Log(job, "Server source file: " + req.RequestUri.AbsolutePath);
// Occasionally, websites are not available and an error page is encountered
// For the case that the content type is just plain wrong, ignore it if the size is higher than 500KB
HttpWebResponse httpResponse = response as HttpWebResponse;
if (httpResponse != null && response.ContentLength < 500000)
{
if (response.ContentType.StartsWith("text/xml") || response.ContentType.StartsWith("application/xml"))
{
// If an XML file is served, maybe we have a PAD file
ApplicationJob padJob = ApplicationJob.ImportFromPad(httpResponse);
if (padJob != null)
{
job.CachedPadFileVersion = padJob.CachedPadFileVersion;
return(this.DoDownload(job, new Uri(padJob.FixedDownloadUrl)));
}
}
if (response.ContentType.StartsWith("text/html"))
{
bool avoidNonBinary = (bool)Settings.GetValue("AvoidDownloadingNonBinaryFiles", true);
if (httpResponse.StatusCode != HttpStatusCode.OK || avoidNonBinary)
{
throw NonBinaryFileException.Create(response.ContentType, httpResponse.StatusCode);
}
}
}
long fileSize = GetContentLength(response);
if (fileSize == 0)
{
throw new IOException("Source file on server is empty (ContentLength = 0).");
}
string targetFileName = job.GetTargetFile(response, urlToRequest.AbsoluteUri);
LogDialog.Log(job, "Determined target file name: " + targetFileName);
// Only download, if the file size or date has changed
if (!ForceDownload && !job.RequiresDownload(response, targetFileName))
{
// If file already exists (created by user),
// the download is not necessary. We still need to
// set the file name.
// If the file exists, but not at the target location
// (after renaming), do not reset the previous location.
if (File.Exists(targetFileName))
{
job.PreviousLocation = targetFileName;
}
job.Save();
return(Status.NoUpdate);
}
// Skip downloading!
// Installing also requires a forced download
if (!ForceDownload && !m_InstallUpdated && (m_OnlyCheck || (job.CheckForUpdatesOnly && !IgnoreCheckForUpdatesOnly)))
{
LogDialog.Log(job, "Skipped downloading updates");
return(Status.UpdateAvailable);
}
// Execute: Default pre-update command
string defaultPreCommand = Settings.GetValue("PreUpdateCommand", "") as string;
// For starting external download managers: {preupdate-url}
defaultPreCommand = UrlVariable.Replace(defaultPreCommand, "preupdate-url", urlToRequest.ToString(), job);
ScriptType defaultPreCommandType = Command.ConvertToScriptType(Settings.GetValue("PreUpdateCommandType", ScriptType.Batch.ToString()) as string);
int exitCode = new Command(defaultPreCommand, defaultPreCommandType).Execute(job, targetFileName);
if (exitCode == 1)
{
LogDialog.Log(job, "Default pre-update command returned '1', download aborted");
throw new CommandErrorException();
}
else if (exitCode == 2)
{
LogDialog.Log(job, "Default pre-update command returned '2', download skipped");
return(Status.UpdateAvailable);
}
// Execute: Application pre-update command
exitCode = new Command(UrlVariable.Replace(job.ExecutePreCommand, "preupdate-url", urlToRequest.ToString(), job), job.ExecutePreCommandType).Execute(job, targetFileName);
if (exitCode == 1)
{
LogDialog.Log(job, "Pre-update command returned '1', download aborted");
throw new CommandErrorException();
}
else if (exitCode == 2)
{
LogDialog.Log(job, "Pre-update command returned '2', download skipped");
return(Status.UpdateAvailable);
}
else if (exitCode == 3)
{
LogDialog.Log(job, "Pre-update command returned '3', external download");
job.LastUpdated = DateTime.Now;
job.Save();
return(Status.UpdateSuccessful);
}
// Read all file contents to a temporary location
string tmpLocation = Path.GetTempFileName();
DateTime lastWriteTime = ApplicationJob.GetLastModified(response);
// Only use segmented downloader with more than one segment.
if (segmentCount > 1)
{
// Response can be closed now, new one will be created.
response.Dispose();
m_Size[job] = fileSize;
Downloader d = new Downloader(new ResourceLocation {
Url = urlToRequest.AbsoluteUri, ProtocolProvider = new KetarinProtocolProvider(job, m_Cookies)
}, null, tmpLocation, segmentCount);
d.Start();
while (d.State < DownloaderState.Ended)
{
if (m_CancelUpdates)
{
d.Pause();
break;
}
this.OnProgressChanged(d.Segments.Sum(x => x.Transfered), fileSize, job);
Thread.Sleep(250);
}
if (d.State == DownloaderState.EndedWithError)
{
throw d.LastError;
}
}
else
{
// Read contents from the web and put into file
using (Stream sourceFile = response.GetResponseStream())
{
using (FileStream targetFile = File.Create(tmpLocation))
{
long byteCount = 0;
int readBytes;
m_Size[job] = fileSize;
// Only create buffer once and re-use.
const int bufferSize = 1024 * 1024;
byte[] buffer = new byte[bufferSize];
do
{
if (m_CancelUpdates)
{
break;
}
// Some adjustment for SCP download: Read only up to the max known bytes
int maxRead = (fileSize > 0) ? (int)Math.Min(fileSize - byteCount, bufferSize) : bufferSize;
if (maxRead == 0)
{
break;
}
readBytes = sourceFile.Read(buffer, 0, maxRead);
if (readBytes > 0)
{
targetFile.Write(buffer, 0, readBytes);
}
byteCount += readBytes;
this.OnProgressChanged(byteCount, fileSize, job);
} while (readBytes > 0);
}
}
}
if (m_CancelUpdates)
{
m_Progress[job] = 0;
OnStatusChanged(job);
return(Status.Failure);
}
// If each version has a different file name (version number),
// we might only want to keep one of them. Also, we might
// want to free some space on the target location.
if (job.DeletePreviousFile)
{
PathEx.TryDeleteFiles(job.PreviousLocation);
}
try
{
File.SetLastWriteTime(tmpLocation, lastWriteTime);
}
catch (ArgumentException)
{
// Invalid file date. Ignore and just use DateTime.Now
}
// File downloaded. Now let's check if the hash value is valid or abort otherwise!
if (!string.IsNullOrEmpty(job.HashVariable) && job.HashType != HashType.None)
{
string varName = job.HashVariable.Trim('{', '}');
string expectedHash = job.Variables.ReplaceAllInString("{" + varName + "}").Trim();
// Compare online hash with actual current hash.
if (!string.IsNullOrEmpty(expectedHash))
{
string currentHash = job.GetFileHash(tmpLocation);
if (string.Compare(expectedHash, currentHash, StringComparison.OrdinalIgnoreCase) != 0)
{
LogDialog.Log(job, string.Format("File downloaded, but hash of downloaded file {0} does not match the expected hash {1}.", currentHash, expectedHash));
File.Delete(tmpLocation);
throw new IOException("Hash verification failed.");
}
}
}
try
{
FileInfo downloadedFileInfo = new FileInfo(tmpLocation);
job.LastFileSize = downloadedFileInfo.Length;
job.LastFileDate = downloadedFileInfo.LastWriteTime;
}
catch (Exception ex)
{
LogDialog.Log(job, ex);
}
try
{
// Before copying, we might have to create the directory
Directory.CreateDirectory(Path.GetDirectoryName(targetFileName));
// Copying might fail if variables have been replaced with bad values.
// However, we cannot rely on functions to clean up the path, since they
// might actually parse the path incorrectly and return an even worse path.
File.Copy(tmpLocation, targetFileName, true);
}
catch (ArgumentException)
{
throw new TargetPathInvalidException(targetFileName);
}
catch (NotSupportedException)
{
throw new TargetPathInvalidException(targetFileName);
}
File.Delete(tmpLocation);
// At this point, the update is complete
job.LastUpdated = DateTime.Now;
job.PreviousLocation = targetFileName;
}
job.Save();
job.ExecutePostUpdateCommands();
return(Status.UpdateSuccessful);
}
/// <summary>
/// Replaces this variable within a given string.
/// </summary>
/// <param name="onlyCached">If true, no new content will be downloaded and only chached content will be used.</param>
/// <param name="fileDate">Current file date, when downloading the modification date of the file being downloaded</param>
public virtual string ReplaceInString(string value, DateTime fileDate, bool onlyCached)
{
if (!IsVariableUsedInString(m_Name, value)) return value;
// Global variable only has static content
if (onlyCached)
{
// We don't want to spam the log with chached only screen updates
if (!onlyCached) LogDialog.Log(this, value, m_CachedContent);
return (m_CachedContent == null) ? value : Replace(value, m_CachedContent);
}
// Ignore missing URLs etc.
if (IsEmpty) return value;
// Using textual content?
if (m_VariableType == Type.Textual)
{
m_CachedContent = GetExpandedTextualContent(fileDate);
LogDialog.Log(this, value, m_CachedContent);
return Replace(value, m_CachedContent);
}
string page = string.Empty;
// Get the content we need to put in
string userAgent = (this.Parent == null) ? null : this.Parent.Parent.UserAgent;
using (WebClient client = new WebClient(userAgent))
{
try
{
string url = ExpandedUrl;
client.SetPostData(this);
page = client.DownloadString(url);
}
catch (ArgumentException)
{
throw new UriFormatException("The URL '" + Url + "' of variable '" + m_Name + "' is not valid.");
}
m_DownloadCount++;
}
// Normalise line-breaks
page = page.Replace("\r\n", "\n");
page = page.Replace("\r", "\n");
// Using a regular expression?
if (m_VariableType == Type.RegularExpression)
{
Regex regex = CreateRegex();
if (regex == null) return value;
Match match = regex.Match(page);
if (match.Success)
{
if (match.Groups.Count == 1)
{
m_CachedContent = match.Value;
LogDialog.Log(this, value, m_CachedContent);
return Replace(value, match.Value);
}
else if (match.Groups.Count >= 2)
{
// Use first group (except index 0, which is complete match) with a match
for (int i = 1; i < match.Groups.Count; i++)
{
if (match.Groups[i].Success)
{
m_CachedContent = match.Groups[i].Value;
LogDialog.Log(this, value, m_CachedContent);
return Replace(value, match.Groups[i].Value);
}
}
// No group matches, use complete match
m_CachedContent = match.Groups[0].Value;
LogDialog.Log(this, value, m_CachedContent);
return Replace(value, match.Groups[0].Value);
}
}
else
{
// No regex match should yield an empty result
return Replace(value, string.Empty);
}
}
// Use whole page if either start or end is missing
if (string.IsNullOrEmpty(m_StartText) || string.IsNullOrEmpty(m_EndText))
{
m_CachedContent = page;
LogDialog.Log(this, value, m_CachedContent);
return Replace(value, page);
}
int startPos = page.IndexOf(m_StartText);
if (startPos < 0) return value;
int endOfStart = startPos + m_StartText.Length;
int endPos = page.IndexOf(m_EndText, endOfStart);
if (endPos < 0) return value;
string result = page.Substring(endOfStart, endPos - endOfStart);
m_CachedContent = result;
LogDialog.Log(this, value, m_CachedContent);
value = Replace(value, result);
return value;
}