public static WebRequest CreateRequest(Uri location, ApplicationJob job, CookieContainer cookies)
        {
            WebRequest req = WebRequest.CreateDefault(WebClient.FixNoProtocolUri(location));

            req.Timeout = Convert.ToInt32(Settings.GetValue("ConnectionTimeout", 10)) * 1000;
            // 10 seconds by default

            HttpWebRequest httpRequest = req as HttpWebRequest;

            if (httpRequest != null)
            {
                httpRequest.Accept = "*/*";

                // Store cookies for future requests. Some sites
                // check for previously stored cookies before allowing to download.
                if (httpRequest.CookieContainer == null)
                {
                    httpRequest.CookieContainer = cookies;
                }
                else
                {
                    httpRequest.CookieContainer.Add(cookies.GetCookies(httpRequest.RequestUri));
                }

                // If we have an HTTP request, some sites may require a correct referer
                // for the download.
                // If there are variables defined (from which most likely the download link
                // or version is being extracted), we'll just use the first variable's URL as referer.
                // The user still has the option to set a custom referer.
                // Note: Some websites don't "like" certain referers
                if (!NoAutoReferer.Contains(GetBaseHost(req.RequestUri)))
                {
                    foreach (UrlVariable urlVar in job.Variables.Values)
                    {
                        httpRequest.Referer = urlVar.Url;
                        break;
                    }
                }

                if (!string.IsNullOrEmpty(job.HttpReferer))
                {
                    httpRequest.Referer = job.Variables.ReplaceAllInString(job.HttpReferer);
                }

                LogDialog.Log(job,
                              "Using referer: " + (string.IsNullOrEmpty(httpRequest.Referer) ? "(none)" : httpRequest.Referer));
                httpRequest.UserAgent = string.IsNullOrEmpty(job.UserAgent)
                    ? WebClient.DefaultUserAgent
                    : job.Variables.ReplaceAllInString(job.UserAgent);

                // PAD files may be compressed
                httpRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
            }

            return(req);
        }
예제 #2
0
        /// <summary>
        /// Executes the actual download from an URL. Does not handle exceptions,
        /// but takes care of proper cleanup.
        /// </summary>
        /// <exception cref="NonBinaryFileException">This exception is thrown, if the resulting file is not of a binary type</exception>
        /// <exception cref="TargetPathInvalidException">This exception is thrown, if the resulting target path of an application is not valid</exception>
        /// <param name="job">The job to process</param>
        /// <param name="urlToRequest">URL from which should be downloaded</param>
        /// <returns>true, if a new update has been found and downloaded, false otherwise</returns>
        protected Status DoDownload(ApplicationJob job, Uri urlToRequest)
        {
            // Determine number of segments to create
            int segmentCount = Convert.ToInt32(Settings.GetValue("SegmentCount", 1));

            job.Variables.ResetDownloadCount();

            WebRequest req = KetarinProtocolProvider.CreateRequest(urlToRequest, job, this.m_Cookies);

            AddRequestToCancel(req);

            using (WebResponse response = WebClient.GetResponse(req))
            {
                LogDialog.Log(job, "Server source file: " + req.RequestUri.AbsolutePath);

                // Occasionally, websites are not available and an error page is encountered
                // For the case that the content type is just plain wrong, ignore it if the size is higher than 500KB
                HttpWebResponse httpResponse = response as HttpWebResponse;
                if (httpResponse != null && response.ContentLength < 500000)
                {
                    if (response.ContentType.StartsWith("text/xml") || response.ContentType.StartsWith("application/xml"))
                    {
                        // If an XML file is served, maybe we have a PAD file
                        ApplicationJob padJob = ApplicationJob.ImportFromPad(httpResponse);
                        if (padJob != null)
                        {
                            job.CachedPadFileVersion = padJob.CachedPadFileVersion;
                            return(this.DoDownload(job, new Uri(padJob.FixedDownloadUrl)));
                        }
                    }

                    if (response.ContentType.StartsWith("text/html"))
                    {
                        bool avoidNonBinary = (bool)Settings.GetValue("AvoidDownloadingNonBinaryFiles", true);
                        if (httpResponse.StatusCode != HttpStatusCode.OK || avoidNonBinary)
                        {
                            throw NonBinaryFileException.Create(response.ContentType, httpResponse.StatusCode);
                        }
                    }
                }

                long fileSize = GetContentLength(response);
                if (fileSize == 0)
                {
                    throw new IOException("Source file on server is empty (ContentLength = 0).");
                }

                string targetFileName = job.GetTargetFile(response, urlToRequest.AbsoluteUri);

                LogDialog.Log(job, "Determined target file name: " + targetFileName);

                // Only download, if the file size or date has changed
                if (!ForceDownload && !job.RequiresDownload(response, targetFileName))
                {
                    // If file already exists (created by user),
                    // the download is not necessary. We still need to
                    // set the file name.
                    // If the file exists, but not at the target location
                    // (after renaming), do not reset the previous location.
                    if (File.Exists(targetFileName))
                    {
                        job.PreviousLocation = targetFileName;
                    }
                    job.Save();
                    return(Status.NoUpdate);
                }

                // Skip downloading!
                // Installing also requires a forced download
                if (!ForceDownload && !m_InstallUpdated && (m_OnlyCheck || (job.CheckForUpdatesOnly && !IgnoreCheckForUpdatesOnly)))
                {
                    LogDialog.Log(job, "Skipped downloading updates");
                    return(Status.UpdateAvailable);
                }

                // Execute: Default pre-update command
                string defaultPreCommand = Settings.GetValue("PreUpdateCommand", "") as string;
                // For starting external download managers: {preupdate-url}
                defaultPreCommand = UrlVariable.Replace(defaultPreCommand, "preupdate-url", urlToRequest.ToString(), job);
                ScriptType defaultPreCommandType = Command.ConvertToScriptType(Settings.GetValue("PreUpdateCommandType", ScriptType.Batch.ToString()) as string);

                int exitCode = new Command(defaultPreCommand, defaultPreCommandType).Execute(job, targetFileName);
                if (exitCode == 1)
                {
                    LogDialog.Log(job, "Default pre-update command returned '1', download aborted");
                    throw new CommandErrorException();
                }
                else if (exitCode == 2)
                {
                    LogDialog.Log(job, "Default pre-update command returned '2', download skipped");
                    return(Status.UpdateAvailable);
                }

                // Execute: Application pre-update command
                exitCode = new Command(UrlVariable.Replace(job.ExecutePreCommand, "preupdate-url", urlToRequest.ToString(), job), job.ExecutePreCommandType).Execute(job, targetFileName);
                if (exitCode == 1)
                {
                    LogDialog.Log(job, "Pre-update command returned '1', download aborted");
                    throw new CommandErrorException();
                }
                else if (exitCode == 2)
                {
                    LogDialog.Log(job, "Pre-update command returned '2', download skipped");
                    return(Status.UpdateAvailable);
                }
                else if (exitCode == 3)
                {
                    LogDialog.Log(job, "Pre-update command returned '3', external download");
                    job.LastUpdated = DateTime.Now;
                    job.Save();
                    return(Status.UpdateSuccessful);
                }

                // Read all file contents to a temporary location
                string   tmpLocation   = Path.GetTempFileName();
                DateTime lastWriteTime = ApplicationJob.GetLastModified(response);

                // Only use segmented downloader with more than one segment.
                if (segmentCount > 1)
                {
                    // Response can be closed now, new one will be created.
                    response.Dispose();

                    m_Size[job] = fileSize;

                    Downloader d = new Downloader(new ResourceLocation {
                        Url = urlToRequest.AbsoluteUri, ProtocolProvider = new KetarinProtocolProvider(job, m_Cookies)
                    }, null, tmpLocation, segmentCount);
                    d.Start();

                    while (d.State < DownloaderState.Ended)
                    {
                        if (m_CancelUpdates)
                        {
                            d.Pause();
                            break;
                        }

                        this.OnProgressChanged(d.Segments.Sum(x => x.Transfered), fileSize, job);
                        Thread.Sleep(250);
                    }

                    if (d.State == DownloaderState.EndedWithError)
                    {
                        throw d.LastError;
                    }
                }
                else
                {
                    // Read contents from the web and put into file
                    using (Stream sourceFile = response.GetResponseStream())
                    {
                        using (FileStream targetFile = File.Create(tmpLocation))
                        {
                            long byteCount = 0;
                            int  readBytes;
                            m_Size[job] = fileSize;

                            // Only create buffer once and re-use.
                            const int bufferSize = 1024 * 1024;
                            byte[]    buffer     = new byte[bufferSize];

                            do
                            {
                                if (m_CancelUpdates)
                                {
                                    break;
                                }

                                // Some adjustment for SCP download: Read only up to the max known bytes
                                int maxRead = (fileSize > 0) ? (int)Math.Min(fileSize - byteCount, bufferSize) : bufferSize;
                                if (maxRead == 0)
                                {
                                    break;
                                }

                                readBytes = sourceFile.Read(buffer, 0, maxRead);
                                if (readBytes > 0)
                                {
                                    targetFile.Write(buffer, 0, readBytes);
                                }
                                byteCount += readBytes;

                                this.OnProgressChanged(byteCount, fileSize, job);
                            } while (readBytes > 0);
                        }
                    }
                }

                if (m_CancelUpdates)
                {
                    m_Progress[job] = 0;
                    OnStatusChanged(job);
                    return(Status.Failure);
                }

                // If each version has a different file name (version number),
                // we might only want to keep one of them. Also, we might
                // want to free some space on the target location.
                if (job.DeletePreviousFile)
                {
                    PathEx.TryDeleteFiles(job.PreviousLocation);
                }

                try
                {
                    File.SetLastWriteTime(tmpLocation, lastWriteTime);
                }
                catch (ArgumentException)
                {
                    // Invalid file date. Ignore and just use DateTime.Now
                }

                // File downloaded. Now let's check if the hash value is valid or abort otherwise!
                if (!string.IsNullOrEmpty(job.HashVariable) && job.HashType != HashType.None)
                {
                    string varName      = job.HashVariable.Trim('{', '}');
                    string expectedHash = job.Variables.ReplaceAllInString("{" + varName + "}").Trim();

                    // Compare online hash with actual current hash.
                    if (!string.IsNullOrEmpty(expectedHash))
                    {
                        string currentHash = job.GetFileHash(tmpLocation);
                        if (string.Compare(expectedHash, currentHash, StringComparison.OrdinalIgnoreCase) != 0)
                        {
                            LogDialog.Log(job, string.Format("File downloaded, but hash of downloaded file {0} does not match the expected hash {1}.", currentHash, expectedHash));
                            File.Delete(tmpLocation);
                            throw new IOException("Hash verification failed.");
                        }
                    }
                }

                try
                {
                    FileInfo downloadedFileInfo = new FileInfo(tmpLocation);
                    job.LastFileSize = downloadedFileInfo.Length;
                    job.LastFileDate = downloadedFileInfo.LastWriteTime;
                }
                catch (Exception ex)
                {
                    LogDialog.Log(job, ex);
                }

                try
                {
                    // Before copying, we might have to create the directory
                    Directory.CreateDirectory(Path.GetDirectoryName(targetFileName));

                    // Copying might fail if variables have been replaced with bad values.
                    // However, we cannot rely on functions to clean up the path, since they
                    // might actually parse the path incorrectly and return an even worse path.
                    File.Copy(tmpLocation, targetFileName, true);
                }
                catch (ArgumentException)
                {
                    throw new TargetPathInvalidException(targetFileName);
                }
                catch (NotSupportedException)
                {
                    throw new TargetPathInvalidException(targetFileName);
                }

                File.Delete(tmpLocation);

                // At this point, the update is complete
                job.LastUpdated      = DateTime.Now;
                job.PreviousLocation = targetFileName;
            }

            job.Save();

            job.ExecutePostUpdateCommands();

            return(Status.UpdateSuccessful);
        }
예제 #3
0
        /// <summary>
        /// Replaces this variable within a given string.
        /// </summary>
        /// <param name="onlyCached">If true, no new content will be downloaded and only chached content will be used.</param>
        /// <param name="fileDate">Current file date, when downloading the modification date of the file being downloaded</param>
        public virtual string ReplaceInString(string value, DateTime fileDate, bool onlyCached)
        {
            if (!IsVariableUsedInString(m_Name, value)) return value;

            // Global variable only has static content
            if (onlyCached)
            {
                // We don't want to spam the log with chached only screen updates
                if (!onlyCached) LogDialog.Log(this, value, m_CachedContent);
                return (m_CachedContent == null) ? value : Replace(value, m_CachedContent);
            }

            // Ignore missing URLs etc.
            if (IsEmpty) return value;

            // Using textual content?
            if (m_VariableType == Type.Textual)
            {
                m_CachedContent = GetExpandedTextualContent(fileDate);
                LogDialog.Log(this, value, m_CachedContent);
                return Replace(value, m_CachedContent);
            }

            string page = string.Empty;
            // Get the content we need to put in
            string userAgent = (this.Parent == null) ? null : this.Parent.Parent.UserAgent;
            using (WebClient client = new WebClient(userAgent))
            {
                try
                {
                    string url = ExpandedUrl;
                    client.SetPostData(this);
                    page = client.DownloadString(url);
                }
                catch (ArgumentException)
                {
                    throw new UriFormatException("The URL '" + Url + "' of variable '" + m_Name + "' is not valid.");
                }
                m_DownloadCount++;
            }

            // Normalise line-breaks
            page = page.Replace("\r\n", "\n");
            page = page.Replace("\r", "\n");

            // Using a regular expression?
            if (m_VariableType == Type.RegularExpression)
            {
                Regex regex = CreateRegex();
                if (regex == null) return value;

                Match match = regex.Match(page);
                if (match.Success)
                {
                    if (match.Groups.Count == 1)
                    {
                        m_CachedContent = match.Value;
                        LogDialog.Log(this, value, m_CachedContent);
                        return Replace(value, match.Value);
                    }
                    else if (match.Groups.Count >= 2)
                    {
                        // Use first group (except index 0, which is complete match) with a match
                        for (int i = 1; i < match.Groups.Count; i++)
                        {
                            if (match.Groups[i].Success)
                            {
                                m_CachedContent = match.Groups[i].Value;
                                LogDialog.Log(this, value, m_CachedContent);
                                return Replace(value, match.Groups[i].Value);
                            }
                        }

                        // No group matches, use complete match
                        m_CachedContent = match.Groups[0].Value;
                        LogDialog.Log(this, value, m_CachedContent);
                        return Replace(value, match.Groups[0].Value);
                    }
                }
                else
                {
                    // No regex match should yield an empty result
                    return Replace(value, string.Empty);
                }
            }

            // Use whole page if either start or end is missing
            if (string.IsNullOrEmpty(m_StartText) || string.IsNullOrEmpty(m_EndText))
            {
                m_CachedContent = page;
                LogDialog.Log(this, value, m_CachedContent);
                return Replace(value, page);
            }

            int startPos = page.IndexOf(m_StartText);
            if (startPos < 0) return value;

            int endOfStart = startPos + m_StartText.Length;

            int endPos = page.IndexOf(m_EndText, endOfStart);
            if (endPos < 0) return value;

            string result = page.Substring(endOfStart, endPos - endOfStart);

            m_CachedContent = result;
            LogDialog.Log(this, value, m_CachedContent);
            value = Replace(value, result);

            return value;
        }