Ejemplo n.º 1
0
        public void Get(string remotename, System.IO.Stream stream)
        {
            HttpWebRequest req = CreateRequest("/" + remotename, "");

            req.Method = "GET";

            Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
            using (WebResponse resp = areq.GetResponse())
                using (System.IO.Stream s = areq.GetResponseStream())
                    using (var mds = new Utility.MD5CalculatingStream(s))
                    {
                        string md5Hash = resp.Headers["ETag"];
                        Utility.Utility.CopyStream(mds, stream, true, m_copybuffer);

                        if (mds.GetFinalHashString().ToLower() != md5Hash.ToLower())
                        {
                            throw new Exception(Strings.CloudFiles.ETagVerificationError);
                        }
                    }
        }
Ejemplo n.º 2
0
        public void Get(string remotename, System.IO.Stream stream)
        {
            var req = CreateRequest("/" + remotename, "");

            req.Method = "GET";

            var areq = new Utility.AsyncHttpRequest(req);

            using (var resp = areq.GetResponse())
                using (var s = areq.GetResponseStream())
                    using (var mds = new Utility.MD5CalculatingStream(s))
                    {
                        string md5Hash = resp.Headers["ETag"];
                        Utility.Utility.CopyStream(mds, stream, true, m_copybuffer);

                        if (!String.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase))
                        {
                            throw new Exception(Strings.CloudFiles.ETagVerificationError);
                        }
                    }
        }
Ejemplo n.º 3
0
        public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
        {
            // Some challenges with uploading to Jottacloud:
            // - Jottacloud supports use of a custom header where we can tell the server the MD5 hash of the file
            //   we are uploading, and then it will verify the content of our request against it. But the HTTP
            //   status code we get back indicates success even if there is a mismatch, so we must dig into the
            //   XML response to see if we were able to correctly upload the new content or not. Another issue
            //   is that if the stream is not seek-able we have a challenge pre-calculating MD5 hash on it before
            //   writing it out on the HTTP request stream. And even if the stream is seek-able it may be throttled.
            //   One way to avoid using the throttled stream for calculating the MD5 is to try to get the
            //   underlying stream from the "m_basestream" field, with fall-back to a temporary file.
            // - We can instead chose to upload the data without setting the MD5 hash header. The server will
            //   calculate the MD5 on its side and return it in the response back to use. We can then compare it
            //   with the MD5 hash of the stream (using a MD5CalculatingStream), and if there is a mismatch we can
            //   request the server to remove the file again and throw an exception. But there is a requirement that
            //   we specify the file size in a custom header. And if the stream is not seek-able we are not able
            //   to use stream.Length, so we are back at square one.
            Duplicati.Library.Utility.TempFile tmpFile = null;
            var baseStream = stream;

            while (baseStream is Duplicati.Library.Utility.OverrideableStream)
            {
                baseStream = typeof(Duplicati.Library.Utility.OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic).GetValue(baseStream) as System.IO.Stream;
            }
            if (baseStream == null)
            {
                throw new Exception(string.Format("Unable to unwrap stream from: {0}", stream.GetType()));
            }
            string md5Hash;

            if (baseStream.CanSeek)
            {
                var originalPosition = baseStream.Position;
                using (var md5 = System.Security.Cryptography.MD5.Create())
                    md5Hash = Library.Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream));
                baseStream.Position = originalPosition;
            }
            else
            {
                // No seeking possible, use a temp file
                tmpFile = new Duplicati.Library.Utility.TempFile();
                using (var os = System.IO.File.OpenWrite(tmpFile))
                    using (var md5 = new Utility.MD5CalculatingStream(baseStream))
                    {
                        await Utility.Utility.CopyStreamAsync(md5, os, true, cancelToken, m_copybuffer);

                        md5Hash = md5.GetFinalHashString();
                    }
                stream = System.IO.File.OpenRead(tmpFile);
            }
            try
            {
                // Create request, with query parameter, and a few custom headers.
                // NB: If we wanted to we could send the same POST request as below but without the file contents
                // and with "cphash=[md5Hash]" as the only query parameter. Then we will get an HTTP 200 (OK) response
                // if an identical file already exists, and we can skip uploading the new file. We will get
                // HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which
                // case we must send a new request to upload the new content.
                var fileSize = stream.Length;
                var req      = CreateRequest(System.Net.WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true);
                req.Headers.Add("JMd5", md5Hash);              // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch.
                req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something  be the total size of the original file!
                // File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps,
                // and then we can just omit the JCreated and JModified and let the server automatically set the current time.
                //req.Headers.Add("JCreated", timeCreated);
                //req.Headers.Add("JModified", timeModified);
                req.ContentType   = "application/octet-stream";
                req.ContentLength = fileSize;

                // Write post data request
                var areq = new Utility.AsyncHttpRequest(req);
                using (var rs = areq.GetRequestStream())
                    await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken, m_copybuffer);
                // Send request, and check response
                using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
                {
                    if (resp.StatusCode != System.Net.HttpStatusCode.Created)
                    {
                        throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, null, System.Net.WebExceptionStatus.ProtocolError, resp);
                    }

                    // Request seems to be successful, but we must verify the response XML content to be sure that the file
                    // was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if
                    // there was mismatch, and it will verify the JMd5 header and mark the file as corrupt if there was a hash
                    // mismatch. The returned XML contains a file element, and if upload was error free it contains a single
                    // child element "currentRevision", which has a "state" child element with the string "COMPLETED".
                    // If there was a problem we should have a "latestRevision" child element, and this will have state with
                    // value "INCOMPLETE" or "CORRUPT". If the file was new or had no previous complete versions the latestRevision
                    // will be the only child, but if not there may also be a "currentRevision" representing the previous
                    // complete version - and then we need to detect the case where our upload failed but there was an existing
                    // complete version!
                    using (var rs = areq.GetResponseStream())
                    {
                        var doc = new System.Xml.XmlDocument();
                        try { doc.Load(rs); }
                        catch (System.Xml.XmlException)
                        {
                            throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError);
                        }
                        bool uploadCompletedSuccessfully = false;
                        var  xFile = doc["file"];
                        if (xFile != null)
                        {
                            var xRevState = xFile.SelectSingleNode("latestRevision");
                            if (xRevState == null)
                            {
                                xRevState = xFile.SelectSingleNode("currentRevision/state");
                                if (xRevState != null)
                                {
                                    uploadCompletedSuccessfully = xRevState.InnerText == "COMPLETED"; // Success: There is no "latestRevision", only a "currentRevision" (and it specifies the file is complete, but I think it always will).
                                }
                            }
                        }
                        if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..)
                        {
                            throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError);
                        }
                    }
                }
            }
            finally
            {
                try
                {
                    if (tmpFile != null)
                    {
                        tmpFile.Dispose();
                    }
                }
                catch { }
            }
        }
Ejemplo n.º 4
0
        public void Put(string remotename, System.IO.Stream stream)
        {
            HttpWebRequest req = CreateRequest("/" + remotename, "");

            req.Method      = "PUT";
            req.ContentType = "application/octet-stream";

            try { req.ContentLength = stream.Length; }
            catch { }

            //If we can pre-calculate the MD5 hash before transmission, do so

            /*if (stream.CanSeek)
             * {
             *  System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create();
             *  req.Headers["ETag"] = Core.Utility.ByteArrayAsHexString(md5.ComputeHash(stream)).ToLower();
             *  stream.Seek(0, System.IO.SeekOrigin.Begin);
             *
             *  using (System.IO.Stream s = req.GetRequestStream())
             *      Core.Utility.CopyStream(stream, s);
             *
             *  //Reset the timeout to the default value of 100 seconds to
             *  // avoid blocking the GetResponse() call
             *  req.Timeout = 100000;
             *
             *  //The server handles the eTag verification for us, and gives an error if the hash was a mismatch
             *  using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
             *      if ((int)resp.StatusCode >= 300)
             *          throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
             *
             * }
             * else //Otherwise use a client-side calculation
             */
            //TODO: We cannot use the local MD5 calculation, because that could involve a throttled read,
            // and may invoke various events
            {
                string fileHash = null;

                long streamLen = -1;
                try { streamLen = stream.Length; }
                catch {}


                Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
                using (System.IO.Stream s = areq.GetRequestStream(streamLen))
                    using (var mds = new Utility.MD5CalculatingStream(s))
                    {
                        Utility.Utility.CopyStream(stream, mds, true, m_copybuffer);
                        fileHash = mds.GetFinalHashString();
                    }

                string md5Hash = null;

                //We need to verify the eTag locally
                try
                {
                    using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
                        if ((int)resp.StatusCode >= 300)
                        {
                            throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
                        }
                        else
                        {
                            md5Hash = resp.Headers["ETag"];
                        }
                }
                catch (WebException wex)
                {
                    //Catch 404 and turn it into a FolderNotFound error
                    if (wex.Response is HttpWebResponse && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound)
                    {
                        throw new FolderMissingException(wex);
                    }

                    //Other error, just re-throw
                    throw;
                }


                if (md5Hash == null || !String.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase))
                {
                    //Remove the broken file
                    try { Delete(remotename); }
                    catch { }

                    throw new Exception(Strings.CloudFiles.ETagVerificationError);
                }
            }
        }
Ejemplo n.º 5
0
        public void Put(string remotename, System.IO.Stream stream)
        {
            HttpWebRequest req = CreateRequest("/" + remotename, "");
            req.Method = "PUT";
            req.ContentType = "application/octet-stream";

            try { req.ContentLength = stream.Length; }
            catch { }

            //If we can pre-calculate the MD5 hash before transmission, do so
            /*if (stream.CanSeek)
            {
                System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create();
                req.Headers["ETag"] = Core.Utility.ByteArrayAsHexString(md5.ComputeHash(stream)).ToLower();
                stream.Seek(0, System.IO.SeekOrigin.Begin);

                using (System.IO.Stream s = req.GetRequestStream())
                    Core.Utility.CopyStream(stream, s);

                //Reset the timeout to the default value of 100 seconds to 
                // avoid blocking the GetResponse() call
                req.Timeout = 100000;

                //The server handles the eTag verification for us, and gives an error if the hash was a mismatch
                using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
                    if ((int)resp.StatusCode >= 300)
                        throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);

            }
            else //Otherwise use a client-side calculation
            */
            //TODO: We cannot use the local MD5 calculation, because that could involve a throttled read,
            // and may invoke various events
            {
                string fileHash = null;

                Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
                using (System.IO.Stream s = areq.GetRequestStream())
                using (var mds = new Utility.MD5CalculatingStream(s))
                {
                    Utility.Utility.CopyStream(stream, mds, true, m_copybuffer);
                    fileHash = mds.GetFinalHashString();
                }

                string md5Hash = null;

                //We need to verify the eTag locally
                try
                {
                    using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
                        if ((int)resp.StatusCode >= 300)
                            throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
                        else
                            md5Hash = resp.Headers["ETag"];
                }
                catch (WebException wex)
                {
                    //Catch 404 and turn it into a FolderNotFound error
                    if (wex.Response is HttpWebResponse && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound)
                        throw new FolderMissingException(wex);

                    //Other error, just re-throw
                    throw;
                }


                if (md5Hash == null || md5Hash.ToLower() != fileHash.ToLower())
                {
                    //Remove the broken file
                    try { Delete(remotename); }
                    catch { }

                    throw new Exception(Strings.CloudFiles.ETagVerificationError);
                }
            }
        }
Ejemplo n.º 6
0
        public void Get(string remotename, System.IO.Stream stream)
        {
            var req = CreateRequest("/" + remotename, "");
            req.Method = "GET";

            var areq = new Utility.AsyncHttpRequest(req);
            using (var resp = areq.GetResponse())
			using (var s = areq.GetResponseStream())
            using (var mds = new Utility.MD5CalculatingStream(s))
            {
                string md5Hash = resp.Headers["ETag"];
                Utility.Utility.CopyStream(mds, stream, true, m_copybuffer);

                if (mds.GetFinalHashString().ToLower() != md5Hash.ToLower())
                    throw new Exception(Strings.CloudFiles.ETagVerificationError);
            }
        }