public void Get(string remotename, System.IO.Stream stream)
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "GET";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (WebResponse resp = areq.GetResponse())
using (System.IO.Stream s = areq.GetResponseStream())
using (var mds = new Utility.MD5CalculatingStream(s))
{
string md5Hash = resp.Headers["ETag"];
Utility.Utility.CopyStream(mds, stream, true, m_copybuffer);
if (mds.GetFinalHashString().ToLower() != md5Hash.ToLower())
{
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
}
public void Get(string remotename, System.IO.Stream stream)
{
var req = CreateRequest("/" + remotename, "");
req.Method = "GET";
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = areq.GetResponse())
using (var s = areq.GetResponseStream())
using (var mds = new Utility.MD5CalculatingStream(s))
{
string md5Hash = resp.Headers["ETag"];
Utility.Utility.CopyStream(mds, stream, true, m_copybuffer);
if (!String.Equals(mds.GetFinalHashString(), md5Hash, StringComparison.OrdinalIgnoreCase))
{
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
}
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
{
// Some challenges with uploading to Jottacloud:
// - Jottacloud supports use of a custom header where we can tell the server the MD5 hash of the file
// we are uploading, and then it will verify the content of our request against it. But the HTTP
// status code we get back indicates success even if there is a mismatch, so we must dig into the
// XML response to see if we were able to correctly upload the new content or not. Another issue
// is that if the stream is not seek-able we have a challenge pre-calculating MD5 hash on it before
// writing it out on the HTTP request stream. And even if the stream is seek-able it may be throttled.
// One way to avoid using the throttled stream for calculating the MD5 is to try to get the
// underlying stream from the "m_basestream" field, with fall-back to a temporary file.
// - We can instead chose to upload the data without setting the MD5 hash header. The server will
// calculate the MD5 on its side and return it in the response back to use. We can then compare it
// with the MD5 hash of the stream (using a MD5CalculatingStream), and if there is a mismatch we can
// request the server to remove the file again and throw an exception. But there is a requirement that
// we specify the file size in a custom header. And if the stream is not seek-able we are not able
// to use stream.Length, so we are back at square one.
Duplicati.Library.Utility.TempFile tmpFile = null;
var baseStream = stream;
while (baseStream is Duplicati.Library.Utility.OverrideableStream)
{
baseStream = typeof(Duplicati.Library.Utility.OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic).GetValue(baseStream) as System.IO.Stream;
}
if (baseStream == null)
{
throw new Exception(string.Format("Unable to unwrap stream from: {0}", stream.GetType()));
}
string md5Hash;
if (baseStream.CanSeek)
{
var originalPosition = baseStream.Position;
using (var md5 = System.Security.Cryptography.MD5.Create())
md5Hash = Library.Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream));
baseStream.Position = originalPosition;
}
else
{
// No seeking possible, use a temp file
tmpFile = new Duplicati.Library.Utility.TempFile();
using (var os = System.IO.File.OpenWrite(tmpFile))
using (var md5 = new Utility.MD5CalculatingStream(baseStream))
{
await Utility.Utility.CopyStreamAsync(md5, os, true, cancelToken, m_copybuffer);
md5Hash = md5.GetFinalHashString();
}
stream = System.IO.File.OpenRead(tmpFile);
}
try
{
// Create request, with query parameter, and a few custom headers.
// NB: If we wanted to we could send the same POST request as below but without the file contents
// and with "cphash=[md5Hash]" as the only query parameter. Then we will get an HTTP 200 (OK) response
// if an identical file already exists, and we can skip uploading the new file. We will get
// HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which
// case we must send a new request to upload the new content.
var fileSize = stream.Length;
var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true);
req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch.
req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file!
// File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps,
// and then we can just omit the JCreated and JModified and let the server automatically set the current time.
//req.Headers.Add("JCreated", timeCreated);
//req.Headers.Add("JModified", timeModified);
req.ContentType = "application/octet-stream";
req.ContentLength = fileSize;
// Write post data request
var areq = new Utility.AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken, m_copybuffer);
// Send request, and check response
using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode != System.Net.HttpStatusCode.Created)
{
throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
// Request seems to be successful, but we must verify the response XML content to be sure that the file
// was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if
// there was mismatch, and it will verify the JMd5 header and mark the file as corrupt if there was a hash
// mismatch. The returned XML contains a file element, and if upload was error free it contains a single
// child element "currentRevision", which has a "state" child element with the string "COMPLETED".
// If there was a problem we should have a "latestRevision" child element, and this will have state with
// value "INCOMPLETE" or "CORRUPT". If the file was new or had no previous complete versions the latestRevision
// will be the only child, but if not there may also be a "currentRevision" representing the previous
// complete version - and then we need to detect the case where our upload failed but there was an existing
// complete version!
using (var rs = areq.GetResponseStream())
{
var doc = new System.Xml.XmlDocument();
try { doc.Load(rs); }
catch (System.Xml.XmlException)
{
throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError);
}
bool uploadCompletedSuccessfully = false;
var xFile = doc["file"];
if (xFile != null)
{
var xRevState = xFile.SelectSingleNode("latestRevision");
if (xRevState == null)
{
xRevState = xFile.SelectSingleNode("currentRevision/state");
if (xRevState != null)
{
uploadCompletedSuccessfully = xRevState.InnerText == "COMPLETED"; // Success: There is no "latestRevision", only a "currentRevision" (and it specifies the file is complete, but I think it always will).
}
}
}
if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..)
{
throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError);
}
}
}
}
finally
{
try
{
if (tmpFile != null)
{
tmpFile.Dispose();
}
}
catch { }
}
}
public void Put(string remotename, System.IO.Stream stream)
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "PUT";
req.ContentType = "application/octet-stream";
try { req.ContentLength = stream.Length; }
catch { }
//If we can pre-calculate the MD5 hash before transmission, do so
/*if (stream.CanSeek)
* {
* System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create();
* req.Headers["ETag"] = Core.Utility.ByteArrayAsHexString(md5.ComputeHash(stream)).ToLower();
* stream.Seek(0, System.IO.SeekOrigin.Begin);
*
* using (System.IO.Stream s = req.GetRequestStream())
* Core.Utility.CopyStream(stream, s);
*
* //Reset the timeout to the default value of 100 seconds to
* // avoid blocking the GetResponse() call
* req.Timeout = 100000;
*
* //The server handles the eTag verification for us, and gives an error if the hash was a mismatch
* using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
* if ((int)resp.StatusCode >= 300)
* throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
*
* }
* else //Otherwise use a client-side calculation
*/
//TODO: We cannot use the local MD5 calculation, because that could involve a throttled read,
// and may invoke various events
{
string fileHash = null;
long streamLen = -1;
try { streamLen = stream.Length; }
catch {}
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream s = areq.GetRequestStream(streamLen))
using (var mds = new Utility.MD5CalculatingStream(s))
{
Utility.Utility.CopyStream(stream, mds, true, m_copybuffer);
fileHash = mds.GetFinalHashString();
}
string md5Hash = null;
//We need to verify the eTag locally
try
{
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
if ((int)resp.StatusCode >= 300)
{
throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
}
else
{
md5Hash = resp.Headers["ETag"];
}
}
catch (WebException wex)
{
//Catch 404 and turn it into a FolderNotFound error
if (wex.Response is HttpWebResponse && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound)
{
throw new FolderMissingException(wex);
}
//Other error, just re-throw
throw;
}
if (md5Hash == null || !String.Equals(md5Hash, fileHash, StringComparison.OrdinalIgnoreCase))
{
//Remove the broken file
try { Delete(remotename); }
catch { }
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
}
public void Put(string remotename, System.IO.Stream stream)
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "PUT";
req.ContentType = "application/octet-stream";
try { req.ContentLength = stream.Length; }
catch { }
//If we can pre-calculate the MD5 hash before transmission, do so
/*if (stream.CanSeek)
{
System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create();
req.Headers["ETag"] = Core.Utility.ByteArrayAsHexString(md5.ComputeHash(stream)).ToLower();
stream.Seek(0, System.IO.SeekOrigin.Begin);
using (System.IO.Stream s = req.GetRequestStream())
Core.Utility.CopyStream(stream, s);
//Reset the timeout to the default value of 100 seconds to
// avoid blocking the GetResponse() call
req.Timeout = 100000;
//The server handles the eTag verification for us, and gives an error if the hash was a mismatch
using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
}
else //Otherwise use a client-side calculation
*/
//TODO: We cannot use the local MD5 calculation, because that could involve a throttled read,
// and may invoke various events
{
string fileHash = null;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream s = areq.GetRequestStream())
using (var mds = new Utility.MD5CalculatingStream(s))
{
Utility.Utility.CopyStream(stream, mds, true, m_copybuffer);
fileHash = mds.GetFinalHashString();
}
string md5Hash = null;
//We need to verify the eTag locally
try
{
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
else
md5Hash = resp.Headers["ETag"];
}
catch (WebException wex)
{
//Catch 404 and turn it into a FolderNotFound error
if (wex.Response is HttpWebResponse && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound)
throw new FolderMissingException(wex);
//Other error, just re-throw
throw;
}
if (md5Hash == null || md5Hash.ToLower() != fileHash.ToLower())
{
//Remove the broken file
try { Delete(remotename); }
catch { }
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
}
public void Get(string remotename, System.IO.Stream stream)
{
var req = CreateRequest("/" + remotename, "");
req.Method = "GET";
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = areq.GetResponse())
using (var s = areq.GetResponseStream())
using (var mds = new Utility.MD5CalculatingStream(s))
{
string md5Hash = resp.Headers["ETag"];
Utility.Utility.CopyStream(mds, stream, true, m_copybuffer);
if (mds.GetFinalHashString().ToLower() != md5Hash.ToLower())
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
