/// <summary>
/// Retrieves info for a single file (used to determine file size for chunking)
/// </summary>
/// <param name="remotename"></param>
/// <returns></returns>
public IFileEntry Info(string remotename)
{
var doc = new System.Xml.XmlDocument();
try
{
// Send request and load XML response.
var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "", false);
var areq = new Utility.AsyncHttpRequest(req);
using (var rs = areq.GetResponseStream())
doc.Load(rs);
}
catch (System.Net.WebException wex)
{
if (wex.Response is System.Net.HttpWebResponse && ((System.Net.HttpWebResponse)wex.Response).StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FileMissingException(wex);
}
throw;
}
// Handle XML response. Since we in the constructor demand a folder below the mount point we know the root
// element must be a "folder", else it could also have been a "mountPoint" (which has a very similar structure).
// We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion)
// so we treat them as non-existant here.
var xFile = doc.DocumentElement;
if (xFile.Attributes["deleted"] != null)
{
throw new FileMissingException(string.Format("{0}: {1}", LC.L("The requested file does not exist"), remotename));
}
return(ToFileEntry(xFile));
}
public void Delete(string remotename)
{
try
{
System.Net.HttpWebRequest req = CreateRequest(remotename);
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FileMissingException();
}
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
{
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
}
}
catch (System.Net.WebException wex)
{
if (wex.Response is System.Net.HttpWebResponse && ((System.Net.HttpWebResponse)wex.Response).StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FileMissingException(wex);
}
else
{
throw;
}
}
}
public void Delete(string remotename)
{
System.Net.HttpWebRequest req = CreateRequest(System.Net.WebRequestMethods.Http.Post, remotename, "rm=true", false); // rm=true means permanent delete, dl=true would be move to trash.
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{ }
}
public string GetRegionEndpoint(string url)
{
try
{
System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url);
req.Method = System.Net.WebRequestMethods.Http.Get;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
{
throw new Exception("Failed to fetch region endpoint");
}
using (var s = areq.GetResponseStream())
{
using (var reader = new StreamReader(s))
{
string endpoint = reader.ReadToEnd();
return(endpoint);
}
}
}
}
catch (System.Net.WebException wex)
{
//Convert to better exception
throw new Exception("Failed to fetch region endpoint");
}
}
public void Delete(string remotename)
{
try
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FileMissingException();
}
if ((int)resp.StatusCode >= 300)
{
throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp);
}
else
{
using (areq.GetResponseStream())
{ }
}
}
}
catch (System.Net.WebException wex)
{
if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FileMissingException(wex);
}
private HttpWebRequest CreateRequest(string remotename, string query)
{
//If this is the first call, get an authentication token
if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl))
{
HttpWebRequest authReq = (HttpWebRequest)HttpWebRequest.Create(m_authUrl);
authReq.Headers.Add("X-Auth-User", m_username);
authReq.Headers.Add("X-Auth-Key", m_password);
authReq.Method = "GET";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(authReq);
using (WebResponse resp = areq.GetResponse())
{
m_storageUrl = resp.Headers["X-Storage-Url"];
m_authToken = resp.Headers["X-Auth-Token"];
}
if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl))
{
throw new Exception(Strings.CloudFiles.UnexpectedResponseError);
}
}
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(m_storageUrl + UrlEncode(m_path + remotename) + query);
req.Headers.Add("X-Auth-Token", UrlEncode(m_authToken));
req.UserAgent = "Duplicati CloudFiles Backend v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString();
req.KeepAlive = false;
req.PreAuthenticate = true;
req.AllowWriteStreamBuffering = false;
return(req);
}
public void Put(string remotename, System.IO.Stream stream)
{
var url = string.Format("{0}/{1}/files/{2}?access_token={3}", WLID_SERVER, FolderID, Utility.Uri.UrlPathEncode(remotename), Library.Utility.Uri.UrlEncode(AccessToken));
var req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = USER_AGENT;
req.Method = "PUT";
try
{
req.ContentLength = stream.Length;
}
catch
{
}
// Docs says not to set this ?
//req.ContentType = "application/octet-stream";
var areq = new Utility.AsyncHttpRequest(req);
using (var reqs = areq.GetRequestStream())
Utility.Utility.CopyStream(stream, reqs, true, m_copybuffer);
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var rs = areq.GetResponseStream())
using (var tr = new System.IO.StreamReader(rs))
using (var jr = new Newtonsoft.Json.JsonTextReader(tr))
{
var nf = new Newtonsoft.Json.JsonSerializer().Deserialize <WLID_FolderItem>(jr);
m_fileidCache[remotename] = nf.id;
}
}
public void Rename(string oldname, string newname)
{
try
{
try
{
var id = GetFileID(oldname);
var url = string.Format("{0}/{1}?access_token={2}", WLID_SERVER, id, Library.Utility.Uri.UrlEncode(m_oauth.AccessToken));
var req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = USER_AGENT;
req.Method = "PUT";
var updateData = new WLID_FolderItem()
{
name = newname
};
var data = System.Text.Encoding.UTF8.GetBytes(Newtonsoft.Json.JsonConvert.SerializeObject(updateData));
req.ContentLength = data.Length;
req.ContentType = "application/json; charset=UTF-8";
using (var requestStream = req.GetRequestStream())
requestStream.Write(data, 0, data.Length);
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FileMissingException();
}
if ((int)resp.StatusCode < 200 || (int)resp.StatusCode > 299)
{
throw new ProtocolViolationException(Strings.OneDrive.UnexpectedError(resp.StatusCode, resp.StatusDescription));
}
m_fileidCache[newname] = id;
m_fileidCache.Remove(oldname);
}
}
catch
{
// Since we don't know the state of file IDs, clear the cache
m_fileidCache.Clear();
throw;
}
}
catch (System.Net.WebException wex)
{
if (wex.Response is System.Net.HttpWebResponse && ((System.Net.HttpWebResponse)wex.Response).StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FileMissingException(wex);
}
else
{
throw;
}
}
}
public void Delete(string remotename)
{
System.Net.HttpWebRequest req = CreateRequest(remotename, "");
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
public void CreateFolder()
{
System.Net.HttpWebRequest req = CreateRequest("", "t=mkdir");
req.Method = System.Net.WebRequestMethods.Http.Post;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
public void Delete(string remotename)
{
System.Net.FtpWebRequest req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Ftp.DeleteFile;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
public void CreateFolder()
{
System.Net.FtpWebRequest req = CreateRequest("", true);
req.Method = System.Net.WebRequestMethods.Ftp.MakeDirectory;
req.KeepAlive = false;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
public void CreateFolder()
{
HttpWebRequest createReq = CreateRequest("", "");
createReq.Method = "PUT";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(createReq);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
{ }
}
public void Get(string remotename, System.IO.Stream output)
{
System.Net.FtpWebRequest req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Ftp.DownloadFile;
req.UseBinary = true;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.WebResponse resp = areq.GetResponse())
using (System.IO.Stream rs = areq.GetResponseStream())
Utility.Utility.CopyStream(rs, output, false);
}
public void Put(string remotename, System.IO.Stream input)
{
System.Net.FtpWebRequest req = null;
try
{
req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Ftp.UploadFile;
req.UseBinary = true;
long streamLen = -1;
try { streamLen = input.Length; }
catch {}
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream rs = areq.GetRequestStream())
Utility.Utility.CopyStream(input, rs, true);
if (m_listVerify)
{
List <IFileEntry> files = List(remotename);
StringBuilder sb = new StringBuilder();
foreach (IFileEntry fe in files)
{
if (fe.Name.Equals(remotename) || fe.Name.EndsWith("/" + remotename) || fe.Name.EndsWith("\\" + remotename))
{
if (fe.Size < 0 || streamLen < 0 || fe.Size == streamLen)
{
return;
}
throw new Exception(string.Format(Strings.FTPBackend.ListVerifySizeFailure, remotename, fe.Size, streamLen));
}
else
{
sb.AppendLine(fe.Name);
}
}
throw new Exception(string.Format(Strings.FTPBackend.ListVerifyFailure, remotename, sb.ToString()));
}
}
catch (System.Net.WebException wex)
{
if (req != null && wex.Response as System.Net.FtpWebResponse != null && (wex.Response as System.Net.FtpWebResponse).StatusCode == System.Net.FtpStatusCode.ActionNotTakenFileUnavailable)
{
throw new Interface.FolderMissingException(string.Format(Strings.FTPBackend.MissingFolderError, req.RequestUri.PathAndQuery, wex.Message), wex);
}
else
{
throw;
}
}
}
public void Get(string remotename, System.IO.Stream stream)
{
// Downloading from Jottacloud: Will only succeed if the file has a completed revision,
// and if there are multiple versions of the file we will only get the latest completed version,
// ignoring any incomplete or corrupt versions.
var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "mode=bin", false);
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
using (var s = areq.GetResponseStream())
Utility.Utility.CopyStream(s, stream, true, m_copybuffer);
}
public void Get(string remotename, string localname)
{
string endpoint = "";
string siafile = m_targetpath + "/" + remotename;
string tmpfilename = localname + ".tmp";
try
{
endpoint = string.Format("/renter/download/{0}/{1}?destination={2}",
m_targetpath,
Library.Utility.Uri.UrlEncode(remotename).Replace("+", "%20"),
Library.Utility.Uri.UrlEncode(tmpfilename).Replace("+", "%20")
);
System.Net.HttpWebRequest req = CreateRequest(endpoint);
req.Method = System.Net.WebRequestMethods.Http.Get;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300)
{
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
while (!IsDownloadComplete(siafile, localname))
{
System.Threading.Thread.Sleep(5000);
}
System.IO.File.Copy(tmpfilename, localname, true);
try
{
System.IO.File.Delete(tmpfilename);
} catch (Exception)
{
}
}
}
catch (System.Net.WebException wex)
{
if (wex.Response is System.Net.HttpWebResponse && ((System.Net.HttpWebResponse)wex.Response).StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FileMissingException(wex);
}
else
{
throw new Exception(getResponseBodyOnError(endpoint, wex));
}
}
}
public void Delete(string remotename)
{
System.Net.HttpWebRequest req = CreateRequest(remotename);
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
{
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
}
}
public void Get(string remotename, System.IO.Stream stream)
{
var id = GetFileID(remotename);
var url = string.Format("{0}/{1}/content?access_token={2}", WLID_SERVER, id, Library.Utility.Uri.UrlEncode(m_oauth.AccessToken));
var req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = USER_AGENT;
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var rs = areq.GetResponseStream())
Utility.Utility.CopyStream(rs, stream, true, m_copybuffer);
}
public void CreateFolder()
{
System.Net.HttpWebRequest req = CreateRequest("");
req.Method = System.Net.WebRequestMethods.Http.MkCol;
req.KeepAlive = false;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
{
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
}
}
public async Task PutAsync(string remotename, System.IO.Stream input, CancellationToken cancelToken)
{
System.Net.FtpWebRequest req = null;
try
{
req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Ftp.UploadFile;
req.UseBinary = true;
long streamLen = -1;
try { streamLen = input.Length; }
catch {}
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream rs = areq.GetRequestStream(streamLen))
await Utility.Utility.CopyStreamAsync(input, rs, true, cancelToken, m_copybuffer).ConfigureAwait(false);
if (m_listVerify)
{
IEnumerable <IFileEntry> files = List(remotename);
foreach (IFileEntry fe in files)
{
if (fe.Name.Equals(remotename) || fe.Name.EndsWith("/" + remotename, StringComparison.Ordinal) || fe.Name.EndsWith("\\" + remotename, StringComparison.Ordinal))
{
if (fe.Size < 0 || streamLen < 0 || fe.Size == streamLen)
{
return;
}
throw new Exception(Strings.FTPBackend.ListVerifySizeFailure(remotename, fe.Size, streamLen));
}
}
throw new Exception(Strings.FTPBackend.ListVerifyFailure(remotename, files.Select(n => n.Name)));
}
}
catch (System.Net.WebException wex)
{
if (req != null && wex.Response as System.Net.FtpWebResponse != null && (wex.Response as System.Net.FtpWebResponse).StatusCode == System.Net.FtpStatusCode.ActionNotTakenFileUnavailable)
{
throw new Interface.FolderMissingException(Strings.FTPBackend.MissingFolderError(req.RequestUri.PathAndQuery, wex.Message), wex);
}
else
{
throw;
}
}
}
public void Delete(string remotename)
{
try
{
System.Net.HttpWebRequest req = CreateRequest(remotename, "");
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
catch (System.Net.WebException wex)
{
if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FileMissingException(wex);
}
public void Delete(string remotename)
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
if ((int)resp.StatusCode >= 300)
{
throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp);
}
else
{
using (areq.GetResponseStream())
{ }
}
}
public void Get(string remotename, System.IO.Stream stream)
{
var req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Http.Get;
try
{
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
{
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
using (var s = areq.GetResponseStream())
Utility.Utility.CopyStream(s, stream, true, m_copybuffer);
}
}
catch (System.Net.WebException wex)
{
if (wex.Response as System.Net.HttpWebResponse != null)
{
if ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict)
{
throw new Interface.FolderMissingException(Strings.WEBDAV.MissingFolderError(m_path, wex.Message), wex);
}
if
(
(wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound
&&
m_filenamelist != null
&&
m_filenamelist.Contains(remotename)
)
{
throw new Exception(Strings.WEBDAV.SeenThenNotFoundError(m_path, remotename, System.IO.Path.GetExtension(remotename), wex.Message), wex);
}
}
throw;
}
}
public IEnumerable <IFileEntry> List()
{
var doc = new System.Xml.XmlDocument();
try
{
// Send request and load XML response.
var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, "", "", false);
var areq = new Utility.AsyncHttpRequest(req);
using (var rs = areq.GetResponseStream())
doc.Load(rs);
}
catch (System.Net.WebException wex)
{
if (wex.Response is System.Net.HttpWebResponse && ((System.Net.HttpWebResponse)wex.Response).StatusCode == System.Net.HttpStatusCode.NotFound)
{
throw new FolderMissingException(wex);
}
throw;
}
// Handle XML response. Since we in the constructor demand a folder below the mount point we know the root
// element must be a "folder", else it could also have been a "mountPoint" (which has a very similar structure).
// We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion)
// so we treat them as non-existant here.
var xRoot = doc.DocumentElement;
if (xRoot.Attributes["deleted"] != null)
{
throw new FolderMissingException();
}
foreach (System.Xml.XmlNode xFolder in xRoot.SelectNodes("folders/folder[not(@deleted)]"))
{
// Subfolders are only listed with name. We can get a timestamp by sending a request for each folder, but that is probably not necessary?
FileEntry fe = new FileEntry(xFolder.Attributes["name"].Value);
fe.IsFolder = true;
yield return(fe);
}
foreach (System.Xml.XmlNode xFile in xRoot.SelectNodes("files/file[not(@deleted)]"))
{
var fe = ToFileEntry(xFile);
if (fe != null)
{
yield return(fe);
}
}
}
public void Get(string remotename, System.IO.Stream stream)
{
System.Net.HttpWebRequest req = CreateRequest(remotename, "");
req.Method = System.Net.WebRequestMethods.Http.Get;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
{
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
using (System.IO.Stream s = areq.GetResponseStream())
Utility.Utility.CopyStream(s, stream);
}
}
public void CreateFolder()
{
// When using custom (backup) device we must create the device first (if not already exists).
if (!m_device_builtin)
{
System.Net.HttpWebRequest req = CreateRequest(System.Net.WebRequestMethods.Http.Post, m_url_device, "type=WORKSTATION"); // Hard-coding device type. Must be one of "WORKSTATION", "LAPTOP", "IMAC", "MACBOOK", "IPAD", "ANDROID", "IPHONE" or "WINDOWS_PHONE".
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{ }
}
// Create the folder path, and if using custom mount point it will be created as well in the same operation.
{
System.Net.HttpWebRequest req = CreateRequest(System.Net.WebRequestMethods.Http.Post, "", "mkDir=true", false);
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{ }
}
}
private T GetJSONData <T>(string url, Action <HttpWebRequest> setup = null)
{
var req = (HttpWebRequest)System.Net.WebRequest.Create(url);
req.UserAgent = USER_AGENT;
if (setup != null)
{
setup(req);
}
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var rs = areq.GetResponseStream())
using (var tr = new System.IO.StreamReader(rs))
using (var jr = new Newtonsoft.Json.JsonTextReader(tr))
return(new Newtonsoft.Json.JsonSerializer().Deserialize <T>(jr));
}
public void Delete(string remotename)
{
var id = GetFileID(remotename);
var url = string.Format("{0}/{1}?access_token={2}", WLID_SERVER, id, Library.Utility.Uri.UrlEncode(AccessToken));
var req = (HttpWebRequest)WebRequest.Create(url);
req.Method = "DELETE";
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if ((int)resp.StatusCode < 200 || (int)resp.StatusCode > 299)
{
throw new ProtocolViolationException(string.Format(LC.L("Unexpected error code: {0} - {1}"), resp.StatusCode, resp.StatusDescription));
}
m_fileidCache.Remove(remotename);
}
}
public void Get(string remotename, System.IO.Stream stream)
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "GET";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (WebResponse resp = areq.GetResponse())
using (System.IO.Stream s = areq.GetResponseStream())
using (MD5CalculatingStream mds = new MD5CalculatingStream(s))
{
string md5Hash = resp.Headers["ETag"];
Utility.Utility.CopyStream(mds, stream, true, m_copybuffer);
if (mds.GetFinalHashString().ToLower() != md5Hash.ToLower())
{
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
}
public List<IFileEntry> List()
{
System.Net.HttpWebRequest req = CreateRequest("");
req.Method = "PROPFIND";
req.Headers.Add("Depth", "1");
req.ContentType = "text/xml";
req.ContentLength = PROPFIND_BODY.Length;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream s = areq.GetRequestStream())
s.Write(PROPFIND_BODY, 0, PROPFIND_BODY.Length);
try
{
System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
if (!string.IsNullOrEmpty(m_debugPropfindFile))
{
using (System.IO.FileStream fs = new System.IO.FileStream(m_debugPropfindFile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
Utility.Utility.CopyStream(areq.GetResponseStream(), fs, false);
doc.Load(m_debugPropfindFile);
}
else
doc.Load(areq.GetResponseStream());
}
System.Xml.XmlNamespaceManager nm = new System.Xml.XmlNamespaceManager(doc.NameTable);
nm.AddNamespace("D", "DAV:");
List<IFileEntry> files = new List<IFileEntry>();
m_filenamelist = new List<string>();
foreach (System.Xml.XmlNode n in doc.SelectNodes("D:multistatus/D:response/D:href", nm))
{
//IIS uses %20 for spaces and %2B for +
//Apache uses %20 for spaces and + for +
string name = System.Web.HttpUtility.UrlDecode(n.InnerText.Replace("+", "%2B"));
string cmp_path;
//TODO: This list is getting ridiculous, should change to regexps
if (name.StartsWith(m_url))
cmp_path = m_url;
else if (name.StartsWith(m_rawurl))
cmp_path = m_rawurl;
else if (name.StartsWith(m_rawurlPort))
cmp_path = m_rawurlPort;
else if (name.StartsWith(m_path))
cmp_path = m_path;
else if (name.StartsWith(m_sanitizedUrl))
cmp_path = m_sanitizedUrl;
else if (name.StartsWith(m_reverseProtocolUrl))
cmp_path = m_reverseProtocolUrl;
else
continue;
if (name.Length <= cmp_path.Length)
continue;
name = name.Substring(cmp_path.Length);
long size = -1;
DateTime lastAccess = new DateTime();
DateTime lastModified = new DateTime();
bool isCollection = false;
System.Xml.XmlNode stat = n.ParentNode.SelectSingleNode("D:propstat/D:prop", nm);
if (stat != null)
{
System.Xml.XmlNode s = stat.SelectSingleNode("D:getcontentlength", nm);
if (s != null)
size = long.Parse(s.InnerText);
s = stat.SelectSingleNode("D:getlastmodified", nm);
if (s != null)
try
{
//Not important if this succeeds
lastAccess = lastModified = DateTime.Parse(s.InnerText, System.Globalization.CultureInfo.InvariantCulture);
}
catch { }
s = stat.SelectSingleNode("D:iscollection", nm);
if (s != null)
isCollection = s.InnerText.Trim() == "1";
else
isCollection = (stat.SelectSingleNode("D:resourcetype/D:collection", nm) != null);
}
FileEntry fe = new FileEntry(name, size, lastAccess, lastModified);
fe.IsFolder = isCollection;
files.Add(fe);
m_filenamelist.Add(name);
}
return files;
}
catch (System.Net.WebException wex)
{
if (wex.Response as System.Net.HttpWebResponse != null &&
((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound || (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict))
throw new Interface.FolderMissingException(string.Format(Strings.WEBDAV.MissingFolderError, m_path, wex.Message), wex);
throw;
}
}
public void Delete(string remotename)
{
System.Net.HttpWebRequest req = CreateRequest(remotename);
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
}
private HttpWebRequest CreateRequest(string remotename, string query)
{
//If this is the first call, get an authentication token
if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl))
{
HttpWebRequest authReq = (HttpWebRequest)HttpWebRequest.Create(m_authUrl);
authReq.Headers.Add("X-Auth-User", m_username);
authReq.Headers.Add("X-Auth-Key", m_password);
authReq.Method = "GET";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(authReq);
using (WebResponse resp = areq.GetResponse())
{
m_storageUrl = resp.Headers["X-Storage-Url"];
m_authToken = resp.Headers["X-Auth-Token"];
}
if (string.IsNullOrEmpty(m_authToken) || string.IsNullOrEmpty(m_storageUrl))
throw new Exception(Strings.CloudFiles.UnexpectedResponseError);
}
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(m_storageUrl + UrlEncode(m_path + remotename) + query);
req.Headers.Add("X-Auth-Token", UrlEncode(m_authToken));
req.UserAgent = "Duplicati CloudFiles Backend v" + System.Reflection.Assembly.GetExecutingAssembly().GetName().Version.ToString();
req.KeepAlive = false;
req.PreAuthenticate = true;
req.AllowWriteStreamBuffering = false;
return req;
}
private WLID_FolderItem FindFolders(bool autocreate)
{
var folders = (m_rootfolder + '/' + m_prefix).Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
if (folders.Length == 0)
{
var url = string.Format("{0}/{1}?access_token={2}", WLID_SERVER, ROOT_FOLDER_ID, Library.Utility.Uri.UrlEncode(m_oauth.AccessToken));
return m_oauth.GetJSONData<WLID_FolderItem>(url);
}
WLID_FolderItem cur = null;
foreach (var f in folders)
{
var n = FindFolder(f, cur == null ? null : cur.id);
if (n == null)
{
if (autocreate)
{
var url = string.Format("{0}/{1}?access_token={2}", WLID_SERVER, cur == null ? ROOT_FOLDER_ID : cur.id, Library.Utility.Uri.UrlEncode(m_oauth.AccessToken));
var req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = USER_AGENT;
req.Method = "POST";
var areq = new Utility.AsyncHttpRequest(req);
using (var ms = new System.IO.MemoryStream())
using (var sw = new System.IO.StreamWriter(ms))
{
new Newtonsoft.Json.JsonSerializer().Serialize(sw, new WLID_CreateFolderData() {
name = f,
description = Strings.OneDrive.AutoCreatedFolderLabel
});
sw.Flush();
ms.Position = 0;
req.ContentLength = ms.Length;
req.ContentType = "application/json";
using (var reqs = areq.GetRequestStream())
Utility.Utility.CopyStream(ms, reqs, true, m_copybuffer);
}
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var rs = areq.GetResponseStream())
using (var tr = new System.IO.StreamReader(rs))
using (var jr = new Newtonsoft.Json.JsonTextReader(tr))
{
if ((int)resp.StatusCode < 200 || (int)resp.StatusCode > 299)
throw new ProtocolViolationException(Strings.OneDrive.UnexpectedError(resp.StatusCode, resp.StatusDescription));
cur = new Newtonsoft.Json.JsonSerializer().Deserialize<WLID_FolderItem>(jr);
}
}
else
throw new FolderMissingException(Strings.OneDrive.MissingFolderError(f));
}
else
cur = n;
}
return cur;
}
public void CreateFolder()
{
System.Net.HttpWebRequest req = CreateRequest("", "t=mkdir");
req.Method = System.Net.WebRequestMethods.Http.Post;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
public void Put(string remotename, System.IO.Stream stream)
{
try
{
Google.Documents.Document folder = GetFolder();
//Special, since uploads can overwrite or create,
// we must figure out if the file exists in advance.
//Unfortunately it would be wastefull to request the list
// for each upload request, so we rely on the cache being
// correct
TaggedFileEntry doc = null;
if (m_files == null)
doc = TryGetFile(remotename);
else
m_files.TryGetValue(remotename, out doc);
try
{
string resumableUri;
if (doc != null)
{
if (doc.MediaUrl == null)
{
//Strange, we could not get the edit url, perhaps it is readonly?
//Fallback strategy is "delete-then-upload"
try { this.Delete(remotename); }
catch { }
doc = TryGetFile(remotename);
if (doc != null || doc.MediaUrl == null)
throw new Exception(string.Format(Strings.GoogleDocs.FileIsReadOnlyError, remotename));
}
}
//File does not exist, we upload a new one
if (doc == null)
{
//First we need to get a resumeable upload url
HttpWebRequest req = (HttpWebRequest)WebRequest.Create("https://docs.google.com/feeds/upload/create-session/default/private/full/" + System.Web.HttpUtility.UrlEncode(folder.ResourceId) + "/contents?convert=false");
req.Method = "POST";
req.Headers.Add("X-Upload-Content-Length", stream.Length.ToString());
req.Headers.Add("X-Upload-Content-Type", "application/octet-stream");
req.UserAgent = USER_AGENT;
req.Headers.Add("GData-Version", "3.0");
//Build the atom entry describing the file we want to create
string labels = "";
foreach (string s in m_labels)
if (s.Trim().Length > 0)
labels += string.Format(ATTRIBUTE_TEMPLATE, s);
//Apply the name and content-type to the not-yet-uploaded file
byte[] data = System.Text.Encoding.UTF8.GetBytes(string.Format(CREATE_ITEM_TEMPLATE, System.Web.HttpUtility.HtmlEncode(remotename), labels));
req.ContentLength = data.Length;
req.ContentType = "application/atom+xml";
//Authenticate our request
m_cla.ApplyAuthenticationToRequest(req);
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream s = areq.GetRequestStream())
s.Write(data, 0, data.Length);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
resumableUri = resp.Headers["Location"];
}
}
else
{
//First we need to get a resumeable upload url
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(doc.MediaUrl);
req.Method = "PUT";
req.Headers.Add("X-Upload-Content-Length", stream.Length.ToString());
req.Headers.Add("X-Upload-Content-Type", "application/octet-stream");
req.UserAgent = USER_AGENT;
req.Headers.Add("If-Match", doc.ETag);
req.Headers.Add("GData-Version", "3.0");
//This is a blank marker request
req.ContentLength = 0;
//Bad... docs say "text/plain" or "text/xml", but really needs to be content type, otherwise overwrite fails
//req.ContentType = "text/plain";
req.ContentType = "application/octet-stream";
//Authenticate our request
m_cla.ApplyAuthenticationToRequest(req);
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
resumableUri = resp.Headers["Location"];
}
}
//Ensure that we have a resumeable upload url
if (resumableUri == null)
throw new Exception(Strings.GoogleDocs.NoResumeURLError);
string id = null;
byte[] buffer = new byte[8 * 1024];
int retries = 0;
long initialPosition;
DateTime initialRequestTime = DateTime.Now;
while (stream.Position != stream.Length)
{
initialPosition = stream.Position;
long postbytes = Math.Min(stream.Length - initialPosition, TRANSFER_CHUNK_SIZE);
//Post a fragment of the file as a partial request
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(resumableUri);
req.Method = "PUT";
req.UserAgent = USER_AGENT;
req.ContentLength = postbytes;
req.ContentType = "application/octet-stream";
req.Headers.Add("Content-Range", string.Format("bytes {0}-{1}/{2}", initialPosition, initialPosition + (postbytes - 1), stream.Length.ToString()));
//Copy the current fragment of bytes
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream s = areq.GetRequestStream())
{
long bytesleft = postbytes;
long written = 0;
int a;
while (bytesleft != 0 && ((a = stream.Read(buffer, 0, (int)Math.Min(buffer.Length, bytesleft))) != 0))
{
s.Write(buffer, 0, a);
bytesleft -= a;
written += a;
}
s.Flush();
if (bytesleft != 0 || postbytes != written)
throw new System.IO.EndOfStreamException();
}
try
{
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
//If all goes well, we should now get an atom entry describing the new element
System.Xml.XmlDocument xml = new XmlDocument();
using (System.IO.Stream s = areq.GetResponseStream())
xml.Load(s);
System.Xml.XmlNamespaceManager mgr = new XmlNamespaceManager(xml.NameTable);
mgr.AddNamespace("atom", "http://www.w3.org/2005/Atom");
mgr.AddNamespace("gd", "http://schemas.google.com/g/2005");
id = xml.SelectSingleNode("atom:entry/atom:id", mgr).InnerText;
string resourceId = xml.SelectSingleNode("atom:entry/gd:resourceId", mgr).InnerText;
string url = xml.SelectSingleNode("atom:entry/atom:content", mgr).Attributes["src"].Value;
string mediaUrl = null;
foreach(XmlNode n in xml.SelectNodes("atom:entry/atom:link", mgr))
if (n.Attributes["rel"] != null && n.Attributes["href"] != null &&n.Attributes["rel"].Value.EndsWith("#resumable-edit-media"))
{
mediaUrl = n.Attributes["href"].Value;
break;
}
if (doc == null)
{
TaggedFileEntry tf = new TaggedFileEntry(remotename, stream.Length, initialRequestTime, initialRequestTime, resourceId, url, mediaUrl, resp.Headers["ETag"]);
m_files.Add(remotename, tf);
}
else
{
//Since we update an existing item, we just need to update the ETag
doc.ETag = resp.Headers["ETag"];
}
}
retries = 0;
}
catch (WebException wex)
{
bool acceptedError =
wex.Status == WebExceptionStatus.ProtocolError &&
wex.Response is HttpWebResponse &&
(int)((HttpWebResponse)wex.Response).StatusCode == 308;
//Mono does not give us the response object,
// so we rely on the error code being present
// in the string, not ideal, but I have found
// no other workaround :(
if (Duplicati.Library.Utility.Utility.IsMono)
{
acceptedError |=
wex.Status == WebExceptionStatus.ProtocolError &&
wex.Message.Contains("308");
}
//Accept the 308 until we are complete
if (acceptedError &&
initialPosition + postbytes != stream.Length)
{
retries = 0;
//Accept the 308 until we are complete
}
else
{
//Retries are handled in Duplicati, but it is much more efficient here,
// because we only re-submit the last TRANSFER_CHUNK_SIZE bytes,
// instead of the entire file
retries++;
if (retries > 2)
throw;
else
System.Threading.Thread.Sleep(2000 * retries);
stream.Position = initialPosition;
}
}
}
if (string.IsNullOrEmpty(id))
throw new Exception(Strings.GoogleDocs.NoIDReturnedError);
}
catch
{
//Clear the cache as we have no idea what happened
m_files = null;
throw;
}
}
catch (Google.GData.Client.CaptchaRequiredException cex)
{
throw new Exception(string.Format(Strings.GoogleDocs.CaptchaRequiredError, CAPTCHA_UNLOCK_URL), cex);
}
}
public void Put(string remotename, System.IO.Stream stream)
{
if (stream.Length > BITS_FILE_SIZE_LIMIT)
{
// Get extra info for BITS
var uid = UserID;
var fid = FolderID.Split('.')[2];
// Create a session
var url = string.Format("https://cid-{0}.users.storage.live.com/items/{1}/{2}?access_token={3}", uid, fid, Utility.Uri.UrlPathEncode(remotename), Library.Utility.Uri.UrlEncode(m_oauth.AccessToken));
var req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = USER_AGENT;
req.Method = "POST";
req.ContentType = "application/json";
req.Headers.Add("X-Http-Method-Override", "BITS_POST");
req.Headers.Add("BITS-Packet-Type", "Create-Session");
req.Headers.Add("BITS-Supported-Protocols", "{7df0354d-249b-430f-820d-3d2a9bef4931}");
req.ContentLength = 0;
var areq = new Utility.AsyncHttpRequest(req);
string sessionid;
using(var resp = (HttpWebResponse)areq.GetResponse())
{
var packtype = resp.Headers["BITS-Packet-Type"];
if (!packtype.Equals("Ack", StringComparison.InvariantCultureIgnoreCase))
throw new Exception(string.Format("Unable to create BITS transfer, got status: {0}", packtype));
sessionid = resp.Headers["BITS-Session-Id"];
}
if (string.IsNullOrEmpty(sessionid))
throw new Exception("BITS session-id was missing");
// Session is now created, start uploading chunks
var offset = 0L;
var retries = 0;
while (offset < stream.Length)
{
try
{
var bytesInChunk = Math.Min(BITS_CHUNK_SIZE, stream.Length - offset);
req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = USER_AGENT;
req.Method = "POST";
req.Headers.Add("X-Http-Method-Override", "BITS_POST");
req.Headers.Add("BITS-Packet-Type", "Fragment");
req.Headers.Add("BITS-Session-Id", sessionid);
req.Headers.Add("Content-Range", string.Format("bytes {0}-{1}/{2}", offset, offset + bytesInChunk - 1, stream.Length));
req.ContentLength = bytesInChunk;
if (stream.Position != offset)
stream.Position = offset;
areq = new Utility.AsyncHttpRequest(req);
var remaining = (int)bytesInChunk;
using(var reqs = areq.GetRequestStream())
{
int read;
while ((read = stream.Read(m_copybuffer, 0, Math.Min(m_copybuffer.Length, remaining))) != 0)
{
reqs.Write(m_copybuffer, 0, read);
remaining -= read;
}
}
using(var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode != HttpStatusCode.OK)
throw new WebException("Invalid partial upload response", null, WebExceptionStatus.UnknownError, resp);
}
offset += bytesInChunk;
retries = 0;
}
catch (Exception ex)
{
var retry = false;
// If we get a 5xx error, or some network issue, we retry
if (ex is WebException && ((WebException)ex).Response is HttpWebResponse)
{
var code = (int)((HttpWebResponse)((WebException)ex).Response).StatusCode;
retry = code >= 500 && code <= 599;
}
else if (ex is System.Net.Sockets.SocketException || ex is System.IO.IOException || ex.InnerException is System.Net.Sockets.SocketException || ex.InnerException is System.IO.IOException)
{
retry = true;
}
// Retry with exponential backoff
if (retry && retries < 5)
{
System.Threading.Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries)));
retries++;
}
else
throw;
}
}
// Transfer completed, now commit the upload and close the session
req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = USER_AGENT;
req.Method = "POST";
req.Headers.Add("X-Http-Method-Override", "BITS_POST");
req.Headers.Add("BITS-Packet-Type", "Close-Session");
req.Headers.Add("BITS-Session-Id", sessionid);
req.ContentLength = 0;
areq = new Utility.AsyncHttpRequest(req);
using(var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode != HttpStatusCode.OK)
throw new Exception("Invalid partial upload commit response");
}
}
else
{
var url = string.Format("{0}/{1}/files/{2}?access_token={3}", WLID_SERVER, FolderID, Utility.Uri.UrlPathEncode(remotename), Library.Utility.Uri.UrlEncode(m_oauth.AccessToken));
var req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = USER_AGENT;
req.Method = "PUT";
try
{
req.ContentLength = stream.Length;
}
catch
{
}
// Docs says not to set this ?
//req.ContentType = "application/octet-stream";
var areq = new Utility.AsyncHttpRequest(req);
using(var reqs = areq.GetRequestStream())
Utility.Utility.CopyStream(stream, reqs, true, m_copybuffer);
using(var resp = (HttpWebResponse)areq.GetResponse())
using(var rs = areq.GetResponseStream())
using(var tr = new System.IO.StreamReader(rs))
using(var jr = new Newtonsoft.Json.JsonTextReader(tr))
{
var nf = new Newtonsoft.Json.JsonSerializer().Deserialize<WLID_FolderItem>(jr);
m_fileidCache[remotename] = nf.id;
}
}
}
public void Delete(string remotename)
{
try
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
throw new FileMissingException();
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError, resp);
else
using (areq.GetResponseStream())
{ }
}
}
catch (System.Net.WebException wex)
{
if (wex.Response is System.Net.HttpWebResponse && ((System.Net.HttpWebResponse)wex.Response).StatusCode == System.Net.HttpStatusCode.NotFound)
throw new FileMissingException(wex);
else
throw;
}
}
public void Put(string remotename, System.IO.Stream stream)
{
try
{
System.Net.HttpWebRequest req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Http.Put;
req.ContentType = "application/octet-stream";
try { req.ContentLength = stream.Length; }
catch { }
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream s = areq.GetRequestStream())
Utility.Utility.CopyStream(stream, s);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
}
catch (System.Net.WebException wex)
{
//Convert to better exception
if (wex.Response as System.Net.HttpWebResponse != null)
if ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict || (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound)
throw new Interface.FolderMissingException(string.Format(Strings.WEBDAV.MissingFolderError, m_path, wex.Message), wex);
throw;
}
}
public void CreateFolder()
{
System.Net.FtpWebRequest req = CreateRequest("", true);
req.Method = System.Net.WebRequestMethods.Ftp.MakeDirectory;
req.KeepAlive = false;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
public void Delete(string remotename)
{
System.Net.FtpWebRequest req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Ftp.DeleteFile;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
public void Get(string remotename, System.IO.Stream output)
{
System.Net.FtpWebRequest req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Ftp.DownloadFile;
req.UseBinary = true;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.WebResponse resp = areq.GetResponse())
using (System.IO.Stream rs = areq.GetResponseStream())
Utility.Utility.CopyStream(rs, output, false);
}
public void Put(string remotename, System.IO.Stream input)
{
System.Net.FtpWebRequest req = null;
try
{
req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Ftp.UploadFile;
req.UseBinary = true;
long streamLen = -1;
try { streamLen = input.Length; }
catch {}
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream rs = areq.GetRequestStream())
Utility.Utility.CopyStream(input, rs, true);
if (m_listVerify)
{
List<IFileEntry> files = List(remotename);
StringBuilder sb = new StringBuilder();
foreach(IFileEntry fe in files)
if (fe.Name.Equals(remotename) || fe.Name.EndsWith("/" + remotename) || fe.Name.EndsWith("\\" + remotename))
{
if (fe.Size < 0 || streamLen < 0 || fe.Size == streamLen)
return;
throw new Exception(string.Format(Strings.FTPBackend.ListVerifySizeFailure, remotename, fe.Size, streamLen));
}
else
sb.AppendLine(fe.Name);
throw new Exception(string.Format(Strings.FTPBackend.ListVerifyFailure, remotename, sb.ToString()));
}
}
catch (System.Net.WebException wex)
{
if (req != null && wex.Response as System.Net.FtpWebResponse != null && (wex.Response as System.Net.FtpWebResponse).StatusCode == System.Net.FtpStatusCode.ActionNotTakenFileUnavailable)
throw new Interface.FolderMissingException(string.Format(Strings.FTPBackend.MissingFolderError, req.RequestUri.PathAndQuery, wex.Message), wex);
else
throw;
}
}
public void Get(string remotename, System.IO.Stream stream)
{
System.Net.HttpWebRequest req = CreateRequest(remotename);
req.Method = System.Net.WebRequestMethods.Http.Get;
try
{
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
using (System.IO.Stream s = areq.GetResponseStream())
Utility.Utility.CopyStream(s, stream);
}
}
catch (System.Net.WebException wex)
{
if (wex.Response as System.Net.HttpWebResponse != null)
{
if ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict)
throw new Interface.FolderMissingException(string.Format(Strings.WEBDAV.MissingFolderError, m_path, wex.Message), wex);
if
(
(wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound
&&
m_filenamelist != null
&&
m_filenamelist.Contains(remotename)
)
throw new Exception(string.Format(Strings.WEBDAV.SeenThenNotFoundError, m_path, remotename, System.IO.Path.GetExtension(remotename), wex.Message), wex);
}
throw;
}
}
public void CreateFolder()
{
System.Net.HttpWebRequest req = CreateRequest("");
req.Method = System.Net.WebRequestMethods.Http.MkCol;
req.KeepAlive = false;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
}
public void Delete(string remotename)
{
try
{
var id = GetFileID(remotename);
var url = string.Format("{0}/{1}?access_token={2}", WLID_SERVER, id, Library.Utility.Uri.UrlEncode(m_oauth.AccessToken));
var req = (HttpWebRequest)WebRequest.Create(url);
req.Method = "DELETE";
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
throw new FileMissingException();
if ((int)resp.StatusCode < 200 || (int)resp.StatusCode > 299)
throw new ProtocolViolationException(Strings.OneDrive.UnexpectedError(resp.StatusCode, resp.StatusDescription));
m_fileidCache.Remove(remotename);
}
}
catch (System.Net.WebException wex)
{
if (wex.Response is System.Net.HttpWebResponse && ((System.Net.HttpWebResponse)wex.Response).StatusCode == System.Net.HttpStatusCode.NotFound)
throw new FileMissingException(wex);
else
throw;
}
}
public void Delete(string remotename)
{
try
{
System.Net.HttpWebRequest req = CreateRequest(remotename);
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode == System.Net.HttpStatusCode.NotFound)
throw new FileMissingException();
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
}
}
catch (System.Net.WebException wex)
{
if (wex.Response is System.Net.HttpWebResponse && ((System.Net.HttpWebResponse)wex.Response).StatusCode == System.Net.HttpStatusCode.NotFound)
throw new FileMissingException(wex);
else
throw;
}
}
public List<IFileEntry> List()
{
List<IFileEntry> files = new List<IFileEntry>();
string extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString();
string markerUrl = "";
bool repeat;
do
{
System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
HttpWebRequest req = CreateRequest("", extraUrl + markerUrl);
try
{
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
using (System.IO.Stream s = areq.GetResponseStream())
doc.Load(s);
}
catch (WebException wex)
{
if (markerUrl == "") //Only check on first itteration
if (wex.Response is HttpWebResponse && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound)
throw new FolderMissingException(wex);
//Other error, just re-throw
throw;
}
System.Xml.XmlNodeList lst = doc.SelectNodes("container/object");
//Perhaps the folder does not exist?
//The response should be 404 from the server, but it is not :(
if (lst.Count == 0 && markerUrl == "") //Only on first itteration
{
try { CreateFolder(); }
catch { } //Ignore
}
string lastItemName = "";
foreach (System.Xml.XmlNode n in lst)
{
string name = n["name"].InnerText;
long size;
DateTime mod;
if (!long.TryParse(n["bytes"].InnerText, out size))
size = -1;
if (!DateTime.TryParse(n["last_modified"].InnerText, out mod))
mod = new DateTime();
lastItemName = name;
files.Add(new FileEntry(name, size, mod, mod));
}
repeat = lst.Count == ITEM_LIST_LIMIT;
if (repeat)
markerUrl = "&marker=" + System.Web.HttpUtility.UrlEncode(lastItemName);
} while (repeat);
return files;
}
public List<IFileEntry> List()
{
TahoeEl data;
try
{
var req = CreateRequest("", "t=json");
req.Method = System.Net.WebRequestMethods.Http.Get;
var areq = new Utility.AsyncHttpRequest(req);
using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
using (var rs = areq.GetResponseStream())
using (var sr = new System.IO.StreamReader(rs))
using (var jr = new Newtonsoft.Json.JsonTextReader(sr))
{
var jsr =new Newtonsoft.Json.JsonSerializer();
jsr.Converters.Add(new TahoeElConverter());
data = jsr.Deserialize<TahoeEl>(jr);
}
}
}
catch (System.Net.WebException wex)
{
//Convert to better exception
if (wex.Response as System.Net.HttpWebResponse != null)
if ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict || (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound)
throw new Interface.FolderMissingException(Strings.TahoeBackend.MissingFolderError(m_url, wex.Message), wex);
throw;
}
if (data == null || data.node == null || data.nodetype != "dirnode")
throw new Exception("Invalid folder listing response");
var files = new List<IFileEntry>();
foreach (var e in data.node.children)
{
if (e.Value == null || e.Value.node == null)
continue;
bool isDir = e.Value.nodetype == "dirnode";
bool isFile = e.Value.nodetype == "filenode";
if (!isDir && !isFile)
continue;
FileEntry fe = new FileEntry(e.Key);
fe.IsFolder = isDir;
if (e.Value.node.metadata != null && e.Value.node.metadata.tahoe != null)
fe.LastModification = Duplicati.Library.Utility.Utility.EPOCH + TimeSpan.FromSeconds(e.Value.node.metadata.tahoe.linkmotime);
if (isFile)
fe.Size = e.Value.node.size;
files.Add(fe);
}
return files;
}
public void Delete(string remotename)
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileDeleteError, null, WebExceptionStatus.ProtocolError , resp);
else
using (areq.GetResponseStream())
{ }
}
public void Get(string remotename, System.IO.Stream stream)
{
var req = CreateRequest(remotename, "");
req.Method = System.Net.WebRequestMethods.Http.Get;
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
{
int code = (int)resp.StatusCode;
if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
using (var s = areq.GetResponseStream())
Utility.Utility.CopyStream(s, stream, true, m_copybuffer);
}
}
public void CreateFolder()
{
HttpWebRequest createReq = CreateRequest("", "");
createReq.Method = "PUT";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(createReq);
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
{ }
}
public void Delete(string remotename)
{
try
{
System.Net.HttpWebRequest req = CreateRequest(remotename, "");
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
catch (System.Net.WebException wex)
{
if (wex.Response is System.Net.HttpWebResponse && ((System.Net.HttpWebResponse)wex.Response).StatusCode == System.Net.HttpStatusCode.NotFound)
throw new FileMissingException(wex);
else
throw;
}
}
public void Get(string remotename, System.IO.Stream stream)
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "GET";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (WebResponse resp = areq.GetResponse())
using (System.IO.Stream s = areq.GetResponseStream())
using (MD5CalculatingStream mds = new MD5CalculatingStream(s))
{
string md5Hash = resp.Headers["ETag"];
Utility.Utility.CopyStream(mds, stream);
if (mds.GetFinalHashString().ToLower() != md5Hash.ToLower())
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
public void Get(string remotename, System.IO.Stream stream)
{
var id = GetFileID(remotename);
var url = string.Format("{0}/{1}/content?access_token={2}", WLID_SERVER, id, Library.Utility.Uri.UrlEncode(m_oauth.AccessToken));
var req = (HttpWebRequest)WebRequest.Create(url);
req.UserAgent = USER_AGENT;
var areq = new Utility.AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var rs = areq.GetResponseStream())
Utility.Utility.CopyStream(rs, stream, true, m_copybuffer);
}
public void Put(string remotename, System.IO.Stream stream)
{
HttpWebRequest req = CreateRequest("/" + remotename, "");
req.Method = "PUT";
req.ContentType = "application/octet-stream";
try { req.ContentLength = stream.Length; }
catch { }
//If we can pre-calculate the MD5 hash before transmission, do so
/*if (stream.CanSeek)
{
System.Security.Cryptography.MD5 md5 = System.Security.Cryptography.MD5.Create();
req.Headers["ETag"] = Core.Utility.ByteArrayAsHexString(md5.ComputeHash(stream)).ToLower();
stream.Seek(0, System.IO.SeekOrigin.Begin);
using (System.IO.Stream s = req.GetRequestStream())
Core.Utility.CopyStream(stream, s);
//Reset the timeout to the default value of 100 seconds to
// avoid blocking the GetResponse() call
req.Timeout = 100000;
//The server handles the eTag verification for us, and gives an error if the hash was a mismatch
using (HttpWebResponse resp = (HttpWebResponse)req.GetResponse())
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
}
else //Otherwise use a client-side calculation
*/
//TODO: We cannot use the local MD5 calculation, because that could involve a throttled read,
// and may invoke various events
{
string fileHash = null;
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.IO.Stream s = areq.GetRequestStream())
using (MD5CalculatingStream mds = new MD5CalculatingStream(s))
{
Utility.Utility.CopyStream(stream, mds);
fileHash = mds.GetFinalHashString();
}
string md5Hash = null;
//We need to verify the eTag locally
try
{
using (HttpWebResponse resp = (HttpWebResponse)areq.GetResponse())
if ((int)resp.StatusCode >= 300)
throw new WebException(Strings.CloudFiles.FileUploadError, null, WebExceptionStatus.ProtocolError, resp);
else
md5Hash = resp.Headers["ETag"];
}
catch (WebException wex)
{
//Catch 404 and turn it into a FolderNotFound error
if (wex.Response is HttpWebResponse && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound)
throw new FolderMissingException(wex);
//Other error, just re-throw
throw;
}
if (md5Hash == null || md5Hash.ToLower() != fileHash.ToLower())
{
//Remove the broken file
try { Delete(remotename); }
catch { }
throw new Exception(Strings.CloudFiles.ETagVerificationError);
}
}
}
public List<IFileEntry> List(string filename)
{
System.Net.FtpWebRequest req = CreateRequest(filename);
req.Method = System.Net.WebRequestMethods.Ftp.ListDirectoryDetails;
req.UseBinary = false;
try
{
List<IFileEntry> lst = new List<IFileEntry>();
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (System.Net.WebResponse resp = areq.GetResponse())
using (System.IO.Stream rs = areq.GetResponseStream())
using (System.IO.StreamReader sr = new System.IO.StreamReader(new StreamReadHelper(rs)))
{
string line;
while ((line = sr.ReadLine()) != null)
{
FileEntry f = ParseLine(line);
if (f != null)
lst.Add(f);
}
}
return lst;
}
catch (System.Net.WebException wex)
{
if (wex.Response as System.Net.FtpWebResponse != null && (wex.Response as System.Net.FtpWebResponse).StatusCode == System.Net.FtpStatusCode.ActionNotTakenFileUnavailable)
throw new Interface.FolderMissingException(string.Format(Strings.FTPBackend.MissingFolderError, req.RequestUri.PathAndQuery, wex.Message), wex);
else
throw;
}
}
public void Delete(string remotename)
{
System.Net.HttpWebRequest req = CreateRequest(remotename, "");
req.Method = "DELETE";
Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
using (areq.GetResponse())
{ }
}
