private static void LogApiCallCounter(HttpRequestMessage request, string opName)
{
if (string.IsNullOrWhiteSpace(opName))
{
// OpName was not provided, so try to figure it out
if (request.RequestUri.Query.Contains("comp=list") &&
request.Method == HttpMethod.Get)
{
opName = "ListContainers";
}
}
if (opName == null)
{
string message = string.Format(
"Failed to determine counter op name from Uri {0}",
request.RequestUri);
#if DEBUG
throw new Exception(message);
#else
Logger.Warning(message);
return;
#endif
}
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(Constants.DimensionNames.ApiCallName, opName));
}
private static void LogApiCallCounter(HttpRequestMessage request)
{
// A backblaze API request should have the following formats:
// https://<domain>/b2api/v1/b2_authorize_account
// https://<domain>/b2api/v1/b2_upload_file/foo/bar
// We want to isolate the 'b2_' segment and extract the operation name
// Find the segment containing "b2api/"
var segments = request.RequestUri.Segments;
int idx = segments.IndexOf("b2api/", true);
if (idx < 0)
{
throw new Exception("Failed to locate 'b2api' segment in URI " + request.RequestUri);
}
// The op name will be 2 segments after the b2api/ segment. Ensure that it exists
if (segments.Length <= idx + 2)
{
throw new Exception("URI " + request.RequestUri + " is not formatted correctly (idx=" + idx + ")");
}
// Isoate the segment
string opNameSegment = segments[idx + 2];
// Ensure it starts with b2_ in case there is a strange call
Pre.Assert(opNameSegment.StartsWith("b2_"));
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(Constants.DimensionNames.ApiCallName, opNameSegment.Trim('/')));
}
public async Task CancelUploadSession(OneDriveUploadSession session)
{
// If the session is already cancelled, nothing to do
if (session.State == OneDriveFileUploadState.Cancelled)
{
return;
}
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(
Constants.DimensionNames.OperationName,
"CancelUploadSession"));
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Delete, session.UploadUrl);
HttpResponseMessage response = await this.SendOneDriveRequest(request).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NoContent)
{
session.State = OneDriveFileUploadState.Cancelled;
return;
}
Logger.Warning(
"Failed to cancel upload session for file {0} (ParenId={1})",
session.ItemName,
session.ParentId);
LogRequest(request, this.oneDriveHttpClient.BaseAddress, true);
LogResponse(response, true);
throw new OneDriveHttpException("Failed to cancel the upload session.", response.StatusCode);
}
// See https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
public async Task <OneDriveUploadSession> CreateUploadSession(string parentItemId, string name, long length)
{
if (string.IsNullOrWhiteSpace(parentItemId))
{
throw new ArgumentNullException(nameof(parentItemId));
}
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException(nameof(name));
}
// TODO: Check for the maximum file size limit
if (length <= 0)
{
throw new ArgumentOutOfRangeException(nameof(length));
}
if (this.uploadSessions.Any(s => s.ParentId == parentItemId && s.ItemName == name))
{
throw new InvalidOperationException("An upload session for this item already exists.");
}
HttpRequestMessage request = new HttpRequestMessage(
HttpMethod.Post,
string.Format("/v1.0/drive/items/{0}:/{1}:/upload.createSession", parentItemId, HttpUtility.UrlEncode(name)));
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(
Constants.DimensionNames.OperationName,
"CreateUploadSession"));
HttpResponseMessage response = await this.SendOneDriveRequest(request).ConfigureAwait(false);
JObject responseObject = await response.Content.ReadAsJObjectAsync().ConfigureAwait(false);
var newSession = new OneDriveUploadSession(
parentItemId,
name,
responseObject["uploadUrl"].Value <string>(),
responseObject["expirationDateTime"].Value <DateTime>(),
length);
Logger.Info(
"Created OneDrive upload session with parentItemId={0}, name={1}, expirationDateTime={2}",
parentItemId,
name,
newSession.ExpirationDateTime);
this.uploadSessions.Add(newSession);
return(newSession);
}
public async Task SendUploadFragment(OneDriveUploadSession uploadSession, byte[] fragmentBuffer, long offset)
{
switch (uploadSession.State)
{
case OneDriveFileUploadState.NotStarted:
uploadSession.State = OneDriveFileUploadState.InProgress;
break;
case OneDriveFileUploadState.Completed:
throw new OneDriveException("Cannot upload fragment to completed upload session.");
case OneDriveFileUploadState.Faulted:
throw new OneDriveException("Cannot upload fragment to faulted upload session.");
case OneDriveFileUploadState.Cancelled:
throw new OneDriveException("Cannot upload fragment to cancelled upload session.");
}
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Put, uploadSession.UploadUrl)
{
Content = new ByteArrayContent(fragmentBuffer)
};
request.Content.Headers.ContentLength = fragmentBuffer.LongLength;
request.Content.Headers.ContentRange = new ContentRangeHeaderValue(
offset,
offset + fragmentBuffer.Length - 1,
uploadSession.Length);
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(
Constants.DimensionNames.OperationName,
"SendUploadFragment"));
var response = await this.SendOneDriveRequest(request).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
uploadSession.State = OneDriveFileUploadState.Faulted;
}
if (response.StatusCode == HttpStatusCode.Created || response.StatusCode == HttpStatusCode.OK)
{
// 201 indicates that the upload is complete.
uploadSession.State = OneDriveFileUploadState.Completed;
uploadSession.Item = await response.Content.ReadAsJsonAsync <Item>().ConfigureAwait(false);
this.uploadSessions.Remove(uploadSession);
}
}
public async Task <Uri> GetDownloadUriForItem(string id)
{
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, "/v1.0/drive/items/" + id + "/content");
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(
Constants.DimensionNames.OperationName,
"GetDownloadUriForItem"));
var response = await this.SendOneDriveRequest(request, this.oneDriveHttpClientNoRedirect).ConfigureAwait(false);
return(response.Headers.Location);
}
private async Task <OneDriveResponse <T> > GetOneDriveItemSet <T>(string requestUri)
{
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(
Constants.DimensionNames.OperationName,
"GetItemSet"));
// Send the request to OneDrive and get the response.
var response = await this.SendOneDriveRequest(new HttpRequestMessage(HttpMethod.Get, requestUri)).ConfigureAwait(false);
// Request was successful. Read the content returned.
string content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
return(JsonConvert.DeserializeObject <OneDriveResponse <T> >(content));
}
public async Task <Item> CreateFolderAsync(ItemContainer parent, string name)
{
string requestUri;
// Build the request specific to the parent
if (parent.IsItem)
{
requestUri = string.Format("/v1.0/drive/items/{0}/children", parent.Item.Id);
}
else
{
requestUri = string.Format("/v1.0/drives/{0}/root/children", parent.Drive.Id);
}
Item newFolder = new Item
{
Name = name,
Folder = new FolderFacet()
};
string jsonContent = JsonConvert.SerializeObject(newFolder, new JsonSerializerSettings()
{
DefaultValueHandling = DefaultValueHandling.Ignore
});
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Post, requestUri);
request.Content = new StringContent(jsonContent, Encoding.UTF8, "application/json");
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(
Constants.DimensionNames.OperationName,
"CreateFolder"));
HttpResponseMessage response = await this.SendOneDriveRequest(request).ConfigureAwait(false);
return(await response.Content.ReadAsJsonAsync <Item>().ConfigureAwait(false));
}
public async Task <HttpResponseMessage> DownloadFileFragment(Uri downloadUri, int offset, int length)
{
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, downloadUri);
request.Headers.Range = new RangeHeaderValue(offset * length, ((offset + 1) * length) - 1);
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(
Constants.DimensionNames.OperationName,
"DownloadFileFragment"));
var response = await this.oneDriveHttpClient.SendAsync(request).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
throw new OneDriveHttpException(response.ReasonPhrase, response.StatusCode);
}
return(response);
}
public async Task <Item> CreateItem(ItemContainer parent, string name)
{
string uri = string.Format("/v1.0/drive/items/{0}:/{1}:/[email protected]=fail", parent.Item.Id, HttpUtility.UrlEncode(name));
HttpContent requestContent = new StringContent(string.Empty, Encoding.ASCII, "text/plain");
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Put, uri)
{
Content = requestContent
};
CounterManager.LogSyncJobCounter(
Constants.CounterNames.ApiCall,
1,
new CounterDimension(
Constants.DimensionNames.OperationName,
"CreateItem"));
var response = await this.SendOneDriveRequest(request).ConfigureAwait(false);
// Request was successful. Read the content returned.
string content = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
return(JsonConvert.DeserializeObject <Item>(content));
}
/// <summary>
/// Transfer data from the source stream to the destination stream. This method also performs the necessary
/// data transforms (throttling, hashing, encrypting, etc) as the data is streamed. This allows a the
/// source stream to only be read a single time while performing multiple operations on the data.
/// </summary>
/// <param name="sourceStream">The stream that the data is read from</param>
/// <param name="destinationStream">The stream that the data is written to</param>
/// <returns>(async) The result of the transfer</returns>
private async Task <TransferResult> TransferDataWithTransformsAsync(
Stream sourceStream,
Stream destinationStream)
{
SHA1Cng sha1 = null;
MD5Cng md5 = null;
try
{
// By default, we will compute the SHA1 and MD5 hashes of the file as it is streamed.
sha1 = new SHA1Cng();
md5 = new MD5Cng();
// Allocate the buffer that will be used to transfer the data. The source stream will be read one
// suffer-size as a time, then written to the destination.
byte[] buffer = new byte[TransferBufferSize];
long readTotal = 0;
long writtenTotal = 0;
while (true)
{
// If we are using a throttling manager, get the necessary number of tokens to
// transfer the data
if (this.throttlingManager != null)
{
int tokens = 0;
while (true)
{
// Get the number of tokens needed. We will require tokens equaling the number of
// bytes to be transferred. This is a non-blocking calling and will return between
// 0 and the number of requested tokens.
tokens += this.throttlingManager.GetTokens(TransferBufferSize - tokens);
if (tokens >= TransferBufferSize)
{
// We have enough tokens to transfer the buffer
break;
}
// We don't (yet) have enough tokens, so wait for a short duration and try again
await Task.Delay(10, this.cancellationToken).ConfigureAwait(false);
}
}
// Read data from the source adapter
int read = sourceStream.Read(buffer, 0, buffer.Length);
CounterManager.LogSyncJobCounter(
"SyncJob/BytesRead",
read);
// Increment the total number of bytes read from the source adapter
readTotal += read;
int bytesWritten;
if (read < buffer.Length)
{
// Compute the last part of the SHA1 and MD5 hashes (this finished the algorithm's work).
sha1.TransformFinalBlock(buffer, 0, read);
md5.TransformFinalBlock(buffer, 0, read);
if (this.encryptionManager != null)
{
bytesWritten = this.encryptionManager.TransformFinalBlock(buffer, 0, read);
}
else
{
destinationStream.Write(buffer, 0, read);
destinationStream.Flush();
bytesWritten = buffer.Length;
}
CounterManager.LogSyncJobCounter(
"SyncJob/BytesWritten",
read);
writtenTotal += bytesWritten;
// Increment the total number of bytes written to the desination adapter
this.bytesCompleted += read;
this.progressChanged(new CopyProgressInfo(this.bytesCompleted, this.updateInfo));
// Read the end of the stream
break;
}
// Pass the data through the required hashing algorithms.
sha1.TransformBlock(buffer, 0, read, buffer, 0);
md5.TransformBlock(buffer, 0, read, buffer, 0);
// Write the data to the destination adapter
if (this.encryptionManager != null)
{
bytesWritten = this.encryptionManager.TransformBlock(buffer, 0, read);
}
else
{
destinationStream.Write(buffer, 0, read);
bytesWritten = buffer.Length;
}
CounterManager.LogSyncJobCounter(
"SyncJob/BytesWritten",
read);
writtenTotal += bytesWritten;
// Increment the total number of bytes written to the desination adapter
this.bytesCompleted += read;
if (this.syncProgressUpdateStopwatch.ElapsedMilliseconds > 100)
{
this.progressChanged(new CopyProgressInfo(this.bytesCompleted, this.updateInfo));
// After reporting the number of bytes copied for this file, set back to 0 so that we are only
// reporting the number of bytes sync we last invoked the callback.
this.bytesCompleted = 0;
this.syncProgressUpdateStopwatch.Restart();
}
}
TransferResult result = new TransferResult
{
BytesRead = readTotal,
BytesWritten = writtenTotal
};
if (this.encryptionManager != null)
{
if (this.encryptionManager.Mode == EncryptionMode.Encrypt)
{
result.Sha1Hash = sha1.Hash;
result.Md5Hash = md5.Hash;
result.TransformedSha1Hash = this.encryptionManager.Sha1Hash;
result.TransformedMd5Hash = this.encryptionManager.Md5Hash;
}
else
{
result.TransformedSha1Hash = sha1.Hash;
result.TransformedMd5Hash = md5.Hash;
result.Sha1Hash = this.encryptionManager.Sha1Hash; // The SHA1 hash of the data written by the encryption manager
result.Md5Hash = this.encryptionManager.Md5Hash;
}
}
else
{
result.Sha1Hash = sha1.Hash;
result.Md5Hash = md5.Hash;
}
return(result);
}
finally
{
sha1?.Dispose();
md5?.Dispose();
}
}
