public void Dispose_InnerContent_InnerContentDisposed()
{
var content = new MultipartContent();
var innerContent = new MockContent();
content.Add(innerContent);
content.Dispose();
Assert.Equal(1, innerContent.DisposeCount);
content.Dispose();
// Inner content is discarded after first dispose.
Assert.Equal(1, innerContent.DisposeCount);
}
public void Dispose_NestedContent_NestedContentDisposed()
{
var outer = new MultipartContent();
var inner = new MultipartContent();
outer.Add(inner);
var mock = new MockContent();
inner.Add(mock);
outer.Dispose();
Assert.Equal(1, mock.DisposeCount);
outer.Dispose();
// Inner content is discarded after first dispose.
Assert.Equal(1, mock.DisposeCount);
}
internal void SendAsyncMultipartRequest(HttpMethod method, String relativePath, MultipartContent multiPartEntity, RemoteRequestCompletionBlock completionHandler)
{
Uri url = null;
try {
var urlStr = BuildRelativeURLString(relativePath);
url = new Uri(urlStr);
} catch (UriFormatException e) {
throw new ArgumentException("Invalid URI format.", e);
}
var message = new HttpRequestMessage(method, url);
message.Content = multiPartEntity;
message.Headers.Add("Accept", "*/*");
var client = clientFactory.GetHttpClient(false);
var authHeader = AuthUtils.GetAuthenticationHeaderValue(Authenticator, message.RequestUri);
if (authHeader != null) {
client.DefaultRequestHeaders.Authorization = authHeader;
}
client.SendAsync(message, CancellationTokenSource.Token).ContinueWith(response=>
{
multiPartEntity.Dispose();
if (response.Status != TaskStatus.RanToCompletion)
{
Log.E(TAG, "SendAsyncRequest did not run to completion.", response.Exception);
client.Dispose();
return null;
}
if ((Int32)response.Result.StatusCode > 300) {
LastError = new HttpResponseException(response.Result.StatusCode);
Log.E(TAG, "Server returned HTTP Error", LastError);
client.Dispose();
return null;
}
return response.Result.Content.ReadAsStreamAsync();
}, CancellationTokenSource.Token).ContinueWith(response=>
{
try {
var hasEmptyResult = response.Result == null || response.Result.Result == null || response.Result.Result.Length == 0;
if (response.Status != TaskStatus.RanToCompletion) {
Log.E (TAG, "SendAsyncRequest did not run to completion.", response.Exception);
} else if (hasEmptyResult) {
Log.E (TAG, "Server returned an empty response.", response.Exception ?? LastError);
}
if (completionHandler != null) {
object fullBody = null;
if (!hasEmptyResult) {
var mapper = Manager.GetObjectMapper();
fullBody = mapper.ReadValue<Object> (response.Result.Result);
}
completionHandler (fullBody, response.Exception);
}
} finally {
client.Dispose();
}
}, CancellationTokenSource.Token);
}
public void Dispose_Empty_Sucess()
{
var content = new MultipartContent();
content.Dispose();
}
internal void SendAsyncMultipartRequest(HttpMethod method, string relativePath, MultipartContent multiPartEntity, RemoteRequestCompletionBlock completionHandler)
{
Uri url = null;
try {
url = _baseUrl.Append(relativePath);
} catch (UriFormatException) {
Log.To.Sync.E(Tag, "Invalid path received for request: {0}, throwing...",
new SecureLogString(relativePath, LogMessageSensitivity.PotentiallyInsecure));
throw new ArgumentException("Invalid path", "relativePath");
}
var message = new HttpRequestMessage(method, url);
message.Content = multiPartEntity;
message.Headers.Add("Accept", "*/*");
var client = default(CouchbaseLiteHttpClient);
if (!_client.AcquireFor(TimeSpan.FromSeconds(1), out client))
{
Log.To.Sync.I(Tag, "Client is disposed, aborting request to {0}", new SecureLogString(relativePath, LogMessageSensitivity.PotentiallyInsecure));
return;
}
var _lastError = default(Exception);
client.Authenticator = Authenticator;
var t = client.SendAsync(message, _cancellationTokenSource.Token).ContinueWith(response =>
{
multiPartEntity.Dispose();
if (response.Status != TaskStatus.RanToCompletion)
{
_lastError = response.Exception;
Log.To.Sync.W(Tag, "SendAsyncRequest did not run to completion, returning null...");
return(Task.FromResult((Stream)null));
}
if ((int)response.Result.StatusCode > 300)
{
_lastError = new HttpResponseException(response.Result.StatusCode);
Log.To.Sync.W(Tag, "Server returned HTTP Error, returning null...");
return(Task.FromResult((Stream)null));
}
return(response.Result.Content.ReadAsStreamAsync());
}, _cancellationTokenSource.Token).ContinueWith(response =>
{
try {
var hasEmptyResult = response.Result == null || response.Result.Result == null || response.Result.Result.Length == 0;
if (response.Status != TaskStatus.RanToCompletion)
{
Log.To.Sync.W(Tag, "SendAsyncRequest phase two did not run to completion, continuing...");
}
else if (hasEmptyResult)
{
Log.To.Sync.W(Tag, "Server returned an empty response, continuing...");
}
if (completionHandler != null)
{
object fullBody = null;
if (!hasEmptyResult)
{
var mapper = Manager.GetObjectMapper();
fullBody = mapper.ReadValue <Object>(response.Result.Result);
}
completionHandler(fullBody, response.Exception ?? _lastError);
}
} finally {
Task dummy;
_requests.TryRemove(message, out dummy);
}
}, _cancellationTokenSource.Token);
_requests.TryAdd(message, t);
}
public override async Task ExecuteResultAsync(IWebDavResponse response, CancellationToken ct)
{
await base.ExecuteResultAsync(response, ct).ConfigureAwait(false);
response.Headers["Accept-Ranges"] = new[] { "bytes" };
var properties = await _document.GetProperties(response.Dispatcher).ToList(ct).ConfigureAwait(false);
var etagProperty = properties.OfType <GetETagProperty>().FirstOrDefault();
if (etagProperty != null)
{
var propValue = await etagProperty.GetValueAsync(ct).ConfigureAwait(false);
response.Headers["ETag"] = new[] { propValue.ToString() };
}
if (!_returnFile)
{
var lastModifiedProp = properties.OfType <LastModifiedProperty>().FirstOrDefault();
if (lastModifiedProp != null)
{
var propValue = await lastModifiedProp.GetValueAsync(ct).ConfigureAwait(false);
response.Headers["Last-Modified"] = new[] { propValue.ToString("R") };
}
return;
}
var views = new List <StreamView>();
try
{
foreach (var rangeItem in _rangeItems)
{
var baseStream = await _document.OpenReadAsync(ct).ConfigureAwait(false);
var streamView = await StreamView
.CreateAsync(baseStream, rangeItem.From, rangeItem.Length, ct)
.ConfigureAwait(false);
views.Add(streamView);
}
string contentType;
var contentTypeProp = properties.OfType <GetContentTypeProperty>().FirstOrDefault();
if (contentTypeProp != null)
{
contentType = await contentTypeProp.GetValueAsync(ct).ConfigureAwait(false);
}
else
{
contentType = MimeTypesMap.DefaultMimeType;
}
HttpContent content;
if (_rangeItems.Count == 1)
{
// No multipart content
var rangeItem = _rangeItems.Single();
var streamView = views.Single();
content = new StreamContent(streamView);
try
{
content.Headers.ContentRange = new ContentRangeHeaderValue(rangeItem.From, rangeItem.To, _document.Length);
content.Headers.ContentLength = rangeItem.Length;
}
catch
{
content.Dispose();
throw;
}
content.Headers.ContentType = MediaTypeHeaderValue.Parse(contentType);
}
else
{
// Multipart content
var multipart = new MultipartContent("byteranges");
try
{
var index = 0;
foreach (var rangeItem in _rangeItems)
{
var streamView = views[index++];
var partContent = new StreamContent(streamView);
partContent.Headers.ContentRange = new ContentRangeHeaderValue(rangeItem.From, rangeItem.To, _document.Length);
partContent.Headers.ContentType = MediaTypeHeaderValue.Parse(contentType);
partContent.Headers.ContentLength = rangeItem.Length;
multipart.Add(partContent);
}
}
catch
{
multipart.Dispose();
throw;
}
content = multipart;
}
using (content)
{
await SetPropertiesToContentHeaderAsync(content, properties, ct)
.ConfigureAwait(false);
foreach (var header in content.Headers)
{
response.Headers.Add(header.Key, header.Value.ToArray());
}
await content.CopyToAsync(response.Body).ConfigureAwait(false);
}
}
finally
{
foreach (var streamView in views)
{
streamView.Dispose();
}
}
}
public void Dispose_Empty_Sucess()
{
var content = new MultipartContent();
content.Dispose();
}
public async Task stowAsMultiPart(List <RoutedItem> batch, int taskID, LifeImageCloudConnection Connection, IHttpManager httpManager)
{
Throw.IfNull(Connection);
Throw.IfNull(httpManager);
var taskInfo = $"task: {taskID} connection: {Connection.name}";
StreamContent streamContent = null;
MultipartContent content = null;
HttpResponseMessage response = null;
string testFile = null;
var firstRecord = batch.First();
var httpClient = _liteHttpClient.GetClient(Connection);
try
{
var stopWatch = new Stopwatch();
stopWatch.Start();
//set the URL
//string stowURL = Connection.URL + "/api/agent/v1/stow/studies";
string stowURL = Connection.URL + CloudAgentConstants.StowStudies;
_logger.Log(LogLevel.Debug, $"{taskInfo} stowURL: {stowURL}");
// generate guid for boundary...boundaries cannot be accidentally found in the content
var boundary = Guid.NewGuid();
_logger.Log(LogLevel.Debug, $"{taskInfo} boundary: {boundary}");
// create the content
content = new MultipartContent("related", boundary.ToString());
//add the sharing headers
List <string> shareHeader = new List <string>();
if (Connection.shareDestinations != null)
{
foreach (var connectionSet in firstRecord.toConnections.FindAll(e =>
e.connectionName.Equals(Connection.name)))
{
if (connectionSet.shareDestinations != null)
{
foreach (var shareDestination in connectionSet.shareDestinations)
{
shareHeader.Add(shareDestination.boxUuid);
_logger.Log(LogLevel.Debug,
$"{taskInfo} sharing to: {shareDestination.boxId} {shareDestination.boxName} {shareDestination.groupId} {shareDestination.groupName} {shareDestination.organizationName} {shareDestination.publishableBoxType}");
}
}
}
}
content.Headers.Add("X-Li-Destination", shareHeader);
long fileSize = 0;
var dir = _profileStorage.Current.tempPath + Path.DirectorySeparatorChar + Connection.name + Path.DirectorySeparatorChar + Constants.Dirs.ToCloud;
Directory.CreateDirectory(dir);
testFile = dir + Path.DirectorySeparatorChar + Guid.NewGuid() + ".gz";
using (FileStream compressedFileStream = File.Create(testFile))
{
using GZipStream compressionStream = new GZipStream(compressedFileStream, CompressionMode.Compress);
foreach (var routedItem in batch)
{
if (File.Exists(routedItem.sourceFileName))
{
routedItem.stream = File.OpenRead(routedItem.sourceFileName);
if (Connection.CalcCompressionStats)
{
routedItem.stream.CopyTo(compressionStream);
}
fileSize += routedItem.length;
streamContent = new StreamContent(routedItem.stream);
streamContent.Headers.ContentDisposition =
new ContentDispositionHeaderValue("attachment")
{
FileName = routedItem.sourceFileName
};
content.Add(streamContent);
//shb 2017-11-01 streamcontent (uncompressed) is inside content (can be compressed below) server is complaining so will comment out
//streamContent.Headers.Add("Content-Transfer-Encoding", "gzip");
//shb 2017-11-10 shb added content-type header to solve
//controller.DicomRSControllerBase: Content-Encoding header has value null !!!
//controller.StowRSController: Unable to process part 1 no content-type parameter received
streamContent.Headers.Add("content-type", "application/dicom");
}
else
{
_logger.Log(LogLevel.Error,
$"{taskInfo} {routedItem.sourceFileName} no longer exists. Increase tempFileRetentionHours for heavy transfer backlogs that may take hours!!");
}
}
}
if (Connection.CalcCompressionStats)
{
FileInfo info = new FileInfo(testFile);
_logger.Log(LogLevel.Information, $"{taskInfo} orgSize: {fileSize} compressedSize: {info.Length} reduction: {(fileSize == 0 ? 0 : (fileSize * 1.0 - info.Length) / (fileSize) * 100)}%");
}
// issue the POST
Task <HttpResponseMessage> task;
if (firstRecord.Compress == true)
{
var compressedContent = new CompressedContent(content, "gzip");
_logger.Log(LogLevel.Debug, $"{taskInfo} compressedContent.Headers {compressedContent.Headers} ");
var cookies = _liteHttpClient.GetCookies(stowURL);
_logger.LogCookies(cookies, taskInfo);
task = httpClient.PostAsync(stowURL, compressedContent, _taskManager.cts.Token);
}
else
{
_logger.Log(LogLevel.Debug, $"{taskInfo} will send content.Headers {content.Headers}");
var cookies = _liteHttpClient.GetCookies(stowURL);
_logger.LogCookies(cookies, taskInfo);
task = httpClient.PostAsync(stowURL, content, _taskManager.cts.Token);
}
response = await task;
// output the result
_logger.LogHttpResponseAndHeaders(response, taskInfo);
_logger.Log(LogLevel.Debug, $"{taskInfo} response.Content.ReadAsStringAsync(): {await response.Content.ReadAsStringAsync()}");
stopWatch.Stop();
_logger.Log(LogLevel.Information, $"{taskInfo} elapsed: {stopWatch.Elapsed} size: {fileSize} rate: {(float)fileSize / stopWatch.Elapsed.TotalMilliseconds * 1000 / 1000000} MB/s");
if (!(response.StatusCode == HttpStatusCode.OK ||
response.StatusCode == HttpStatusCode.Accepted))
{
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
httpManager.loginNeeded = true;
}
_logger.Log(LogLevel.Warning,
$"stow of {firstRecord.sourceFileName} and others in batch failed with {response.StatusCode}");
_liteHttpClient.DumpHttpClientDetails();
}
else
{
//dequeue the work, we're done!
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
foreach (var ri in batch)
{
_routedItemManager.Init(ri);
_routedItemManager.Dequeue(Connection, Connection.toCloud, nameof(Connection.toCloud));
}
}
//delete the compression test file
File.Delete(testFile);
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"{taskInfo} Task was canceled.");
}
catch (HttpRequestException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} Exception: {e.Message} {e.StackTrace}");
if (e.InnerException != null)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} Inner Exception: {e.InnerException}");
}
_liteHttpClient.DumpHttpClientDetails();
}
catch (FileNotFoundException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} {e.Message} {e.StackTrace}");
}
catch (IOException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} {e.Message} {e.StackTrace}");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
_liteHttpClient.DumpHttpClientDetails();
}
finally
{
try
{
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
File.Delete(testFile);
_taskManager.Stop($"{Connection.name}.Stow");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
}
}
}
public void Dispose_NestedContent_NestedContentDisposed()
{
var outer = new MultipartContent();
var inner = new MultipartContent();
outer.Add(inner);
var mock = new MockContent();
inner.Add(mock);
outer.Dispose();
Assert.Equal(1, mock.DisposeCount);
outer.Dispose();
// Inner content is discarded after first dispose.
Assert.Equal(1, mock.DisposeCount);
}
public void Dispose_InnerContent_InnerContentDisposed()
{
var content = new MultipartContent();
var innerContent = new MockContent();
content.Add(innerContent);
content.Dispose();
Assert.Equal(1, innerContent.DisposeCount);
content.Dispose();
// Inner content is discarded after first dispose.
Assert.Equal(1, innerContent.DisposeCount);
}
internal void SendAsyncMultipartRequest(HttpMethod method, string relativePath, MultipartContent multiPartEntity, RemoteRequestCompletionBlock completionHandler)
{
Uri url = null;
try {
url = _baseUrl.Append(relativePath);
} catch(UriFormatException) {
Log.To.Sync.E(Tag, "Invalid path received for request: {0}, throwing...",
new SecureLogString(relativePath, LogMessageSensitivity.PotentiallyInsecure));
throw new ArgumentException("Invalid path", "relativePath");
}
var message = new HttpRequestMessage(method, url);
message.Content = multiPartEntity;
message.Headers.Add("Accept", "*/*");
var client = default(CouchbaseLiteHttpClient);
if(!_client.AcquireFor(TimeSpan.FromSeconds(1), out client)) {
Log.To.Sync.I(Tag, "Client is disposed, aborting request to {0}", new SecureLogString(relativePath, LogMessageSensitivity.PotentiallyInsecure));
return;
}
var _lastError = default(Exception);
client.Authenticator = Authenticator;
var t = client.SendAsync(message, _cancellationTokenSource.Token).ContinueWith(response =>
{
multiPartEntity.Dispose();
if(response.Status != TaskStatus.RanToCompletion) {
_lastError = response.Exception;
Log.To.Sync.W(Tag, "SendAsyncRequest did not run to completion, returning null...");
return Task.FromResult((Stream)null);
}
if((int)response.Result.StatusCode > 300) {
_lastError = new HttpResponseException(response.Result.StatusCode);
Log.To.Sync.W(Tag, "Server returned HTTP Error, returning null...");
return Task.FromResult((Stream)null);
}
return response.Result.Content.ReadAsStreamAsync();
}, _cancellationTokenSource.Token).ContinueWith(response =>
{
try {
var hasEmptyResult = response.Result == null || response.Result.Result == null || response.Result.Result.Length == 0;
if(response.Status != TaskStatus.RanToCompletion) {
Log.To.Sync.W(Tag, "SendAsyncRequest phase two did not run to completion, continuing...");
} else if(hasEmptyResult) {
Log.To.Sync.W(Tag, "Server returned an empty response, continuing...");
}
if(completionHandler != null) {
object fullBody = null;
if(!hasEmptyResult) {
var mapper = Manager.GetObjectMapper();
fullBody = mapper.ReadValue<Object>(response.Result.Result);
}
completionHandler(fullBody, response.Exception ?? _lastError);
}
} finally {
Task dummy;
_requests.TryRemove(message, out dummy);
}
}, _cancellationTokenSource.Token);
_requests.TryAdd(message, t);
}
public async Task putHL7(RoutedItem routedItem, int taskID, LifeImageCloudConnection connection, IHttpManager httpManager)
{
var Connection = connection;
var httpClient = _liteHttpClient.GetClient(connection);
var taskInfo = $"task: {taskID} connection: {Connection.name}";
MultipartContent content = null;
StreamContent streamContent = null;
HttpResponseMessage response = null;
try
{
if (!File.Exists(routedItem.sourceFileName))
{
routedItem.Error = "File Not Found";
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.toCloud, nameof(Connection.toCloud), error: true);
return;
}
var stopWatch = new Stopwatch();
stopWatch.Start();
//set theConnection.URL http://localhost:8080/universal-inbox/api/agent/v1/hl7-upload
//string putHL7URL = Connection.URL + "/api/agent/v1/hl7-upload?connectionName=" + routedItem.fromConnection;
string putHL7URL = Connection.URL + CloudAgentConstants.GetPutHl7Url(routedItem.fromConnection);
_logger.Log(LogLevel.Debug, $"{taskInfo} putHL7URL: {putHL7URL}");
//generate guid for boundary...boundaries cannot be accidentally found in the content
var boundary = Guid.NewGuid();
_logger.Log(LogLevel.Debug, $"{taskInfo} boundary: {boundary}");
// create the content
content = new MultipartContent("related", boundary.ToString());
//add the sharing headers
List <string> shareHeader = new List <string>();
if (Connection.shareDestinations != null)
{
foreach (var connectionSet in routedItem.toConnections.FindAll(e =>
e.connectionName.Equals(Connection.name)))
{
if (connectionSet.shareDestinations != null)
{
foreach (var shareDestination in connectionSet.shareDestinations)
{
shareHeader.Add(shareDestination.boxUuid);
}
}
}
}
content.Headers.Add("X-Li-Destination", shareHeader);
//
//var fileSize = routedItem.stream.Length;
var fileSize = new FileInfo(routedItem.sourceFileName).Length;
//var streamContent = new StreamContent(routedItem.stream);
streamContent = new StreamContent(File.OpenRead(routedItem.sourceFileName));
streamContent.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
// FileName = filename
FileName = routedItem.sourceFileName
};
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
//streamContent.Headers.Add("Content-Transfer-Encoding", "gzip");
content.Add(streamContent);
// issue the POST
Task <HttpResponseMessage> task;
var cookies = _liteHttpClient.GetCookies(putHL7URL);
_logger.LogCookies(cookies, taskInfo);
if (routedItem.Compress == true)
{
task = httpClient.PostAsync(putHL7URL, new CompressedContent(content, "gzip"), _taskManager.cts.Token);
}
else
{
task = httpClient.PostAsync(putHL7URL, content, _taskManager.cts.Token);
}
response = await task;
// output the result
_logger.LogHttpResponseAndHeaders(response, taskInfo);
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
httpManager.loginNeeded = true;
}
_logger.Log(LogLevel.Debug,
$"{taskInfo} response.Content.ReadAsStringAsync(): {await response.Content.ReadAsStringAsync()}");
// convert from stream to JSON
//var serializer = new DataContractJsonSerializer(typeof(LoginJSON));
//var loginJSON = serializer.ReadObject(await response.Content.ReadAsStreamAsync()) as LoginJSON;
stopWatch.Stop();
_logger.Log(LogLevel.Information,
$"{taskInfo} elapsed: {stopWatch.Elapsed} size: {fileSize} rate: {(float)fileSize / stopWatch.Elapsed.TotalMilliseconds * 1000 / 1000000} MB/s");
//dequeue the work, we're done!
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.toCloud, nameof(Connection.toCloud));
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"{taskInfo} Task was canceled.");
}
catch (HttpRequestException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} Exception: {e.Message} {e.StackTrace}");
if (e.InnerException != null)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} Inner Exception: {e.InnerException}");
}
_liteHttpClient.DumpHttpClientDetails();
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
_liteHttpClient.DumpHttpClientDetails();
}
finally
{
try
{
_taskManager.Stop($"{Connection.name}.putHL7");
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
}
}
}
/// <summary>
/// Store takes a batch of RoutedItem, all going to the same share destination, and uploads them as a single operation. This is done to solve the many small files problem.Larger files can go individually.
/// </summary>
/// <param name="batch"></param>
/// <param name="taskID"></param>
/// <param name="connection"></param>
/// <returns></returns>
public async Task store(List <RoutedItem> batch, int taskID, LITEConnection connection)
{
Connection = connection;
var taskInfo = $"task: {taskID} connection: {Connection.name}";
StreamContent streamContent = null;
MultipartContent content = null;
HttpResponseMessage response = null;
string testFile = null;
var firstRecord = batch.First();
try
{
var stopWatch = new Stopwatch();
stopWatch.Start();
//set the URL
//string resourceURL = Connection.URL + "/api/File";
string resourceURL = Connection.URL + FileAgentConstants.BaseUrl;
_logger.Log(LogLevel.Debug, $"{taskInfo} URL: {resourceURL}");
// generate guid for boundary...boundaries cannot be accidentally found in the content
var boundary = Guid.NewGuid();
_logger.Log(LogLevel.Debug, $"{taskInfo} boundary: {boundary}");
// create the content
content = new MultipartContent("related", boundary.ToString());
//add the sharing headers
List <string> shareHeader = new List <string>();
if (Connection.shareDestinations != null)
{
foreach (var connectionSet in firstRecord.toConnections.FindAll(e => e.connectionName.Equals(Connection.name)))
{
if (connectionSet.shareDestinations != null)
{
foreach (var shareDestination in connectionSet.shareDestinations)
{
shareHeader.Add(shareDestination.boxUuid);
_logger.Log(LogLevel.Debug, $"{taskInfo} sharing to: {shareDestination.boxId} {shareDestination.boxName} {shareDestination.groupId} {shareDestination.groupName} {shareDestination.organizationName} {shareDestination.publishableBoxType}");
}
}
}
}
content.Headers.Add("X-Li-Destination", shareHeader);
long fileSize = 0;
var profile = _profileStorage.Current;
var dir = profile.tempPath + Path.DirectorySeparatorChar + Connection.name + Path.DirectorySeparatorChar + "toEGS";
Directory.CreateDirectory(dir);
testFile = dir + Path.DirectorySeparatorChar + Guid.NewGuid() + ".gz";
using (FileStream compressedFileStream = File.Create(testFile))
{
using GZipStream compressionStream = new GZipStream(compressedFileStream, CompressionMode.Compress);
foreach (var routedItem in batch)
{
if (File.Exists(routedItem.sourceFileName))
{
routedItem.stream = File.OpenRead(routedItem.sourceFileName);
if (Connection.calcCompressionStats)
{
routedItem.stream.CopyTo(compressionStream);
}
fileSize += routedItem.length;
streamContent = new StreamContent(routedItem.stream);
streamContent.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = routedItem.sourceFileName
};
content.Add(streamContent);
streamContent.Headers.Add("content-type", "application/octet-stream");
}
else
{
_logger.Log(LogLevel.Error, $"{taskInfo} {routedItem.sourceFileName} no longer exists. Increase tempFileRetentionHours for heavy transfer backlogs that may take hours!!");
routedItem.Error = "File no longer exists";
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.toEGS, nameof(Connection.toEGS), error: true);
}
}
}
if (Connection.calcCompressionStats)
{
FileInfo info = new FileInfo(testFile);
_logger.Log(LogLevel.Information, $"{taskInfo} orgSize: {fileSize} compressedSize: {info.Length} reduction: {(fileSize == 0 ? 0 : (fileSize * 1.0 - info.Length) / (fileSize) * 100)}%");
}
// issue the POST
Task <HttpResponseMessage> task;
var httpClient = _liteHttpClient.GetClient(connection);
if (firstRecord.Compress == true)
{
var compressedContent = new CompressedContent(content, "gzip");
_logger.Log(LogLevel.Debug, $"{taskInfo} compressedContent.Headers {compressedContent.Headers} ");
compressedContent.Headers.Remove("Content-Encoding");
var cookies = _liteHttpClient.GetCookies(resourceURL);
_logger.LogCookies(cookies, taskInfo);
task = httpClient.PostAsync(resourceURL, compressedContent);
}
else
{
_logger.Log(LogLevel.Debug, $"{taskInfo} will send content.Headers {content.Headers}");
var cookies = _liteHttpClient.GetCookies(resourceURL);
_logger.LogCookies(cookies, taskInfo);
task = httpClient.PostAsync(resourceURL, content);
}
response = await task;
// output the result
_logger.LogHttpResponseAndHeaders(response, taskInfo);
_logger.Log(LogLevel.Debug, $"{taskInfo} response.Content.ReadAsStringAsync(): {await response.Content.ReadAsStringAsync()}");
stopWatch.Stop();
_logger.Log(LogLevel.Information, $"{taskInfo} elapsed: {stopWatch.Elapsed} size: {fileSize} rate: {(float)fileSize / stopWatch.Elapsed.TotalMilliseconds * 1000 / 1000000} MB/s");
switch (response.StatusCode)
{
case HttpStatusCode.Created:
//dequeue the work, we're done!
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
foreach (var ri in batch)
{
_routedItemManager.Init(ri);
_routedItemManager.Dequeue(Connection, Connection.toEGS, nameof(Connection.toEGS));
}
//let EGGS know it's available, or when we convert udt to .net core then perhaps push so no open socket required on client.
//await SendToAllHubs(LITEServicePoint, batch);
break;
case HttpStatusCode.UnprocessableEntity:
//dequeue the work, we're done!
_logger.Log(LogLevel.Warning, $"creation of {firstRecord.sourceFileName} and others in batch failed with {response.StatusCode}");
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
foreach (var ri in batch)
{
ri.Error = HttpStatusCode.UnprocessableEntity.ToString();
_routedItemManager.Init(ri);
_routedItemManager.Dequeue(Connection, Connection.toEGS, nameof(Connection.toEGS), error: true);
}
break;
default:
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
Connection.loginNeeded = true;
}
_logger.Log(LogLevel.Warning, $"creation of {firstRecord.sourceFileName} and others in batch failed with {response.StatusCode}");
_liteHttpClient.DumpHttpClientDetails();
break;
}
//delete the compression test file
File.Delete(testFile);
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (HttpRequestException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} Exception: {e.Message} {e.StackTrace}");
if (e.InnerException != null)
{
_logger.Log(LogLevel.Warning, $"Inner Exception: {e.InnerException}");
}
_liteHttpClient.DumpHttpClientDetails();
}
catch (FileNotFoundException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} {e.Message} {e.StackTrace}");
}
catch (IOException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} {e.Message} {e.StackTrace}");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
_liteHttpClient.DumpHttpClientDetails();
}
finally
{
try
{
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
File.Delete(testFile);
_taskManager.Stop($"{Connection.name}.Store");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
}
}
}
