public async Task Upload(int taskID, LITEConnection connection, SemaphoreSlim toEGSSignal, ISendToAllHubsService sendToAllHubsService)
{
Connection = connection;
ToEGSSignal = toEGSSignal;
SendToAllHubsService = sendToAllHubsService;
var taskInfo = $"task: {taskID} connection: {Connection.name}";
_logger.Log(LogLevel.Debug, $"{taskInfo} Entering Upload");
try
{
await UploadImpl(taskID);
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (OperationCanceledException)
{
_logger.Log(LogLevel.Warning, $"Wait Operation Canceled. Exiting Upload");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
_logger.Log(LogLevel.Critical, $"{taskInfo} Exiting Upload");
}
finally
{
_taskManager.Stop($"{Connection.name}.Upload");
}
}
public void Purge(int taskID, LITEConnection connection)
{
Connection = connection;
var taskInfo = $"task: {taskID}";
var profile = _profileStorage.Current;
//purge temp
Directory.CreateDirectory(Connection.resourcePath);
_logger.Log(LogLevel.Debug, $"{taskInfo} Purging {Connection.resourcePath}");
var resources = _util.DirSearch(Connection.resourcePath, "*.*");
foreach (var file in resources)
{
try
{
PurgeFile(profile, file, taskInfo);
}
catch (Exception e)
{
_logger.LogFullException(e);
}
}
_util.CleanUpDirectory(Connection.resourcePath);
}
private async Task ProcessItem(int taskID, RoutedItem routedItem, LITEConnection connection, IHttpManager httpManager)
{
var taskInfo = $"task: {taskID} connection: {Connection.name}";
if (routedItem.lastAttempt == null || routedItem.lastAttempt >= DateTime.Now.AddMinutes(-connection.retryDelayMinutes)) //not attempted lately
{
await Task.CompletedTask;
return;
}
routedItem.attempts++;
if (routedItem.attempts > 1)
{
_logger.Log(LogLevel.Debug, $"{taskInfo} {routedItem.sourceFileName} second attempt.");
}
routedItem.lastAttempt = DateTime.Now;
if (routedItem.attempts > connection.maxAttempts)
{
_logger.Log(LogLevel.Warning, $"Resource {routedItem.resource} exceeded max attempts. Deleting item.");
if (await _deleteEGSResourceService.DeleteEGSResource(taskID, routedItem, connection, httpManager))
{
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(connection, connection.fromEGS, nameof(connection.fromEGS), error: false);
}
else
{
routedItem.Error = "Exceeded maxAttempts";
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(connection, connection.fromEGS, nameof(connection.fromEGS), error: true);
}
}
switch (Connection.protocol)
{
case Protocol.Http:
// while (!LITETask.CanStart($"{name}.DownloadViaHttp"))
// {
// await Task.Delay(LITE.profile.taskDelay).ConfigureAwait(false);
// }
var newTaskID = _taskManager.NewTaskID();
Task task = new Task(new Action(async() => await _downloadViaHttpService.DownloadViaHttp(newTaskID, routedItem, connection, httpManager)), _taskManager.cts.Token);
await _taskManager.Start(newTaskID, task, $"{Connection.name}.DownloadViaHttp", routedItem.resource, isLongRunning : false).ConfigureAwait(false);
//await DownloadViaHttp(newTaskID, ri).ConfigureAwait(false);
break;
// case Protocol.UDT:
// await DownloadViaUDTShell(remoteHostname, remotePort, $"{routedItem.box + "/" + routedItem.resource}", LITE.profile.tempPath + Path.DirectorySeparatorChar + "toScanner", taskID);
// break;
}
}
public async Task PresentAsResource(List <RoutedItem> batch, int taskID, LITEConnection connection, ISendToAllHubsService sendToAllHubs)
{
Connection = connection;
try
{
var path = Connection.resourcePath + Path.DirectorySeparatorChar + Connection.name + Path.DirectorySeparatorChar + "Resource" + Path.DirectorySeparatorChar + "batch" + Path.DirectorySeparatorChar + System.Guid.NewGuid();
Directory.CreateDirectory(path);
//Move all files in batch to separate folder
foreach (var ri in batch)
{
File.Move(ri.sourceFileName, path + Path.DirectorySeparatorChar + ri.sourceFileName.Substring(ri.sourceFileName.LastIndexOf(Path.DirectorySeparatorChar)));
ri.sourceFileName = path + Path.DirectorySeparatorChar + ri.sourceFileName.Substring(ri.sourceFileName.LastIndexOf(Path.DirectorySeparatorChar));
}
var file = path + ".zip";
//zip up files and meta
ZipFile.CreateFromDirectory(path, file);
//protect the file
//var protected file = Crypto.Protect(file)
//let EGGS know it's available, or when we convert udt to .net core then perhaps push so no open socket required on client.
await sendToAllHubs.SendToAllHubs(X509CertificateService.ServicePointName, file);
//Dequeue
foreach (var ri in batch)
{
_routedItemManager.Init(ri);
_routedItemManager.Dequeue(Connection, Connection.toEGS, nameof(Connection.toEGS), error: false);
}
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (Exception e)
{
_logger.LogFullException(e);
}
}
public async Task <bool> ping(LITEConnection connection, IHttpManager httpManager)
{
var taskInfo = $"connection: {Connection.name}";
var httpClient = _liteHttpClient.GetClient(connection);
try
{
if (httpClient != null)
{
try
{
//var task = httpClient.GetAsync(Connection.URL + "/api/LITE");
var task = httpClient.GetAsync(Connection.URL + LiteAgentConstants.BaseUrl);
var result = await task;
if (result.StatusCode == HttpStatusCode.OK)
{
return(true);
}
}
catch (Exception e)
{
_logger.LogFullException(e, $"{taskInfo} ping failed:");
_liteHttpClient.DumpHttpClientDetails();
}
_logger.Log(LogLevel.Warning, $"{taskInfo} ping failed: setting loginNeeded = true");
}
httpManager.loginNeeded = true;
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
}
return(false);
}
public async Task RegisterWithEGS(int taskID, LITEConnection connection, IHttpManager httpManager)
{
Connection = connection;
var taskInfo = $"task: {taskID} connection: {Connection.name}";
var httpClient = _liteHttpClient.GetClient(connection);
try
{
//contact each EGS and register presence with LITEServicePoint and share dest info which will be used for address oriented routing.
IPHostEntry hostEntry = Dns.GetHostEntry(Connection.remoteHostname);
//look up on known dns the other LITE EGS instances and register presence
foreach (var iPAddress in hostEntry.AddressList)
{
//set the URL
//string url = Connection.URL + "/api/LITE";
string url = Connection.URL + LiteAgentConstants.BaseUrl;
_logger.Log(LogLevel.Debug, $"{taskInfo} url: {url}");
// issue the POST
HttpResponseMessage response = null;
var cookies = _liteHttpClient.GetCookies(url);
_logger.LogCookies(cookies, taskInfo);
var profile = _profileStorage.Current;
string profilejson = JsonSerializer.Serialize(profile);
var stream = new MemoryStream();
var writer = new StreamWriter(stream);
writer.Write(profilejson);
writer.Flush();
stream.Position = 0;
StreamContent content = new StreamContent(stream);
var task = httpClient.PostAsync(url, content, _taskManager.cts.Token);
response = await task;
// output the result
_logger.LogHttpResponseAndHeaders(response, taskInfo);
_logger.Log(LogLevel.Debug, $"{taskInfo} await response.Content.ReadAsStringAsync(): {await response.Content.ReadAsStringAsync()}");
if (response.StatusCode == HttpStatusCode.Created)
{
connection.loginAttempts = 0;
_logger.Log(LogLevel.Debug, $"{taskInfo} LITE Successfully Registered with EGS!");
httpManager.loginNeeded = false;
}
else
{
_liteHttpClient.DumpHttpClientDetails();
httpManager.loginNeeded = true;
if (response.StatusCode == HttpStatusCode.Unauthorized && connection.loginAttempts == Connection.maxAttempts)
{
LiteEngine.shutdown(null, null);
Environment.Exit(0);
}
}
}
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"{taskInfo} Task Canceled");
}
catch (HttpRequestException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} HttpRequestException: Unable to login: {e.Message} {e.StackTrace}");
if (e.InnerException != null)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} HttpRequestException: (Inner Exception) {e.InnerException.Message} {e.InnerException.StackTrace}");
}
_liteHttpClient.DumpHttpClientDetails();
}
catch (Exception e)
{
_logger.LogFullException(e, $"{taskInfo} Unable to login");
_liteHttpClient.DumpHttpClientDetails();
}
}
public async Task <bool> DeleteEGSResource(int taskID, RoutedItem routedItem, LITEConnection connection, IHttpManager httpManager)
{
Connection = connection;
var taskInfo = $"task: {taskID} connection: {Connection.name}";
HttpResponseMessage response = null;
//string resourceURL = Connection.URL + "/api/File/" + routedItem.box + "/" + routedItem.resource;
string resourceURL = Connection.URL + FileAgentConstants.GetDeleteUrl(routedItem);
var httpClient = _liteHttpClient.GetClient(Connection);
try
{
var stopWatch = new Stopwatch();
stopWatch.Start();
//set the URL
_logger.Log(LogLevel.Debug, $"{taskInfo} URL: {resourceURL}");
// issue the POST
Task <HttpResponseMessage> task;
task = httpClient.DeleteAsync(resourceURL, _taskManager.cts.Token);
response = await task.ConfigureAwait(false);
// output the result
_logger.LogHttpResponseAndHeaders(response, taskInfo);
stopWatch.Stop();
_logger.Log(LogLevel.Information, $"{taskInfo} elapsed: {stopWatch.Elapsed}");
switch (response.StatusCode)
{
case HttpStatusCode.NoContent:
return(true);
default:
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
httpManager.loginNeeded = true;
}
_logger.Log(LogLevel.Warning, $"Deletion of {resourceURL} failed with {response.StatusCode}");
_liteHttpClient.DumpHttpClientDetails();
return(false);
}
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (HttpRequestException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} Exception: {e.Message} {e.StackTrace}");
if (e.InnerException != null)
{
_logger.Log(LogLevel.Warning, $"Inner Exception: {e.InnerException}");
}
_liteHttpClient.DumpHttpClientDetails();
}
catch (FileNotFoundException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} {e.Message} {e.StackTrace}");
}
catch (IOException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} {e.Message} {e.StackTrace}");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
_liteHttpClient.DumpHttpClientDetails();
}
finally
{
if (response != null)
{
response.Dispose();
}
}
return(false);
}
public async Task <int> GetResources(int taskID, LITEConnection connection, IHttpManager manager)
{
Connection = connection;
//get the resource list from EGS
try
{
var httpClient = _liteHttpClient.GetClient(connection);
var taskInfo = $"task: {taskID} connection: {Connection.name}";
foreach (var shareDestination in Connection.boxes)
{
//set the URL
string url = Connection.URL + "/api/File/" + shareDestination.boxUuid; //add summary
_logger.Log(LogLevel.Debug, $"{taskInfo} URL: {url}");
var cookies = _liteHttpClient.GetCookies(url);
_logger.LogCookies(cookies, taskInfo);
// issue the GET
var task = httpClient.GetAsync(url);
var response = await task;
// output the result
_logger.LogHttpResponseAndHeaders(response, taskInfo);
_logger.Log(LogLevel.Debug, $"{taskInfo} response.Content.ReadAsStringAsync(): {await response.Content.ReadAsStringAsync()}");
if (response.StatusCode != HttpStatusCode.OK)
{
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
manager.loginNeeded = true;
}
_logger.Log(LogLevel.Warning, $"{taskInfo} {response.StatusCode} {response.ReasonPhrase}");
_liteHttpClient.DumpHttpClientDetails();
}
//2018-02-06 shb convert from stream to JSON and clean up any non UTF-8 that appears like it did
// when receiving "contains invalid UTF8 bytes" exception
// var serializer = new DataContractJsonSerializer(typeof(Files));
// var streamReader = new StreamReader(await response.Content.ReadAsStreamAsync(), Encoding.UTF8);
// byte[] byteArray = Encoding.UTF8.GetBytes(streamReader.ReadToEnd());
// MemoryStream stream = new MemoryStream(byteArray);
// var newResources = serializer.ReadObject(stream) as Files; //List<EGSFileInfo>
var newResources = JsonSerializer.Deserialize <FilesModel>(await response.Content.ReadAsStringAsync());
if (newResources != null && newResources.files.Count > 0)
{
lock (Connection.fromEGS)
{
//take the new studies from cloud and merge with existing
foreach (var ri in newResources.files)
{
if (!Connection.fromEGS.Any(e => e.resource == ri.resource))
{
_logger.Log(LogLevel.Information, $"Adding {ri.resource}");
_routedItemManager.Init(ri);
_routedItemManager.Enqueue(connection, connection.fromEGS, nameof(connection.fromEGS), copy: false);
}
else
{
_logger.Log(LogLevel.Error, $"Resource already exists: {ri.resource}");
}
}
}
return(newResources.files.Count);
}
}
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (System.Runtime.Serialization.SerializationException e)
{
//eat it for now
_logger.Log(LogLevel.Warning, $"{e.Message} {e.StackTrace}");
}
catch (System.Net.Http.HttpRequestException e)
{
_logger.Log(LogLevel.Warning, $"{e.Message} {e.StackTrace}");
if (e.InnerException != null)
{
_logger.Log(LogLevel.Warning, $"Inner Exception: {e.InnerException}");
}
}
catch (Exception e)
{
_logger.LogFullException(e);
//throw e;
}
return(0);
}
public async Task SendToEGS(int taskID, LITEConnection connection, ISendToAllHubsService sendToAllHubs)
{
Connection = connection;
//2018-02-02 shb RoutedItem does not necessarily have an open stream at this point any more
var taskInfo = $"task: {taskID} connection: {Connection.name}";
_logger.Log(LogLevel.Information, $"{taskInfo} toEGS: {(Connection.toEGS == null ? 0 : Connection.toEGS.Count)} items to send.");
Dictionary <string, List <RoutedItem> > shareSet = new Dictionary <string, List <RoutedItem> >(); //I need a set for each sharing dest set
List <List <RoutedItem> > sizeSet = new List <List <RoutedItem> >(); //I need a set for each minEGSBatchSize
try
{
Task.WaitAll(_taskManager.FindByType($"{Connection.name}.PresentAsResource"));
Task.WaitAll(_taskManager.FindByType($"{Connection.name}.Store"));
int retryDelayed = 0;
if (Connection.toEGS.Count > 0)
{
var toEGSTemp = Connection.toEGS.ToArray();
//remove the toCloud item if it has exceeded maxAttempts
foreach (var routedItem in toEGSTemp)
{
if (_taskManager.cts.IsCancellationRequested)
{
return;
}
if (routedItem.lastAttempt != null && routedItem.lastAttempt < DateTime.Now.AddMinutes(-Connection.retryDelayMinutes)) //not attempted lately
{
routedItem.attempts++;
if (routedItem.attempts > 1)
{
_logger.Log(LogLevel.Debug, $"{taskInfo} {routedItem.sourceFileName} second attempt.");
}
routedItem.lastAttempt = DateTime.Now;
if (routedItem.attempts > Connection.maxAttempts)
{
_logger.Log(LogLevel.Error, $"{taskInfo} {routedItem.sourceFileName} has exceeded maxAttempts of {Connection.maxAttempts}. Will move to errors and not try again.");
routedItem.Error = "Exceeded maxAttempts";
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.toEGS, nameof(Connection.toEGS), error: true);
}
else
{
_logger.Log(LogLevel.Information, $"{taskInfo} {routedItem.sourceFileName} attempts: {routedItem.attempts}");
}
//inspect the sharing headers and batch by set
string shareString = "";
if (Connection.shareDestinations != null)
{
foreach (var connectionSet in routedItem.toConnections.FindAll(e => e.connectionName.Equals(Connection.name)))
{
if (connectionSet.shareDestinations != null)
{
foreach (var shareDestination in connectionSet.shareDestinations)
{
shareString += shareDestination.boxUuid;
}
}
}
}
if (shareSet.ContainsKey(shareString))
{
_logger.Log(LogLevel.Debug, $"{taskInfo} Adding {routedItem.sourceFileName} to shareString: {shareString}");
shareSet.GetValueOrDefault(shareString).Add(routedItem);
}
else
{
var list = new List <RoutedItem>
{
routedItem
};
_logger.Log(LogLevel.Debug, $"{taskInfo} Adding {routedItem.sourceFileName} to shareString: {shareString}");
shareSet.Add(shareString, list);
}
}
else
{
retryDelayed++;
}
}
//Now that each key in the Dictionary is to a single set of sharing destinations, let's break it up further by minEGSBatchSize
//What we want is a big enough upload to solve the small file problem, but small enough so the upload makes forward progress.
//If this is not the first attempt, then disable batching and send individually.
foreach (var share in shareSet.Values)
{
if (_taskManager.cts.IsCancellationRequested)
{
return;
}
var batch = new List <RoutedItem>();
long bytes = 0;
foreach (var element in share)
{
if (File.Exists(element.sourceFileName))
{
try
{
element.length = new FileInfo(element.sourceFileName).Length;
}
catch (FileNotFoundException e)
{
_logger.Log(LogLevel.Error, $"{taskInfo} {e.Message} {e.StackTrace}");
continue;
}
catch (IOException e)
{
_logger.Log(LogLevel.Error, $"{taskInfo} {e.Message} {e.StackTrace}");
//condition may be transient like file in use so skip for the moment
continue;
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
//condition may be transient like file so skip for the moment
continue;
}
}
else
{
element.Error = $"File {element.sourceFileName} does not exist";
_routedItemManager.Init(element);
_routedItemManager.Dequeue(Connection, Connection.toEGS, nameof(Connection.toEGS), true);
}
//If this is not the first attempt, then disable batching and send individually.
if (element.length < Connection.minEGSBatchSize && bytes < Connection.minEGSBatchSize && element.attempts == 1)
{
bytes += element.length;
_logger.Log(LogLevel.Debug, $"{taskInfo} Adding {element.sourceFileName} to batch...");
batch.Add(element);
}
else
{
_logger.Log(LogLevel.Debug, $"{taskInfo} Batch is full with count: {batch.Count} size: {bytes} attempts: {element.attempts} {(element.attempts > 1 ? "items are sent individually after 1st attempt!" : "")}");
sizeSet.Add(batch);
batch = new List <RoutedItem>();
bytes = element.length;
batch.Add(element);
}
}
if (!sizeSet.Contains(batch) && batch.Count > 0)
{
_logger.Log(LogLevel.Debug, $"{taskInfo} Add final batch to set with count: {batch.Count} size: {bytes}");
sizeSet.Add(batch);
}
}
int tempcount = 0;
foreach (var batch in sizeSet)
{
tempcount += batch.Count;
}
_logger.Log(LogLevel.Information, $"{taskInfo} {sizeSet.Count} batches to send, selected: {tempcount}/{toEGSTemp.Length} retry delayed: {retryDelayed}");
foreach (var batch in sizeSet)
{
if (_taskManager.cts.IsCancellationRequested)
{
return;
}
//if (loginNeeded) break;
if (batch.Count > 0)
{
switch (Connection.PushPull)
{
case PushPullEnum.pull:
switch (Connection.protocol)
{
case Protocol.Http:
case Protocol.UDT:
var newTaskID = _taskManager.NewTaskID();
Task task = new Task(new Action(async() => await _presentAsResourceService.PresentAsResource(batch, newTaskID, connection, sendToAllHubs)));
await _taskManager.Start(newTaskID, task, $"{Connection.name}.PresentAsResource", $"{Connection.name}.PresentAsResource batch {batch.Count}", isLongRunning : false);
break;
}
break;
case PushPullEnum.push:
switch (Connection.protocol)
{
case Protocol.Http:
var newTaskID2 = _taskManager.NewTaskID();
Task task2 = new Task(new Action(async() => await _liteStoreService.store(batch, newTaskID2, connection)));
await _taskManager.Start(newTaskID2, task2, $"{Connection.name}.Store", $"{Connection.name}.Store batch {batch.Count}", isLongRunning : false);
break;
case Protocol.UDT:
break;
}
break;
case PushPullEnum.both:
//since each method dequeues it's own work we would need a separate queue before we can do both, like toEGSPull toEGSPush.
break;
}
}
}
}
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
}
}
public async Task Download(int taskID, LITEConnection connection, IHttpManager httpManager, SemaphoreSlim FromEGSSignal)
{
Connection = connection;
var taskInfo = $"task: {taskID} connection: {Connection.name}";
var profile = _profileStorage.Current;
DateTime lastGetResources = DateTime.MinValue;
int GetResourcesInterval = 120000;
int lastResourceCount = 0;
try
{
do
{
var temp = Connection.fromEGS.ToList();
if (temp.Count == 0 || !temp.Any(e => e.attempts == 0))
{
var getResourcesTime = DateTime.Now.AddMilliseconds(GetResourcesInterval * -1);
if (lastGetResources.CompareTo(getResourcesTime) < 0 || lastResourceCount > 0)
{
lastResourceCount = await _getLiteReresourcesService.GetResources(taskID, connection, httpManager);
lastGetResources = DateTime.Now;
}
bool success = await FromEGSSignal.WaitAsync(profile.KickOffInterval, _taskManager.cts.Token).ConfigureAwait(false);
// FromEGSSignal.Dispose();
// FromEGSSignal = new SemaphoreSlim(0, 1);
}
else
{
await Task.Delay(profile.taskDelay).ConfigureAwait(false);
}
foreach (var routedItem in Connection.fromEGS.ToArray())
{
await ProcessItem(taskID, routedItem, connection, httpManager);
}
Task.WaitAll(_taskManager.FindByType($"{Connection.name}.DownloadViaHttp"));
} while (Connection.responsive);
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (OperationCanceledException)
{
_logger.Log(LogLevel.Warning, $"Wait Operation Canceled. Exiting Download");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
_logger.Log(LogLevel.Critical, $"{taskInfo} Exiting Download");
}
finally
{
_taskManager.Stop($"{Connection.name}.Download");
}
}
/// <summary>
/// wado downloads studies from liCloud. ImagingStudy is required while Series and Instance are optional. RAM utilization remains low regardless of download size.
/// </summary>
/// <param name="taskID"></param>
/// <param name="routedItem"></param>
/// <param name="connection"></param>
/// <param name="httpManager"></param>
/// <param name="compress"></param>
/// <returns></returns>
public async Task DownloadViaHttp(int taskID, RoutedItem routedItem, LITEConnection connection, IHttpManager httpManager, bool compress = true)
{
Connection = connection;
var taskInfo = $"task: {taskID} connection: {Connection.name} resource: {routedItem.resource}";
var profile = _profileStorage.Current;
var stopWatch = new Stopwatch();
stopWatch.Start();
//string url = Connection.URL + $"/api/File/{routedItem.box}/{routedItem.resource}";
string url = Connection.URL + FileAgentConstants.GetDownloadUrl(routedItem);
string dir = profile.tempPath + Path.DirectorySeparatorChar + Connection.name + Path.DirectorySeparatorChar + Constants.Dirs.ToRules + Path.DirectorySeparatorChar + Guid.NewGuid();
Directory.CreateDirectory(dir);
long fileSize = 0;
HttpResponseMessage response = null;
MultipartFileStreamProvider streamProvider = null;
MultipartFileStreamProvider contents = null;
var httpClient = _liteHttpClient.GetClient(connection);
try
{
_logger.Log(LogLevel.Debug, $"{taskInfo} download dir will be {dir}");
_logger.Log(LogLevel.Debug, $"{taskInfo} url: {url} attempt: {routedItem.attempts}");
var cookies = _liteHttpClient.GetCookies(url);
_logger.LogCookies(cookies, taskInfo);
// issue the GET
var task = httpClient.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, _taskManager.cts.Token);
try
{
response = await task.ConfigureAwait(false);
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"{taskInfo} Task Canceled");
}
// output the result
_logger.LogHttpResponseAndHeaders(response, taskInfo);
//if(Logger.logger.FileTraceLevel == "Verbose") _logger.Log(LogLevel.Debug,$"{taskInfo} await response.Content.ReadAsStringAsync(): {await response.Content.ReadAsStringAsync()}");
switch (response.StatusCode)
{
case HttpStatusCode.OK:
break;
case HttpStatusCode.NotFound:
routedItem.Error = HttpStatusCode.NotFound.ToString();
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.fromEGS, nameof(Connection.fromEGS), error: true);
return;
case HttpStatusCode.Unauthorized:
httpManager.loginNeeded = true;
_liteHttpClient.DumpHttpClientDetails();
return;
default:
_liteHttpClient.DumpHttpClientDetails();
return;
}
if (!_util.IsDiskAvailable(dir, profile, routedItem.length))
{
_logger.Log(LogLevel.Debug, $"{taskInfo} Insufficient disk to write {url} to {dir} guessing it could be 16GB");
return;
}
streamProvider = new MultipartFileStreamProvider(dir, 1024000);
try
{
contents = await response.Content.ReadAsMultipartAsync(streamProvider, _taskManager.cts.Token).ConfigureAwait(false);
}
catch (Exception e)
{
//MIME is corrupt such as Unexpected end of MIME multipart stream. MIME multipart message is not complete.
//This usually happens if the upload does not complete. Catch as "normal" and remove resource as if success
//since retrying will not help this condition.
_logger.LogFullException(e, taskInfo);
_liteHttpClient.DumpHttpClientDetails();
if (await _deleteEGSResourceService.DeleteEGSResource(taskID, routedItem, connection, httpManager).ConfigureAwait(false))
{
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.fromEGS, nameof(Connection.fromEGS), error: false);
}
else
{
routedItem.Error = "Unable to delete EGS resource";
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.fromEGS, nameof(Connection.fromEGS), error: true);
}
return;
}
int index = 0;
_logger.Log(LogLevel.Debug, $"{taskInfo} Splitting {contents?.FileData.Count} files into RoutedItems.");
foreach (var part in contents.FileData)
{
try
{
index++;
fileSize += new FileInfo(part.LocalFileName).Length;
_logger.Log(LogLevel.Debug, $"{taskInfo} downloaded file: {part.LocalFileName}");
RoutedItem ri = new RoutedItem(fromConnection: Connection.name, sourceFileName: part.LocalFileName, taskID: taskID, fileIndex: index, fileCount: contents.FileData.Count)
{
type = RoutedItem.Type.FILE
};
_logger.Log(LogLevel.Debug, $"{taskInfo} Enqueuing RoutedItem {routedItem.sourceFileName}");
_routedItemManager.Init(ri);
_routedItemManager.Enqueue(Connection, Connection.toRules, nameof(Connection.toRules));
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
}
}
if (await _deleteEGSResourceService.DeleteEGSResource(taskID, routedItem, connection, httpManager).ConfigureAwait(false))
{
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.fromEGS, nameof(Connection.fromEGS), error: false);
}
else
{
routedItem.Error = "Unable to delete EGS resource";
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.fromEGS, nameof(Connection.fromEGS), error: true);
}
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (NullReferenceException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} Exception: {e.Message} {e.StackTrace}");
//throw e;
}
catch (HttpRequestException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} Exception: {e.Message} {e.StackTrace}");
if (e.InnerException != null)
{
_logger.Log(LogLevel.Warning, $"Inner Exception: {e.InnerException}");
}
_liteHttpClient.DumpHttpClientDetails();
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
_liteHttpClient.DumpHttpClientDetails();
}
finally
{
if (response != null)
{
response.Dispose();
}
_taskManager.Stop($"{Connection.name}.DownloadViaHttp");
}
stopWatch.Stop();
_logger.Log(LogLevel.Information, $"{taskInfo} elapsed: {stopWatch.Elapsed} size: {fileSize} rate: {(float)fileSize / stopWatch.Elapsed.TotalMilliseconds * 1000 / 1000000} MB/s");
}
/// <summary>
/// Store takes a batch of RoutedItem, all going to the same share destination, and uploads them as a single operation. This is done to solve the many small files problem.Larger files can go individually.
/// </summary>
/// <param name="batch"></param>
/// <param name="taskID"></param>
/// <param name="connection"></param>
/// <returns></returns>
public async Task store(List <RoutedItem> batch, int taskID, LITEConnection connection)
{
Connection = connection;
var taskInfo = $"task: {taskID} connection: {Connection.name}";
StreamContent streamContent = null;
MultipartContent content = null;
HttpResponseMessage response = null;
string testFile = null;
var firstRecord = batch.First();
try
{
var stopWatch = new Stopwatch();
stopWatch.Start();
//set the URL
//string resourceURL = Connection.URL + "/api/File";
string resourceURL = Connection.URL + FileAgentConstants.BaseUrl;
_logger.Log(LogLevel.Debug, $"{taskInfo} URL: {resourceURL}");
// generate guid for boundary...boundaries cannot be accidentally found in the content
var boundary = Guid.NewGuid();
_logger.Log(LogLevel.Debug, $"{taskInfo} boundary: {boundary}");
// create the content
content = new MultipartContent("related", boundary.ToString());
//add the sharing headers
List <string> shareHeader = new List <string>();
if (Connection.shareDestinations != null)
{
foreach (var connectionSet in firstRecord.toConnections.FindAll(e => e.connectionName.Equals(Connection.name)))
{
if (connectionSet.shareDestinations != null)
{
foreach (var shareDestination in connectionSet.shareDestinations)
{
shareHeader.Add(shareDestination.boxUuid);
_logger.Log(LogLevel.Debug, $"{taskInfo} sharing to: {shareDestination.boxId} {shareDestination.boxName} {shareDestination.groupId} {shareDestination.groupName} {shareDestination.organizationName} {shareDestination.publishableBoxType}");
}
}
}
}
content.Headers.Add("X-Li-Destination", shareHeader);
long fileSize = 0;
var profile = _profileStorage.Current;
var dir = profile.tempPath + Path.DirectorySeparatorChar + Connection.name + Path.DirectorySeparatorChar + "toEGS";
Directory.CreateDirectory(dir);
testFile = dir + Path.DirectorySeparatorChar + Guid.NewGuid() + ".gz";
using (FileStream compressedFileStream = File.Create(testFile))
{
using GZipStream compressionStream = new GZipStream(compressedFileStream, CompressionMode.Compress);
foreach (var routedItem in batch)
{
if (File.Exists(routedItem.sourceFileName))
{
routedItem.stream = File.OpenRead(routedItem.sourceFileName);
if (Connection.calcCompressionStats)
{
routedItem.stream.CopyTo(compressionStream);
}
fileSize += routedItem.length;
streamContent = new StreamContent(routedItem.stream);
streamContent.Headers.ContentDisposition = new ContentDispositionHeaderValue("attachment")
{
FileName = routedItem.sourceFileName
};
content.Add(streamContent);
streamContent.Headers.Add("content-type", "application/octet-stream");
}
else
{
_logger.Log(LogLevel.Error, $"{taskInfo} {routedItem.sourceFileName} no longer exists. Increase tempFileRetentionHours for heavy transfer backlogs that may take hours!!");
routedItem.Error = "File no longer exists";
_routedItemManager.Init(routedItem);
_routedItemManager.Dequeue(Connection, Connection.toEGS, nameof(Connection.toEGS), error: true);
}
}
}
if (Connection.calcCompressionStats)
{
FileInfo info = new FileInfo(testFile);
_logger.Log(LogLevel.Information, $"{taskInfo} orgSize: {fileSize} compressedSize: {info.Length} reduction: {(fileSize == 0 ? 0 : (fileSize * 1.0 - info.Length) / (fileSize) * 100)}%");
}
// issue the POST
Task <HttpResponseMessage> task;
var httpClient = _liteHttpClient.GetClient(connection);
if (firstRecord.Compress == true)
{
var compressedContent = new CompressedContent(content, "gzip");
_logger.Log(LogLevel.Debug, $"{taskInfo} compressedContent.Headers {compressedContent.Headers} ");
compressedContent.Headers.Remove("Content-Encoding");
var cookies = _liteHttpClient.GetCookies(resourceURL);
_logger.LogCookies(cookies, taskInfo);
task = httpClient.PostAsync(resourceURL, compressedContent);
}
else
{
_logger.Log(LogLevel.Debug, $"{taskInfo} will send content.Headers {content.Headers}");
var cookies = _liteHttpClient.GetCookies(resourceURL);
_logger.LogCookies(cookies, taskInfo);
task = httpClient.PostAsync(resourceURL, content);
}
response = await task;
// output the result
_logger.LogHttpResponseAndHeaders(response, taskInfo);
_logger.Log(LogLevel.Debug, $"{taskInfo} response.Content.ReadAsStringAsync(): {await response.Content.ReadAsStringAsync()}");
stopWatch.Stop();
_logger.Log(LogLevel.Information, $"{taskInfo} elapsed: {stopWatch.Elapsed} size: {fileSize} rate: {(float)fileSize / stopWatch.Elapsed.TotalMilliseconds * 1000 / 1000000} MB/s");
switch (response.StatusCode)
{
case HttpStatusCode.Created:
//dequeue the work, we're done!
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
foreach (var ri in batch)
{
_routedItemManager.Init(ri);
_routedItemManager.Dequeue(Connection, Connection.toEGS, nameof(Connection.toEGS));
}
//let EGGS know it's available, or when we convert udt to .net core then perhaps push so no open socket required on client.
//await SendToAllHubs(LITEServicePoint, batch);
break;
case HttpStatusCode.UnprocessableEntity:
//dequeue the work, we're done!
_logger.Log(LogLevel.Warning, $"creation of {firstRecord.sourceFileName} and others in batch failed with {response.StatusCode}");
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
foreach (var ri in batch)
{
ri.Error = HttpStatusCode.UnprocessableEntity.ToString();
_routedItemManager.Init(ri);
_routedItemManager.Dequeue(Connection, Connection.toEGS, nameof(Connection.toEGS), error: true);
}
break;
default:
if (response.StatusCode == HttpStatusCode.Unauthorized)
{
Connection.loginNeeded = true;
}
_logger.Log(LogLevel.Warning, $"creation of {firstRecord.sourceFileName} and others in batch failed with {response.StatusCode}");
_liteHttpClient.DumpHttpClientDetails();
break;
}
//delete the compression test file
File.Delete(testFile);
}
catch (TaskCanceledException)
{
_logger.Log(LogLevel.Information, $"Task was canceled.");
}
catch (HttpRequestException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} Exception: {e.Message} {e.StackTrace}");
if (e.InnerException != null)
{
_logger.Log(LogLevel.Warning, $"Inner Exception: {e.InnerException}");
}
_liteHttpClient.DumpHttpClientDetails();
}
catch (FileNotFoundException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} {e.Message} {e.StackTrace}");
}
catch (IOException e)
{
_logger.Log(LogLevel.Warning, $"{taskInfo} {e.Message} {e.StackTrace}");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
_liteHttpClient.DumpHttpClientDetails();
}
finally
{
try
{
if (streamContent != null)
{
streamContent.Dispose();
}
if (response != null)
{
response.Dispose();
}
if (content != null)
{
content.Dispose();
}
File.Delete(testFile);
_taskManager.Stop($"{Connection.name}.Store");
}
catch (Exception e)
{
_logger.LogFullException(e, taskInfo);
}
}
}