public async Task <System.IO.Stream> GetPackage(Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier packageIdentifier)
{
var response = await this._Client.GetAsync(this._Host + AdaptersPath + "/GetPackage?identifier=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Identifier) + "&version=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Version));
if (response.StatusCode != System.Net.HttpStatusCode.OK)
{
throw new Exception("An error occurred while trying to download the package:" + packageIdentifier.PackageName() + "/n" + response.ReasonPhrase + "/nStatusCode:" + response.StatusCode.ToString());
}
return(await response.Content.ReadAsStreamAsync());
}
public async Task <System.IO.Stream> GetPackage(Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier packageIdentifier)
{
string path = "GetPackage?identifier=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Identifier) + "&version=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Version);
string identifier;
_log.Debug(ExecutingMessage(path, out identifier));
var response = await this._Client.GetAsync(this._Host + AdaptersPath + "/GetPackage?identifier=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Identifier) + "&version=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Version));
if (response.StatusCode != System.Net.HttpStatusCode.OK)
{
_log.Error(CompletionMessage(identifier, path, false) + "\r\n" + response.GetMessage());
throw new Exception("An error occurred while trying to download the package:" + packageIdentifier.PackageName() + "/n" + response.ReasonPhrase + "/nStatusCode:" + response.StatusCode.ToString());
}
_log.Debug(CompletionMessage(identifier, path, true));
return(await response.Content.ReadAsStreamAsync());
}
private void AutoProcess(KeyValuePair <string, HubRequest> input)
{
var request = input.Value;
var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId);
if (datamartDescription.ProcessQueriesAndUploadAutomatically == false && datamartDescription.ProcessQueriesAndNotUpload == false)
{
//just notify, do not process
string message = $"New query submitted and awaiting processing in { request.ProjectName } Project: { request.Source.Name } ({request.Source.Identifier})";
SystemTray.DisplayNewQueryNotificationToolTip(message);
RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload);
return;
}
var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID);
var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier {
Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion
};
if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
{
DnsServiceManager.DownloadPackage(_networkSetting, packageIdentifier);
}
var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath);
try
{
domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion);
IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId);
ProcessorManager.UpdateProcessorSettings(modelDescription, processor);
processor.Settings.Add("NetworkId", request.NetworkId);
Lib.Caching.DocumentCacheManager cache = new Lib.Caching.DocumentCacheManager(request.NetworkId, request.DataMartId, request.Source.ID, request.Source.Responses.Where(x => x.DataMartID == request.DataMartId).FirstOrDefault().ResponseID);
//need to initialize before checking the capabilities and settings of the processor since they may change based on the type of request being sent.
if (processor is IEarlyInitializeModelProcessor)
{
((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, _networkSetting))).ToArray());
}
if (processor != null &&
processor.ModelMetadata != null &&
processor.ModelMetadata.Capabilities != null &&
processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") &&
!(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"])
{
//can't be run, don't attempt autoprocessing
RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload);
domainManager.Dispose();
return;
}
request.Processor = processor;
if (cache.HasResponseDocuments == false && (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted || request.RoutingStatus == DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted))
{
if (processor != null)
{
SystemTray.GenerateNotification(request, request.NetworkId);
StartProcessingRequest(request, processor, datamartDescription, domainManager, cache);
return;
}
}
else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval)
{
if (datamartDescription.ProcessQueriesAndUploadAutomatically)
{
// Increment counter
System.Threading.Interlocked.Increment(ref _queriesProcessedCount);
SystemTray.UpdateNotifyText(_queriesProcessedCount, request.DataMartName, request.NetworkId);
StartUploadingRequest(request, domainManager, cache);
return;
}
}
else if (cache.HasResponseDocuments)
{
RequestStatuses.TryAdd(input.Key, ProcessingStatus.PendingUpload);
}
domainManager.Dispose();
}
catch (Exception ex)
{
Log.Error($"Error autoprocessing Request: { request.Source.Identifier }, DataMartId: { request.DataMartId }, NetworkId: {request.NetworkId}", ex);
domainManager.Dispose();
throw;
}
}
protected override void OnDoWork(DoWorkEventArgs e)
{
_log.Info("Automated processing worker started.");
while (true)
{
_log.Info("Checking Master Request Queue for requests requiring automated processing.");
SystemTray.UpdateNotificationIcon(IconType.IconBusy, "Processing Requests");
var reqs = from ns in Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Cast <NetWorkSetting>().ToObservable()
where ns.NetworkStatus == Util.ConnectionOKStatus
let dmIds = ns.DataMartList
// BMS: Note the ProcessAndNotUpload feature has been temporarilty disabled until we fix the processing around this feature
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || /* dm.ProcessQueriesAndNotUpload || */ dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId)
.ToArray()
where dmIds.Any()
from list in DnsServiceManager.GetRequestList(ns, 0, Properties.Settings.Default.AutoProcessingBatchSize,
new RequestFilter {
Statuses = new [] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted }, DataMartIds = dmIds
}, null, null)
from rl in list.Segment.EmptyIfNull().ToObservable()
where rl.AllowUnattendedProcessing
from r in RequestCache.ForNetwork(ns).LoadRequest(rl.ID, rl.DataMartID)
select new { Request = r, NetworkSetting = ns };
reqs
.Do(r =>
{
var request = r.Request;
var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId);
var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID);
var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier {
Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion
};
if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
{
DnsServiceManager.DownloadPackage(r.NetworkSetting, packageIdentifier);
}
using (var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath))
{
domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion);
IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId);
ProcessorManager.UpdateProcessorSettings(modelDescription, processor);
if (processor is IEarlyInitializeModelProcessor)
{
((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, r.NetworkSetting))).ToArray());
}
if (processor != null &&
processor.ModelMetadata != null &&
processor.ModelMetadata.Capabilities != null &&
processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") &&
!(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"])
{
//can't be run, don't attempt autoprocessing
return;
}
request.Processor = processor;
if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted)
{
if (processor != null)
{
SystemTray.generate_notification(request, request.NetworkId);
if (datamartDescription.NotifyOfNewQueries)
{
SystemTray.UpdateNotificationIcon(IconType.IconBusy, string.Format("Query Submitted by {0}", request.Source.Author.Username));
}
else
{
processor.SetRequestProperties(request.Source.ID.ToString(), request.Properties);
ProcessRequest(request);
var statusCode = request.Processor.Status(request.Source.ID.ToString()).Code;
if (datamartDescription.ProcessQueriesAndUploadAutomatically && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage))
{
// Post process requests that are automatically uploaded
processor.PostProcess(request.Source.ID.ToString());
// Increment counter
_queriesProcessedCount++;
SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
UploadRequest(request);
}
}
}
}
else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval)
{
if (datamartDescription.ProcessQueriesAndUploadAutomatically)
{
// Increment counter
_queriesProcessedCount++;
SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
UploadRequest(request);
}
}
}
})
.LogExceptions(_log.Error)
.Catch()
.LastOrDefault();
SystemTray.UpdateNotificationIcon(IconType.IconDefault, null);
Thread.Sleep(DMClient.Properties.Settings.Default.RefreshRate);
}
}
