public async Task OriginalGetRequestsForAutoProcessor()
{
//If there are more than 10 pending requests within the last 30 days that are possible for autoprocessing, only the first 10 will be retrieved
var datamartIDs = _networkSetting.DataMartList
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || dm.ProcessQueriesAndNotUpload || dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId).ToArray();
if (datamartIDs.Length == 0)
{
throw new NotSupportedException("No datamarts enabled for autoprocessing.");
}
var requestFilter = new RequestFilter
{
Statuses = new[] { DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted, DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted },
DataMartIds = datamartIDs
};
int batchSize = 10;
var requests = from list in DnsServiceManager.GetRequestList("AutoProcessor", _networkSetting, 0, batchSize, requestFilter, null, null)
from rl in list.Segment.EmptyIfNull().ToObservable()
where rl.AllowUnattendedProcessing
from r in RequestCache.ForNetwork(_networkSetting).LoadRequest(rl.ID, rl.DataMartID)
select r;
var rq = await requests.ToArray();
foreach (var r in rq)
{
_logger.Debug($"{ r.DataMartName }: { r.Source.Name} [ { r.Source.Identifier } ]");
}
}
async Task <HubRequest[]> GetRequestsAsync()
{
if (_networkSetting.NetworkStatus != Util.ConnectionOKStatus)
{
return(new HubRequest[0]);
}
var datamartIDs = _networkSetting.DataMartList
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || dm.ProcessQueriesAndNotUpload || dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId).ToArray();
if (datamartIDs.Length == 0)
{
return(new HubRequest[0]);
}
var requestFilter = new RequestFilter
{
Statuses = new[] { DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted, DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted },
DataMartIds = datamartIDs
};
var requests = Observable.Create <DTO.DataMartClient.RequestList>(async observer =>
{
int index = 0;
int batchSize = Properties.Settings.Default.AutoProcessingBatchSize;
DTO.DataMartClient.RequestList rl = null;
while (rl == null || (index < rl.TotalCount))
{
rl = await DnsServiceManager.GetRequestList("AutoProcessor", _networkSetting, index, batchSize, requestFilter, DTO.DataMartClient.RequestSortColumn.RequestTime, true);
if (rl == null || rl.TotalCount == 0)
{
break;
}
observer.OnNext(rl);
index += batchSize;
}
observer.OnCompleted();
}).DefaultIfEmpty()
.Aggregate((requestList1, requestList2) =>
{
if (requestList1 == null && requestList2 == null)
{
return(new DTO.DataMartClient.RequestList
{
Segment = Array.Empty <DTO.DataMartClient.RequestListRow>(),
SortedAscending = true,
SortedByColumn = DTO.DataMartClient.RequestSortColumn.RequestTime
});
}
else if (requestList1 != null && requestList2 == null)
{
return(requestList1);
}
else if (requestList1 == null && requestList2 != null)
{
return(requestList2);
}
else
{
return(new DTO.DataMartClient.RequestList
{
Segment = requestList1.Segment.EmptyIfNull().Concat(requestList2.Segment.EmptyIfNull()).ToArray(),
SortedAscending = requestList1.SortedAscending,
SortedByColumn = requestList1.SortedByColumn,
StartIndex = requestList1.StartIndex,
TotalCount = requestList1.TotalCount
});
}
})
.SelectMany(requestList =>
{
if (requestList == null)
{
return(Array.Empty <DTO.DataMartClient.RequestListRow>());
}
return(requestList.Segment.DefaultIfEmpty().Where(s => s.AllowUnattendedProcessing));
})
.SelectMany(rlr => RequestCache.ForNetwork(_networkSetting).LoadRequest(rlr.ID, rlr.DataMartID))
.ToArray();
return(await requests);
}
protected override void OnDoWork(DoWorkEventArgs e)
{
_log.Info("Automated processing worker started.");
while (true)
{
_log.Info("Checking Master Request Queue for requests requiring automated processing.");
SystemTray.UpdateNotificationIcon(IconType.IconBusy, "Processing Requests");
var reqs = from ns in Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Cast <NetWorkSetting>().ToObservable()
where ns.NetworkStatus == Util.ConnectionOKStatus
let dmIds = ns.DataMartList
// BMS: Note the ProcessAndNotUpload feature has been temporarilty disabled until we fix the processing around this feature
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || /* dm.ProcessQueriesAndNotUpload || */ dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId)
.ToArray()
where dmIds.Any()
from list in DnsServiceManager.GetRequestList(ns, 0, Properties.Settings.Default.AutoProcessingBatchSize,
new RequestFilter {
Statuses = new [] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted }, DataMartIds = dmIds
}, null, null)
from rl in list.Segment.EmptyIfNull().ToObservable()
where rl.AllowUnattendedProcessing
from r in RequestCache.ForNetwork(ns).LoadRequest(rl.ID, rl.DataMartID)
select new { Request = r, NetworkSetting = ns };
reqs
.Do(r =>
{
var request = r.Request;
var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId);
var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID);
var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier {
Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion
};
if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
{
DnsServiceManager.DownloadPackage(r.NetworkSetting, packageIdentifier);
}
using (var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath))
{
domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion);
IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId);
ProcessorManager.UpdateProcessorSettings(modelDescription, processor);
if (processor is IEarlyInitializeModelProcessor)
{
((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, r.NetworkSetting))).ToArray());
}
if (processor != null &&
processor.ModelMetadata != null &&
processor.ModelMetadata.Capabilities != null &&
processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") &&
!(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"])
{
//can't be run, don't attempt autoprocessing
return;
}
request.Processor = processor;
if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted)
{
if (processor != null)
{
SystemTray.generate_notification(request, request.NetworkId);
if (datamartDescription.NotifyOfNewQueries)
{
SystemTray.UpdateNotificationIcon(IconType.IconBusy, string.Format("Query Submitted by {0}", request.Source.Author.Username));
}
else
{
processor.SetRequestProperties(request.Source.ID.ToString(), request.Properties);
ProcessRequest(request);
var statusCode = request.Processor.Status(request.Source.ID.ToString()).Code;
if (datamartDescription.ProcessQueriesAndUploadAutomatically && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage))
{
// Post process requests that are automatically uploaded
processor.PostProcess(request.Source.ID.ToString());
// Increment counter
_queriesProcessedCount++;
SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
UploadRequest(request);
}
}
}
}
else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval)
{
if (datamartDescription.ProcessQueriesAndUploadAutomatically)
{
// Increment counter
_queriesProcessedCount++;
SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
UploadRequest(request);
}
}
}
})
.LogExceptions(_log.Error)
.Catch()
.LastOrDefault();
SystemTray.UpdateNotificationIcon(IconType.IconDefault, null);
Thread.Sleep(DMClient.Properties.Settings.Default.RefreshRate);
}
}
async Task <HubRequest[]> GetRequestsAsync()
{
var datamartIDs = _networkSetting.DataMartList
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || dm.ProcessQueriesAndNotUpload || dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId).ToArray();
if (datamartIDs.Length == 0)
{
return(Array.Empty <HubRequest>());
}
var requestFilter = new RequestFilter
{
Statuses = new[] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted, Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted },
DataMartIds = datamartIDs,
FromDate = DateTime.UtcNow.AddYears(-1),
DateRange = DateRangeKind.Exact
};
var ob = Observable.Create <DTO.DataMartClient.RequestList>(async observer =>
{
int index = 0;
int batchSize = 2;
DTO.DataMartClient.RequestList rl = null;
while (rl == null || (index < rl.TotalCount))
{
_logger.Debug($"Observer loop: pageIndex={ index }.");
rl = await DnsServiceManager.GetRequestList("RxQueryTests-DefaultDSM", _networkSetting, index, batchSize, requestFilter, null, null);
_logger.Debug($"Observer loop: pageIndex={ index }, results returned={ rl.Segment.DefaultIfEmpty().Count() }, total results={ rl.TotalCount }");
if (rl.TotalCount == 0)
{
break;
}
observer.OnNext(rl);
index += batchSize;
}
_logger.Debug("Observer loop firing OnComplete");
observer.OnCompleted();
}).DefaultIfEmpty().Aggregate((requestList1, requestList2) => {
if (requestList1 == null && requestList2 == null)
{
return(new DTO.DataMartClient.RequestList
{
Segment = Array.Empty <DTO.DataMartClient.RequestListRow>(),
SortedAscending = false,
SortedByColumn = DTO.DataMartClient.RequestSortColumn.RequestTime
});
}
else if (requestList1 != null && requestList2 == null)
{
return(requestList1);
}
else if (requestList1 == null && requestList2 != null)
{
return(requestList2);
}
else
{
return(new DTO.DataMartClient.RequestList
{
Segment = requestList1.Segment.EmptyIfNull().Concat(requestList2.Segment.EmptyIfNull()).ToArray(),
SortedAscending = requestList1.SortedAscending,
SortedByColumn = requestList1.SortedByColumn,
StartIndex = requestList1.StartIndex,
TotalCount = requestList1.TotalCount
});
}
})
.SelectMany(requestList => {
if (requestList == null)
{
return(Array.Empty <DTO.DataMartClient.RequestListRow>());
}
return(requestList.Segment.DefaultIfEmpty().Where(s => s.AllowUnattendedProcessing));
})
.SelectMany(rlr => RequestCache.ForNetwork(_networkSetting).LoadRequest(rlr.ID, rlr.DataMartID))
.ToArray();
return(await ob);
}
public void Reload()
{
this.connectionError.Visible = Network.NetworkStatus != Util.ConnectionOKStatus;
this.errorMessage.Text = Network.NetworkMessage;
if (Network.NetworkStatus != Util.ConnectionOKStatus)
{
return;
}
var reload =
DnsServiceManager.GetRequestList("filtered requests lookup", Network, Page * _pageSize, _pageSize, Network.Filter, _sortColumn, _isSortAscending)
.TakeLast(1)
.ObserveOn(this)
.Select(list =>
{
PageCount = list.TotalCount / _pageSize + (list.TotalCount == 0 || list.TotalCount % _pageSize > 0 ? 1 : 0);
lblTotalPages.Text = PageCount.ToString();
if (Page > PageCount)
{
Page = PageCount;
txtPageNumber.Text = (Page + 1).ToString();
}
var selected = SelectedRequest;
//update request statuses that have cached results that are awaiting upload - Pending Upload is a local status and is not known by the API
foreach (var seg in list.Segment.Where(s => s.Status == DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted || s.Status == DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted))
{
var cache = new Lib.Caching.DocumentCacheManager(Network.NetworkId, seg.DataMartID, seg.ID, seg.ResponseID);
if (cache.HasResponseDocuments)
{
seg.Status = DTO.DataMartClient.Enums.DMCRoutingStatus.PendingUpload;
}
}
dgvRequestList.DataSource = list.Segment;
SelectedRequest = selected == null ? null : list.Segment.FirstOrDefault(rq => rq.ID == selected.ID && rq.DataMartID == selected.DataMartID);
DisplaySortGlyph();
EnableDisablePageButtons();
return(Unit.Default);
});
var now = DateTime.Now;
var newReqsLookup =
DnsServiceManager.GetRequestList("new requests lookup", Network, 0, 1, new RequestFilter {
DateRange = DateRangeKind.Exact, FromDate = _lastNewRequestsLookup
}, null, null)
.Take(1)
.ObserveOn(this)
.Select(n =>
{
if (n.TotalCount > 0)
{
NewRequestsAvailable = true;
}
_lastNewRequestsLookup = now.AddMilliseconds(1);
return(Unit.Default);
});
var invFilter = InvertFilter(_lastInvisibleRequestsLookup);
var invisibleReqsLookup = invFilter == null?Observable.Return(Unit.Default) :
DnsServiceManager.GetRequestList("hidden requests lookup", Network, 0, 1, invFilter.Value, null, null)
.Take(1)
.ObserveOn(this)
.Select(n =>
{
if (n.TotalCount > 0)
{
InvisibleRequestsAvailable = true;
}
_lastInvisibleRequestsLookup = now.AddMilliseconds(1);
return(Unit.Default);
});
_currentReload.CallDispose();
_currentReload = Observable
.Merge(reload, newReqsLookup, invisibleReqsLookup)
.TakeLast(1)
.Do(_ =>
{
_currentReload = null;
Reloaded.Raise(this);
})
.Finally(() => _currentReload = null)
.Catch((Exception ex) =>
{
_log.Error(ex);
Network.NetworkStatus = Util.ConnectionFailedStatus;
errorMessage.Text = Network.NetworkMessage = ex.Message;
connectionError.Show();
_currentReload = null;
ReloadFailed.Raise(this);
return(Observable.Empty <Unit>());
})
.Subscribe();
ReloadStarted.Raise(this);
}
public void Reload()
{
this.connectionError.Visible = Network.NetworkStatus != Util.ConnectionOKStatus;
this.errorMessage.Text = Network.NetworkMessage;
if (Network.NetworkStatus != Util.ConnectionOKStatus)
{
return;
}
var reload =
DnsServiceManager.GetRequestList(Network, Page * _pageSize, _pageSize, Network.Filter, _sortColumn, _isSortAscending)
.TakeLast(1)
.ObserveOn(this)
.Select(list =>
{
PageCount = list.TotalCount / _pageSize + (list.TotalCount == 0 || list.TotalCount % _pageSize > 0 ? 1 : 0);
lblTotalPages.Text = PageCount.ToString();
if (Page > PageCount)
{
Page = PageCount;
txtPageNumber.Text = (Page + 1).ToString();
}
var selected = SelectedRequest;
dgvRequestList.DataSource = list.Segment;
SelectedRequest = selected == null ? null : list.Segment.FirstOrDefault(rq => rq.ID == selected.ID && rq.DataMartID == selected.DataMartID);
DisplaySortGlyph();
EnableDisablePageButtons();
return(Unit.Default);
});
var now = DateTime.Now;
var newReqsLookup =
DnsServiceManager.GetRequestList(Network, 0, 1, new RequestFilter {
DateRange = DateRangeKind.Exact, FromDate = _lastNewRequestsLookup
}, null, null)
.Take(1)
.ObserveOn(this)
.Select(n =>
{
if (n.TotalCount > 0)
{
NewRequestsAvailable = true;
}
_lastNewRequestsLookup = now.AddMilliseconds(1);
return(Unit.Default);
});
var invFilter = InvertFilter(_lastInvisibleRequestsLookup);
var invisibleReqsLookup = invFilter == null?Observable.Return(Unit.Default) :
DnsServiceManager.GetRequestList(Network, 0, 1, invFilter.Value, null, null)
.Take(1)
.ObserveOn(this)
.Select(n =>
{
if (n.TotalCount > 0)
{
InvisibleRequestsAvailable = true;
}
_lastInvisibleRequestsLookup = now.AddMilliseconds(1);
return(Unit.Default);
});
_currentReload.CallDispose();
_currentReload = Observable
.Merge(reload, newReqsLookup, invisibleReqsLookup)
.TakeLast(1)
.Do(_ =>
{
_currentReload = null;
Reloaded.Raise(this);
})
.Finally(() => _currentReload = null)
.Catch((Exception ex) =>
{
_log.Error(ex);
Network.NetworkStatus = Util.ConnectionFailedStatus;
errorMessage.Text = Network.NetworkMessage = ex.Message;
connectionError.Show();
_currentReload = null;
ReloadFailed.Raise(this);
return(Observable.Empty <Unit>());
})
.Subscribe();
ReloadStarted.Raise(this);
}
