public async Task OriginalGetRequestsForAutoProcessor()
{
//If there are more than 10 pending requests within the last 30 days that are possible for autoprocessing, only the first 10 will be retrieved
var datamartIDs = _networkSetting.DataMartList
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || dm.ProcessQueriesAndNotUpload || dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId).ToArray();
if (datamartIDs.Length == 0)
{
throw new NotSupportedException("No datamarts enabled for autoprocessing.");
}
var requestFilter = new RequestFilter
{
Statuses = new[] { DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted, DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted },
DataMartIds = datamartIDs
};
int batchSize = 10;
var requests = from list in DnsServiceManager.GetRequestList("AutoProcessor", _networkSetting, 0, batchSize, requestFilter, null, null)
from rl in list.Segment.EmptyIfNull().ToObservable()
where rl.AllowUnattendedProcessing
from r in RequestCache.ForNetwork(_networkSetting).LoadRequest(rl.ID, rl.DataMartID)
select r;
var rq = await requests.ToArray();
foreach (var r in rq)
{
_logger.Debug($"{ r.DataMartName }: { r.Source.Name} [ { r.Source.Identifier } ]");
}
}
/// <summary>
/// Execute query only and do not upload results
/// </summary>
/// <param name="qdm">Query to be processed</param>
private void ProcessRequest(HubRequest request)
{
NetWorkSetting networkSetting = Configuration.Instance.GetNetworkSetting(request.NetworkId);
_log.Info(String.Format("BackgroundProcess: Processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID));
try
{
string requestId = ProcessRequest(request, request.Processor, networkSetting);
RequestStatus.StatusCode statusCode = request.Processor.Status(requestId).Code;
if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed)
{
HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId));
hubRequestStatus.Message = request.Processor.Status(requestId).Message;
DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, networkSetting);
}
else
{
DnsServiceManager.SetRequestStatus(request, DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId)), request.Properties, networkSetting);
}
_log.Info(String.Format("The following query {3} (ID: {0}) was executed on behalf of {1}:\n\n{2}",
request.Source.ID, networkSetting.Profile.Username, "", request.Source.Identifier));
}
catch (Exception ex)
{
string message = string.Format("BackgroundProcess: An error occurred when processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2}", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID);
_log.Error(message, ex);
DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed, message), request.Properties, networkSetting);
}
_log.Info(String.Format("BackgroundProcess: Finished processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID));
}
/// <summary>
/// Execute query only and do not upload results
/// </summary>
/// <param name="qdm">Query to be processed</param>
private void ProcessRequest(HubRequest request)
{
Log.Info(string.Format("BackgroundProcess: Processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, request.NetworkId, request.Source.ID));
try
{
var key = MakeKey(request);
if (RequestStatuses.ContainsKey(key))
{
RequestStatuses[key] = ProcessingStatus.InProcessing;
}
string requestId = ProcessRequest(request, request.Processor);
RequestStatus.StatusCode statusCode = request.Processor.Status(requestId).Code;
if (statusCode == RequestStatus.StatusCode.Error)
{
HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId));
hubRequestStatus.Message = request.Processor.Status(requestId).Message;
DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, _networkSetting);
}
Log.Info(String.Format("The following query {3} (ID: {0}) was executed on behalf of {1}:\n\n{2}", request.Source.ID, _networkSetting.Profile.Username, "", request.Source.Identifier));
}
catch (Exception ex)
{
string message = string.Format("BackgroundProcess: An error occurred when processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2}", request.Source.Identifier, request.DataMartId, request.NetworkId, request.Source.ID);
Log.Error(message, ex);
DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed, message), request.Properties, _networkSetting);
}
Log.Info(string.Format("BackgroundProcess: Finished processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, request.NetworkId, request.Source.ID));
}
IObservable <HubRequest> LoadRequestImpl(Guid id, Guid dataMartId)
{
var dm = _networkSetting.DataMartList.FirstOrDefault(d => d.DataMartId == dataMartId);
return(from r in DnsServiceManager.GetRequests(_networkSetting, new[] { id }, dataMartId).SkipWhile(r => r.ID != id).Take(1)
let rt = r.Routings.EmptyIfNull().FirstOrDefault(x => x.DataMartID == dataMartId)
where rt != null
select new HubRequest
{
Source = r,
DataMartId = dataMartId,
NetworkId = _networkSetting.NetworkId,
NetworkName = _networkSetting.NetworkName,
DataMartName = dm == null ? null : dm.DataMartName,
DataMartOrgId = dm == null ? null : dm.OrganizationId,
DataMartOrgName = dm == null ? null : dm.OrganizationName,
ProjectName = r != null && r.Project != null ? r.Project.Name : null,
Properties = rt.Properties.EmptyIfNull().GroupBy(p => p.Name).ToDictionary(pp => pp.Key, pp => pp.First().Value),
Rights = r.Routings.Where(rn => rn.DataMartID == dataMartId).Select(rn => (HubRequestRights)rn.Rights).FirstOrDefault(),
RoutingStatus = rt.Status,
SubmittedDataMarts = string.Join(", ", from rtng in r.Routings
join dmt in _networkSetting.DataMartList on rtng.DataMartID equals dmt.DataMartId
select dmt.DataMartName),
Documents = r.Documents.EmptyIfNull().ToArray()
});
}
private void TestConnections()
{
int numConnectionsOk = DnsServiceManager.TestConnections(Configuration.Instance.NetworkSettingCollection.AsEnumerable());
this.Invoke(new Action(() =>
{
if (numConnectionsOk > 0)
{
DisplayInformationalStatus(string.Format("Successfully connected to {0} out of {1} Network(s)", numConnectionsOk, Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count), TextColor.Normal, toolStripStatusNetworkConnectivityStatus);
DisplayInformationalStatus(string.Format("Connected", numConnectionsOk, Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count), TextColor.Normal, toolStripLoginStatus);
DisplayInformationalStatus("Ready", TextColor.Normal, toolStripStatusNetworkConnectivityStatus);
}
else
{
int numNetworks = Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count;
if (numNetworks > 0)
{
DisplayInformationalStatus(string.Format("Failed to connect to all {0} Network(s). Check Settings.", Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count), TextColor.Error, toolStripStatusNetworkConnectivityStatus);
}
else
{
DisplayInformationalStatus(string.Format("No Network configured. Check Settings.", Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count), TextColor.Error, toolStripStatusNetworkConnectivityStatus);
}
DisplayInformationalStatus("Disconnected", TextColor.Alert, toolStripLoginStatus);
}
}));
}
public void LoadModels(NetWorkSetting ns)
{
//Get the current model information from the API.
var models = DnsServiceManager.GetModels(ns);
foreach (var d in ns.DataMartList)
{
HubModel[] m = models[d.DataMartId];
if (d.ModelList != null && !d.ModelList.IsEmpty())
{
try
{
//add missing models
var newModels = m.Where(x => !d.ModelList.Any(a => a.ModelId == x.Id)).ToArray();
if (newModels.Length > 0)
{
d.ModelList.AddRange(newModels.Select(nm => new ModelDescription {
ModelId = nm.Id, ModelName = nm.Name, ModelDisplayName = nm.Name, ProcessorId = nm.ModelProcessorId
}));
}
//remove old models
d.ModelList.RemoveAll(rm => !m.Any(a => a.Id == rm.ModelId));
}
catch (ArgumentNullException ex)
{
// Ignore if nothing to remove, which will throw an exception.
}
}
}
}
public override int Read(byte[] buffer, int offset, int count)
{
byte[] DocContent = DnsServiceManager.GetDocumentChunk(documentID, documentOffset, count, networkSetting);
int size = DocContent.Length;
Buffer.BlockCopy(DocContent, 0, buffer, offset, size);
documentOffset += size;
return(size);
}
public void LoadModels(NetWorkSetting ns)
{
foreach (var d in ns.DataMartList)
{
d.ModelList = (from m in DnsServiceManager.GetModels(d.DataMartId, ns).EmptyIfNull()
join existing in d.ModelList.EmptyIfNull() on m.Id equals existing.ModelId into exts
from existing in exts.DefaultIfEmpty()
select existing ?? new ModelDescription {
ModelId = m.Id, ModelName = m.Name, ProcessorId = m.ModelProcessorId
}
).ToList();
}
}
private void NetworkList_Load(object sender, EventArgs e)
{
try
{
this.Cursor = Cursors.WaitCursor;
DnsServiceManager.TestConnections(Configuration.Instance.NetworkSettingCollection.AsEnumerable());
LoadPage();
}
finally
{
this.Cursor = Cursors.Default;
}
}
private void reconnect_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e)
{
var progress = new ProgressForm("Connecting to the Network", "Connecting to the Network");
Observable
.Start(() => DnsServiceManager.TestConnections(new[] { Network }), Scheduler.Default)
.TakeUntil(progress.ShowAndWaitForCancel(this))
.ObserveOn(this)
.Finally(progress.Dispose)
.Do(_ => Reload())
.LogExceptions(_log.Error)
.Catch()
.Subscribe();
}
private void btnTestConnection_Click(object sender, EventArgs e)
{
try
{
this.Cursor = Cursors.WaitCursor;
lblStatusMessage.Text = string.Empty;
if (!IsValidNetworkSetting())
{
return;
}
// Test the network configuration on the UI (may or may not be saved).
//NetWorkSetting _NetworkSetting = new NetWorkSetting();
if (_NetworkSetting == null)
{
_NetworkSetting = new NetWorkSetting();
}
SaveFields(_NetworkSetting);
if (DnsServiceManager.LogIn(_NetworkSetting))
{
MessageBox.Show("Connection Successful", Util.ConnectionOKStatus, MessageBoxButtons.OK, MessageBoxIcon.Information);
}
else
{
MessageBox.Show(_NetworkSetting.NetworkMessage, Util.ConnectionFailedStatus, MessageBoxButtons.OK, MessageBoxIcon.Information);
}
PopulateDataMarts(ref _NetworkSetting);
EnableDMControls(_NetworkSetting.IsAuthenticated);
return;
}
catch (Exception ex)
{
log.Error(ex.Message, ex);
MessageBox.Show(ex.Message, "Unexpected Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
finally
{
this.Cursor = Cursors.Default;
}
}
private void SettingsForm_Load(object sender, EventArgs e)
{
try
{
chkAllowUnattendedOperation.Checked = _DataMartDescription.AllowUnattendedOperation;
rbModeProcess.Checked = _DataMartDescription.ProcessQueriesAndUploadAutomatically;
rbModeNotify.Checked = _DataMartDescription.NotifyOfNewQueries;
rbSemiAutomatic.Checked = _DataMartDescription.ProcessQueriesAndNotUpload;
update_processing_mode_controls();
Guid dataMartId = DataMartDescription.DataMartId;
dgvModels.DataSource = DnsServiceManager.GetModels(dataMartId, NetworkSetting).Where(m => m.ModelProcessorId != Guid.Empty).ToArray();
btnEdit.Enabled = dgvModels.DataSource != null && (dgvModels.DataSource as HubModel[]).Length > 0;
}
catch (Exception ex)
{
log.Error(ex.Message, ex);
}
}
private void SettingsForm_Load(object sender, EventArgs e)
{
try
{
chkAllowUnattendedOperation.Checked = _dataMartDescription.AllowUnattendedOperation;
rbModeProcess.Checked = _dataMartDescription.ProcessQueriesAndUploadAutomatically;
rbModeNotify.Checked = _dataMartDescription.NotifyOfNewQueries;
rbSemiAutomatic.Checked = _dataMartDescription.ProcessQueriesAndNotUpload;
update_processing_mode_controls();
Guid dataMartId = DataMartDescription.DataMartId;
var models = DnsServiceManager.GetModels(NetworkSetting);
if (models.ContainsKey(dataMartId))
{
dgvModels.DataSource = (models[dataMartId] ?? Array.Empty <HubModel>()).Where(m => m.ModelProcessorId != Guid.Empty).ToArray();
}
else
{
dgvModels.DataSource = Array.Empty <HubModel>();
}
btnEdit.Enabled = dgvModels.DataSource != null && (dgvModels.DataSource as HubModel[]).Length > 0;
_cache = new Lib.Caching.DataMartCacheManager(NetworkSetting.NetworkId, DataMartDescription.DataMartId);
chkEncryptCacheItems.Checked = _dataMartDescription.EncryptCacheItems;
chkExplicitCacheRemoval.Checked = _dataMartDescription.EnableExplictCacheRemoval;
nmCacheRetentionPeriod.Value = _dataMartDescription.DaysToRetainCacheItems;
chkEnableCache.Checked = _dataMartDescription.EnableResponseCaching;
chkEnableCache_CheckedChanged(null, EventArgs.Empty);
chkEnableCache.CheckedChanged += chkEnableCache_CheckedChanged;
}
catch (Exception ex)
{
log.Error(ex.Message, ex);
}
}
private void NetworkLogin()
{
try
{
if (_NetworkSetting != null && _NetworkSetting.IsAuthenticated)
{
//_NetworkSetting.LogOut();
DnsServiceManager.LogOut(_NetworkSetting);
//txtServiceUrl.ReadOnly = txtUsername.ReadOnly = txtPassword.ReadOnly = false;
//btnLogOut.Visible = false;
//btnTestConnection.Visible = true;
btnRefreshDMlist.Enabled = false;
//RefreshNetworlListDelegate();
}
}
catch (Exception ex)
{
//if (_NetworkSetting != null && !_NetworkSetting.IsAuthenticated)
// txtServiceUrl.ReadOnly = txtUsername.ReadOnly = txtPassword.ReadOnly = true;
throw ex;
}
}
private void UploadRequest(HubRequest request, Lib.Caching.DocumentCacheManager cache)
{
string requestId = request.Source.ID.ToString();
Document[] responseDocuments;
if (cache.Enabled)
{
responseDocuments = cache.GetResponseDocuments().ToArray();
}
else
{
responseDocuments = request.Processor.Response(requestId);
}
if (responseDocuments != null && responseDocuments.Length > 0)
{
Log.Info(string.Format("Uploading {4} documents for query: {0} (DataMart:{1}, Network:{2}, RequestID: {3:D})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID, responseDocuments.Length));
//since the ID of the document returned from the API is not known when posting, and the order cannot be assumed in the response
//loop over each document and post the metadata and the content as a single unit of work per document.
for (int i = 0; i < responseDocuments.Length; i++)
{
string uploadIdentifier = ("[" + Utilities.Crypto.Hash(Guid.NewGuid()) + "]").PadRight(16);
Guid[] metaDataPostResponse = DnsServiceManager.PostResponseDocuments(uploadIdentifier, requestId, request.DataMartId, new[] { responseDocuments[i] }, _networkSetting);
System.IO.Stream contentStream = null;
try
{
if (cache.Enabled)
{
contentStream = cache.GetDocumentStream(Guid.Parse(responseDocuments[i].DocumentID));
}
else
{
request.Processor.ResponseDocument(requestId, responseDocuments[i].DocumentID, out contentStream, 60000);
}
DnsServiceManager.PostResponseDocumentContent(uploadIdentifier, requestId, request.DataMartId, metaDataPostResponse[0], responseDocuments[i].Filename, contentStream, _networkSetting)
.LogExceptions(Log.Error)
.Catch()
.Wait();
}
finally
{
contentStream.CallDispose();
}
}
}
else
{
Log.Info(string.Format("No documents to upload for query: {0} (DataMart:{1}, Network:{2}, RequestID: {3:D})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID, responseDocuments.Length));
}
if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed)
{
HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId));
hubRequestStatus.Message = request.Processor.Status(requestId).Message;
DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, _networkSetting);
Log.Info(string.Format("The following query {0} (ID: {3}, DataMart: {4}, Network: {5}) failed to execute on behalf of {1}:\n\n{2}", request.Source.Identifier, _networkSetting.Profile.Username, hubRequestStatus.Message, request.Source.ID, request.DataMartName, request.NetworkName));
}
else
{
Log.Info(string.Format("The following query {0} (ID: {2}, DataMart: {3}, Network: {4}) was executed and results were uploaded automatically on behalf of {1}", request.Source.Identifier, _networkSetting.Profile.Username, request.Source.ID, request.DataMartName, request.NetworkName));
}
}
private void UploadRequest(HubRequest request)
{
NetWorkSetting networkSetting = Configuration.Instance.GetNetworkSetting(request.NetworkId);
_log.Info(String.Format("BackgroundProcess: Processing query {0} (RequestId: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID));
try
{
string requestId = request.Source.ID.ToString();
Lpp.Dns.DataMart.Model.Document[] responseDocuments = request.Processor.Response(requestId);
Guid[] documentIds = DnsServiceManager.PostResponseDocuments(requestId, request.DataMartId, responseDocuments, networkSetting);
_log.Info("Number of portal document ids returned by PostResponseDocuments: " + documentIds.Length);
for (int i = 0; i < documentIds.Length; i++)
{
_log.Info("About to post content for portal document id: " + documentIds[i] + " corresponding to response document id: " + responseDocuments[i].DocumentID);
Stream contentStream = null;
try
{
request.Processor.ResponseDocument(requestId, responseDocuments[i].DocumentID, out contentStream, 60000);
DnsServiceManager.PostResponseDocumentContent(documentIds[i], contentStream, networkSetting)
.LogExceptions(_log.Error)
.Catch()
.SubscribeOn(Scheduler.Default)
.LastOrDefault();
}
finally
{
contentStream.CallDispose();
}
}
if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed)
{
HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId));
hubRequestStatus.Message = request.Processor.Status(requestId).Message;
DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, networkSetting);
_log.Info(String.Format("The following query {0} (ID: {3}) failed to execute on behalf of {1}:\n\n{2}",
request.Source.Identifier, networkSetting.Profile.Username, hubRequestStatus.Message, request.Source.ID));
}
else
{
//MantisID:6331:Zero out low-cell counts if they exist.
//string strReason = request.Processor.Status(requestId).Code == RequestStatus.StatusCode.CompleteWithMessage ? request.Processor.Status(requestId).Message : string.Empty;
//DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Completed), request.Properties, networkSetting);
//if (string.IsNullOrEmpty(strReason))
//{
//DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(HubRequestStatus.StatusCode.Completed), request.Properties, networkSetting);
_log.Info(String.Format("The following query {0} (ID: {2}) was executed and results were uploaded automatically on behalf of {1}",
request.Source.Identifier, networkSetting.Profile.Username, request.Source.ID));
//}
//else
//{
// //DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(HubRequestStatus.StatusCode.Completed, strReason), request.Properties, networkSetting);
// _log.Info(String.Format("The following query {0} (ID: {3}) was executed and results were uploaded automatically on behalf of {1}:\n\n{2}",
// request.Source.Identifier, networkSetting.Profile.Username, strReason, request.Source.ID));
//}
}
}
catch (Exception ex)
{
string message = string.Format("An error occurred while attempting unattended processing of the following query {0} (ID: {1})", request.Source.Identifier, request.Source.ID);
_log.Error(message, ex);
DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed, message), request.Properties, networkSetting);
}
_log.Info(String.Format("BackgroundProcess: Finished Processing / Uploading results for query {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID));
}
private void UploadRequest(HubRequest request, Lib.Caching.DocumentCacheManager cache)
{
string requestId = request.Source.ID.ToString();
Document[] responseDocuments;
if (cache.Enabled)
{
responseDocuments = cache.GetResponseDocuments().ToArray();
}
else
{
responseDocuments = request.Processor.Response(requestId);
}
if (responseDocuments != null && responseDocuments.Length > 0)
{
Log.Info(string.Format("Uploading {4} documents for query: {0} (DataMart:{1}, Network:{2}, RequestID: {3:D})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID, responseDocuments.Length));
if (_networkSetting.SupportsUploadV2)
{
Log.Debug($"Network settings indicate document upload v2 supported for query: {request.Source.Identifier}, (DataMart: {request.DataMartName}, Network: { request.NetworkName }, RequestID: { request.Source.ID }");
try
{
Parallel.ForEach(responseDocuments,
new ParallelOptions {
MaxDegreeOfParallelism = 4
},
(doc) =>
{
string uploadIdentifier = ("[" + Utilities.Crypto.Hash(Guid.NewGuid()) + "]").PadRight(16);
System.IO.Stream stream = null;
try
{
if (cache.Enabled)
{
stream = cache.GetDocumentStream(Guid.Parse(doc.DocumentID));
}
else
{
request.Processor.ResponseDocument(requestId, doc.DocumentID, out stream, doc.Size);
}
var dto = new DTO.DataMartClient.Criteria.DocumentMetadata
{
ID = Guid.Parse(doc.DocumentID),
DataMartID = request.DataMartId,
RequestID = Guid.Parse(requestId),
IsViewable = doc.IsViewable,
Size = doc.Size,
MimeType = doc.MimeType,
Kind = doc.Kind,
Name = doc.Filename,
CurrentChunkIndex = 0
};
DnsServiceManager.PostDocumentChunk(uploadIdentifier, dto, stream, _networkSetting);
}
finally
{
stream.Dispose();
}
});
}
catch (Exception ex)
{
Log.Error($"An error occurred during concurrent uploading of response documents for query: {request.Source.Identifier}, (DataMart: {request.DataMartName}, Network: { request.NetworkName }, RequestID: { request.Source.ID }", ex);
throw;
}
}
else
{
//since the ID of the document returned from the API is not known when posting, and the order cannot be assumed in the response
//loop over each document and post the metadata and the content as a single unit of work per document.
Log.Debug($"Network settings indicates document upload v2 IS NOT supported for query: {request.Source.Identifier}, (DataMart: {request.DataMartName}, Network: { request.NetworkName }, RequestID: { request.Source.ID }");
for (int i = 0; i < responseDocuments.Length; i++)
{
string uploadIdentifier = ("[" + Utilities.Crypto.Hash(Guid.NewGuid()) + "]").PadRight(16);
Guid[] metaDataPostResponse = DnsServiceManager.PostResponseDocuments(uploadIdentifier, requestId, request.DataMartId, new[] { responseDocuments[i] }, _networkSetting);
System.IO.Stream contentStream = null;
try
{
if (cache.Enabled)
{
contentStream = cache.GetDocumentStream(Guid.Parse(responseDocuments[i].DocumentID));
}
else
{
request.Processor.ResponseDocument(requestId, responseDocuments[i].DocumentID, out contentStream, 60000);
}
DnsServiceManager.PostResponseDocumentContent(uploadIdentifier, requestId, request.DataMartId, metaDataPostResponse[0], responseDocuments[i].Filename, contentStream, _networkSetting)
.LogExceptions(Log.Error)
.Catch()
.Wait();
}
finally
{
contentStream.CallDispose();
}
}
}
}
else
{
Log.Info(string.Format("No documents to upload for query: {0} (DataMart:{1}, Network:{2}, RequestID: {3:D})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID, responseDocuments.Length));
}
if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed)
{
HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId));
hubRequestStatus.Message = request.Processor.Status(requestId).Message;
DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, _networkSetting);
Log.Info(string.Format("The following query {0} (ID: {3}, DataMart: {4}, Network: {5}) failed to execute on behalf of {1}:\n\n{2}", request.Source.Identifier, _networkSetting.Profile.Username, hubRequestStatus.Message, request.Source.ID, request.DataMartName, request.NetworkName));
}
else
{
Log.Info(string.Format("The following query {0} (ID: {2}, DataMart: {3}, Network: {4}) was executed and results were uploaded automatically on behalf of {1}", request.Source.Identifier, _networkSetting.Profile.Username, request.Source.ID, request.DataMartName, request.NetworkName));
}
}
private void PopulateDataMarts(ref NetWorkSetting ns)
{
try
{
if (!DnsServiceManager.LogIn(ns))
{
return;
}
// Find matching existing datamarts and update their info. Append new datamarts and remove deleted ones.
var newDMs = ns.GetDataMartsByUser().EmptyIfNull();
if ((dgvDataMarts.DataSource as IEnumerable <DataMartDescription>).NullOrEmpty())
{
dgvDataMarts.DataSource = newDMs;
}
else
{
(from oldDataMarts in dgvDataMarts.DataSource as IEnumerable <DataMartDescription>
join newDataMarts in newDMs on oldDataMarts.DataMartId equals newDataMarts.DataMartId
select new { oldDM = oldDataMarts, newDM = newDataMarts }).ForEach(d =>
{
// Append new datamarts.
// For each existing datamart, find matching existing model and update their info.
d.oldDM.DataMartName = d.newDM.DataMartName;
d.oldDM.OrganizationId = d.newDM.OrganizationId;
d.oldDM.OrganizationName = d.newDM.OrganizationName;
if (d.oldDM.ModelList != null && d.newDM.ModelList != null)
{
(from oldModel in d.oldDM.ModelList.Cast <ModelDescription>()
join newModel in d.newDM.ModelList.Cast <ModelDescription>() on oldModel.ModelId equals newModel.ModelId
select new { oldModel = oldModel, newModel = newModel }).ForEach(m =>
{
m.oldModel.ModelName = m.newModel.ModelName;
m.oldModel.ModelDisplayName = m.newModel.ModelDisplayName;
});
}
});
var ds = new List <DataMartDescription>();
ds.AddRange(dgvDataMarts.DataSource as IEnumerable <DataMartDescription>);
ds.AddRange(from newDataMart in newDMs
where !ds.Any(es => (es.DataMartId == newDataMart.DataMartId))
select newDataMart);
ds.RemoveAll(r => !newDMs.Any(es => es.DataMartId == r.DataMartId));
dgvDataMarts.DataSource = ds;
}
dgvDataMarts.Refresh();
if (ns.DataMartList != null && ns.DataMartList.Count > 0)
{
Configuration.Instance.LoadModels(ns);
btnEdit.Enabled = btnRefreshDMlist.Enabled = true;
btnApply.Enabled = btnOk.Enabled = true;
dgvDataMarts.Enabled = true;
}
else
{
dgvDataMarts.Enabled = false;
btnEdit.Enabled = false;
MessageBox.Show("No network DataMart(s) found", Application.ProductName);
}
}
catch (Exception ex)
{
throw ex;
}
}
private void btnOk_Click(object sender, EventArgs e)
{
try
{
this.Cursor = Cursors.WaitCursor;
// Get the network setting for this service url if it exists already.
NetWorkSetting ns = null;
try
{
ns = (from n in Configuration.Instance.NetworkSettingCollection.NetWorkSettings.ToArray(typeof(NetWorkSetting)) as NetWorkSetting[]
where n.HubWebServiceUrl == txtServiceUrl.Text
select n).FirstOrDefault();
}
catch
{
Configuration.CreateNewNetworkSettingsFile();
}
// If not, create one.
if (ns == null)
{
ns = new NetWorkSetting();
ns.HubWebServiceUrl = txtServiceUrl.Text;
Configuration.Instance.AddNetworkSetting(ns);
}
else
{
if (MessageBox.Show("This will override your existing configuration for this network.\nAre you sure you want to do that?", "Override Network Configuration?", MessageBoxButtons.OKCancel, MessageBoxIcon.Question) == DialogResult.Cancel)
{
this.Close();
return;
}
}
ns.Username = txtUsername.Text;
ns.EncryptedPassword = txtPassword.Text;
ns.NetworkName = txtNetworkname.Text;
var selectedCert = cbCertificates.SelectedItem as Cert;
ns.X509CertThumbprint = selectedCert == null ? null : selectedCert.Thumbprint;
ns.GetDataMartsByUser();
Configuration.Instance.LoadModels(ns);
CredentialManager.SaveCredential(ns.CredentialKey, ns.Username, ns.Password);
// Get the configuration for all models for all datamarts at this service url.
var dmc = DnsServiceManager.GetDataMartModelConfigurations(ns);
// Find the model processor in the existing network if exists. If not, create it.
XmlSerializer serializer = new XmlSerializer(typeof(ModelProperties));
foreach (var c in dmc)
{
var d = ns.DataMartList.FirstOrDefault(dm => dm.DataMartId == c.DataMartId);
var mp = d.ModelList.FirstOrDefault(m => m.ModelId == c.ModelId && m.ProcessorId == c.ModelProcessorId);
if (d != null && mp != null && !string.IsNullOrEmpty(c.Properties))
{
using (XmlTextReader reader = new XmlTextReader(new MemoryStream(new System.Text.UTF8Encoding().GetBytes(c.Properties))))
{
d.AllowUnattendedOperation = c.UnattendedMode != Lpp.Dns.DataMart.Lib.Classes.UnattendedMode.NoUnattendedOperation;
d.NotifyOfNewQueries = c.UnattendedMode == Lpp.Dns.DataMart.Lib.Classes.UnattendedMode.NotifyOnly;
d.ProcessQueriesAndNotUpload = c.UnattendedMode == Lpp.Dns.DataMart.Lib.Classes.UnattendedMode.ProcessNoUpload;
d.ProcessQueriesAndUploadAutomatically = c.UnattendedMode == Lpp.Dns.DataMart.Lib.Classes.UnattendedMode.ProcessAndUpload;
var p = new List <PropertyData>();
p.AddRange(((ModelProperties)serializer.Deserialize(reader)).Properties.Where(prop => prop.Name != "ConfirmPassword"));
mp.Properties = p;
if (mp.ProcessorId != Guid.Empty)
{
try
{
//TODO: confirm the latest package exists for the model/processor combination
var packageIdentifier = DnsServiceManager.GetRequestTypeIdentifier(ns, mp.ModelId, mp.ProcessorId);
if (packageIdentifier != null && !File.Exists(Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.Identifier + "." + packageIdentifier.Version + ".zip")))
{
DnsServiceManager.DownloadPackage(ns, packageIdentifier);
}
//string processorPath, className;
//ProcessorManager.FindProcessor(mp.ModelId, mp.ProcessorId, out processorPath, out className);
//mp.ProcessorPath = processorPath;
//mp.ClassName = className;
}
catch (Exception ex)
{
log.Error("Unable to load processor.", ex);
}
}
}
}
}
Configuration.SaveNetworkSettings();
if (NetworkSettingChanged != null)
{
NetworkSettingChanged(this, ns);
}
}
catch (Exception ex)
{
MessageBox.Show(ex.Message, "Update Failed", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
finally
{
this.Cursor = Cursors.Default;
this.Close();
}
}
public void Reload()
{
this.connectionError.Visible = Network.NetworkStatus != Util.ConnectionOKStatus;
this.errorMessage.Text = Network.NetworkMessage;
if (Network.NetworkStatus != Util.ConnectionOKStatus)
{
return;
}
var reload =
DnsServiceManager.GetRequestList("filtered requests lookup", Network, Page * _pageSize, _pageSize, Network.Filter, _sortColumn, _isSortAscending)
.TakeLast(1)
.ObserveOn(this)
.Select(list =>
{
PageCount = list.TotalCount / _pageSize + (list.TotalCount == 0 || list.TotalCount % _pageSize > 0 ? 1 : 0);
lblTotalPages.Text = PageCount.ToString();
if (Page > PageCount)
{
Page = PageCount;
txtPageNumber.Text = (Page + 1).ToString();
}
var selected = SelectedRequest;
//update request statuses that have cached results that are awaiting upload - Pending Upload is a local status and is not known by the API
foreach (var seg in list.Segment.Where(s => s.Status == DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted || s.Status == DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted))
{
var cache = new Lib.Caching.DocumentCacheManager(Network.NetworkId, seg.DataMartID, seg.ID, seg.ResponseID);
if (cache.HasResponseDocuments)
{
seg.Status = DTO.DataMartClient.Enums.DMCRoutingStatus.PendingUpload;
}
}
dgvRequestList.DataSource = list.Segment;
SelectedRequest = selected == null ? null : list.Segment.FirstOrDefault(rq => rq.ID == selected.ID && rq.DataMartID == selected.DataMartID);
DisplaySortGlyph();
EnableDisablePageButtons();
return(Unit.Default);
});
var now = DateTime.Now;
var newReqsLookup =
DnsServiceManager.GetRequestList("new requests lookup", Network, 0, 1, new RequestFilter {
DateRange = DateRangeKind.Exact, FromDate = _lastNewRequestsLookup
}, null, null)
.Take(1)
.ObserveOn(this)
.Select(n =>
{
if (n.TotalCount > 0)
{
NewRequestsAvailable = true;
}
_lastNewRequestsLookup = now.AddMilliseconds(1);
return(Unit.Default);
});
var invFilter = InvertFilter(_lastInvisibleRequestsLookup);
var invisibleReqsLookup = invFilter == null?Observable.Return(Unit.Default) :
DnsServiceManager.GetRequestList("hidden requests lookup", Network, 0, 1, invFilter.Value, null, null)
.Take(1)
.ObserveOn(this)
.Select(n =>
{
if (n.TotalCount > 0)
{
InvisibleRequestsAvailable = true;
}
_lastInvisibleRequestsLookup = now.AddMilliseconds(1);
return(Unit.Default);
});
_currentReload.CallDispose();
_currentReload = Observable
.Merge(reload, newReqsLookup, invisibleReqsLookup)
.TakeLast(1)
.Do(_ =>
{
_currentReload = null;
Reloaded.Raise(this);
})
.Finally(() => _currentReload = null)
.Catch((Exception ex) =>
{
_log.Error(ex);
Network.NetworkStatus = Util.ConnectionFailedStatus;
errorMessage.Text = Network.NetworkMessage = ex.Message;
connectionError.Show();
_currentReload = null;
ReloadFailed.Raise(this);
return(Observable.Empty <Unit>());
})
.Subscribe();
ReloadStarted.Raise(this);
}
public void Reload()
{
this.connectionError.Visible = Network.NetworkStatus != Util.ConnectionOKStatus;
this.errorMessage.Text = Network.NetworkMessage;
if (Network.NetworkStatus != Util.ConnectionOKStatus)
{
return;
}
var reload =
DnsServiceManager.GetRequestList(Network, Page * _pageSize, _pageSize, Network.Filter, _sortColumn, _isSortAscending)
.TakeLast(1)
.ObserveOn(this)
.Select(list =>
{
PageCount = list.TotalCount / _pageSize + (list.TotalCount == 0 || list.TotalCount % _pageSize > 0 ? 1 : 0);
lblTotalPages.Text = PageCount.ToString();
if (Page > PageCount)
{
Page = PageCount;
txtPageNumber.Text = (Page + 1).ToString();
}
var selected = SelectedRequest;
dgvRequestList.DataSource = list.Segment;
SelectedRequest = selected == null ? null : list.Segment.FirstOrDefault(rq => rq.ID == selected.ID && rq.DataMartID == selected.DataMartID);
DisplaySortGlyph();
EnableDisablePageButtons();
return(Unit.Default);
});
var now = DateTime.Now;
var newReqsLookup =
DnsServiceManager.GetRequestList(Network, 0, 1, new RequestFilter {
DateRange = DateRangeKind.Exact, FromDate = _lastNewRequestsLookup
}, null, null)
.Take(1)
.ObserveOn(this)
.Select(n =>
{
if (n.TotalCount > 0)
{
NewRequestsAvailable = true;
}
_lastNewRequestsLookup = now.AddMilliseconds(1);
return(Unit.Default);
});
var invFilter = InvertFilter(_lastInvisibleRequestsLookup);
var invisibleReqsLookup = invFilter == null?Observable.Return(Unit.Default) :
DnsServiceManager.GetRequestList(Network, 0, 1, invFilter.Value, null, null)
.Take(1)
.ObserveOn(this)
.Select(n =>
{
if (n.TotalCount > 0)
{
InvisibleRequestsAvailable = true;
}
_lastInvisibleRequestsLookup = now.AddMilliseconds(1);
return(Unit.Default);
});
_currentReload.CallDispose();
_currentReload = Observable
.Merge(reload, newReqsLookup, invisibleReqsLookup)
.TakeLast(1)
.Do(_ =>
{
_currentReload = null;
Reloaded.Raise(this);
})
.Finally(() => _currentReload = null)
.Catch((Exception ex) =>
{
_log.Error(ex);
Network.NetworkStatus = Util.ConnectionFailedStatus;
errorMessage.Text = Network.NetworkMessage = ex.Message;
connectionError.Show();
_currentReload = null;
ReloadFailed.Raise(this);
return(Observable.Empty <Unit>());
})
.Subscribe();
ReloadStarted.Raise(this);
}
protected override void OnDoWork(DoWorkEventArgs e)
{
_log.Info("Automated processing worker started.");
while (true)
{
_log.Info("Checking Master Request Queue for requests requiring automated processing.");
SystemTray.UpdateNotificationIcon(IconType.IconBusy, "Processing Requests");
var reqs = from ns in Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Cast <NetWorkSetting>().ToObservable()
where ns.NetworkStatus == Util.ConnectionOKStatus
let dmIds = ns.DataMartList
// BMS: Note the ProcessAndNotUpload feature has been temporarilty disabled until we fix the processing around this feature
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || /* dm.ProcessQueriesAndNotUpload || */ dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId)
.ToArray()
where dmIds.Any()
from list in DnsServiceManager.GetRequestList(ns, 0, Properties.Settings.Default.AutoProcessingBatchSize,
new RequestFilter {
Statuses = new [] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted }, DataMartIds = dmIds
}, null, null)
from rl in list.Segment.EmptyIfNull().ToObservable()
where rl.AllowUnattendedProcessing
from r in RequestCache.ForNetwork(ns).LoadRequest(rl.ID, rl.DataMartID)
select new { Request = r, NetworkSetting = ns };
reqs
.Do(r =>
{
var request = r.Request;
var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId);
var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID);
var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier {
Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion
};
if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
{
DnsServiceManager.DownloadPackage(r.NetworkSetting, packageIdentifier);
}
using (var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath))
{
domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion);
IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId);
ProcessorManager.UpdateProcessorSettings(modelDescription, processor);
if (processor is IEarlyInitializeModelProcessor)
{
((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, r.NetworkSetting))).ToArray());
}
if (processor != null &&
processor.ModelMetadata != null &&
processor.ModelMetadata.Capabilities != null &&
processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") &&
!(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"])
{
//can't be run, don't attempt autoprocessing
return;
}
request.Processor = processor;
if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted)
{
if (processor != null)
{
SystemTray.generate_notification(request, request.NetworkId);
if (datamartDescription.NotifyOfNewQueries)
{
SystemTray.UpdateNotificationIcon(IconType.IconBusy, string.Format("Query Submitted by {0}", request.Source.Author.Username));
}
else
{
processor.SetRequestProperties(request.Source.ID.ToString(), request.Properties);
ProcessRequest(request);
var statusCode = request.Processor.Status(request.Source.ID.ToString()).Code;
if (datamartDescription.ProcessQueriesAndUploadAutomatically && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage))
{
// Post process requests that are automatically uploaded
processor.PostProcess(request.Source.ID.ToString());
// Increment counter
_queriesProcessedCount++;
SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
UploadRequest(request);
}
}
}
}
else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval)
{
if (datamartDescription.ProcessQueriesAndUploadAutomatically)
{
// Increment counter
_queriesProcessedCount++;
SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
UploadRequest(request);
}
}
}
})
.LogExceptions(_log.Error)
.Catch()
.LastOrDefault();
SystemTray.UpdateNotificationIcon(IconType.IconDefault, null);
Thread.Sleep(DMClient.Properties.Settings.Default.RefreshRate);
}
}
private void AutoProcess(KeyValuePair <string, HubRequest> input)
{
var request = input.Value;
var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId);
if (datamartDescription.ProcessQueriesAndUploadAutomatically == false && datamartDescription.ProcessQueriesAndNotUpload == false)
{
//just notify, do not process
string message = $"New query submitted and awaiting processing in { request.ProjectName } Project: { request.Source.Name } ({request.Source.Identifier})";
SystemTray.DisplayNewQueryNotificationToolTip(message);
RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload);
return;
}
var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID);
var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier {
Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion
};
if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
{
DnsServiceManager.DownloadPackage(_networkSetting, packageIdentifier);
}
var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath);
try
{
domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion);
IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId);
ProcessorManager.UpdateProcessorSettings(modelDescription, processor);
processor.Settings.Add("NetworkId", request.NetworkId);
Lib.Caching.DocumentCacheManager cache = new Lib.Caching.DocumentCacheManager(request.NetworkId, request.DataMartId, request.Source.ID, request.Source.Responses.Where(x => x.DataMartID == request.DataMartId).FirstOrDefault().ResponseID);
//need to initialize before checking the capabilities and settings of the processor since they may change based on the type of request being sent.
if (processor is IEarlyInitializeModelProcessor)
{
((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, _networkSetting))).ToArray());
}
if (processor != null &&
processor.ModelMetadata != null &&
processor.ModelMetadata.Capabilities != null &&
processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") &&
!(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"])
{
//can't be run, don't attempt autoprocessing
RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload);
domainManager.Dispose();
return;
}
request.Processor = processor;
if (cache.HasResponseDocuments == false && (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted || request.RoutingStatus == DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted))
{
if (processor != null)
{
SystemTray.GenerateNotification(request, request.NetworkId);
StartProcessingRequest(request, processor, datamartDescription, domainManager, cache);
return;
}
}
else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval)
{
if (datamartDescription.ProcessQueriesAndUploadAutomatically)
{
// Increment counter
System.Threading.Interlocked.Increment(ref _queriesProcessedCount);
SystemTray.UpdateNotifyText(_queriesProcessedCount, request.DataMartName, request.NetworkId);
StartUploadingRequest(request, domainManager, cache);
return;
}
}
else if (cache.HasResponseDocuments)
{
RequestStatuses.TryAdd(input.Key, ProcessingStatus.PendingUpload);
}
domainManager.Dispose();
}
catch (Exception ex)
{
Log.Error($"Error autoprocessing Request: { request.Source.Identifier }, DataMartId: { request.DataMartId }, NetworkId: {request.NetworkId}", ex);
domainManager.Dispose();
throw;
}
}
async Task <HubRequest[]> GetRequestsAsync()
{
var datamartIDs = _networkSetting.DataMartList
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || dm.ProcessQueriesAndNotUpload || dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId).ToArray();
if (datamartIDs.Length == 0)
{
return(Array.Empty <HubRequest>());
}
var requestFilter = new RequestFilter
{
Statuses = new[] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted, Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted },
DataMartIds = datamartIDs,
FromDate = DateTime.UtcNow.AddYears(-1),
DateRange = DateRangeKind.Exact
};
var ob = Observable.Create <DTO.DataMartClient.RequestList>(async observer =>
{
int index = 0;
int batchSize = 2;
DTO.DataMartClient.RequestList rl = null;
while (rl == null || (index < rl.TotalCount))
{
_logger.Debug($"Observer loop: pageIndex={ index }.");
rl = await DnsServiceManager.GetRequestList("RxQueryTests-DefaultDSM", _networkSetting, index, batchSize, requestFilter, null, null);
_logger.Debug($"Observer loop: pageIndex={ index }, results returned={ rl.Segment.DefaultIfEmpty().Count() }, total results={ rl.TotalCount }");
if (rl.TotalCount == 0)
{
break;
}
observer.OnNext(rl);
index += batchSize;
}
_logger.Debug("Observer loop firing OnComplete");
observer.OnCompleted();
}).DefaultIfEmpty().Aggregate((requestList1, requestList2) => {
if (requestList1 == null && requestList2 == null)
{
return(new DTO.DataMartClient.RequestList
{
Segment = Array.Empty <DTO.DataMartClient.RequestListRow>(),
SortedAscending = false,
SortedByColumn = DTO.DataMartClient.RequestSortColumn.RequestTime
});
}
else if (requestList1 != null && requestList2 == null)
{
return(requestList1);
}
else if (requestList1 == null && requestList2 != null)
{
return(requestList2);
}
else
{
return(new DTO.DataMartClient.RequestList
{
Segment = requestList1.Segment.EmptyIfNull().Concat(requestList2.Segment.EmptyIfNull()).ToArray(),
SortedAscending = requestList1.SortedAscending,
SortedByColumn = requestList1.SortedByColumn,
StartIndex = requestList1.StartIndex,
TotalCount = requestList1.TotalCount
});
}
})
.SelectMany(requestList => {
if (requestList == null)
{
return(Array.Empty <DTO.DataMartClient.RequestListRow>());
}
return(requestList.Segment.DefaultIfEmpty().Where(s => s.AllowUnattendedProcessing));
})
.SelectMany(rlr => RequestCache.ForNetwork(_networkSetting).LoadRequest(rlr.ID, rlr.DataMartID))
.ToArray();
return(await ob);
}
private void StartProcessingRequest(HubRequest request, IModelProcessor processor, DataMartDescription datamartDescription, DomainManger.DomainManager domainManager, Lib.Caching.DocumentCacheManager cache)
{
Action process = () =>
{
processor.SetRequestProperties(request.Source.ID.ToString(), request.Properties);
ProcessRequest(request);
};
Action <Task> continuation = (completed) =>
{
HubRequestStatus hubRequestStatus = null;
var statusCode = request.Processor.Status(request.Source.ID.ToString()).Code;
if (cache.Enabled)
{
Document[] responseDocuments = processor.Response(request.Source.ID.ToString());
cache.Add(responseDocuments.Select(doc => {
System.IO.Stream data;
processor.ResponseDocument(request.Source.ID.ToString(), doc.DocumentID, out data, doc.Size);
Guid documentID;
if (!Guid.TryParse(doc.DocumentID, out documentID))
{
documentID = Utilities.DatabaseEx.NewGuid();
doc.DocumentID = documentID.ToString();
}
return(new DocumentWithStream(documentID, doc, data));
}));
}
if (datamartDescription.ProcessQueriesAndNotUpload && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage))
{
RequestStatuses.TryAdd(MakeKey(request), ProcessingStatus.PendingUpload);
}
else if (datamartDescription.ProcessQueriesAndUploadAutomatically && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage))
{
// Post process requests that are automatically uploaded
processor.PostProcess(request.Source.ID.ToString());
if (cache.Enabled)
{
cache.ClearCache();
Document[] responseDocuments = processor.Response(request.Source.ID.ToString());
cache.Add(responseDocuments.Select(doc => {
System.IO.Stream data;
processor.ResponseDocument(request.Source.ID.ToString(), doc.DocumentID, out data, doc.Size);
Guid documentID;
if (!Guid.TryParse(doc.DocumentID, out documentID))
{
documentID = Utilities.DatabaseEx.NewGuid();
doc.DocumentID = documentID.ToString();
}
return(new DocumentWithStream(documentID, doc, data));
}));
}
// Increment counter
System.Threading.Interlocked.Increment(ref _queriesProcessedCount);
statusCode = request.Processor.Status(request.Source.ID.ToString()).Code;
if (statusCode == RequestStatus.StatusCode.Error)
{
hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(request.Source.ID.ToString()));
hubRequestStatus.Message = request.Processor.Status(request.Source.ID.ToString()).Message;
}
else if (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage)
{
SystemTray.UpdateNotifyText(_queriesProcessedCount, request.DataMartName, request.NetworkId);
try
{
UploadRequest(request, cache);
statusCode = request.Processor.Status(request.Source.ID.ToString()).Code;
hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(request.Source.ID.ToString()));
hubRequestStatus.Message = request.Processor.Status(request.Source.ID.ToString()).Message;
}
catch (Exception ex)
{
string message = string.Format("An error occurred while attempting unattended processing of the following query {0} (ID: {1}, DataMart: {2}, Network: {3})", request.Source.Identifier, request.Source.ID, request.DataMartName, request.NetworkName);
Log.Error(message, ex);
hubRequestStatus = new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed, message);
}
}
else
{
statusCode = request.Processor.Status(request.Source.ID.ToString()).Code;
hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(request.Source.ID.ToString()));
hubRequestStatus.Message = request.Processor.Status(request.Source.ID.ToString()).Message;
}
DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, _networkSetting);
Log.Info(string.Format("BackgroundProcess: Finished Processing / Uploading results for query {0} (RequestID: {3}, DataMart: {1}, Network: {2})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID));
if (statusCode == RequestStatus.StatusCode.Error || statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage || statusCode == RequestStatus.StatusCode.AwaitingResponseApproval)
{
//mark auto-processing complete in the status cache if the processing resulted in either an error or a completed status
var key = MakeKey(request);
if (RequestStatuses.ContainsKey(key))
{
RequestStatuses[key] = ProcessingStatus.Complete;
}
}
}
};
RunTask(MakeKey(request), domainManager, process, continuation);
}
private void SettingsForm_Load(object sender, EventArgs e)
{
Lpp.Dns.DataMart.Client.DomainManger.DomainManager domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath);
try
{
if (ModelDescription == null)
{
return;
}
if (ModelDescription.ProcessorId == Guid.Empty)
{
ModelDescription.ProcessorId = HubModel.ModelProcessorId;
}
//get the package identifier and version
var packageIdentifier = DnsServiceManager.GetRequestTypeIdentifier(NetworkSetting, ModelDescription.ModelId, ModelDescription.ProcessorId);
if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
{
DnsServiceManager.DownloadPackage(NetworkSetting, packageIdentifier);
}
domainManager.Load(packageIdentifier.Identifier, packageIdentifier.Version);
modelProcessor = domainManager.GetProcessor(ModelDescription.ProcessorId);
ProcessorManager.UpdateProcessorSettings(ModelDescription, modelProcessor);
if (modelProcessor is IEarlyInitializeModelProcessor)
{
((IEarlyInitializeModelProcessor)modelProcessor).Initialize(ModelDescription.ModelId, Array.Empty <DocumentWithStream>());
}
this.SuspendLayout();
if (modelProcessor.ModelMetadata.Settings == null || !modelProcessor.ModelMetadata.Settings.Any())
{
var noSettingsLabel = new Label();
noSettingsLabel.Text = "This model processor does not have any settings.";
noSettingsLabel.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left;
noSettingsLabel.TextAlign = ContentAlignment.MiddleCenter;
tableLayoutPanel1.Controls.Add(noSettingsLabel, 0, 0);
tableLayoutPanel1.SetColumnSpan(noSettingsLabel, 2);
}
else
{
_editors = new HashSet <Control>();
tableLayoutPanel1.RowStyles.Clear();
int rowIndex = 0;
IEnumerable <Lpp.Dns.DataMart.Model.Settings.ProcessorSetting> settings = modelProcessor.ModelMetadata.Settings;
if (modelProcessor.ModelMetadata.SQlProviders != null && modelProcessor.ModelMetadata.SQlProviders.Any())
{
DataMart.Client.Controls.DataSourceEditor dsEditor = new Controls.DataSourceEditor(modelProcessor.ModelMetadata, ModelDescription.Properties);
dsEditor.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left;
tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.AutoSize));
tableLayoutPanel1.Controls.Add(dsEditor, 0, rowIndex++);
tableLayoutPanel1.SetColumnSpan(dsEditor, 2);
settings = settings.Where(s => !Lpp.Dns.DataMart.Model.Settings.ProcessorSettings.IsDbSetting(s.Key)).ToArray();
_editors.Add(dsEditor);
}
settings.ToList().ForEach(s => {
string value = ModelDescription.Properties.Where(p => string.Equals(p.Name, s.Key, StringComparison.OrdinalIgnoreCase)).Select(p => p.Value).FirstOrDefault();
if (string.IsNullOrEmpty(value) && s.Required && !string.IsNullOrEmpty(s.DefaultValue))
{
value = s.DefaultValue;
}
Label lbl = new Label();
lbl.AutoSize = true;
lbl.Anchor = AnchorStyles.Right;
lbl.TextAlign = ContentAlignment.MiddleRight;
lbl.Text = s.Title.EndsWith(":") ? s.Title : s.Title + ":";
Control editor = null;
if (s.ValidValues != null)
{
ComboBox combo = new ComboBox();
combo.DropDownStyle = ComboBoxStyle.DropDownList;
combo.Anchor = AnchorStyles.Right | AnchorStyles.Left;
combo.Name = s.Key;
combo.Items.AddRange(s.ValidValues.Select(v => new PropertyData(v.Key, v.Value.ToString())).ToArray());
if (!string.IsNullOrEmpty(value))
{
foreach (PropertyData p in combo.Items)
{
if (string.Equals(p.Value, value, StringComparison.OrdinalIgnoreCase))
{
combo.SelectedItem = p;
break;
}
}
}
if (combo.SelectedIndex < 0)
{
combo.SelectedIndex = 0;
}
editor = combo;
}
else
{
if (s.ValueType == typeof(bool) || s.ValueType == typeof(Nullable <bool>))
{
CheckBox chkbox = new CheckBox();
chkbox.Anchor = AnchorStyles.Left;
chkbox.Text = s.Title;
chkbox.TextAlign = ContentAlignment.MiddleLeft;
chkbox.AutoSize = true;
if (!string.IsNullOrEmpty(value))
{
bool isChecked = false;
bool.TryParse(value, out isChecked);
chkbox.Checked = isChecked;
}
editor = chkbox;
lbl = null;
}
else if (s.ValueType == typeof(Lpp.Dns.DataMart.Model.Settings.FilePickerEditor))
{
SelectFileButton btn = new SelectFileButton(s.EditorSettings as Lpp.Dns.DataMart.Model.Settings.FilePickerEditor);
if (btn.Multiselect)
{
btn.FileNames = ((value ?? "").Trim(',')).Split(',');
}
else
{
btn.FileName = value;
}
btn.Anchor = AnchorStyles.Right | AnchorStyles.Left;
editor = btn;
}
else if (s.ValueType == typeof(Lpp.Dns.DataMart.Model.Settings.FolderSelectorEditor))
{
SelectFolderButton btn = new SelectFolderButton(s.EditorSettings as Lpp.Dns.DataMart.Model.Settings.FolderSelectorEditor);
btn.FolderPath = value;
btn.Anchor = AnchorStyles.Right | AnchorStyles.Left;
editor = btn;
}
else
{
TextBox txtbox = new TextBox();
txtbox.Anchor = AnchorStyles.Right | AnchorStyles.Left;
txtbox.Text = value;
editor = txtbox;
}
}
if (editor != null)
{
editor.Tag = s.Key;
_editors.Add(editor);
tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.AutoSize));
if (lbl != null)
{
tableLayoutPanel1.Controls.Add(lbl, 0, rowIndex);
tableLayoutPanel1.Controls.Add(editor, 1, rowIndex++);
}
else
{
tableLayoutPanel1.Controls.Add(editor, 0, rowIndex++);
tableLayoutPanel1.SetColumnSpan(editor, 2);
}
}
});
//add auto expanding row to bottom to fill empty space
Label emptyLabel = new Label();
emptyLabel.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left;
emptyLabel.Text = string.Empty;
tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.Percent, 100f));
tableLayoutPanel1.Controls.Add(emptyLabel, 0, rowIndex);
tableLayoutPanel1.SetColumnSpan(emptyLabel, 2);
}
this.ResumeLayout(true);
}
catch (Exception ex)
{
log.Error(ex);
Close();
}
finally
{
domainManager.Dispose();
this.Cursor = Cursors.Default;
}
}
async Task <HubRequest[]> GetRequestsAsync()
{
if (_networkSetting.NetworkStatus != Util.ConnectionOKStatus)
{
return(new HubRequest[0]);
}
var datamartIDs = _networkSetting.DataMartList
.Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || dm.ProcessQueriesAndNotUpload || dm.ProcessQueriesAndUploadAutomatically))
.Select(dm => dm.DataMartId).ToArray();
if (datamartIDs.Length == 0)
{
return(new HubRequest[0]);
}
var requestFilter = new RequestFilter
{
Statuses = new[] { DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted, DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted },
DataMartIds = datamartIDs
};
var requests = Observable.Create <DTO.DataMartClient.RequestList>(async observer =>
{
int index = 0;
int batchSize = Properties.Settings.Default.AutoProcessingBatchSize;
DTO.DataMartClient.RequestList rl = null;
while (rl == null || (index < rl.TotalCount))
{
rl = await DnsServiceManager.GetRequestList("AutoProcessor", _networkSetting, index, batchSize, requestFilter, DTO.DataMartClient.RequestSortColumn.RequestTime, true);
if (rl == null || rl.TotalCount == 0)
{
break;
}
observer.OnNext(rl);
index += batchSize;
}
observer.OnCompleted();
}).DefaultIfEmpty()
.Aggregate((requestList1, requestList2) =>
{
if (requestList1 == null && requestList2 == null)
{
return(new DTO.DataMartClient.RequestList
{
Segment = Array.Empty <DTO.DataMartClient.RequestListRow>(),
SortedAscending = true,
SortedByColumn = DTO.DataMartClient.RequestSortColumn.RequestTime
});
}
else if (requestList1 != null && requestList2 == null)
{
return(requestList1);
}
else if (requestList1 == null && requestList2 != null)
{
return(requestList2);
}
else
{
return(new DTO.DataMartClient.RequestList
{
Segment = requestList1.Segment.EmptyIfNull().Concat(requestList2.Segment.EmptyIfNull()).ToArray(),
SortedAscending = requestList1.SortedAscending,
SortedByColumn = requestList1.SortedByColumn,
StartIndex = requestList1.StartIndex,
TotalCount = requestList1.TotalCount
});
}
})
.SelectMany(requestList =>
{
if (requestList == null)
{
return(Array.Empty <DTO.DataMartClient.RequestListRow>());
}
return(requestList.Segment.DefaultIfEmpty().Where(s => s.AllowUnattendedProcessing));
})
.SelectMany(rlr => RequestCache.ForNetwork(_networkSetting).LoadRequest(rlr.ID, rlr.DataMartID))
.ToArray();
return(await requests);
}
public void UploadMultipleSmallFiles()
{
//Guid requestID = new Guid("63f074ee-cf4c-4838-ae27-aa4700d7ca87");
//Guid dataMartID = new Guid("84C9440A-0B1F-4EC0-AF75-A85500CC8F16");
//string sourceFolderPath = @"C:\work\DR-UAT\DP2\v_psu_01_346\msoc";
string sourceFolderPath = @"C:\work\DR-UAT\DP2\v_psu_01_353\TEST";
Guid requestID = new Guid("2D35C33F-11D2-47E3-A2D8-AA4700BCCBBA");
Guid dataMartID = new Guid("6D55D9BA-9787-40DC-B9B7-A49F00C83946");
List <DTO.DataMartClient.Criteria.DocumentMetadata> documents = new List <DTO.DataMartClient.Criteria.DocumentMetadata>();
foreach (var file in System.IO.Directory.GetFiles(sourceFolderPath))
{
System.IO.FileInfo fi = new System.IO.FileInfo(file);
Guid documentID = Lpp.Utilities.DatabaseEx.NewGuid();
documents.Add(
new DTO.DataMartClient.Criteria.DocumentMetadata
{
CurrentChunkIndex = 0,
DataMartID = dataMartID,
ID = documentID,
IsViewable = false,
Kind = string.Empty,
MimeType = Lpp.Utilities.FileEx.GetMimeTypeByExtension(fi.Name),
Name = fi.Name,
RequestID = requestID
}
);
logger.Debug($"Adding document: {fi.Name} (ID: {documentID})");
}
//documents = documents.Skip(12).Take(4).ToList();
Parallel.ForEach(documents,
new ParallelOptions {
MaxDegreeOfParallelism = 4
},
(doc) =>
{
string uploadIdentifier = ("[" + Utilities.Crypto.Hash(Guid.NewGuid()) + "]").PadRight(16);
System.IO.Stream stream = null;
try
{
//if (cache.Enabled)
//{
// stream = cache.GetDocumentStream(Guid.Parse(doc.DocumentID));
//}
//else
//{
// request.Processor.ResponseDocument(requestId, doc.DocumentID, out stream, doc.Size);
//}
stream = new System.IO.FileStream(System.IO.Path.Combine(sourceFolderPath, doc.Name), System.IO.FileMode.Open);
var dto = new DTO.DataMartClient.Criteria.DocumentMetadata
{
ID = doc.ID,
DataMartID = dataMartID,
RequestID = requestID,
IsViewable = doc.IsViewable,
Size = doc.Size,
MimeType = doc.MimeType,
Kind = doc.Kind,
Name = doc.Name,
CurrentChunkIndex = 0
};
DnsServiceManager.PostDocumentChunk(uploadIdentifier, dto, stream, _networkSetting);
}
finally
{
stream.Dispose();
}
});
}