public async Task OriginalGetRequestsForAutoProcessor() { //If there are more than 10 pending requests within the last 30 days that are possible for autoprocessing, only the first 10 will be retrieved var datamartIDs = _networkSetting.DataMartList .Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || dm.ProcessQueriesAndNotUpload || dm.ProcessQueriesAndUploadAutomatically)) .Select(dm => dm.DataMartId).ToArray(); if (datamartIDs.Length == 0) { throw new NotSupportedException("No datamarts enabled for autoprocessing."); } var requestFilter = new RequestFilter { Statuses = new[] { DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted, DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted }, DataMartIds = datamartIDs }; int batchSize = 10; var requests = from list in DnsServiceManager.GetRequestList("AutoProcessor", _networkSetting, 0, batchSize, requestFilter, null, null) from rl in list.Segment.EmptyIfNull().ToObservable() where rl.AllowUnattendedProcessing from r in RequestCache.ForNetwork(_networkSetting).LoadRequest(rl.ID, rl.DataMartID) select r; var rq = await requests.ToArray(); foreach (var r in rq) { _logger.Debug($"{ r.DataMartName }: { r.Source.Name} [ { r.Source.Identifier } ]"); } }
/// <summary> /// Execute query only and do not upload results /// </summary> /// <param name="qdm">Query to be processed</param> private void ProcessRequest(HubRequest request) { NetWorkSetting networkSetting = Configuration.Instance.GetNetworkSetting(request.NetworkId); _log.Info(String.Format("BackgroundProcess: Processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID)); try { string requestId = ProcessRequest(request, request.Processor, networkSetting); RequestStatus.StatusCode statusCode = request.Processor.Status(requestId).Code; if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed) { HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId)); hubRequestStatus.Message = request.Processor.Status(requestId).Message; DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, networkSetting); } else { DnsServiceManager.SetRequestStatus(request, DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId)), request.Properties, networkSetting); } _log.Info(String.Format("The following query {3} (ID: {0}) was executed on behalf of {1}:\n\n{2}", request.Source.ID, networkSetting.Profile.Username, "", request.Source.Identifier)); } catch (Exception ex) { string message = string.Format("BackgroundProcess: An error occurred when processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2}", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID); _log.Error(message, ex); DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed, message), request.Properties, networkSetting); } _log.Info(String.Format("BackgroundProcess: Finished processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID)); }
/// <summary> /// Execute query only and do not upload results /// </summary> /// <param name="qdm">Query to be processed</param> private void ProcessRequest(HubRequest request) { Log.Info(string.Format("BackgroundProcess: Processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, request.NetworkId, request.Source.ID)); try { var key = MakeKey(request); if (RequestStatuses.ContainsKey(key)) { RequestStatuses[key] = ProcessingStatus.InProcessing; } string requestId = ProcessRequest(request, request.Processor); RequestStatus.StatusCode statusCode = request.Processor.Status(requestId).Code; if (statusCode == RequestStatus.StatusCode.Error) { HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId)); hubRequestStatus.Message = request.Processor.Status(requestId).Message; DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, _networkSetting); } Log.Info(String.Format("The following query {3} (ID: {0}) was executed on behalf of {1}:\n\n{2}", request.Source.ID, _networkSetting.Profile.Username, "", request.Source.Identifier)); } catch (Exception ex) { string message = string.Format("BackgroundProcess: An error occurred when processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2}", request.Source.Identifier, request.DataMartId, request.NetworkId, request.Source.ID); Log.Error(message, ex); DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed, message), request.Properties, _networkSetting); } Log.Info(string.Format("BackgroundProcess: Finished processing request {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, request.NetworkId, request.Source.ID)); }
IObservable <HubRequest> LoadRequestImpl(Guid id, Guid dataMartId) { var dm = _networkSetting.DataMartList.FirstOrDefault(d => d.DataMartId == dataMartId); return(from r in DnsServiceManager.GetRequests(_networkSetting, new[] { id }, dataMartId).SkipWhile(r => r.ID != id).Take(1) let rt = r.Routings.EmptyIfNull().FirstOrDefault(x => x.DataMartID == dataMartId) where rt != null select new HubRequest { Source = r, DataMartId = dataMartId, NetworkId = _networkSetting.NetworkId, NetworkName = _networkSetting.NetworkName, DataMartName = dm == null ? null : dm.DataMartName, DataMartOrgId = dm == null ? null : dm.OrganizationId, DataMartOrgName = dm == null ? null : dm.OrganizationName, ProjectName = r != null && r.Project != null ? r.Project.Name : null, Properties = rt.Properties.EmptyIfNull().GroupBy(p => p.Name).ToDictionary(pp => pp.Key, pp => pp.First().Value), Rights = r.Routings.Where(rn => rn.DataMartID == dataMartId).Select(rn => (HubRequestRights)rn.Rights).FirstOrDefault(), RoutingStatus = rt.Status, SubmittedDataMarts = string.Join(", ", from rtng in r.Routings join dmt in _networkSetting.DataMartList on rtng.DataMartID equals dmt.DataMartId select dmt.DataMartName), Documents = r.Documents.EmptyIfNull().ToArray() }); }
private void TestConnections() { int numConnectionsOk = DnsServiceManager.TestConnections(Configuration.Instance.NetworkSettingCollection.AsEnumerable()); this.Invoke(new Action(() => { if (numConnectionsOk > 0) { DisplayInformationalStatus(string.Format("Successfully connected to {0} out of {1} Network(s)", numConnectionsOk, Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count), TextColor.Normal, toolStripStatusNetworkConnectivityStatus); DisplayInformationalStatus(string.Format("Connected", numConnectionsOk, Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count), TextColor.Normal, toolStripLoginStatus); DisplayInformationalStatus("Ready", TextColor.Normal, toolStripStatusNetworkConnectivityStatus); } else { int numNetworks = Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count; if (numNetworks > 0) { DisplayInformationalStatus(string.Format("Failed to connect to all {0} Network(s). Check Settings.", Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count), TextColor.Error, toolStripStatusNetworkConnectivityStatus); } else { DisplayInformationalStatus(string.Format("No Network configured. Check Settings.", Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Count), TextColor.Error, toolStripStatusNetworkConnectivityStatus); } DisplayInformationalStatus("Disconnected", TextColor.Alert, toolStripLoginStatus); } })); }
public void LoadModels(NetWorkSetting ns) { //Get the current model information from the API. var models = DnsServiceManager.GetModels(ns); foreach (var d in ns.DataMartList) { HubModel[] m = models[d.DataMartId]; if (d.ModelList != null && !d.ModelList.IsEmpty()) { try { //add missing models var newModels = m.Where(x => !d.ModelList.Any(a => a.ModelId == x.Id)).ToArray(); if (newModels.Length > 0) { d.ModelList.AddRange(newModels.Select(nm => new ModelDescription { ModelId = nm.Id, ModelName = nm.Name, ModelDisplayName = nm.Name, ProcessorId = nm.ModelProcessorId })); } //remove old models d.ModelList.RemoveAll(rm => !m.Any(a => a.Id == rm.ModelId)); } catch (ArgumentNullException ex) { // Ignore if nothing to remove, which will throw an exception. } } } }
public override int Read(byte[] buffer, int offset, int count) { byte[] DocContent = DnsServiceManager.GetDocumentChunk(documentID, documentOffset, count, networkSetting); int size = DocContent.Length; Buffer.BlockCopy(DocContent, 0, buffer, offset, size); documentOffset += size; return(size); }
public void LoadModels(NetWorkSetting ns) { foreach (var d in ns.DataMartList) { d.ModelList = (from m in DnsServiceManager.GetModels(d.DataMartId, ns).EmptyIfNull() join existing in d.ModelList.EmptyIfNull() on m.Id equals existing.ModelId into exts from existing in exts.DefaultIfEmpty() select existing ?? new ModelDescription { ModelId = m.Id, ModelName = m.Name, ProcessorId = m.ModelProcessorId } ).ToList(); } }
private void NetworkList_Load(object sender, EventArgs e) { try { this.Cursor = Cursors.WaitCursor; DnsServiceManager.TestConnections(Configuration.Instance.NetworkSettingCollection.AsEnumerable()); LoadPage(); } finally { this.Cursor = Cursors.Default; } }
private void reconnect_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e) { var progress = new ProgressForm("Connecting to the Network", "Connecting to the Network"); Observable .Start(() => DnsServiceManager.TestConnections(new[] { Network }), Scheduler.Default) .TakeUntil(progress.ShowAndWaitForCancel(this)) .ObserveOn(this) .Finally(progress.Dispose) .Do(_ => Reload()) .LogExceptions(_log.Error) .Catch() .Subscribe(); }
private void btnTestConnection_Click(object sender, EventArgs e) { try { this.Cursor = Cursors.WaitCursor; lblStatusMessage.Text = string.Empty; if (!IsValidNetworkSetting()) { return; } // Test the network configuration on the UI (may or may not be saved). //NetWorkSetting _NetworkSetting = new NetWorkSetting(); if (_NetworkSetting == null) { _NetworkSetting = new NetWorkSetting(); } SaveFields(_NetworkSetting); if (DnsServiceManager.LogIn(_NetworkSetting)) { MessageBox.Show("Connection Successful", Util.ConnectionOKStatus, MessageBoxButtons.OK, MessageBoxIcon.Information); } else { MessageBox.Show(_NetworkSetting.NetworkMessage, Util.ConnectionFailedStatus, MessageBoxButtons.OK, MessageBoxIcon.Information); } PopulateDataMarts(ref _NetworkSetting); EnableDMControls(_NetworkSetting.IsAuthenticated); return; } catch (Exception ex) { log.Error(ex.Message, ex); MessageBox.Show(ex.Message, "Unexpected Error", MessageBoxButtons.OK, MessageBoxIcon.Error); } finally { this.Cursor = Cursors.Default; } }
private void SettingsForm_Load(object sender, EventArgs e) { try { chkAllowUnattendedOperation.Checked = _DataMartDescription.AllowUnattendedOperation; rbModeProcess.Checked = _DataMartDescription.ProcessQueriesAndUploadAutomatically; rbModeNotify.Checked = _DataMartDescription.NotifyOfNewQueries; rbSemiAutomatic.Checked = _DataMartDescription.ProcessQueriesAndNotUpload; update_processing_mode_controls(); Guid dataMartId = DataMartDescription.DataMartId; dgvModels.DataSource = DnsServiceManager.GetModels(dataMartId, NetworkSetting).Where(m => m.ModelProcessorId != Guid.Empty).ToArray(); btnEdit.Enabled = dgvModels.DataSource != null && (dgvModels.DataSource as HubModel[]).Length > 0; } catch (Exception ex) { log.Error(ex.Message, ex); } }
private void SettingsForm_Load(object sender, EventArgs e) { try { chkAllowUnattendedOperation.Checked = _dataMartDescription.AllowUnattendedOperation; rbModeProcess.Checked = _dataMartDescription.ProcessQueriesAndUploadAutomatically; rbModeNotify.Checked = _dataMartDescription.NotifyOfNewQueries; rbSemiAutomatic.Checked = _dataMartDescription.ProcessQueriesAndNotUpload; update_processing_mode_controls(); Guid dataMartId = DataMartDescription.DataMartId; var models = DnsServiceManager.GetModels(NetworkSetting); if (models.ContainsKey(dataMartId)) { dgvModels.DataSource = (models[dataMartId] ?? Array.Empty <HubModel>()).Where(m => m.ModelProcessorId != Guid.Empty).ToArray(); } else { dgvModels.DataSource = Array.Empty <HubModel>(); } btnEdit.Enabled = dgvModels.DataSource != null && (dgvModels.DataSource as HubModel[]).Length > 0; _cache = new Lib.Caching.DataMartCacheManager(NetworkSetting.NetworkId, DataMartDescription.DataMartId); chkEncryptCacheItems.Checked = _dataMartDescription.EncryptCacheItems; chkExplicitCacheRemoval.Checked = _dataMartDescription.EnableExplictCacheRemoval; nmCacheRetentionPeriod.Value = _dataMartDescription.DaysToRetainCacheItems; chkEnableCache.Checked = _dataMartDescription.EnableResponseCaching; chkEnableCache_CheckedChanged(null, EventArgs.Empty); chkEnableCache.CheckedChanged += chkEnableCache_CheckedChanged; } catch (Exception ex) { log.Error(ex.Message, ex); } }
private void NetworkLogin() { try { if (_NetworkSetting != null && _NetworkSetting.IsAuthenticated) { //_NetworkSetting.LogOut(); DnsServiceManager.LogOut(_NetworkSetting); //txtServiceUrl.ReadOnly = txtUsername.ReadOnly = txtPassword.ReadOnly = false; //btnLogOut.Visible = false; //btnTestConnection.Visible = true; btnRefreshDMlist.Enabled = false; //RefreshNetworlListDelegate(); } } catch (Exception ex) { //if (_NetworkSetting != null && !_NetworkSetting.IsAuthenticated) // txtServiceUrl.ReadOnly = txtUsername.ReadOnly = txtPassword.ReadOnly = true; throw ex; } }
private void UploadRequest(HubRequest request, Lib.Caching.DocumentCacheManager cache) { string requestId = request.Source.ID.ToString(); Document[] responseDocuments; if (cache.Enabled) { responseDocuments = cache.GetResponseDocuments().ToArray(); } else { responseDocuments = request.Processor.Response(requestId); } if (responseDocuments != null && responseDocuments.Length > 0) { Log.Info(string.Format("Uploading {4} documents for query: {0} (DataMart:{1}, Network:{2}, RequestID: {3:D})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID, responseDocuments.Length)); //since the ID of the document returned from the API is not known when posting, and the order cannot be assumed in the response //loop over each document and post the metadata and the content as a single unit of work per document. for (int i = 0; i < responseDocuments.Length; i++) { string uploadIdentifier = ("[" + Utilities.Crypto.Hash(Guid.NewGuid()) + "]").PadRight(16); Guid[] metaDataPostResponse = DnsServiceManager.PostResponseDocuments(uploadIdentifier, requestId, request.DataMartId, new[] { responseDocuments[i] }, _networkSetting); System.IO.Stream contentStream = null; try { if (cache.Enabled) { contentStream = cache.GetDocumentStream(Guid.Parse(responseDocuments[i].DocumentID)); } else { request.Processor.ResponseDocument(requestId, responseDocuments[i].DocumentID, out contentStream, 60000); } DnsServiceManager.PostResponseDocumentContent(uploadIdentifier, requestId, request.DataMartId, metaDataPostResponse[0], responseDocuments[i].Filename, contentStream, _networkSetting) .LogExceptions(Log.Error) .Catch() .Wait(); } finally { contentStream.CallDispose(); } } } else { Log.Info(string.Format("No documents to upload for query: {0} (DataMart:{1}, Network:{2}, RequestID: {3:D})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID, responseDocuments.Length)); } if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed) { HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId)); hubRequestStatus.Message = request.Processor.Status(requestId).Message; DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, _networkSetting); Log.Info(string.Format("The following query {0} (ID: {3}, DataMart: {4}, Network: {5}) failed to execute on behalf of {1}:\n\n{2}", request.Source.Identifier, _networkSetting.Profile.Username, hubRequestStatus.Message, request.Source.ID, request.DataMartName, request.NetworkName)); } else { Log.Info(string.Format("The following query {0} (ID: {2}, DataMart: {3}, Network: {4}) was executed and results were uploaded automatically on behalf of {1}", request.Source.Identifier, _networkSetting.Profile.Username, request.Source.ID, request.DataMartName, request.NetworkName)); } }
private void UploadRequest(HubRequest request) { NetWorkSetting networkSetting = Configuration.Instance.GetNetworkSetting(request.NetworkId); _log.Info(String.Format("BackgroundProcess: Processing query {0} (RequestId: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID)); try { string requestId = request.Source.ID.ToString(); Lpp.Dns.DataMart.Model.Document[] responseDocuments = request.Processor.Response(requestId); Guid[] documentIds = DnsServiceManager.PostResponseDocuments(requestId, request.DataMartId, responseDocuments, networkSetting); _log.Info("Number of portal document ids returned by PostResponseDocuments: " + documentIds.Length); for (int i = 0; i < documentIds.Length; i++) { _log.Info("About to post content for portal document id: " + documentIds[i] + " corresponding to response document id: " + responseDocuments[i].DocumentID); Stream contentStream = null; try { request.Processor.ResponseDocument(requestId, responseDocuments[i].DocumentID, out contentStream, 60000); DnsServiceManager.PostResponseDocumentContent(documentIds[i], contentStream, networkSetting) .LogExceptions(_log.Error) .Catch() .SubscribeOn(Scheduler.Default) .LastOrDefault(); } finally { contentStream.CallDispose(); } } if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed) { HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId)); hubRequestStatus.Message = request.Processor.Status(requestId).Message; DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, networkSetting); _log.Info(String.Format("The following query {0} (ID: {3}) failed to execute on behalf of {1}:\n\n{2}", request.Source.Identifier, networkSetting.Profile.Username, hubRequestStatus.Message, request.Source.ID)); } else { //MantisID:6331:Zero out low-cell counts if they exist. //string strReason = request.Processor.Status(requestId).Code == RequestStatus.StatusCode.CompleteWithMessage ? request.Processor.Status(requestId).Message : string.Empty; //DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Completed), request.Properties, networkSetting); //if (string.IsNullOrEmpty(strReason)) //{ //DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(HubRequestStatus.StatusCode.Completed), request.Properties, networkSetting); _log.Info(String.Format("The following query {0} (ID: {2}) was executed and results were uploaded automatically on behalf of {1}", request.Source.Identifier, networkSetting.Profile.Username, request.Source.ID)); //} //else //{ // //DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(HubRequestStatus.StatusCode.Completed, strReason), request.Properties, networkSetting); // _log.Info(String.Format("The following query {0} (ID: {3}) was executed and results were uploaded automatically on behalf of {1}:\n\n{2}", // request.Source.Identifier, networkSetting.Profile.Username, strReason, request.Source.ID)); //} } } catch (Exception ex) { string message = string.Format("An error occurred while attempting unattended processing of the following query {0} (ID: {1})", request.Source.Identifier, request.Source.ID); _log.Error(message, ex); DnsServiceManager.SetRequestStatus(request, new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed, message), request.Properties, networkSetting); } _log.Info(String.Format("BackgroundProcess: Finished Processing / Uploading results for query {0} (RequestID: {3}, DataMartId: {1}, NetworkId: {2})", request.Source.Identifier, request.DataMartId, networkSetting.NetworkId, request.Source.ID)); }
private void UploadRequest(HubRequest request, Lib.Caching.DocumentCacheManager cache) { string requestId = request.Source.ID.ToString(); Document[] responseDocuments; if (cache.Enabled) { responseDocuments = cache.GetResponseDocuments().ToArray(); } else { responseDocuments = request.Processor.Response(requestId); } if (responseDocuments != null && responseDocuments.Length > 0) { Log.Info(string.Format("Uploading {4} documents for query: {0} (DataMart:{1}, Network:{2}, RequestID: {3:D})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID, responseDocuments.Length)); if (_networkSetting.SupportsUploadV2) { Log.Debug($"Network settings indicate document upload v2 supported for query: {request.Source.Identifier}, (DataMart: {request.DataMartName}, Network: { request.NetworkName }, RequestID: { request.Source.ID }"); try { Parallel.ForEach(responseDocuments, new ParallelOptions { MaxDegreeOfParallelism = 4 }, (doc) => { string uploadIdentifier = ("[" + Utilities.Crypto.Hash(Guid.NewGuid()) + "]").PadRight(16); System.IO.Stream stream = null; try { if (cache.Enabled) { stream = cache.GetDocumentStream(Guid.Parse(doc.DocumentID)); } else { request.Processor.ResponseDocument(requestId, doc.DocumentID, out stream, doc.Size); } var dto = new DTO.DataMartClient.Criteria.DocumentMetadata { ID = Guid.Parse(doc.DocumentID), DataMartID = request.DataMartId, RequestID = Guid.Parse(requestId), IsViewable = doc.IsViewable, Size = doc.Size, MimeType = doc.MimeType, Kind = doc.Kind, Name = doc.Filename, CurrentChunkIndex = 0 }; DnsServiceManager.PostDocumentChunk(uploadIdentifier, dto, stream, _networkSetting); } finally { stream.Dispose(); } }); } catch (Exception ex) { Log.Error($"An error occurred during concurrent uploading of response documents for query: {request.Source.Identifier}, (DataMart: {request.DataMartName}, Network: { request.NetworkName }, RequestID: { request.Source.ID }", ex); throw; } } else { //since the ID of the document returned from the API is not known when posting, and the order cannot be assumed in the response //loop over each document and post the metadata and the content as a single unit of work per document. Log.Debug($"Network settings indicates document upload v2 IS NOT supported for query: {request.Source.Identifier}, (DataMart: {request.DataMartName}, Network: { request.NetworkName }, RequestID: { request.Source.ID }"); for (int i = 0; i < responseDocuments.Length; i++) { string uploadIdentifier = ("[" + Utilities.Crypto.Hash(Guid.NewGuid()) + "]").PadRight(16); Guid[] metaDataPostResponse = DnsServiceManager.PostResponseDocuments(uploadIdentifier, requestId, request.DataMartId, new[] { responseDocuments[i] }, _networkSetting); System.IO.Stream contentStream = null; try { if (cache.Enabled) { contentStream = cache.GetDocumentStream(Guid.Parse(responseDocuments[i].DocumentID)); } else { request.Processor.ResponseDocument(requestId, responseDocuments[i].DocumentID, out contentStream, 60000); } DnsServiceManager.PostResponseDocumentContent(uploadIdentifier, requestId, request.DataMartId, metaDataPostResponse[0], responseDocuments[i].Filename, contentStream, _networkSetting) .LogExceptions(Log.Error) .Catch() .Wait(); } finally { contentStream.CallDispose(); } } } } else { Log.Info(string.Format("No documents to upload for query: {0} (DataMart:{1}, Network:{2}, RequestID: {3:D})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID, responseDocuments.Length)); } if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed) { HubRequestStatus hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(requestId)); hubRequestStatus.Message = request.Processor.Status(requestId).Message; DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, _networkSetting); Log.Info(string.Format("The following query {0} (ID: {3}, DataMart: {4}, Network: {5}) failed to execute on behalf of {1}:\n\n{2}", request.Source.Identifier, _networkSetting.Profile.Username, hubRequestStatus.Message, request.Source.ID, request.DataMartName, request.NetworkName)); } else { Log.Info(string.Format("The following query {0} (ID: {2}, DataMart: {3}, Network: {4}) was executed and results were uploaded automatically on behalf of {1}", request.Source.Identifier, _networkSetting.Profile.Username, request.Source.ID, request.DataMartName, request.NetworkName)); } }
private void PopulateDataMarts(ref NetWorkSetting ns) { try { if (!DnsServiceManager.LogIn(ns)) { return; } // Find matching existing datamarts and update their info. Append new datamarts and remove deleted ones. var newDMs = ns.GetDataMartsByUser().EmptyIfNull(); if ((dgvDataMarts.DataSource as IEnumerable <DataMartDescription>).NullOrEmpty()) { dgvDataMarts.DataSource = newDMs; } else { (from oldDataMarts in dgvDataMarts.DataSource as IEnumerable <DataMartDescription> join newDataMarts in newDMs on oldDataMarts.DataMartId equals newDataMarts.DataMartId select new { oldDM = oldDataMarts, newDM = newDataMarts }).ForEach(d => { // Append new datamarts. // For each existing datamart, find matching existing model and update their info. d.oldDM.DataMartName = d.newDM.DataMartName; d.oldDM.OrganizationId = d.newDM.OrganizationId; d.oldDM.OrganizationName = d.newDM.OrganizationName; if (d.oldDM.ModelList != null && d.newDM.ModelList != null) { (from oldModel in d.oldDM.ModelList.Cast <ModelDescription>() join newModel in d.newDM.ModelList.Cast <ModelDescription>() on oldModel.ModelId equals newModel.ModelId select new { oldModel = oldModel, newModel = newModel }).ForEach(m => { m.oldModel.ModelName = m.newModel.ModelName; m.oldModel.ModelDisplayName = m.newModel.ModelDisplayName; }); } }); var ds = new List <DataMartDescription>(); ds.AddRange(dgvDataMarts.DataSource as IEnumerable <DataMartDescription>); ds.AddRange(from newDataMart in newDMs where !ds.Any(es => (es.DataMartId == newDataMart.DataMartId)) select newDataMart); ds.RemoveAll(r => !newDMs.Any(es => es.DataMartId == r.DataMartId)); dgvDataMarts.DataSource = ds; } dgvDataMarts.Refresh(); if (ns.DataMartList != null && ns.DataMartList.Count > 0) { Configuration.Instance.LoadModels(ns); btnEdit.Enabled = btnRefreshDMlist.Enabled = true; btnApply.Enabled = btnOk.Enabled = true; dgvDataMarts.Enabled = true; } else { dgvDataMarts.Enabled = false; btnEdit.Enabled = false; MessageBox.Show("No network DataMart(s) found", Application.ProductName); } } catch (Exception ex) { throw ex; } }
private void btnOk_Click(object sender, EventArgs e) { try { this.Cursor = Cursors.WaitCursor; // Get the network setting for this service url if it exists already. NetWorkSetting ns = null; try { ns = (from n in Configuration.Instance.NetworkSettingCollection.NetWorkSettings.ToArray(typeof(NetWorkSetting)) as NetWorkSetting[] where n.HubWebServiceUrl == txtServiceUrl.Text select n).FirstOrDefault(); } catch { Configuration.CreateNewNetworkSettingsFile(); } // If not, create one. if (ns == null) { ns = new NetWorkSetting(); ns.HubWebServiceUrl = txtServiceUrl.Text; Configuration.Instance.AddNetworkSetting(ns); } else { if (MessageBox.Show("This will override your existing configuration for this network.\nAre you sure you want to do that?", "Override Network Configuration?", MessageBoxButtons.OKCancel, MessageBoxIcon.Question) == DialogResult.Cancel) { this.Close(); return; } } ns.Username = txtUsername.Text; ns.EncryptedPassword = txtPassword.Text; ns.NetworkName = txtNetworkname.Text; var selectedCert = cbCertificates.SelectedItem as Cert; ns.X509CertThumbprint = selectedCert == null ? null : selectedCert.Thumbprint; ns.GetDataMartsByUser(); Configuration.Instance.LoadModels(ns); CredentialManager.SaveCredential(ns.CredentialKey, ns.Username, ns.Password); // Get the configuration for all models for all datamarts at this service url. var dmc = DnsServiceManager.GetDataMartModelConfigurations(ns); // Find the model processor in the existing network if exists. If not, create it. XmlSerializer serializer = new XmlSerializer(typeof(ModelProperties)); foreach (var c in dmc) { var d = ns.DataMartList.FirstOrDefault(dm => dm.DataMartId == c.DataMartId); var mp = d.ModelList.FirstOrDefault(m => m.ModelId == c.ModelId && m.ProcessorId == c.ModelProcessorId); if (d != null && mp != null && !string.IsNullOrEmpty(c.Properties)) { using (XmlTextReader reader = new XmlTextReader(new MemoryStream(new System.Text.UTF8Encoding().GetBytes(c.Properties)))) { d.AllowUnattendedOperation = c.UnattendedMode != Lpp.Dns.DataMart.Lib.Classes.UnattendedMode.NoUnattendedOperation; d.NotifyOfNewQueries = c.UnattendedMode == Lpp.Dns.DataMart.Lib.Classes.UnattendedMode.NotifyOnly; d.ProcessQueriesAndNotUpload = c.UnattendedMode == Lpp.Dns.DataMart.Lib.Classes.UnattendedMode.ProcessNoUpload; d.ProcessQueriesAndUploadAutomatically = c.UnattendedMode == Lpp.Dns.DataMart.Lib.Classes.UnattendedMode.ProcessAndUpload; var p = new List <PropertyData>(); p.AddRange(((ModelProperties)serializer.Deserialize(reader)).Properties.Where(prop => prop.Name != "ConfirmPassword")); mp.Properties = p; if (mp.ProcessorId != Guid.Empty) { try { //TODO: confirm the latest package exists for the model/processor combination var packageIdentifier = DnsServiceManager.GetRequestTypeIdentifier(ns, mp.ModelId, mp.ProcessorId); if (packageIdentifier != null && !File.Exists(Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.Identifier + "." + packageIdentifier.Version + ".zip"))) { DnsServiceManager.DownloadPackage(ns, packageIdentifier); } //string processorPath, className; //ProcessorManager.FindProcessor(mp.ModelId, mp.ProcessorId, out processorPath, out className); //mp.ProcessorPath = processorPath; //mp.ClassName = className; } catch (Exception ex) { log.Error("Unable to load processor.", ex); } } } } } Configuration.SaveNetworkSettings(); if (NetworkSettingChanged != null) { NetworkSettingChanged(this, ns); } } catch (Exception ex) { MessageBox.Show(ex.Message, "Update Failed", MessageBoxButtons.OK, MessageBoxIcon.Error); } finally { this.Cursor = Cursors.Default; this.Close(); } }
public void Reload() { this.connectionError.Visible = Network.NetworkStatus != Util.ConnectionOKStatus; this.errorMessage.Text = Network.NetworkMessage; if (Network.NetworkStatus != Util.ConnectionOKStatus) { return; } var reload = DnsServiceManager.GetRequestList("filtered requests lookup", Network, Page * _pageSize, _pageSize, Network.Filter, _sortColumn, _isSortAscending) .TakeLast(1) .ObserveOn(this) .Select(list => { PageCount = list.TotalCount / _pageSize + (list.TotalCount == 0 || list.TotalCount % _pageSize > 0 ? 1 : 0); lblTotalPages.Text = PageCount.ToString(); if (Page > PageCount) { Page = PageCount; txtPageNumber.Text = (Page + 1).ToString(); } var selected = SelectedRequest; //update request statuses that have cached results that are awaiting upload - Pending Upload is a local status and is not known by the API foreach (var seg in list.Segment.Where(s => s.Status == DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted || s.Status == DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted)) { var cache = new Lib.Caching.DocumentCacheManager(Network.NetworkId, seg.DataMartID, seg.ID, seg.ResponseID); if (cache.HasResponseDocuments) { seg.Status = DTO.DataMartClient.Enums.DMCRoutingStatus.PendingUpload; } } dgvRequestList.DataSource = list.Segment; SelectedRequest = selected == null ? null : list.Segment.FirstOrDefault(rq => rq.ID == selected.ID && rq.DataMartID == selected.DataMartID); DisplaySortGlyph(); EnableDisablePageButtons(); return(Unit.Default); }); var now = DateTime.Now; var newReqsLookup = DnsServiceManager.GetRequestList("new requests lookup", Network, 0, 1, new RequestFilter { DateRange = DateRangeKind.Exact, FromDate = _lastNewRequestsLookup }, null, null) .Take(1) .ObserveOn(this) .Select(n => { if (n.TotalCount > 0) { NewRequestsAvailable = true; } _lastNewRequestsLookup = now.AddMilliseconds(1); return(Unit.Default); }); var invFilter = InvertFilter(_lastInvisibleRequestsLookup); var invisibleReqsLookup = invFilter == null?Observable.Return(Unit.Default) : DnsServiceManager.GetRequestList("hidden requests lookup", Network, 0, 1, invFilter.Value, null, null) .Take(1) .ObserveOn(this) .Select(n => { if (n.TotalCount > 0) { InvisibleRequestsAvailable = true; } _lastInvisibleRequestsLookup = now.AddMilliseconds(1); return(Unit.Default); }); _currentReload.CallDispose(); _currentReload = Observable .Merge(reload, newReqsLookup, invisibleReqsLookup) .TakeLast(1) .Do(_ => { _currentReload = null; Reloaded.Raise(this); }) .Finally(() => _currentReload = null) .Catch((Exception ex) => { _log.Error(ex); Network.NetworkStatus = Util.ConnectionFailedStatus; errorMessage.Text = Network.NetworkMessage = ex.Message; connectionError.Show(); _currentReload = null; ReloadFailed.Raise(this); return(Observable.Empty <Unit>()); }) .Subscribe(); ReloadStarted.Raise(this); }
public void Reload() { this.connectionError.Visible = Network.NetworkStatus != Util.ConnectionOKStatus; this.errorMessage.Text = Network.NetworkMessage; if (Network.NetworkStatus != Util.ConnectionOKStatus) { return; } var reload = DnsServiceManager.GetRequestList(Network, Page * _pageSize, _pageSize, Network.Filter, _sortColumn, _isSortAscending) .TakeLast(1) .ObserveOn(this) .Select(list => { PageCount = list.TotalCount / _pageSize + (list.TotalCount == 0 || list.TotalCount % _pageSize > 0 ? 1 : 0); lblTotalPages.Text = PageCount.ToString(); if (Page > PageCount) { Page = PageCount; txtPageNumber.Text = (Page + 1).ToString(); } var selected = SelectedRequest; dgvRequestList.DataSource = list.Segment; SelectedRequest = selected == null ? null : list.Segment.FirstOrDefault(rq => rq.ID == selected.ID && rq.DataMartID == selected.DataMartID); DisplaySortGlyph(); EnableDisablePageButtons(); return(Unit.Default); }); var now = DateTime.Now; var newReqsLookup = DnsServiceManager.GetRequestList(Network, 0, 1, new RequestFilter { DateRange = DateRangeKind.Exact, FromDate = _lastNewRequestsLookup }, null, null) .Take(1) .ObserveOn(this) .Select(n => { if (n.TotalCount > 0) { NewRequestsAvailable = true; } _lastNewRequestsLookup = now.AddMilliseconds(1); return(Unit.Default); }); var invFilter = InvertFilter(_lastInvisibleRequestsLookup); var invisibleReqsLookup = invFilter == null?Observable.Return(Unit.Default) : DnsServiceManager.GetRequestList(Network, 0, 1, invFilter.Value, null, null) .Take(1) .ObserveOn(this) .Select(n => { if (n.TotalCount > 0) { InvisibleRequestsAvailable = true; } _lastInvisibleRequestsLookup = now.AddMilliseconds(1); return(Unit.Default); }); _currentReload.CallDispose(); _currentReload = Observable .Merge(reload, newReqsLookup, invisibleReqsLookup) .TakeLast(1) .Do(_ => { _currentReload = null; Reloaded.Raise(this); }) .Finally(() => _currentReload = null) .Catch((Exception ex) => { _log.Error(ex); Network.NetworkStatus = Util.ConnectionFailedStatus; errorMessage.Text = Network.NetworkMessage = ex.Message; connectionError.Show(); _currentReload = null; ReloadFailed.Raise(this); return(Observable.Empty <Unit>()); }) .Subscribe(); ReloadStarted.Raise(this); }
protected override void OnDoWork(DoWorkEventArgs e) { _log.Info("Automated processing worker started."); while (true) { _log.Info("Checking Master Request Queue for requests requiring automated processing."); SystemTray.UpdateNotificationIcon(IconType.IconBusy, "Processing Requests"); var reqs = from ns in Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Cast <NetWorkSetting>().ToObservable() where ns.NetworkStatus == Util.ConnectionOKStatus let dmIds = ns.DataMartList // BMS: Note the ProcessAndNotUpload feature has been temporarilty disabled until we fix the processing around this feature .Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || /* dm.ProcessQueriesAndNotUpload || */ dm.ProcessQueriesAndUploadAutomatically)) .Select(dm => dm.DataMartId) .ToArray() where dmIds.Any() from list in DnsServiceManager.GetRequestList(ns, 0, Properties.Settings.Default.AutoProcessingBatchSize, new RequestFilter { Statuses = new [] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted }, DataMartIds = dmIds }, null, null) from rl in list.Segment.EmptyIfNull().ToObservable() where rl.AllowUnattendedProcessing from r in RequestCache.ForNetwork(ns).LoadRequest(rl.ID, rl.DataMartID) select new { Request = r, NetworkSetting = ns }; reqs .Do(r => { var request = r.Request; var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId); var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID); var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier { Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion }; if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName()))) { DnsServiceManager.DownloadPackage(r.NetworkSetting, packageIdentifier); } using (var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath)) { domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion); IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId); ProcessorManager.UpdateProcessorSettings(modelDescription, processor); if (processor is IEarlyInitializeModelProcessor) { ((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, r.NetworkSetting))).ToArray()); } if (processor != null && processor.ModelMetadata != null && processor.ModelMetadata.Capabilities != null && processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") && !(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"]) { //can't be run, don't attempt autoprocessing return; } request.Processor = processor; if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted) { if (processor != null) { SystemTray.generate_notification(request, request.NetworkId); if (datamartDescription.NotifyOfNewQueries) { SystemTray.UpdateNotificationIcon(IconType.IconBusy, string.Format("Query Submitted by {0}", request.Source.Author.Username)); } else { processor.SetRequestProperties(request.Source.ID.ToString(), request.Properties); ProcessRequest(request); var statusCode = request.Processor.Status(request.Source.ID.ToString()).Code; if (datamartDescription.ProcessQueriesAndUploadAutomatically && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage)) { // Post process requests that are automatically uploaded processor.PostProcess(request.Source.ID.ToString()); // Increment counter _queriesProcessedCount++; SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId); UploadRequest(request); } } } } else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval) { if (datamartDescription.ProcessQueriesAndUploadAutomatically) { // Increment counter _queriesProcessedCount++; SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId); UploadRequest(request); } } } }) .LogExceptions(_log.Error) .Catch() .LastOrDefault(); SystemTray.UpdateNotificationIcon(IconType.IconDefault, null); Thread.Sleep(DMClient.Properties.Settings.Default.RefreshRate); } }
private void AutoProcess(KeyValuePair <string, HubRequest> input) { var request = input.Value; var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId); if (datamartDescription.ProcessQueriesAndUploadAutomatically == false && datamartDescription.ProcessQueriesAndNotUpload == false) { //just notify, do not process string message = $"New query submitted and awaiting processing in { request.ProjectName } Project: { request.Source.Name } ({request.Source.Identifier})"; SystemTray.DisplayNewQueryNotificationToolTip(message); RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload); return; } var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID); var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier { Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion }; if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName()))) { DnsServiceManager.DownloadPackage(_networkSetting, packageIdentifier); } var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath); try { domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion); IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId); ProcessorManager.UpdateProcessorSettings(modelDescription, processor); processor.Settings.Add("NetworkId", request.NetworkId); Lib.Caching.DocumentCacheManager cache = new Lib.Caching.DocumentCacheManager(request.NetworkId, request.DataMartId, request.Source.ID, request.Source.Responses.Where(x => x.DataMartID == request.DataMartId).FirstOrDefault().ResponseID); //need to initialize before checking the capabilities and settings of the processor since they may change based on the type of request being sent. if (processor is IEarlyInitializeModelProcessor) { ((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, _networkSetting))).ToArray()); } if (processor != null && processor.ModelMetadata != null && processor.ModelMetadata.Capabilities != null && processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") && !(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"]) { //can't be run, don't attempt autoprocessing RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload); domainManager.Dispose(); return; } request.Processor = processor; if (cache.HasResponseDocuments == false && (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted || request.RoutingStatus == DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted)) { if (processor != null) { SystemTray.GenerateNotification(request, request.NetworkId); StartProcessingRequest(request, processor, datamartDescription, domainManager, cache); return; } } else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval) { if (datamartDescription.ProcessQueriesAndUploadAutomatically) { // Increment counter System.Threading.Interlocked.Increment(ref _queriesProcessedCount); SystemTray.UpdateNotifyText(_queriesProcessedCount, request.DataMartName, request.NetworkId); StartUploadingRequest(request, domainManager, cache); return; } } else if (cache.HasResponseDocuments) { RequestStatuses.TryAdd(input.Key, ProcessingStatus.PendingUpload); } domainManager.Dispose(); } catch (Exception ex) { Log.Error($"Error autoprocessing Request: { request.Source.Identifier }, DataMartId: { request.DataMartId }, NetworkId: {request.NetworkId}", ex); domainManager.Dispose(); throw; } }
async Task <HubRequest[]> GetRequestsAsync() { var datamartIDs = _networkSetting.DataMartList .Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || dm.ProcessQueriesAndNotUpload || dm.ProcessQueriesAndUploadAutomatically)) .Select(dm => dm.DataMartId).ToArray(); if (datamartIDs.Length == 0) { return(Array.Empty <HubRequest>()); } var requestFilter = new RequestFilter { Statuses = new[] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted, Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted }, DataMartIds = datamartIDs, FromDate = DateTime.UtcNow.AddYears(-1), DateRange = DateRangeKind.Exact }; var ob = Observable.Create <DTO.DataMartClient.RequestList>(async observer => { int index = 0; int batchSize = 2; DTO.DataMartClient.RequestList rl = null; while (rl == null || (index < rl.TotalCount)) { _logger.Debug($"Observer loop: pageIndex={ index }."); rl = await DnsServiceManager.GetRequestList("RxQueryTests-DefaultDSM", _networkSetting, index, batchSize, requestFilter, null, null); _logger.Debug($"Observer loop: pageIndex={ index }, results returned={ rl.Segment.DefaultIfEmpty().Count() }, total results={ rl.TotalCount }"); if (rl.TotalCount == 0) { break; } observer.OnNext(rl); index += batchSize; } _logger.Debug("Observer loop firing OnComplete"); observer.OnCompleted(); }).DefaultIfEmpty().Aggregate((requestList1, requestList2) => { if (requestList1 == null && requestList2 == null) { return(new DTO.DataMartClient.RequestList { Segment = Array.Empty <DTO.DataMartClient.RequestListRow>(), SortedAscending = false, SortedByColumn = DTO.DataMartClient.RequestSortColumn.RequestTime }); } else if (requestList1 != null && requestList2 == null) { return(requestList1); } else if (requestList1 == null && requestList2 != null) { return(requestList2); } else { return(new DTO.DataMartClient.RequestList { Segment = requestList1.Segment.EmptyIfNull().Concat(requestList2.Segment.EmptyIfNull()).ToArray(), SortedAscending = requestList1.SortedAscending, SortedByColumn = requestList1.SortedByColumn, StartIndex = requestList1.StartIndex, TotalCount = requestList1.TotalCount }); } }) .SelectMany(requestList => { if (requestList == null) { return(Array.Empty <DTO.DataMartClient.RequestListRow>()); } return(requestList.Segment.DefaultIfEmpty().Where(s => s.AllowUnattendedProcessing)); }) .SelectMany(rlr => RequestCache.ForNetwork(_networkSetting).LoadRequest(rlr.ID, rlr.DataMartID)) .ToArray(); return(await ob); }
private void StartProcessingRequest(HubRequest request, IModelProcessor processor, DataMartDescription datamartDescription, DomainManger.DomainManager domainManager, Lib.Caching.DocumentCacheManager cache) { Action process = () => { processor.SetRequestProperties(request.Source.ID.ToString(), request.Properties); ProcessRequest(request); }; Action <Task> continuation = (completed) => { HubRequestStatus hubRequestStatus = null; var statusCode = request.Processor.Status(request.Source.ID.ToString()).Code; if (cache.Enabled) { Document[] responseDocuments = processor.Response(request.Source.ID.ToString()); cache.Add(responseDocuments.Select(doc => { System.IO.Stream data; processor.ResponseDocument(request.Source.ID.ToString(), doc.DocumentID, out data, doc.Size); Guid documentID; if (!Guid.TryParse(doc.DocumentID, out documentID)) { documentID = Utilities.DatabaseEx.NewGuid(); doc.DocumentID = documentID.ToString(); } return(new DocumentWithStream(documentID, doc, data)); })); } if (datamartDescription.ProcessQueriesAndNotUpload && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage)) { RequestStatuses.TryAdd(MakeKey(request), ProcessingStatus.PendingUpload); } else if (datamartDescription.ProcessQueriesAndUploadAutomatically && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage)) { // Post process requests that are automatically uploaded processor.PostProcess(request.Source.ID.ToString()); if (cache.Enabled) { cache.ClearCache(); Document[] responseDocuments = processor.Response(request.Source.ID.ToString()); cache.Add(responseDocuments.Select(doc => { System.IO.Stream data; processor.ResponseDocument(request.Source.ID.ToString(), doc.DocumentID, out data, doc.Size); Guid documentID; if (!Guid.TryParse(doc.DocumentID, out documentID)) { documentID = Utilities.DatabaseEx.NewGuid(); doc.DocumentID = documentID.ToString(); } return(new DocumentWithStream(documentID, doc, data)); })); } // Increment counter System.Threading.Interlocked.Increment(ref _queriesProcessedCount); statusCode = request.Processor.Status(request.Source.ID.ToString()).Code; if (statusCode == RequestStatus.StatusCode.Error) { hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(request.Source.ID.ToString())); hubRequestStatus.Message = request.Processor.Status(request.Source.ID.ToString()).Message; } else if (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage) { SystemTray.UpdateNotifyText(_queriesProcessedCount, request.DataMartName, request.NetworkId); try { UploadRequest(request, cache); statusCode = request.Processor.Status(request.Source.ID.ToString()).Code; hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(request.Source.ID.ToString())); hubRequestStatus.Message = request.Processor.Status(request.Source.ID.ToString()).Message; } catch (Exception ex) { string message = string.Format("An error occurred while attempting unattended processing of the following query {0} (ID: {1}, DataMart: {2}, Network: {3})", request.Source.Identifier, request.Source.ID, request.DataMartName, request.NetworkName); Log.Error(message, ex); hubRequestStatus = new HubRequestStatus(Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Failed, message); } } else { statusCode = request.Processor.Status(request.Source.ID.ToString()).Code; hubRequestStatus = DnsServiceManager.ConvertModelRequestStatus(request.Processor.Status(request.Source.ID.ToString())); hubRequestStatus.Message = request.Processor.Status(request.Source.ID.ToString()).Message; } DnsServiceManager.SetRequestStatus(request, hubRequestStatus, request.Properties, _networkSetting); Log.Info(string.Format("BackgroundProcess: Finished Processing / Uploading results for query {0} (RequestID: {3}, DataMart: {1}, Network: {2})", request.Source.Identifier, request.DataMartName, request.NetworkName, request.Source.ID)); if (statusCode == RequestStatus.StatusCode.Error || statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage || statusCode == RequestStatus.StatusCode.AwaitingResponseApproval) { //mark auto-processing complete in the status cache if the processing resulted in either an error or a completed status var key = MakeKey(request); if (RequestStatuses.ContainsKey(key)) { RequestStatuses[key] = ProcessingStatus.Complete; } } } }; RunTask(MakeKey(request), domainManager, process, continuation); }
private void SettingsForm_Load(object sender, EventArgs e) { Lpp.Dns.DataMart.Client.DomainManger.DomainManager domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath); try { if (ModelDescription == null) { return; } if (ModelDescription.ProcessorId == Guid.Empty) { ModelDescription.ProcessorId = HubModel.ModelProcessorId; } //get the package identifier and version var packageIdentifier = DnsServiceManager.GetRequestTypeIdentifier(NetworkSetting, ModelDescription.ModelId, ModelDescription.ProcessorId); if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName()))) { DnsServiceManager.DownloadPackage(NetworkSetting, packageIdentifier); } domainManager.Load(packageIdentifier.Identifier, packageIdentifier.Version); modelProcessor = domainManager.GetProcessor(ModelDescription.ProcessorId); ProcessorManager.UpdateProcessorSettings(ModelDescription, modelProcessor); if (modelProcessor is IEarlyInitializeModelProcessor) { ((IEarlyInitializeModelProcessor)modelProcessor).Initialize(ModelDescription.ModelId, Array.Empty <DocumentWithStream>()); } this.SuspendLayout(); if (modelProcessor.ModelMetadata.Settings == null || !modelProcessor.ModelMetadata.Settings.Any()) { var noSettingsLabel = new Label(); noSettingsLabel.Text = "This model processor does not have any settings."; noSettingsLabel.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left; noSettingsLabel.TextAlign = ContentAlignment.MiddleCenter; tableLayoutPanel1.Controls.Add(noSettingsLabel, 0, 0); tableLayoutPanel1.SetColumnSpan(noSettingsLabel, 2); } else { _editors = new HashSet <Control>(); tableLayoutPanel1.RowStyles.Clear(); int rowIndex = 0; IEnumerable <Lpp.Dns.DataMart.Model.Settings.ProcessorSetting> settings = modelProcessor.ModelMetadata.Settings; if (modelProcessor.ModelMetadata.SQlProviders != null && modelProcessor.ModelMetadata.SQlProviders.Any()) { DataMart.Client.Controls.DataSourceEditor dsEditor = new Controls.DataSourceEditor(modelProcessor.ModelMetadata, ModelDescription.Properties); dsEditor.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left; tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.AutoSize)); tableLayoutPanel1.Controls.Add(dsEditor, 0, rowIndex++); tableLayoutPanel1.SetColumnSpan(dsEditor, 2); settings = settings.Where(s => !Lpp.Dns.DataMart.Model.Settings.ProcessorSettings.IsDbSetting(s.Key)).ToArray(); _editors.Add(dsEditor); } settings.ToList().ForEach(s => { string value = ModelDescription.Properties.Where(p => string.Equals(p.Name, s.Key, StringComparison.OrdinalIgnoreCase)).Select(p => p.Value).FirstOrDefault(); if (string.IsNullOrEmpty(value) && s.Required && !string.IsNullOrEmpty(s.DefaultValue)) { value = s.DefaultValue; } Label lbl = new Label(); lbl.AutoSize = true; lbl.Anchor = AnchorStyles.Right; lbl.TextAlign = ContentAlignment.MiddleRight; lbl.Text = s.Title.EndsWith(":") ? s.Title : s.Title + ":"; Control editor = null; if (s.ValidValues != null) { ComboBox combo = new ComboBox(); combo.DropDownStyle = ComboBoxStyle.DropDownList; combo.Anchor = AnchorStyles.Right | AnchorStyles.Left; combo.Name = s.Key; combo.Items.AddRange(s.ValidValues.Select(v => new PropertyData(v.Key, v.Value.ToString())).ToArray()); if (!string.IsNullOrEmpty(value)) { foreach (PropertyData p in combo.Items) { if (string.Equals(p.Value, value, StringComparison.OrdinalIgnoreCase)) { combo.SelectedItem = p; break; } } } if (combo.SelectedIndex < 0) { combo.SelectedIndex = 0; } editor = combo; } else { if (s.ValueType == typeof(bool) || s.ValueType == typeof(Nullable <bool>)) { CheckBox chkbox = new CheckBox(); chkbox.Anchor = AnchorStyles.Left; chkbox.Text = s.Title; chkbox.TextAlign = ContentAlignment.MiddleLeft; chkbox.AutoSize = true; if (!string.IsNullOrEmpty(value)) { bool isChecked = false; bool.TryParse(value, out isChecked); chkbox.Checked = isChecked; } editor = chkbox; lbl = null; } else if (s.ValueType == typeof(Lpp.Dns.DataMart.Model.Settings.FilePickerEditor)) { SelectFileButton btn = new SelectFileButton(s.EditorSettings as Lpp.Dns.DataMart.Model.Settings.FilePickerEditor); if (btn.Multiselect) { btn.FileNames = ((value ?? "").Trim(',')).Split(','); } else { btn.FileName = value; } btn.Anchor = AnchorStyles.Right | AnchorStyles.Left; editor = btn; } else if (s.ValueType == typeof(Lpp.Dns.DataMart.Model.Settings.FolderSelectorEditor)) { SelectFolderButton btn = new SelectFolderButton(s.EditorSettings as Lpp.Dns.DataMart.Model.Settings.FolderSelectorEditor); btn.FolderPath = value; btn.Anchor = AnchorStyles.Right | AnchorStyles.Left; editor = btn; } else { TextBox txtbox = new TextBox(); txtbox.Anchor = AnchorStyles.Right | AnchorStyles.Left; txtbox.Text = value; editor = txtbox; } } if (editor != null) { editor.Tag = s.Key; _editors.Add(editor); tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.AutoSize)); if (lbl != null) { tableLayoutPanel1.Controls.Add(lbl, 0, rowIndex); tableLayoutPanel1.Controls.Add(editor, 1, rowIndex++); } else { tableLayoutPanel1.Controls.Add(editor, 0, rowIndex++); tableLayoutPanel1.SetColumnSpan(editor, 2); } } }); //add auto expanding row to bottom to fill empty space Label emptyLabel = new Label(); emptyLabel.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left; emptyLabel.Text = string.Empty; tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.Percent, 100f)); tableLayoutPanel1.Controls.Add(emptyLabel, 0, rowIndex); tableLayoutPanel1.SetColumnSpan(emptyLabel, 2); } this.ResumeLayout(true); } catch (Exception ex) { log.Error(ex); Close(); } finally { domainManager.Dispose(); this.Cursor = Cursors.Default; } }
async Task <HubRequest[]> GetRequestsAsync() { if (_networkSetting.NetworkStatus != Util.ConnectionOKStatus) { return(new HubRequest[0]); } var datamartIDs = _networkSetting.DataMartList .Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || dm.ProcessQueriesAndNotUpload || dm.ProcessQueriesAndUploadAutomatically)) .Select(dm => dm.DataMartId).ToArray(); if (datamartIDs.Length == 0) { return(new HubRequest[0]); } var requestFilter = new RequestFilter { Statuses = new[] { DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted, DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted }, DataMartIds = datamartIDs }; var requests = Observable.Create <DTO.DataMartClient.RequestList>(async observer => { int index = 0; int batchSize = Properties.Settings.Default.AutoProcessingBatchSize; DTO.DataMartClient.RequestList rl = null; while (rl == null || (index < rl.TotalCount)) { rl = await DnsServiceManager.GetRequestList("AutoProcessor", _networkSetting, index, batchSize, requestFilter, DTO.DataMartClient.RequestSortColumn.RequestTime, true); if (rl == null || rl.TotalCount == 0) { break; } observer.OnNext(rl); index += batchSize; } observer.OnCompleted(); }).DefaultIfEmpty() .Aggregate((requestList1, requestList2) => { if (requestList1 == null && requestList2 == null) { return(new DTO.DataMartClient.RequestList { Segment = Array.Empty <DTO.DataMartClient.RequestListRow>(), SortedAscending = true, SortedByColumn = DTO.DataMartClient.RequestSortColumn.RequestTime }); } else if (requestList1 != null && requestList2 == null) { return(requestList1); } else if (requestList1 == null && requestList2 != null) { return(requestList2); } else { return(new DTO.DataMartClient.RequestList { Segment = requestList1.Segment.EmptyIfNull().Concat(requestList2.Segment.EmptyIfNull()).ToArray(), SortedAscending = requestList1.SortedAscending, SortedByColumn = requestList1.SortedByColumn, StartIndex = requestList1.StartIndex, TotalCount = requestList1.TotalCount }); } }) .SelectMany(requestList => { if (requestList == null) { return(Array.Empty <DTO.DataMartClient.RequestListRow>()); } return(requestList.Segment.DefaultIfEmpty().Where(s => s.AllowUnattendedProcessing)); }) .SelectMany(rlr => RequestCache.ForNetwork(_networkSetting).LoadRequest(rlr.ID, rlr.DataMartID)) .ToArray(); return(await requests); }
public void UploadMultipleSmallFiles() { //Guid requestID = new Guid("63f074ee-cf4c-4838-ae27-aa4700d7ca87"); //Guid dataMartID = new Guid("84C9440A-0B1F-4EC0-AF75-A85500CC8F16"); //string sourceFolderPath = @"C:\work\DR-UAT\DP2\v_psu_01_346\msoc"; string sourceFolderPath = @"C:\work\DR-UAT\DP2\v_psu_01_353\TEST"; Guid requestID = new Guid("2D35C33F-11D2-47E3-A2D8-AA4700BCCBBA"); Guid dataMartID = new Guid("6D55D9BA-9787-40DC-B9B7-A49F00C83946"); List <DTO.DataMartClient.Criteria.DocumentMetadata> documents = new List <DTO.DataMartClient.Criteria.DocumentMetadata>(); foreach (var file in System.IO.Directory.GetFiles(sourceFolderPath)) { System.IO.FileInfo fi = new System.IO.FileInfo(file); Guid documentID = Lpp.Utilities.DatabaseEx.NewGuid(); documents.Add( new DTO.DataMartClient.Criteria.DocumentMetadata { CurrentChunkIndex = 0, DataMartID = dataMartID, ID = documentID, IsViewable = false, Kind = string.Empty, MimeType = Lpp.Utilities.FileEx.GetMimeTypeByExtension(fi.Name), Name = fi.Name, RequestID = requestID } ); logger.Debug($"Adding document: {fi.Name} (ID: {documentID})"); } //documents = documents.Skip(12).Take(4).ToList(); Parallel.ForEach(documents, new ParallelOptions { MaxDegreeOfParallelism = 4 }, (doc) => { string uploadIdentifier = ("[" + Utilities.Crypto.Hash(Guid.NewGuid()) + "]").PadRight(16); System.IO.Stream stream = null; try { //if (cache.Enabled) //{ // stream = cache.GetDocumentStream(Guid.Parse(doc.DocumentID)); //} //else //{ // request.Processor.ResponseDocument(requestId, doc.DocumentID, out stream, doc.Size); //} stream = new System.IO.FileStream(System.IO.Path.Combine(sourceFolderPath, doc.Name), System.IO.FileMode.Open); var dto = new DTO.DataMartClient.Criteria.DocumentMetadata { ID = doc.ID, DataMartID = dataMartID, RequestID = requestID, IsViewable = doc.IsViewable, Size = doc.Size, MimeType = doc.MimeType, Kind = doc.Kind, Name = doc.Name, CurrentChunkIndex = 0 }; DnsServiceManager.PostDocumentChunk(uploadIdentifier, dto, stream, _networkSetting); } finally { stream.Dispose(); } }); }