public async Task <System.IO.Stream> GetPackage(Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier packageIdentifier) { var response = await this._Client.GetAsync(this._Host + AdaptersPath + "/GetPackage?identifier=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Identifier) + "&version=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Version)); if (response.StatusCode != System.Net.HttpStatusCode.OK) { throw new Exception("An error occurred while trying to download the package:" + packageIdentifier.PackageName() + "/n" + response.ReasonPhrase + "/nStatusCode:" + response.StatusCode.ToString()); } return(await response.Content.ReadAsStreamAsync()); }
public async Task <System.IO.Stream> GetPackage(Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier packageIdentifier) { string path = "GetPackage?identifier=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Identifier) + "&version=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Version); string identifier; _log.Debug(ExecutingMessage(path, out identifier)); var response = await this._Client.GetAsync(this._Host + AdaptersPath + "/GetPackage?identifier=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Identifier) + "&version=" + System.Web.HttpUtility.UrlEncode(packageIdentifier.Version)); if (response.StatusCode != System.Net.HttpStatusCode.OK) { _log.Error(CompletionMessage(identifier, path, false) + "\r\n" + response.GetMessage()); throw new Exception("An error occurred while trying to download the package:" + packageIdentifier.PackageName() + "/n" + response.ReasonPhrase + "/nStatusCode:" + response.StatusCode.ToString()); } _log.Debug(CompletionMessage(identifier, path, true)); return(await response.Content.ReadAsStreamAsync()); }
private void AutoProcess(KeyValuePair <string, HubRequest> input) { var request = input.Value; var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId); if (datamartDescription.ProcessQueriesAndUploadAutomatically == false && datamartDescription.ProcessQueriesAndNotUpload == false) { //just notify, do not process string message = $"New query submitted and awaiting processing in { request.ProjectName } Project: { request.Source.Name } ({request.Source.Identifier})"; SystemTray.DisplayNewQueryNotificationToolTip(message); RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload); return; } var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID); var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier { Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion }; if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName()))) { DnsServiceManager.DownloadPackage(_networkSetting, packageIdentifier); } var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath); try { domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion); IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId); ProcessorManager.UpdateProcessorSettings(modelDescription, processor); processor.Settings.Add("NetworkId", request.NetworkId); Lib.Caching.DocumentCacheManager cache = new Lib.Caching.DocumentCacheManager(request.NetworkId, request.DataMartId, request.Source.ID, request.Source.Responses.Where(x => x.DataMartID == request.DataMartId).FirstOrDefault().ResponseID); //need to initialize before checking the capabilities and settings of the processor since they may change based on the type of request being sent. if (processor is IEarlyInitializeModelProcessor) { ((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, _networkSetting))).ToArray()); } if (processor != null && processor.ModelMetadata != null && processor.ModelMetadata.Capabilities != null && processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") && !(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"]) { //can't be run, don't attempt autoprocessing RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload); domainManager.Dispose(); return; } request.Processor = processor; if (cache.HasResponseDocuments == false && (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted || request.RoutingStatus == DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted)) { if (processor != null) { SystemTray.GenerateNotification(request, request.NetworkId); StartProcessingRequest(request, processor, datamartDescription, domainManager, cache); return; } } else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval) { if (datamartDescription.ProcessQueriesAndUploadAutomatically) { // Increment counter System.Threading.Interlocked.Increment(ref _queriesProcessedCount); SystemTray.UpdateNotifyText(_queriesProcessedCount, request.DataMartName, request.NetworkId); StartUploadingRequest(request, domainManager, cache); return; } } else if (cache.HasResponseDocuments) { RequestStatuses.TryAdd(input.Key, ProcessingStatus.PendingUpload); } domainManager.Dispose(); } catch (Exception ex) { Log.Error($"Error autoprocessing Request: { request.Source.Identifier }, DataMartId: { request.DataMartId }, NetworkId: {request.NetworkId}", ex); domainManager.Dispose(); throw; } }
protected override void OnDoWork(DoWorkEventArgs e) { _log.Info("Automated processing worker started."); while (true) { _log.Info("Checking Master Request Queue for requests requiring automated processing."); SystemTray.UpdateNotificationIcon(IconType.IconBusy, "Processing Requests"); var reqs = from ns in Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Cast <NetWorkSetting>().ToObservable() where ns.NetworkStatus == Util.ConnectionOKStatus let dmIds = ns.DataMartList // BMS: Note the ProcessAndNotUpload feature has been temporarilty disabled until we fix the processing around this feature .Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || /* dm.ProcessQueriesAndNotUpload || */ dm.ProcessQueriesAndUploadAutomatically)) .Select(dm => dm.DataMartId) .ToArray() where dmIds.Any() from list in DnsServiceManager.GetRequestList(ns, 0, Properties.Settings.Default.AutoProcessingBatchSize, new RequestFilter { Statuses = new [] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted }, DataMartIds = dmIds }, null, null) from rl in list.Segment.EmptyIfNull().ToObservable() where rl.AllowUnattendedProcessing from r in RequestCache.ForNetwork(ns).LoadRequest(rl.ID, rl.DataMartID) select new { Request = r, NetworkSetting = ns }; reqs .Do(r => { var request = r.Request; var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId); var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID); var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier { Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion }; if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName()))) { DnsServiceManager.DownloadPackage(r.NetworkSetting, packageIdentifier); } using (var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath)) { domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion); IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId); ProcessorManager.UpdateProcessorSettings(modelDescription, processor); if (processor is IEarlyInitializeModelProcessor) { ((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, r.NetworkSetting))).ToArray()); } if (processor != null && processor.ModelMetadata != null && processor.ModelMetadata.Capabilities != null && processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") && !(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"]) { //can't be run, don't attempt autoprocessing return; } request.Processor = processor; if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted) { if (processor != null) { SystemTray.generate_notification(request, request.NetworkId); if (datamartDescription.NotifyOfNewQueries) { SystemTray.UpdateNotificationIcon(IconType.IconBusy, string.Format("Query Submitted by {0}", request.Source.Author.Username)); } else { processor.SetRequestProperties(request.Source.ID.ToString(), request.Properties); ProcessRequest(request); var statusCode = request.Processor.Status(request.Source.ID.ToString()).Code; if (datamartDescription.ProcessQueriesAndUploadAutomatically && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage)) { // Post process requests that are automatically uploaded processor.PostProcess(request.Source.ID.ToString()); // Increment counter _queriesProcessedCount++; SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId); UploadRequest(request); } } } } else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval) { if (datamartDescription.ProcessQueriesAndUploadAutomatically) { // Increment counter _queriesProcessedCount++; SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId); UploadRequest(request); } } } }) .LogExceptions(_log.Error) .Catch() .LastOrDefault(); SystemTray.UpdateNotificationIcon(IconType.IconDefault, null); Thread.Sleep(DMClient.Properties.Settings.Default.RefreshRate); } }