예제 #1
0
        protected override void OnDoWork(DoWorkEventArgs e)
        {
            _log.Info("Automated processing worker started.");
            while (true)
            {
                _log.Info("Checking Master Request Queue for requests requiring automated processing.");
                SystemTray.UpdateNotificationIcon(IconType.IconBusy, "Processing Requests");

                var reqs = from ns in Configuration.Instance.NetworkSettingCollection.NetWorkSettings.Cast <NetWorkSetting>().ToObservable()
                           where ns.NetworkStatus == Util.ConnectionOKStatus

                           let dmIds = ns.DataMartList
                                       // BMS: Note the ProcessAndNotUpload feature has been temporarilty disabled until we fix the processing around this feature
                                       .Where(dm => dm.AllowUnattendedOperation && (dm.NotifyOfNewQueries || /* dm.ProcessQueriesAndNotUpload || */ dm.ProcessQueriesAndUploadAutomatically))
                                       .Select(dm => dm.DataMartId)
                                       .ToArray()
                                       where dmIds.Any()

                                       from list in DnsServiceManager.GetRequestList(ns, 0, Properties.Settings.Default.AutoProcessingBatchSize,
                                                                                     new RequestFilter {
                    Statuses = new [] { Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted }, DataMartIds = dmIds
                }, null, null)

                                       from rl in list.Segment.EmptyIfNull().ToObservable()
                                       where rl.AllowUnattendedProcessing
                                       from r in RequestCache.ForNetwork(ns).LoadRequest(rl.ID, rl.DataMartID)
                                       select new { Request = r, NetworkSetting = ns };

                reqs
                .Do(r =>
                {
                    var request             = r.Request;
                    var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId);
                    var modelDescription    = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID);


                    var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier {
                        Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion
                    };
                    if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
                    {
                        DnsServiceManager.DownloadPackage(r.NetworkSetting, packageIdentifier);
                    }

                    using (var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath))
                    {
                        domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion);
                        IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId);
                        ProcessorManager.UpdateProcessorSettings(modelDescription, processor);

                        if (processor is IEarlyInitializeModelProcessor)
                        {
                            ((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, r.NetworkSetting))).ToArray());
                        }

                        if (processor != null &&
                            processor.ModelMetadata != null &&
                            processor.ModelMetadata.Capabilities != null &&
                            processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") &&
                            !(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"])
                        {
                            //can't be run, don't attempt autoprocessing
                            return;
                        }


                        request.Processor = processor;

                        if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted)
                        {
                            if (processor != null)
                            {
                                SystemTray.generate_notification(request, request.NetworkId);

                                if (datamartDescription.NotifyOfNewQueries)
                                {
                                    SystemTray.UpdateNotificationIcon(IconType.IconBusy, string.Format("Query Submitted by {0}", request.Source.Author.Username));
                                }
                                else
                                {
                                    processor.SetRequestProperties(request.Source.ID.ToString(), request.Properties);
                                    ProcessRequest(request);

                                    var statusCode = request.Processor.Status(request.Source.ID.ToString()).Code;
                                    if (datamartDescription.ProcessQueriesAndUploadAutomatically && (statusCode == RequestStatus.StatusCode.Complete || statusCode == RequestStatus.StatusCode.CompleteWithMessage))
                                    {
                                        // Post process requests that are automatically uploaded
                                        processor.PostProcess(request.Source.ID.ToString());
                                        // Increment counter
                                        _queriesProcessedCount++;
                                        SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
                                        UploadRequest(request);
                                    }
                                }
                            }
                        }
                        else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval)
                        {
                            if (datamartDescription.ProcessQueriesAndUploadAutomatically)
                            {
                                // Increment counter
                                _queriesProcessedCount++;
                                SystemTray.update_notify_text(_queriesProcessedCount, request.DataMartName, request.NetworkId);
                                UploadRequest(request);
                            }
                        }
                    }
                })
                .LogExceptions(_log.Error)
                .Catch()
                .LastOrDefault();

                SystemTray.UpdateNotificationIcon(IconType.IconDefault, null);
                Thread.Sleep(DMClient.Properties.Settings.Default.RefreshRate);
            }
        }
예제 #2
0
        private void AutoProcess(KeyValuePair <string, HubRequest> input)
        {
            var request = input.Value;

            var datamartDescription = Configuration.Instance.GetDataMartDescription(request.NetworkId, request.DataMartId);

            if (datamartDescription.ProcessQueriesAndUploadAutomatically == false && datamartDescription.ProcessQueriesAndNotUpload == false)
            {
                //just notify, do not process
                string message = $"New query submitted and awaiting processing in { request.ProjectName } Project: { request.Source.Name } ({request.Source.Identifier})";
                SystemTray.DisplayNewQueryNotificationToolTip(message);
                RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload);
                return;
            }


            var modelDescription = Configuration.Instance.GetModelDescription(request.NetworkId, request.DataMartId, request.Source.ModelID);

            var packageIdentifier = new Lpp.Dns.DTO.DataMartClient.RequestTypeIdentifier {
                Identifier = request.Source.RequestTypePackageIdentifier, Version = request.Source.AdapterPackageVersion
            };

            if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
            {
                DnsServiceManager.DownloadPackage(_networkSetting, packageIdentifier);
            }

            var domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath);

            try
            {
                domainManager.Load(request.Source.RequestTypePackageIdentifier, request.Source.AdapterPackageVersion);
                IModelProcessor processor = domainManager.GetProcessor(modelDescription.ProcessorId);
                ProcessorManager.UpdateProcessorSettings(modelDescription, processor);
                processor.Settings.Add("NetworkId", request.NetworkId);

                Lib.Caching.DocumentCacheManager cache = new Lib.Caching.DocumentCacheManager(request.NetworkId, request.DataMartId, request.Source.ID, request.Source.Responses.Where(x => x.DataMartID == request.DataMartId).FirstOrDefault().ResponseID);

                //need to initialize before checking the capabilities and settings of the processor since they may change based on the type of request being sent.
                if (processor is IEarlyInitializeModelProcessor)
                {
                    ((IEarlyInitializeModelProcessor)processor).Initialize(modelDescription.ModelId, request.Documents.Select(d => new DocumentWithStream(d.ID, new Document(d.ID, d.Document.MimeType, d.Document.Name, d.Document.IsViewable, Convert.ToInt32(d.Document.Size), d.Document.Kind), new DocumentChunkStream(d.ID, _networkSetting))).ToArray());
                }

                if (processor != null &&
                    processor.ModelMetadata != null &&
                    processor.ModelMetadata.Capabilities != null &&
                    processor.ModelMetadata.Capabilities.ContainsKey("CanRunAndUpload") &&
                    !(bool)processor.ModelMetadata.Capabilities["CanRunAndUpload"])
                {
                    //can't be run, don't attempt autoprocessing
                    RequestStatuses.TryAdd(input.Key, ProcessingStatus.CannotRunAndUpload);

                    domainManager.Dispose();
                    return;
                }

                request.Processor = processor;

                if (cache.HasResponseDocuments == false && (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.Submitted || request.RoutingStatus == DTO.DataMartClient.Enums.DMCRoutingStatus.Resubmitted))
                {
                    if (processor != null)
                    {
                        SystemTray.GenerateNotification(request, request.NetworkId);
                        StartProcessingRequest(request, processor, datamartDescription, domainManager, cache);
                        return;
                    }
                }
                else if (request.RoutingStatus == Lpp.Dns.DTO.DataMartClient.Enums.DMCRoutingStatus.AwaitingResponseApproval)
                {
                    if (datamartDescription.ProcessQueriesAndUploadAutomatically)
                    {
                        // Increment counter
                        System.Threading.Interlocked.Increment(ref _queriesProcessedCount);

                        SystemTray.UpdateNotifyText(_queriesProcessedCount, request.DataMartName, request.NetworkId);

                        StartUploadingRequest(request, domainManager, cache);
                        return;
                    }
                }
                else if (cache.HasResponseDocuments)
                {
                    RequestStatuses.TryAdd(input.Key, ProcessingStatus.PendingUpload);
                }

                domainManager.Dispose();
            }
            catch (Exception ex)
            {
                Log.Error($"Error autoprocessing Request: { request.Source.Identifier }, DataMartId: { request.DataMartId }, NetworkId: {request.NetworkId}", ex);

                domainManager.Dispose();
                throw;
            }
        }
예제 #3
0
        private void SettingsForm_Load(object sender, EventArgs e)
        {
            Lpp.Dns.DataMart.Client.DomainManger.DomainManager domainManager = new DomainManger.DomainManager(Configuration.PackagesFolderPath);
            try
            {
                if (ModelDescription == null)
                {
                    return;
                }

                if (ModelDescription.ProcessorId == Guid.Empty)
                {
                    ModelDescription.ProcessorId = HubModel.ModelProcessorId;
                }

                //get the package identifier and version
                var packageIdentifier = DnsServiceManager.GetRequestTypeIdentifier(NetworkSetting, ModelDescription.ModelId, ModelDescription.ProcessorId);
                if (!System.IO.File.Exists(System.IO.Path.Combine(Configuration.PackagesFolderPath, packageIdentifier.PackageName())))
                {
                    DnsServiceManager.DownloadPackage(NetworkSetting, packageIdentifier);
                }
                domainManager.Load(packageIdentifier.Identifier, packageIdentifier.Version);
                modelProcessor = domainManager.GetProcessor(ModelDescription.ProcessorId);
                ProcessorManager.UpdateProcessorSettings(ModelDescription, modelProcessor);

                if (modelProcessor is IEarlyInitializeModelProcessor)
                {
                    ((IEarlyInitializeModelProcessor)modelProcessor).Initialize(ModelDescription.ModelId, Array.Empty <DocumentWithStream>());
                }

                this.SuspendLayout();

                if (modelProcessor.ModelMetadata.Settings == null || !modelProcessor.ModelMetadata.Settings.Any())
                {
                    var noSettingsLabel = new Label();
                    noSettingsLabel.Text      = "This model processor does not have any settings.";
                    noSettingsLabel.Anchor    = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left;
                    noSettingsLabel.TextAlign = ContentAlignment.MiddleCenter;
                    tableLayoutPanel1.Controls.Add(noSettingsLabel, 0, 0);
                    tableLayoutPanel1.SetColumnSpan(noSettingsLabel, 2);
                }
                else
                {
                    _editors = new HashSet <Control>();
                    tableLayoutPanel1.RowStyles.Clear();
                    int rowIndex = 0;

                    IEnumerable <Lpp.Dns.DataMart.Model.Settings.ProcessorSetting> settings = modelProcessor.ModelMetadata.Settings;
                    if (modelProcessor.ModelMetadata.SQlProviders != null && modelProcessor.ModelMetadata.SQlProviders.Any())
                    {
                        DataMart.Client.Controls.DataSourceEditor dsEditor = new Controls.DataSourceEditor(modelProcessor.ModelMetadata, ModelDescription.Properties);
                        dsEditor.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left;

                        tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.AutoSize));
                        tableLayoutPanel1.Controls.Add(dsEditor, 0, rowIndex++);
                        tableLayoutPanel1.SetColumnSpan(dsEditor, 2);

                        settings = settings.Where(s => !Lpp.Dns.DataMart.Model.Settings.ProcessorSettings.IsDbSetting(s.Key)).ToArray();
                        _editors.Add(dsEditor);
                    }

                    settings.ToList().ForEach(s => {
                        string value = ModelDescription.Properties.Where(p => string.Equals(p.Name, s.Key, StringComparison.OrdinalIgnoreCase)).Select(p => p.Value).FirstOrDefault();

                        if (string.IsNullOrEmpty(value) && s.Required && !string.IsNullOrEmpty(s.DefaultValue))
                        {
                            value = s.DefaultValue;
                        }

                        Label lbl     = new Label();
                        lbl.AutoSize  = true;
                        lbl.Anchor    = AnchorStyles.Right;
                        lbl.TextAlign = ContentAlignment.MiddleRight;
                        lbl.Text      = s.Title.EndsWith(":") ? s.Title : s.Title + ":";

                        Control editor = null;
                        if (s.ValidValues != null)
                        {
                            ComboBox combo      = new ComboBox();
                            combo.DropDownStyle = ComboBoxStyle.DropDownList;
                            combo.Anchor        = AnchorStyles.Right | AnchorStyles.Left;
                            combo.Name          = s.Key;

                            combo.Items.AddRange(s.ValidValues.Select(v => new PropertyData(v.Key, v.Value.ToString())).ToArray());
                            if (!string.IsNullOrEmpty(value))
                            {
                                foreach (PropertyData p in combo.Items)
                                {
                                    if (string.Equals(p.Value, value, StringComparison.OrdinalIgnoreCase))
                                    {
                                        combo.SelectedItem = p;
                                        break;
                                    }
                                }
                            }

                            if (combo.SelectedIndex < 0)
                            {
                                combo.SelectedIndex = 0;
                            }

                            editor = combo;
                        }
                        else
                        {
                            if (s.ValueType == typeof(bool) || s.ValueType == typeof(Nullable <bool>))
                            {
                                CheckBox chkbox  = new CheckBox();
                                chkbox.Anchor    = AnchorStyles.Left;
                                chkbox.Text      = s.Title;
                                chkbox.TextAlign = ContentAlignment.MiddleLeft;
                                chkbox.AutoSize  = true;

                                if (!string.IsNullOrEmpty(value))
                                {
                                    bool isChecked = false;
                                    bool.TryParse(value, out isChecked);
                                    chkbox.Checked = isChecked;
                                }

                                editor = chkbox;
                                lbl    = null;
                            }
                            else if (s.ValueType == typeof(Lpp.Dns.DataMart.Model.Settings.FilePickerEditor))
                            {
                                SelectFileButton btn = new SelectFileButton(s.EditorSettings as Lpp.Dns.DataMart.Model.Settings.FilePickerEditor);
                                if (btn.Multiselect)
                                {
                                    btn.FileNames = ((value ?? "").Trim(',')).Split(',');
                                }
                                else
                                {
                                    btn.FileName = value;
                                }
                                btn.Anchor = AnchorStyles.Right | AnchorStyles.Left;
                                editor     = btn;
                            }
                            else if (s.ValueType == typeof(Lpp.Dns.DataMart.Model.Settings.FolderSelectorEditor))
                            {
                                SelectFolderButton btn = new SelectFolderButton(s.EditorSettings as Lpp.Dns.DataMart.Model.Settings.FolderSelectorEditor);
                                btn.FolderPath         = value;
                                btn.Anchor             = AnchorStyles.Right | AnchorStyles.Left;
                                editor = btn;
                            }
                            else
                            {
                                TextBox txtbox = new TextBox();
                                txtbox.Anchor  = AnchorStyles.Right | AnchorStyles.Left;
                                txtbox.Text    = value;
                                editor         = txtbox;
                            }
                        }

                        if (editor != null)
                        {
                            editor.Tag = s.Key;
                            _editors.Add(editor);

                            tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.AutoSize));
                            if (lbl != null)
                            {
                                tableLayoutPanel1.Controls.Add(lbl, 0, rowIndex);
                                tableLayoutPanel1.Controls.Add(editor, 1, rowIndex++);
                            }
                            else
                            {
                                tableLayoutPanel1.Controls.Add(editor, 0, rowIndex++);
                                tableLayoutPanel1.SetColumnSpan(editor, 2);
                            }
                        }
                    });

                    //add auto expanding row to bottom to fill empty space
                    Label emptyLabel = new Label();
                    emptyLabel.Anchor = AnchorStyles.Top | AnchorStyles.Right | AnchorStyles.Bottom | AnchorStyles.Left;
                    emptyLabel.Text   = string.Empty;
                    tableLayoutPanel1.RowStyles.Add(new RowStyle(SizeType.Percent, 100f));
                    tableLayoutPanel1.Controls.Add(emptyLabel, 0, rowIndex);
                    tableLayoutPanel1.SetColumnSpan(emptyLabel, 2);
                }
                this.ResumeLayout(true);
            }
            catch (Exception ex)
            {
                log.Error(ex);
                Close();
            }
            finally
            {
                domainManager.Dispose();
                this.Cursor = Cursors.Default;
            }
        }