Esempio n. 1
0
        public void Index(FileInfo item, Clue clue, AgentJobProcessorState <FileSystemCrawlJobData> state)
        {
            if (item == null)
            {
                throw new ArgumentNullException(nameof(item));
            }

            if (clue == null)
            {
                throw new ArgumentNullException(nameof(clue));
            }

            if (item.Exists == false)
            {
                return;
            }

            if (item.Length == 0)
            {
                return;
            }

            if (!ConfigurationManager.AppSettings.GetFlag("Crawl.InitialCrawl.FileIndexing", true))
            {
                return;
            }

            FileCrawlingUtility.IndexFile(item, clue.Data, clue, args, context);
        }
Esempio n. 2
0
 protected ContactObserverBase([NotNull] ApplicationContext appContext,
                               [NotNull] AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
                               [NotNull] IClueFactory factory)
     : base(appContext, state, factory)
 {
     this.state = state;
 }
Esempio n. 3
0
        public static IEnumerable <AttachmentModel> FilterAttachments(
            IEnumerable <AttachmentModel> attachments,
            AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state)
        {
            var skippedFirstContactPhoto = false;

            foreach (var model in attachments)
            {
                var attachment = model.Attachment;

                if (attachment is FileAttachment fileAttachment && fileAttachment.IsContactPhoto && !skippedFirstContactPhoto)
                {
                    skippedFirstContactPhoto = true;
                    continue;
                }

                if (AttachmentHelper.IsFiltered(state, attachment))
                {
                    continue;
                }

                if (!(attachment is FileAttachment))
                {
                    continue;
                }

                yield return(model);
            }
        }
Esempio n. 4
0
 protected ItemObserverBase([NotNull] ApplicationContext appContext,
                            [NotNull] AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
                            [NotNull] IClueFactory factory)
 {
     this.appContext = appContext ?? throw new ArgumentNullException(nameof(appContext));
     this.state      = state ?? throw new ArgumentNullException(nameof(state));
     this.factory    = factory ?? throw new ArgumentNullException(nameof(factory));
 }
        public override IEnumerable <Clue> DoProcess(ExecutionContext context, WebhookDataCommand command)
        {
            try
            {
                if (ConfigurationManager.AppSettings.GetFlag("Feature.Webhooks.Log.Posts", false))
                {
                    context.Log.LogDebug(command.HttpPostData);
                }

                var configurationDataStore = context.ApplicationContext.Container.Resolve <IConfigurationRepository>();
                if (command.WebhookDefinition.ProviderDefinitionId != null)
                {
                    var providerDefinition = context.Organization.Providers.GetProviderDefinition(context, command.WebhookDefinition.ProviderDefinitionId.Value);
                    var jobDataCheck       = context.ApplicationContext.Container.ResolveAll <IProvider>().FirstOrDefault(providerInstance => providerDefinition != null && providerInstance.Id == providerDefinition.ProviderId);
                    var configStoreData    = configurationDataStore.GetConfigurationById(context, command.WebhookDefinition.ProviderDefinitionId.Value);

                    // If you have stopped the provider then don't process the webhooks
                    if (providerDefinition?.WebHooks != null)
                    {
                        if (providerDefinition.WebHooks == false || providerDefinition.IsEnabled == false)
                        {
                            return(new List <Clue>());
                        }
                    }

                    if (jobDataCheck != null)
                    {
                        var crawlJobData = new SalesforceCrawlJobData();

                        var clues = new List <Clue>();

                        IAgentJobProcessorArguments jobArgs = new QueuedJob(new AgentJob(Guid.NewGuid(), AgentJobPriority.Normal, "CluedIn" + SalesforceConstants.ProviderName, ProcessingRestriction.Any, null, null))
                        {
                            TaskScheduler = TaskScheduler.Default,
                        };

                        var processorState = new AgentJobProcessorState <SalesforceCrawlJobData>(jobArgs, AppContext)
                        {
                            JobData = crawlJobData,
                            Status  = new AgentJobStatus {
                                Statistics = new AgentJobStatusStatistics()
                            }
                        };

                        throw new NotImplementedException($"TODO: Implement this to populate '{clues.GetType()}' with '{processorState}'");
                    }
                }
            }
            catch (Exception exception)
            {
                using (context.Log.BeginScope(new { command.HttpHeaders, command.HttpQueryString, command.HttpPostData, command.WebhookDefinitionId }))
                {
                    context.Log.LogError(exception, "Could not process web hook message");
                }
            }

            return(new List <Clue>());
        }
Esempio n. 6
0
 public MailboxObserver([NotNull] IClueFactory factory,
                        [NotNull] IObserver <Clue> observer,
                        [NotNull] AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
                        [NotNull] ApplicationContext appContext)
 {
     this.factory    = factory ?? throw new ArgumentNullException(nameof(factory));
     this.observer   = observer ?? throw new ArgumentNullException(nameof(observer));
     this.state      = state ?? throw new ArgumentNullException(nameof(state));
     this.appContext = appContext ?? throw new ArgumentNullException(nameof(appContext));
 }
Esempio n. 7
0
        public static void ExLoad(
            this FileAttachment obj,
            AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
            string fileName)
        {
            state.Status.Ping();

            ActionExtensions.ExecuteWithRetry(
                () => obj.Load(fileName),
                retryIntervalMilliseconds: retryInterval.Milliseconds,
                isTransient: ex => TransientExceptionHandler(state, ex));
        }
Esempio n. 8
0
        public static void ExLoad(
            this ServiceObject obj,
            AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
            PropertySet propertySet)
        {
            state.Status.Ping();

            ActionExtensions.ExecuteWithRetry(
                () => obj.Load(propertySet),
                retryIntervalMilliseconds: retryInterval.Milliseconds,
                isTransient: ex => TransientExceptionHandler(state, ex));
        }
Esempio n. 9
0
        public static FindItemsResults <Item> ExFindItems(
            this ExchangeService service,
            AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
            FolderId parentFolderId,
            ItemView view)
        {
            state.Status.Ping();

            return(ActionExtensions.ExecuteWithRetry(
                       () => PrepareRequest(service).FindItems(parentFolderId, view),
                       retryIntervalMilliseconds: retryInterval.Milliseconds,
                       isTransient: ex => TransientExceptionHandler(state, ex)));
        }
Esempio n. 10
0
        public static GetSearchableMailboxesResponse ExGetSearchableMailboxes(
            this ExchangeService service,
            AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
            string searchFilter,
            bool expandGroupMembership)
        {
            state.Status.Ping();

            return(ActionExtensions.ExecuteWithRetry(
                       () => PrepareRequest(service).GetSearchableMailboxes(searchFilter, expandGroupMembership),
                       retryIntervalMilliseconds: retryInterval.Milliseconds,
                       isTransient: ex => TransientExceptionHandler(state, ex)));
        }
Esempio n. 11
0
        public static NameResolutionCollection ExResolveName(
            this ExchangeService service,
            AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
            string nameToResolve,
            ResolveNameSearchLocation searchScope,
            bool returnContactDetails)
        {
            state.Status.Ping();

            return(ActionExtensions.ExecuteWithRetry(
                       () => PrepareRequest(service).ResolveName(nameToResolve, searchScope, returnContactDetails),
                       retryIntervalMilliseconds: retryInterval.Milliseconds,
                       isTransient: ex => TransientExceptionHandler(state, ex)));
        }
Esempio n. 12
0
        private static bool TransientExceptionHandler(AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state, Exception ex)
        {
            state.Status.Ping();

            if (ex is ServerBusyException busyException)
            {
                if (busyException.BackOffMilliseconds > 0)
                {
                    state.Log.Warn(() => $"ServerBusy - waiting for {busyException.BackOffMilliseconds} ms");
                    System.Threading.Tasks.Task.Delay(busyException.BackOffMilliseconds, state.CancellationTokenSource.Token).Wait(state.CancellationTokenSource.Token);
                }

                return(true);
            }

            if (ex is ServiceResponseException responseException && responseException.Message.StartsWith("You have exceeded the available concurrent connections for your account"))
            {
                state.Log.Warn("Too many connections - waiting for 30 secs");
                System.Threading.Tasks.Task.Delay(TimeSpan.FromSeconds(30), state.CancellationTokenSource.Token).Wait(state.CancellationTokenSource.Token);
                return(true);
            }

            if (ex.InnerException != null && ex.InnerException is ServerBusyException busyException2)
            {
                if (busyException2.BackOffMilliseconds > 0)
                {
                    state.Log.Warn(() => $"ServerBusy - waiting for {busyException2.BackOffMilliseconds} ms");
                    System.Threading.Tasks.Task.Delay(busyException2.BackOffMilliseconds, state.CancellationTokenSource.Token).Wait(state.CancellationTokenSource.Token);
                }

                return(true);
            }

            if (ex.InnerException != null && ex.InnerException is ServiceResponseException responseException2 && responseException2.Message.StartsWith("You have exceeded the available concurrent connections for your account"))
            {
                state.Log.Warn("Too many connections - waiting for 30 secs");
                System.Threading.Tasks.Task.Delay(TimeSpan.FromSeconds(30), state.CancellationTokenSource.Token).Wait(state.CancellationTokenSource.Token);
                return(true);
            }

            return(false);
        }
Esempio n. 13
0
        public virtual void LoadSchemaProperties(AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state)
        {
            if (this.schemaPropertiesLoaded)
            {
                return;
            }

            try
            {
                var version = (int)this.Service.RequestedServerVersion;

                var additional = new List <PropertyDefinitionBase>()
                {
                    MeetingMessageSchema.AssociatedAppointmentId,
                    MeetingMessageSchema.IsDelegated,
                    MeetingMessageSchema.IsOutOfDate,
                    MeetingMessageSchema.HasBeenProcessed,
                    MeetingMessageSchema.ResponseType,
                    MeetingMessageSchema.ICalUid,
                    MeetingMessageSchema.ICalRecurrenceId,
                    MeetingMessageSchema.ICalDateTimeStamp,
                    MeetingMessageSchema.IsOrganizer,
                    ItemSchema.Attachments,
                };

                var         filtered = additional.Where(p => ((int)p.Version) <= version).ToArray();
                PropertySet tmp      = new PropertySet(BasePropertySet.FirstClassProperties, filtered);

                if (this.Object.Id != null && filtered.Any())
                {
                    // this.Object.ExLoad(tmp);
                }
            }
            catch (Exception ex)
            {
                state.Log.Error(() => "Could not load MeetingMessage schema properties", ex);
            }

            this.schemaPropertiesLoaded = true;
        }
Esempio n. 14
0
 public EmailAddressObserver([NotNull] IClueFactory factory, [NotNull] IObserver <Clue> observer, [NotNull] AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state, ApplicationContext appContext)
     : base(appContext, state, factory)
 {
     this.factory  = factory ?? throw new ArgumentNullException(nameof(factory));
     this.observer = observer ?? throw new ArgumentNullException(nameof(observer));
     this.state    = state;
 }