public void Index(FileInfo item, Clue clue, AgentJobProcessorState <FileSystemCrawlJobData> state)
{
if (item == null)
{
throw new ArgumentNullException(nameof(item));
}
if (clue == null)
{
throw new ArgumentNullException(nameof(clue));
}
if (item.Exists == false)
{
return;
}
if (item.Length == 0)
{
return;
}
if (!ConfigurationManager.AppSettings.GetFlag("Crawl.InitialCrawl.FileIndexing", true))
{
return;
}
FileCrawlingUtility.IndexFile(item, clue.Data, clue, args, context);
}
protected ContactObserverBase([NotNull] ApplicationContext appContext,
[NotNull] AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
[NotNull] IClueFactory factory)
: base(appContext, state, factory)
{
this.state = state;
}
public static IEnumerable <AttachmentModel> FilterAttachments(
IEnumerable <AttachmentModel> attachments,
AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state)
{
var skippedFirstContactPhoto = false;
foreach (var model in attachments)
{
var attachment = model.Attachment;
if (attachment is FileAttachment fileAttachment && fileAttachment.IsContactPhoto && !skippedFirstContactPhoto)
{
skippedFirstContactPhoto = true;
continue;
}
if (AttachmentHelper.IsFiltered(state, attachment))
{
continue;
}
if (!(attachment is FileAttachment))
{
continue;
}
yield return(model);
}
}
protected ItemObserverBase([NotNull] ApplicationContext appContext,
[NotNull] AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
[NotNull] IClueFactory factory)
{
this.appContext = appContext ?? throw new ArgumentNullException(nameof(appContext));
this.state = state ?? throw new ArgumentNullException(nameof(state));
this.factory = factory ?? throw new ArgumentNullException(nameof(factory));
}
public override IEnumerable <Clue> DoProcess(ExecutionContext context, WebhookDataCommand command)
{
try
{
if (ConfigurationManager.AppSettings.GetFlag("Feature.Webhooks.Log.Posts", false))
{
context.Log.LogDebug(command.HttpPostData);
}
var configurationDataStore = context.ApplicationContext.Container.Resolve <IConfigurationRepository>();
if (command.WebhookDefinition.ProviderDefinitionId != null)
{
var providerDefinition = context.Organization.Providers.GetProviderDefinition(context, command.WebhookDefinition.ProviderDefinitionId.Value);
var jobDataCheck = context.ApplicationContext.Container.ResolveAll <IProvider>().FirstOrDefault(providerInstance => providerDefinition != null && providerInstance.Id == providerDefinition.ProviderId);
var configStoreData = configurationDataStore.GetConfigurationById(context, command.WebhookDefinition.ProviderDefinitionId.Value);
// If you have stopped the provider then don't process the webhooks
if (providerDefinition?.WebHooks != null)
{
if (providerDefinition.WebHooks == false || providerDefinition.IsEnabled == false)
{
return(new List <Clue>());
}
}
if (jobDataCheck != null)
{
var crawlJobData = new SalesforceCrawlJobData();
var clues = new List <Clue>();
IAgentJobProcessorArguments jobArgs = new QueuedJob(new AgentJob(Guid.NewGuid(), AgentJobPriority.Normal, "CluedIn" + SalesforceConstants.ProviderName, ProcessingRestriction.Any, null, null))
{
TaskScheduler = TaskScheduler.Default,
};
var processorState = new AgentJobProcessorState <SalesforceCrawlJobData>(jobArgs, AppContext)
{
JobData = crawlJobData,
Status = new AgentJobStatus {
Statistics = new AgentJobStatusStatistics()
}
};
throw new NotImplementedException($"TODO: Implement this to populate '{clues.GetType()}' with '{processorState}'");
}
}
}
catch (Exception exception)
{
using (context.Log.BeginScope(new { command.HttpHeaders, command.HttpQueryString, command.HttpPostData, command.WebhookDefinitionId }))
{
context.Log.LogError(exception, "Could not process web hook message");
}
}
return(new List <Clue>());
}
public MailboxObserver([NotNull] IClueFactory factory,
[NotNull] IObserver <Clue> observer,
[NotNull] AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
[NotNull] ApplicationContext appContext)
{
this.factory = factory ?? throw new ArgumentNullException(nameof(factory));
this.observer = observer ?? throw new ArgumentNullException(nameof(observer));
this.state = state ?? throw new ArgumentNullException(nameof(state));
this.appContext = appContext ?? throw new ArgumentNullException(nameof(appContext));
}
public static void ExLoad(
this FileAttachment obj,
AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
string fileName)
{
state.Status.Ping();
ActionExtensions.ExecuteWithRetry(
() => obj.Load(fileName),
retryIntervalMilliseconds: retryInterval.Milliseconds,
isTransient: ex => TransientExceptionHandler(state, ex));
}
public static void ExLoad(
this ServiceObject obj,
AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
PropertySet propertySet)
{
state.Status.Ping();
ActionExtensions.ExecuteWithRetry(
() => obj.Load(propertySet),
retryIntervalMilliseconds: retryInterval.Milliseconds,
isTransient: ex => TransientExceptionHandler(state, ex));
}
public static FindItemsResults <Item> ExFindItems(
this ExchangeService service,
AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
FolderId parentFolderId,
ItemView view)
{
state.Status.Ping();
return(ActionExtensions.ExecuteWithRetry(
() => PrepareRequest(service).FindItems(parentFolderId, view),
retryIntervalMilliseconds: retryInterval.Milliseconds,
isTransient: ex => TransientExceptionHandler(state, ex)));
}
public static GetSearchableMailboxesResponse ExGetSearchableMailboxes(
this ExchangeService service,
AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
string searchFilter,
bool expandGroupMembership)
{
state.Status.Ping();
return(ActionExtensions.ExecuteWithRetry(
() => PrepareRequest(service).GetSearchableMailboxes(searchFilter, expandGroupMembership),
retryIntervalMilliseconds: retryInterval.Milliseconds,
isTransient: ex => TransientExceptionHandler(state, ex)));
}
public static NameResolutionCollection ExResolveName(
this ExchangeService service,
AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state,
string nameToResolve,
ResolveNameSearchLocation searchScope,
bool returnContactDetails)
{
state.Status.Ping();
return(ActionExtensions.ExecuteWithRetry(
() => PrepareRequest(service).ResolveName(nameToResolve, searchScope, returnContactDetails),
retryIntervalMilliseconds: retryInterval.Milliseconds,
isTransient: ex => TransientExceptionHandler(state, ex)));
}
private static bool TransientExceptionHandler(AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state, Exception ex)
{
state.Status.Ping();
if (ex is ServerBusyException busyException)
{
if (busyException.BackOffMilliseconds > 0)
{
state.Log.Warn(() => $"ServerBusy - waiting for {busyException.BackOffMilliseconds} ms");
System.Threading.Tasks.Task.Delay(busyException.BackOffMilliseconds, state.CancellationTokenSource.Token).Wait(state.CancellationTokenSource.Token);
}
return(true);
}
if (ex is ServiceResponseException responseException && responseException.Message.StartsWith("You have exceeded the available concurrent connections for your account"))
{
state.Log.Warn("Too many connections - waiting for 30 secs");
System.Threading.Tasks.Task.Delay(TimeSpan.FromSeconds(30), state.CancellationTokenSource.Token).Wait(state.CancellationTokenSource.Token);
return(true);
}
if (ex.InnerException != null && ex.InnerException is ServerBusyException busyException2)
{
if (busyException2.BackOffMilliseconds > 0)
{
state.Log.Warn(() => $"ServerBusy - waiting for {busyException2.BackOffMilliseconds} ms");
System.Threading.Tasks.Task.Delay(busyException2.BackOffMilliseconds, state.CancellationTokenSource.Token).Wait(state.CancellationTokenSource.Token);
}
return(true);
}
if (ex.InnerException != null && ex.InnerException is ServiceResponseException responseException2 && responseException2.Message.StartsWith("You have exceeded the available concurrent connections for your account"))
{
state.Log.Warn("Too many connections - waiting for 30 secs");
System.Threading.Tasks.Task.Delay(TimeSpan.FromSeconds(30), state.CancellationTokenSource.Token).Wait(state.CancellationTokenSource.Token);
return(true);
}
return(false);
}
public virtual void LoadSchemaProperties(AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state)
{
if (this.schemaPropertiesLoaded)
{
return;
}
try
{
var version = (int)this.Service.RequestedServerVersion;
var additional = new List <PropertyDefinitionBase>()
{
MeetingMessageSchema.AssociatedAppointmentId,
MeetingMessageSchema.IsDelegated,
MeetingMessageSchema.IsOutOfDate,
MeetingMessageSchema.HasBeenProcessed,
MeetingMessageSchema.ResponseType,
MeetingMessageSchema.ICalUid,
MeetingMessageSchema.ICalRecurrenceId,
MeetingMessageSchema.ICalDateTimeStamp,
MeetingMessageSchema.IsOrganizer,
ItemSchema.Attachments,
};
var filtered = additional.Where(p => ((int)p.Version) <= version).ToArray();
PropertySet tmp = new PropertySet(BasePropertySet.FirstClassProperties, filtered);
if (this.Object.Id != null && filtered.Any())
{
// this.Object.ExLoad(tmp);
}
}
catch (Exception ex)
{
state.Log.Error(() => "Could not load MeetingMessage schema properties", ex);
}
this.schemaPropertiesLoaded = true;
}
public EmailAddressObserver([NotNull] IClueFactory factory, [NotNull] IObserver <Clue> observer, [NotNull] AgentJobProcessorState <ExchangeSharedMailboxCrawlJobData> state, ApplicationContext appContext)
: base(appContext, state, factory)
{
this.factory = factory ?? throw new ArgumentNullException(nameof(factory));
this.observer = observer ?? throw new ArgumentNullException(nameof(observer));
this.state = state;
}