public override async Task <IDictionary <string, object> > GetHelperConfiguration(
ProviderUpdateContext context,
[NotNull] CrawlJobData jobData,
Guid organizationId,
Guid userId,
Guid providerDefinitionId)
{
if (jobData == null)
{
throw new ArgumentNullException(nameof(jobData));
}
if (jobData is HubSpotCrawlJobData hubspotCrawlJobData)
{
if (_notifications != null)
{
_notifications.Publish <ProviderMessageCommand>(new ProviderMessageCommand()
{
OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = this.Id, ProviderName = this.Name, Message = "Authenticating", UserId = userId
});
}
var result = hubspotCrawlJobData.ToDictionary();
result.Add("webhooks", new List <object>()
{
new { DisplayName = "New Contact", Name = "contact.creation", Status = "ACTIVE", Description = "When a new contact is created." },
new { DisplayName = "Deleted Contact", Name = "contact.deletion", Status = "ACTIVE", Description = "When a contact is deleted." },
new { DisplayName = "Contact Update", Name = "contact.propertyChange", Status = "ACTIVE", Description = "When a contact is updated." },
new { DisplayName = "New Company", Name = "company.creation", Status = "ACTIVE", Description = "When a company is created." },
new { DisplayName = "Deleted Company", Name = "company.deletion", Status = "ACTIVE", Description = "When a company is deleted." },
new { DisplayName = "Company Update", Name = "company.propertyChange", Status = "ACTIVE", Description = "When a company is updated." },
new { DisplayName = "New Deal", Name = "deal.creation", Status = "ACTIVE", Description = "When a deal is created." },
new { DisplayName = "Deleted Deal", Name = "deal.deletion", Status = "ACTIVE", Description = "When a deal is deleted" },
new { DisplayName = "Deal Update", Name = "deal.propertyChange", Status = "ACTIVE", Description = "When a deal is updated." }
});
result.Add("expectedStatistics", hubspotCrawlJobData.ExpectedStatistics);
return(await Task.FromResult(result));
}
throw new InvalidOperationException($"Unpexected data type for HubSpotJobCrawlData, {jobData.GetType()}");
}
public override async Task <IDictionary <string, object> > GetHelperConfiguration(
ProviderUpdateContext context,
CrawlJobData jobData,
Guid organizationId,
Guid userId,
Guid providerDefinitionId,
string folderId)
{
if (jobData == null)
{
throw new ArgumentNullException(nameof(jobData));
}
var configuration = new Dictionary <string, object>();
if (jobData is DropBoxCrawlJobData dropBoxCrawlJobData)
{
configuration.Add(DropBoxConstants.KeyName.ClientId, dropBoxCrawlJobData.ClientId);
configuration.Add(DropBoxConstants.KeyName.ClientSecret, dropBoxCrawlJobData.ClientSecret);
try
{
if (dropBoxCrawlJobData.Token == null || string.IsNullOrEmpty(dropBoxCrawlJobData.Token.AccessToken))
{
throw new ApplicationException("The crawl data does not contain an access token");
}
_notifications?.Publish(new ProviderMessageCommand {
OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Authenticating", UserId = userId
});
var client = _dropboxClientFactory.CreateNew(dropBoxCrawlJobData);
_notifications?.Publish(new ProviderMessageCommand {
OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Fetching Folders", UserId = userId
});
var full = await client.ListFolderAsync(string.Empty, DropBoxConstants.FetchLimit, false);
var folderProjection = full.Entries.Where(i => i.IsFolder).Select(item => new FolderProjection {
Id = item.PathLower, Name = item.Name, Parent = string.IsNullOrEmpty(item.PathLower.Remove(item.PathLower.LastIndexOf('/'))) ? "/" : item.PathLower.Remove(item.PathLower.LastIndexOf('/')), Sensitive = ConfigurationManager.AppSettings["Configuration.Sensitive"] != null && ConfigurationManager.AppSettings["Configuration.Sensitive"].Split(',').Contains(item.Name), Permissions = new List <CluedInPermission>(), Active = true
}).ToList();
folderProjection.Add(new FolderProjection {
Id = "/", Name = "Root Folder", Parent = (string)null, Sensitive = false, Permissions = (List <CluedInPermission>)null, Active = true
});
var cursor = full.Cursor;
var hasMore = full.HasMore;
while (cursor != null && hasMore)
{
_notifications?.Publish(new ProviderMessageCommand {
OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Fetching Folders", UserId = userId
});
var fullWithCursor = await client.ListFolderContinueAsync(cursor);
foreach (var fol in fullWithCursor.Entries.Where(i => i.IsFolder))
{
if (!folderProjection.Select(d => d.Id).Contains(fol.PathLower))
{
folderProjection.Add(new FolderProjection {
Id = fol.PathLower, Name = fol.Name, Parent = string.IsNullOrEmpty(fol.PathLower.Remove(fol.PathLower.LastIndexOf('/'))) ? "/" : fol.PathLower.Remove(fol.PathLower.LastIndexOf('/')), Sensitive = ConfigurationManager.AppSettings["Configuration.Sensitive"] != null && ConfigurationManager.AppSettings["Configuration.Sensitive"].Split(',').Contains(fol.Name), Permissions = new List <CluedInPermission>(), Active = true
});
}
}
cursor = fullWithCursor.Cursor;
hasMore = fullWithCursor.HasMore;
}
_notifications?.Publish(new ProviderMessageCommand {
OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Fetching Shared Folders", UserId = userId
});
var responseFromBox = await client.GetFolderListViaRestAsync();
if (responseFromBox != null)
{
foreach (var sharedFolder in responseFromBox.entries)
{
var responseFromBox1 = await client.GetFolderPermissions(sharedFolder);
if (responseFromBox1 != null)
{
if (sharedFolder.path_lower != null)
{
if (!folderProjection.Select(d => d.Id).Contains(sharedFolder.path_lower))
{
folderProjection.Add(new FolderProjection {
Id = sharedFolder.path_lower, Name = sharedFolder.name, Parent = "/", Sensitive = false, Permissions = responseFromBox1.groups.Select(s => new CluedInPermission {
}).Concat(responseFromBox1.invitees.Select(s => new CluedInPermission {
})).Concat(responseFromBox1.users.Select(s => new CluedInPermission {
})).ToList(), Active = true
});
}
}
}
}
}
//Created,Uploaded,Commented,Downloaded,Previewed,Moved,Copied
configuration.Add("webhooks", new List <object>
{
new { DisplayName = "Created", Name = "Created", Status = "ACTIVE", Description = "When a new file or folder is created." },
new { DisplayName = "Uploaded", Name = "Uploaded", Status = "ACTIVE", Description = "When a new file or folder is uploaded." },
new { DisplayName = "Commented", Name = "Commented", Status = "ACTIVE", Description = "When a file or folder is commented upon." },
new { DisplayName = "Downloaded", Name = "Downloaded", Status = "ACTIVE", Description = "When a file or folder is downloaded." },
new { DisplayName = "Previewed", Name = "Previewed", Status = "ACTIVE", Description = "When a file is previewed." },
new { DisplayName = "Moved", Name = "Moved", Status = "ACTIVE", Description = "When a file or folder is moved." },
new { DisplayName = "Copied", Name = "Copied", Status = "ACTIVE", Description = "When a file or folder is copied." }
});
configuration.Add("folders", folderProjection);
if (configuration.ContainsKey("previousfolderProjectionCount"))
{
if (int.Parse(configuration["previousfolderProjectionCount"].ToString()) != folderProjection.Count)
{
configuration["previousfolderProjectionCount"] = folderProjection.Count;
}
}
else
{
configuration.Add("previousfolderProjectionCount", folderProjection.Count);
}
//This MUST be called after the previosFolderCount has been assigned
CheckForNewConfiguration(context, organizationId, providerDefinitionId, dropBoxCrawlJobData, folderProjection.Count);
_notifications?.Publish(new ProviderMessageCommand {
OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Forecasting projected processing time", UserId = userId
});
var usage = await client.GetSpaceUsageAsync();
configuration.Add("usage", new Usage {
UsedSpace = (long)usage.Used, NumberOfClues = null, TotalSpace = null
});
dropBoxCrawlJobData.ExpectedStatistics?.EntityTypeStatistics?.Add(new EntityTypeStatistics(EntityType.Files.Directory, folderProjection.Count, 0));
configuration.Add("expectedStatistics", dropBoxCrawlJobData.ExpectedStatistics);
}
catch (Exception exception)
{
_log.Warn(() => "Could not add DropBox provider", exception);
return(new Dictionary <string, object> {
{ "error", "Could not fetch configuration data from DropBox" }
});
}
}
return(await Task.FromResult(configuration));
}
