public override async Task <IDictionary <string, object> > GetHelperConfiguration( ProviderUpdateContext context, [NotNull] CrawlJobData jobData, Guid organizationId, Guid userId, Guid providerDefinitionId) { if (jobData == null) { throw new ArgumentNullException(nameof(jobData)); } if (jobData is HubSpotCrawlJobData hubspotCrawlJobData) { if (_notifications != null) { _notifications.Publish <ProviderMessageCommand>(new ProviderMessageCommand() { OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = this.Id, ProviderName = this.Name, Message = "Authenticating", UserId = userId }); } var result = hubspotCrawlJobData.ToDictionary(); result.Add("webhooks", new List <object>() { new { DisplayName = "New Contact", Name = "contact.creation", Status = "ACTIVE", Description = "When a new contact is created." }, new { DisplayName = "Deleted Contact", Name = "contact.deletion", Status = "ACTIVE", Description = "When a contact is deleted." }, new { DisplayName = "Contact Update", Name = "contact.propertyChange", Status = "ACTIVE", Description = "When a contact is updated." }, new { DisplayName = "New Company", Name = "company.creation", Status = "ACTIVE", Description = "When a company is created." }, new { DisplayName = "Deleted Company", Name = "company.deletion", Status = "ACTIVE", Description = "When a company is deleted." }, new { DisplayName = "Company Update", Name = "company.propertyChange", Status = "ACTIVE", Description = "When a company is updated." }, new { DisplayName = "New Deal", Name = "deal.creation", Status = "ACTIVE", Description = "When a deal is created." }, new { DisplayName = "Deleted Deal", Name = "deal.deletion", Status = "ACTIVE", Description = "When a deal is deleted" }, new { DisplayName = "Deal Update", Name = "deal.propertyChange", Status = "ACTIVE", Description = "When a deal is updated." } }); result.Add("expectedStatistics", hubspotCrawlJobData.ExpectedStatistics); return(await Task.FromResult(result)); } throw new InvalidOperationException($"Unpexected data type for HubSpotJobCrawlData, {jobData.GetType()}"); }
public override async Task <IDictionary <string, object> > GetHelperConfiguration( ProviderUpdateContext context, CrawlJobData jobData, Guid organizationId, Guid userId, Guid providerDefinitionId, string folderId) { if (jobData == null) { throw new ArgumentNullException(nameof(jobData)); } var configuration = new Dictionary <string, object>(); if (jobData is DropBoxCrawlJobData dropBoxCrawlJobData) { configuration.Add(DropBoxConstants.KeyName.ClientId, dropBoxCrawlJobData.ClientId); configuration.Add(DropBoxConstants.KeyName.ClientSecret, dropBoxCrawlJobData.ClientSecret); try { if (dropBoxCrawlJobData.Token == null || string.IsNullOrEmpty(dropBoxCrawlJobData.Token.AccessToken)) { throw new ApplicationException("The crawl data does not contain an access token"); } _notifications?.Publish(new ProviderMessageCommand { OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Authenticating", UserId = userId }); var client = _dropboxClientFactory.CreateNew(dropBoxCrawlJobData); _notifications?.Publish(new ProviderMessageCommand { OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Fetching Folders", UserId = userId }); var full = await client.ListFolderAsync(string.Empty, DropBoxConstants.FetchLimit, false); var folderProjection = full.Entries.Where(i => i.IsFolder).Select(item => new FolderProjection { Id = item.PathLower, Name = item.Name, Parent = string.IsNullOrEmpty(item.PathLower.Remove(item.PathLower.LastIndexOf('/'))) ? "/" : item.PathLower.Remove(item.PathLower.LastIndexOf('/')), Sensitive = ConfigurationManager.AppSettings["Configuration.Sensitive"] != null && ConfigurationManager.AppSettings["Configuration.Sensitive"].Split(',').Contains(item.Name), Permissions = new List <CluedInPermission>(), Active = true }).ToList(); folderProjection.Add(new FolderProjection { Id = "/", Name = "Root Folder", Parent = (string)null, Sensitive = false, Permissions = (List <CluedInPermission>)null, Active = true }); var cursor = full.Cursor; var hasMore = full.HasMore; while (cursor != null && hasMore) { _notifications?.Publish(new ProviderMessageCommand { OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Fetching Folders", UserId = userId }); var fullWithCursor = await client.ListFolderContinueAsync(cursor); foreach (var fol in fullWithCursor.Entries.Where(i => i.IsFolder)) { if (!folderProjection.Select(d => d.Id).Contains(fol.PathLower)) { folderProjection.Add(new FolderProjection { Id = fol.PathLower, Name = fol.Name, Parent = string.IsNullOrEmpty(fol.PathLower.Remove(fol.PathLower.LastIndexOf('/'))) ? "/" : fol.PathLower.Remove(fol.PathLower.LastIndexOf('/')), Sensitive = ConfigurationManager.AppSettings["Configuration.Sensitive"] != null && ConfigurationManager.AppSettings["Configuration.Sensitive"].Split(',').Contains(fol.Name), Permissions = new List <CluedInPermission>(), Active = true }); } } cursor = fullWithCursor.Cursor; hasMore = fullWithCursor.HasMore; } _notifications?.Publish(new ProviderMessageCommand { OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Fetching Shared Folders", UserId = userId }); var responseFromBox = await client.GetFolderListViaRestAsync(); if (responseFromBox != null) { foreach (var sharedFolder in responseFromBox.entries) { var responseFromBox1 = await client.GetFolderPermissions(sharedFolder); if (responseFromBox1 != null) { if (sharedFolder.path_lower != null) { if (!folderProjection.Select(d => d.Id).Contains(sharedFolder.path_lower)) { folderProjection.Add(new FolderProjection { Id = sharedFolder.path_lower, Name = sharedFolder.name, Parent = "/", Sensitive = false, Permissions = responseFromBox1.groups.Select(s => new CluedInPermission { }).Concat(responseFromBox1.invitees.Select(s => new CluedInPermission { })).Concat(responseFromBox1.users.Select(s => new CluedInPermission { })).ToList(), Active = true }); } } } } } //Created,Uploaded,Commented,Downloaded,Previewed,Moved,Copied configuration.Add("webhooks", new List <object> { new { DisplayName = "Created", Name = "Created", Status = "ACTIVE", Description = "When a new file or folder is created." }, new { DisplayName = "Uploaded", Name = "Uploaded", Status = "ACTIVE", Description = "When a new file or folder is uploaded." }, new { DisplayName = "Commented", Name = "Commented", Status = "ACTIVE", Description = "When a file or folder is commented upon." }, new { DisplayName = "Downloaded", Name = "Downloaded", Status = "ACTIVE", Description = "When a file or folder is downloaded." }, new { DisplayName = "Previewed", Name = "Previewed", Status = "ACTIVE", Description = "When a file is previewed." }, new { DisplayName = "Moved", Name = "Moved", Status = "ACTIVE", Description = "When a file or folder is moved." }, new { DisplayName = "Copied", Name = "Copied", Status = "ACTIVE", Description = "When a file or folder is copied." } }); configuration.Add("folders", folderProjection); if (configuration.ContainsKey("previousfolderProjectionCount")) { if (int.Parse(configuration["previousfolderProjectionCount"].ToString()) != folderProjection.Count) { configuration["previousfolderProjectionCount"] = folderProjection.Count; } } else { configuration.Add("previousfolderProjectionCount", folderProjection.Count); } //This MUST be called after the previosFolderCount has been assigned CheckForNewConfiguration(context, organizationId, providerDefinitionId, dropBoxCrawlJobData, folderProjection.Count); _notifications?.Publish(new ProviderMessageCommand { OrganizationId = organizationId, ProviderDefinitionId = providerDefinitionId, ProviderId = Id, ProviderName = Name, Message = "Forecasting projected processing time", UserId = userId }); var usage = await client.GetSpaceUsageAsync(); configuration.Add("usage", new Usage { UsedSpace = (long)usage.Used, NumberOfClues = null, TotalSpace = null }); dropBoxCrawlJobData.ExpectedStatistics?.EntityTypeStatistics?.Add(new EntityTypeStatistics(EntityType.Files.Directory, folderProjection.Count, 0)); configuration.Add("expectedStatistics", dropBoxCrawlJobData.ExpectedStatistics); } catch (Exception exception) { _log.Warn(() => "Could not add DropBox provider", exception); return(new Dictionary <string, object> { { "error", "Could not fetch configuration data from DropBox" } }); } } return(await Task.FromResult(configuration)); }