/********************************************************************************************************** * METHODS **********************************************************************************************************/ public override async Task <CrawlJobData> GetCrawlJobData( ProviderUpdateContext context, IDictionary <string, object> configuration, Guid organizationId, Guid userId, Guid providerDefinitionId) { var jobData = new FileSystemCrawlJobData(configuration); if (jobData.StartingPoints == null) { return(null); } //long fileCount = 0; //long folderCount = 0; //foreach (var folder in jobData.StartingPoints) //{ // fileCount += System.IO.Directory.GetFiles(folder.EntryPoint, "*.*", SearchOption.AllDirectories).Count(); // folderCount += System.IO.Directory.GetDirectories(folder.EntryPoint, "*.*", SearchOption.AllDirectories).Count(); //} ////Remove the current folder //if (fileCount > 0) // fileCount = fileCount - 1; //jobData.ExpectedTaskCount = fileCount + folderCount; var entityTypeStatistics = new ExpectedStatistics(); //entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.Directory, folderCount, 0)); //entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.File, fileCount, 0)); jobData.ExpectedData = 0; jobData.ExpectedTime = new TimeSpan(0, 0, 0); //jobData.ExpectedTaskCount = folderCount + fileCount; jobData.ExpectedStatistics = entityTypeStatistics; return(await Task.FromResult(jobData)); }
public override async Task <CrawlJobData> GetCrawlJobData( ProviderUpdateContext context, IDictionary <string, object> configuration, Guid organizationId, Guid userId, Guid providerDefinitionId) { var dropBoxProviderId = Id; if (_tokenStore == null) { return(new CrawlJobData { Errors = new Dictionary <string, string> { { "error", "Please contact CluedIn support in the top menu to help you setup with Dropbox." } } }); } var accounts = await _tokenStore.SelectAsync(context, account => account.ProviderId == dropBoxProviderId && account.OrganizationId == organizationId && account.UserId == userId && account.ConfigurationId == providerDefinitionId); if (!accounts.Any()) { accounts = await _tokenStore.SelectAsync(context, account => account.ProviderId == dropBoxProviderId && account.OrganizationId == organizationId && account.UserId == userId && account.ConfigurationId == null); // You have more than one token that could authenticate with this if (accounts.Count() > 1) { context.Log.Warn(() => "While binding a token to a provider, there was more than one token that would be viable and we chose the first."); } if (accounts.Count() == 1) { if (accounts.First().ConfigurationId == null) { accounts.First().ConfigurationId = providerDefinitionId; _tokenStore.Update(context, accounts.First()); } } } //Add Authentication for all Accounts here var agentToken = accounts.Select(i => new AgentToken { AccessToken = i.AccessToken, RefreshToken = i.RefreshToken, ExpiresIn = i.ExpiresIn }).FirstOrDefault(); if (agentToken == null) { return(new CrawlJobData { Errors = new Dictionary <string, string> { { "error", "Please contact CluedIn support in the top menu to help you setup with Dropbox." } } }); } if (!configuration.ContainsKey("baseUri")) { configuration.Add("baseUri", "https://www.dropbox.com/home/"); } if (!configuration.ContainsKey("Providers.DropBoxClientId")) { configuration.Add("Providers.DropBoxClientId", ConfigurationManager.AppSettings["Providers.DropBoxClientId"]); } if (!configuration.ContainsKey("Providers.DropBoxClientSecret")) { configuration.Add("Providers.DropBoxClientSecret", ConfigurationManager.AppSettings["Providers.DropBoxClientSecret"]); } if (!configuration.ContainsKey("Accounts")) { configuration.Add("Accounts", agentToken); } var jobData = new DropBoxCrawlJobData(configuration); var client = _dropboxClientFactory.CreateNew(jobData); if (jobData.LastestCursors == null || !jobData.LastestCursors.ContainsKey("Files")) { // HACK: Populate latest cursor if none is set try { jobData.LastestCursors = jobData.LastestCursors ?? new Dictionary <string, string>(); var cursor = await client.ListFolderGetLatestCursorAsync(string.Empty, recursive : true, includeMediaInfo : false).ConfigureAwait(false); if (cursor != null) { jobData.LastestCursors["Files"] = cursor.Cursor; } } catch (Exception ex) { context.Log.Error(() => "Failed to get latest cursor Dropbox", ex); } } try { var entityTypeStatistics = new ExpectedStatistics(); entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.File, 0, 0)); entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Person, 0, 0)); entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Infrastructure.Group, 0, 0)); entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Documents.Document, 0, 0)); entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Documents.Presentation, 0, 0)); entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Documents.Spreadsheet, 0, 0)); jobData.ExpectedStatistics = entityTypeStatistics; jobData.ExpectedData = 0; // accountInformation.quota_info.quota; jobData.ExpectedTime = new TimeSpan(0, 0, 0); //Maybe use the last run time from the agent statistics } catch (Exception ex) { context.Log.Error(() => "Failed to refresh the access token for Dropbox", ex); context.Organization.Providers.SetProviderDefinitionAuthenticationError(context, providerDefinitionId, ex); } //No need to look for a refresh token as it will only stop working if the person uninstalls the app. https://www.dropbox.com/developers/reference/oauthguide if (!jobData.IsAuthenticated) { return(new CrawlJobData { Errors = new Dictionary <string, string> { { "error", "Please contact CluedIn support in the top menu to help you setup with Dropbox." } } }); } if (ConfigurationManager.AppSettings.GetFlag("Feature.Webhooks.Enabled", false) && SupportsWebHooks) { var organizationProviderDataStore = context.Organization.DataStores.GetDataStore <ProviderDefinition>(); if (organizationProviderDataStore != null) { var webhookEnabled = organizationProviderDataStore.GetById(context, providerDefinitionId); if (webhookEnabled != null) { if (webhookEnabled.WebHooks != null) { if (webhookEnabled.WebHooks.Value == false) { var webhookDefinition = WebHookDefinitionBuilder.CreateWebhookDefinition(webhookEnabled); var results = await CreateWebHook(context, jobData, webhookDefinition, configuration); foreach (var result in results) { webhookDefinition.Signature = result.Signature; webhookDefinition.ExternalVersion = result.ExternalVersion; webhookDefinition.ExternalId = result.ExternalId; var webhookDefinitionDataStore = context.Organization.DataStores.GetDataStore <WebhookDefinition>(); if (webhookDefinitionDataStore != null) { webhookDefinitionDataStore.Insert(context, webhookDefinition); } } } } else { var webhookDefinition = WebHookDefinitionBuilder.CreateWebhookDefinition(webhookEnabled); var results = await CreateWebHook(context, jobData, webhookDefinition, configuration); foreach (var result in results) { webhookDefinition.Signature = result.Signature; webhookDefinition.ExternalVersion = result.ExternalVersion; webhookDefinition.ExternalId = result.ExternalId; var webhookDefinitionDataStore = context.Organization.DataStores.GetDataStore <WebhookDefinition>(); if (webhookDefinitionDataStore != null) { webhookDefinitionDataStore.Insert(context, webhookDefinition); } } } webhookEnabled.WebHooks = true; organizationProviderDataStore.Update(context, webhookEnabled); } } } return(jobData); }