/**********************************************************************************************************
* METHODS
**********************************************************************************************************/
public override async Task <CrawlJobData> GetCrawlJobData(
ProviderUpdateContext context,
IDictionary <string, object> configuration,
Guid organizationId,
Guid userId,
Guid providerDefinitionId)
{
var jobData = new FileSystemCrawlJobData(configuration);
if (jobData.StartingPoints == null)
{
return(null);
}
//long fileCount = 0;
//long folderCount = 0;
//foreach (var folder in jobData.StartingPoints)
//{
// fileCount += System.IO.Directory.GetFiles(folder.EntryPoint, "*.*", SearchOption.AllDirectories).Count();
// folderCount += System.IO.Directory.GetDirectories(folder.EntryPoint, "*.*", SearchOption.AllDirectories).Count();
//}
////Remove the current folder
//if (fileCount > 0)
// fileCount = fileCount - 1;
//jobData.ExpectedTaskCount = fileCount + folderCount;
var entityTypeStatistics = new ExpectedStatistics();
//entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.Directory, folderCount, 0));
//entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.File, fileCount, 0));
jobData.ExpectedData = 0;
jobData.ExpectedTime = new TimeSpan(0, 0, 0);
//jobData.ExpectedTaskCount = folderCount + fileCount;
jobData.ExpectedStatistics = entityTypeStatistics;
return(await Task.FromResult(jobData));
}
public override async Task <CrawlJobData> GetCrawlJobData(
ProviderUpdateContext context,
IDictionary <string, object> configuration,
Guid organizationId,
Guid userId,
Guid providerDefinitionId)
{
var dropBoxProviderId = Id;
if (_tokenStore == null)
{
return(new CrawlJobData {
Errors = new Dictionary <string, string> {
{ "error", "Please contact CluedIn support in the top menu to help you setup with Dropbox." }
}
});
}
var accounts = await _tokenStore.SelectAsync(context, account => account.ProviderId == dropBoxProviderId && account.OrganizationId == organizationId && account.UserId == userId && account.ConfigurationId == providerDefinitionId);
if (!accounts.Any())
{
accounts = await _tokenStore.SelectAsync(context, account => account.ProviderId == dropBoxProviderId && account.OrganizationId == organizationId && account.UserId == userId && account.ConfigurationId == null);
// You have more than one token that could authenticate with this
if (accounts.Count() > 1)
{
context.Log.Warn(() => "While binding a token to a provider, there was more than one token that would be viable and we chose the first.");
}
if (accounts.Count() == 1)
{
if (accounts.First().ConfigurationId == null)
{
accounts.First().ConfigurationId = providerDefinitionId;
_tokenStore.Update(context, accounts.First());
}
}
}
//Add Authentication for all Accounts here
var agentToken = accounts.Select(i => new AgentToken {
AccessToken = i.AccessToken, RefreshToken = i.RefreshToken, ExpiresIn = i.ExpiresIn
}).FirstOrDefault();
if (agentToken == null)
{
return(new CrawlJobData {
Errors = new Dictionary <string, string> {
{ "error", "Please contact CluedIn support in the top menu to help you setup with Dropbox." }
}
});
}
if (!configuration.ContainsKey("baseUri"))
{
configuration.Add("baseUri", "https://www.dropbox.com/home/");
}
if (!configuration.ContainsKey("Providers.DropBoxClientId"))
{
configuration.Add("Providers.DropBoxClientId", ConfigurationManager.AppSettings["Providers.DropBoxClientId"]);
}
if (!configuration.ContainsKey("Providers.DropBoxClientSecret"))
{
configuration.Add("Providers.DropBoxClientSecret", ConfigurationManager.AppSettings["Providers.DropBoxClientSecret"]);
}
if (!configuration.ContainsKey("Accounts"))
{
configuration.Add("Accounts", agentToken);
}
var jobData = new DropBoxCrawlJobData(configuration);
var client = _dropboxClientFactory.CreateNew(jobData);
if (jobData.LastestCursors == null || !jobData.LastestCursors.ContainsKey("Files"))
{
// HACK: Populate latest cursor if none is set
try
{
jobData.LastestCursors = jobData.LastestCursors ?? new Dictionary <string, string>();
var cursor = await client.ListFolderGetLatestCursorAsync(string.Empty, recursive : true, includeMediaInfo : false).ConfigureAwait(false);
if (cursor != null)
{
jobData.LastestCursors["Files"] = cursor.Cursor;
}
}
catch (Exception ex)
{
context.Log.Error(() => "Failed to get latest cursor Dropbox", ex);
}
}
try
{
var entityTypeStatistics = new ExpectedStatistics();
entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.File, 0, 0));
entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Person, 0, 0));
entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Infrastructure.Group, 0, 0));
entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Documents.Document, 0, 0));
entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Documents.Presentation, 0, 0));
entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Documents.Spreadsheet, 0, 0));
jobData.ExpectedStatistics = entityTypeStatistics;
jobData.ExpectedData = 0; // accountInformation.quota_info.quota;
jobData.ExpectedTime = new TimeSpan(0, 0, 0); //Maybe use the last run time from the agent statistics
}
catch (Exception ex)
{
context.Log.Error(() => "Failed to refresh the access token for Dropbox", ex);
context.Organization.Providers.SetProviderDefinitionAuthenticationError(context, providerDefinitionId, ex);
}
//No need to look for a refresh token as it will only stop working if the person uninstalls the app. https://www.dropbox.com/developers/reference/oauthguide
if (!jobData.IsAuthenticated)
{
return(new CrawlJobData {
Errors = new Dictionary <string, string> {
{ "error", "Please contact CluedIn support in the top menu to help you setup with Dropbox." }
}
});
}
if (ConfigurationManager.AppSettings.GetFlag("Feature.Webhooks.Enabled", false) && SupportsWebHooks)
{
var organizationProviderDataStore = context.Organization.DataStores.GetDataStore <ProviderDefinition>();
if (organizationProviderDataStore != null)
{
var webhookEnabled = organizationProviderDataStore.GetById(context, providerDefinitionId);
if (webhookEnabled != null)
{
if (webhookEnabled.WebHooks != null)
{
if (webhookEnabled.WebHooks.Value == false)
{
var webhookDefinition = WebHookDefinitionBuilder.CreateWebhookDefinition(webhookEnabled);
var results = await CreateWebHook(context, jobData, webhookDefinition, configuration);
foreach (var result in results)
{
webhookDefinition.Signature = result.Signature;
webhookDefinition.ExternalVersion = result.ExternalVersion;
webhookDefinition.ExternalId = result.ExternalId;
var webhookDefinitionDataStore = context.Organization.DataStores.GetDataStore <WebhookDefinition>();
if (webhookDefinitionDataStore != null)
{
webhookDefinitionDataStore.Insert(context, webhookDefinition);
}
}
}
}
else
{
var webhookDefinition = WebHookDefinitionBuilder.CreateWebhookDefinition(webhookEnabled);
var results = await CreateWebHook(context, jobData, webhookDefinition, configuration);
foreach (var result in results)
{
webhookDefinition.Signature = result.Signature;
webhookDefinition.ExternalVersion = result.ExternalVersion;
webhookDefinition.ExternalId = result.ExternalId;
var webhookDefinitionDataStore = context.Organization.DataStores.GetDataStore <WebhookDefinition>();
if (webhookDefinitionDataStore != null)
{
webhookDefinitionDataStore.Insert(context, webhookDefinition);
}
}
}
webhookEnabled.WebHooks = true;
organizationProviderDataStore.Update(context, webhookEnabled);
}
}
}
return(jobData);
}
