public FileSystemItem(T item, FileSystemCrawlJobData fylesystemcrawlJobData)
: this(
item,
fylesystemcrawlJobData.FetchOwners
? FileSystemPrincipal.GetOwner(item, fylesystemcrawlJobData.LookupCache)
: null)
{
}
protected IEnumerable <T> FilterFileSystemInfos <T>(IEnumerable <T> items, FileSystemCrawlJobData filesystemCrawlJobData)
where T : FileSystemInfo
{
return(items.Where(
c =>
FileSystemInfoFilter.ShouldIndex(c) &&
(
filesystemCrawlJobData.LastCrawlFinishTime == DateTimeOffset.MinValue ||
c.LastWriteTimeUtc > filesystemCrawlJobData.LastCrawlFinishTime.UtcDateTime ||
c.CreationTimeUtc > filesystemCrawlJobData.LastCrawlFinishTime.UtcDateTime
)
));
}
public FileSystemClient(ILogger log, FileSystemCrawlJobData filesystemCrawlJobData, IRestClient client) // TODO: pass on any extra dependencies
{
if (filesystemCrawlJobData == null)
{
throw new ArgumentNullException(nameof(filesystemCrawlJobData));
}
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
_log = log ?? throw new ArgumentNullException(nameof(log));
_filesystemCrawlJobData = filesystemCrawlJobData;
// TODO use info from filesystemCrawlJobData to instantiate the connection
client.BaseUrl = new Uri(s_baseUri);
// client.AddDefaultParameter("api_key", filesystemCrawlJobData.ApiKey, ParameterType.QueryString);
}
/**********************************************************************************************************
* METHODS
**********************************************************************************************************/
public override async Task <CrawlJobData> GetCrawlJobData(
ProviderUpdateContext context,
IDictionary <string, object> configuration,
Guid organizationId,
Guid userId,
Guid providerDefinitionId)
{
var jobData = new FileSystemCrawlJobData(configuration);
if (jobData.StartingPoints == null)
{
return(null);
}
//long fileCount = 0;
//long folderCount = 0;
//foreach (var folder in jobData.StartingPoints)
//{
// fileCount += System.IO.Directory.GetFiles(folder.EntryPoint, "*.*", SearchOption.AllDirectories).Count();
// folderCount += System.IO.Directory.GetDirectories(folder.EntryPoint, "*.*", SearchOption.AllDirectories).Count();
//}
////Remove the current folder
//if (fileCount > 0)
// fileCount = fileCount - 1;
//jobData.ExpectedTaskCount = fileCount + folderCount;
var entityTypeStatistics = new ExpectedStatistics();
//entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.Directory, folderCount, 0));
//entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.File, fileCount, 0));
jobData.ExpectedData = 0;
jobData.ExpectedTime = new TimeSpan(0, 0, 0);
//jobData.ExpectedTaskCount = folderCount + fileCount;
jobData.ExpectedStatistics = entityTypeStatistics;
return(await Task.FromResult(jobData));
}
public FileInfoClueProducer(IClueFactory factory, FileSystemCrawlJobData crawlJobData)
{
_factory = factory ?? throw new ArgumentNullException(nameof(factory));
_crawlJobData = crawlJobData ?? throw new ArgumentNullException(nameof(crawlJobData));
}
protected IEnumerable <object> CrawlDirectory(DirectoryInfo info, CrawlOptions options, FileSystemCrawlJobData filesystemcrawlJobData)
{
var directory = new FileSystemItem <DirectoryInfo>(info, filesystemcrawlJobData);
yield return(directory.Owner);
yield return(directory);
foreach (var fileInfo in info.GetFiles())
{
var file = new FileSystemItem <FileInfo>(fileInfo, filesystemcrawlJobData);
yield return(file.Owner);
yield return(file);
}
if (options == CrawlOptions.Recursive)
{
foreach (var subDirectory in FilterFileSystemInfos(info.GetDirectories().OrderBy(d => Guid.NewGuid()), filesystemcrawlJobData))
{
CrawlDirectory(subDirectory, options, filesystemcrawlJobData);
}
}
}
