public FileSystemItem(T item, FileSystemCrawlJobData fylesystemcrawlJobData) : this( item, fylesystemcrawlJobData.FetchOwners ? FileSystemPrincipal.GetOwner(item, fylesystemcrawlJobData.LookupCache) : null) { }
protected IEnumerable <T> FilterFileSystemInfos <T>(IEnumerable <T> items, FileSystemCrawlJobData filesystemCrawlJobData) where T : FileSystemInfo { return(items.Where( c => FileSystemInfoFilter.ShouldIndex(c) && ( filesystemCrawlJobData.LastCrawlFinishTime == DateTimeOffset.MinValue || c.LastWriteTimeUtc > filesystemCrawlJobData.LastCrawlFinishTime.UtcDateTime || c.CreationTimeUtc > filesystemCrawlJobData.LastCrawlFinishTime.UtcDateTime ) )); }
public FileSystemClient(ILogger log, FileSystemCrawlJobData filesystemCrawlJobData, IRestClient client) // TODO: pass on any extra dependencies { if (filesystemCrawlJobData == null) { throw new ArgumentNullException(nameof(filesystemCrawlJobData)); } if (client == null) { throw new ArgumentNullException(nameof(client)); } _log = log ?? throw new ArgumentNullException(nameof(log)); _filesystemCrawlJobData = filesystemCrawlJobData; // TODO use info from filesystemCrawlJobData to instantiate the connection client.BaseUrl = new Uri(s_baseUri); // client.AddDefaultParameter("api_key", filesystemCrawlJobData.ApiKey, ParameterType.QueryString); }
/********************************************************************************************************** * METHODS **********************************************************************************************************/ public override async Task <CrawlJobData> GetCrawlJobData( ProviderUpdateContext context, IDictionary <string, object> configuration, Guid organizationId, Guid userId, Guid providerDefinitionId) { var jobData = new FileSystemCrawlJobData(configuration); if (jobData.StartingPoints == null) { return(null); } //long fileCount = 0; //long folderCount = 0; //foreach (var folder in jobData.StartingPoints) //{ // fileCount += System.IO.Directory.GetFiles(folder.EntryPoint, "*.*", SearchOption.AllDirectories).Count(); // folderCount += System.IO.Directory.GetDirectories(folder.EntryPoint, "*.*", SearchOption.AllDirectories).Count(); //} ////Remove the current folder //if (fileCount > 0) // fileCount = fileCount - 1; //jobData.ExpectedTaskCount = fileCount + folderCount; var entityTypeStatistics = new ExpectedStatistics(); //entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.Directory, folderCount, 0)); //entityTypeStatistics.EntityTypeStatistics.Add(new EntityTypeStatistics(EntityType.Files.File, fileCount, 0)); jobData.ExpectedData = 0; jobData.ExpectedTime = new TimeSpan(0, 0, 0); //jobData.ExpectedTaskCount = folderCount + fileCount; jobData.ExpectedStatistics = entityTypeStatistics; return(await Task.FromResult(jobData)); }
public FileInfoClueProducer(IClueFactory factory, FileSystemCrawlJobData crawlJobData) { _factory = factory ?? throw new ArgumentNullException(nameof(factory)); _crawlJobData = crawlJobData ?? throw new ArgumentNullException(nameof(crawlJobData)); }
protected IEnumerable <object> CrawlDirectory(DirectoryInfo info, CrawlOptions options, FileSystemCrawlJobData filesystemcrawlJobData) { var directory = new FileSystemItem <DirectoryInfo>(info, filesystemcrawlJobData); yield return(directory.Owner); yield return(directory); foreach (var fileInfo in info.GetFiles()) { var file = new FileSystemItem <FileInfo>(fileInfo, filesystemcrawlJobData); yield return(file.Owner); yield return(file); } if (options == CrawlOptions.Recursive) { foreach (var subDirectory in FilterFileSystemInfos(info.GetDirectories().OrderBy(d => Guid.NewGuid()), filesystemcrawlJobData)) { CrawlDirectory(subDirectory, options, filesystemcrawlJobData); } } }