public CachedDirectory AddToCache(DirectoryEntry directory, IReadOnlyList <FileExplorerEntry> entries, bool directoriesOnly) { var cachedDirectory = new CachedDirectory(directory, directoriesOnly, entries); var cacheKey = NormalizePath(directory.Path); _localCache[cacheKey] = cachedDirectory; DirectoryEntriesUpdated?.Invoke(this, new DirectoryEntriesUpdatedEventArgs(directory.Path, cachedDirectory.Entries, directoriesOnly)); return(cachedDirectory); }
public void RunIndexOperations( ICloudProvider CloudProvider ) { try { // default CachedDirectory stores cache in local temp folder using ( CachedDirectory cachedDirectory = new CachedDirectory( CloudProvider ) ) { bool findexExists = IndexReader.IndexExists( cachedDirectory ); using ( StandardAnalyzer analyzer = new StandardAnalyzer( Version.LUCENE_CURRENT ) ) { using ( IndexWriter indexWriter = GetIndexWriter( cachedDirectory, analyzer ) ) { indexWriter.SetRAMBufferSizeMB( 10.0 ); //indexWriter.SetUseCompoundFile(false); //indexWriter.SetMaxMergeDocs(10000); //indexWriter.SetMergeFactor(100); Console.WriteLine( "Total existing docs is {0}", indexWriter.NumDocs() ); Console.WriteLine( "Creating docs:" ); int maxDocs = 10000; for ( int iDoc = 0; iDoc < maxDocs; iDoc++ ) { if ( iDoc % 1000 == 0 ) { Console.WriteLine( "- " + iDoc + "/" + maxDocs ); } Document doc = new Document(); doc.Add( new Field( "id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO ) ); doc.Add( new Field( "Title", GeneratePhrase( 10 ), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO ) ); doc.Add( new Field( "Body", GeneratePhrase( 40 ), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO ) ); indexWriter.AddDocument( doc ); } Console.WriteLine( "Total docs is {0}", indexWriter.NumDocs() ); } } using ( new AutoStopWatch( "Creating searcher" ) ) { using ( IndexSearcher searcher = new IndexSearcher( cachedDirectory ) ) { SearchForPhrase( searcher, "dog" ); SearchForPhrase( searcher, _random.Next( 32768 ).ToString() ); SearchForPhrase( searcher, _random.Next( 32768 ).ToString() ); } } } } catch ( Exception ex ) { Console.WriteLine( "Error: " + ex.Message ); Console.WriteLine( ex.StackTrace ); } Console.WriteLine( "Push return to exit" ); Console.Read(); }
public void RunIndexOperations(ICloudProvider CloudProvider) { try { // default CachedDirectory stores cache in local temp folder using (CachedDirectory cachedDirectory = new CachedDirectory(CloudProvider)) { bool findexExists = IndexReader.IndexExists(cachedDirectory); using (StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT)) { using (IndexWriter indexWriter = GetIndexWriter(cachedDirectory, analyzer)) { indexWriter.SetRAMBufferSizeMB(10.0); //indexWriter.SetUseCompoundFile(false); //indexWriter.SetMaxMergeDocs(10000); //indexWriter.SetMergeFactor(100); Console.WriteLine("Total existing docs is {0}", indexWriter.NumDocs()); Console.WriteLine("Creating docs:"); int maxDocs = 10000; for (int iDoc = 0; iDoc < maxDocs; iDoc++) { if (iDoc % 1000 == 0) { Console.WriteLine("- " + iDoc + "/" + maxDocs); } Document doc = new Document(); doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); indexWriter.AddDocument(doc); } Console.WriteLine("Total docs is {0}", indexWriter.NumDocs()); } } using (new AutoStopWatch("Creating searcher")) { using (IndexSearcher searcher = new IndexSearcher(cachedDirectory)) { SearchForPhrase(searcher, "dog"); SearchForPhrase(searcher, _random.Next(32768).ToString()); SearchForPhrase(searcher, _random.Next(32768).ToString()); } } } } catch (Exception ex) { Console.WriteLine("Error: " + ex.Message); Console.WriteLine(ex.StackTrace); } Console.WriteLine("Push return to exit"); Console.Read(); }
private IndexWriter GetIndexWriter(CachedDirectory cachedDirectory, StandardAnalyzer analyzer) { IndexWriter indexWriter = null; while (indexWriter == null) { try { indexWriter = new IndexWriter(cachedDirectory, analyzer, !IndexReader.IndexExists(cachedDirectory), new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); } catch (LockObtainFailedException) { Console.WriteLine("Lock is taken, Hit 'Y' to clear the lock, or anything else to try again"); if ((Console.ReadLine() ?? "").ToLower().Trim() == "y") { cachedDirectory.ClearLock("write.lock"); } } } Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index"); return(indexWriter); }
public async Task <RootElementsDto> GetRoot() { var dto = await FileExplorerResource.GetRoot(_restClient); dto.ComputerDirectory.Migrate(); foreach (var rootDirectory in dto.RootDirectories) { rootDirectory.Migrate(); } foreach (var computerDirectoryEntry in dto.ComputerDirectoryEntries) { computerDirectoryEntry.Migrate(dto.ComputerDirectory); } _computerDirectory = AddToCache(dto.ComputerDirectory, dto.ComputerDirectoryEntries, false); return(dto); }
public IList <string> AddDirectoryFilesToCache(String directoryPath, bool recursive) { lock (cache) { try { String fileSpec; if (recursive) { fileSpec = String.Format("{0}/...", directoryPath); } else { fileSpec = String.Format("{0}/*", directoryPath); } var key = GetKey(directoryPath); if (CachedDirectories == null) { CachedDirectories = new Dictionary <CacheKey, CachedDirectory>(); } else if ((CachedDirectories.ContainsKey(key)) && (CachedDirectories[key] > DateTime.Now - TimeSpan.FromSeconds(25))) { return(null); } //make sure the added path is under the client's root directory if (_scm != null && !string.IsNullOrEmpty(_scm.Connection.WorkspaceRoot)) { var wsRoot = GetKey(_scm.Connection.WorkspaceRoot); logger.Trace("wsRoot: {0}", wsRoot); if (!key.ToString().StartsWith(wsRoot.ToString(), StringComparison.OrdinalIgnoreCase)) { CachedDirectories[key] = new CachedDirectory(directoryPath); logger.Trace("AddDir: {0}", key); return(null); } } // add/update cache folder to the hash of folders that have been cached CachedDirectories[key] = new CachedDirectory(directoryPath); if (recursive) { // non recursive directory walk to add/update all the subdirectories // to the hash table of directories that have been cached Queue <string> subdirs = new Queue <string>(); subdirs.Enqueue(directoryPath); while (subdirs.Count > 0) { string subdir = subdirs.Dequeue(); foreach (string sd in System.IO.Directory.GetDirectories(subdir)) { // add to queue for so it'll be walked for subdirectories subdirs.Enqueue(sd); // add/update this subdirectory to the hash of cached directories CachedDirectories[GetKey(sd)] = new CachedDirectory(sd); } } } P4.Options opts = new P4.Options(); opts["-Olhp"] = null; opts["-m"] = "100000"; logger.Trace("WorkspaceRoot: {0}", _scm.Connection.WorkspaceRoot); IList <P4.FileMetaData> files = _repository.GetFileMetaData(opts, P4.FileSpec.LocalSpec(fileSpec)); IList <string> updated = null; if (files != null) { updated = UpdateFileMetaData(files); } else { P4.P4CommandResult results = _repository.Connection.LastResults; if ((results != null) && (results.ErrorList != null) && (results.ErrorList.Count > 0) && ((results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDb_NotUnderRoot) || (results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDb_NotUnderClient) || (results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDm_IntegMovedUnmapped) || (results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDm_ExVIEW) || (results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDm_ExVIEW2))) { if (CachedDirectories == null) { CachedDirectories = new Dictionary <CacheKey, CachedDirectory>(); } CachedDirectories[key] = new CachedDirectory(directoryPath); } } return(updated); } catch (Exception ex) { // If the error is because the repository is now null, it means // the connection was closed in the middle of a command, so ignore it. if (_repository != null) { _scm.ShowException(ex, false, true, (Preferences.LocalSettings.GetBool("Log_reporting", false) == true)); } if (CachedDirectories == null) { CachedDirectories = new Dictionary <CacheKey, CachedDirectory>(); } CachedDirectories[GetKey(directoryPath)] = new CachedDirectory(directoryPath, DateTime.MinValue); return(null); } } }
private bool TryGetCachedDirectory(string path, out CachedDirectory cachedDirectory) { return(_localCache.TryGetValue(NormalizePath(path), out cachedDirectory)); }
public async Task <PathContent> FetchPath(string path, bool ignoreEntriesCache, bool ignorePathCache, CancellationToken token) { if (PathHelper.ContainsEnvironmentVariables(path)) { path = await FileSystemResource.ExpandEnvironmentVariables(path, _restClient); } path = NormalizePath(path); var request = new PathTreeRequestDto { Path = path, RequestedDirectories = new List <int>() }; if (ignoreEntriesCache || !TryGetCachedDirectory(path, out var cachedDirectory) || cachedDirectory.DirectoriesOnly) { request.RequestEntries = true; } var parts = PathHelper.GetPathDirectories(path).ToList(); for (var i = 0; i < parts.Count; i++) { var partPath = parts[i]; if (ignorePathCache || !TryGetCachedDirectory(partPath, out _)) { request.RequestedDirectories.Add(i); } } PathTreeResponseDto queryResponse = null; if (request.RequestEntries || request.RequestedDirectories.Any()) { queryResponse = await FileExplorerResource.GetPathTree(request, DirectoryHelper.IsComputerDirectory(path), token, _restClient); } parts.Add(path); DirectoryEntry directory = null; var pathDirectories = new List <DirectoryEntry>(); IReadOnlyList <FileExplorerEntry> directoryEntries = null; for (var i = 0; i < parts.Count; i++) { var directoryPath = parts[i]; if (directory == null) { if (TryGetCachedDirectory(directoryPath, out cachedDirectory)) { directory = cachedDirectory.Directory; } else { directory = _computerDirectory.Entries.OfType <DirectoryEntry>() .FirstOrDefault(x => string.Equals(x.Path, directoryPath, PathStringComparison)); } } if (directory == null) //Special folders like trash can etc. { directory = await FileSystemResource.GetDirectoryEntry(directoryPath, _restClient); directory.Migrate(); } if (request.RequestEntries && i == parts.Count - 1) { // ReSharper disable once PossibleNullReferenceException directoryEntries = queryResponse.Entries; foreach (var entry in directoryEntries) { entry.Migrate(directory); } AddToCache(directory, directoryEntries, false); } else if (queryResponse?.Directories != null && queryResponse.Directories.TryGetValue(i, out var subDirectories)) { directoryEntries = subDirectories; foreach (var directoryEntry in directoryEntries) { directoryEntry.Migrate(directory); } AddToCache(directory, directoryEntries, true); } else { directoryEntries = _localCache[NormalizePath(directoryPath)].Entries.ToList(); } pathDirectories.Add(directory); //if we are not at the very last part (so there is a child directory) if (i < parts.Count - 1) { var normalizedPath = NormalizePath(parts[i + 1]); //path of the child directory var nextDirectory = directoryEntries.OfType <DirectoryEntry>().FirstOrDefault(x => string.Equals(normalizedPath, NormalizePath(x.Path), PathStringComparison)); if (nextDirectory == null) { nextDirectory = new DirectoryEntry { HasSubFolder = true, Parent = directory, Name = GetDirectoryName(normalizedPath) }; directoryEntries = directoryEntries.Concat(new[] { nextDirectory }).ToList(); //update cache var key = NormalizePath(directoryPath); var oldCache = _localCache[key]; _localCache[key] = new CachedDirectory(oldCache.Directory, oldCache.DirectoriesOnly, directoryEntries); } directory = nextDirectory; } } return(new PathContent(directory, directoryEntries, pathDirectories)); }
private IndexWriter GetIndexWriter( CachedDirectory cachedDirectory, StandardAnalyzer analyzer ) { IndexWriter indexWriter = null; while ( indexWriter == null ) { try { indexWriter = new IndexWriter( cachedDirectory, analyzer, !IndexReader.IndexExists( cachedDirectory ), new IndexWriter.MaxFieldLength( IndexWriter.DEFAULT_MAX_FIELD_LENGTH ) ); } catch ( LockObtainFailedException ) { Console.WriteLine( "Lock is taken, Hit 'Y' to clear the lock, or anything else to try again" ); if ( ( Console.ReadLine() ?? "" ).ToLower().Trim() == "y" ) { cachedDirectory.ClearLock( "write.lock" ); } } } Console.WriteLine( "IndexWriter lock obtained, this process has exclusive write access to index" ); return indexWriter; }