public CachedDirectory AddToCache(DirectoryEntry directory, IReadOnlyList <FileExplorerEntry> entries, bool directoriesOnly)
{
var cachedDirectory = new CachedDirectory(directory, directoriesOnly, entries);
var cacheKey = NormalizePath(directory.Path);
_localCache[cacheKey] = cachedDirectory;
DirectoryEntriesUpdated?.Invoke(this,
new DirectoryEntriesUpdatedEventArgs(directory.Path, cachedDirectory.Entries, directoriesOnly));
return(cachedDirectory);
}
public void RunIndexOperations( ICloudProvider CloudProvider )
{
try {
// default CachedDirectory stores cache in local temp folder
using ( CachedDirectory cachedDirectory = new CachedDirectory( CloudProvider ) ) {
bool findexExists = IndexReader.IndexExists( cachedDirectory );
using ( StandardAnalyzer analyzer = new StandardAnalyzer( Version.LUCENE_CURRENT ) ) {
using ( IndexWriter indexWriter = GetIndexWriter( cachedDirectory, analyzer ) ) {
indexWriter.SetRAMBufferSizeMB( 10.0 );
//indexWriter.SetUseCompoundFile(false);
//indexWriter.SetMaxMergeDocs(10000);
//indexWriter.SetMergeFactor(100);
Console.WriteLine( "Total existing docs is {0}", indexWriter.NumDocs() );
Console.WriteLine( "Creating docs:" );
int maxDocs = 10000;
for ( int iDoc = 0; iDoc < maxDocs; iDoc++ ) {
if ( iDoc % 1000 == 0 ) {
Console.WriteLine( "- " + iDoc + "/" + maxDocs );
}
Document doc = new Document();
doc.Add( new Field( "id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO ) );
doc.Add( new Field( "Title", GeneratePhrase( 10 ), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO ) );
doc.Add( new Field( "Body", GeneratePhrase( 40 ), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO ) );
indexWriter.AddDocument( doc );
}
Console.WriteLine( "Total docs is {0}", indexWriter.NumDocs() );
}
}
using ( new AutoStopWatch( "Creating searcher" ) ) {
using ( IndexSearcher searcher = new IndexSearcher( cachedDirectory ) ) {
SearchForPhrase( searcher, "dog" );
SearchForPhrase( searcher, _random.Next( 32768 ).ToString() );
SearchForPhrase( searcher, _random.Next( 32768 ).ToString() );
}
}
}
} catch ( Exception ex ) {
Console.WriteLine( "Error: " + ex.Message );
Console.WriteLine( ex.StackTrace );
}
Console.WriteLine( "Push return to exit" );
Console.Read();
}
public void RunIndexOperations(ICloudProvider CloudProvider)
{
try {
// default CachedDirectory stores cache in local temp folder
using (CachedDirectory cachedDirectory = new CachedDirectory(CloudProvider)) {
bool findexExists = IndexReader.IndexExists(cachedDirectory);
using (StandardAnalyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT)) {
using (IndexWriter indexWriter = GetIndexWriter(cachedDirectory, analyzer)) {
indexWriter.SetRAMBufferSizeMB(10.0);
//indexWriter.SetUseCompoundFile(false);
//indexWriter.SetMaxMergeDocs(10000);
//indexWriter.SetMergeFactor(100);
Console.WriteLine("Total existing docs is {0}", indexWriter.NumDocs());
Console.WriteLine("Creating docs:");
int maxDocs = 10000;
for (int iDoc = 0; iDoc < maxDocs; iDoc++)
{
if (iDoc % 1000 == 0)
{
Console.WriteLine("- " + iDoc + "/" + maxDocs);
}
Document doc = new Document();
doc.Add(new Field("id", DateTime.Now.ToFileTimeUtc().ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
doc.Add(new Field("Title", GeneratePhrase(10), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
doc.Add(new Field("Body", GeneratePhrase(40), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO));
indexWriter.AddDocument(doc);
}
Console.WriteLine("Total docs is {0}", indexWriter.NumDocs());
}
}
using (new AutoStopWatch("Creating searcher")) {
using (IndexSearcher searcher = new IndexSearcher(cachedDirectory)) {
SearchForPhrase(searcher, "dog");
SearchForPhrase(searcher, _random.Next(32768).ToString());
SearchForPhrase(searcher, _random.Next(32768).ToString());
}
}
}
} catch (Exception ex) {
Console.WriteLine("Error: " + ex.Message);
Console.WriteLine(ex.StackTrace);
}
Console.WriteLine("Push return to exit");
Console.Read();
}
private IndexWriter GetIndexWriter(CachedDirectory cachedDirectory, StandardAnalyzer analyzer)
{
IndexWriter indexWriter = null;
while (indexWriter == null)
{
try {
indexWriter = new IndexWriter(cachedDirectory, analyzer, !IndexReader.IndexExists(cachedDirectory), new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH));
} catch (LockObtainFailedException) {
Console.WriteLine("Lock is taken, Hit 'Y' to clear the lock, or anything else to try again");
if ((Console.ReadLine() ?? "").ToLower().Trim() == "y")
{
cachedDirectory.ClearLock("write.lock");
}
}
}
Console.WriteLine("IndexWriter lock obtained, this process has exclusive write access to index");
return(indexWriter);
}
public async Task <RootElementsDto> GetRoot()
{
var dto = await FileExplorerResource.GetRoot(_restClient);
dto.ComputerDirectory.Migrate();
foreach (var rootDirectory in dto.RootDirectories)
{
rootDirectory.Migrate();
}
foreach (var computerDirectoryEntry in dto.ComputerDirectoryEntries)
{
computerDirectoryEntry.Migrate(dto.ComputerDirectory);
}
_computerDirectory = AddToCache(dto.ComputerDirectory, dto.ComputerDirectoryEntries, false);
return(dto);
}
public IList <string> AddDirectoryFilesToCache(String directoryPath, bool recursive)
{
lock (cache)
{
try
{
String fileSpec;
if (recursive)
{
fileSpec = String.Format("{0}/...", directoryPath);
}
else
{
fileSpec = String.Format("{0}/*", directoryPath);
}
var key = GetKey(directoryPath);
if (CachedDirectories == null)
{
CachedDirectories = new Dictionary <CacheKey, CachedDirectory>();
}
else if ((CachedDirectories.ContainsKey(key)) &&
(CachedDirectories[key] > DateTime.Now - TimeSpan.FromSeconds(25)))
{
return(null);
}
//make sure the added path is under the client's root directory
if (_scm != null && !string.IsNullOrEmpty(_scm.Connection.WorkspaceRoot))
{
var wsRoot = GetKey(_scm.Connection.WorkspaceRoot);
logger.Trace("wsRoot: {0}", wsRoot);
if (!key.ToString().StartsWith(wsRoot.ToString(), StringComparison.OrdinalIgnoreCase))
{
CachedDirectories[key] = new CachedDirectory(directoryPath);
logger.Trace("AddDir: {0}", key);
return(null);
}
}
// add/update cache folder to the hash of folders that have been cached
CachedDirectories[key] = new CachedDirectory(directoryPath);
if (recursive)
{
// non recursive directory walk to add/update all the subdirectories
// to the hash table of directories that have been cached
Queue <string> subdirs = new Queue <string>();
subdirs.Enqueue(directoryPath);
while (subdirs.Count > 0)
{
string subdir = subdirs.Dequeue();
foreach (string sd in System.IO.Directory.GetDirectories(subdir))
{
// add to queue for so it'll be walked for subdirectories
subdirs.Enqueue(sd);
// add/update this subdirectory to the hash of cached directories
CachedDirectories[GetKey(sd)] = new CachedDirectory(sd);
}
}
}
P4.Options opts = new P4.Options();
opts["-Olhp"] = null;
opts["-m"] = "100000";
logger.Trace("WorkspaceRoot: {0}", _scm.Connection.WorkspaceRoot);
IList <P4.FileMetaData> files =
_repository.GetFileMetaData(opts, P4.FileSpec.LocalSpec(fileSpec));
IList <string> updated = null;
if (files != null)
{
updated = UpdateFileMetaData(files);
}
else
{
P4.P4CommandResult results = _repository.Connection.LastResults;
if ((results != null) && (results.ErrorList != null) && (results.ErrorList.Count > 0) &&
((results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDb_NotUnderRoot) ||
(results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDb_NotUnderClient) ||
(results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDm_IntegMovedUnmapped) ||
(results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDm_ExVIEW) ||
(results.ErrorList[0].ErrorCode == P4.P4ClientError.MsgDm_ExVIEW2)))
{
if (CachedDirectories == null)
{
CachedDirectories = new Dictionary <CacheKey, CachedDirectory>();
}
CachedDirectories[key] = new CachedDirectory(directoryPath);
}
}
return(updated);
}
catch (Exception ex)
{
// If the error is because the repository is now null, it means
// the connection was closed in the middle of a command, so ignore it.
if (_repository != null)
{
_scm.ShowException(ex, false, true, (Preferences.LocalSettings.GetBool("Log_reporting", false) == true));
}
if (CachedDirectories == null)
{
CachedDirectories = new Dictionary <CacheKey, CachedDirectory>();
}
CachedDirectories[GetKey(directoryPath)] = new CachedDirectory(directoryPath, DateTime.MinValue);
return(null);
}
}
}
private bool TryGetCachedDirectory(string path, out CachedDirectory cachedDirectory)
{
return(_localCache.TryGetValue(NormalizePath(path), out cachedDirectory));
}
public async Task <PathContent> FetchPath(string path, bool ignoreEntriesCache, bool ignorePathCache,
CancellationToken token)
{
if (PathHelper.ContainsEnvironmentVariables(path))
{
path = await FileSystemResource.ExpandEnvironmentVariables(path, _restClient);
}
path = NormalizePath(path);
var request = new PathTreeRequestDto {
Path = path, RequestedDirectories = new List <int>()
};
if (ignoreEntriesCache || !TryGetCachedDirectory(path, out var cachedDirectory) ||
cachedDirectory.DirectoriesOnly)
{
request.RequestEntries = true;
}
var parts = PathHelper.GetPathDirectories(path).ToList();
for (var i = 0; i < parts.Count; i++)
{
var partPath = parts[i];
if (ignorePathCache || !TryGetCachedDirectory(partPath, out _))
{
request.RequestedDirectories.Add(i);
}
}
PathTreeResponseDto queryResponse = null;
if (request.RequestEntries || request.RequestedDirectories.Any())
{
queryResponse = await FileExplorerResource.GetPathTree(request,
DirectoryHelper.IsComputerDirectory(path), token, _restClient);
}
parts.Add(path);
DirectoryEntry directory = null;
var pathDirectories = new List <DirectoryEntry>();
IReadOnlyList <FileExplorerEntry> directoryEntries = null;
for (var i = 0; i < parts.Count; i++)
{
var directoryPath = parts[i];
if (directory == null)
{
if (TryGetCachedDirectory(directoryPath, out cachedDirectory))
{
directory = cachedDirectory.Directory;
}
else
{
directory = _computerDirectory.Entries.OfType <DirectoryEntry>()
.FirstOrDefault(x => string.Equals(x.Path, directoryPath, PathStringComparison));
}
}
if (directory == null) //Special folders like trash can etc.
{
directory = await FileSystemResource.GetDirectoryEntry(directoryPath, _restClient);
directory.Migrate();
}
if (request.RequestEntries && i == parts.Count - 1)
{
// ReSharper disable once PossibleNullReferenceException
directoryEntries = queryResponse.Entries;
foreach (var entry in directoryEntries)
{
entry.Migrate(directory);
}
AddToCache(directory, directoryEntries, false);
}
else if (queryResponse?.Directories != null && queryResponse.Directories.TryGetValue(i, out var subDirectories))
{
directoryEntries = subDirectories;
foreach (var directoryEntry in directoryEntries)
{
directoryEntry.Migrate(directory);
}
AddToCache(directory, directoryEntries, true);
}
else
{
directoryEntries = _localCache[NormalizePath(directoryPath)].Entries.ToList();
}
pathDirectories.Add(directory);
//if we are not at the very last part (so there is a child directory)
if (i < parts.Count - 1)
{
var normalizedPath = NormalizePath(parts[i + 1]); //path of the child directory
var nextDirectory = directoryEntries.OfType <DirectoryEntry>().FirstOrDefault(x =>
string.Equals(normalizedPath, NormalizePath(x.Path), PathStringComparison));
if (nextDirectory == null)
{
nextDirectory = new DirectoryEntry
{
HasSubFolder = true,
Parent = directory,
Name = GetDirectoryName(normalizedPath)
};
directoryEntries = directoryEntries.Concat(new[] { nextDirectory }).ToList();
//update cache
var key = NormalizePath(directoryPath);
var oldCache = _localCache[key];
_localCache[key] = new CachedDirectory(oldCache.Directory, oldCache.DirectoriesOnly,
directoryEntries);
}
directory = nextDirectory;
}
}
return(new PathContent(directory, directoryEntries, pathDirectories));
}
private IndexWriter GetIndexWriter( CachedDirectory cachedDirectory, StandardAnalyzer analyzer )
{
IndexWriter indexWriter = null;
while ( indexWriter == null ) {
try {
indexWriter = new IndexWriter( cachedDirectory, analyzer, !IndexReader.IndexExists( cachedDirectory ), new IndexWriter.MaxFieldLength( IndexWriter.DEFAULT_MAX_FIELD_LENGTH ) );
} catch ( LockObtainFailedException ) {
Console.WriteLine( "Lock is taken, Hit 'Y' to clear the lock, or anything else to try again" );
if ( ( Console.ReadLine() ?? "" ).ToLower().Trim() == "y" ) {
cachedDirectory.ClearLock( "write.lock" );
}
}
}
Console.WriteLine( "IndexWriter lock obtained, this process has exclusive write access to index" );
return indexWriter;
}
