public Task SearchAsync(string searchPattern, FileSearchMode mode, int count, IFileCollection files, CancellationToken cancellationToken)
{
if (String.IsNullOrEmpty(searchPattern))
{
files.Clear();
foreach (string filePath in pinStateService.GetList())
files.Add(Path.GetFileNameWithoutExtension(filePath), filePath, true);
if (lastCancellation != null)
{
lastCancellation.Cancel();
lastCancellation = null;
}
return Task.FromResult(true);
}
if (cancellationToken.IsCancellationRequested)
return Async.CompletedTask;
lastCancellation = new CancellationTokenSource();
cancellationToken.Register(() => lastCancellation.Cancel());
Task result = innerService.SearchAsync(searchPattern, mode, count, files, lastCancellation.Token);
return result;
}
public BtdbTimeTests(bool inMemory, bool memoryMapped)
{
_inMemory = inMemory;
_memoryMapped = memoryMapped;
_fileCollection = CreateTestFileCollection();
}
internal SerializedFile(IFileCollection collection, IAssemblyManager manager, SerializedFileScheme scheme)
{
if (scheme == null)
{
throw new ArgumentNullException(nameof(scheme));
}
}
static ObjectDB CreateDb(IFileCollection fc)
{
var lowDb = new KeyValueDB(fc);
var db = new ObjectDB();
db.Open(lowDb, true);
return db;
}
public void Dispose()
{
_keyValueDb?.Dispose();
_keyValueDb = null;
_memoryFileCollection?.Dispose();
_memoryFileCollection = null;
}
public SerializedFile ReadFile(IFileCollection collection, IAssemblyManager manager)
{
SerializedFile file = new SerializedFile(collection, manager, this);
m_stream.Dispose();
return(file);
}
IFileCollection CreateTestFileCollection()
{
if (_fastInMemory)
{
return(null);
}
if (_inMemory)
{
_fileCollection = new InMemoryFileCollection();
return(_fileCollection);
}
const string dbfilename = "data";
if (Directory.Exists(dbfilename))
{
Directory.Delete(dbfilename, true);
}
Directory.CreateDirectory(dbfilename);
if (_memoryMapped)
{
return(new OnDiskMemoryMappedFileCollection(dbfilename));
}
return(new OnDiskFileCollection(dbfilename));
}
IKeyValueDB CreateKeyValueDB(IFileCollection fileCollection, ICompressionStrategy compressionStrategy = null)
{
if (fileCollection == null)
{
switch (_kvType)
{
case KVType.Managed:
return(new InMemoryKeyValueDB());
default:
throw new NotImplementedException();
}
}
switch (_kvType)
{
case KVType.Managed:
if (compressionStrategy == null)
{
return(new KeyValueDB(fileCollection));
}
return(new KeyValueDB(fileCollection, compressionStrategy));
case KVType.BTree:
if (compressionStrategy == null)
{
return(new BTreeKeyValueDB(fileCollection));
}
return(new BTreeKeyValueDB(fileCollection, compressionStrategy));
default:
throw new NotImplementedException();
}
}
public ArchiveFile ReadFile(IFileCollection collection, IAssemblyManager manager)
{
ArchiveFile archive = new ArchiveFile(collection, this);
archive.AddFile(WebScheme, collection, manager);
return(archive);
}
internal SerializedFile(IFileCollection collection, IAssemblyManager manager, SerializedFileScheme scheme)
{
if (collection == null)
{
throw new ArgumentNullException(nameof(collection));
}
if (manager == null)
{
throw new ArgumentNullException(nameof(manager));
}
if (scheme == null)
{
throw new ArgumentNullException(nameof(scheme));
}
Collection = collection;
AssemblyManager = manager;
FilePath = scheme.FilePath;
NameOrigin = scheme.Name;
Name = FilenameUtils.FixFileIdentifier(scheme.Name);
Flags = scheme.Flags;
Header = scheme.Header;
Metadata = scheme.Metadata;
}
void UploadNewFiles(IFileCollection fileUploadViewModel)
{
//iterate through each fileViewModel in collection
foreach (IFileWrapper file in fileUploadViewModel.Files)
{
UploadNewFile(file);
}
}
static ObjectDB CreateDb(IFileCollection fc)
{
var lowDb = new KeyValueDB(fc);
var db = new ObjectDB();
db.Open(lowDb, true);
return(db);
}
internal ArchiveFile(IFileCollection collection, ArchiveFileScheme scheme)
{
if (scheme == null)
{
throw new ArgumentNullException(nameof(scheme));
}
Header = scheme.Header;
}
internal BundleFile(IFileCollection collection, BundleFileScheme scheme)
{
if (scheme == null)
{
throw new ArgumentNullException(nameof(scheme));
}
Header = scheme.Header;
Metadata = scheme.Metadata;
}
public BundleFile ReadFile(IFileCollection collection, IAssemblyManager manager)
{
BundleFile bundle = new BundleFile(collection, this);
foreach (FileScheme scheme in Schemes)
{
bundle.AddFile(scheme, collection, manager);
}
return(bundle);
}
public WebFile ReadFile(IFileCollection collection, IAssemblyManager manager)
{
WebFile web = new WebFile(collection, this);
foreach (FileScheme scheme in Schemes)
{
web.AddFile(scheme, collection, manager);
}
return(web);
}
internal void AddFile(FileScheme scheme, IFileCollection collection, IAssemblyManager manager)
{
switch (scheme.SchemeType)
{
case FileEntryType.Serialized:
{
SerializedFileScheme serializedScheme = (SerializedFileScheme)scheme;
SerializedFile file = serializedScheme.ReadFile(collection, manager);
m_serializedFiles.Add(file);
OnSerializedFileAdded(file);
}
break;
case FileEntryType.Bundle:
{
BundleFileScheme bundleScheme = (BundleFileScheme)scheme;
BundleFile bundle = bundleScheme.ReadFile(collection, manager);
m_fileLists.Add(bundle);
OnFileListAdded(bundle);
}
break;
case FileEntryType.Archive:
{
ArchiveFileScheme archiveScheme = (ArchiveFileScheme)scheme;
ArchiveFile archive = archiveScheme.ReadFile(collection, manager);
m_fileLists.Add(archive);
OnFileListAdded(archive);
}
break;
case FileEntryType.Web:
{
WebFileScheme webScheme = (WebFileScheme)scheme;
WebFile webFile = webScheme.ReadFile(collection, manager);
m_fileLists.Add(webFile);
OnFileListAdded(webFile);
}
break;
case FileEntryType.Resource:
{
ResourceFileScheme resourceScheme = (ResourceFileScheme)scheme;
ResourceFile resource = resourceScheme.ReadFile();
m_resourceFiles.Add(resource);
OnResourceFileAdded(resource);
}
break;
default:
throw new NotSupportedException(scheme.SchemeType.ToString());
}
}
IKeyValueDB CreateKeyValueDB(IFileCollection fileCollection, ICompressionStrategy compressionStrategy = null)
{
if (fileCollection == null)
{
return(new InMemoryKeyValueDB());
}
if (compressionStrategy == null)
{
return(new KeyValueDB(fileCollection));
}
return(new KeyValueDB(fileCollection, compressionStrategy));
}
bool VerifyMultipleFileType(IFileCollection fileUploadViewModel)
{
//iterate through files in upload collection and verify that they are on the allowed files list
foreach (IFileWrapper fileWrapper in fileUploadViewModel.Files)
{
if (!VerifyFileType(fileWrapper))
{
return(false);
}
}
return(true);
}
bool CollectionContainsFiles(IFileCollection fileUploadViewModel)
{
//iterate thorugh upload collection and check if each object contains a file
foreach (IFileWrapper file in fileUploadViewModel.Files)
{
if (!ContainsFile(file))
{
return(false);
}
}
return(true);
}
public SerializedFile ReadFile(IFileCollection collection, IAssemblyManager manager)
{
SerializedFile file = new SerializedFile(collection, manager, this);
using (PartialStream stream = new PartialStream(m_stream, m_offset, m_size))
{
EndianType endianess = Header.SwapEndianess ? EndianType.BigEndian : EndianType.LittleEndian;
using (EndianReader reader = new EndianReader(stream, endianess, stream.Position))
{
file.Read(reader);
}
}
return(file);
}
public SerializedFile ReadSerializedFile(IFileCollection collection, Action <string> requestDependencyCallback)
{
SerializedFile file = new SerializedFile(collection, m_filePath, Name);
m_stream.Position = m_offset;
file.Read(m_stream, requestDependencyCallback);
long read = m_stream.Position - m_offset;
if (read != m_size)
{
//throw new System.Exception($"Read {read} but expected {m_length}");
}
return(file);
}
internal SerializedFile(IFileCollection collection, SerializedFileScheme scheme)
{
Collection = collection ?? throw new ArgumentNullException(nameof(collection));
FilePath = scheme.FilePath;
NameOrigin = scheme.Name;
Name = FilenameUtils.FixFileIdentifier(scheme.Name);
Flags = scheme.Flags;
Header = scheme.Header;
Metadata = scheme.Metadata;
for (int i = 0; i < Metadata.Entries.Length; i++)
{
m_assetEntryLookup.Add(Metadata.Entries[i].PathID, i);
}
}
IFileCollection CreateTestFileCollection()
{
if (_fastInMemory)
return null;
if (_inMemory)
{
_fileCollection = new InMemoryFileCollection();
return _fileCollection;
}
const string dbfilename = "data";
if (Directory.Exists(dbfilename))
Directory.Delete(dbfilename, true);
Directory.CreateDirectory(dbfilename);
if (_memoryMapped)
return new OnDiskMemoryMappedFileCollection(dbfilename);
return new OnDiskFileCollection(dbfilename);
}
public SerializedFile(IFileCollection collection, string filePath, string fileName)
{
if (collection == null)
{
throw new ArgumentNullException(nameof(collection));
}
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentNullException(nameof(fileName));
}
Collection = collection;
FilePath = filePath;
Name = fileName.ToLower();
Header = new SerializedFileHeader(Name);
Metadata = new SerializedFileMetadata(Name);
}
public DiskChunkCache(IFileCollection fileCollection, int keySize, long cacheCapacity)
{
if (keySize != 20)
{
throw new NotSupportedException("Only keySize of 20 (Useful for SHA1) is supported for now");
}
if (cacheCapacity < 1000)
{
throw new ArgumentOutOfRangeException(nameof(cacheCapacity), "Minimum for cache capacity is 1kB");
}
_fileCollection = fileCollection;
_keySize = keySize;
_cacheCapacity = cacheCapacity;
cacheCapacity = cacheCapacity / 1000 * (980 - keySize); // decrease for size of HashIndex
if (cacheCapacity / 8 > int.MaxValue)
{
_maxValueFileCount = checked ((int)(cacheCapacity / int.MaxValue));
_sizeLimitOfOneValueFile = int.MaxValue;
}
else
{
_maxValueFileCount = 8;
_sizeLimitOfOneValueFile = (int)(cacheCapacity / 8);
}
try
{
LoadContent();
}
catch
{
_cache.Clear();
}
if (_cache.Count != 0)
{
return;
}
foreach (var collectionFile in _fileInfos.Keys)
{
_fileCollection.GetFile(collectionFile).Remove();
}
_fileInfos.Clear();
_fileGeneration = 0;
}
IKeyValueDB CreateKeyValueDB(IFileCollection fileCollection, ICompressionStrategy compressionStrategy = null)
{
if (fileCollection == null)
{
if (_memoryMapped)
{
return(new ArtInMemoryKeyValueDB(new HGlobalAllocator()));
}
else
{
return(new InMemoryKeyValueDB());
}
}
if (compressionStrategy == null)
{
return(new KeyValueDB(fileCollection));
}
return(new KeyValueDB(fileCollection, compressionStrategy));
}
public SerializedFile(IFileCollection collection, string filePath, string fileName, TransferInstructionFlags flags)
{
if (collection == null)
{
throw new ArgumentNullException(nameof(collection));
}
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentNullException(nameof(fileName));
}
Collection = collection;
FilePath = filePath;
Name = fileName.ToLower(CultureInfo.InvariantCulture);
Flags = flags;
Header = new SerializedFileHeader(Name);
Metadata = new SerializedFileMetadata(Name);
}
public PersistentBuildCache(string dir)
{
var cacheIndex = 0;
while (cacheIndex < 100)
{
_mutex = new Mutex(false, @"Global\bbcoreCache" + cacheIndex);
if (_mutex.WaitOne(10))
{
break;
}
_mutex.Dispose();
_mutex = null;
cacheIndex++;
}
if (_mutex == null)
{
return;
}
_dir = dir + "/cache" + (cacheIndex == 0 ? "" : cacheIndex.ToString());
if (!new DirectoryInfo(_dir).Exists)
{
Directory.CreateDirectory(_dir);
}
_diskFileCollection = new OnDiskFileCollection(_dir);
_kvdb = new KeyValueDB(new KeyValueDBOptions
{
FileCollection = _diskFileCollection,
Compression = new SnappyCompressionStrategy(),
FileSplitSize = 100000000
});
_odb = new ObjectDB();
_odb.Open(_kvdb, false);
using (var tr = _odb.StartWritingTransaction().Result)
{
_tsConfiguration = tr.InitRelation <ITSConfigurationTable>("tsconf");
_tsRelation = tr.InitRelation <ITSFileBuildCacheTable>("ts");
_tsHashedContent = tr.InitRelation <IHashedContentTable>("hashedContent");
tr.Commit();
}
}
private SerializedFile(IFileCollection collection, IAssemblyManager manager, string filePath, string name, TransferInstructionFlags flags)
{
if (collection == null)
{
throw new ArgumentNullException(nameof(collection));
}
if (string.IsNullOrEmpty(filePath))
{
throw new ArgumentNullException(nameof(filePath));
}
Collection = collection;
AssemblyManager = manager;
FilePath = filePath;
Name = FilenameUtils.FixFileIdentifier(name);
Flags = flags;
Header = new SerializedFileHeader(Name);
Metadata = new SerializedFileMetadata(Name);
}
public Task SearchAsync(string searchPattern, FileSearchMode mode, int count, IFileCollection files, CancellationToken cancellationToken)
{
TaskCompletionSource<bool> result = new TaskCompletionSource<bool>();
Task.Factory.StartNew(
DelayedSearchHandler,
new Context()
{
RequestIndex = ++currentRequestIndex,
CompletionSource = result,
SearchPattern = searchPattern,
Mode = mode,
Count = count,
Files = files,
CancellationToken = cancellationToken
},
cancellationToken
);
return result.Task;
}
public MainViewModel Create()
{
UiBackgroundContext backgroundContext = new UiBackgroundContext();
MainViewModel viewModel = new MainViewModel(
new PinnedForEmptyPatternFileSearchService(
//new DelayedFileSearchService(
// Dispatcher,
// CreateFileSearchService()
//),
//this
CreateFileSearchService(backgroundContext),
pinStateService
),
backgroundContext,
() => settings.FileSearchMode,
() => settings.FileSearchCount
);
if (propertyChangedHandler != null)
{
viewModel.PropertyChanged += propertyChangedHandler;
}
ApplicationFilteredCollection applications = new ApplicationFilteredCollection(settings, viewModel);
mainApplicationLoader.Add(applications);
AdditionalApplicationLoader additionalLoader = new AdditionalApplicationLoader(settings);
additionalLoader.Add(viewModel);
IFileCollection files = viewModel;
foreach (string filePath in pinStateService.Enumerate())
{
files.Add(Path.GetFileNameWithoutExtension(filePath), filePath, true);
}
return(viewModel);
}
public PersistentBuildCache(string dir)
{
_dir = dir + "/cache";
if (!new DirectoryInfo(_dir).Exists)
{
Directory.CreateDirectory(_dir);
}
_diskFileCollection = new OnDiskFileCollection(_dir);
_kvdb = new KeyValueDB(new KeyValueDBOptions {
FileCollection = _diskFileCollection,
Compression = new SnappyCompressionStrategy(),
FileSplitSize = 100000000
});
_odb = new ObjectDB();
_odb.Open(_kvdb, false);
using (var tr = _odb.StartWritingTransaction().Result)
{
_tsConfiguration = tr.InitRelation <ITSConfigurationTable>("tsconf");
_tsRelation = tr.InitRelation <ITSFileBuildCacheTable>("ts");
tr.Commit();
}
}
public DiskChunkCache(IFileCollection fileCollection, int keySize, long cacheCapacity)
{
if (keySize != 20) throw new NotSupportedException("Only keySize of 20 (Usefull for SHA1) is supported for now");
if (cacheCapacity < 1000) throw new ArgumentOutOfRangeException(nameof(cacheCapacity), "Minimum for cache capacity is 1kB");
_fileCollection = fileCollection;
_keySize = keySize;
_cacheCapacity = cacheCapacity;
cacheCapacity = cacheCapacity / 1000 * (980 - keySize); // decrease for size of HashIndex
if (cacheCapacity / 8 > int.MaxValue)
{
_maxValueFileCount = checked((int)(cacheCapacity / int.MaxValue));
_sizeLimitOfOneValueFile = int.MaxValue;
}
else
{
_maxValueFileCount = 8;
_sizeLimitOfOneValueFile = (int)(cacheCapacity / 8);
}
try
{
LoadContent();
}
catch
{
_cache.Clear();
}
if (_cache.Count == 0)
{
foreach (var collectionFile in _fileInfos.Keys)
{
_fileCollection.GetFile(collectionFile).Remove();
}
_fileInfos.Clear();
_fileGeneration = 0;
}
}
public KeyValueDB(IFileCollection fileCollection, ICompressionStrategy compression, uint fileSplitSize, ICompactorScheduler compactorScheduler)
{
if (fileCollection == null)
{
throw new ArgumentNullException(nameof(fileCollection));
}
if (compression == null)
{
throw new ArgumentNullException(nameof(compression));
}
if (fileSplitSize < 1024 || fileSplitSize > int.MaxValue)
{
throw new ArgumentOutOfRangeException(nameof(fileSplitSize), "Allowed range 1024 - 2G");
}
_compactorScheduler = compactorScheduler;
MaxTrLogFileSize = fileSplitSize;
_compression = compression;
DurableTransactions = false;
_fileCollection = new FileCollectionWithFileInfos(fileCollection);
_lastCommited = new BTreeRoot(0);
LoadInfoAboutFiles();
_compactFunc = _compactorScheduler?.AddCompactAction(Compact);
_compactorScheduler?.AdviceRunning();
}
public void Dispose()
{
_keyValueDb?.Dispose();
_keyValueDb = null;
_memoryFileCollection?.Dispose();
_memoryFileCollection = null;
}
public FindUnusedKeysVisitor()
{
_memoryFileCollection = new InMemoryFileCollection();
_keyValueDb = new KeyValueDB(_memoryFileCollection);
}
public FileCollectionWithFileInfos(IFileCollection fileCollection)
{
_fileCollection = fileCollection;
Guid = null;
LoadInfoAboutFiles();
}
public Task SearchAsync(string searchPattern, FileSearchMode mode, int count, IFileCollection files, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
return Async.CompletedTask;
files.Clear();
Func<FileModel, bool> filter = matcherFactory.Create(searchPattern, mode);
foreach (FileModel model in storage.Where(f => filter(f)).Take(count))
files.Add(model.Name, model.Path, pinStateService.IsPinned(model.Path));
return Async.CompletedTask;
}
public void CreateChunkStorageSubDB()
{
_fileCollection = new InMemoryFileCollection();
_kvdb = new KeyValueDB(_fileCollection);
_cs = _kvdb.GetSubDB<IChunkStorage>(1);
}
IKeyValueDB CreateKeyValueDB(IFileCollection fileCollection, ICompressionStrategy compressionStrategy = null)
{
if (fileCollection == null) return new InMemoryKeyValueDB();
if (compressionStrategy == null)
return new KeyValueDB(fileCollection);
return new KeyValueDB(fileCollection, compressionStrategy);
}
