public StorageSignatureRepository(ITransactionalStorage storage, string fileName)
{
_tempDirectory = TempDirectoryTools.Create();
_storage = storage;
_fileName = fileName;
_createdFiles = new Dictionary<string, FileStream>();
}
private static void when_there_are_multiple_map_results_for_multiple_indexes(ITransactionalStorage transactionalStorage)
{
transactionalStorage.Initialize(new DummyUuidGenerator());
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.True(actionsAccessor.Staleness.IsReduceStale("a"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("b"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("c"));
});
}
private SynchronizingFileStream(ITransactionalStorage transactionalStorage, string fileName,
StorageStreamAccess storageStreamAccess, NameValueCollection metadata,
IndexStorage indexStorage, StorageOperationsTask operations)
: base(transactionalStorage, fileName, storageStreamAccess, metadata, indexStorage, operations)
{
md5Hasher = new MD5CryptoServiceProvider();
}
public static SynchronizingFileStream CreatingOrOpeningAndWritting(ITransactionalStorage storage, IndexStorage search,
StorageOperationsTask operationsTask,
string fileName, NameValueCollection metadata)
{
return new SynchronizingFileStream(storage, fileName, StorageStreamAccess.CreateAndWrite, metadata, search,
operationsTask) { PreventUploadComplete = true };
}
protected StorageStream(ITransactionalStorage transactionalStorage, string fileName,
StorageStreamAccess storageStreamAccess,
RavenJObject metadata, IndexStorage indexStorage, StorageOperationsTask operations)
{
TransactionalStorage = transactionalStorage;
StorageStreamAccess = storageStreamAccess;
Name = fileName;
switch (storageStreamAccess)
{
case StorageStreamAccess.Read:
TransactionalStorage.Batch(accessor => fileHeader = accessor.ReadFile(fileName));
if (fileHeader.TotalSize == null)
{
throw new FileNotFoundException("File is not uploaded yet");
}
Metadata = fileHeader.Metadata;
Seek(0, SeekOrigin.Begin);
break;
case StorageStreamAccess.CreateAndWrite:
TransactionalStorage.Batch(accessor =>
{
operations.IndicateFileToDelete(fileName);
accessor.PutFile(fileName, null, metadata);
indexStorage.Index(fileName, metadata);
});
Metadata = metadata;
break;
default:
throw new ArgumentOutOfRangeException("storageStreamAccess", storageStreamAccess, "Unknown value");
}
}
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState recievedTransportState = null)
{
this.Name = name;
this.systemConfiguration = systemConfiguration;
var storageType = systemConfiguration.FileSystem.DefaultStorageTypeName;
storage = CreateTransactionalStorage(storageType, systemConfiguration);
search = new IndexStorage(systemConfiguration.FileSystem.IndexStoragePath, systemConfiguration.Settings);
sigGenerator = new SigGenerator();
var replicationHiLo = new SynchronizationHiLo(storage);
var sequenceActions = new SequenceActions(storage);
transportState = recievedTransportState ?? new TransportState();
notificationPublisher = new NotificationPublisher(transportState);
fileLockManager = new FileLockManager();
storage.Initialize();
search.Initialize();
var uuidGenerator = new UuidGenerator(sequenceActions);
historian = new Historian(storage, replicationHiLo, uuidGenerator);
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog));
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher);
metricsCounters = new MetricsCountersManager();
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
protected StorageStream(ITransactionalStorage transactionalStorage, string fileName,
StorageStreamAccess storageStreamAccess,
RavenJObject metadata, IndexStorage indexStorage, StorageOperationsTask operations)
{
TransactionalStorage = transactionalStorage;
StorageStreamAccess = storageStreamAccess;
Name = fileName;
switch (storageStreamAccess)
{
case StorageStreamAccess.Read:
TransactionalStorage.Batch(accessor => fileHeader = accessor.ReadFile(fileName));
if (fileHeader.TotalSize == null)
{
throw new FileNotFoundException("File is not uploaded yet");
}
Metadata = fileHeader.Metadata;
Seek(0, SeekOrigin.Begin);
break;
case StorageStreamAccess.CreateAndWrite:
TransactionalStorage.Batch(accessor =>
{
operations.IndicateFileToDelete(fileName);
accessor.PutFile(fileName, null, metadata);
indexStorage.Index(fileName, metadata);
});
Metadata = metadata;
break;
default:
throw new ArgumentOutOfRangeException("storageStreamAccess", storageStreamAccess, "Unknown value");
}
}
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState recievedTransportState = null)
{
this.Name = name;
this.systemConfiguration = systemConfiguration;
var storageType = systemConfiguration.FileSystem.DefaultStorageTypeName;
storage = CreateTransactionalStorage(storageType, systemConfiguration);
search = new IndexStorage(systemConfiguration.FileSystem.IndexStoragePath, systemConfiguration.Settings);
sigGenerator = new SigGenerator();
var replicationHiLo = new SynchronizationHiLo(storage);
var sequenceActions = new SequenceActions(storage);
transportState = recievedTransportState ?? new TransportState();
notificationPublisher = new NotificationPublisher(transportState);
fileLockManager = new FileLockManager();
storage.Initialize();
search.Initialize();
var uuidGenerator = new UuidGenerator(sequenceActions);
historian = new Historian(storage, replicationHiLo, uuidGenerator);
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog));
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher);
metricsCounters = new MetricsCountersManager();
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
private void ReadIndexesFromCatalog(IEnumerable<AbstractViewGenerator> compiledGenerators, ITransactionalStorage transactionalStorage)
{
foreach (var generator in compiledGenerators)
{
var copy = generator;
var displayNameAtt = TypeDescriptor.GetAttributes(copy)
.OfType<DisplayNameAttribute>()
.FirstOrDefault();
var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name;
transactionalStorage.Batch(actions =>
{
if (actions.Indexing.GetIndexesStats().Any(x => x.Name == name))
return;
actions.Indexing.AddIndex(name, copy.ReduceDefinition != null);
});
var indexDefinition = new IndexDefinition
{
Name = name,
Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName,
// need to supply this so the index storage will create map/reduce index
Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName,
Indexes = generator.Indexes,
Stores = generator.Stores,
IsCompiled = true
};
indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy);
indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition);
}
}
public TisSecurityMngr(
ITransactionalStorage oApplicationResources,
string sAppName,
ITisSecurityPolicyProviderService securityPolicy,
PermissionValidationInvocator permissionValidationInvocator)
{
m_oApplicationResources = oApplicationResources as ITransactionalStorage;
DataContractSerializer rolesDataContractSerializer = new DataContractSerializer(typeof(TisRolesMngr), new Type[] { typeof(TisRole), typeof(TisSystemUsersMngr), typeof(TisSystemUser) });
DataContractSerializer permissionsDataContractSerializer = new DataContractSerializer(typeof(TisSupportedPermissionsMngr), new Type[] { typeof(TisSupportedPermissionsSet) });
m_rolesStorage = new ObjectStorage(
m_oApplicationResources,
new ObjectReadDelegate(rolesDataContractSerializer.ReadObject),
new ObjectWriteDelegate(rolesDataContractSerializer.WriteObject));
m_permissionsStorage = new ObjectStorage(
m_oApplicationResources,
new ObjectReadDelegate(permissionsDataContractSerializer.ReadObject),
new ObjectWriteDelegate(permissionsDataContractSerializer.WriteObject));
m_permissionValidationInvocator = permissionValidationInvocator;
m_securityPolicy = securityPolicy;
InitTransactionManager();
m_bLoaded = false;
}
private static void when_there_are_updates_to_map_reduce_results(ITransactionalStorage transactionalStorage)
{
var dummyUuidGenerator = new DummyUuidGenerator();
transactionalStorage.Initialize(dummyUuidGenerator);
Guid a = Guid.Empty;
Guid b = Guid.Empty;
Guid c = Guid.Empty;
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
a = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
b = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
c = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("a", a, false, 100).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("b", b, false, 100).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("c", c, false, 100).Count());
});
}
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState receivedTransportState = null)
{
ExtensionsState = new AtomicDictionary <object>();
Name = name;
this.systemConfiguration = systemConfiguration;
systemConfiguration.Container.SatisfyImportsOnce(this);
transportState = receivedTransportState ?? new TransportState();
storage = CreateTransactionalStorage(systemConfiguration);
sigGenerator = new SigGenerator();
fileLockManager = new FileLockManager();
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog));
notificationPublisher = new NotificationPublisher(transportState);
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
metricsCounters = new MetricsCountersManager();
search = new IndexStorage(name, systemConfiguration);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
storageOperationsTask = new StorageOperationsTask(storage, DeleteTriggers, search, notificationPublisher);
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
public IndexDefinitionStorage(
InMemoryRavenConfiguration configuration,
ITransactionalStorage transactionalStorage,
string path,
IEnumerable <AbstractViewGenerator> compiledGenerators,
OrderedPartCollection <AbstractDynamicCompilationExtension> extensions)
{
this.configuration = configuration;
this.extensions = extensions; // this is used later in the ctor, so it must appears first
this.path = Path.Combine(path, IndexDefDir);
if (Directory.Exists(this.path) == false && configuration.RunInMemory == false)
{
Directory.CreateDirectory(this.path);
}
this.extensions = extensions;
if (configuration.RunInMemory == false)
{
ReadIndexesFromDisk();
}
//compiled view generators always overwrite dynamic views
ReadIndexesFromCatalog(compiledGenerators, transactionalStorage);
}
private SynchronizingFileStream(ITransactionalStorage transactionalStorage, string fileName,
StorageStreamAccess storageStreamAccess, RavenJObject metadata,
IndexStorage indexStorage, StorageOperationsTask operations)
: base(transactionalStorage, fileName, storageStreamAccess, metadata, indexStorage, operations)
{
md5Hasher = Encryptor.Current.CreateHash();
}
public EtagSynchronizer(EtagSynchronizerType type, ITransactionalStorage transactionalStorage)
{
this.type = type;
this.transactionalStorage = transactionalStorage;
LoadSynchronizationState();
}
public IndexDefinitionStorage(
InMemoryRavenConfiguration configuration,
ITransactionalStorage transactionalStorage,
string path,
IEnumerable <AbstractViewGenerator> compiledGenerators,
OrderedPartCollection <AbstractDynamicCompilationExtension> extensions)
{
IndexMergeSuggestions = new List <IndexMergeSuggestion>();
this.configuration = configuration;
this.transactionalStorage = transactionalStorage;
this.extensions = extensions; // this is used later in the ctor, so it must appears first
this.path = Path.Combine(path, IndexDefDir);
if (Directory.Exists(this.path) == false && configuration.RunInMemory == false)
{
Directory.CreateDirectory(this.path);
}
if (configuration.RunInMemory == false)
{
ReadFromDisk();
}
newDefinitionsThisSession.Clear();
}
public LocalRdcManager(ISignatureRepository signatureRepository, ITransactionalStorage transactionalStorage,
SigGenerator sigGenerator)
{
_signatureRepository = signatureRepository;
_transactionalStorage = transactionalStorage;
_sigGenerator = sigGenerator;
}
protected AbstractIndexingExecuter(ITransactionalStorage transactionalStorage, WorkContext context, TaskScheduler scheduler)
{
log = LogManager.GetLogger(GetType());
this.transactionalStorage = transactionalStorage;
this.context = context;
this.scheduler = scheduler;
}
public LocalRdcManager(ISignatureRepository signatureRepository, ITransactionalStorage transactionalStorage,
SigGenerator sigGenerator)
{
_signatureRepository = signatureRepository;
_transactionalStorage = transactionalStorage;
_sigGenerator = sigGenerator;
}
protected AbstractIndexingExecuter(
ITransactionalStorage transactionalStorage, WorkContext context, TaskScheduler scheduler)
{
this.transactionalStorage = transactionalStorage;
this.context = context;
this.scheduler = scheduler;
}
private SynchronizingFileStream(ITransactionalStorage transactionalStorage, string fileName,
StorageStreamAccess storageStreamAccess, RavenJObject metadata,
IndexStorage indexStorage, StorageOperationsTask operations)
: base(transactionalStorage, fileName, storageStreamAccess, metadata, indexStorage, operations)
{
md5Hasher = Encryptor.Current.CreateHash();
}
protected AbstractIndexingExecuter(
ITransactionalStorage transactionalStorage, WorkContext context, TaskScheduler scheduler)
{
this.transactionalStorage = transactionalStorage;
this.context = context;
this.scheduler = scheduler;
}
private static void when_there_are_multiple_map_results_for_multiple_indexes(ITransactionalStorage transactionalStorage)
{
transactionalStorage.Initialize(new DummyUuidGenerator());
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a",true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.True(actionsAccessor.Staleness.IsReduceStale("a"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("b"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("c"));
});
}
protected AbstractIndexingExecuter(WorkContext context, IndexReplacer indexReplacer)
{
this.transactionalStorage = context.TransactionalStorage;
this.context = context;
this.indexReplacer = indexReplacer;
this.scheduler = context.TaskScheduler;
}
public EtagSynchronizer(EtagSynchronizerType type, ITransactionalStorage transactionalStorage)
{
this.type = type;
this.transactionalStorage = transactionalStorage;
LoadSynchronizationState();
}
protected AbstractIndexingExecuter(WorkContext context, IndexReplacer indexReplacer)
{
this.transactionalStorage = context.TransactionalStorage;
this.context = context;
this.indexReplacer = indexReplacer;
this.scheduler = context.TaskScheduler;
}
public StorageSignatureRepository(ITransactionalStorage storage, string fileName)
{
_tempDirectory = TempDirectoryTools.Create();
_storage = storage;
_fileName = fileName;
_createdFiles = new Dictionary <string, FileStream>();
}
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState receivedTransportState = null)
{
ExtensionsState = new AtomicDictionary<object>();
Name = name;
this.systemConfiguration = systemConfiguration;
systemConfiguration.Container.SatisfyImportsOnce(this);
transportState = receivedTransportState ?? new TransportState();
storage = CreateTransactionalStorage(systemConfiguration);
sigGenerator = new SigGenerator();
fileLockManager = new FileLockManager();
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog));
notificationPublisher = new NotificationPublisher(transportState);
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
metricsCounters = new MetricsCountersManager();
search = new IndexStorage(name, systemConfiguration);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
storageOperationsTask = new StorageOperationsTask(storage, DeleteTriggers, search, notificationPublisher);
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
public IndexDefinitionStorage(
ITransactionalStorage transactionalStorage,
string path,
IEnumerable<AbstractViewGenerator> compiledGenerators,
AbstractDynamicCompilationExtension[] extensions)
{
this.extensions = extensions;// this is used later in the ctor, so it must appears first
this.path = Path.Combine(path, IndexDefDir);
if (Directory.Exists(this.path) == false)
Directory.CreateDirectory(this.path);
this.extensions = extensions;
foreach (var index in Directory.GetFiles(this.path, "*.index"))
{
try
{
AddAndCompileIndex(
MonoHttpUtility.UrlDecode(Path.GetFileNameWithoutExtension(index)),
JsonConvert.DeserializeObject<IndexDefinition>(File.ReadAllText(index), new JsonEnumConverter())
);
}
catch (Exception e)
{
logger.Warn("Could not compile index " + index + ", skipping bad index", e);
}
}
//compiled view generators always overwrite dynamic views
foreach (var generator in compiledGenerators)
{
var copy = generator;
var displayNameAtt = TypeDescriptor.GetAttributes(copy)
.OfType<DisplayNameAttribute>()
.FirstOrDefault();
var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name;
transactionalStorage.Batch(actions =>
{
if (actions.Indexing.GetIndexesStats().Any(x => x.Name == name))
return;
actions.Indexing.AddIndex(name);
});
var indexDefinition = new IndexDefinition
{
Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName,
// need to supply this so the index storage will create map/reduce index
Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName,
Indexes = generator.Indexes,
Stores = generator.Stores,
IsCompiled = true
};
indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy);
indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition);
}
}
protected AbstractIndexingExecuter(WorkContext context, IndexReplacer indexReplacer)
{
Log = LogManager.GetLogger(GetType());
this.transactionalStorage = context.TransactionalStorage;
this.context = context;
this.indexReplacer = indexReplacer;
this.scheduler = context.TaskScheduler;
}
public static SynchronizationConfig GetOrDefault(ITransactionalStorage storage)
{
SynchronizationConfig result = null;
storage.Batch(accessor => result = GetOrDefault(accessor));
return(result ?? new SynchronizationConfig());
}
public bool TimeoutExceeded(string fileName, ITransactionalStorage storage)
{
var result = false;
storage.Batch(accessor => result = TimeoutExceeded(fileName, accessor));
return result;
}
public StorageOperationsTask(ITransactionalStorage storage, IndexStorage search, INotificationPublisher notificationPublisher)
{
this.storage = storage;
this.search = search;
this.notificationPublisher = notificationPublisher;
InitializeTimer();
}
public StorageOperationsTask(ITransactionalStorage storage, IndexStorage search, INotificationPublisher notificationPublisher)
{
this.storage = storage;
this.search = search;
this.notificationPublisher = notificationPublisher;
InitializeTimer();
}
public bool TimeoutExceeded(string fileName, ITransactionalStorage storage)
{
var result = false;
storage.Batch(accessor => result = TimeoutExceeded(fileName, accessor));
return(result);
}
protected AbstractIndexingExecuter(ITransactionalStorage transactionalStorage, WorkContext context, TaskScheduler scheduler)
{
this.transactionalStorage = transactionalStorage;
this.context = context;
this.scheduler = scheduler;
autoTuner = new IndexBatchSizeAutoTuner(context);
}
public static SynchronizationConfig GetOrDefault(ITransactionalStorage storage)
{
SynchronizationConfig result = null;
storage.Batch(accessor => result = GetOrDefault(accessor));
return result ?? new SynchronizationConfig();
}
protected AbstractIndexingExecuter(WorkContext context, IndexReplacer indexReplacer)
{
Log = LogManager.GetLogger(GetType());
this.transactionalStorage = context.TransactionalStorage;
this.context = context;
this.indexReplacer = indexReplacer;
this.scheduler = context.TaskScheduler;
}
public static SynchronizingFileStream CreatingOrOpeningAndWriting(ITransactionalStorage storage, IndexStorage search,
StorageOperationsTask operationsTask,
string fileName, RavenJObject metadata)
{
return(new SynchronizingFileStream(storage, fileName, StorageStreamAccess.CreateAndWrite, metadata, search, operationsTask)
{
PreventUploadComplete = true
});
}
public ReadFileToDatabase(BufferPool bufferPool, ITransactionalStorage storage, Stream inputStream, string filename)
{
this.bufferPool = bufferPool;
this.inputStream = inputStream;
this.storage = storage;
this.filename = filename;
buffer = bufferPool.TakeBuffer(StorageConstants.MaxPageSize);
md5Hasher = Encryptor.Current.CreateHash();
}
public ViewStorage()
{
transactionalStorage = new TransactionalStorage(new RavenConfiguration
{
DataDirectory = "raven.db.test.esent",
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = true
}, () => { });
transactionalStorage.Initialize(new DummyUuidGenerator());
}
public ViewStorage()
{
transactionalStorage = new TransactionalStorage(new RavenConfiguration
{
DataDirectory = "raven.db.test.esent",
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = true
}, () => { });
transactionalStorage.Initialize();
}
public ViewStorage()
{
transactionalStorage = new TransactionalStorage(new RavenConfiguration
{
DataDirectory = DataDir,
RunInUnreliableYetFastModeThatIsNotSuitableForProduction = true
}, () => { });
transactionalStorage.Initialize(new DummyUuidGenerator());
}
public RavenFileSystem(InMemoryRavenConfiguration config, string name, TransportState receivedTransportState = null)
{
ExtensionsState = new AtomicDictionary <object>();
Name = name;
ResourceName = string.Concat(Constants.FileSystem.UrlPrefix, "/", name);
configuration = config;
try
{
ValidateStorage();
configuration.Container.SatisfyImportsOnce(this);
transportState = receivedTransportState ?? new TransportState();
storage = CreateTransactionalStorage(configuration);
sigGenerator = new SigGenerator();
fileLockManager = new FileLockManager();
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(configuration.Catalog));
notificationPublisher = new NotificationPublisher(transportState);
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, configuration);
metricsCounters = new MetricsCountersManager();
search = new IndexStorage(name, configuration);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
TimerManager = new ResourceTimerManager();
Tasks = new TaskActions(this, Log);
Files = new FileActions(this, Log);
Synchronizations = new SynchronizationActions(this, Log);
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
catch (Exception e)
{
Log.ErrorException(string.Format("Could not create file system '{0}'", Name ?? "unknown name"), e);
try
{
Dispose();
}
catch (Exception ex)
{
Log.FatalException("Failed to dispose when already getting an error in file system ctor", ex);
}
throw;
}
}
public StorageOperationsTask(ITransactionalStorage storage, OrderedPartCollection<AbstractFileDeleteTrigger> deleteTriggers, IndexStorage search, INotificationPublisher notificationPublisher)
{
this.storage = storage;
this.deleteTriggers = deleteTriggers;
this.search = search;
this.notificationPublisher = notificationPublisher;
InitializeTimer();
}
public RavenFileSystem(InMemoryRavenConfiguration config, string name, TransportState receivedTransportState = null)
{
ExtensionsState = new AtomicDictionary<object>();
Name = name;
ResourceName = string.Concat(Abstractions.Data.Constants.FileSystem.UrlPrefix, "/", name);
configuration = config;
try
{
ValidateStorage();
configuration.Container.SatisfyImportsOnce(this);
transportState = receivedTransportState ?? new TransportState();
storage = CreateTransactionalStorage(configuration);
sigGenerator = new SigGenerator();
fileLockManager = new FileLockManager();
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(configuration.Catalog));
notificationPublisher = new NotificationPublisher(transportState);
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, configuration);
metricsCounters = new MetricsCountersManager();
search = new IndexStorage(name, configuration);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
TimerManager = new ResourceTimerManager();
Tasks = new TaskActions(this, Log);
Files = new FileActions(this, Log);
Synchronizations = new SynchronizationActions(this, Log);
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
catch (Exception e)
{
Log.ErrorException(string.Format("Could not create file system '{0}'", Name ?? "unknown name"), e);
try
{
Dispose();
}
catch (Exception ex)
{
Log.FatalException("Failed to dispose when already getting an error in file system ctor", ex);
}
throw;
}
}
public static bool TryToCreateTransactionalStorage(InMemoryRavenConfiguration ravenConfiguration,
bool hasCompression, EncryptionConfiguration encryption, bool isRavenFs,
out ITransactionalStorage storage, out Database.FileSystem.Storage.ITransactionalStorage fileStorage)
{
storage = null;
fileStorage = null;
if (File.Exists(Path.Combine(ravenConfiguration.DataDirectory, Voron.Impl.Constants.DatabaseFilename)))
{
if (isRavenFs)
{
fileStorage = RavenFileSystem.CreateTransactionalStorage(ravenConfiguration);
}
else
{
storage = ravenConfiguration.CreateTransactionalStorage(InMemoryRavenConfiguration.VoronTypeName, () => { }, () => { });
}
}
else if (File.Exists(Path.Combine(ravenConfiguration.DataDirectory, "Data.jfm")))
{
if (isRavenFs)
{
fileStorage = RavenFileSystem.CreateTransactionalStorage(ravenConfiguration);
}
else
{
storage = ravenConfiguration.CreateTransactionalStorage(InMemoryRavenConfiguration.EsentTypeName, () => { }, () => { });
}
}
if (storage == null && fileStorage == null)
{
return(false);
}
if (isRavenFs)
{
var filesOrderedPartCollection = new OrderedPartCollection <Database.FileSystem.Plugins.AbstractFileCodec>();
fileStorage.Initialize(new UuidGenerator(), filesOrderedPartCollection);
return(true);
}
var orderedPartCollection = new OrderedPartCollection <AbstractDocumentCodec>();
if (encryption != null)
{
var documentEncryption = new DocumentEncryption();
documentEncryption.SetSettings(new EncryptionSettings(encryption.EncryptionKey, encryption.SymmetricAlgorithmType,
encryption.EncryptIndexes, encryption.PreferedEncryptionKeyBitsSize));
orderedPartCollection.Add(documentEncryption);
}
if (hasCompression)
{
orderedPartCollection.Add(new DocumentCompression());
}
storage.Initialize(new SequentialUuidGenerator {
EtagBase = 0
}, orderedPartCollection);
return(true);
}
public StorageOperationsTask(ITransactionalStorage storage, OrderedPartCollection <AbstractFileDeleteTrigger> deleteTriggers, IndexStorage search, INotificationPublisher notificationPublisher)
{
this.storage = storage;
this.deleteTriggers = deleteTriggers;
this.search = search;
this.notificationPublisher = notificationPublisher;
InitializeTimer();
}
public SynchronizationTask(ITransactionalStorage storage, SigGenerator sigGenerator, NotificationPublisher publisher, InMemoryRavenConfiguration systemConfiguration)
{
this.storage = storage;
this.publisher = publisher;
this.systemConfiguration = systemConfiguration;
context = new SynchronizationTaskContext();
synchronizationQueue = new SynchronizationQueue();
synchronizationStrategy = new SynchronizationStrategy(storage, sigGenerator);
}
public SynchronizationTask(ITransactionalStorage storage, SigGenerator sigGenerator, NotificationPublisher publisher, RavenConfiguration systemConfiguration)
{
this.storage = storage;
this.publisher = publisher;
this.systemConfiguration = systemConfiguration;
context = new SynchronizationTaskContext();
synchronizationQueue = new SynchronizationQueue();
synchronizationStrategy = new SynchronizationStrategy(storage, sigGenerator, systemConfiguration);
}
public ReadFileToDatabase(BufferPool bufferPool, ITransactionalStorage storage, OrderedPartCollection <AbstractFilePutTrigger> putTriggers, Stream inputStream, string filename, RavenJObject headers)
{
this.bufferPool = bufferPool;
this.inputStream = inputStream;
this.storage = storage;
this.putTriggers = putTriggers;
this.filename = filename;
this.headers = headers;
buffer = bufferPool.TakeBuffer(StorageConstants.MaxPageSize);
md5Hasher = Encryptor.Current.CreateHash();
}
public static StorageStream CreatingNewAndWritting(ITransactionalStorage transactionalStorage,
IndexStorage indexStorage, StorageOperationsTask operations,
string fileName, RavenJObject metadata)
{
if (indexStorage == null)
{
throw new ArgumentNullException("indexStorage", "indexStorage == null");
}
return(new StorageStream(transactionalStorage, fileName, StorageStreamAccess.CreateAndWrite, metadata, indexStorage, operations));
}
public ReadFileToDatabase(BufferPool bufferPool, ITransactionalStorage storage, OrderedPartCollection<AbstractFilePutTrigger> putTriggers, Stream inputStream, string filename, RavenJObject headers)
{
this.bufferPool = bufferPool;
this.inputStream = inputStream;
this.storage = storage;
this.putTriggers = putTriggers;
this.filename = filename;
this.headers = headers;
buffer = bufferPool.TakeBuffer(StorageConstants.MaxPageSize);
md5Hasher = Encryptor.Current.CreateHash();
}
public SynchronizationTask(ITransactionalStorage storage, SigGenerator sigGenerator, NotificationPublisher publisher,
InMemoryRavenConfiguration systemConfiguration)
{
this.storage = storage;
this.publisher = publisher;
this.systemConfiguration = systemConfiguration;
synchronizationQueue = new SynchronizationQueue();
synchronizationStrategy = new SynchronizationStrategy(storage, sigGenerator);
LastSuccessfulSynchronizationTime = DateTime.MinValue;
InitializeTimer();
}
public SynchronizationTask(ITransactionalStorage storage, SigGenerator sigGenerator, NotificationPublisher publisher,
InMemoryRavenConfiguration systemConfiguration)
{
this.storage = storage;
this.publisher = publisher;
this.systemConfiguration = systemConfiguration;
synchronizationQueue = new SynchronizationQueue();
synchronizationStrategy = new SynchronizationStrategy(storage, sigGenerator);
LastSuccessfulSynchronizationTime = DateTime.MinValue;
InitializeTimer();
}
protected SynchronizationWorkItem(string fileName, string sourceServerUrl, ITransactionalStorage storage)
{
Storage = storage;
FileName = fileName;
FileAndPagesInformation fileAndPages = null;
Storage.Batch(accessor => fileAndPages = accessor.GetFile(fileName, 0, 0));
FileMetadata = fileAndPages.Metadata;
FileSystemInfo = new FileSystemInfo
{
Id = Storage.Id,
Url = sourceServerUrl
};
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(null, null);
}
public IndexDefinitionStorage(
InMemoryRavenConfiguration configuration,
ITransactionalStorage transactionalStorage,
string path,
OrderedPartCollection<AbstractDynamicCompilationExtension> extensions)
{
this.configuration = configuration;
this.transactionalStorage = transactionalStorage;
this.extensions = extensions; // this is used later in the ctor, so it must appears first
this.path = Path.Combine(path, IndexDefDir);
if (Directory.Exists(this.path) == false && configuration.RunInMemory == false)
Directory.CreateDirectory(this.path);
if (configuration.RunInMemory == false)
ReadFromDisk();
newDefinitionsThisSession.Clear();
}
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, TransportState transportState, string name)
{
this.Name = name;
this.systemConfiguration = systemConfiguration;
var storageType = systemConfiguration.DefaultFileSystemStorageTypeName;
if (string.Equals(InMemoryRavenConfiguration.VoronTypeName, storageType, StringComparison.OrdinalIgnoreCase) == false)
{
if (Directory.Exists(systemConfiguration.FileSystemDataDirectory) &&
Directory.EnumerateFileSystemEntries(systemConfiguration.FileSystemDataDirectory).Any())
throw new InvalidOperationException(
string.Format(
"We do not allow to run on a storage engine other then Voron, while we are in the early pre-release phase of RavenDB 3.0. You are currently running on {0}",
storageType));
Trace.WriteLine("Forcing filesystem to run on Voron - pre release behavior only, mind " + Path.GetFileName(Path.GetDirectoryName(systemConfiguration.FileSystemDataDirectory)));
storageType = InMemoryRavenConfiguration.VoronTypeName;
}
storage = CreateTransactionalStorage(storageType, systemConfiguration.FileSystemDataDirectory, systemConfiguration.Settings);
search = new IndexStorage(systemConfiguration.FileSystemIndexStoragePath, systemConfiguration.Settings);
sigGenerator = new SigGenerator();
var replicationHiLo = new SynchronizationHiLo(storage);
var sequenceActions = new SequenceActions(storage);
this.transportState = transportState;
notificationPublisher = new NotificationPublisher(transportState);
fileLockManager = new FileLockManager();
storage.Initialize();
search.Initialize();
var uuidGenerator = new UuidGenerator(sequenceActions);
historian = new Historian(storage, replicationHiLo, uuidGenerator);
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver();
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher);
metricsCounters = new MetricsCountersManager();
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
public IndexDefinitionStorage(
InMemoryRavenConfiguration configuration,
ITransactionalStorage transactionalStorage,
string path,
IEnumerable<AbstractViewGenerator> compiledGenerators,
OrderedPartCollection<AbstractDynamicCompilationExtension> extensions)
{
this.configuration = configuration;
this.extensions = extensions; // this is used later in the ctor, so it must appears first
this.path = Path.Combine(path, IndexDefDir);
if (Directory.Exists(this.path) == false && configuration.RunInMemory == false)
Directory.CreateDirectory(this.path);
if (configuration.RunInMemory == false)
ReadIndexesFromDisk();
//compiled view generators always overwrite dynamic views
ReadIndexesFromCatalog(compiledGenerators, transactionalStorage);
}
protected StorageStream(RavenFileSystem fileSystem, ITransactionalStorage storage, string fileName, RavenJObject metadata, StorageStreamAccess storageStreamAccess)
{
this.fileSystem = fileSystem;
this.storage = storage;
StorageStreamAccess = storageStreamAccess;
Name = fileName;
switch (storageStreamAccess)
{
case StorageStreamAccess.Read:
storage.Batch(accessor => fileHeader = accessor.ReadFile(fileName));
if (fileHeader.TotalSize == null)
{
throw new FileNotFoundException("File is not uploaded yet");
}
Metadata = fileHeader.Metadata;
Seek(0, SeekOrigin.Begin);
break;
case StorageStreamAccess.CreateAndWrite:
if (this.fileSystem == null)
throw new ArgumentNullException("fileSystem");
storage.Batch(accessor =>
{
using (fileSystem.DisableAllTriggersForCurrentThread())
{
fileSystem.Files.IndicateFileToDelete(fileName, null);
}
var putResult = accessor.PutFile(fileName, null, metadata);
fileSystem.Search.Index(fileName, metadata, putResult.Etag);
});
Metadata = metadata;
break;
default:
throw new ArgumentOutOfRangeException("storageStreamAccess", storageStreamAccess, "Unknown value");
}
}
