private bool ExecuteTasks()
{
bool foundWork = false;
transactionalStorage.Batch(actions =>
{
Task task = GetApplicableTask(actions);
if (task == null)
{
return;
}
context.UpdateFoundWork();
Log.Debug("Executing {0}", task);
foundWork = true;
context.CancellationToken.ThrowIfCancellationRequested();
try
{
task.Execute(context);
}
catch (Exception e)
{
Log.WarnException(
string.Format("Task {0} has failed and was deleted without completing any work", task),
e);
}
});
return(foundWork);
}
public async Task Execute()
{
while (true)
{
var read = await inputStream.ReadAsync(buffer);
TotalSizeRead += read;
if (read == 0) // nothing left to read
{
FileHash = IOExtensions.GetMD5Hex(md5Hasher.TransformFinalBlock());
headers["Content-MD5"] = FileHash;
storage.Batch(accessor =>
{
accessor.CompleteFileUpload(filename);
putTriggers.Apply(trigger => trigger.AfterUpload(filename, headers));
});
return; // task is done
}
ConcurrencyAwareExecutor.Execute(() => storage.Batch(accessor =>
{
var hashKey = accessor.InsertPage(buffer, read);
accessor.AssociatePage(filename, hashKey, pos, read);
putTriggers.Apply(trigger => trigger.OnUpload(filename, headers, hashKey, pos, read));
}));
md5Hasher.TransformBlock(buffer, 0, read);
pos++;
}
}
private static void when_there_are_multiple_map_results_for_multiple_indexes(ITransactionalStorage transactionalStorage)
{
transactionalStorage.Initialize(new DummyUuidGenerator());
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a",true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.True(actionsAccessor.Staleness.IsReduceStale("a"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("b"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("c"));
});
}
protected StorageStream(ITransactionalStorage transactionalStorage, string fileName,
StorageStreamAccess storageStreamAccess,
RavenJObject metadata, IndexStorage indexStorage, StorageOperationsTask operations)
{
TransactionalStorage = transactionalStorage;
StorageStreamAccess = storageStreamAccess;
Name = fileName;
switch (storageStreamAccess)
{
case StorageStreamAccess.Read:
TransactionalStorage.Batch(accessor => fileHeader = accessor.ReadFile(fileName));
if (fileHeader.TotalSize == null)
{
throw new FileNotFoundException("File is not uploaded yet");
}
Metadata = fileHeader.Metadata;
Seek(0, SeekOrigin.Begin);
break;
case StorageStreamAccess.CreateAndWrite:
TransactionalStorage.Batch(accessor =>
{
operations.IndicateFileToDelete(fileName);
accessor.PutFile(fileName, null, metadata);
indexStorage.Index(fileName, metadata);
});
Metadata = metadata;
break;
default:
throw new ArgumentOutOfRangeException("storageStreamAccess", storageStreamAccess, "Unknown value");
}
}
private static void when_there_are_multiple_map_results_and_we_ask_for_results(ITransactionalStorage transactionalStorage)
{
transactionalStorage.Initialize(new DummyUuidGenerator());
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.Equal(2, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("a", Guid.Empty, false, 100).Count());
Assert.Equal(2, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("b", Guid.Empty, false, 100).Count());
Assert.Equal(2, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("c", Guid.Empty, false, 100).Count());
});
}
private static void when_there_are_multiple_map_results_for_multiple_indexes(ITransactionalStorage transactionalStorage)
{
transactionalStorage.Initialize(new DummyUuidGenerator());
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.True(actionsAccessor.Staleness.IsReduceStale("a"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("b"));
Assert.True(actionsAccessor.Staleness.IsReduceStale("c"));
});
}
protected StorageStream(ITransactionalStorage transactionalStorage, string fileName,
StorageStreamAccess storageStreamAccess,
RavenJObject metadata, IndexStorage indexStorage, StorageOperationsTask operations)
{
TransactionalStorage = transactionalStorage;
StorageStreamAccess = storageStreamAccess;
Name = fileName;
switch (storageStreamAccess)
{
case StorageStreamAccess.Read:
TransactionalStorage.Batch(accessor => fileHeader = accessor.ReadFile(fileName));
if (fileHeader.TotalSize == null)
{
throw new FileNotFoundException("File is not uploaded yet");
}
Metadata = fileHeader.Metadata;
Seek(0, SeekOrigin.Begin);
break;
case StorageStreamAccess.CreateAndWrite:
TransactionalStorage.Batch(accessor =>
{
operations.IndicateFileToDelete(fileName);
accessor.PutFile(fileName, null, metadata);
indexStorage.Index(fileName, metadata);
});
Metadata = metadata;
break;
default:
throw new ArgumentOutOfRangeException("storageStreamAccess", storageStreamAccess, "Unknown value");
}
}
private static void when_there_are_updates_to_map_reduce_results(ITransactionalStorage transactionalStorage)
{
var dummyUuidGenerator = new DummyUuidGenerator();
transactionalStorage.Initialize(dummyUuidGenerator);
Guid a = Guid.Empty;
Guid b = Guid.Empty;
Guid c = Guid.Empty;
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
a = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
b = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
c = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("a", a, false, 100).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("b", b, false, 100).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("c", c, false, 100).Count());
});
}
public async Task Execute()
{
while (true)
{
var totalSizeRead = await inputStream.ReadAsync(buffer);
TotalSizeRead += totalSizeRead;
if (totalSizeRead == 0) // nothing left to read
{
storage.Batch(accessor => accessor.CompleteFileUpload(filename));
FileHash = IOExtensions.GetMD5Hex(md5Hasher.TransformFinalBlock());
return; // task is done
}
ConcurrencyAwareExecutor.Execute(() => storage.Batch(accessor =>
{
var hashKey = accessor.InsertPage(buffer, totalSizeRead);
accessor.AssociatePage(filename, hashKey, pos, totalSizeRead);
}));
md5Hasher.TransformBlock(buffer, 0, totalSizeRead);
pos++;
}
}
public async Task Execute()
{
while (true)
{
var read = await inputStream.ReadAsync(buffer).ConfigureAwait(false);
TotalSizeRead += read;
if (read == 0) // nothing left to read
{
FileHash = IOExtensions.GetAsHex(md5Hasher.TransformFinalBlock());
headers["Content-MD5"] = FileHash;
storage.Batch(accessor =>
{
accessor.CompleteFileUpload(filename);
putTriggers.Apply(trigger => trigger.AfterUpload(filename, headers));
});
return; // task is done
}
int retries = 50;
bool shouldRetry;
do
{
try
{
storage.Batch(accessor =>
{
var hashKey = accessor.InsertPage(buffer, read);
accessor.AssociatePage(filename, hashKey, pos, read);
putTriggers.Apply(trigger => trigger.OnUpload(filename, headers, hashKey, pos, read));
});
shouldRetry = false;
}
catch (ConcurrencyException)
{
if (retries-- > 0)
{
shouldRetry = true;
Thread.Sleep(50);
continue;
}
throw;
}
} while (shouldRetry);
md5Hasher.TransformBlock(buffer, 0, read);
pos++;
}
}
private void PersistSynchronizationState()
{
using (transactionalStorage.DisableBatchNesting())
{
transactionalStorage.Batch(
actions => actions.Lists.Set("Raven/Etag/Synchronization", type.ToString(), RavenJObject.FromObject(new
{
etag = GetEtagForPersistance()
}), UuidType.EtagSynchronization));
}
}
private void RemoveIndexAndCleanup(string name)
{
var index = GetIndexDefinition(name);
if (index == null)
return;
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.PrepareIndexForDeletion(index.IndexId);
accessor.Indexing.DeleteIndex(index.IndexId, CancellationToken.None);
});
RemoveIndex(index.IndexId);
}
private bool ExecuteTasks()
{
bool foundWork = false;
transactionalStorage.Batch(actions =>
{
int tasks;
Task task = actions.Tasks.GetMergedTask(out tasks);
if (task == null)
{
return;
}
log.Debug("Executing {0}", task);
foundWork = true;
try
{
task.Execute(context);
}
catch (Exception e)
{
log.WarnException(
string.Format("Task {0} has failed and was deleted without completing any work", task),
e);
}
});
return(foundWork);
}
private void ReadIndexesFromCatalog(IEnumerable<AbstractViewGenerator> compiledGenerators, ITransactionalStorage transactionalStorage)
{
foreach (var generator in compiledGenerators)
{
var copy = generator;
var displayNameAtt = TypeDescriptor.GetAttributes(copy)
.OfType<DisplayNameAttribute>()
.FirstOrDefault();
var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name;
transactionalStorage.Batch(actions =>
{
if (actions.Indexing.GetIndexesStats().Any(x => x.Name == name))
return;
actions.Indexing.AddIndex(name, copy.ReduceDefinition != null);
});
var indexDefinition = new IndexDefinition
{
Name = name,
Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName,
// need to supply this so the index storage will create map/reduce index
Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName,
Indexes = generator.Indexes,
Stores = generator.Stores,
IsCompiled = true
};
indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy);
indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition);
}
}
private void ReadIndexesFromCatalog(IEnumerable <AbstractViewGenerator> compiledGenerators, ITransactionalStorage transactionalStorage)
{
foreach (var generator in compiledGenerators)
{
var copy = generator;
var displayNameAtt = TypeDescriptor.GetAttributes(copy)
.OfType <DisplayNameAttribute>()
.FirstOrDefault();
var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name;
transactionalStorage.Batch(actions =>
{
if (actions.Indexing.GetIndexesStats().Any(x => x.Name == name))
{
return;
}
actions.Indexing.AddIndex(name, copy.ReduceDefinition != null);
});
var indexDefinition = new IndexDefinition
{
Name = name,
Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName,
// need to supply this so the index storage will create map/reduce index
Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName,
Indexes = generator.Indexes,
Stores = generator.Stores,
IsCompiled = true
};
indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy);
indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition);
}
}
private SynchronizationConfig GetSynchronizationConfig()
{
try
{
SynchronizationConfig configDoc = null;
storage.Batch(
accessor =>
{
if (accessor.ConfigExists(SynchronizationConstants.RavenSynchronizationConfig) == false)
{
return;
}
configDoc = accessor.GetConfig(SynchronizationConstants.RavenSynchronizationConfig).JsonDeserialization <SynchronizationConfig>();
});
return(configDoc);
}
catch (Exception e)
{
Log.Warn("Could not deserialize a synchronization config", e);
return(null);
}
}
public IndexDefinitionStorage(
ITransactionalStorage transactionalStorage,
string path,
IEnumerable<AbstractViewGenerator> compiledGenerators,
AbstractDynamicCompilationExtension[] extensions)
{
this.extensions = extensions;// this is used later in the ctor, so it must appears first
this.path = Path.Combine(path, IndexDefDir);
if (Directory.Exists(this.path) == false)
Directory.CreateDirectory(this.path);
this.extensions = extensions;
foreach (var index in Directory.GetFiles(this.path, "*.index"))
{
try
{
AddAndCompileIndex(
MonoHttpUtility.UrlDecode(Path.GetFileNameWithoutExtension(index)),
JsonConvert.DeserializeObject<IndexDefinition>(File.ReadAllText(index), new JsonEnumConverter())
);
}
catch (Exception e)
{
logger.Warn("Could not compile index " + index + ", skipping bad index", e);
}
}
//compiled view generators always overwrite dynamic views
foreach (var generator in compiledGenerators)
{
var copy = generator;
var displayNameAtt = TypeDescriptor.GetAttributes(copy)
.OfType<DisplayNameAttribute>()
.FirstOrDefault();
var name = displayNameAtt != null ? displayNameAtt.DisplayName : copy.GetType().Name;
transactionalStorage.Batch(actions =>
{
if (actions.Indexing.GetIndexesStats().Any(x => x.Name == name))
return;
actions.Indexing.AddIndex(name);
});
var indexDefinition = new IndexDefinition
{
Map = "Compiled map function: " + generator.GetType().AssemblyQualifiedName,
// need to supply this so the index storage will create map/reduce index
Reduce = generator.ReduceDefinition == null ? null : "Compiled reduce function: " + generator.GetType().AssemblyQualifiedName,
Indexes = generator.Indexes,
Stores = generator.Stores,
IsCompiled = true
};
indexCache.AddOrUpdate(name, copy, (s, viewGenerator) => copy);
indexDefinitions.AddOrUpdate(name, indexDefinition, (s1, definition) => indexDefinition);
}
}
public bool TimeoutExceeded(string fileName, ITransactionalStorage storage)
{
var result = false;
storage.Batch(accessor => result = TimeoutExceeded(fileName, accessor));
return result;
}
public static SynchronizationConfig GetOrDefault(ITransactionalStorage storage)
{
SynchronizationConfig result = null;
storage.Batch(accessor => result = GetOrDefault(accessor));
return(result ?? new SynchronizationConfig());
}
public bool TimeoutExceeded(string fileName, ITransactionalStorage storage)
{
var result = false;
storage.Batch(accessor => result = TimeoutExceeded(fileName, accessor));
return(result);
}
public static SynchronizationConfig GetOrDefault(ITransactionalStorage storage)
{
SynchronizationConfig result = null;
storage.Batch(accessor => result = GetOrDefault(accessor));
return result ?? new SynchronizationConfig();
}
public void Create(string fileName, ConflictItem conflict)
{
RavenJObject metadata = null;
storage.Batch(
accessor =>
{
metadata = accessor.GetFile(fileName, 0, 0).Metadata;
accessor.SetConfig(RavenFileNameHelper.ConflictConfigNameForFile(fileName), JsonExtensions.ToJObject(conflict));
metadata[SynchronizationConstants.RavenSynchronizationConflict] = true;
accessor.UpdateFileMetadata(fileName, metadata);
});
if (metadata != null)
{
index.Index(fileName, metadata);
}
}
protected StorageStream(RavenFileSystem fileSystem, ITransactionalStorage storage, string fileName, RavenJObject metadata, StorageStreamAccess storageStreamAccess)
{
this.fileSystem = fileSystem;
this.storage = storage;
StorageStreamAccess = storageStreamAccess;
Name = fileName;
switch (storageStreamAccess)
{
case StorageStreamAccess.Read:
storage.Batch(accessor => fileHeader = accessor.ReadFile(fileName));
if (fileHeader.TotalSize == null)
{
throw new FileNotFoundException("File is not uploaded yet");
}
Metadata = fileHeader.Metadata;
Seek(0, SeekOrigin.Begin);
break;
case StorageStreamAccess.CreateAndWrite:
if (this.fileSystem == null)
{
throw new ArgumentNullException("fileSystem");
}
storage.Batch(accessor =>
{
using (fileSystem.DisableAllTriggersForCurrentThread())
{
fileSystem.Files.IndicateFileToDelete(fileName, null);
}
var putResult = accessor.PutFile(fileName, null, metadata);
fileSystem.Search.Index(fileName, metadata, putResult.Etag);
});
Metadata = metadata;
break;
default:
throw new ArgumentOutOfRangeException("storageStreamAccess", storageStreamAccess, "Unknown value");
}
}
private async Task <IEnumerable <Task <SynchronizationReport> > > SynchronizeDestinationAsync(SynchronizationDestination destination, bool forceSyncingAll)
{
ICredentials credentials = null;
if (string.IsNullOrEmpty(destination.Username) == false)
{
credentials = string.IsNullOrEmpty(destination.Domain)
? new NetworkCredential(destination.Username, destination.Password)
: new NetworkCredential(destination.Username, destination.Password, destination.Domain);
}
var synchronizationServerClient = new SynchronizationServerClient(destination.ServerUrl, destination.FileSystem, destination.ApiKey, credentials);
var lastETag = await synchronizationServerClient.GetLastSynchronizationFromAsync(storage.Id).ConfigureAwait(false);
var activeTasks = synchronizationQueue.Active;
var filesNeedConfirmation = GetSyncingConfigurations(destination).Where(sync => activeTasks.All(x => x.FileName != sync.FileName)).ToList();
var confirmations = await ConfirmPushedFiles(filesNeedConfirmation, synchronizationServerClient).ConfigureAwait(false);
var needSyncingAgain = new List <FileHeader>();
foreach (var confirmation in confirmations)
{
if (confirmation.Status == FileStatus.Safe)
{
Log.Debug("Destination server {0} said that file '{1}' is safe", destination, confirmation.FileName);
RemoveSyncingConfiguration(confirmation.FileName, destination.Url);
}
else
{
storage.Batch(accessor =>
{
var fileHeader = accessor.ReadFile(confirmation.FileName);
if (fileHeader != null)
{
needSyncingAgain.Add(fileHeader);
Log.Debug("Destination server {0} said that file '{1}' is {2}.", destination, confirmation.FileName, confirmation.Status);
}
});
}
}
if (synchronizationQueue.NumberOfPendingSynchronizationsFor(destination.Url) < AvailableSynchronizationRequestsTo(destination.Url))
{
await EnqueueMissingUpdatesAsync(synchronizationServerClient, lastETag, needSyncingAgain).ConfigureAwait(false);
}
return(SynchronizePendingFilesAsync(synchronizationServerClient, forceSyncingAll));
}
public Stream GetContentForReading(string sigName)
{
SignatureReadOnlyStream signatureStream = null;
_storage.Batch(
accessor =>
{
var signatureLevel = GetSignatureLevel(sigName, accessor);
if (signatureLevel != null)
{
signatureStream = new SignatureReadOnlyStream(_storage, signatureLevel.Id, signatureLevel.Level);
}
else
{
throw new FileNotFoundException(sigName + " not found in the repo");
}
});
signatureStream.Position = 0;
return(signatureStream);
}
public void StorageStream_should_write_to_storage_by_64kB_pages()
{
using (var stream = StorageStream.CreatingNewAndWritting(fs, "file", new RavenJObject()))
{
var buffer = new byte[StorageConstants.MaxPageSize];
new Random().NextBytes(buffer);
stream.Write(buffer, 0, 32768);
stream.Write(buffer, 32767, 32768);
stream.Write(buffer, 0, 1);
}
FileAndPagesInformation fileAndPages = null;
transactionalStorage.Batch(accessor => fileAndPages = accessor.GetFile("file", 0, 10));
Assert.Equal(2, fileAndPages.Pages.Count);
Assert.Equal(StorageConstants.MaxPageSize, fileAndPages.Pages[0].Size);
Assert.Equal(1, fileAndPages.Pages[1].Size);
}
protected StorageStream(RavenFileSystem fileSystem, ITransactionalStorage storage, string fileName, RavenJObject metadata, StorageStreamAccess storageStreamAccess)
{
this.fileSystem = fileSystem;
this.storage = storage;
StorageStreamAccess = storageStreamAccess;
Name = fileName;
switch (storageStreamAccess)
{
case StorageStreamAccess.Read:
storage.Batch(accessor => fileHeader = accessor.ReadFile(fileName));
if (fileHeader.TotalSize == null)
{
throw new FileNotFoundException("File is not uploaded yet");
}
Metadata = fileHeader.Metadata;
Seek(0, SeekOrigin.Begin);
break;
case StorageStreamAccess.CreateAndWrite:
if (this.fileSystem == null)
throw new ArgumentNullException("fileSystem");
storage.Batch(accessor =>
{
using (fileSystem.DisableAllTriggersForCurrentThread())
{
fileSystem.Files.IndicateFileToDelete(fileName, null);
}
var putResult = accessor.PutFile(fileName, null, metadata);
fileSystem.Search.Index(fileName, metadata, putResult.Etag);
});
Metadata = metadata;
break;
default:
throw new ArgumentOutOfRangeException("storageStreamAccess", storageStreamAccess, "Unknown value");
}
}
private long GetNextHi()
{
long result = 0;
storage.Batch(
accessor =>
{
accessor.TryGetConfigurationValue(SynchronizationConstants.RavenSynchronizationVersionHiLo, out result);
result++;
accessor.SetConfigurationValue(SynchronizationConstants.RavenSynchronizationVersionHiLo, result);
});
return(result);
}
private RavenJObject GetMetadata(string fileName)
{
try
{
FileAndPagesInformation fileAndPages = null;
storage.Batch(accessor => fileAndPages = accessor.GetFile(fileName, 0, 0));
return(fileAndPages.Metadata);
}
catch (FileNotFoundException)
{
return(new RavenJObject());
}
}
private void MovePageFrame(long offset)
{
offset = Math.Min(Length, offset);
if (offset < currentPageFrameOffset || fileAndPages == null)
{
storage.Batch(accessor => fileAndPages = accessor.GetFile(Name, 0, PagesBatchSize));
currentPageFrameOffset = 0;
}
while (currentPageFrameOffset + CurrentPageFrameSize - 1 < offset)
{
var lastPageFrameSize = CurrentPageFrameSize;
var nextPageIndex = fileAndPages.Start + fileAndPages.Pages.Count;
storage.Batch(accessor => fileAndPages = accessor.GetFile(Name, nextPageIndex, PagesBatchSize));
if (fileAndPages.Pages.Count < 1)
{
fileAndPages.Start = 0;
break;
}
currentPageFrameOffset += lastPageFrameSize;
}
currentOffset = offset;
}
public override int Read(byte[] buffer, int offset, int count)
{
var readBytes = 0;
storage.Batch(
accessor => accessor.GetSignatureStream(id, level, sigContent =>
{
sigContent.Position = Position;
readBytes = sigContent.Read(buffer, offset, count);
Position = sigContent.Position;
}));
return(readBytes);
}
public long GetNextValue(string name)
{
long result = 1;
storage.Batch(
accessor =>
{
var sequenceName = SequenceName(name);
accessor.TryGetConfigurationValue(sequenceName, out result);
result++;
accessor.SetConfigurationValue(sequenceName, result);
});
return(result);
}
public static bool TryToCreateTransactionalStorage(InMemoryRavenConfiguration ravenConfiguration,
bool hasCompression, EncryptionConfiguration encryption, out ITransactionalStorage storage)
{
storage = null;
if (File.Exists(Path.Combine(ravenConfiguration.DataDirectory, Voron.Impl.Constants.DatabaseFilename)))
{
storage = ravenConfiguration.CreateTransactionalStorage(InMemoryRavenConfiguration.VoronTypeName, () => { }, () => { });
}
else if (File.Exists(Path.Combine(ravenConfiguration.DataDirectory, "Data")))
{
storage = ravenConfiguration.CreateTransactionalStorage(InMemoryRavenConfiguration.EsentTypeName, () => { }, () => { });
}
if (storage == null)
{
return(false);
}
var orderedPartCollection = new OrderedPartCollection <AbstractDocumentCodec>();
if (encryption != null)
{
var documentEncryption = new DocumentEncryption();
documentEncryption.SetSettings(new EncryptionSettings(encryption.EncryptionKey, encryption.SymmetricAlgorithmType,
encryption.EncryptIndexes, encryption.PreferedEncryptionKeyBitsSize));
orderedPartCollection.Add(documentEncryption);
}
if (hasCompression)
{
orderedPartCollection.Add(new DocumentCompression());
}
uuidGenerator = new SequentialUuidGenerator();
storage.Initialize(uuidGenerator, orderedPartCollection);
/*
* Added configuration steps
*/
storage.Batch(actions =>
{
var nextIdentityValue = actions.General.GetNextIdentityValue("Raven/Etag");
uuidGenerator.EtagBase = nextIdentityValue;
});
return(true);
}
protected SynchronizationWorkItem(string fileName, string sourceServerUrl, ITransactionalStorage storage)
{
Storage = storage;
FileName = fileName;
FileAndPagesInformation fileAndPages = null;
Storage.Batch(accessor => fileAndPages = accessor.GetFile(fileName, 0, 0));
FileMetadata = fileAndPages.Metadata;
FileSystemInfo = new FileSystemInfo
{
Id = Storage.Id,
Url = sourceServerUrl
};
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(null, null);
}
protected SynchronizationWorkItem(string fileName, string sourceServerUrl, ITransactionalStorage storage)
{
Storage = storage;
FileName = fileName;
FileAndPagesInformation fileAndPages = null;
Storage.Batch(accessor => fileAndPages = accessor.GetFile(fileName, 0, 0));
FileMetadata = fileAndPages.Metadata;
ServerInfo = new ServerInfo
{
Id = Storage.Id,
FileSystemUrl = sourceServerUrl
};
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(null, null);
}
public void RenameFile(RenameFileOperation operation)
{
var configName = RavenFileNameHelper.RenameOperationConfigNameForFile(operation.Name);
notificationPublisher.Publish(new FileChangeNotification
{
File = FilePathTools.Cannoicalise(operation.Name),
Action = FileChangeAction.Renaming
});
storage.Batch(accessor =>
{
var previousRenameTombstone = accessor.ReadFile(operation.Rename);
if (previousRenameTombstone != null &&
previousRenameTombstone.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if there is a tombstone delete it
accessor.Delete(previousRenameTombstone.FullPath);
}
accessor.RenameFile(operation.Name, operation.Rename, true);
accessor.UpdateFileMetadata(operation.Rename, operation.MetadataAfterOperation);
// copy renaming file metadata and set special markers
var tombstoneMetadata = new RavenJObject(operation.MetadataAfterOperation).WithRenameMarkers(operation.Rename);
accessor.PutFile(operation.Name, 0, tombstoneMetadata, true); // put rename tombstone
accessor.DeleteConfig(configName);
search.Delete(operation.Name);
search.Index(operation.Rename, operation.MetadataAfterOperation);
});
notificationPublisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
notificationPublisher.Publish(new FileChangeNotification
{
File = FilePathTools.Cannoicalise(operation.Rename),
Action = FileChangeAction.Renamed
});
}
public void Execute()
{
while (context.DoWork)
{
var foundWork = false;
Task task = null;
try
{
int tasks = 0;
transactionalStorage.Batch(actions =>
{
task = actions.Tasks.GetMergedTask(out tasks);
if (task == null)
{
return;
}
log.DebugFormat("Executing {0}", task);
foundWork = true;
try
{
task.Execute(context);
}
catch (Exception e)
{
log.WarnFormat(e, "Task {0} has failed and was deleted without completing any work", task);
}
});
context.PerformanceCounters.IncrementProcessedTask(tasks);
}
catch (Exception e)
{
log.Error("Failed to execute task: " + task, e);
}
if (foundWork == false)
{
context.WaitForWork(TimeSpan.FromSeconds(1));
}
}
}
private static void when_there_are_multiple_map_results_and_we_ask_for_results(ITransactionalStorage transactionalStorage)
{
transactionalStorage.Initialize(new DummyUuidGenerator());
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.Equal(2, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("a", Guid.Empty).Count());
Assert.Equal(2, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("b", Guid.Empty).Count());
Assert.Equal(2, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("c", Guid.Empty).Count());
});
}
private static void when_there_are_updates_to_map_reduce_results(ITransactionalStorage transactionalStorage)
{
var dummyUuidGenerator = new DummyUuidGenerator();
transactionalStorage.Initialize(dummyUuidGenerator);
Guid a = Guid.Empty;
Guid b = Guid.Empty;
Guid c = Guid.Empty;
transactionalStorage.Batch(accessor =>
{
accessor.Indexing.AddIndex("a", true);
accessor.Indexing.AddIndex("b", true);
accessor.Indexing.AddIndex("c", true);
accessor.MappedResults.PutMappedResult("a", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
a = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("a", "a/2", "a", new RavenJObject(), MapReduceIndex.ComputeHash("a", "a"));
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
b = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("b", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("b", "a"));
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
c = dummyUuidGenerator.CreateSequentialUuid();
accessor.MappedResults.PutMappedResult("c", "a/1", "a", new RavenJObject(), MapReduceIndex.ComputeHash("c", "a"));
});
transactionalStorage.Batch(actionsAccessor =>
{
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("a", a).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("b", b).Count());
Assert.Equal(1, actionsAccessor.MappedResults.GetMappedResultsReduceKeysAfter("c", c).Count());
});
}
public void CanStoreValues()
{
transactionalStorage.Batch(actions =>
{
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "123", "1", RavenJObject.Parse("{'a': 'abc'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "324", "2", RavenJObject.Parse("{'a': 'def'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "2"));
actions.MappedResults.PutMappedResult("CommentCountsByBlog", "321", "1", RavenJObject.Parse("{'a': 'ijg'}"), MapReduceIndex.ComputeHash("CommentCountsByBlog", "1"));
});
}