public void Ctor_and_dispose()
{
using (var tested = new SigGenerator())
{
Assert.NotNull(tested);
}
}
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState recievedTransportState = null)
{
this.Name = name;
this.systemConfiguration = systemConfiguration;
storage = CreateTransactionalStorage(systemConfiguration);
search = new IndexStorage(systemConfiguration.FileSystem.IndexStoragePath, systemConfiguration.Settings);
sigGenerator = new SigGenerator();
var replicationHiLo = new SynchronizationHiLo(storage);
var sequenceActions = new SequenceActions(storage);
transportState = recievedTransportState ?? new TransportState();
notificationPublisher = new NotificationPublisher(transportState);
fileLockManager = new FileLockManager();
storage.Initialize();
search.Initialize();
var uuidGenerator = new UuidGenerator(sequenceActions);
historian = new Historian(storage, replicationHiLo, uuidGenerator);
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog));
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher);
metricsCounters = new MetricsCountersManager();
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
public LocalRdcManager(ISignatureRepository signatureRepository, ITransactionalStorage transactionalStorage,
SigGenerator sigGenerator)
{
_signatureRepository = signatureRepository;
_transactionalStorage = transactionalStorage;
_sigGenerator = sigGenerator;
}
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState receivedTransportState = null)
{
ExtensionsState = new AtomicDictionary<object>();
Name = name;
this.systemConfiguration = systemConfiguration;
systemConfiguration.Container.SatisfyImportsOnce(this);
transportState = receivedTransportState ?? new TransportState();
storage = CreateTransactionalStorage(systemConfiguration);
sigGenerator = new SigGenerator();
fileLockManager = new FileLockManager();
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog));
notificationPublisher = new NotificationPublisher(transportState);
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
metricsCounters = new MetricsCountersManager();
search = new IndexStorage(name, systemConfiguration);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
storageOperationsTask = new StorageOperationsTask(storage, DeleteTriggers, search, notificationPublisher);
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
public void Generate_check()
{
IList<SignatureInfo> sourceSignatureInfos;
IList<SignatureInfo> seedSignatureInfos;
using (var sourceSignatureRepository = CreateSignatureRepositoryFor("test"))
using (var seedSignatureRepository = CreateSignatureRepositoryFor("test"))
{
using (var generator = new SigGenerator())
{
seedSignatureInfos = generator.GenerateSignatures(GetSeedStream(), "test", seedSignatureRepository);
}
var sourceStream = GetSourceStream();
using (var generator = new SigGenerator())
{
sourceSignatureInfos = generator.GenerateSignatures(sourceStream, "test", sourceSignatureRepository);
}
var sourceSize = sourceStream.Length;
using (var tested = new NeedListGenerator(sourceSignatureRepository, seedSignatureRepository))
{
var result = tested.CreateNeedsList(seedSignatureInfos.Last(), sourceSignatureInfos.Last());
Assert.NotNull(result);
Assert.Equal(0, sourceSize - result.Sum(x => Convert.ToInt32(x.BlockLength)));
}
}
}
public RavenFileSystem(InMemoryRavenConfiguration config, string name, TransportState receivedTransportState = null)
{
ExtensionsState = new AtomicDictionary<object>();
Name = name;
ResourceName = string.Concat(Abstractions.Data.Constants.FileSystem.UrlPrefix, "/", name);
configuration = config;
try
{
ValidateStorage();
configuration.Container.SatisfyImportsOnce(this);
transportState = receivedTransportState ?? new TransportState();
storage = CreateTransactionalStorage(configuration);
sigGenerator = new SigGenerator();
fileLockManager = new FileLockManager();
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(configuration.Catalog));
notificationPublisher = new NotificationPublisher(transportState);
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, configuration);
metricsCounters = new MetricsCountersManager();
search = new IndexStorage(name, configuration);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
TimerManager = new ResourceTimerManager();
Tasks = new TaskActions(this, Log);
Files = new FileActions(this, Log);
Synchronizations = new SynchronizationActions(this, Log);
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
catch (Exception e)
{
Log.ErrorException(string.Format("Could not create file system '{0}'", Name ?? "unknown name"), e);
try
{
Dispose();
}
catch (Exception ex)
{
Log.FatalException("Failed to dispose when already getting an error in file system ctor", ex);
}
throw;
}
}
public SynchronizationTask(ITransactionalStorage storage, SigGenerator sigGenerator, NotificationPublisher publisher, InMemoryRavenConfiguration systemConfiguration)
{
this.storage = storage;
this.publisher = publisher;
this.systemConfiguration = systemConfiguration;
context = new SynchronizationTaskContext();
synchronizationQueue = new SynchronizationQueue();
synchronizationStrategy = new SynchronizationStrategy(storage, sigGenerator);
}
public void Should_be_the_same_signatures()
{
const int size = 1024 * 1024 * 5;
var randomStream = new RandomStream(size);
var buffer = new byte[size];
randomStream.Read(buffer, 0, size);
var stream = new MemoryStream(buffer);
var firstSigContentHashes = new List<string>();
using (var signatureRepository = new VolatileSignatureRepository("test"))
using (var rested = new SigGenerator())
{
var result = rested.GenerateSignatures(stream, "test", signatureRepository);
foreach (var signatureInfo in result)
{
using (var content = signatureRepository.GetContentForReading(signatureInfo.Name))
{
firstSigContentHashes.Add(content.GetMD5Hash());
}
}
}
stream.Position = 0;
var secondSigContentHashes = new List<string>();
using (var signatureRepository = new VolatileSignatureRepository("test"))
using (var rested = new SigGenerator())
{
var result = rested.GenerateSignatures(stream, "test", signatureRepository);
foreach (var signatureInfo in result)
{
using (var content = signatureRepository.GetContentForReading(signatureInfo.Name))
{
secondSigContentHashes.Add(content.GetMD5Hash());
}
}
}
Assert.Equal(firstSigContentHashes.Count, secondSigContentHashes.Count);
for (var i = 0; i < firstSigContentHashes.Count; i++)
{
Assert.Equal(firstSigContentHashes[i], secondSigContentHashes[i]);
}
}
public SynchronizationTask(ITransactionalStorage storage, SigGenerator sigGenerator, NotificationPublisher publisher,
InMemoryRavenConfiguration systemConfiguration)
{
this.storage = storage;
this.publisher = publisher;
this.systemConfiguration = systemConfiguration;
this.timer = Observable.Interval(systemConfiguration.FileSystem.MaximumSynchronizationInterval);
synchronizationQueue = new SynchronizationQueue();
synchronizationStrategy = new SynchronizationStrategy(storage, sigGenerator);
LastSuccessfulSynchronizationTime = DateTime.MinValue;
InitializeTimer();
}
public void Generate_check()
{
using (var signatureRepository = new VolatileSignatureRepository("test"))
using (var rested = new SigGenerator())
{
var result = rested.GenerateSignatures(_stream, "test", signatureRepository);
Assert.Equal(2, result.Count);
using (var content = signatureRepository.GetContentForReading(result[0].Name))
{
Assert.Equal("91b64180c75ef27213398979cc20bfb7", content.GetMD5Hash());
}
using (var content = signatureRepository.GetContentForReading(result[1].Name))
{
Assert.Equal("9fe9d408aed35769e25ece3a56f2d12f", content.GetMD5Hash());
}
}
}
public void Should_be_only_work_with_greater_etag_in_pending_queue()
{
using (var sigGenerator = new SigGenerator())
{
transactionalStorage.Batch(accessor => accessor.PutFile(FileName, 0, EmptyETagMetadata));
queue.EnqueueSynchronization(Destination, new ContentUpdateWorkItem(FileName, "http://localhost:12345", transactionalStorage, sigGenerator));
Assert.Equal(1, queue.Pending.Count());
var greaterGuid = Guid.NewGuid();
transactionalStorage.Batch(accessor => accessor.UpdateFileMetadata(FileName, new RavenJObject().WithETag(greaterGuid)));
queue.EnqueueSynchronization(Destination, new ContentUpdateWorkItem(FileName, "http://localhost:12345", transactionalStorage, new SigGenerator()));
Assert.Equal(1, queue.Pending.Count());
Assert.Equal(greaterGuid, queue.Pending.ToArray()[0].FileETag);
}
}
public void Should_be_only_work_with_greater_etag_in_pending_queue()
{
using (var sigGenerator = new SigGenerator())
{
transactionalStorage.Batch(accessor => accessor.PutFile(FileName, 0, new RavenJObject()));
queue.EnqueueSynchronization(Destination, new ContentUpdateWorkItem(FileName, "http://localhost:12345", transactionalStorage, sigGenerator));
Assert.Equal(1, queue.Pending.Count());
Etag newerEtag = null;
transactionalStorage.Batch(accessor => newerEtag = accessor.UpdateFileMetadata(FileName, new RavenJObject(), null).Etag);
queue.EnqueueSynchronization(Destination, new ContentUpdateWorkItem(FileName, "http://localhost:12345", transactionalStorage, new SigGenerator()));
Assert.Equal(1, queue.Pending.Count());
Assert.True(newerEtag.CompareTo(queue.Pending.ToArray()[0].FileETag) == 0);
}
}
public void Synchronize_file_with_different_beginning()
{
const int size = 5000;
var differenceChunk = new MemoryStream();
var sw = new StreamWriter(differenceChunk);
sw.Write("Coconut is Stupid");
sw.Flush();
var sourceContent = PrepareSourceStream(size);
sourceContent.Position = 0;
var seedContent = new CombinedStream(differenceChunk, sourceContent);
using (var sourceSignatureRepository = CreateSignatureRepositoryFor("test2"))
using (var seedSignatureRepository = CreateSignatureRepositoryFor("test1"))
{
IList<SignatureInfo> seedSignatureInfos;
using (var generator = new SigGenerator())
{
seedContent.Seek(0, SeekOrigin.Begin);
seedSignatureInfos = generator.GenerateSignatures(seedContent, "test1", seedSignatureRepository);
}
IList<SignatureInfo> sourceSignatureInfos;
using (var generator = new SigGenerator())
{
sourceContent.Seek(0, SeekOrigin.Begin);
sourceSignatureInfos = generator.GenerateSignatures(sourceContent, "test2", sourceSignatureRepository);
}
var sourceSize = sourceContent.Length;
using (var tested = new NeedListGenerator(seedSignatureRepository, sourceSignatureRepository))
{
var result = tested.CreateNeedsList(seedSignatureInfos.Last(), sourceSignatureInfos.Last());
Assert.NotNull(result);
Assert.Equal(2, result.Count);
Assert.Equal(0, sourceSize - result.Sum(x => Convert.ToInt32(x.BlockLength)));
}
}
}
public void Should_detect_that_different_work_is_being_perfomed()
{
using (var sigGenerator = new SigGenerator())
{
transactionalStorage.Batch(accessor => accessor.PutFile(FileName, 0, EmptyETagMetadata));
var contentUpdateWorkItem = new ContentUpdateWorkItem(FileName, "http://localhost:12345", transactionalStorage, sigGenerator);
queue.EnqueueSynchronization(Destination, contentUpdateWorkItem);
queue.SynchronizationStarted(contentUpdateWorkItem, Destination);
Assert.True(queue.IsDifferentWorkForTheSameFileBeingPerformed(new RenameWorkItem(FileName, "rename.txt", "http://localhost:12345", transactionalStorage), Destination));
}
}
public ContentUpdateWorkItem(string file, string sourceServerUrl, ITransactionalStorage storage, SigGenerator sigGenerator) : base(file, sourceServerUrl, storage)
{
this.sigGenerator = sigGenerator;
}
public SynchronizationStrategy(ITransactionalStorage storage, SigGenerator sigGenerator)
{
this.storage = storage;
this.sigGenerator = sigGenerator;
}
public void Signatures_can_be_generated_on_the_same_repository()
{
const int size = 1024 * 1024 * 5;
var randomStream = new RandomStream(size);
var buffer = new byte[size];
randomStream.Read(buffer, 0, size);
var stream = new MemoryStream(buffer);
foreach (var fileName in new [] { "test", "content/test", "/content/test"})
{
using (var signatureRepository = new VolatileSignatureRepository(fileName))
using (var rested = new SigGenerator())
{
var signatures = signatureRepository.GetByFileName();
Assert.Equal(0, signatures.Count());
stream.Position = 0;
var result = rested.GenerateSignatures(stream, fileName, signatureRepository);
signatures = signatureRepository.GetByFileName();
Assert.Equal(2, signatures.Count());
stream.Position = 0;
result = rested.GenerateSignatures(stream, fileName, signatureRepository);
signatures = signatureRepository.GetByFileName();
Assert.Equal(2, signatures.Count());
}
}
}
public void Synchronize_file_with_different_sizes_and_completely_different_content()
{
var seedBuffer = new byte[1024 * 1024 * 2]; // 2 MB
new Random().NextBytes(seedBuffer);
var seedContent = new MemoryStream(seedBuffer);
var smallerBuffer = new byte[1024 * 1024];
new Random().NextBytes(smallerBuffer);
var changedContent = new MemoryStream(smallerBuffer);
using (var seedSignatureRepository = CreateSignatureRepositoryFor("seed"))
using (var sourceSignatureRepository = CreateSignatureRepositoryFor("source"))
{
IList<SignatureInfo> seedSignatureInfos;
using (var generator = new SigGenerator())
{
seedContent.Seek(0, SeekOrigin.Begin);
seedSignatureInfos = generator.GenerateSignatures(seedContent, "seed", seedSignatureRepository);
}
IList<SignatureInfo> sourceSignatureInfos;
using (var generator = new SigGenerator())
{
changedContent.Seek(0, SeekOrigin.Begin);
sourceSignatureInfos = generator.GenerateSignatures(changedContent, "source", sourceSignatureRepository);
}
var sourceSize = changedContent.Length;
using (var tested = new NeedListGenerator(seedSignatureRepository, sourceSignatureRepository))
{
var result = tested.CreateNeedsList(seedSignatureInfos.Last(), sourceSignatureInfos.Last());
Assert.NotNull(result);
Assert.Equal(0, sourceSize - result.Sum(x => Convert.ToInt32(x.BlockLength)));
}
}
}
