public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState recievedTransportState = null)
{
this.Name = name;
this.systemConfiguration = systemConfiguration;
var storageType = systemConfiguration.FileSystem.DefaultStorageTypeName;
storage = CreateTransactionalStorage(storageType, systemConfiguration);
search = new IndexStorage(systemConfiguration.FileSystem.IndexStoragePath, systemConfiguration.Settings);
sigGenerator = new SigGenerator();
var replicationHiLo = new SynchronizationHiLo(storage);
var sequenceActions = new SequenceActions(storage);
transportState = recievedTransportState ?? new TransportState();
notificationPublisher = new NotificationPublisher(transportState);
fileLockManager = new FileLockManager();
storage.Initialize();
search.Initialize();
var uuidGenerator = new UuidGenerator(sequenceActions);
historian = new Historian(storage, replicationHiLo, uuidGenerator);
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog));
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher);
metricsCounters = new MetricsCountersManager();
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
private void FinishSynchronization(string fileName, SynchronizationReport report, ServerInfo sourceServer, Guid sourceFileETag)
{
try
{
// we want to execute those operation in a single batch but we also have to ensure that
// Raven/Synchronization/Sources/sourceServerId config is modified only by one finishing synchronization at the same time
SynchronizationFinishLocks.GetOrAdd(sourceServer.Id, new ReaderWriterLockSlim()).EnterWriteLock();
SynchronizationTask.IncomingSynchronizationFinished(fileName, sourceServer, sourceFileETag);
Storage.Batch(accessor =>
{
SaveSynchronizationReport(fileName, accessor, report);
FileLockManager.UnlockByDeletingSyncConfiguration(fileName, accessor);
if (report.Exception == null)
{
SaveSynchronizationSourceInformation(sourceServer, sourceFileETag, accessor);
}
});
}
catch (Exception ex)
{
Log.ErrorException(
string.Format("Failed to finish synchronization of a file '{0}' from {1}", fileName, sourceServer), ex);
}
finally
{
SynchronizationFinishLocks.GetOrAdd(sourceServer.Id, new ReaderWriterLockSlim()).ExitWriteLock();
}
}
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, string name, TransportState receivedTransportState = null)
{
ExtensionsState = new AtomicDictionary <object>();
Name = name;
this.systemConfiguration = systemConfiguration;
systemConfiguration.Container.SatisfyImportsOnce(this);
transportState = receivedTransportState ?? new TransportState();
storage = CreateTransactionalStorage(systemConfiguration);
sigGenerator = new SigGenerator();
fileLockManager = new FileLockManager();
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(systemConfiguration.Catalog));
notificationPublisher = new NotificationPublisher(transportState);
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
metricsCounters = new MetricsCountersManager();
search = new IndexStorage(name, systemConfiguration);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
storageOperationsTask = new StorageOperationsTask(storage, DeleteTriggers, search, notificationPublisher);
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
public RavenFileSystem(InMemoryRavenConfiguration config, string name, TransportState receivedTransportState = null)
{
ExtensionsState = new AtomicDictionary <object>();
Name = name;
ResourceName = string.Concat(Constants.FileSystem.UrlPrefix, "/", name);
configuration = config;
try
{
ValidateStorage();
configuration.Container.SatisfyImportsOnce(this);
transportState = receivedTransportState ?? new TransportState();
storage = CreateTransactionalStorage(configuration);
sigGenerator = new SigGenerator();
fileLockManager = new FileLockManager();
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver(storage, new CompositionContainer(configuration.Catalog));
notificationPublisher = new NotificationPublisher(transportState);
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, configuration);
metricsCounters = new MetricsCountersManager();
search = new IndexStorage(name, configuration);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
TimerManager = new ResourceTimerManager();
Tasks = new TaskActions(this, Log);
Files = new FileActions(this, Log);
Synchronizations = new SynchronizationActions(this, Log);
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
catch (Exception e)
{
Log.ErrorException(string.Format("Could not create file system '{0}'", Name ?? "unknown name"), e);
try
{
Dispose();
}
catch (Exception ex)
{
Log.FatalException("Failed to dispose when already getting an error in file system ctor", ex);
}
throw;
}
}
public async Task <HttpResponseMessage> Start(string fileName)
{
var destination = await ReadJsonObjectAsync <SynchronizationDestination>();
Log.Debug("Starting to synchronize a file '{0}' to {1}", fileName, destination.FileSystemUrl);
var result = await SynchronizationTask.SynchronizeFileToAsync(fileName, destination);
return(this.GetMessageWithObject(result, HttpStatusCode.OK));
}
public HttpResponseMessage ResolveConflict(string filename, ConflictResolutionStrategy strategy)
{
var canonicalFilename = FileHeader.Canonize(filename);
Log.Debug("Resolving conflict of a file '{0}' by using {1} strategy", filename, strategy);
switch (strategy)
{
case ConflictResolutionStrategy.CurrentVersion:
Storage.Batch(accessor =>
{
var localMetadata = accessor.GetFile(canonicalFilename, 0, 0).Metadata;
var conflict = accessor.GetConfigurationValue <ConflictItem>(RavenFileNameHelper.ConflictConfigNameForFile(canonicalFilename));
ConflictResolver.ApplyCurrentStrategy(canonicalFilename, conflict, localMetadata);
accessor.UpdateFileMetadata(canonicalFilename, localMetadata);
ConflictArtifactManager.Delete(canonicalFilename, accessor);
});
Publisher.Publish(new ConflictNotification
{
FileName = filename,
Status = ConflictStatus.Resolved
});
break;
case ConflictResolutionStrategy.RemoteVersion:
Storage.Batch(accessor =>
{
var localMetadata = accessor.GetFile(canonicalFilename, 0, 0).Metadata;
var conflict = accessor.GetConfig(RavenFileNameHelper.ConflictConfigNameForFile(canonicalFilename)).JsonDeserialization <ConflictItem>();
ConflictResolver.ApplyRemoteStrategy(canonicalFilename, conflict, localMetadata);
accessor.UpdateFileMetadata(canonicalFilename, localMetadata);
// ConflictArtifactManager.Delete(canonicalFilename, accessor); - intentionally not deleting, conflict item will be removed when a remote file is put
});
Task.Run(() => SynchronizationTask.SynchronizeDestinationsAsync(true));
break;
default:
throw new NotSupportedException(string.Format("{0} is not the valid strategy to resolve a conflict", strategy));
}
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
public async Task <HttpResponseMessage> Start(string filename)
{
var canonicalFilename = FileHeader.Canonize(filename);
var destination = await ReadJsonObjectAsync <SynchronizationDestination>();
Log.Debug("Starting to synchronize a file '{0}' to {1}", canonicalFilename, destination.Url);
var result = await SynchronizationTask.SynchronizeFileToAsync(canonicalFilename, destination);
return(GetMessageWithObject(result));
}
public async Task <HttpResponseMessage> ToDestinations(bool forceSyncingAll)
{
var tasks = SynchronizationTask.Execute(forceSyncingAll);
var result = new List <DestinationSyncResult>();
foreach (var task in tasks)
{
result.Add(await SynchronizationTask.CreateDestinationResult(task.Key, await task.Value));
}
return(GetMessageWithObject(result.ToArray()));
}
public RavenFileSystem(InMemoryRavenConfiguration systemConfiguration, TransportState transportState, string name)
{
this.Name = name;
this.systemConfiguration = systemConfiguration;
var storageType = systemConfiguration.DefaultFileSystemStorageTypeName;
if (string.Equals(InMemoryRavenConfiguration.VoronTypeName, storageType, StringComparison.OrdinalIgnoreCase) == false)
{
if (Directory.Exists(systemConfiguration.FileSystemDataDirectory) &&
Directory.EnumerateFileSystemEntries(systemConfiguration.FileSystemDataDirectory).Any())
{
throw new InvalidOperationException(
string.Format(
"We do not allow to run on a storage engine other then Voron, while we are in the early pre-release phase of RavenDB 3.0. You are currently running on {0}",
storageType));
}
Trace.WriteLine("Forcing filesystem to run on Voron - pre release behavior only, mind " + Path.GetFileName(Path.GetDirectoryName(systemConfiguration.FileSystemDataDirectory)));
storageType = InMemoryRavenConfiguration.VoronTypeName;
}
storage = CreateTransactionalStorage(storageType, systemConfiguration);
search = new IndexStorage(systemConfiguration.FileSystemIndexStoragePath, systemConfiguration.Settings);
sigGenerator = new SigGenerator();
var replicationHiLo = new SynchronizationHiLo(storage);
var sequenceActions = new SequenceActions(storage);
this.transportState = transportState;
notificationPublisher = new NotificationPublisher(transportState);
fileLockManager = new FileLockManager();
storage.Initialize();
search.Initialize();
var uuidGenerator = new UuidGenerator(sequenceActions);
historian = new Historian(storage, replicationHiLo, uuidGenerator);
BufferPool = new BufferPool(1024 * 1024 * 1024, 65 * 1024);
conflictArtifactManager = new ConflictArtifactManager(storage, search);
conflictDetector = new ConflictDetector();
conflictResolver = new ConflictResolver();
synchronizationTask = new SynchronizationTask(storage, sigGenerator, notificationPublisher, systemConfiguration);
storageOperationsTask = new StorageOperationsTask(storage, search, notificationPublisher);
metricsCounters = new MetricsCountersManager();
AppDomain.CurrentDomain.ProcessExit += ShouldDispose;
AppDomain.CurrentDomain.DomainUnload += ShouldDispose;
}
public HttpResponseMessage UpdateMetadata(string fileSystemName, string fileName)
{
bool isConflictResolved = false;
var canonicalFilename = FileHeader.Canonize(fileName);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
// REVIEW: (Oren) It works, but it seems to me it is not an scalable solution.
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
Log.Debug("Starting to update a metadata of file '{0}' with ETag {1} from {2} because of synchronization", fileName,
sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.MetadataUpdate);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.MetadataUpdate);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
var localMetadata = GetLocalMetadata(canonicalFilename);
var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Historian.UpdateLastModified(sourceMetadata);
Storage.Batch(accessor => accessor.UpdateFileMetadata(canonicalFilename, sourceMetadata));
Search.Index(canonicalFilename, sourceMetadata);
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
PublishFileNotification(fileName, FileChangeAction.Update);
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(
string.Format("Error was occurred during metadata synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
}
finally
{
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
}
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
if (report.Exception == null)
{
Log.Debug("Metadata of file '{0}' was synchronized successfully from {1}", fileName, sourceServerInfo);
}
return(this.GetMessageWithObject(report, HttpStatusCode.OK));
}
private void StartSynchronizeDestinationsInBackground()
{
Task.Factory.StartNew(async() => await SynchronizationTask.SynchronizeDestinationsAsync(), CancellationToken.None,
TaskCreationOptions.None, TaskScheduler.Default);
}
public async Task <HttpResponseMessage> Put(string name, string uploadId = null)
{
try
{
FileSystem.MetricsCounters.FilesPerSecond.Mark();
name = RavenFileNameHelper.RavenPath(name);
var headers = this.GetFilteredMetadataFromHeaders(InnerHeaders);
Historian.UpdateLastModified(headers);
headers["Creation-Date"] = DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.fffffff", CultureInfo.InvariantCulture);
Historian.Update(name, headers);
SynchronizationTask.Cancel(name);
long?size = -1;
ConcurrencyAwareExecutor.Execute(() => Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(name, accessor, true);
StorageOperationsTask.IndicateFileToDelete(name);
var contentLength = Request.Content.Headers.ContentLength;
var sizeHeader = GetHeader("RavenFS-size");
long sizeForParse;
if (contentLength == 0 || long.TryParse(sizeHeader, out sizeForParse) == false)
{
size = contentLength;
if (Request.Headers.TransferEncodingChunked ?? false)
{
size = null;
}
}
else
{
size = sizeForParse;
}
accessor.PutFile(name, size, headers);
Search.Index(name, headers);
}));
log.Debug("Inserted a new file '{0}' with ETag {1}", name, headers.Value <Guid>(Constants.MetadataEtagField));
using (var contentStream = await Request.Content.ReadAsStreamAsync())
using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, contentStream, name))
{
await readFileToDatabase.Execute();
if (readFileToDatabase.TotalSizeRead != size)
{
Storage.Batch(accessor => { StorageOperationsTask.IndicateFileToDelete(name); });
throw new HttpResponseException(HttpStatusCode.BadRequest);
}
Historian.UpdateLastModified(headers); // update with the final file size
log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name);
headers["Content-MD5"] = readFileToDatabase.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(name, headers));
int totalSizeRead = readFileToDatabase.TotalSizeRead;
headers["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture);
Search.Index(name, headers);
Publisher.Publish(new FileChangeNotification {
Action = FileChangeAction.Add, File = FilePathTools.Cannoicalise(name)
});
log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, headers.Value <Guid>(Constants.MetadataEtagField));
StartSynchronizeDestinationsInBackground();
}
}
catch (Exception ex)
{
if (uploadId != null)
{
Guid uploadIdentifier;
if (Guid.TryParse(uploadId, out uploadIdentifier))
{
Publisher.Publish(new CancellationNotification {
UploadId = uploadIdentifier, File = name
});
}
}
log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex);
var concurrencyException = ex as ConcurrencyException;
if (concurrencyException != null)
{
throw ConcurrencyResponseException(concurrencyException);
}
throw;
}
return(GetEmptyMessage(HttpStatusCode.Created));
}
public async Task <HttpResponseMessage> ToDestination(string destination, bool forceSyncingAll)
{
var result = await SynchronizationTask.SynchronizeDestinationAsync(destination + "/fs/" + this.FileSystemName, forceSyncingAll);
return(this.GetMessageWithObject(result, HttpStatusCode.OK));
}
public async Task <HttpResponseMessage> ToDestination(string destination, bool forceSyncingAll)
{
var result = await SynchronizationTask.SynchronizeDestinationAsync(destination + "/fs/" + FileSystemName, forceSyncingAll).ConfigureAwait(false);
return(GetMessageWithObject(result));
}
public async Task <HttpResponseMessage> ToDestinations(bool forceSyncingAll)
{
var result = await SynchronizationTask.Execute(forceSyncingAll);
return(GetMessageWithObject(result));
}
public async Task <HttpResponseMessage> ToDestinations(bool forceSyncingAll)
{
var result = await SynchronizationTask.SynchronizeDestinationsAsync(forceSyncingAll);
return(this.GetMessageWithObject(result, HttpStatusCode.OK));
}
public HttpResponseMessage Delete(string fileSystemName, string fileName)
{
var canonicalFilename = FileHeader.Canonize(fileName);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Delete);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Delete);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
var localMetadata = GetLocalMetadata(canonicalFilename);
if (localMetadata != null)
{
// REVIEW: Use InnerHeaders for consistency?
var sourceMetadata = GetFilteredMetadataFromHeaders(Request.Headers); // Request.Headers.FilterHeadersToObject();
bool isConflictResolved;
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(canonicalFilename);
var tombstoneMetadata = new RavenJObject
{
{
SynchronizationConstants.RavenSynchronizationHistory,
localMetadata[SynchronizationConstants.RavenSynchronizationHistory]
},
{
SynchronizationConstants.RavenSynchronizationVersion,
localMetadata[SynchronizationConstants.RavenSynchronizationVersion]
},
{
SynchronizationConstants.RavenSynchronizationSource,
localMetadata[SynchronizationConstants.RavenSynchronizationSource]
}
}.WithDeleteMarker();
Historian.UpdateLastModified(tombstoneMetadata);
accessor.PutFile(canonicalFilename, 0, tombstoneMetadata, true);
});
PublishFileNotification(fileName, FileChangeAction.Delete);
}
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(string.Format("Error was occurred during deletion synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
}
finally
{
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
}
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (report.Exception == null)
{
Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceServerInfo);
}
return(this.GetMessageWithObject(report, HttpStatusCode.OK));
}
public async Task PutAsync(string name, Etag etag, RavenJObject metadata, Func <Task <Stream> > streamAsync, PutOperationOptions options)
{
try
{
FileSystem.MetricsCounters.FilesPerSecond.Mark();
name = FileHeader.Canonize(name);
if (options.PreserveTimestamps)
{
if (!metadata.ContainsKey(Constants.RavenCreationDate))
{
if (metadata.ContainsKey(Constants.CreationDate))
{
metadata[Constants.RavenCreationDate] = metadata[Constants.CreationDate];
}
else
{
throw new InvalidOperationException("Preserve Timestamps requires that the client includes the Raven-Creation-Date header.");
}
}
Historian.UpdateLastModified(metadata, options.LastModified.HasValue ? options.LastModified.Value : DateTimeOffset.UtcNow);
}
else
{
metadata[Constants.RavenCreationDate] = DateTimeOffset.UtcNow;
Historian.UpdateLastModified(metadata);
}
// TODO: To keep current filesystems working. We should remove when adding a new migration.
metadata[Constants.CreationDate] = metadata[Constants.RavenCreationDate].Value <DateTimeOffset>().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture);
Historian.Update(name, metadata);
long?size = -1;
Storage.Batch(accessor =>
{
FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name);
AssertPutOperationNotVetoed(name, metadata);
SynchronizationTask.Cancel(name);
var contentLength = options.ContentLength;
var contentSize = options.ContentSize;
if (contentLength == 0 || contentSize.HasValue == false)
{
size = contentLength;
if (options.TransferEncodingChunked)
{
size = null;
}
}
else
{
size = contentSize;
}
FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(name, metadata));
using (FileSystem.DisableAllTriggersForCurrentThread())
{
IndicateFileToDelete(name, etag);
}
var putResult = accessor.PutFile(name, size, metadata);
FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(name, size, metadata));
Search.Index(name, metadata, putResult.Etag);
});
Log.Debug("Inserted a new file '{0}' with ETag {1}", name, metadata.Value <string>(Constants.MetadataEtagField));
using (var contentStream = await streamAsync())
using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, FileSystem.PutTriggers, contentStream, name, metadata))
{
await readFileToDatabase.Execute();
if (readFileToDatabase.TotalSizeRead != size)
{
using (FileSystem.DisableAllTriggersForCurrentThread())
{
IndicateFileToDelete(name, null);
}
throw new HttpResponseException(HttpStatusCode.BadRequest);
}
if (options.PreserveTimestamps == false)
{
Historian.UpdateLastModified(metadata); // update with the final file size.
}
Log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name);
metadata["Content-MD5"] = readFileToDatabase.FileHash;
MetadataUpdateResult updateMetadata = null;
Storage.Batch(accessor => updateMetadata = accessor.UpdateFileMetadata(name, metadata, null));
int totalSizeRead = readFileToDatabase.TotalSizeRead;
metadata["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture);
Search.Index(name, metadata, updateMetadata.Etag);
Publisher.Publish(new FileChangeNotification {
Action = FileChangeAction.Add, File = name
});
Log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, metadata.Value <string>(Constants.MetadataEtagField));
}
}
catch (Exception ex)
{
Log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex);
throw;
}
}
public HttpResponseMessage Rename(string fileSystemName, string fileName, string rename)
{
bool isConflictResolved = false;
var canonicalFilename = FileHeader.Canonize(fileName);
var canonicalRename = FileHeader.Canonize(rename);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName,
rename, sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Rename);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Rename);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
var localMetadata = GetLocalMetadata(canonicalFilename);
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
StorageOperationsTask.RenameFile(new RenameFileOperation
{
FileSystem = FileSystem.Name,
Name = canonicalFilename,
Rename = canonicalRename,
MetadataAfterOperation = sourceMetadata.WithETag(sourceFileETag).DropRenameMarkers()
});
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(string.Format("Error was occurred during renaming synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
}
finally
{
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
}
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
if (report.Exception == null)
{
Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceServerInfo);
}
return(GetMessageWithObject(report));
}
public void StartSynchronizeDestinationsInBackground()
{
Task.Factory.StartNew(() => SynchronizationTask.Execute(false), CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default);
}
public async Task <HttpResponseMessage> MultipartProceed(string fileSystemName)
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault();
var canonicalFilename = FileHeader.Canonize(fileName);
var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate);
Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo);
StorageStream localFile = null;
var isNewFile = false;
var isConflictResolved = false;
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
var localMetadata = GetLocalMetadata(canonicalFilename);
if (localMetadata != null)
{
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
localFile = StorageStream.Reading(Storage, canonicalFilename);
}
else
{
isNewFile = true;
}
Historian.UpdateLastModified(sourceMetadata);
var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata);
var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);
Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName);
await Request.Content.ReadAsMultipartAsync(provider);
Log.Debug("Multipart content of a file '{0}' was processed/read", fileName);
report.BytesCopied = provider.BytesCopied;
report.BytesTransfered = provider.BytesTransfered;
report.NeedListLength = provider.NumberOfFileParts;
synchronizingFile.PreventUploadComplete = false;
synchronizingFile.Flush();
synchronizingFile.Dispose();
sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata));
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(canonicalFilename);
accessor.RenameFile(tempFileName, canonicalFilename);
Search.Delete(tempFileName);
Search.Index(canonicalFilename, sourceMetadata);
});
if (isNewFile)
{
Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName);
}
else
{
Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName);
}
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
}
}
finally
{
if (localFile != null)
{
localFile.Dispose();
}
}
if (report.Exception == null)
{
Log.Debug(
"File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
else
{
Log.WarnException(
string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo),
report.Exception);
}
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update);
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
return(GetMessageWithObject(report));
}
public async Task <HttpResponseMessage> Put(string name, string uploadId = null, bool preserveTimestamps = false)
{
try
{
FileSystem.MetricsCounters.FilesPerSecond.Mark();
name = FileHeader.Canonize(name);
var headers = this.GetFilteredMetadataFromHeaders(ReadInnerHeaders);
if (preserveTimestamps)
{
if (!headers.ContainsKey(Constants.RavenCreationDate))
{
if (headers.ContainsKey(Constants.CreationDate))
{
headers[Constants.RavenCreationDate] = headers[Constants.CreationDate];
}
else
{
throw new InvalidOperationException("Preserve Timestamps requires that the client includes the Raven-Creation-Date header.");
}
}
var lastModified = GetHeader(Constants.RavenLastModified);
if (lastModified != null)
{
DateTimeOffset when;
if (!DateTimeOffset.TryParse(lastModified, out when))
{
when = DateTimeOffset.UtcNow;
}
Historian.UpdateLastModified(headers, when);
}
else
{
Historian.UpdateLastModified(headers);
}
}
else
{
headers[Constants.RavenCreationDate] = DateTimeOffset.UtcNow;
Historian.UpdateLastModified(headers);
}
// TODO: To keep current filesystems working. We should remove when adding a new migration.
headers[Constants.CreationDate] = headers[Constants.RavenCreationDate].Value <DateTimeOffset>().ToString("yyyy-MM-ddTHH:mm:ss.fffffffZ", CultureInfo.InvariantCulture);
Historian.Update(name, headers);
SynchronizationTask.Cancel(name);
long?size = -1;
ConcurrencyAwareExecutor.Execute(() => Storage.Batch(accessor =>
{
AssertPutOperationNotVetoed(name, headers);
AssertFileIsNotBeingSynced(name, accessor, true);
var contentLength = Request.Content.Headers.ContentLength;
var sizeHeader = GetHeader("RavenFS-size");
long sizeForParse;
if (contentLength == 0 || long.TryParse(sizeHeader, out sizeForParse) == false)
{
size = contentLength;
if (Request.Headers.TransferEncodingChunked ?? false)
{
size = null;
}
}
else
{
size = sizeForParse;
}
FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(name, headers));
using (FileSystem.DisableAllTriggersForCurrentThread())
{
StorageOperationsTask.IndicateFileToDelete(name);
}
accessor.PutFile(name, size, headers);
FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(name, size, headers));
Search.Index(name, headers);
}));
log.Debug("Inserted a new file '{0}' with ETag {1}", name, headers.Value <Guid>(Constants.MetadataEtagField));
using (var contentStream = await Request.Content.ReadAsStreamAsync())
using (var readFileToDatabase = new ReadFileToDatabase(BufferPool, Storage, FileSystem.PutTriggers, contentStream, name, headers))
{
await readFileToDatabase.Execute();
if (readFileToDatabase.TotalSizeRead != size)
{
StorageOperationsTask.IndicateFileToDelete(name);
throw new HttpResponseException(HttpStatusCode.BadRequest);
}
if (!preserveTimestamps)
{
Historian.UpdateLastModified(headers); // update with the final file size.
}
log.Debug("File '{0}' was uploaded. Starting to update file metadata and indexes", name);
headers["Content-MD5"] = readFileToDatabase.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(name, headers));
int totalSizeRead = readFileToDatabase.TotalSizeRead;
headers["Content-Length"] = totalSizeRead.ToString(CultureInfo.InvariantCulture);
Search.Index(name, headers);
Publisher.Publish(new FileChangeNotification {
Action = FileChangeAction.Add, File = FilePathTools.Cannoicalise(name)
});
log.Debug("Updates of '{0}' metadata and indexes were finished. New file ETag is {1}", name, headers.Value <Guid>(Constants.MetadataEtagField));
StartSynchronizeDestinationsInBackground();
}
}
catch (Exception ex)
{
if (uploadId != null)
{
Guid uploadIdentifier;
if (Guid.TryParse(uploadId, out uploadIdentifier))
{
Publisher.Publish(new CancellationNotification {
UploadId = uploadIdentifier, File = name
});
}
}
log.WarnException(string.Format("Failed to upload a file '{0}'", name), ex);
var concurrencyException = ex as ConcurrencyException;
if (concurrencyException != null)
{
throw ConcurrencyResponseException(concurrencyException);
}
throw;
}
return(GetEmptyMessage(HttpStatusCode.Created));
}
