public void SynchronizingFileStream_should_write_to_storage_by_64kB_pages()
{
using (var stream = SynchronizingFileStream.CreatingOrOpeningAndWriting(
transactionalStorage, new MockIndexStorage(),
new StorageOperationsTask(transactionalStorage, new MockIndexStorage(), new EmptyNotificationsPublisher()),
"file", EmptyETagMetadata))
{
var buffer = new byte[StorageConstants.MaxPageSize];
new Random().NextBytes(buffer);
stream.Write(buffer, 0, 32768);
stream.Write(buffer, 32767, 32768);
stream.Write(buffer, 0, 1);
stream.PreventUploadComplete = false;
}
FileAndPagesInformation fileAndPages = null;
transactionalStorage.Batch(accessor => fileAndPages = accessor.GetFile("file", 0, 10));
Assert.Equal(2, fileAndPages.Pages.Count);
Assert.Equal(StorageConstants.MaxPageSize, fileAndPages.Pages[0].Size);
Assert.Equal(1, fileAndPages.Pages[1].Size);
}
public MultipartSyncStreamProvider(SynchronizingFileStream synchronizingFile, StorageStream localFile)
{
this.synchronizingFile = synchronizingFile;
this.localFile = localFile;
BytesTransfered = BytesCopied = NumberOfFileParts = 0;
}
public MultipartSyncStreamProvider(SynchronizingFileStream synchronizingFile, StorageStream localFile)
{
this.synchronizingFile = synchronizingFile;
this.localFile = localFile;
BytesTransfered = BytesCopied = NumberOfFileParts = 0;
}
public void SynchronizingFileStream_should_write_to_storage_by_64kB_pages()
{
using (var stream = SynchronizingFileStream.CreatingOrOpeningAndWriting(fs, "file", new RavenJObject()))
{
var buffer = new byte[StorageConstants.MaxPageSize];
new Random().NextBytes(buffer);
stream.Write(buffer, 0, 32768);
stream.Write(buffer, 32767, 32768);
stream.Write(buffer, 0, 1);
stream.PreventUploadComplete = false;
}
FileAndPagesInformation fileAndPages = null;
transactionalStorage.Batch(accessor => fileAndPages = accessor.GetFile("file", 0, 10));
Assert.Equal(2, fileAndPages.Pages.Count);
Assert.Equal(StorageConstants.MaxPageSize, fileAndPages.Pages[0].Size);
Assert.Equal(1, fileAndPages.Pages[1].Size);
}
public async Task <HttpResponseMessage> MultipartProceed(string fileSystemName)
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault();
var canonicalFilename = FileHeader.Canonize(fileName);
var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate);
Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo);
StorageStream localFile = null;
var isNewFile = false;
var isConflictResolved = false;
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
var localMetadata = GetLocalMetadata(canonicalFilename);
if (localMetadata != null)
{
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
localFile = StorageStream.Reading(Storage, canonicalFilename);
}
else
{
isNewFile = true;
}
Historian.UpdateLastModified(sourceMetadata);
var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata);
var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);
Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName);
await Request.Content.ReadAsMultipartAsync(provider);
Log.Debug("Multipart content of a file '{0}' was processed/read", fileName);
report.BytesCopied = provider.BytesCopied;
report.BytesTransfered = provider.BytesTransfered;
report.NeedListLength = provider.NumberOfFileParts;
synchronizingFile.PreventUploadComplete = false;
synchronizingFile.Flush();
synchronizingFile.Dispose();
sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata));
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(canonicalFilename);
accessor.RenameFile(tempFileName, canonicalFilename);
Search.Delete(tempFileName);
Search.Index(canonicalFilename, sourceMetadata);
});
if (isNewFile)
{
Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName);
}
else
{
Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName);
}
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
}
}
finally
{
if (localFile != null)
{
localFile.Dispose();
}
}
if (report.Exception == null)
{
Log.Debug(
"File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
else
{
Log.WarnException(
string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo),
report.Exception);
}
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update);
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
return(GetMessageWithObject(report));
}
private async Task ExecuteContentUpdate(RavenJObject localMetadata, SynchronizationReport report)
{
var tempFileName = RavenFileNameHelper.DownloadingFileName(fileName);
using (var localFile = localMetadata != null ? StorageStream.Reading(fs.Storage, fileName) : null)
{
fs.PutTriggers.Apply(trigger => trigger.OnPut(tempFileName, sourceMetadata));
fs.Historian.UpdateLastModified(sourceMetadata);
var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(fs, tempFileName, sourceMetadata);
fs.PutTriggers.Apply(trigger => trigger.AfterPut(tempFileName, null, sourceMetadata));
var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);
if (Log.IsDebugEnabled)
{
Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName);
}
await MultipartContent.ReadAsMultipartAsync(provider).ConfigureAwait(false);
if (Log.IsDebugEnabled)
{
Log.Debug("Multipart content of a file '{0}' was processed/read", fileName);
}
report.BytesCopied = provider.BytesCopied;
report.BytesTransfered = provider.BytesTransfered;
report.NeedListLength = provider.NumberOfFileParts;
synchronizingFile.PreventUploadComplete = false;
synchronizingFile.Flush();
synchronizingFile.Dispose();
sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;
FileUpdateResult updateResult = null;
fs.Storage.Batch(accessor => updateResult = accessor.UpdateFileMetadata(tempFileName, sourceMetadata, null));
fs.Storage.Batch(accessor =>
{
using (fs.DisableAllTriggersForCurrentThread())
{
fs.Files.IndicateFileToDelete(fileName, null);
}
accessor.RenameFile(tempFileName, fileName);
fs.Search.Delete(tempFileName);
fs.Search.Index(fileName, sourceMetadata, updateResult.Etag);
});
if (Log.IsDebugEnabled)
{
var message = localFile == null
? string.Format("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName)
: string.Format("Old file '{0}' was deleted. Indexes were updated.", fileName);
Log.Debug(message);
}
fs.Publisher.Publish(new FileChangeNotification {
File = fileName, Action = localFile == null ? FileChangeAction.Add : FileChangeAction.Update
});
}
}
