public void Should_create_apropriate_config_after_indicating_file_to_delete()
{
var filename = FileHeader.Canonize("toDelete.bin");
var client = NewAsyncClient();
var rfs = GetFileSystem();
client.UploadAsync(filename, new MemoryStream(new byte[] { 1, 2, 3, 4, 5 })).Wait();
rfs.Files.IndicateFileToDelete(filename, null);
DeleteFileOperation deleteFile = null;
rfs.Storage.Batch(accessor =>
deleteFile = accessor.GetConfigurationValue <DeleteFileOperation>(RavenFileNameHelper.DeleteOperationConfigNameForFile(RavenFileNameHelper.DeletingFileName(filename))));
Assert.Equal(RavenFileNameHelper.DeletingFileName(filename), deleteFile.CurrentFileName);
Assert.Equal(filename, deleteFile.OriginalFileName);
}
public void Should_reuse_second_page_if_only_first_one_changed()
{
string filename = FileHeader.Canonize("test");
var file = SyncTestUtils.PreparePagesStream(2);
file.Position = 0;
var sourceContent = new MemoryStream();
file.CopyTo(sourceContent);
sourceContent.Position = 0;
sourceContent.Write(new byte[] { 0, 0, 0, 0 }, 0, 4); // change content of the 1st page
var destinationContent = file;
sourceContent.Position = 0;
source.UploadAsync(filename, sourceContent).Wait();
destinationContent.Position = 0;
destination.UploadAsync(filename, destinationContent).Wait();
var contentUpdate = new ContentUpdateWorkItem(filename, "http://localhost:12345", sourceRfs.Storage,
sourceRfs.SigGenerator);
sourceContent.Position = 0;
// force to upload entire file, we just want to check which pages will be reused
contentUpdate.UploadToAsync(destination.Synchronization).Wait();
destination.Synchronization.ResolveConflictAsync(filename, ConflictResolutionStrategy.RemoteVersion).Wait();
contentUpdate.UploadToAsync(destination.Synchronization).Wait();
FileAndPagesInformation fileAndPages = null;
destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile(filename, 0, 256));
Assert.Equal(2, fileAndPages.Pages.Count);
Assert.Equal(3, fileAndPages.Pages[0].Id); // new page -> id == 3
Assert.Equal(2, fileAndPages.Pages[1].Id); // reused page -> id still == 2
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync(filename).Result["Content-MD5"]);
}
private void StreamExportToClient(Stream stream, string[] fileNames)
{
using (var bufferedStream = new BufferedStream(stream))
{
var binaryWriter = new BinaryWriter(bufferedStream);
var buffer = new byte[StorageConstants.MaxPageSize];
var pageBuffer = new byte[StorageConstants.MaxPageSize];
foreach (var name in fileNames)
{
FileAndPagesInformation fileAndPages = null;
var cannonizedName = FileHeader.Canonize(name);
Storage.Batch(accessor => fileAndPages = accessor.GetFile(cannonizedName, 0, 0));
// if we didn't find the document, we'll write "-1" to the sourceStream, signaling that
if (fileAndPages.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
{
if (log.IsDebugEnabled)
{
log.Debug("File '{0}' is not accessible to get (Raven-Delete-Marker set)", name);
}
binaryWriter.Write(-1);
continue;
}
var fileSize = fileAndPages.UploadedSize;
binaryWriter.Write(fileSize);
var readingStream = StorageStream.Reading(Storage, cannonizedName);
var bytesRead = 0;
do
{
bytesRead = readingStream.ReadUsingExternalTempBuffer(buffer, 0, buffer.Length, pageBuffer);
bufferedStream.Write(buffer, 0, bytesRead);
} while (bytesRead > 0);
}
}
}
public HttpResponseMessage Post(string name)
{
name = FileHeader.Canonize(name);
var metadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
var etag = GetEtag();
Storage.Batch(accessor =>
{
Synchronizations.AssertFileIsNotBeingSynced(name);
Historian.Update(name, metadata);
Files.UpdateMetadata(name, metadata, etag);
Synchronizations.StartSynchronizeDestinationsInBackground();
});
//Hack needed by jquery on the client side. We need to find a better solution for this
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
public async Task <HttpResponseMessage> Delete(string fileName)
{
fileName = FileHeader.Canonize(fileName);
var sourceInfo = GetSourceFileSystemInfo();
var sourceFileETag = GetEtag();
var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceInfo);
var report = await new SynchronizationBehavior(fileName, sourceFileETag, sourceMetadata, sourceInfo, SynchronizationType.Delete, FileSystem)
.Execute();
if (report.Exception == null)
{
Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceInfo);
}
return(GetMessageWithObject(report));
}
public async Task <HttpResponseMessage> Confirm()
{
var contentStream = await Request.Content.ReadAsStreamAsync();
var confirmingFiles = new JsonSerializer().Deserialize <IEnumerable <Tuple <string, Guid> > >(new JsonTextReader(new StreamReader(contentStream)));
var result = confirmingFiles.Select(x =>
{
string canonicalFilename = FileHeader.Canonize(x.Item1);
return(new SynchronizationConfirmation
{
FileName = canonicalFilename,
Status = CheckSynchronizedFileStatus(canonicalFilename, x.Item2)
});
});
return(this.GetMessageWithObject(result)
.WithNoCache());
}
public async Task Resolve_with_local_should_create_marker_in_conflict_item()
{
var server1 = NewAsyncClient(0);
var server2 = NewAsyncClient(1);
await server1.UploadAsync("test", new MemoryStream(new byte[] { 1, 2, 3 }));
await server2.UploadAsync("test", new MemoryStream(new byte[] { 1, 2 }));
var shouldBeConflict = await server1.Synchronization.StartAsync("test", server2);
Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test")), shouldBeConflict.Exception.Message);
await server2.Synchronization.ResolveConflictAsync("test", ConflictResolutionStrategy.RemoteVersion);
var conflicts = await server2.Synchronization.GetConflictsAsync(0, 5);
Assert.Equal(1, conflicts.Items.Count);
Assert.True(conflicts.Items.First().ResolveUsingRemote);
}
public async void Should_reuse_pages_when_data_appended(int numberOfPages)
{
string filename = FileHeader.Canonize("test");
var file = SyncTestUtils.PreparePagesStream(numberOfPages);
var sourceContent = new CombinedStream(file, SyncTestUtils.PreparePagesStream(numberOfPages));
// add new pages at the end
var destinationContent = file;
sourceContent.Position = 0;
await source.UploadAsync(filename, sourceContent);
destinationContent.Position = 0;
await destination.UploadAsync(filename, destinationContent);
var contentUpdate = new ContentUpdateWorkItem(filename, "http://localhost:12345", sourceRfs.Storage, sourceRfs.SigGenerator);
// force to upload entire file, we just want to check which pages will be reused
await contentUpdate.UploadToAsync(destination.Synchronization);
await destination.Synchronization.ResolveConflictAsync(filename, ConflictResolutionStrategy.RemoteVersion);
await contentUpdate.UploadToAsync(destination.Synchronization);
FileAndPagesInformation fileAndPages = null;
destinationRfs.Storage.Batch(accessor => fileAndPages = accessor.GetFile(filename, 0, 2 * numberOfPages));
Assert.Equal(2 * numberOfPages, fileAndPages.Pages.Count);
for (var i = 0; i < numberOfPages; i++)
{
Assert.Equal(i + 1, fileAndPages.Pages[i].Id);
// if page ids are in the original order it means that they were used the existing pages
}
sourceContent.Position = 0;
Assert.Equal(sourceContent.GetMD5Hash(), destination.GetMetadataForAsync(filename).Result["Content-MD5"]);
}
public async void Should_detect_conflict_on_destination()
{
var destination = (IAsyncFilesCommandsImpl)NewAsyncClient(1);
const string fileName = "test.txt";
await destination.UploadAsync(fileName, new MemoryStream(new byte[] { 1 }));
var request = (HttpWebRequest)WebRequest.Create(destination.ServerUrl + "/fs/" + destination.FileSystem + "/synchronization/updatemetadata/" + fileName);
request.Method = "POST";
request.ContentLength = 0;
var conflictedMetadata = new RavenJObject
{
{ SynchronizationConstants.RavenSynchronizationVersion, new RavenJValue(1) },
{ SynchronizationConstants.RavenSynchronizationSource, new RavenJValue(Guid.Empty) },
{ SynchronizationConstants.RavenSynchronizationHistory, "[]" }
}
.WithETag(Guid.Empty);
request.AddHeaders(conflictedMetadata);
request.Headers[SyncingMultipartConstants.SourceServerInfo] = new ServerInfo {
Id = Guid.Empty, FileSystemUrl = "http://localhost:12345"
}.AsJson();
var response = await request.GetResponseAsync();
using (var stream = response.GetResponseStream())
{
Assert.NotNull(stream);
if (stream == null)
{
return;
}
var report = new JsonSerializer().Deserialize <SynchronizationReport>(new JsonTextReader(new StreamReader(stream)));
Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test.txt")), report.Exception.Message);
}
}
public async void Conflict_item_should_have_remote_server_url()
{
var source = (IAsyncFilesCommandsImpl)NewAsyncClient(0);
var destination = NewAsyncClient(1);
await source.UploadAsync("test", new MemoryStream(new byte[] { 1, 2, 3 }));
await destination.UploadAsync("test", new MemoryStream(new byte[] { 1, 2 }));
var shouldBeConflict = await source.Synchronization.StartAsync("test", destination);
Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test")), shouldBeConflict.Exception.Message);
var pages = await destination.Synchronization.GetConflictsAsync();
var remoteServerUrl = pages.Items[0].RemoteServerUrl;
Assert.NotNull(remoteServerUrl);
Assert.Equal(new Uri(source.ServerUrl).Port, new Uri(remoteServerUrl).Port);
}
public void Delete_conflicted_document_should_delete_conflict_items_as_well()
{
var source = NewAsyncClient(0);
var destination = NewAsyncClient(1);
source.UploadAsync("test", new MemoryStream(new byte[] { 1, 2, 3 })).Wait();
destination.UploadAsync("test", new MemoryStream(new byte[] { 1, 2 })).Wait();
var shouldBeConflict = source.Synchronization.StartAsync("test", destination).Result;
Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test")), shouldBeConflict.Exception.Message);
var pages = destination.Synchronization.GetConflictsAsync().Result;
Assert.Equal(1, pages.TotalCount);
destination.DeleteAsync("test").Wait();
pages = destination.Synchronization.GetConflictsAsync().Result;
Assert.Equal(0, pages.TotalCount);
}
public async Task Source_should_save_configuration_record_after_synchronization()
{
var sourceClient = NewAsyncClient(0);
var sourceContent = new RandomStream(10000);
var destinationClient = NewAsyncClient(1);
await sourceClient.UploadAsync("test.bin", sourceContent);
await sourceClient.Synchronization.SetDestinationsAsync(destinationClient.ToSynchronizationDestination());
await sourceClient.Synchronization.SynchronizeAsync();
var fullDstUrl = destinationClient.ToSynchronizationDestination().Url;
var synchronizationDetails = sourceClient.Configuration.GetKeyAsync <SynchronizationDetails>(RavenFileNameHelper.SyncNameForFile("test.bin", fullDstUrl)).Result;
Assert.Equal(FileHeader.Canonize("test.bin"), synchronizationDetails.FileName);
Assert.Equal(fullDstUrl, synchronizationDetails.DestinationUrl);
Assert.NotEqual(Guid.Empty, synchronizationDetails.FileETag);
Assert.Equal(SynchronizationType.ContentUpdate, synchronizationDetails.Type);
}
public async Task <HttpResponseMessage> MultipartProceed()
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var fileName = FileHeader.Canonize(Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault());
var sourceInfo = GetSourceFileSystemInfo();
var sourceFileETag = GetEtag();
var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
if (Log.IsDebugEnabled)
{
Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceInfo);
}
var report = await new SynchronizationBehavior(fileName, sourceFileETag, sourceMetadata, sourceInfo, SynchronizationType.ContentUpdate, FileSystem)
{
MultipartContent = Request.Content
}.Execute().ConfigureAwait(false);
if (report.Exception == null)
{
if (Log.IsDebugEnabled)
{
Log.Debug(
"File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
fileName, sourceInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
}
else
{
Log.WarnException(string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceInfo), report.Exception);
}
return(GetMessageWithObject(report));
}
public void Should_resume_to_rename_file_if_appropriate_config_exists()
{
var client = NewAsyncClient();
var rfs = GetFileSystem();
string fileName = FileHeader.Canonize("file.bin");
string rename = FileHeader.Canonize("renamed.bin");
client.UploadAsync(fileName, new RandomStream(1)).Wait();
// create config to say to the server that rename operation performed last time were not finished
var renameOpConfig = RavenFileNameHelper.RenameOperationConfigNameForFile(fileName);
var renameOperation = new RenameFileOperation
{
Name = fileName,
Rename = rename,
MetadataAfterOperation = new RavenJObject().WithETag(Guid.Empty)
};
rfs.Storage.Batch(accessor => accessor.SetConfigurationValue(renameOpConfig, renameOperation));
rfs.StorageOperationsTask.ResumeFileRenamingAsync().Wait();
IEnumerable <string> configNames = null;
rfs.Storage.Batch(accessor => configNames = accessor.GetConfigNames(0, 10).ToArray());
Assert.DoesNotContain(renameOpConfig, configNames);
var renamedMetadata = client.GetMetadataForAsync(rename).Result;
Assert.NotNull(renamedMetadata);
var results = client.SearchOnDirectoryAsync("/").Result; // make sure that indexes are updated
Assert.Equal(1, results.FileCount);
Assert.Equal(rename, results.Files[0].FullPath);
}
public HttpResponseMessage Head(string name)
{
name = FileHeader.Canonize(name);
FileAndPagesInformation fileAndPages = null;
Storage.Batch(accessor => fileAndPages = accessor.GetFile(name, 0, 0));
if (fileAndPages.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
{
log.Debug("Cannot get metadata of a file '{0}' because file was deleted", name);
throw new FileNotFoundException();
}
var httpResponseMessage = GetEmptyMessage();
var etag = new Etag(fileAndPages.Metadata.Value <string>(Constants.MetadataEtagField));
fileAndPages.Metadata.Remove(Constants.MetadataEtagField);
WriteHeaders(fileAndPages.Metadata, etag, httpResponseMessage);
return(httpResponseMessage);
}
public async Task <HttpResponseMessage> UpdateMetadata(string fileName)
{
fileName = FileHeader.Canonize(fileName);
var sourceInfo = GetSourceFileSystemInfo();
var sourceFileETag = GetEtag();
var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
if (Log.IsDebugEnabled)
{
Log.Debug("Starting to update a metadata of file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceInfo);
}
var report = await new SynchronizationBehavior(fileName, sourceFileETag, sourceMetadata, sourceInfo, SynchronizationType.MetadataUpdate, FileSystem)
.Execute().ConfigureAwait(false);
if (Log.IsDebugEnabled && report.Exception == null)
{
Log.Debug("Metadata of file '{0}' was synchronized successfully from {1}", fileName, sourceInfo);
}
return(GetMessageWithObject(report));
}
public void Should_detect_conflict_on_renaming_synchronization()
{
var content = new MemoryStream(new byte[] { 1, 2, 3, 4 });
var sourceClient = NewAsyncClient(0);
var destinationClient = NewAsyncClient(1);
sourceClient.UploadAsync("test.bin", content, new RavenJObject {
{ "key", "value" }
}).Wait();
content.Position = 0;
destinationClient.UploadAsync("test.bin", content, new RavenJObject {
{ "key", "value" }
}).Wait();
sourceClient.RenameAsync("test.bin", "renamed.bin").Wait();
// we need to indicate old file name, otherwise content update would be performed because renamed file does not exist on dest
var report = sourceClient.Synchronization.StartAsync("test.bin", destinationClient).Result;
Assert.Equal(SynchronizationType.Rename, report.Type);
Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test.bin")), report.Exception.Message);
}
public async Task <HttpResponseMessage> Manifest(string id)
{
var canonicalFilename = FileHeader.Canonize(id);
FileAndPagesInformation fileAndPages = null;
try
{
Storage.Batch(accessor => fileAndPages = accessor.GetFile(canonicalFilename, 0, 0));
}
catch (FileNotFoundException)
{
Log.Debug("Signature manifest for a file '{0}' was not found", id);
return(Request.CreateResponse(HttpStatusCode.NotFound));
}
long?fileLength = fileAndPages.TotalSize;
using (var signatureRepository = new StorageSignatureRepository(Storage, canonicalFilename))
{
var rdcManager = new LocalRdcManager(signatureRepository, Storage, SigGenerator);
var signatureManifest = await rdcManager.GetSignatureManifestAsync(
new DataInfo
{
Name = canonicalFilename,
LastModified = fileAndPages.Metadata.Value <DateTime>(Constants.LastModified)
.ToUniversalTime()
});
signatureManifest.FileLength = fileLength ?? 0;
Log.Debug("Signature manifest for a file '{0}' was downloaded. Signatures count was {1}", id, signatureManifest.Signatures.Count);
return(GetMessageWithObject(signatureManifest)
.WithNoCache());
}
}
public HttpResponseMessage Post(string name)
{
name = FileHeader.Canonize(name);
var headers = this.GetFilteredMetadataFromHeaders(InnerHeaders);
Historian.UpdateLastModified(headers);
Historian.Update(name, headers);
try
{
ConcurrencyAwareExecutor.Execute(() =>
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(name, accessor, true);
accessor.UpdateFileMetadata(name, headers);
}), ConcurrencyResponseException);
}
catch (FileNotFoundException)
{
log.Debug("Cannot update metadata because file '{0}' was not found", name);
return(GetEmptyMessage(HttpStatusCode.NotFound));
}
Search.Index(name, headers);
Publisher.Publish(new FileChangeNotification {
File = FilePathTools.Cannoicalise(name), Action = FileChangeAction.Update
});
StartSynchronizeDestinationsInBackground();
log.Debug("Metadata of a file '{0}' was updated", name);
//Hack needed by jquery on the client side. We need to find a better solution for this
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
public async Task Should_report_that_file_is_broken_if_last_synchronization_set_exception()
{
var destinationClient = NewAsyncClient(0);
var sampleGuid = Guid.NewGuid();
var failureSynchronization = new SynchronizationReport("test.bin", sampleGuid, SynchronizationType.Unknown)
{
Exception = new Exception("There was an exception in last synchronization.")
};
await destinationClient.Configuration.SetKeyAsync(RavenFileNameHelper.SyncResultNameForFile("test.bin"), failureSynchronization);
var confirmations = await destinationClient.Synchronization
.GetConfirmationForFilesAsync(new List <Tuple <string, Guid> >
{
new Tuple <string, Guid>(
"test.bin", sampleGuid)
});
Assert.Equal(1, confirmations.Count());
Assert.Equal(FileStatus.Broken, confirmations[0].Status);
Assert.Equal(FileHeader.Canonize("test.bin"), confirmations[0].FileName);
}
public HttpResponseMessage Delete(string name)
{
name = FileHeader.Canonize(name);
Storage.Batch(accessor =>
{
Synchronizations.AssertFileIsNotBeingSynced(name);
var fileAndPages = accessor.GetFile(name, 0, 0);
var metadata = fileAndPages.Metadata;
if (metadata == null)
{
throw new FileNotFoundException();
}
if (metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
{
throw new FileNotFoundException();
}
Historian.Update(name, metadata);
Files.IndicateFileToDelete(name, GetEtag());
if (name.EndsWith(RavenFileNameHelper.DownloadingFileSuffix) == false) // don't create a tombstone for .downloading file
{
Files.PutTombstone(name, metadata);
accessor.DeleteConfig(RavenFileNameHelper.ConflictConfigNameForFile(name)); // delete conflict item too
}
});
SynchronizationTask.Context.NotifyAboutWork();
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
public async Task <HttpResponseMessage> Rename(string fileName, string rename)
{
fileName = FileHeader.Canonize(fileName);
rename = FileHeader.Canonize(rename);
var sourceInfo = GetSourceFileSystemInfo();
var sourceFileEtag = GetEtag();
var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName,
rename, sourceFileEtag, sourceInfo);
var report = await new SynchronizationBehavior(fileName, sourceFileEtag, sourceMetadata, sourceInfo, SynchronizationType.Rename, FileSystem)
{
Rename = rename
}.Execute().ConfigureAwait(false);
if (report.Exception == null)
{
Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceInfo);
}
return(GetMessageWithObject(report));
}
public void Can_resolve_all_conflicts()
{
var sourceClient = NewAsyncClient(0);
var destinationClient = NewAsyncClient(1);
for (var i = 0; i < 10; i++)
{
sourceClient.UploadAsync("test" + i, new MemoryStream(new byte[] { 1, 2, 3 })).Wait();
destinationClient.UploadAsync("test" + i, new MemoryStream(new byte[] { 1, 2 })).Wait();
}
for (var i = 0; i < 10; i++)
{
var shouldBeConflict = sourceClient.Synchronization.StartAsync("test" + i, destinationClient).Result;
Assert.Equal(string.Format("File {0} is conflicted", FileHeader.Canonize("test" + i)), shouldBeConflict.Exception.Message);
}
destinationClient.Synchronization.ResolveConflictsAsync(ConflictResolutionStrategy.CurrentVersion).Wait();
var conflicts = destinationClient.Synchronization.GetConflictsAsync(0, 100).Result;
Assert.Equal(0, conflicts.TotalCount);
Assert.Equal(0, conflicts.Items.Count);
}
public async Task Should_refuse_to_synchronize_if_limit_of_concurrent_synchronizations_exceeded()
{
var sourceContent = new RandomStream(1);
var sourceClient = NewAsyncClient(0);
var destinationClient = (IAsyncFilesCommandsImpl)NewAsyncClient(1);
await sourceClient.Configuration.SetKeyAsync(SynchronizationConstants.RavenSynchronizationLimit, -1);
await sourceClient.UploadAsync("test.bin", sourceContent);
var synchronizationReport = await sourceClient.Synchronization.StartAsync("test.bin", destinationClient);
Assert.Contains("The limit of active synchronizations to " + destinationClient.ServerUrl, synchronizationReport.Exception.Message);
Assert.Contains(string.Format("server has been achieved. Cannot process a file '{0}'.", FileHeader.Canonize("test.bin")), synchronizationReport.Exception.Message);
}
public async Task <HttpResponseMessage> MultipartProceed(string fileSystemName)
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault();
var canonicalFilename = FileHeader.Canonize(fileName);
var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate);
Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo);
StorageStream localFile = null;
var isNewFile = false;
var isConflictResolved = false;
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
var localMetadata = GetLocalMetadata(canonicalFilename);
if (localMetadata != null)
{
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
localFile = StorageStream.Reading(Storage, canonicalFilename);
}
else
{
isNewFile = true;
}
Historian.UpdateLastModified(sourceMetadata);
var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata);
var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile);
Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName);
await Request.Content.ReadAsMultipartAsync(provider);
Log.Debug("Multipart content of a file '{0}' was processed/read", fileName);
report.BytesCopied = provider.BytesCopied;
report.BytesTransfered = provider.BytesTransfered;
report.NeedListLength = provider.NumberOfFileParts;
synchronizingFile.PreventUploadComplete = false;
synchronizingFile.Flush();
synchronizingFile.Dispose();
sourceMetadata["Content-MD5"] = synchronizingFile.FileHash;
Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata));
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(canonicalFilename);
accessor.RenameFile(tempFileName, canonicalFilename);
Search.Delete(tempFileName);
Search.Index(canonicalFilename, sourceMetadata);
});
if (isNewFile)
{
Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName);
}
else
{
Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName);
}
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
}
}
finally
{
if (localFile != null)
{
localFile.Dispose();
}
}
if (report.Exception == null)
{
Log.Debug(
"File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}",
fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength);
}
else
{
Log.WarnException(
string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo),
report.Exception);
}
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update);
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
return(GetMessageWithObject(report));
}
public HttpResponseMessage Rename(string fileSystemName, string fileName, string rename)
{
bool isConflictResolved = false;
var canonicalFilename = FileHeader.Canonize(fileName);
var canonicalRename = FileHeader.Canonize(rename);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
Log.Debug("Starting to rename a file '{0}' to '{1}' with ETag {2} from {3} because of synchronization", fileName,
rename, sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Rename);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Rename);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
var localMetadata = GetLocalMetadata(canonicalFilename);
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
StorageOperationsTask.RenameFile(new RenameFileOperation
{
FileSystem = FileSystem.Name,
Name = canonicalFilename,
Rename = canonicalRename,
MetadataAfterOperation = sourceMetadata.WithETag(sourceFileETag).DropRenameMarkers()
});
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(string.Format("Error was occurred during renaming synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
}
finally
{
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
}
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
if (report.Exception == null)
{
Log.Debug("File '{0}' was renamed to '{1}' during synchronization from {2}", fileName, rename, sourceServerInfo);
}
return(GetMessageWithObject(report));
}
public HttpResponseMessage Delete(string fileSystemName, string fileName)
{
var canonicalFilename = FileHeader.Canonize(fileName);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
Log.Debug("Starting to delete a file '{0}' with ETag {1} from {2} because of synchronization", fileName, sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.Delete);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.Delete);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
var localMetadata = GetLocalMetadata(canonicalFilename);
if (localMetadata != null)
{
// REVIEW: Use InnerHeaders for consistency?
var sourceMetadata = GetFilteredMetadataFromHeaders(Request.Headers); // Request.Headers.FilterHeadersToObject();
bool isConflictResolved;
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Storage.Batch(accessor =>
{
StorageOperationsTask.IndicateFileToDelete(canonicalFilename);
var tombstoneMetadata = new RavenJObject
{
{
SynchronizationConstants.RavenSynchronizationHistory,
localMetadata[SynchronizationConstants.RavenSynchronizationHistory]
},
{
SynchronizationConstants.RavenSynchronizationVersion,
localMetadata[SynchronizationConstants.RavenSynchronizationVersion]
},
{
SynchronizationConstants.RavenSynchronizationSource,
localMetadata[SynchronizationConstants.RavenSynchronizationSource]
}
}.WithDeleteMarker();
Historian.UpdateLastModified(tombstoneMetadata);
accessor.PutFile(canonicalFilename, 0, tombstoneMetadata, true);
});
PublishFileNotification(fileName, FileChangeAction.Delete);
}
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(string.Format("Error was occurred during deletion synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
}
finally
{
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
}
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (report.Exception == null)
{
Log.Debug("File '{0}' was deleted during synchronization from {1}", fileName, sourceServerInfo);
}
return(this.GetMessageWithObject(report, HttpStatusCode.OK));
}
public HttpResponseMessage UpdateMetadata(string fileSystemName, string fileName)
{
bool isConflictResolved = false;
var canonicalFilename = FileHeader.Canonize(fileName);
var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo);
// REVIEW: (Oren) It works, but it seems to me it is not an scalable solution.
var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"'));
Log.Debug("Starting to update a metadata of file '{0}' with ETag {1} from {2} because of synchronization", fileName,
sourceFileETag, sourceServerInfo);
var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.MetadataUpdate);
try
{
Storage.Batch(accessor =>
{
AssertFileIsNotBeingSynced(canonicalFilename, accessor);
FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor);
});
SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.MetadataUpdate);
PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start);
Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor));
var localMetadata = GetLocalMetadata(canonicalFilename);
var sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders);
AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved);
Historian.UpdateLastModified(sourceMetadata);
Storage.Batch(accessor => accessor.UpdateFileMetadata(canonicalFilename, sourceMetadata));
Search.Index(canonicalFilename, sourceMetadata);
if (isConflictResolved)
{
ConflictArtifactManager.Delete(canonicalFilename);
}
PublishFileNotification(fileName, FileChangeAction.Update);
}
catch (Exception ex)
{
if (ShouldAddExceptionToReport(ex))
{
report.Exception = ex;
Log.WarnException(
string.Format("Error was occurred during metadata synchronization of file '{0}' from {1}", fileName, sourceServerInfo), ex);
}
}
finally
{
FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag);
}
PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish);
if (isConflictResolved)
{
Publisher.Publish(new ConflictNotification
{
FileName = fileName,
Status = ConflictStatus.Resolved
});
}
if (report.Exception == null)
{
Log.Debug("Metadata of file '{0}' was synchronized successfully from {1}", fileName, sourceServerInfo);
}
return(this.GetMessageWithObject(report, HttpStatusCode.OK));
}
public HttpResponseMessage Patch(string name, string rename)
{
name = FileHeader.Canonize(name);
rename = FileHeader.Canonize(rename);
var etag = GetEtag();
if (rename.Length > SystemParameters.KeyMost)
{
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename);
}
return(GetMessageWithString(string.Format("File '{0}' was not renamed to '{1}' due to illegal name length", name, rename), HttpStatusCode.BadRequest));
}
Storage.Batch(accessor =>
{
FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name);
var existingFile = accessor.ReadFile(name);
if (existingFile == null || existingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker))
{
throw new FileNotFoundException();
}
var renamingFile = accessor.ReadFile(rename);
if (renamingFile != null && renamingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker) == false)
{
throw new FileExistsException("Cannot rename because file " + rename + " already exists");
}
var metadata = existingFile.Metadata;
if (etag != null && existingFile.Etag != etag)
{
throw new ConcurrencyException("Operation attempted on file '" + name + "' using a non current etag")
{
ActualETag = existingFile.Etag,
ExpectedETag = etag
}
}
;
Historian.UpdateLastModified(metadata);
var operation = new RenameFileOperation(name, rename, existingFile.Etag, metadata);
accessor.SetConfig(RavenFileNameHelper.RenameOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation));
accessor.PulseTransaction(); // commit rename operation config
Files.ExecuteRenameOperation(operation);
});
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was renamed to '{1}'", name, rename);
}
SynchronizationTask.Context.NotifyAboutWork();
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
public HttpResponseMessage Get([FromUri] string[] fileNames)
{
var list = new List <FileHeader>();
var startsWith = GetQueryStringValue("startsWith");
if (string.IsNullOrEmpty(startsWith) == false)
{
var matches = GetQueryStringValue("matches");
var endsWithSlash = startsWith.EndsWith("/") || startsWith.EndsWith("\\");
startsWith = FileHeader.Canonize(startsWith);
if (endsWithSlash)
{
startsWith += "/";
}
Storage.Batch(accessor =>
{
var actualStart = 0;
var filesToSkip = Paging.Start;
int fileCount, matchedFiles = 0, addedFiles = 0;
do
{
fileCount = 0;
foreach (var file in accessor.GetFilesStartingWith(startsWith, actualStart, Paging.PageSize))
{
fileCount++;
var keyTest = file.FullPath.Substring(startsWith.Length);
if (WildcardMatcher.Matches(matches, keyTest) == false)
{
continue;
}
if (FileSystem.ReadTriggers.CanReadFile(file.FullPath, file.Metadata, ReadOperation.Load) == false)
{
continue;
}
matchedFiles++;
if (matchedFiles <= filesToSkip)
{
continue;
}
list.Add(file);
addedFiles++;
}
actualStart += Paging.PageSize;
}while (fileCount > 0 && addedFiles < Paging.PageSize && actualStart > 0 && actualStart < int.MaxValue);
});
}
else
{
if (fileNames != null && fileNames.Length > 0)
{
Storage.Batch(accessor =>
{
foreach (var path in fileNames.Where(x => x != null).Select(FileHeader.Canonize))
{
var file = accessor.ReadFile(path);
if (file == null || file.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker))
{
list.Add(null);
continue;
}
list.Add(file);
}
});
}
else
{
int results;
long durationInMs;
var keys = Search.Query(null, null, Paging.Start, Paging.PageSize, out results, out durationInMs);
Storage.Batch(accessor => list.AddRange(keys.Select(accessor.ReadFile).Where(x => x != null)));
}
}
return(GetMessageWithObject(list)
.WithNoCache());
}
