public async Task Should_resume_file_copying_from_client()
{
var client = NewAsyncClient();
var rfs = GetFileSystem();
string fileName = FileHeader.Canonize("file.bin");
string newName = FileHeader.Canonize("file2.bin");
await client.UploadAsync(fileName, new RandomStream(1));
// create config to say to the server that rename operation performed last time were not finished
var copyOpConfig = RavenFileNameHelper.CopyOperationConfigNameForFile(fileName, newName);
var copyOperation = new CopyFileOperation
{
SourceFilename = fileName,
TargetFilename = newName,
MetadataAfterOperation = new RavenJObject()
};
rfs.Storage.Batch(accessor => accessor.SetConfigurationValue(copyOpConfig, copyOperation));
await client.Storage.RetryCopyingAsync();
IEnumerable <string> configNames = await client.Configuration.GetKeyNamesAsync();
Assert.DoesNotContain(copyOpConfig, configNames);
var renamedMetadata = await client.GetMetadataForAsync(newName);
Assert.NotNull(renamedMetadata);
}
/// <summary>
/// Will deploy local file and its attributes to remote server.
/// </summary>
/// <param name="sourceFile"></param>
/// <param name="destFile"></param>
/// <returns></returns>
public static IOfferRemoteDeployment File(this IOfferRemoteDeployment remote, string sourceFile, string destFile)
{
var copyFileOperation = new CopyFileOperation(sourceFile, destFile);
OperationExecutor.Execute((RemoteBuilder)remote, copyFileOperation);
return(remote);
}
public HttpResponseMessage Copy(string name, string targetFilename)
{
name = FileHeader.Canonize(name);
targetFilename = FileHeader.Canonize(targetFilename);
var etag = GetEtag();
Storage.Batch(accessor =>
{
FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name);
var existingFile = accessor.ReadFile(name);
if (existingFile == null || existingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker))
{
throw new FileNotFoundException();
}
var renamingFile = accessor.ReadFile(targetFilename);
if (renamingFile != null && renamingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker) == false)
{
throw new FileExistsException("Cannot copy because file " + targetFilename + " already exists");
}
var metadata = existingFile.Metadata;
if (etag != null && existingFile.Etag != etag)
{
throw new ConcurrencyException("Operation attempted on file '" + name + "' using a non current etag")
{
ActualETag = existingFile.Etag,
ExpectedETag = etag
}
}
;
Historian.UpdateLastModified(metadata);
var operation = new CopyFileOperation
{
FileSystem = FileSystem.Name,
SourceFilename = name,
TargetFilename = targetFilename,
MetadataAfterOperation = metadata
};
accessor.SetConfig(RavenFileNameHelper.CopyOperationConfigNameForFile(name), JsonExtensions.ToJObject(operation));
accessor.PulseTransaction(); // commit rename operation config
Files.ExecuteCopyOperation(operation);
});
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was copied to '{1}'", name, targetFilename);
}
FileSystem.Synchronizations.StartSynchronizeDestinationsInBackground();
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
/// <summary>
/// Will deploy local file and its attributes to remote server.
/// </summary>
/// <param name="sourceFile"></param>
/// <param name="destFile"></param>
/// <returns></returns>
public static IOfferRemoteDeployment File(this IOfferRemoteDeployment remote, string sourceFile, string destFile)
{
var copyFileOperation = new CopyFileOperation(sourceFile, destFile);
Configure.Operation(remote, copyFileOperation);
//Configure.DeploymentOperations.AddOperation(copyFileOperation);
return(remote);
}
public void ExecuteCopyOperation(CopyFileOperation operation)
{
var configName = RavenFileNameHelper.CopyOperationConfigNameForFile(operation.SourceFilename, operation.TargetFilename);
Storage.Batch(accessor =>
{
AssertPutOperationNotVetoed(operation.TargetFilename, operation.MetadataAfterOperation);
var targetTombstrone = accessor.ReadFile(operation.TargetFilename);
if (targetTombstrone != null &&
targetTombstrone.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if there is a tombstone delete it
accessor.Delete(targetTombstrone.FullPath);
}
FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(operation.TargetFilename, operation.MetadataAfterOperation));
accessor.CopyFile(operation.SourceFilename, operation.TargetFilename, true);
var putResult = accessor.UpdateFileMetadata(operation.TargetFilename, operation.MetadataAfterOperation, null);
FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(operation.TargetFilename, null, operation.MetadataAfterOperation));
accessor.DeleteConfig(configName);
Search.Index(operation.TargetFilename, operation.MetadataAfterOperation, putResult.Etag);
});
Publisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
Publisher.Publish(new FileChangeNotification {
File = operation.TargetFilename, Action = FileChangeAction.Add
});
}
public void Run()
{
var operations = new List <IFileSystemCloningOperation>();
var sourceDirectoryFileSystemSite = this.DeploymentSource_SecretsDirectory_FileSystemSiteProvider.GetDeploymentSourceSecretsDirectoryFileSystemSite();
var destinationDirectoryFileSystemSite = this.DeploymentDestination_SecretsDirectory_FileSystemSiteProvider.GetDeploymentDestinationSecretsDirectoryFileSystemSite();
var secretsFileNames = this.SecretsFileNamesProvider.GetSecretsFileNames();
foreach (var secretsFileName in secretsFileNames)
{
var sourceFilePath = this.StringlyTypedPathOperator.Combine(sourceDirectoryFileSystemSite.DirectoryPath, secretsFileName);
var destinationFilePath = this.StringlyTypedPathOperator.Combine(destinationDirectoryFileSystemSite.DirectoryPath, secretsFileName);
var copyFileOperation = new CopyFileOperation(sourceFilePath, destinationFilePath);
operations.Add(copyFileOperation);
}
foreach (var operation in operations)
{
operation.Execute(sourceDirectoryFileSystemSite.FileSystemOperator, destinationDirectoryFileSystemSite.FileSystemOperator);
}
}
public HttpResponseMessage Copy(string name, string targetFilename)
{
name = FileHeader.Canonize(name);
targetFilename = FileHeader.Canonize(targetFilename);
var etag = GetEtag();
var retriesCount = 0;
while (true)
{
try
{
Storage.Batch(accessor =>
{
FileSystem.Synchronizations.AssertFileIsNotBeingSynced(name);
var existingFile = accessor.ReadFile(name);
if (existingFile == null || existingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker))
{
throw new FileNotFoundException();
}
var renamingFile = accessor.ReadFile(targetFilename);
if (renamingFile != null && renamingFile.Metadata.Value <bool>(SynchronizationConstants.RavenDeleteMarker) == false)
{
throw new FileExistsException("Cannot copy because file " + targetFilename + " already exists");
}
var metadata = existingFile.Metadata;
if (etag != null && existingFile.Etag != etag)
{
throw new ConcurrencyException("Operation attempted on file '" + name + "' using a non current etag")
{
ActualETag = existingFile.Etag,
ExpectedETag = etag
}
}
;
Historian.UpdateLastModified(metadata);
var operation = new CopyFileOperation
{
FileSystem = FileSystem.Name,
SourceFilename = name,
TargetFilename = targetFilename,
MetadataAfterOperation = metadata
};
accessor.SetConfig(RavenFileNameHelper.CopyOperationConfigNameForFile(name, targetFilename), JsonExtensions.ToJObject(operation));
var configName = RavenFileNameHelper.CopyOperationConfigNameForFile(operation.SourceFilename, operation.TargetFilename);
Files.AssertPutOperationNotVetoed(operation.TargetFilename, operation.MetadataAfterOperation);
var targetTombstrone = accessor.ReadFile(operation.TargetFilename);
if (targetTombstrone != null &&
targetTombstrone.Metadata[SynchronizationConstants.RavenDeleteMarker] != null)
{
// if there is a tombstone delete it
accessor.Delete(targetTombstrone.FullPath);
}
FileSystem.PutTriggers.Apply(trigger => trigger.OnPut(operation.TargetFilename, operation.MetadataAfterOperation));
accessor.CopyFile(operation.SourceFilename, operation.TargetFilename, true);
var putResult = accessor.UpdateFileMetadata(operation.TargetFilename, operation.MetadataAfterOperation, null);
FileSystem.PutTriggers.Apply(trigger => trigger.AfterPut(operation.TargetFilename, null, operation.MetadataAfterOperation));
accessor.DeleteConfig(configName);
Search.Index(operation.TargetFilename, operation.MetadataAfterOperation, putResult.Etag);
Publisher.Publish(new ConfigurationChangeNotification {
Name = configName, Action = ConfigurationChangeAction.Set
});
Publisher.Publish(new FileChangeNotification {
File = operation.TargetFilename, Action = FileChangeAction.Add
});
});
break;
}
catch (ConcurrencyException)
{
// due to IncrementUsageCount performed concurrently on Voron storage
// Esent deals with that much better using Api.EscrowUpdate
if (retriesCount++ > 100)
{
throw;
}
Thread.Sleep(new Random().Next(1, 13));
}
}
if (Log.IsDebugEnabled)
{
Log.Debug("File '{0}' was copied to '{1}'", name, targetFilename);
}
SynchronizationTask.Context.NotifyAboutWork();
return(GetEmptyMessage(HttpStatusCode.NoContent));
}
public void Clone(FileSystemSite source, FileSystemSite destination, FileSystemCloningOptions options)
{
// Ensure the source and destination directories are directory indicated.
var ensuredSource = source.EnsureSiteDirectoryPathIsDirectoryIndicated(this.StringlyTypedPathOperator);
var ensuredDestination = destination.EnsureSiteDirectoryPathIsDirectoryIndicated(this.StringlyTypedPathOperator);
// Get all source file-system entries.
var sourceFileSystemEntries = ensuredSource.FileSystemOperator.EnumerateFileSystemEntries(ensuredSource.DirectoryPath, true)
.ToList();
// Get all destination file-system entries.
var destinationFileSystemEntries = ensuredDestination.FileSystemOperator.EnumerateFileSystemEntries(ensuredDestination.DirectoryPath, true)
.ToList();
// Create relative-path source and destination file-system entries.
FileSystemEntry MakeRelativeEntry(string baseDirectoryPath, FileSystemEntry entry)
{
var sourceBaseDirectoryRelativeEntryPath = this.StringlyTypedPathOperator.GetRelativePath(baseDirectoryPath, entry.Path);
var relativeEntry = FileSystemEntry.New(sourceBaseDirectoryRelativeEntryPath, entry.Type, entry.LastModifiedUTC);
return(relativeEntry);
}
var sourceBaseDirectoryRelativePathEntries = sourceFileSystemEntries.Select(entry => MakeRelativeEntry(ensuredSource.DirectoryPath, entry))
.Select(fileSystemEntry =>
{
// Make sure we are using a common path format.
var standardPathFileSystemEntry = fileSystemEntry.GetStandardPathFormatEntry(this.StringlyTypedPathOperator);
return(standardPathFileSystemEntry);
})
.ToList();
var destinationBaseDirectoryRelativePathEntries = destinationFileSystemEntries.Select(entry => MakeRelativeEntry(ensuredDestination.DirectoryPath, entry))
.Select(fileSystemEntry =>
{
// Make sure we are using a common path format.
var standardPathFileSystemEntry = fileSystemEntry.GetStandardPathFormatEntry(this.StringlyTypedPathOperator);
return(standardPathFileSystemEntry);
})
.ToList();
// Write out source and destination data.
// Get the file-system cloning difference.
var difference = this.FileSystemCloningDifferencer.PerformDifference(sourceBaseDirectoryRelativePathEntries, destinationBaseDirectoryRelativePathEntries, options);
// Create a list of operations, using absolute paths.
var operations = new List <IFileSystemCloningOperation>();
// Special case: the destination directory does not exist. If so, make sure it is created first to allow files to be copied into it!
var destinationDirectoryExists = ensuredDestination.FileSystemOperator.ExistsDirectory(ensuredDestination.DirectoryPath);
if (!destinationDirectoryExists)
{
var createDestinationDirectoryOperation = new CreateDirectoryOperation(ensuredDestination.DirectoryPath);
operations.Add(createDestinationDirectoryOperation);
}
foreach (var directoryToCreate in difference.RelativeDirectoryPathsToCreate)
{
string destinationDirectoryToCreate = this.StringlyTypedPathOperator.Combine(ensuredDestination.DirectoryPath, directoryToCreate);
var createDirectoryOperation = new CreateDirectoryOperation(destinationDirectoryToCreate);
operations.Add(createDirectoryOperation);
}
foreach (var directoryToDelete in difference.RelativeDirectoryPathsToDelete)
{
string destinationDirectoryToDelete = this.StringlyTypedPathOperator.Combine(ensuredDestination.DirectoryPath, directoryToDelete);
var deleteDirectoryOperation = new DeleteDirectoryOperation(destinationDirectoryToDelete);
operations.Add(deleteDirectoryOperation);
}
foreach (var fileToCopy in difference.RelativeFilePathsToCopy)
{
string sourceFilePath = this.StringlyTypedPathOperator.Combine(ensuredSource.DirectoryPath, fileToCopy);
string destinationFilePath = this.StringlyTypedPathOperator.Combine(ensuredDestination.DirectoryPath, fileToCopy);
var copyFileOperation = new CopyFileOperation(sourceFilePath, destinationFilePath);
operations.Add(copyFileOperation);
}
foreach (var fileToUpdate in difference.RelativeFilePathsToUpdate)
{
string sourceFilePath = this.StringlyTypedPathOperator.Combine(ensuredSource.DirectoryPath, fileToUpdate);
string destinationFilePath = this.StringlyTypedPathOperator.Combine(ensuredDestination.DirectoryPath, fileToUpdate);
var copyFileOperation = new CopyFileOperation(sourceFilePath, destinationFilePath);
operations.Add(copyFileOperation);
}
foreach (var fileToDelete in difference.RelativeFilePathsToDelete)
{
string destinationFilePath = this.StringlyTypedPathOperator.Combine(ensuredDestination.DirectoryPath, fileToDelete);
var deleteFileOperation = new DeleteFileOperation(destinationFilePath);
operations.Add(deleteFileOperation);
}
// Write out and allow approval of the list of operations?
// Execute the list of operations.
foreach (var operation in operations)
{
operation.Execute(ensuredSource.FileSystemOperator, ensuredDestination.FileSystemOperator);
}
}
