/// <nodoc />
public DistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <T> fileCopier,
IFileExistenceChecker <T> fileExistenceChecker,
IPathTransformer <T> pathTransformer,
IContentLocationStore contentLocationStore)
{
Contract.Requires(settings != null);
Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);
_settings = settings;
_tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp");
_remoteFileCopier = fileCopier;
_remoteFileExistenceChecker = fileExistenceChecker;
_contentLocationStore = contentLocationStore;
_pathTransformer = pathTransformer;
_fileSystem = fileSystem;
_workingDirectory = _tempFolderForCopies.Path;
// TODO: Use hashers from IContentStoreInternal instead?
_hashers = HashInfoLookup.CreateAll();
_ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
_retryIntervals = settings.RetryIntervalForCopies;
}
private static void StartWatching(CommandOptions options)
{
ILogger logger = new ConsoleLogger();
logger.LogInfo("Watching has started...");
options.SourceDirectoryPath = string.IsNullOrWhiteSpace(options.SourceDirectoryPath)
? Directory.GetCurrentDirectory()
: options.SourceDirectoryPath;
IPluginLoader loader = new PluginLoader(logger, options.Debug);
var fileCopier = new FileCopier(logger, options);
IFileCopier copier = fileCopier;
if (options.Delay > 0)
{
copier = new QueuedFileCopier(fileCopier, logger, options);
}
IFileWatcher fileWatcher = new FileWatcher(copier, logger);
loader.Subscribe((IPreCopyEventBroadcaster)copier, (IPostCopyEventBroadcaster)copier);
fileWatcher.Watch(options);
}
/// <nodoc />
public DistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <T> fileCopier,
IFileExistenceChecker <T> fileExistenceChecker,
ICopyRequester copyRequester,
IPathTransformer <T> pathTransformer,
IContentLocationStore contentLocationStore)
{
Contract.Requires(settings != null);
Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);
_settings = settings;
_tempFolderForCopies = new DisposableDirectory(fileSystem, workingDirectory / "Temp");
_remoteFileCopier = fileCopier;
_remoteFileExistenceChecker = fileExistenceChecker;
_copyRequester = copyRequester;
_contentLocationStore = contentLocationStore;
_pathTransformer = pathTransformer;
_fileSystem = fileSystem;
_workingDirectory = _tempFolderForCopies.Path;
_ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
_proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations);
_retryIntervals = settings.RetryIntervalForCopies;
_maxRetryCount = settings.MaxRetryCount;
_timeoutForPoractiveCopies = settings.TimeoutForProactiveCopies;
}
private void AddCopier(IFileCopier copier)
{
foreach (var ext in copier.FromExtensions)
{
_copiers.Add(ext, copier);
}
}
public DistributedContentCopier(
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <T> fileCopier,
IFileExistenceChecker <T> fileExistenceChecker,
IContentCommunicationManager copyRequester,
IPathTransformer <T> pathTransformer,
IClock clock)
{
Contract.Requires(settings != null);
Contract.Requires(settings.ParallelHashingFileSizeBoundary >= -1);
_settings = settings;
_remoteFileCopier = fileCopier;
_remoteFileExistenceChecker = fileExistenceChecker;
_copyRequester = copyRequester;
_pathTransformer = pathTransformer;
FileSystem = fileSystem;
_clock = clock;
_ioGate = new SemaphoreSlim(_settings.MaxConcurrentCopyOperations);
_proactiveCopyIoGate = new SemaphoreSlim(_settings.MaxConcurrentProactiveCopyOperations);
_retryIntervals = settings.RetryIntervalForCopies;
_maxRetryCount = settings.MaxRetryCount;
_timeoutForProactiveCopies = settings.TimeoutForProactiveCopies;
}
public PipelineFactory(IFileCopier fileCopier)
{
if (fileCopier == null)
{
throw new ArgumentNullException(nameof(fileCopier));
}
_fileCopier = fileCopier;
}
public DataAnalysisPipeline(
IFileCopier fileCopier,
RepositoryUrl repositoryUrl,
RepositoryDestination repositoryDestination)
{
_fileCopier = fileCopier;
_repositoryUrl = repositoryUrl;
_repositoryDestination = repositoryDestination;
}
/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
ICopyRequester copyRequester,
ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
int locationStoreBatchSize,
IReadOnlyList <TimeSpan> retryIntervalForCopies = null,
PinConfiguration pinConfiguration = null,
int?replicaCreditInMinutes = null,
IClock clock = null,
bool enableRepairHandling = false,
TimeSpan?contentHashBumpTime = null,
bool useTrustedHash = false,
int trustedHashFileSizeBoundary = -1,
long parallelHashingFileSizeBoundary = -1,
int maxConcurrentCopyOperations = 512,
ContentStoreSettings contentStoreSettings = null,
bool enableProactiveCopy = false)
: this(
localMachineLocation,
innerContentStoreFunc,
contentLocationStoreFactory,
fileExistenceChecker,
fileCopier,
pathTransform,
copyRequester,
contentAvailabilityGuarantee,
tempFolderForCopies,
fileSystem,
locationStoreBatchSize,
new DistributedContentStoreSettings()
{
UseTrustedHash = useTrustedHash,
TrustedHashFileSizeBoundary = trustedHashFileSizeBoundary,
ParallelHashingFileSizeBoundary = parallelHashingFileSizeBoundary,
MaxConcurrentCopyOperations = maxConcurrentCopyOperations,
RetryIntervalForCopies = retryIntervalForCopies,
PinConfiguration = pinConfiguration,
EnableProactiveCopy = enableProactiveCopy
},
replicaCreditInMinutes,
clock,
enableRepairHandling,
contentHashBumpTime,
contentStoreSettings)
{
// This constructor is used from tests,
// so we need to complete _postInitializationCompletion when startup is done.
_setPostInitializationCompletionAfterStartup = true;
}
public FileServices(
[NotNull] ILogger logger, [NotNull] IFileCopier fileCopier, [NotNull] IFileMover fileMover,
[NotNull] IStreamComparer streamComparer, [NotNull] ITypeHelper typeHelper)
{
_Logger = logger ?? throw new ArgumentNullException(nameof(logger));
_FileCopier = fileCopier ?? throw new ArgumentNullException(nameof(fileCopier));
_FileMover = fileMover ?? throw new ArgumentNullException(nameof(fileMover));
_StreamComparer = streamComparer ?? throw new ArgumentNullException(nameof(streamComparer));
_TypeHelper = typeHelper ?? throw new ArgumentNullException(nameof(typeHelper));
}
public QueuedFileCopier(IFileCopier fileCopier, ILogger logger, CommandOptions options)
{
_fileCopier = fileCopier;
_logger = logger;
_options = options;
if (_options.Debug)
{
_logger.LogDebug($"Delay option has been specified. QueuedFileCopier is chosen as the copier strategy.");
}
}
public TestDistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <AbsolutePath> fileCopier,
IFileExistenceChecker <AbsolutePath> fileExistenceChecker,
IContentCommunicationManager copyRequester,
IPathTransformer <AbsolutePath> pathTransformer,
IContentLocationStore contentLocationStore)
: base(workingDirectory, settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransformer, TestSystemClock.Instance, contentLocationStore)
{
}
public InstallCommand(IRuntime runtime, IFileSystem fileSystem, ICakeEnvironment environment,
ICakeLog log, INuGetPackageConfigurationCreator packageConfigCreator,
IFileCopier fileCopier, IHttpDownloader downloader, IGitIgnorePatcher gitIgnorePatcher)
{
_runtime = runtime;
_fileSystem = fileSystem;
_environment = environment;
_log = log;
_packageConfigCreator = packageConfigCreator;
_fileCopier = fileCopier;
_downloader = downloader;
_gitIgnorePatcher = gitIgnorePatcher;
}
public InstallCommand(IRuntime runtime, IFileSystem fileSystem, ICakeEnvironment environment,
ICakeLog log, INuGetPackageConfigurationCreator packageConfigCreator,
IFileCopier fileCopier, IHttpDownloader downloader, IGitIgnorePatcher gitIgnorePatcher)
{
_runtime = runtime;
_fileSystem = fileSystem;
_environment = environment;
_log = log;
_packageConfigCreator = packageConfigCreator;
_fileCopier = fileCopier;
_downloader = downloader;
_gitIgnorePatcher = gitIgnorePatcher;
}
public EnsureMinAgentUpdater(ISleeper sleeper, IUnzipper unzipper, IFileCopier fileCopier, IFinalizer finalizer, IServiceStopper serviceStopper, IServiceStarter serviceStarter, ILogger logger, IBackupUpdater backupUpdater, IExtractEmbededResource extractEmbededResource, IVersionChecker versionChecker)
{
_sleeper = sleeper;
_unzipper = unzipper;
_fileCopier = fileCopier;
_finalizer = finalizer;
_serviceStopper = serviceStopper;
_serviceStarter = serviceStarter;
_logger = logger;
_backupUpdater = backupUpdater;
_extractEmbededResource = extractEmbededResource;
_versionChecker = versionChecker;
}
/// <nodoc />
public static Task <CopyFileResult> CopyToWithOperationContextAsync <T>(
this IFileCopier <T> copier,
OperationContext context,
T sourcePath,
Stream destinationStream,
long expectedContentSize) where T : PathBase
{
if (copier is ITraceableFileCopier <T> traceable)
{
return(traceable.CopyToAsync(context, sourcePath, destinationStream, expectedContentSize));
}
return(copier.CopyToAsync(sourcePath, destinationStream, expectedContentSize, context.Token));
}
public UpdaterUpdate(ISleeper sleeper, IDownloader downloader, IChecksumValidator checksumValidator, IUnzipper unzipper, IFileCopier fileCopier, IFinalizer finalizer, IServiceStopper serviceStopper, IServiceStarter serviceStarter, IConnectionChecker connectionChecker, IAgentUpdateMessageHandler agentUpdateMessageHandler, ILogger logger)
{
_sleeper = sleeper;
_downloader = downloader;
_checksumValidator = checksumValidator;
_unzipper = unzipper;
_fileCopier = fileCopier;
_finalizer = finalizer;
_serviceStopper = serviceStopper;
_serviceStarter = serviceStarter;
_connectionChecker = connectionChecker;
_agentUpdateMessageHandler = agentUpdateMessageHandler;
_logger = logger;
}
public UpdaterUpdate(ISleeper sleeper, IDownloader downloader, IChecksumValidator checksumValidator, IUnzipper unzipper, IFileCopier fileCopier, IFinalizer finalizer, IServiceStopper serviceStopper, IServiceStarter serviceStarter, IConnectionChecker connectionChecker, IAgentUpdateMessageHandler agentUpdateMessageHandler, ILogger logger)
{
_sleeper = sleeper;
_downloader = downloader;
_checksumValidator = checksumValidator;
_unzipper = unzipper;
_fileCopier = fileCopier;
_finalizer = finalizer;
_serviceStopper = serviceStopper;
_serviceStarter = serviceStarter;
_connectionChecker = connectionChecker;
_agentUpdateMessageHandler = agentUpdateMessageHandler;
_logger = logger;
}
public TestDistributedContentCopier(
AbsolutePath workingDirectory,
DistributedContentStoreSettings settings,
IAbsFileSystem fileSystem,
IFileCopier <AbsolutePath> fileCopier,
IFileExistenceChecker <AbsolutePath> fileExistenceChecker,
IContentCommunicationManager copyRequester,
IPathTransformer <AbsolutePath> pathTransformer,
MachineLocation[] designatedLocations)
: base(settings, fileSystem, fileCopier, fileExistenceChecker, copyRequester, pathTransformer, TestSystemClock.Instance)
{
WorkingFolder = workingDirectory;
PathTransformer = pathTransformer as NoOpPathTransformer;
_designatedLocations = designatedLocations;
}
public CopyFilesToDestinationCommand(
IFileCopier fileCopier,
RepositoryDestination repositoryDestination)
{
if (fileCopier == null)
{
throw new ArgumentNullException(nameof(fileCopier));
}
if (repositoryDestination == null)
{
throw new ArgumentNullException(nameof(repositoryDestination));
}
_fileCopier = fileCopier;
_repositoryDestination = repositoryDestination;
}
/// <summary>
/// Copy this file to the given file (<paramref name="a_dest"/>) using the given file copier (<paramref name="a_fileCopier"/>).
/// </summary>
/// <typeparam name="TDest">Type of destination file.</typeparam>
/// <param name="a_dest">Destination file.</param>
/// <param name="a_fileCopier">File copier.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="a_dest"/> is null.</exception>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="a_fileCopier"/> is null.</exception>
public void CopyTo <TDest>(TDest a_dest, IFileCopier <PathFile <TLeaf>, TDest> a_fileCopier)
where TDest : IFile
{
#region Argument Validation
if (a_dest == null)
{
throw new ArgumentNullException(nameof(a_dest));
}
if (a_fileCopier == null)
{
throw new ArgumentNullException(nameof(a_fileCopier));
}
#endregion
a_fileCopier.Copy(this, a_dest);
}
public void Setup()
{
_agentUpdateInfo = "http://something.com/file.zip,544564abc453de787ad";
_downloader = MockRepository.GenerateMock <IDownloader>();
_checksumValidator = MockRepository.GenerateMock <IChecksumValidator>();
_unzipper = MockRepository.GenerateMock <IUnzipper>();
_fileCopier = MockRepository.GenerateMock <IFileCopier>();
_finalizer = MockRepository.GenerateMock <IFinalizer>();
_connectionChecker = MockRepository.GenerateMock <IConnectionChecker>();
_sleeper = MockRepository.GenerateMock <ISleeper>();
_logger = MockRepository.GenerateMock <ILogger>();
_serviceStopper = MockRepository.GenerateMock <IServiceStopper>();
_serviceStarter = MockRepository.GenerateMock <IServiceStarter>();
_agentUpdateMessageHandler = new AgentUpdateMessageHandler();
_logger.Stub(x => x.Log(Arg <string> .Is.Anything));
_updaterUpdate = new UpdaterUpdate(_sleeper, _downloader, _checksumValidator, _unzipper, _fileCopier, _finalizer, _serviceStopper, _serviceStarter, _connectionChecker, _agentUpdateMessageHandler, _logger);
}
public void Setup()
{
_agentUpdateInfo = "http://something.com/file.zip,544564abc453de787ad";
_downloader = MockRepository.GenerateMock<IDownloader>();
_checksumValidator = MockRepository.GenerateMock<IChecksumValidator>();
_unzipper = MockRepository.GenerateMock<IUnzipper>();
_fileCopier = MockRepository.GenerateMock<IFileCopier>();
_finalizer = MockRepository.GenerateMock<IFinalizer>();
_connectionChecker = MockRepository.GenerateMock<IConnectionChecker>();
_sleeper = MockRepository.GenerateMock<ISleeper>();
_logger = MockRepository.GenerateMock<ILogger>();
_serviceStopper = MockRepository.GenerateMock<IServiceStopper>();
_serviceStarter = MockRepository.GenerateMock<IServiceStarter>();
_agentUpdateMessageHandler = new AgentUpdateMessageHandler();
_logger.Stub(x => x.Log(Arg<string>.Is.Anything));
_updaterUpdate = new UpdaterUpdate(_sleeper, _downloader, _checksumValidator, _unzipper, _fileCopier, _finalizer, _serviceStopper, _serviceStarter, _connectionChecker, _agentUpdateMessageHandler, _logger);
}
public FileSynchronizer(
IAppConfig appConfig,
ILogger <FileSynchronizer> logger,
IDirectoryStructureComparer directoryStructureComparer,
IFileComparer fileComparer,
IFileCopier fileCopier,
IFileDeleter fileDeleter,
IFileMerger fileMerger,
IFileFilter fileFilter)
{
_appConfig = appConfig ?? throw new ArgumentNullException(nameof(appConfig));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_directoryStructureComparer = directoryStructureComparer ?? throw new ArgumentNullException(nameof(directoryStructureComparer));
_fileComparer = fileComparer ?? throw new ArgumentNullException(nameof(fileComparer));
_fileCopier = fileCopier ?? throw new ArgumentNullException(nameof(fileCopier));
_fileDeleter = fileDeleter ?? throw new ArgumentNullException(nameof(fileDeleter));
_fileMerger = fileMerger ?? throw new ArgumentNullException(nameof(fileMerger));
_fileFilter = fileFilter;
_srcFileSystem = _appConfig.Src;
_destFileSystem = _appConfig.Dest;
}
/// <nodoc />
public DistributedContentStore(
byte[] localMachineLocation,
Func <NagleQueue <ContentHash>, DistributedEvictionSettings, ContentStoreSettings, TrimBulkAsync, IContentStore> innerContentStoreFunc,
IContentLocationStoreFactory contentLocationStoreFactory,
IFileExistenceChecker <T> fileExistenceChecker,
IFileCopier <T> fileCopier,
IPathTransformer <T> pathTransform,
ICopyRequester copyRequester,
ReadOnlyDistributedContentSession <T> .ContentAvailabilityGuarantee contentAvailabilityGuarantee,
AbsolutePath tempFolderForCopies,
IAbsFileSystem fileSystem,
int locationStoreBatchSize,
DistributedContentStoreSettings settings,
int?replicaCreditInMinutes = null,
IClock clock = null,
bool enableRepairHandling = false,
TimeSpan?contentHashBumpTime = null,
ContentStoreSettings contentStoreSettings = null)
{
Contract.Requires(settings != null);
LocalMachineLocation = new MachineLocation(localMachineLocation);
_enableRepairHandling = enableRepairHandling;
_contentLocationStoreFactory = contentLocationStoreFactory;
_contentAvailabilityGuarantee = contentAvailabilityGuarantee;
_locationStoreBatchSize = locationStoreBatchSize;
contentStoreSettings = contentStoreSettings ?? ContentStoreSettings.DefaultSettings;
_settings = settings;
// Queue is created in unstarted state because the eviction function
// requires the context passed at startup.
_evictionNagleQueue = NagleQueue <ContentHash> .CreateUnstarted(
Redis.RedisContentLocationStoreConstants.BatchDegreeOfParallelism,
Redis.RedisContentLocationStoreConstants.BatchInterval,
_locationStoreBatchSize);
_distributedCopierFactory = (contentLocationStore) =>
{
return(new DistributedContentCopier <T>(
tempFolderForCopies,
_settings,
fileSystem,
fileCopier,
fileExistenceChecker,
copyRequester,
pathTransform,
contentLocationStore));
};
_enableDistributedEviction = replicaCreditInMinutes != null;
var distributedEvictionSettings = _enableDistributedEviction ? SetUpDistributedEviction(replicaCreditInMinutes, locationStoreBatchSize) : null;
var enableTouch = contentHashBumpTime.HasValue;
if (enableTouch)
{
_contentTrackerUpdater = new ContentTrackerUpdater(ScheduleBulkTouch, contentHashBumpTime.Value, clock: clock);
}
TrimBulkAsync trimBulkAsync = null;
InnerContentStore = innerContentStoreFunc(_evictionNagleQueue, distributedEvictionSettings, contentStoreSettings, trimBulkAsync);
if (settings.PinConfiguration?.UsePinCache == true)
{
_pinCache = new PinCache(clock: clock);
}
}
public DiffProcessor(IFileCopier fileCopier)
{
IsNotNull(fileCopier);
_fileCopier = fileCopier;
}
protected override Task <CopyFileResult> CopyFileAsync(IFileCopier <AbsolutePath> copier, AbsolutePath sourcePath, AbsolutePath destinationPath, long expectedContentSize, bool overwrite, CancellationToken cancellationToken)
{
return(copier.CopyToAsync(sourcePath, null, expectedContentSize, cancellationToken));
}
public FileWatcher(IFileCopier fileCopier, ILogger logger)
{
_fileCopier = fileCopier;
_logger = logger;
}
public FolderCopier(IFileCopier fileCopier)
{
_fileCopier = fileCopier ?? new FileCopier();
}
public BackupUpdater(ILogger logger, IFileCopier fileCopier)
{
_logger = logger;
_fileCopier = fileCopier;
}
public SimpleFileMerger(IFileCopier fileCopier, IFileDeleter fileDeleter)
{
_fileCopier = fileCopier ?? throw new ArgumentNullException(nameof(fileCopier));
_fileDeleter = fileDeleter ?? throw new ArgumentNullException(nameof(fileDeleter));
}
public BackupUpdater(ILogger logger, IFileCopier fileCopier)
{
_logger = logger;
_fileCopier = fileCopier;
}
/// <nodoc />
public BandwidthCheckedCopier(IFileCopier <T> inner, BandwidthChecker.Configuration config, ILogger logger)
{
_inner = inner;
_checker = new BandwidthChecker(config);
_logger = logger;
}
public FileCopyQueueConsumer(
[NotNull] IFileCopier fileCopier)
{
_fileCopier = fileCopier;
}
