protected async Task <bool> GetSignatureBlobForTransfer() { DownloadOptions options = new DownloadOptions(); SingleTransferContext context = new SingleTransferContext(); context.LogLevel = LogLevel.Informational; context.SetAttributesCallback = (destination) => { SignatureBlob = destination as CloudBlockBlob; }; context.FileTransferred += Context_BlobTransferred; context.FileFailed += Context_BlobFailed; try { ICloudBlob sig = await DestinationStorage.GetCloudBlobAsync(DestinationContainerName, SignatureBlobName); L.Info("Azure Storage blob signature has size {size}.", sig.Properties.Length, sig.Name); context.ProgressHandler = new SingleBlobTransferProgressReporter(sig); if (sig.Properties.BlobType == BlobType.BlockBlob) { SignatureBlob = (CloudBlockBlob)sig; } using (Operation azOp = L.Begin("Download signature")) using (MemoryStream ms = new MemoryStream()) { DownloadSignatureTask = TransferManager.DownloadAsync(SignatureBlob, ms); await DownloadSignatureTask; if (DownloadSignatureTask.Status == TaskStatus.RanToCompletion) { IFormatter formatter = new BinaryFormatter(); Signature = (SingleFileSignature)formatter.Deserialize(ms); ComputeSignatureTask = Task.CompletedTask; L.Info("Using Azure Storage blob signature for synchroniztion built on {date}.", Signature.ComputedDateTime); azOp.Complete(); return(true); } else { return(false); } } } catch (Exception e) { LogTransferException(e, "download signature"); if (e is AggregateException && e.InnerException != null && (e.InnerException is TaskCanceledException || e.InnerException is OperationCanceledException || e.InnerException is TransferSkippedException)) { return(true); } else if (e is TaskCanceledException || e is OperationCanceledException || e is TransferSkippedException) { return(true); } else { return(false); } } }
protected async Task <bool> UploadSignature() { UploadOptions options = new UploadOptions(); SingleTransferContext context = new SingleTransferContext(); bool transferred = false; context.LogLevel = LogLevel.Informational; context.ProgressHandler = new SingleFileTransferProgressReporter(SignatureFile); context.SetAttributesCallback = (destination) => { SignatureBlob = destination as CloudBlockBlob; SignatureBlob.Properties.ContentType = "application/octet-stream"; }; context.ShouldOverwriteCallback = (source, destination) => { return(true); }; context.FileTransferred += (sender, e) => { Context_FileTransferred(sender, e); transferred = true; }; context.FileFailed += (sender, e) => { Context_FileFailed(sender, e); transferred = false; }; context.FileSkipped += (sender, e) => { Context_FileSkipped(sender, e); transferred = false; }; try { SignatureBlob = await DestinationStorage.GetorCreateCloudBlobAsync(DestinationContainerName, DestinationBlob.Name + ".sig", BlobType.BlockBlob) as CloudBlockBlob; if (await SignatureBlob.ExistsAsync()) { L.Warn("The existing signature blob {blob} will be overwritten.", SignatureBlob.Name); } await TransferManager.UploadAsync(SignatureFile.FullName, SignatureBlob, options, context, CT); transferred = true; } catch (Exception e) { LogTransferException(e, $"upload signature to cloud blob {SignatureBlob.Name}"); if (e is OperationCanceledException || e is TaskCanceledException || (e is TransferException && CT.IsCancellationRequested)) { transferred = true; } else { transferred = false; } } return(transferred); }
public override void Process() { Directory destinationDirectory = DestinationStorage.GetDirectory(DestinationPath); File destination = destinationDirectory.CreateFile(DestinationName); using (Stream sourceStream = Source.Open(FileAccess.Read)) using (Stream destinationStream = destination.Open(FileAccess.Write)) sourceStream.CopyTo(destinationStream); destination.Date = Source.Date; }
public FileTransferServiceTestData() { OriginalStorage.GetFileContent(Arg.Any <File>()).Returns(new System.IO.MemoryStream()); DestinationStorage.SaveFileAsync(Arg.Any <File>(), Arg.Any <System.IO.MemoryStream>()).Returns(Task.CompletedTask); TestFiles = new List <File> { new File("/folder/file1.txt", 100), new File("/file1.txt", 100) }; CompareResult = new StorageCompareResult(TestFiles); }
public ModuleModel() { //TODO: inject logger! //DI is for Dependency injection! sheets = new SheetCache(); destinations = new DestinationStorage(); //TODO:Конфигурация! sorter = new GroupsSorter <Sheet>("Формат"); PropertiesExractor propertiesExractor = new PropertiesExractor(); sheetExtractor = new SheetsExtractor(propertiesExractor, new SheetCreator(new BoundsCalculator())); destinationsExtractor = new DestinationsExtractor(propertiesExractor, new DestinationCreator()); }
public override void ExecuteCommand() { if (!String.IsNullOrEmpty(DatabaseName)) { ConnectionString.InitialCatalog = DatabaseName; } Log.Info("Exporting {0} on {1} to {2}", ConnectionString.InitialCatalog, Util.GetDatabaseServerName(ConnectionString), DestinationStorage.Credentials.AccountName); string serverName = ConnectionString.DataSource; if (serverName.StartsWith("tcp:")) { serverName = serverName.Substring(4); } WASDImportExport.ImportExportHelper helper = new WASDImportExport.ImportExportHelper(Log) { EndPointUri = SqlDacEndpoint.AbsoluteUri, DatabaseName = ConnectionString.InitialCatalog, ServerName = serverName, UserName = ConnectionString.UserID, Password = ConnectionString.Password, StorageKey = Convert.ToBase64String(DestinationStorage.Credentials.ExportKey()) }; // Prep the blob string blobUrl = null; if (!WhatIf) { var client = DestinationStorage.CreateCloudBlobClient(); var container = client.GetContainerReference(DestinationContainer); container.CreateIfNotExists(); var blob = container.GetBlockBlobReference(ConnectionString.InitialCatalog + ".bacpac"); if (blob.Exists()) { Log.Info("Skipping export of {0} because the blob already exists", blob.Name); } else { Log.Info("Starting export to {0}", blob.Uri.AbsoluteUri); // Export! blobUrl = helper.DoExport(blob.Uri.AbsoluteUri, WhatIf); } } Log.Info("Export Complete"); }
public override void ExecuteCommand() { var destClient = DestinationStorage.CreateCloudBlobClient(); var destContainer = destClient.GetContainerReference(DestinationContainer); // Iterate through the blobs int index = 0; var blobs = Util.EnumerateBlobs(Log, destContainer, Prefix ?? String.Empty); Parallel.ForEach(blobs, new ParallelOptions() { MaxDegreeOfParallelism = 10 }, blob => { Interlocked.Increment(ref index); if (blob.CopyState.Status != CopyStatus.Pending) { DateTime start = DateTime.UtcNow; int counter = 0; while (blob.CopyState.Status == CopyStatus.Pending) { Thread.Sleep(1000); counter++; blob.FetchAttributes(); if (counter % 5 == 0) { Log.Info("{1}Waiting on {0} ...", blob.Name, counter > 5 ? "Still " : ""); } } if (counter > 2) { Log.Info("Copy of {0} has finished!", blob.Name); } } index++; }); Log.Info("{0} Copies Complete!", index); }
public override void ExecuteCommand() { var sourceClient = SourceStorage.CreateCloudBlobClient(); var sourceContainer = sourceClient.GetContainerReference(SourceContainer); if (!sourceContainer.Exists()) { Log.Warn("No blobs in container!"); return; } var destClient = DestinationStorage.CreateCloudBlobClient(); var destContainer = destClient.GetContainerReference(DestinationContainer); destContainer.CreateIfNotExists(); Log.Info("Collecting blob names in {0} to copy to {1}", SourceStorage.Credentials.AccountName, DestinationStorage.Credentials.AccountName); var blobs = Util.CollectBlobs( Log, sourceContainer, Prefix ?? String.Empty, condition: b => (!PackageBlobsOnly || (!b.Name.StartsWith("/") && String.Equals(b.Name.ToLowerInvariant(), b.Name, StringComparison.Ordinal))), countEstimate: 140000); var count = blobs.Count; int index = 0; Parallel.ForEach(blobs, new ParallelOptions { MaxDegreeOfParallelism = 10 }, blob => { int currentIndex = Interlocked.Increment(ref index); var percentage = (((double)currentIndex / (double)count) * 100); var destBlob = destContainer.GetBlockBlobReference(blob.Name); try { if (!destBlob.Exists() || Overwrite) { Log.Info("[{1:000000}/{2:000000}] ({3:000.00}%) Started Async Copy of {0}.", blob.Name, currentIndex, count, percentage); if (!WhatIf) { destBlob.StartCopyFromBlob(blob); } } else { Log.Info("[{1:000000}/{2:000000}] ({3:000.00}%) Skipped {0}. Blob already Exists", blob.Name, index, count, percentage); } } catch (StorageException stex) { if (stex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict && stex.RequestInformation.ExtendedErrorInformation.ErrorCode == BlobErrorCodeStrings.PendingCopyOperation) { Log.Info("[{1:000000}/{2:000000}] ({3:000.00}%) Skipped {0}. Already being copied", blob.Name, index, count, percentage); } } catch (Exception ex) { Log.Error("Error processing {0}: {1}", blob.Name, ex.ToString()); throw; } }); Log.Info("Copies started. Run checkblobcopies with similar parameters to wait on blob copy completion"); }
/// <summary> /// コンストラクタ /// </summary> /// <param name="projectDir"></param> public SiteGenerator(string src, string dest, bool isPreviewMode) { _srcRepository = new SourceStorage(src, isPreviewMode); _outputRepository = new DestinationStorage(dest, isPreviewMode); }
protected async Task <bool> UploadSingleFile(string fileName, bool writeSignature = false) { FileInfo file = new FileInfo(fileName); L.Info("{file} has size {size}", file.FullName, PrintBytes(file.Length, "")); if (string.IsNullOrEmpty(JournalFilePath)) { JournalFilePath = file.FullName + ".azsj"; } if (writeSignature && string.IsNullOrEmpty(SignatureFilePath)) { SignatureFilePath = file.FullName + ".sig"; } OpenTransferJournal(); UploadOptions options = new UploadOptions(); SingleTransferContext context = new SingleTransferContext(JournalStream); context.LogLevel = LogLevel.Informational; context.ProgressHandler = new SingleFileTransferProgressReporter(file); context.SetAttributesCallback = (destination) => { DestinationBlob = destination as CloudBlob; DestinationBlob.Properties.ContentType = this.ContentType; }; context.ShouldOverwriteCallback = (source, destination) => { CloudBlob b = (CloudBlob)destination; bool o = this.Overwrite; if (o) { L.Info("Overwriting existing blob {container}/{blob}.", DestinationContainerName, b.Name); } else { L.Warn("Not overwriting existing blob {container}/{blob}.", DestinationContainerName, b.Name); } return(o); }; context.FileTransferred += Context_FileTransferred; context.FileFailed += Context_FileFailed; context.FileSkipped += Context_FileSkipped; try { DestinationBlob = await DestinationStorage.GetorCreateCloudBlobAsync(DestinationContainerName, file.Name, BlobType.BlockBlob); if (writeSignature) { Signature = new SingleFileSignature(file, DestinationBlob); ComputeSignatureTask = Task.Factory.StartNew(() => Signature.Compute(), CT, TaskCreationOptions.None, TaskScheduler.Default); } else { ComputeSignatureTask = Task.CompletedTask; } using (Operation azOp = L.Begin("Upload file")) { TransferTask = TransferManager.UploadAsync(file.FullName, DestinationBlob, options, context, CT); Task.WaitAll(TransferTask, ComputeSignatureTask); if (TransferTask.Status == TaskStatus.RanToCompletion) { if (!writeSignature) { azOp.Complete(); return(true); } else { azOp.Complete(); } } else { return(false); } if (writeSignature && ComputeSignatureTask.Status == TaskStatus.RanToCompletion) { L.Info("Compute file signature has length {0} bytes.", Signature.ComputedSignature.Length); } else if (writeSignature) { L.Error("Could not compute signature for file {file}.", file.FullName); return(false); } } } catch (Exception e) { LogTransferException(e, "upload file"); if (e is AggregateException && e.InnerException != null && (e.InnerException is TaskCanceledException || e.InnerException is OperationCanceledException || e.InnerException is TransferSkippedException)) { return(true); } else if (e is TaskCanceledException || e is OperationCanceledException || e is TransferSkippedException) { return(true); } else { return(false); } } finally { CloseTransferJournal(); } using (Operation azOp = L.Begin("Upload and write file signature to disk")) { WriteSignatureTask = Task.Run(() => WriteSignatureToFile(), CT); try { WriteSignatureTask.Wait(); if (WriteSignatureTask.Result) { UploadSignatureTask = UploadSignature(); if (await UploadSignatureTask) { azOp.Complete(); return(true); } else { return(false); } } else { return(false); } } catch (Exception e) { LogTransferException(e, "upload and write file signature to disk"); azOp.Cancel(); if (e is TaskCanceledException || e is OperationCanceledException) { return(true); } else { return(false); } } } }
public Task<BackgroundDownloadPackage> CreateDownloadPackage(Uri uri, DestinationStorage destination, string fileNameAtDestination) { throw new NotImplementedException(); }
public Task <BackgroundDownloadPackage> CreateDownloadPackage(Uri uri, DestinationStorage destination, string fileNameAtDestination) { throw new NotImplementedException(); }