protected async Task <bool> GetSignatureBlobForTransfer()
{
DownloadOptions options = new DownloadOptions();
SingleTransferContext context = new SingleTransferContext();
context.LogLevel = LogLevel.Informational;
context.SetAttributesCallback = (destination) =>
{
SignatureBlob = destination as CloudBlockBlob;
};
context.FileTransferred += Context_BlobTransferred;
context.FileFailed += Context_BlobFailed;
try
{
ICloudBlob sig = await DestinationStorage.GetCloudBlobAsync(DestinationContainerName, SignatureBlobName);
L.Info("Azure Storage blob signature has size {size}.", sig.Properties.Length, sig.Name);
context.ProgressHandler = new SingleBlobTransferProgressReporter(sig);
if (sig.Properties.BlobType == BlobType.BlockBlob)
{
SignatureBlob = (CloudBlockBlob)sig;
}
using (Operation azOp = L.Begin("Download signature"))
using (MemoryStream ms = new MemoryStream())
{
DownloadSignatureTask = TransferManager.DownloadAsync(SignatureBlob, ms);
await DownloadSignatureTask;
if (DownloadSignatureTask.Status == TaskStatus.RanToCompletion)
{
IFormatter formatter = new BinaryFormatter();
Signature = (SingleFileSignature)formatter.Deserialize(ms);
ComputeSignatureTask = Task.CompletedTask;
L.Info("Using Azure Storage blob signature for synchroniztion built on {date}.", Signature.ComputedDateTime);
azOp.Complete();
return(true);
}
else
{
return(false);
}
}
}
catch (Exception e)
{
LogTransferException(e, "download signature");
if (e is AggregateException && e.InnerException != null && (e.InnerException is TaskCanceledException || e.InnerException is OperationCanceledException || e.InnerException is TransferSkippedException))
{
return(true);
}
else if (e is TaskCanceledException || e is OperationCanceledException || e is TransferSkippedException)
{
return(true);
}
else
{
return(false);
}
}
}
protected async Task <bool> UploadSignature()
{
UploadOptions options = new UploadOptions();
SingleTransferContext context = new SingleTransferContext();
bool transferred = false;
context.LogLevel = LogLevel.Informational;
context.ProgressHandler = new SingleFileTransferProgressReporter(SignatureFile);
context.SetAttributesCallback = (destination) =>
{
SignatureBlob = destination as CloudBlockBlob;
SignatureBlob.Properties.ContentType = "application/octet-stream";
};
context.ShouldOverwriteCallback = (source, destination) =>
{
return(true);
};
context.FileTransferred += (sender, e) =>
{
Context_FileTransferred(sender, e);
transferred = true;
};
context.FileFailed += (sender, e) =>
{
Context_FileFailed(sender, e);
transferred = false;
};
context.FileSkipped += (sender, e) =>
{
Context_FileSkipped(sender, e);
transferred = false;
};
try
{
SignatureBlob = await DestinationStorage.GetorCreateCloudBlobAsync(DestinationContainerName, DestinationBlob.Name + ".sig", BlobType.BlockBlob) as CloudBlockBlob;
if (await SignatureBlob.ExistsAsync())
{
L.Warn("The existing signature blob {blob} will be overwritten.", SignatureBlob.Name);
}
await TransferManager.UploadAsync(SignatureFile.FullName, SignatureBlob, options, context, CT);
transferred = true;
}
catch (Exception e)
{
LogTransferException(e, $"upload signature to cloud blob {SignatureBlob.Name}");
if (e is OperationCanceledException || e is TaskCanceledException || (e is TransferException && CT.IsCancellationRequested))
{
transferred = true;
}
else
{
transferred = false;
}
}
return(transferred);
}
public override void Process()
{
Directory destinationDirectory = DestinationStorage.GetDirectory(DestinationPath);
File destination = destinationDirectory.CreateFile(DestinationName);
using (Stream sourceStream = Source.Open(FileAccess.Read))
using (Stream destinationStream = destination.Open(FileAccess.Write))
sourceStream.CopyTo(destinationStream);
destination.Date = Source.Date;
}
public FileTransferServiceTestData()
{
OriginalStorage.GetFileContent(Arg.Any <File>()).Returns(new System.IO.MemoryStream());
DestinationStorage.SaveFileAsync(Arg.Any <File>(), Arg.Any <System.IO.MemoryStream>()).Returns(Task.CompletedTask);
TestFiles = new List <File>
{
new File("/folder/file1.txt", 100),
new File("/file1.txt", 100)
};
CompareResult = new StorageCompareResult(TestFiles);
}
public ModuleModel()
{
//TODO: inject logger!
//DI is for Dependency injection!
sheets = new SheetCache();
destinations = new DestinationStorage();
//TODO:Конфигурация!
sorter = new GroupsSorter <Sheet>("Формат");
PropertiesExractor propertiesExractor = new PropertiesExractor();
sheetExtractor = new SheetsExtractor(propertiesExractor, new SheetCreator(new BoundsCalculator()));
destinationsExtractor = new DestinationsExtractor(propertiesExractor, new DestinationCreator());
}
public override void ExecuteCommand()
{
if (!String.IsNullOrEmpty(DatabaseName))
{
ConnectionString.InitialCatalog = DatabaseName;
}
Log.Info("Exporting {0} on {1} to {2}", ConnectionString.InitialCatalog, Util.GetDatabaseServerName(ConnectionString), DestinationStorage.Credentials.AccountName);
string serverName = ConnectionString.DataSource;
if (serverName.StartsWith("tcp:"))
{
serverName = serverName.Substring(4);
}
WASDImportExport.ImportExportHelper helper = new WASDImportExport.ImportExportHelper(Log)
{
EndPointUri = SqlDacEndpoint.AbsoluteUri,
DatabaseName = ConnectionString.InitialCatalog,
ServerName = serverName,
UserName = ConnectionString.UserID,
Password = ConnectionString.Password,
StorageKey = Convert.ToBase64String(DestinationStorage.Credentials.ExportKey())
};
// Prep the blob
string blobUrl = null;
if (!WhatIf)
{
var client = DestinationStorage.CreateCloudBlobClient();
var container = client.GetContainerReference(DestinationContainer);
container.CreateIfNotExists();
var blob = container.GetBlockBlobReference(ConnectionString.InitialCatalog + ".bacpac");
if (blob.Exists())
{
Log.Info("Skipping export of {0} because the blob already exists", blob.Name);
}
else
{
Log.Info("Starting export to {0}", blob.Uri.AbsoluteUri);
// Export!
blobUrl = helper.DoExport(blob.Uri.AbsoluteUri, WhatIf);
}
}
Log.Info("Export Complete");
}
public override void ExecuteCommand()
{
var destClient = DestinationStorage.CreateCloudBlobClient();
var destContainer = destClient.GetContainerReference(DestinationContainer);
// Iterate through the blobs
int index = 0;
var blobs = Util.EnumerateBlobs(Log, destContainer, Prefix ?? String.Empty);
Parallel.ForEach(blobs, new ParallelOptions()
{
MaxDegreeOfParallelism = 10
}, blob =>
{
Interlocked.Increment(ref index);
if (blob.CopyState.Status != CopyStatus.Pending)
{
DateTime start = DateTime.UtcNow;
int counter = 0;
while (blob.CopyState.Status == CopyStatus.Pending)
{
Thread.Sleep(1000);
counter++;
blob.FetchAttributes();
if (counter % 5 == 0)
{
Log.Info("{1}Waiting on {0} ...", blob.Name, counter > 5 ? "Still " : "");
}
}
if (counter > 2)
{
Log.Info("Copy of {0} has finished!", blob.Name);
}
}
index++;
});
Log.Info("{0} Copies Complete!", index);
}
public override void ExecuteCommand()
{
var sourceClient = SourceStorage.CreateCloudBlobClient();
var sourceContainer = sourceClient.GetContainerReference(SourceContainer);
if (!sourceContainer.Exists())
{
Log.Warn("No blobs in container!");
return;
}
var destClient = DestinationStorage.CreateCloudBlobClient();
var destContainer = destClient.GetContainerReference(DestinationContainer);
destContainer.CreateIfNotExists();
Log.Info("Collecting blob names in {0} to copy to {1}", SourceStorage.Credentials.AccountName, DestinationStorage.Credentials.AccountName);
var blobs = Util.CollectBlobs(
Log,
sourceContainer,
Prefix ?? String.Empty,
condition: b => (!PackageBlobsOnly || (!b.Name.StartsWith("/") && String.Equals(b.Name.ToLowerInvariant(), b.Name, StringComparison.Ordinal))),
countEstimate: 140000);
var count = blobs.Count;
int index = 0;
Parallel.ForEach(blobs, new ParallelOptions {
MaxDegreeOfParallelism = 10
}, blob =>
{
int currentIndex = Interlocked.Increment(ref index);
var percentage = (((double)currentIndex / (double)count) * 100);
var destBlob = destContainer.GetBlockBlobReference(blob.Name);
try
{
if (!destBlob.Exists() || Overwrite)
{
Log.Info("[{1:000000}/{2:000000}] ({3:000.00}%) Started Async Copy of {0}.", blob.Name, currentIndex, count, percentage);
if (!WhatIf)
{
destBlob.StartCopyFromBlob(blob);
}
}
else
{
Log.Info("[{1:000000}/{2:000000}] ({3:000.00}%) Skipped {0}. Blob already Exists", blob.Name, index, count, percentage);
}
}
catch (StorageException stex)
{
if (stex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict &&
stex.RequestInformation.ExtendedErrorInformation.ErrorCode == BlobErrorCodeStrings.PendingCopyOperation)
{
Log.Info("[{1:000000}/{2:000000}] ({3:000.00}%) Skipped {0}. Already being copied", blob.Name, index, count, percentage);
}
}
catch (Exception ex)
{
Log.Error("Error processing {0}: {1}", blob.Name, ex.ToString());
throw;
}
});
Log.Info("Copies started. Run checkblobcopies with similar parameters to wait on blob copy completion");
}
/// <summary>
/// コンストラクタ
/// </summary>
/// <param name="projectDir"></param>
public SiteGenerator(string src, string dest, bool isPreviewMode)
{
_srcRepository = new SourceStorage(src, isPreviewMode);
_outputRepository = new DestinationStorage(dest, isPreviewMode);
}
protected async Task <bool> UploadSingleFile(string fileName, bool writeSignature = false)
{
FileInfo file = new FileInfo(fileName);
L.Info("{file} has size {size}", file.FullName, PrintBytes(file.Length, ""));
if (string.IsNullOrEmpty(JournalFilePath))
{
JournalFilePath = file.FullName + ".azsj";
}
if (writeSignature && string.IsNullOrEmpty(SignatureFilePath))
{
SignatureFilePath = file.FullName + ".sig";
}
OpenTransferJournal();
UploadOptions options = new UploadOptions();
SingleTransferContext context = new SingleTransferContext(JournalStream);
context.LogLevel = LogLevel.Informational;
context.ProgressHandler = new SingleFileTransferProgressReporter(file);
context.SetAttributesCallback = (destination) =>
{
DestinationBlob = destination as CloudBlob;
DestinationBlob.Properties.ContentType = this.ContentType;
};
context.ShouldOverwriteCallback = (source, destination) =>
{
CloudBlob b = (CloudBlob)destination;
bool o = this.Overwrite;
if (o)
{
L.Info("Overwriting existing blob {container}/{blob}.", DestinationContainerName, b.Name);
}
else
{
L.Warn("Not overwriting existing blob {container}/{blob}.", DestinationContainerName, b.Name);
}
return(o);
};
context.FileTransferred += Context_FileTransferred;
context.FileFailed += Context_FileFailed;
context.FileSkipped += Context_FileSkipped;
try
{
DestinationBlob = await DestinationStorage.GetorCreateCloudBlobAsync(DestinationContainerName, file.Name, BlobType.BlockBlob);
if (writeSignature)
{
Signature = new SingleFileSignature(file, DestinationBlob);
ComputeSignatureTask = Task.Factory.StartNew(() => Signature.Compute(), CT, TaskCreationOptions.None, TaskScheduler.Default);
}
else
{
ComputeSignatureTask = Task.CompletedTask;
}
using (Operation azOp = L.Begin("Upload file"))
{
TransferTask = TransferManager.UploadAsync(file.FullName, DestinationBlob, options, context, CT);
Task.WaitAll(TransferTask, ComputeSignatureTask);
if (TransferTask.Status == TaskStatus.RanToCompletion)
{
if (!writeSignature)
{
azOp.Complete();
return(true);
}
else
{
azOp.Complete();
}
}
else
{
return(false);
}
if (writeSignature && ComputeSignatureTask.Status == TaskStatus.RanToCompletion)
{
L.Info("Compute file signature has length {0} bytes.", Signature.ComputedSignature.Length);
}
else if (writeSignature)
{
L.Error("Could not compute signature for file {file}.", file.FullName);
return(false);
}
}
}
catch (Exception e)
{
LogTransferException(e, "upload file");
if (e is AggregateException && e.InnerException != null && (e.InnerException is TaskCanceledException || e.InnerException is OperationCanceledException || e.InnerException is TransferSkippedException))
{
return(true);
}
else if (e is TaskCanceledException || e is OperationCanceledException || e is TransferSkippedException)
{
return(true);
}
else
{
return(false);
}
}
finally
{
CloseTransferJournal();
}
using (Operation azOp = L.Begin("Upload and write file signature to disk"))
{
WriteSignatureTask = Task.Run(() => WriteSignatureToFile(), CT);
try
{
WriteSignatureTask.Wait();
if (WriteSignatureTask.Result)
{
UploadSignatureTask = UploadSignature();
if (await UploadSignatureTask)
{
azOp.Complete();
return(true);
}
else
{
return(false);
}
}
else
{
return(false);
}
}
catch (Exception e)
{
LogTransferException(e, "upload and write file signature to disk");
azOp.Cancel();
if (e is TaskCanceledException || e is OperationCanceledException)
{
return(true);
}
else
{
return(false);
}
}
}
}
public Task<BackgroundDownloadPackage> CreateDownloadPackage(Uri uri, DestinationStorage destination, string fileNameAtDestination)
{
throw new NotImplementedException();
}
public Task <BackgroundDownloadPackage> CreateDownloadPackage(Uri uri, DestinationStorage destination, string fileNameAtDestination)
{
throw new NotImplementedException();
}