private async Task LoadMediaFiles()
{
MediaItems.Clear();
var localFiles = await _localMediaFolder.GetFilesAsync();
var localMediaItems = localFiles.Select(file => new MediaItem(FolderType.Local, ReservationId, file.Name));
foreach (var mediaItem in localMediaItems)
{
MediaItems.Add(mediaItem);
}
UpdateOperationsCount(+1);
var files = await _data.GetAzureStorageFilesByTrip(ReservationId);
foreach (var fileMetadata in files)
{
StorageFile destinationFile = await _tempMediaFolder.CreateFileAsync(fileMetadata.FileName, CreationCollisionOption.ReplaceExisting);
await _transferManager.DownloadAsync(fileMetadata.Uri, destinationFile);
var mediaItem = new MediaItem(FolderType.Temp, ReservationId, fileMetadata.FileName);
MediaItems.Add(mediaItem);
}
UpdateOperationsCount(-1);
}
/// 高级接口下载对象
public async void TransferDownloadObject()
{
//.cssg-snippet-body-start:[transfer-download-object]
// 初始化 TransferConfig
TransferConfig transferConfig = new TransferConfig();
// 初始化 TransferManager
TransferManager transferManager = new TransferManager(cosXml, transferConfig);
String bucket = "examplebucket-1250000000"; //存储桶，格式：BucketName-APPID
String cosPath = "exampleobject"; //对象在存储桶中的位置标识符，即称对象键
string localDir = System.IO.Path.GetTempPath(); //本地文件夹
string localFileName = "my-local-temp-file"; //指定本地保存的文件名
// 下载对象
COSXMLDownloadTask downloadTask = new COSXMLDownloadTask(bucket, cosPath,
localDir, localFileName);
downloadTask.progressCallback = delegate(long completed, long total)
{
Console.WriteLine(String.Format("progress = {0:##.##}%", completed * 100.0 / total));
};
try {
COSXML.Transfer.COSXMLDownloadTask.DownloadTaskResult result = await
transferManager.DownloadAsync(downloadTask);
Console.WriteLine(result.GetResultInfo());
string eTag = result.eTag;
} catch (Exception e) {
Console.WriteLine("CosException: " + e);
}
//.cssg-snippet-body-end
}
internal static async Task GetObject(CosXmlServer cosXml, string cosKey)
{
TransferConfig transferConfig = new TransferConfig();
// 初始化 TransferManager
TransferManager transferManager = new TransferManager(cosXml, transferConfig);
String cosPath = cosKey; //对象在存储桶中的位置标识符，即称对象键
string localDir = System.IO.Path.GetTempPath(); //本地文件夹
string localFileName = "my-local-temp-file"; //指定本地保存的文件名
// 下载对象
COSXMLDownloadTask downloadTask = new COSXMLDownloadTask(bucket, cosPath,
localDir, localFileName);
downloadTask.progressCallback = delegate(long completed, long total)
{
Console.WriteLine(String.Format("progress = {0:##.##}%", completed * 100.0 / total));
};
try {
COSXML.Transfer.COSXMLDownloadTask.DownloadTaskResult result = await
transferManager.DownloadAsync(downloadTask);
Console.WriteLine(result.GetResultInfo());
string eTag = result.eTag;
} catch (Exception e) {
Console.WriteLine("CosException: " + e);
}
}
public async Task DownloadAzureBlobToLocalFile(string DestinationFullFilePath, string ContainerName, string BlobName)
{
CloudBlockBlob blob = GetBlob(ContainerName, BlobName);
TransferCheckpoint checkpoint = null;
SingleTransferContext context = GetSingleTransferContext(checkpoint);
CancellationTokenSource cancellationSource = new CancellationTokenSource(10000000);
Stopwatch stopWatch = Stopwatch.StartNew();
Task task;
try
{
task = TransferManager.DownloadAsync(blob, DestinationFullFilePath, null, context, cancellationSource.Token); //(LocalSourceFilePath, blob, null, context, cancellationSource.Token);
await task;
}
catch (Exception ex)
{
if (Error != null)
{
Error(ex);
}
}
if (cancellationSource.IsCancellationRequested)
{
//autoretry
}
stopWatch.Stop();
}
protected async Task <bool> GetSignatureBlobForTransfer()
{
DownloadOptions options = new DownloadOptions();
SingleTransferContext context = new SingleTransferContext();
context.LogLevel = LogLevel.Informational;
context.SetAttributesCallback = (destination) =>
{
SignatureBlob = destination as CloudBlockBlob;
};
context.FileTransferred += Context_BlobTransferred;
context.FileFailed += Context_BlobFailed;
try
{
ICloudBlob sig = await DestinationStorage.GetCloudBlobAsync(DestinationContainerName, SignatureBlobName);
L.Info("Azure Storage blob signature has size {size}.", sig.Properties.Length, sig.Name);
context.ProgressHandler = new SingleBlobTransferProgressReporter(sig);
if (sig.Properties.BlobType == BlobType.BlockBlob)
{
SignatureBlob = (CloudBlockBlob)sig;
}
using (Operation azOp = L.Begin("Download signature"))
using (MemoryStream ms = new MemoryStream())
{
DownloadSignatureTask = TransferManager.DownloadAsync(SignatureBlob, ms);
await DownloadSignatureTask;
if (DownloadSignatureTask.Status == TaskStatus.RanToCompletion)
{
IFormatter formatter = new BinaryFormatter();
Signature = (SingleFileSignature)formatter.Deserialize(ms);
ComputeSignatureTask = Task.CompletedTask;
L.Info("Using Azure Storage blob signature for synchroniztion built on {date}.", Signature.ComputedDateTime);
azOp.Complete();
return(true);
}
else
{
return(false);
}
}
}
catch (Exception e)
{
LogTransferException(e, "download signature");
if (e is AggregateException && e.InnerException != null && (e.InnerException is TaskCanceledException || e.InnerException is OperationCanceledException || e.InnerException is TransferSkippedException))
{
return(true);
}
else if (e is TaskCanceledException || e is OperationCanceledException || e is TransferSkippedException)
{
return(true);
}
else
{
return(false);
}
}
}
/// <summary>
/// Download data from Azure storage.
/// 1. Download a CloudBlob to a Stream instance
/// 2. Download the same CloudBlob with step #1 to a different Stream instance
/// 3. Download another CloudBlob to a different stream with content MD5 validation disabled
/// 4. Show the overall progress of all transfers
/// </summary>
private static async Task BlobDownloadToStreamSample()
{
string sourceBlobName1 = "azure_blockblob.png";
string sourceBlobName2 = "azure_blockblob2.png";
// Create the source CloudBlob instances
CloudBlob sourceBlob1 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName1, BlobType.BlockBlob);
CloudBlob sourceBlob2 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName2, BlobType.BlockBlob);
// Create a TransferContext shared by both transfers
SingleTransferContext sharedTransferContext = new SingleTransferContext();
// Record the overall progress
ProgressRecorder recorder = new ProgressRecorder();
sharedTransferContext.ProgressHandler = recorder;
MemoryStream memoryStream1_1 = new MemoryStream();
MemoryStream memoryStream1_2 = new MemoryStream();
MemoryStream memoryStream2 = new MemoryStream();
try
{
// Start the blob download
Task task1 = TransferManager.DownloadAsync(sourceBlob1, memoryStream1_1, null /* options */, sharedTransferContext);
// Start to download the same blob to another Stream
// Please note, DataMovement Library will download blob once for each of downloads.
// For example, if you start two downloads from the same source blob to two different Stream instance,
// DataMovement Library will download the blob content twice.
Task task2 = TransferManager.DownloadAsync(sourceBlob1, memoryStream1_2, null /* options */, sharedTransferContext);
// Create a DownloadOptions to disable md5 check after data is downloaded. Otherwise, data movement
// library will check the md5 checksum stored in the ContentMD5 property of the source CloudFile/CloudBlob
// You can uncomment following codes, enable ContentMD5Validation and have a try.
// sourceBlob2.Properties.ContentMD5 = "WrongMD5";
// sourceBlob2.SetProperties();
DownloadOptions options = new DownloadOptions();
options.DisableContentMD5Validation = true;
// Start the download
Task task3 = TransferManager.DownloadAsync(sourceBlob2, memoryStream2, options, sharedTransferContext);
// Wait for all transfers to finish
await task1;
await task2;
await task3;
// Print out the final transfer state
Console.WriteLine("Final transfer state: {0}", recorder.ToString());
}
finally
{
memoryStream1_1.Dispose();
memoryStream1_2.Dispose();
memoryStream2.Dispose();
}
}
/// <summary>
/// Download a cloud file from azure storage to local folder.
/// </summary>
public static async Task FileDownloadFromAzure(string destinationFileFullPath, string cloudFileName, string accountName, string accountKey, string shareName)
{
// Create the destination CloudFile instance
CloudFile sourceCloudFile = await Util.GetCloudFileAsync(shareName, cloudFileName, accountName, accountKey);
// Start the async download
await TransferManager.DownloadAsync(sourceCloudFile, destinationFileFullPath);
}
public static async Task <dynamic> ExtractAndUploadFiles(ILogger log, CloudFile sourceFileReference,
CloudFileDirectory destinationDirectory)
{
dynamic returnObject = new ExpandoObject();
returnObject.Output = new List <string>();
string archivedFilePath = GetTempFileName();
IArchive archive = null;
try
{
await TransferManager.DownloadAsync(sourceFileReference, archivedFilePath,
new DownloadOptions { DisableContentMD5Validation = true }, new SingleTransferContext());
archive = ArchiveFactory.Open(archivedFilePath);
IReader reader = archive.ExtractAllEntries();
while (reader.MoveToNextEntry())
{
if (reader.Entry.IsDirectory)
{
continue;
}
//The file is a compressed archive container. The library will only decompress the stream to an archive. We need to reprocess the archive to get the contents.
if (reader.Entry.Key == null)
{
string tempArchivedFilePath = GetTempFileName();
reader.WriteEntryToFile(tempArchivedFilePath);
archive.Dispose();
File.Delete(archivedFilePath);
archivedFilePath = tempArchivedFilePath;
archive = ArchiveFactory.Open(archivedFilePath);
reader = archive.ExtractAllEntries();
if (!reader.MoveToNextEntry())
{
break;
}
}
await UploadFile(destinationDirectory, reader, returnObject);
}
}
finally
{
archive?.Dispose();
File.Delete(archivedFilePath);
}
return(returnObject);
}
public async Task <Stream> DownloadBlobAsStreamAsync(string containerName, string blobName)
{
try
{
var blockBlob = GetBlockBlob(containerName, blobName);
var context = PrepareTransfer();
var memoryStream = new MemoryStream();
await TransferManager.DownloadAsync(blockBlob, memoryStream, null, context, CancellationToken.None);
memoryStream.Position = 0;
return(memoryStream);
}
catch (Exception ex)
{
throw new BlobStorageException($"Container: {containerName} Blob: {blobName} - Error while downloading blob.", ex);
}
}
private async void FillCurrentTripMediaItems()
{
int ReservationId = 1;
var files = await _data.GetAzurePublicStorageFilesByLocation(CurrentTrip.DepartureFlight.FlightInfo.Flight.Destination.LocationId);
foreach (var fileMetadata in files)
{
StorageFile file = await _tempMediaFolder.CreateFileAsync(fileMetadata.FileName, CreationCollisionOption.ReplaceExisting);
await _transferManager.DownloadAsync(fileMetadata.Uri, file);
var mediaItem = new MediaItem(FolderType.Temp, _tempFolderBase, fileMetadata.FileName);
CurrentDestinationImages.Add(mediaItem);
if (CurrentDestinationImages.Count == 5)
{
break;
}
}
}
static void retryTransferencia(SingleTransferContext context, CloudBlockBlob cloudBlockBlob, string localFile)
{
CancellationTokenSource cancellationSource = new CancellationTokenSource();
Stopwatch stopwatchTareaRetry = new Stopwatch();
stopwatchTareaRetry.Start();
TransferCheckpoint checkpoint = context.LastCheckpoint;
context = GetSingleTransferContext(checkpoint, stopwatchTareaRetry);
Console.WriteLine("\nReanudando la transferencia...\n");
try
{
DownloadOptions optionsWithDisableContentMD5Validation2 = new DownloadOptions()
{
DisableContentMD5Validation = true
};
var taskRetry = TransferManager.DownloadAsync(cloudBlockBlob, localFile, optionsWithDisableContentMD5Validation2, context, cancellationSource.Token);
evaluacionDeCaracteresDeErrorYretry(taskRetry, stopwatchTareaRetry);
taskRetry.Wait();
}
catch (Exception e)
{
Log.Error("\nLa transferencia fue cancelada : {0}", e.Message);
cancellationSource.Cancel();
}
if (cancellationSource.IsCancellationRequested)
{
Console.WriteLine("\nLa transferencia será reanudada en 3 segundos...");
Thread.Sleep(3000);
retryTransferencia(context, cloudBlockBlob, localFile);
}
}
private Task Download(dynamic sourceObject, TransferItem item)
{
DownloadOptions downloadOptions = item.Options as DownloadOptions;
SingleTransferContext transferContext = item.TransferContext as SingleTransferContext;
CancellationToken cancellationToken = item.CancellationToken;
string destPath = item.DestObject as string;
Stream destStream = item.DestObject as Stream;
if (cancellationToken != null && cancellationToken != CancellationToken.None)
{
if (destPath != null)
{
return(TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext, cancellationToken));
}
else
{
return(TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext, cancellationToken));
}
}
else if (transferContext != null || downloadOptions != null)
{
if (destPath != null)
{
return(TransferManager.DownloadAsync(sourceObject, destPath, downloadOptions, transferContext));
}
else
{
return(TransferManager.DownloadAsync(sourceObject, destStream, downloadOptions, transferContext));
}
}
else
{
if (destPath != null)
{
return(TransferManager.DownloadAsync(sourceObject, destPath));
}
else
{
return(TransferManager.DownloadAsync(sourceObject, destStream));
}
}
}
public async Task <Stream> GetFileContentAsync(string folder, string filename)
{
var(containerName, blobName) = GetContainerAndBlobName(folder, filename);
var cloudBlobContainer = await CreateContainer(containerName);
var blob = cloudBlobContainer.GetBlockBlobReference(blobName);
ServicePointManager.DefaultConnectionLimit = Environment.ProcessorCount * 8;
ServicePointManager.Expect100Continue = false;
TransferManager.Configurations.ParallelOperations = 64;
var stream = new MemoryStream();
await TransferManager.DownloadAsync(blob, stream);
stream.Position = 0;
return(stream);
}
private async void FillCurrentTripMediaItems()
{
if (CurrentTrip == null)
{
CurrentDestinationImages.Clear();
return;
}
var locationId = CurrentTrip.DepartureFlight.FlightInfo.Flight.Destination.LocationId;
var files = await _data.GetAzureStorageFilesByLocation(locationId, 5);
foreach (var fileMetadata in files)
{
StorageFile destinationFile = await _tempMediaFolder.CreateFileAsync(fileMetadata.FileName, CreationCollisionOption.ReplaceExisting);
await _transferManager.DownloadAsync(fileMetadata.Uri, destinationFile);
var mediaItem = new MediaItem(FolderType.Temp, CurrentTripMediaFolder, fileMetadata.FileName);
CurrentDestinationImages.Add(mediaItem);
}
}
/// 批量下载
public async void TransferBatchDownloadObjects()
{
//.cssg-snippet-body-start:[transfer-batch-download-objects]
TransferConfig transferConfig = new TransferConfig();
// 初始化 TransferManager
TransferManager transferManager = new TransferManager(cosXml, transferConfig);
string bucket = "examplebucket-1250000000"; //存储桶，格式：BucketName-APPID
string localDir = System.IO.Path.GetTempPath(); //本地文件夹
for (int i = 0; i < 5; i++)
{
// 下载对象
string cosPath = "exampleobject" + i; //对象在存储桶中的位置标识符，即称对象键
string localFileName = "my-local-temp-file"; //指定本地保存的文件名
COSXMLDownloadTask downloadTask = new COSXMLDownloadTask(bucket, cosPath,
localDir, localFileName);
await transferManager.DownloadAsync(downloadTask);
}
//.cssg-snippet-body-end
}
public async Task DownloadObject(string key, string localDir, string localFileName)
{
var downloadTask = GetDownloadTask(key, localDir, localFileName);
// 初始化 TransferConfig
TransferConfig transferConfig = new TransferConfig();
// 初始化 TransferManager
TransferManager transferManager = new TransferManager(cosXml, transferConfig);
try
{
COSXML.Transfer.COSXMLDownloadTask.DownloadTaskResult result = await transferManager.DownloadAsync(downloadTask);
string eTag = result.eTag;
}
catch (Exception e)
{
logger.LogError(e.ToLogMessage());
}
}
/// 下载时对单链接限速
public async void DownloadObjectTrafficLimit()
{
//.cssg-snippet-body-start:[download-object-traffic-limit]
TransferConfig transferConfig = new TransferConfig();
// 初始化 TransferManager
TransferManager transferManager = new TransferManager(cosXml, transferConfig);
String bucket = "examplebucket-1250000000"; //存储桶，格式：BucketName-APPID
String cosPath = "exampleobject"; //对象在存储桶中的位置标识符，即称对象键
string localDir = System.IO.Path.GetTempPath(); //本地文件夹
string localFileName = "my-local-temp-file"; //指定本地保存的文件名
GetObjectRequest request = new GetObjectRequest(bucket,
cosPath, localDir, localFileName);
request.LimitTraffic(8 * 1000 * 1024); // 限制为1MB/s
COSXMLDownloadTask downloadTask = new COSXMLDownloadTask(request);
await transferManager.DownloadAsync(downloadTask);
//.cssg-snippet-body-end
}
/// <summary>
/// Download a file, verify the hash, and return a throughput record of the transfer
/// </summary>
private static async Task <PerformanceRecorder> DownloadBlob <T>(string blobName, byte[] hash, bool useManager = true, bool useRecorder = true, bool useMD5Validation = false) where T : Stream, new()
{
Console.WriteLine($"Starting download of {blobName}");
// Create the source CloudBlob instances
var blob = await Util.GetCloudBlobAsync(ContainerName, blobName, BlobType.BlockBlob);
PerformanceRecorder recorder = null;
using (var stream = new T())
{
recorder = new PerformanceRecorder(useRecorder ? stream : null);
// Start the blob download
recorder.Start();
if (useManager)
{
await TransferManager.DownloadAsync(blob, stream, new DownloadOptions { DisableContentMD5Validation = !useMD5Validation, UseTransactionalMD5 = false }, new SingleTransferContext());
}
else
{
await blob.DownloadToStreamAsync(stream);
}
recorder.Stop();
recorder.Bytes = stream.Length;
if (VerifyStream(stream, hash))
{
Console.WriteLine($"Downloaded {blobName}: Average speed {recorder.AvgMbps} Mbps");
}
else
{
Console.WriteLine($"Failed to download {blobName}: Invalid hash");
}
}
return(recorder);
}
/// <summary>
/// Download an Azure blob from Azure Blob Storage.
/// </summary>
/// <param name="sourceBlob">The Microsoft.WindowsAzure.Storage.Blob.CloudBlob that is the source Azure blob.</param>
/// <param name="destPath">Path to the destination file.</param>
/// <param name="options">A Microsoft.WindowsAzure.Storage.DataMovement.DownloadOptions object that specifies
/// additional options for the operation.</param>
/// <param name="context">A Microsoft.WindowsAzure.Storage.DataMovement.TransferContext object that represents
/// the context for the current operation.</param>
/// <param name="cancellationToken">A System.Threading.CancellationToken object to observe while waiting for a task
/// to complete.</param>
/// <returns>A System.Threading.Tasks.Task object that represents the asynchronous operation.</returns>
public Task DownloadAsync(CloudBlob sourceBlob, string destPath, DownloadOptions options, TransferContext context, CancellationToken cancellationToken)
{
return(TransferManager.DownloadAsync(sourceBlob, destPath, options, context, cancellationToken));
}
/// <summary>
/// Downloading a blob using Azure Storage Data Movement library (with progress tracking)
/// </summary>
/// <returns></returns>
private async Task StorageDataMovementBlockBlobDownloadAsync()
{
try
{
#if FALLCREATORSUPDATE
// Dirty check to see if we are running Fall Creators Update (FCU, build 16299)
// otherwise we cannot make calls to the Azure Storage Data Movement library
// Change the FALLCREATORSUPDATE precompiler directive in build settings
AddResult("Downloading BlockBlob with ASDM Library");
// Create a blob client for interacting with the blob service.
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
// Create a container for organizing blobs within the storage account.
AddResult("1. Opening Blob Container");
CloudBlobContainer container = blobClient.GetContainerReference(BlockBlobContainerName);
try
{
await container.CreateIfNotExistsAsync();
}
catch (StorageException)
{
AddResult("If you are running with the default configuration please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample.");
throw;
}
// Get a reference to the blob we want in the container
AddResult("2. Get Specific Blob in Container");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(TestMediaFile);
if (blockBlob != null)
{
// Setup the number of the concurrent operations
TransferManager.Configurations.ParallelOperations = 64;
// Setup the transfer context and track the upload progress
SingleTransferContext context = new SingleTransferContext();
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
AddResult(" Bytes downloaded: " + progress.BytesTransferred.ToString());
});
// Download a blob to your file system
string path;
AddResult(string.Format("3. Download Blob from {0}", blockBlob.Uri.AbsoluteUri));
string fileName = string.Format("CopyOf{0}", TestMediaFile);
var sw = Stopwatch.StartNew();
#if WINDOWS_UWP
// TO DO: UWP code is not complete and has not been tested yet. This is the old single operation code
// I have only tested Data Movement in the Unity editor
StorageFolder storageFolder = ApplicationData.Current.TemporaryFolder;
StorageFile sf = await storageFolder.CreateFileAsync(fileName, CreationCollisionOption.ReplaceExisting);
path = sf.Path;
Stream sfs = await sf.OpenStreamForWriteAsync();
//Stream sfs = await sf.OpenAsync(FileAccessMode.ReadWrite) as Stream;
// Download a local blob with progress updates
DownloadOptions dOptions = new DownloadOptions();
dOptions.DisableContentMD5Validation = true; // TO DO: Need to test if MD5 works, currently disabled
await TransferManager.DownloadAsync(blockBlob, sfs, dOptions, context, CancellationToken.None);
//await blockBlob.DownloadToFileAsync(sf);
#else
path = Path.Combine(Application.temporaryCachePath, fileName);
// Download a local blob with progress updates
DownloadOptions dOptions = new DownloadOptions();
dOptions.DisableContentMD5Validation = true; // TO DO: Need to test if MD5 works, currently disabled
await TransferManager.DownloadAsync(blockBlob, path, dOptions, context, CancellationToken.None);
#endif
sw.Stop();
TimeSpan time = sw.Elapsed;
AddResult(string.Format("4. Blob file downloaded to {0} in {1}s", path, time.TotalSeconds.ToString()));
AddResult("-- Download Test Complete --");
}
#else
AddResult("Cannot use Azure Storage DMLib. You are currently not running Fall Creators Update. Change minimum target to 16299, add FALLCREATORSUPDATE in build conditional compilation symbols, and re-add DMLib DLL reference.");
#endif
}
catch (Exception ex)
{
// Woops!
AddResult("Error: " + ex.ToString());
AddResult("Error: " + ex.InnerException.ToString());
}
}
static void Main(string[] args)
{
string path = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
path = Path.Combine(path, "..", "..", "..");
var builder = new ConfigurationBuilder()
.SetBasePath(path)
.AddJsonFile("appsettings.json");
Configuration = builder.Build();
crearArchivoLogeo();
Log.Information("/////////// Ejecutando el script de BD ///////////");
ejecutarScriptBD();
//test
//var cloudBlockBlob = new CloudBlockBlob(new Uri(Configuration.GetSection("URLS_ToGetTheInfo")["blob_web_1M"]));
//var cloudBlockBlob = new CloudBlockBlob(new Uri(Configuration.GetSection("URLS_ToGetTheInfo")["blob_web_test_5ok_4repetidos"]));
//var cloudBlockBlob = new CloudBlockBlob(new Uri(Configuration.GetSection("URLS_ToGetTheInfo")["blob_web_stock103mil"]));
//var cloudBlockBlob = new CloudBlockBlob(new Uri(Configuration.GetSection("URLS_ToGetTheInfo")["blob_web_Stock3M"]));
var cloudBlockBlob = new CloudBlockBlob(new Uri(Configuration.GetSection("URLS_ToGetTheInfo")["blob_web_Stock"]));
TransferManager.Configurations.ParallelOperations = 5;
DownloadOptions options = new DownloadOptions();
SingleTransferContext context = new SingleTransferContext();
CancellationTokenSource cancellationSource = new CancellationTokenSource();
string localFile = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location) + "\\" + "STOCK_" + DateTime.Now.ToString("yyyy_dd_M_HH_mm_ss") + ".CSV";
Log.Information("Comenzando descarga...");
Stopwatch stopwatchgral = new Stopwatch();
stopwatchgral.Start();
Stopwatch stopwatchTarea = new Stopwatch();
stopwatchTarea.Start();
try
{
DownloadOptions optionsWithDisableContentMD5Validation1 = new DownloadOptions()
{
DisableContentMD5Validation = true
};
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
if (progress.BytesTransferred > 0)
{
Log.Information("Mb descargados {0:F}\n", (progress.BytesTransferred / 1024) / 1024);
stopwatchTarea.Restart();
}
});
var task = TransferManager.DownloadAsync(cloudBlockBlob, localFile, optionsWithDisableContentMD5Validation1, context, cancellationSource.Token);
evaluacionDeCaracteresDeErrorYretry(task, stopwatchTarea);
task.Wait();
}
catch (Exception e) {
Log.Error("\nLa transferencia fue cancelada: {0}", e.Message);
cancellationSource.Cancel();
}
stopwatchTarea.Stop();
if (cancellationSource.IsCancellationRequested)
{
Console.WriteLine("\nLa transferencia será reanudada en 3 segundos...");
Thread.Sleep(3000);
retryTransferencia(context, cloudBlockBlob, localFile);
}
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
List <string> records = new List <string>();
Log.Information("\n *******Operacion de transferencia completada con exito " + stopwatchgral.Elapsed.TotalSeconds + " segundos ******");
using (StreamReader reader = new StreamReader(localFile, Encoding.GetEncoding(1251), true))
{
//Salteo la primera linae del encabezado
if (!reader.EndOfStream)
{
reader.ReadLine();
}
string file = reader.ReadToEnd();
records = new List <string>(file.Split('\n'));
Log.Information("El largo del archivo es: " + (reader.BaseStream.Length / 1024) / 1024 + " mb");
}
stopwatchgral.Stop();
Log.Information("El tiempo en seg tomado es: : " + (stopwatchgral.ElapsedMilliseconds / 1000) + " segundos");
Log.Information("El tiempo en min tomado es: : " + (stopwatchgral.ElapsedMilliseconds / 1000) / 60 + " minutos");
stopwatchgral.Reset();
stopwatchgral.Start();
List <SaleStock> stcklist = new List <SaleStock>();
Log.Information("Comenzando a copiar el archivo a memoria");
foreach (string record in records)
{
if (record != "")
{
SaleStock stck = new SaleStock();
string[] textpart = record.Split(';');
stck.PointOfSale = textpart[0];
stck.Product = textpart[1];
stck.Date = Convert.ToDateTime(textpart[2]);
stck.Stock = Convert.ToInt32(textpart[3]);
stcklist.Add(stck);
}
}
Log.Information("El tiempo tomado en importar el CSV en memoria es: " + (stopwatchgral.ElapsedMilliseconds / 1000) / 60 + " minutos");
var copyParameters = new[]
{
nameof(SaleStock.PointOfSale),
nameof(SaleStock.Product),
nameof(SaleStock.Date),
nameof(SaleStock.Stock)
};
Log.Information("Comenzando a copiar el archivo a la BD");
stopwatchgral.Reset();
stopwatchgral.Start();
using (var sqlCopy = new SqlBulkCopy(Configuration.GetConnectionString("BloggingDatabase")))
{
sqlCopy.DestinationTableName = "[Stock]";
sqlCopy.BatchSize = 500;
using (var reader = ObjectReader.Create(stcklist, copyParameters))
{
sqlCopy.WriteToServer(reader);
}
}
Log.Information("Fin de copiado del archivo a la BD!");
Log.Information("El tiempo tomado en importar el CSV en seg es: " + (stopwatchgral.ElapsedMilliseconds / 1000) + " segundos");
Log.Information("El tiempo tomado en importar el CSV en min es: " + (stopwatchgral.ElapsedMilliseconds / 1000) / 60 + " minutos");
stopwatchgral.Stop();
}
protected override async Task DownloadFileCore(string id, string file)
{
var context = new SingleTransferContext();
var blob = _container.GetBlockBlobReference(id);
await TransferManager.DownloadAsync(blob, file, null, context);
}
/// <summary>
/// Download data from Azure storage.
/// 1. Download a CloudBlob to an exsiting local file
/// 2. Query the user to overwrite the local file or not in the OverwriteCallback
/// 3. Download another CloudBlob to local with content MD5 validation disabled
/// 4. Show the overall progress of both transfers
/// </summary>
private static async Task BlobDownloadSample()
{
string sourceBlobName1 = "azure_blockblob.png";
string sourceBlobName2 = "azure_blockblob2.png";
string destinationFileName1 = "azure.png";
string destinationFileName2 = "azure_new.png";
// Create the source CloudBlob instances
CloudBlob sourceBlob1 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName1, BlobType.BlockBlob);
CloudBlob sourceBlob2 = await Util.GetCloudBlobAsync(ContainerName, sourceBlobName2, BlobType.BlockBlob);
// Create a TransferContext shared by both transfers
SingleTransferContext sharedTransferContext = new SingleTransferContext();
// Show overwrite prompt in console when OverwriteCallback is triggered
sharedTransferContext.ShouldOverwriteCallbackAsync = async(source, destination) =>
{
Console.WriteLine("{0} already exists. Do you want to overwrite it with {1}? (Y/N)", destination, source);
while (true)
{
ConsoleKeyInfo keyInfo = Console.ReadKey(true);
char key = keyInfo.KeyChar;
if (key == 'y' || key == 'Y')
{
Console.WriteLine("User choose to overwrite the destination.");
return(true);
}
else if (key == 'n' || key == 'N')
{
Console.WriteLine("User choose NOT to overwrite the destination.");
return(false);
}
Console.WriteLine("Please press 'y' or 'n'.");
}
};
// Record the overall progress
ProgressRecorder recorder = new ProgressRecorder();
sharedTransferContext.ProgressHandler = recorder;
// Start the blob download
Task task1 = TransferManager.DownloadAsync(sourceBlob1, destinationFileName1, null /* options */, sharedTransferContext);
// Create a DownloadOptions to disable md5 check after data is downloaded. Otherwise, data movement
// library will check the md5 checksum stored in the ContentMD5 property of the source CloudFile/CloudBlob
// You can uncomment following codes, enable ContentMD5Validation and have a try.
// sourceBlob2.Properties.ContentMD5 = "WrongMD5";
// sourceBlob2.SetProperties();
DownloadOptions options = new DownloadOptions();
options.DisableContentMD5Validation = true;
// Start the download
Task task2 = TransferManager.DownloadAsync(sourceBlob2, destinationFileName2, options, sharedTransferContext);
// Wait for both transfers to finish
try
{
await task1;
}
catch (Exception e)
{
// Data movement library will throw a TransferException when user choose to not overwrite the existing destination
Console.WriteLine(e.Message);
}
await task2;
// Print out the final transfer state
Console.WriteLine("Final transfer state: {0}", recorder.ToString());
}
// TO DO: Add support for uploads
// Download using the Azure Storage Data Movement Library.
// The Azure Storage Data Movement library currently only works in the Unity Editor, not in UWP targets.
private async Task StorageDataMovementBlockBlobDownloadAsync()
{
#if UNITY_EDITOR
WriteLine("Downloading BlockBlob with ASDM Library");
// Create a blob client for interacting with the blob service.
CloudBlobClient blobClient = StorageAccount.CreateCloudBlobClient();
// Create a container for organizing blobs within the storage account.
WriteLine("1. Opening Blob Container");
CloudBlobContainer container = blobClient.GetContainerReference(BlockBlobContainerName);
try
{
await container.CreateIfNotExistsAsync();
}
catch (StorageException)
{
WriteLine("If you are running with the default configuration please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample.");
throw;
}
// Get a reference to the blob we want in the container
WriteLine("2. Get Specific Blob in Container");
CloudBlockBlob blockBlob = container.GetBlockBlobReference(TestMediaFile);
if (blockBlob != null)
{
// Setup the number of the concurrent operations
TransferManager.Configurations.ParallelOperations = 64;
// Setup the transfer context and track the upload progress
SingleTransferContext context = new SingleTransferContext();
context.ProgressHandler = new Progress <TransferStatus>((progress) =>
{
WriteLine(" Bytes downloaded: " + progress.BytesTransferred.ToString());
});
// Download a blob to your file system
string path;
WriteLine(string.Format("3. Download Blob from {0}", blockBlob.Uri.AbsoluteUri));
string fileName = string.Format("CopyOf{0}", TestMediaFile);
// All this UWP code should currently stay commented since DMLib doesn't work in Unity UWP at this time
// I have only tested Data Movement in the Unity editor
/* START OF UWP CODE
* StorageFolder storageFolder = ApplicationData.Current.TemporaryFolder;
* StorageFile sf = await storageFolder.CreateFileAsync(fileName, CreationCollisionOption.ReplaceExisting);
* path = sf.Path;
* Stream sfs = await sf.OpenStreamForWriteAsync();
*
* // Download a local blob with progress updates
* DownloadOptions dOptions = new DownloadOptions();
* dOptions.DisableContentMD5Validation = true; // TO DO: Need to test if MD5 works, currently disabled
* await TransferManager.DownloadAsync(blockBlob, sfs, dOptions, context, CancellationToken.None);
* END OF UWP CODE */
var sw = Stopwatch.StartNew();
path = Path.Combine(Application.temporaryCachePath, fileName);
// Download a local blob with progress updates
DownloadOptions dOptions = new DownloadOptions();
dOptions.DisableContentMD5Validation = true; // TO DO: Need to test if MD5 works, currently disabled
await TransferManager.DownloadAsync(blockBlob, path, dOptions, context, CancellationToken.None);
sw.Stop();
TimeSpan time = sw.Elapsed;
WriteLine(string.Format("4. Blob file downloaded to {0} in {1}s", path, time.TotalSeconds.ToString()));
}
#else
WriteLine("The Azure Storage Data Movement library currently only works in the Unity Editor, not in UWP targets.");
#endif
}
