public void DownloadCatalog(bool force = false)
{
var catalogPath = GetCatalogPath();
var url = ApiCatalogModel.Url;
var blobClient = new BlobClient(new Uri(url));
if (!force && File.Exists(catalogPath))
{
Console.WriteLine("Checking catalog...");
var localTimetamp = File.GetLastWriteTimeUtc(catalogPath);
var properties = blobClient.GetProperties();
var blobTimestamp = properties.Value.LastModified.UtcDateTime;
var blobIsNewer = blobTimestamp > localTimetamp;
if (!blobIsNewer)
{
Console.WriteLine("Catalog is up-to-date.");
return;
}
}
try
{
Console.WriteLine("Downloading catalog...");
blobClient.DownloadTo(catalogPath);
var properties = blobClient.GetProperties();
File.SetLastWriteTimeUtc(catalogPath, properties.Value.LastModified.UtcDateTime);
}
catch (Exception ex)
{
Console.Error.WriteLine($"error: can't download catalog: {ex.Message}");
Environment.Exit(1);
}
}
/*
* Checks replication status of given blob in given container. Returns true if replication is completed or failed. If it failed,
* there is an additional output that notifies user. Returns false if there is no status yet
*/
public static bool CheckReplicationStatus(
BlobContainerClient sourceContainerClient,
String blobName)
{
BlobClient sourceBlob = sourceContainerClient.GetBlobClient(blobName);
Response <BlobProperties> source_response = sourceBlob.GetProperties();
IList <ObjectReplicationPolicy> policyList = source_response.Value.ObjectReplicationSourceProperties;
// If policyList is null, then replication still in progress
if (policyList != null)
{
foreach (ObjectReplicationPolicy policy in policyList)
{
foreach (ObjectReplicationRule rule in policy.Rules)
{
if (rule.ReplicationStatus.ToString() != "Complete")
{
Console.WriteLine("Replication failed for " + blobName);
}
return(true);
}
}
}
return(false);
}
public async Task GivenABlobFile_WhenExecutorWithoutAnonymize_DataShouldBeSame(string connectionString, string containerName, string blobName)
{
string targetContainerName = Guid.NewGuid().ToString("N");
string targetBlobName = Guid.NewGuid().ToString("N");
BlobContainerClient containerClient = new BlobContainerClient(connectionString, targetContainerName);
await containerClient.CreateIfNotExistsAsync();
try
{
BlobClient sourceBlobClient = new BlobClient(connectionString, containerName, blobName, DataFactoryCustomActivity.BlobClientOptions.Value);
BlockBlobClient targetBlobClient = new BlockBlobClient(connectionString, targetContainerName, targetBlobName, DataFactoryCustomActivity.BlobClientOptions.Value);
using FhirBlobDataStream stream = new FhirBlobDataStream(sourceBlobClient);
using FhirStreamReader reader = new FhirStreamReader(stream);
FhirBlobConsumer consumer = new FhirBlobConsumer(targetBlobClient);
var executor = new FhirPartitionedExecutor <string, string>(reader, consumer, content => content);
await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
Assert.Equal(sourceBlobClient.GetProperties().Value.ContentLength, targetBlobClient.GetProperties().Value.ContentLength);
}
finally
{
await containerClient.DeleteIfExistsAsync().ConfigureAwait(false);
}
}
private static void ListBlobMetadata()
{
try
{
BlobContainerClient containerClient = new BlobContainerClient(connectionString, containerName);
BlobClient blobClient = containerClient.GetBlobClient(blobName);
// Get the blob properties
BlobProperties properties = blobClient.GetProperties();
// Display blob's metadata values
Console.WriteLine("-----------------------------------");
Console.WriteLine($" Blob name: {blobClient.Name} - (Metadata)");
foreach (var entry in properties.Metadata)
{
Console.WriteLine($" Metadata: {{key: {entry.Key} - Value: {entry.Value}}}");
}
Console.WriteLine("-----------------------------------");
}
catch (RequestFailedException e)
{
Console.WriteLine($"HTTP error code {e.Status}: {e.ErrorCode}");
Console.WriteLine(e.Message);
Console.ReadLine();
}
}
private static void ListBlobProperties()
{
try
{
BlobContainerClient containerClient = new BlobContainerClient(connectionString, containerName);
BlobClient blobClient = containerClient.GetBlobClient(blobName);
// Get the blob properties
BlobProperties properties = blobClient.GetProperties();
// Display some of the blob's property values
Console.WriteLine("-----------------------------------");
Console.WriteLine($" Blob name: {blobClient.Name} - (System Properties)");
Console.WriteLine($" BlobType: {properties.BlobType}");
Console.WriteLine($" ContentType: {properties.ContentType}");
Console.WriteLine($" CreatedOn: {properties.CreatedOn}");
Console.WriteLine($" LastModified: {properties.LastModified}");
Console.WriteLine($" IsLatestVersion: {properties.IsLatestVersion}");
Console.WriteLine("-----------------------------------");
}
catch (RequestFailedException e)
{
Console.WriteLine($"HTTP error code {e.Status}: {e.ErrorCode}");
Console.WriteLine(e.Message);
Console.ReadLine();
}
}
private static bool SetBlobMetadata(string key, string value)
{
try
{
BlobContainerClient containerClient = new BlobContainerClient(connectionString, containerName);
BlobClient blobClient = containerClient.GetBlobClient(blobName);
// Get the blob properties
BlobProperties properties = blobClient.GetProperties();
// Add blob's metadata
properties.Metadata.Add(key, value);
blobClient.SetMetadata(properties.Metadata);
Console.WriteLine("-----------------------------------");
Console.WriteLine($" Metadata pair {{ {key} - {value} }} added to blob");
Console.WriteLine("-----------------------------------");
return(true);
}
catch (RequestFailedException e)
{
Console.WriteLine($"HTTP error code {e.Status}: {e.ErrorCode}");
Console.WriteLine(e.Message);
Console.ReadLine();
}
return(false);
}
/// <summary>
///
/// <para>GetFileChecksum:</para>
///
/// <para>Gets MD5 checksum of a file from File Service, caller thread will be blocked before it is done</para>
///
/// <para>Check <seealso cref="IBFileServiceInterface.GetFileChecksum"/> for detailed documentation</para>
///
/// </summary>
public bool GetFileChecksum(string _BucketName, string _KeyInBucket, out string _Checksum, Action <string> _ErrorMessageAction = null)
{
try
{
BlobContainerClient ContainerClient = AServiceClient.GetBlobContainerClient(_BucketName);
BlobClient Blob = ContainerClient.GetBlobClient(_KeyInBucket);
Response <BlobProperties> Response = Blob.GetProperties();
if (Response.Value != null)
{
_Checksum = Response.Value.ETag.ToString().Trim('"').ToLower();
return(true);
}
else
{
_ErrorMessageAction?.Invoke($"BFileServiceAZ -> GetFileChecksum : Service response was empty");
_Checksum = null;
return(false);
}
}
catch (Exception ex)
{
_ErrorMessageAction?.Invoke($"BFileServiceAZ -> GetFileChecksum : {ex.Message}\n{ex.StackTrace}");
_Checksum = null;
return(false);
}
}
/// <summary>
///
/// <para>GetFileSize:</para>
///
/// <para>Gets size of a file in bytes from File Service, caller thread will be blocked before it is done</para>
///
/// <para>Check <seealso cref="IBFileServiceInterface.GetFileSize"/> for detailed documentation</para>
///
/// </summary>
public bool GetFileSize(string _BucketName, string _KeyInBucket, out ulong _FileSize, Action <string> _ErrorMessageAction = null)
{
try
{
BlobContainerClient ContainerClient = AServiceClient.GetBlobContainerClient(_BucketName);
BlobClient Blob = ContainerClient.GetBlobClient(_KeyInBucket);
Response <BlobProperties> Response = Blob.GetProperties();
if (Response.Value != null)
{
_FileSize = (ulong)Response.Value.ContentLength;
return(true);
}
else
{
_ErrorMessageAction?.Invoke($"BFileServiceAZ -> GetFileSize : Service response was empty");
_FileSize = 0;
return(false);
}
}
catch (Exception ex)
{
_ErrorMessageAction?.Invoke($"BFileServiceAZ -> GetFileSize : {ex.Message}\n{ex.StackTrace}");
_FileSize = 0;
return(false);
}
}
public HydrationStatusModel CheckHydrationStatus(BlobHydrateModel model, string queueName)
{
var ret = new HydrationStatusModel();
ret.Hydrate(model);
var accountClient = new BlobServiceClient(_cs);
var containerClient = accountClient.GetBlobContainerClient(model.ContainerName);
BlobClient blobClient = containerClient.GetBlobClient(model.BlobName);
var properties = blobClient.GetProperties();
if (properties.Value.ArchiveStatus == "rehydrate-pending-to-hot")
{
ret.Status = HydrationStatus.NotHydrated;
QueueClient queueClient = new QueueClient(_cs, queueName);
var json = JsonConvert.SerializeObject(model);
string requeueMessage = Convert.ToBase64String(Encoding.UTF8.GetBytes(json));
queueClient.SendMessage(requeueMessage, visibilityTimeout: TimeSpan.FromMinutes(_visibilityTimeout));
}
else
{
ret.Status = HydrationStatus.Hydrated;
ret.HydratedFileDataTime = DateTime.Now;
}
return(ret);
}
public AzureBlobJsonLogReader(Uri blobUri, IDictionary <string, string> fieldMapping, SourceJsonFormat format) : base(fieldMapping, format)
{
Blob = new BlobClient(blobUri);
var properties = Blob.GetProperties();
blobSize = properties.Value.ContentLength;
}
public FhirBlobDataStream(BlobClient blobClient)
{
_blobClient = blobClient;
_blobLength = new Lazy <long>(() => _blobClient.GetProperties().Value.ContentLength);
_downloadTasks = new Queue <Task <Stream> >();
_position = 0;
}
/*
* Uploads a single blob (or updates it in the case where blob already exists in container), checks replication completion,
* then demonstrates that source and destination blobs have identical contents
*/
public static void BlobUpdate(BlobContainerClient sourceContainerClient,
BlobContainerClient destContainerClient,
String blobName,
String blobContent,
int timeInterval)
{
// Uploading blobs
Console.WriteLine("Demonstrating replication of blob in source has same contents in destination container");
BlobClient sourceBlob = sourceContainerClient.GetBlobClient(blobName);
Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(blobContent));
sourceBlob.Upload(stream, true);
Console.WriteLine("Added blob " + blobName + " containing content: " + blobContent);
// Check if replication in dest container is finished in interval of 1 min
Console.WriteLine("Checking to see if replication finished");
while (true)
{
Thread.Sleep(timeInterval);
Response <BlobProperties> source_response = sourceBlob.GetProperties();
IList <ObjectReplicationPolicy> policyList = source_response.Value.ObjectReplicationSourceProperties;
// If policyList is null, then replication still in progress
if (policyList != null)
{
foreach (ObjectReplicationPolicy policy in policyList)
{
foreach (ObjectReplicationRule rule in policy.Rules)
{
if (rule.ReplicationStatus.ToString() != "Complete")
{
Console.WriteLine("Blob replication failed");
return;
}
// Comparing source and dest blobs
Console.WriteLine("Blob successfully replicated");
String sourceContent = "";
String destContent = "";
using (MemoryStream sourceStream = new MemoryStream())
{
sourceBlob.DownloadTo(sourceStream);
sourceContent = Encoding.UTF8.GetString(sourceStream.ToArray());
}
Console.WriteLine("Source Blob Content: " + sourceContent);
using (MemoryStream destStream = new MemoryStream())
{
BlobClient destBlob = destContainerClient.GetBlobClient(blobName);
destBlob.DownloadTo(destStream);
destContent = Encoding.UTF8.GetString(destStream.ToArray());
}
Console.WriteLine("Destination Blob Content: " + destContent);
return;
}
}
}
}
}
public virtual DateTime GetLastModified(string blobName)
{
Debug.WriteLine("Get Last Modified Date Time of blob:\n\t {0}\n", blobName);
BlobClient blobClient = GetBlobClient(blobName);
BlobProperties blobProperties = blobClient.GetProperties().Value;
return(blobProperties.LastModified.DateTime);
}
/// <summary>Gets a blob from Azure Storage as just raw bytes with metadata</summary>
/// <param name="containerName">container name</param>
/// <param name="blobName">blob name</param>
/// <returns>Wrapped raw bytes with some metadata</returns>
public RawFileWrapper GetRawBlob(string containerName, string blobName)
{
RawFileWrapper results = new RawFileWrapper();
// validate input
if (String.IsNullOrWhiteSpace(containerName) || String.IsNullOrWhiteSpace(blobName))
{
return(results);
}
containerName = containerName.Trim();
blobName = blobName.Trim();
// Get a reference to a share and then create it
BlobContainerClient container = new BlobContainerClient(this.ConnectionString, containerName);
// check the container exists
Response <bool> exists = container.Exists();
if (!exists.Value)
{
return(results);
}
// set options
BlobOpenReadOptions op = new BlobOpenReadOptions(false);
// read the blob to an array
BlobClient blob = container.GetBlobClient(blobName);
using Stream stream = blob.OpenRead(op);
results.Data = new byte[stream.Length];
stream.Read(results.Data, 0, results.Data.Length);
stream.Close();
// get the properties
BlobProperties props = blob.GetProperties().Value;
if (props == null)
{
return(results);
}
results.ContentType = props.ContentType;
// get a filename
if (props.Metadata.ContainsKey("filename"))
{
results.Filename = props.Metadata["filename"].ToString();
}
else
{
results.Filename = blob.Name;
}
return(results);
}
private static void ReadSetMetaData(BlobClient _blob)
{
BlobProperties props = _blob.GetProperties();
IDictionary <string, string> metadata = props.Metadata;
metadata.Add("Test", "Test");
_blob.SetMetadata(metadata);
}
public FhirBlobDataStream(Uri blobUri, TokenCredential credential)
{
BlobClientOptions options = new BlobClientOptions();
_blobClient = new BlobClient(blobUri, credential, options);
_blobLength = new Lazy <long>(() => _blobClient.GetProperties().Value.ContentLength);
_downloadTasks = new Queue <Task <Stream> >();
_position = 0;
}
public static void GetProperties()
{
BlobContainerClient containerClient = client.GetBlobContainerClient(containerName);
BlobClient blob = containerClient.GetBlobClient(filename);
BlobProperties props = blob.GetProperties();
Console.WriteLine("\nAccess Tier::" + props.AccessTier);
Console.WriteLine("\nContent length::" + props.ContentLength);
}
public void GetBlobProperties(string containerName, string blobName)
{
var blob = new BlobClient(connectionString, containerName, blobName);
var properties = blob.GetProperties().Value;
Console.WriteLine($"File size {ByteSize.FromBytes(properties.ContentLength).MebiBytes.ToString("#.##")} MB");
Console.WriteLine($"Content type {properties.ContentType}");
Console.WriteLine($"Created On {properties.CreatedOn}");
Console.WriteLine($"Updated On {properties.LastModified}");
}
public AzureIndexInput(AzureDirectory azureDirectory, string name, BlobClient blob)
: base(name)
{
this._name = name;
this._azureDirectory = azureDirectory;
#if FULLDEBUG
Debug.WriteLine($"{_azureDirectory.Name} opening {name} ");
#endif
_fileMutex = BlobMutexManager.GrabMutex(name);
_fileMutex.WaitOne();
try
{
_blobContainer = azureDirectory.BlobContainer;
_blob = blob;
bool fileNeeded = false;
if (!CacheDirectory.FileExists(name))
{
fileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(name);
var properties = blob.GetProperties();
long blobLength = properties.Value?.ContentLength ?? 0;
if (cachedLength != blobLength)
{
fileNeeded = true;
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fileNeeded)
{
using (StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(name))
{
// get the blob
_blob.DownloadTo(fileStream);
fileStream.Flush();
Debug.WriteLine($"{_azureDirectory.Name} GET {_name} RETREIVED {fileStream.Length} bytes");
}
}
#if FULLDEBUG
Debug.WriteLine($"{_azureDirectory.Name} Using cached file for {name}");
#endif
// and open it as our input, this is now available forevers until new file comes along
_indexInput = CacheDirectory.OpenInput(name, IOContext.DEFAULT);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
private bool IsNewer(BlobClient blob, string filePath)
{
var blobProperties = blob.GetProperties();
// Any operation that modifies a blob, including an update of the blob's metadata or properties, changes the last modified time of the blob
var blobLastModified = blobProperties.Value.LastModified.UtcDateTime;
// returns date of local file was last written to
DateTime fileLastWrite = File.GetLastWriteTimeUtc(filePath);
return(blobLastModified > fileLastWrite);
}
static void GetMetadata()
{
BlobContainerClient containerClient = client.GetBlobContainerClient(containerName);
BlobClient blob = containerClient.GetBlobClient(filename);
BlobProperties props = blob.GetProperties();
foreach (var metadata in props.Metadata)
{
Console.WriteLine(metadata.Key.ToString() + "::" + metadata.Value.ToString());
}
}
public long TotalBytes()
{
if (IsNormalFile)
{
var prop = BlobClient.GetProperties().Value;
return(prop.ContentLength);
}
return(JsonSerializer.Deserialize <DirectoryMetaData>(File.ReadAllText(LocalCachePath.FullName))
.TotalBytesUsed);
}
protected byte[] RecieveBytes(BlobClient blob, out long size)
{
size = blob.GetProperties().Value.ContentLength;
byte[] buffer = new byte[size];
using (MemoryStream ms = new MemoryStream(buffer))
{
blob.DownloadTo(ms);
return(ms.ToArray());
}
}
private static File GetFileInfo(BlobClient blob, BlobSasBuilder readPermissions)
{
File file = null;
if (blob.Exists())
{
var props = blob.GetProperties().Value;
file = GetFileInfo(blob, readPermissions, props);
}
return(file);
}
static void GetProperties()
{
BlobServiceClient blobServiceClient = new BlobServiceClient(storageconnstring);
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(containerName);
BlobClient blob = containerClient.GetBlobClient(filename);
BlobProperties properties = blob.GetProperties();
Console.WriteLine("The Access tier of the blob is {0}", properties.AccessTier);
Console.WriteLine("The Content Length of the blob is {0}", properties.ContentLength);
}
public async Task <IActionResult> MoveFile([FromForm] ProjectFileMoveVM formdata)
{
try
{
// Check Model State
if (!ModelState.IsValid)
{
return(BadRequest(ModelState));
}
// Find User
var blobFile = await _dbContext.BlobFiles.FindAsync(formdata.FileID);
if (blobFile == null)
{
return(NotFound(new { message = "File Not Found" }));
}
var filePath = formdata.SubDirectory + blobFile.Name + blobFile.Extension;
BlobClient blobClient = await _blobService.MoveBlobAsync(blobFile, filePath);
BlobProperties properties = blobClient.GetProperties();
blobFile.Directory = formdata.SubDirectory;
blobFile.Uri = blobClient.Uri.ToString();
blobFile.LastModified = properties.LastModified.LocalDateTime;
// Set Entity State
_dbContext.Entry(blobFile).State = EntityState.Modified;
// Update Database with entry
await _dbContext.SaveChangesAsync();
// Return Ok Status
return(Ok(new
{
result = blobFile,
message = "File Successfully Moved"
}));
}
catch (Exception e)
{
// Return Bad Request If There Is Any Error
return(BadRequest(new
{
error = e
}));
}
}
private static void DownloadBlobUsingSasURI(BlobContainerClient containerClient)
{
string blobName = "Course1.json";
Uri sasURL = GenerateSasURL(containerClient.Name, blobName, containerClient);
BlobClient _clientBlob = new BlobClient(sasURL);
Azure.Storage.Blobs.Models.BlobProperties props = _clientBlob.GetProperties();
IDictionary <string, string> metadata = props.Metadata;
metadata.Add("test", "test");
_clientBlob.SetMetadata(metadata);
_clientBlob.DownloadTo(@"E:\sasJSONDemo.json");
}
public virtual bool CancelCopy(string destBlobName, string copyId)
{
// Fetch the destination blob's properties before checking the copy state.
BlobClient destinationBlobClient = GetBlobClient(destBlobName);
BlobProperties destinationBlobProperties = destinationBlobClient.GetProperties().Value;
// Check the copy status. If it is still pending, abort the copy operation.
if (destinationBlobProperties.CopyStatus == CopyStatus.Pending)
{
destinationBlobClient.AbortCopyFromUri(copyId);
Debug.WriteLine("Copy operation {0} has been aborted.", copyId);
return(true);
}
return(false);
}
// </Snippet_CopyBlob>
//-------------------------------------------------
// Stop a blob copy operation
//-------------------------------------------------
private static async Task StopBlobCopyAsync(BlobContainerClient container)
{
try
{
// Get the name of the first blob in the container to use as the source.
string blobName = container.GetBlobs().FirstOrDefault().Name;
// Create a BlobClient representing the source blob to copy.
BlobClient sourceBlob = container.GetBlobClient(blobName);
// Ensure that the source blob exists.
if (await sourceBlob.ExistsAsync())
{
// Get a BlobClient representing the destination blob with a unique name.
BlobClient destBlob =
container.GetBlobClient(Guid.NewGuid() + "-" + sourceBlob.Name);
// Start the copy operation.
destBlob.StartCopyFromUri(sourceBlob.Uri);
// <Snippet_StopBlobCopy>
// Get the destination blob's properties to check the copy status.
BlobProperties destProperties = destBlob.GetProperties();
// Check the copy status. If the status is pending, abort the copy operation.
if (destProperties.CopyStatus == CopyStatus.Pending)
{
await destBlob.AbortCopyFromUriAsync(destProperties.CopyId);
Console.WriteLine($"Copy operation {destProperties.CopyId} has been aborted.");
}
// </Snippet_StopBlobCopy>
else
{
Console.WriteLine($"Copy status: {destProperties.CopyStatus}");
Console.WriteLine($"Copy progress: {destProperties.CopyProgress}");
Console.WriteLine($"Completion time: {destProperties.CopyCompletedOn}");
Console.WriteLine($"Total bytes: {destProperties.ContentLength}");
}
}
}
catch (RequestFailedException ex)
{
Console.WriteLine(ex.Message);
Console.ReadLine();
throw;
}
}
static void GetMetadata()
{
BlobServiceClient blobServiceClient = new BlobServiceClient(storageconnstring);
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(containerName);
BlobClient blob = containerClient.GetBlobClient(filename);
BlobProperties properties = blob.GetProperties();
foreach (var metadata in properties.Metadata)
{
Console.WriteLine(metadata.Key.ToString());
Console.WriteLine(metadata.Value.ToString());
}
}
