public void UploadToAppendBlob()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("my-append-blobs");
container.CreateIfNotExists();
CloudAppendBlob appendBlob = container.GetAppendBlobReference("append-blob.log");
if (!appendBlob.Exists())
{
appendBlob.CreateOrReplace();
}
int numBlocks = 10;
Random rnd = new Random();
byte[] bytes = new byte[numBlocks];
rnd.NextBytes(bytes);
//Simulate a logging operation by writing text data and byte data to the end of the append blob.
for (int i = 0; i < numBlocks; i++)
{
appendBlob.AppendText(string.Format("Timestamp: {0:u} \tLog Entry: {1}{2}", DateTime.UtcNow, bytes[i], Environment.NewLine));
}
Console.WriteLine(appendBlob.DownloadText());
}
private bool AppendToBlob(CloudAppendBlob blob, MemoryStream stream)
{
try
{
if (!blob.Exists())
{
return(false);
}
blob.AppendBlock(stream,
accessCondition: AccessCondition.GenerateIfMaxSizeLessThanOrEqualCondition(_maxBlobSize));
}
catch (StorageException ex)
{
if (ex.RequestInformation.HttpStatusCode == 412) // HttpStatusCode.PreconditionFailed
{
return(false);
}
else
{
throw;
}
}
return(true);
}
public static string GetLogs()
{
string connectionString = @"DefaultEndpointsProtocol=https;AccountName=hotelmanagerblob;AccountKey=0rIspWPXwm7U22uvlHZshW6mHeJKLw1W5Tu3OQbi8jUasSRVju61eU0teHvNHwtzuGW68vmtRH9uE/rlhoqAnQ==;EndpointSuffix=core.windows.net";
string file = "appendBlop.txt";
string logs = "No Log or file not found";
CloudStorageAccount account = CloudStorageAccount.Parse(connectionString);
if (account == null) return logs;
CloudBlobClient client = account.CreateCloudBlobClient();
if (client == null) return logs;
CloudBlobContainer container = client.GetContainerReference("hotelmanagerlog");
if (!container.Exists()) container.Create();
CloudAppendBlob blob = container.GetAppendBlobReference(file);
if (!blob.Exists()) return logs;
logs = blob.DownloadText();
return logs;
}
public static void Read([QueueTrigger("customers", Connection = "AzureWebJobsStorage")] string message, TraceWriter log)
{
//Deserialize customer from the queue
Customer customer = JsonConvert.DeserializeObject <Customer>(message);
#if Debug
var builder = new ConfigurationBuilder()
.AddJsonFile("local.settings.json", optional: true, reloadOnChange: true);
IConfiguration Configuration = builder.Build();
string con = Configuration["Values:AzureWebJobsStorage"];
#endif
string con = System.Environment.GetEnvironmentVariable("AzureWebJobsStorage");
CloudBlobClient blobClient = CloudStorageAccount.Parse(con).CreateCloudBlobClient();
var folder = "files";
CloudBlobContainer container = blobClient.GetContainerReference(folder);
container.CreateIfNotExists();
CloudAppendBlob blob = container.GetAppendBlobReference("customers.txt");
if (!blob.Exists())
{
blob.CreateOrReplace();
}
blob.AppendText(customer.Name + ";" + customer.Age + ";" + customer.Phone + "\n");
// blob.UploadText(customer.Name + ";" + customer.Age + ";" + customer.Phone);
}
protected override void Write(LogEventInfo logEvent)
{
if (_client == null)
{
return;
}
string containerName = Container.Render(logEvent);
string blobName = BlobName.Render(logEvent);
if (_container == null || _container.Name != containerName)
{
_container = _client.GetContainerReference(containerName);
_blob = null;
}
if (_blob == null || _blob.Name != blobName)
{
_blob = _container.GetAppendBlobReference(blobName);
if (!_blob.Exists())
{
try
{
_blob.Properties.ContentType = "text/plain";
_blob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition(), null, null);
}
catch (StorageException)
{ }
}
}
var logMessage = this.Layout.Render(logEvent);
_blob.AppendText(logMessage + "\r\n", Encoding.UTF8);
}
private void uploadToBlob(string fileName, string message)
{
CloudAppendBlob blob = this.container.GetAppendBlobReference(fileName);
//make sure the file exsits
if (!blob.Exists())
{
blob.CreateOrReplace();
}
try {
blob.AppendText(message);
}
catch (StorageException e1)
{
//retry after sleep
System.Threading.Thread.Sleep(10);
try
{
blob.AppendText(message);
}
catch (StorageException e)
{
writeToLocalFile(fileName, message + "\r\n" + e.ToString() + "\r\n");
}
}
}
public static void WriteLogs(string log)
{
string connectionString = @"DefaultEndpointsProtocol=https;AccountName=hotelmanagerblob;AccountKey=0rIspWPXwm7U22uvlHZshW6mHeJKLw1W5Tu3OQbi8jUasSRVju61eU0teHvNHwtzuGW68vmtRH9uE/rlhoqAnQ==;EndpointSuffix=core.windows.net";
string file = "appendBlop.txt";
CloudStorageAccount account = CloudStorageAccount.Parse(connectionString);
if (account == null) return;
CloudBlobClient client = account.CreateCloudBlobClient();
if (client == null) return;
CloudBlobContainer container = client.GetContainerReference("hotelmanagerlog");
try
{
container.CreateIfNotExists();
}
catch (Exception e)
{
Debug.WriteLine(e.Message);
}
CloudAppendBlob blob = container.GetAppendBlobReference(file);
if (!blob.Exists()) blob.CreateOrReplace();
blob.AppendText($"{log} {Environment.NewLine}");
}
public void CreateAppendBlob(string filePath)
{
CloudAppendBlob blockBlob = this.cloudBlobContainer.GetAppendBlobReference(filePath);
if (!blockBlob.Exists())
{
blockBlob.CreateOrReplace();
}
}
private CloudAppendBlob GetAppendBlob()
{
CloudAppendBlob blob = this.container.GetAppendBlobReference($"{DateTimeOffset.Now.ToString("yyyyMMdd")}/{DateTimeOffset.Now.ToString("yyyyMMddTHH")}.nm4");
if (!blob.Exists())
{
blob.CreateOrReplace();
}
return(blob);
}
private async Task <CloudAppendBlob> GetAppendBlobAsync()
{
CloudAppendBlob blob = this.container.GetAppendBlobReference($"raw/{DateTimeOffset.Now.ToString("yyyy")}/{DateTimeOffset.Now.ToString("MM")}/{DateTimeOffset.Now.ToString("dd")}/{DateTimeOffset.Now.ToString("yyyyMMddTHH")}.nm4");
if (!blob.Exists())
{
await blob.CreateOrReplaceAsync().ConfigureAwait(false);
}
return(blob);
}
public WeatherMeasurements GetWeatherMeasurements(string deviceId, string measuredDate, string sensorType)
{
try
{
CreateStorageAccountFromConnectionString();
// Create file path from deviceId, sensorType and measuredDate
var prefix = string.Join("/", deviceId, sensorType);
var fileName = string.Join(".", measuredDate, "csv");
var path = string.Join("/", prefix, fileName);
// Create a blob client for interacting with the blob service.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Create a container for organizing blobs within the storage account.
CloudBlobContainer container = blobClient.GetContainerReference(CONTAINER);
CloudAppendBlob appendBlob = container.GetAppendBlobReference(path);
if (appendBlob.Exists())
{
WeatherMeasurements weatherMeasurements = new WeatherMeasurements(deviceId, measuredDate, sensorType);
using (var memoryStream = new MemoryStream())
{
appendBlob.DownloadToStream(memoryStream);
memoryStream.Position = 0;
var text = Encoding.ASCII.GetString(memoryStream.ToArray());
using (var reader = new StreamReader(memoryStream, Encoding.ASCII))
{
string line = String.Empty;
while ((line = reader.ReadLine()) != null)
{
string[] val = line.Split(';');
AtomicMeasurements atomic = new AtomicMeasurements(val[0], val[1]);
weatherMeasurements.Measurements.Add(atomic);
}
}
}
return(weatherMeasurements);
}
else
{
throw new FileNotFoundException();
}
}
catch (Exception ex)
{
throw ex;
}
}
private CloudAppendBlob NewCloudAppendBlob(string userId)
{
lock (cloudAppendBlobDictionary)
{
CloudAppendBlob cloudAppendBlob =
this.container.GetAppendBlobReference($"{userId}_{DateTime.Now.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture)}.log");
if (!cloudAppendBlob.Exists())
{
cloudAppendBlob.CreateOrReplace();
}
cloudAppendBlobDictionary[userId] = cloudAppendBlob;
return(cloudAppendBlobDictionary[userId]);
}
}
/// <summary>
/// Sends the events.
/// </summary>
/// <param name="events">The events that need to be send.</param>
/// <remarks>
/// <para>
/// The subclass must override this method to process the buffered events.
/// </para>
/// </remarks>
protected override void SendBuffer(LoggingEvent[] events)
{
CloudAppendBlob appendBlob = _cloudBlobContainer.GetAppendBlobReference(Filename(_directoryName));
if (!appendBlob.Exists())
{
appendBlob.CreateOrReplace();
}
else
{
_lineFeed = Environment.NewLine;
}
Parallel.ForEach(events, ProcessEvent);
}
private CloudAppendBlob GetBlob(DateTime eventTime)
{
string tagName = eventTime.ToString("yyyy-MM-dd");
if (tagName != _currentTagName)
{
_currentTagName = tagName;
_appendBlob = _blobContainer.GetAppendBlobReference($"{_blobNamePrefix}{_currentTagName}.txt");
if (!_appendBlob.Exists())
{
_appendBlob.CreateOrReplace();
}
}
return(_appendBlob);
}
protected override void Write(LogEventInfo logEvent)
{
var connectionString = ConnectionString.Render(logEvent);
if (_client == null || !string.Equals(_connectionString, connectionString, System.StringComparison.OrdinalIgnoreCase))
{
_client = CloudStorageAccount.Parse(connectionString).CreateCloudBlobClient();
InternalLogger.Debug("Initialized connection to {0}", connectionString);
}
string containerName = Container.Render(logEvent);
string blobName = BlobName.Render(logEvent);
if (_container == null || _container.Name != containerName)
{
_container = _client.GetContainerReference(containerName);
InternalLogger.Debug("Got container reference to {0}", containerName);
if (_container.CreateIfNotExists())
{
InternalLogger.Debug("Created container {0}", containerName);
}
_blob = null;
}
if (_blob == null || _blob.Name != blobName)
{
_blob = _container.GetAppendBlobReference(blobName);
if (!_blob.Exists())
{
try
{
_blob.Properties.ContentType = "text/plain";
_blob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition());
InternalLogger.Debug("Created blob: {0}", blobName);
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict)
{
// to be expected
}
}
}
_blob.AppendText(Layout.Render(logEvent) + "\r\n", Encoding.UTF8);
}
public async Task SaveDataToBlobAsync(string data)
{
try
{
CloudAppendBlob blob = _dataContainer.GetAppendBlobReference(GetBlobName());
if (!blob.Exists())
{
blob.CreateOrReplace();
}
await blob.AppendTextAsync(data);
}
catch (Exception ex)
{
_errorLogger.ErrorFormat("SaveDataToBlobAsync function encounter error : {0}",
ex.ToString().Replace("\r\n", " "));
}
}
public void Log(object data, string source, string message = null)
{
try
{
string dataString = (data != null) ? data as string : "null";
if (dataString == null)
{
dataString = JsonConvert.SerializeObject(data);
}
var sasUri = new Uri(this.configuration.ObjectLogSASUrl);
var accountName = sasUri.Host.TrimEndString(".blob.core.windows.net");
StorageCredentials creds = new StorageCredentials(sasUri.Query);
CloudStorageAccount strAcc = new CloudStorageAccount(creds, accountName, endpointSuffix: null, useHttps: true);
CloudBlobClient blobClient = strAcc.CreateCloudBlobClient();
//Setup our container we are going to use and create it.
CloudBlobContainer container = blobClient.GetContainerReference(configuration.ObjectLogBlobContainer);
container.CreateIfNotExistsAsync();
// Build my typical log file name.
DateTime date = DateTime.Now;
// This creates a reference to the append blob we are going to use.
CloudAppendBlob appBlob = container.GetAppendBlobReference(
$"{date.ToString("yyyy-MM")}/{date.ToString("dd")}/{date.ToString("HH")}-{source}.log");
// Now we are going to check if todays file exists and if it doesn't we create it.
if (!appBlob.Exists())
{
appBlob.CreateOrReplace();
}
// Add the entry to our log.
var logMessage = $"{date.ToString("o")}|{message}|{dataString}{Environment.NewLine}";
using (MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(logMessage)))
{
// Append a block. AppendText is not safe across multiple threads & servers so use AppendBlock instead.
appBlob.AppendBlock(ms);
}
}
catch (Exception ex)
{
this.logger.Error("Error logging data to the permanent store", ex);
}
}
/// <summary>
/// Sends the events.
/// </summary>
/// <param name="events">The events that need to be send.</param>
/// <remarks>
/// <para>
/// The subclass must override this method to process the buffered events.
/// </para>
/// </remarks>
protected override void SendBuffer(LoggingEvent[] events)
{
CloudAppendBlob appendBlob = _cloudBlobContainer.GetAppendBlobReference(Filename(_directoryName, _filenamePrefixDateTimeFormat, _filenameSuffix));
if (!appendBlob.Exists())
{
appendBlob.CreateOrReplace();
}
else
{
_lineFeed = Environment.NewLine;
}
foreach (var e in events)
{
ProcessEvent(e);
}
}
protected override void SendBuffer(LoggingEvent[] events)
{
CloudAppendBlob appendBlob = _cloudBlobContainer.GetAppendBlobReference(Filename(DirectoryName));
if (!appendBlob.Exists())
{
appendBlob.CreateOrReplace();
}
StringBuilder sb = new StringBuilder();
foreach (var loggingEvent in events)
{
sb.Append(RenderLoggingEvent(loggingEvent));
}
appendBlob.AppendText(sb.ToString());
}
private void WriteToAppendBlob(string logLevel, string log)
{
DateTime date = DateTime.Today;
DateTime dateLogEntry = DateTime.UtcNow;
try
{
CloudAppendBlob appBlob = _container.GetAppendBlobReference(string.Format("{0}{1}", date.ToString("yyyy-MM-dd"), ".log"));
if (!appBlob.Exists())
{
appBlob.CreateOrReplace();
}
appBlob.AppendText(string.Format("{0}\t{1}\t{2}\r\n", dateLogEntry.ToString("o"), logLevel, log));
}
catch (Exception)
{
;
}
}
private void InstanciateStorageBlob(string formattedStorageContainerName, string formattedStorageBlobName)
{
if ((_appendBlob == null) || (_appendBlob.Name != formattedStorageBlobName))
{
_appendBlob = _blobContainer.GetAppendBlobReference(formattedStorageBlobName);
if (!_appendBlob.Exists())
{
try
{
_appendBlob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition(), null, null);
}
catch (StorageException storageException)
{
WriteDebugError(String.Format("NLog.AzureStorage - Failed to create Azure Storage Append Blob '{0}' in the Container '{1}' - Storage Exception: {2} {3}", formattedStorageBlobName, formattedStorageContainerName, storageException.Message, GetStorageExceptionHttpStatusMessage(storageException)));
throw;
}
}
}
}
/// <summary>
/// Create the blob storage refrences
/// </summary>
/// <param name="containerref">name of the container references
///</param>
/// <param name="blockref">name of the blob reference</param>
public StorageClient(string containerref, string blockref)
{
// Parse the connection string and return a reference to the storage account.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve a reference to a container.
CloudBlobContainer container = blobClient.GetContainerReference(containerref);
// Create the container if it doesn't already exist.
container.CreateIfNotExists();
// Retrieve reference to a blob named "myblob".
_blockBlob = container.GetAppendBlobReference(blockref);
if (!_blockBlob.Exists())
{
_blockBlob.CreateOrReplace();
}
}
/// <summary>
/// Initializes the BLOB.
/// </summary>
/// <param name="blobName">Name of the BLOB.</param>
private void InitializeBlob(string blobName)
{
if (_appendBlob == null || _appendBlob.Name != blobName)
{
_appendBlob = _container.GetAppendBlobReference(blobName);
if (!_appendBlob.Exists())
{
try
{
_appendBlob.Properties.ContentType = "text/plain";
_appendBlob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition());
}
catch (StorageException ex)
{
InternalLogger.Error(ex, "Initialize Blob Exception");
throw;
}
}
}
}
public static void AppendToBlob(string fileName, string textToAppend)
{
CloudStorageAccount account = CloudStorageAccount.Parse(AmbientConnectionStringProvider.Instance.GetConnectionString("blobConnection"));
CloudBlobClient client = account.CreateCloudBlobClient();
CloudBlobContainer container = client.GetContainerReference("images");
CloudAppendBlob appendBlob = container.GetAppendBlobReference(fileName);
if (!appendBlob.Exists())
{
appendBlob.CreateOrReplace();
}
MemoryStream stream = new MemoryStream();
StreamWriter writer = new StreamWriter(stream);
writer.Write(textToAppend + Environment.NewLine);
writer.Flush();
stream.Position = 0;
appendBlob.AppendBlock(stream);
}
protected override void Write(LogEventInfo logEvent)
{
if (_client == null)
{
return;
}
string containerName = Container.Render(logEvent);
string blobName = BlobName.Render(logEvent);
if (_container == null || _container.Name != containerName)
{
_container = _client.GetContainerReference(containerName);
_blob = null;
}
if (_blob == null || _blob.Name != blobName)
{
_blob = _container.GetAppendBlobReference(blobName);
if (!_blob.Exists())
{
try
{
_blob.Properties.ContentType = "text/plain";
_blob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition());
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict)
{
// to be expected
}
}
}
_blob.AppendText(Layout.Render(logEvent) + "\r\n", Encoding.UTF8);
}
static void Main(string[] args)
{
Assembly execAssembly = Assembly.GetCallingAssembly();
AssemblyName assemblyName = execAssembly.GetName();
int traceLevel = 3;
int event_count = 0;
int total_event_count = 0;
int updowncount = 0;
List <EventData> e = new List <EventData>();
//EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(connectionString);
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(CloudConfigurationManager.GetSetting("EventHubConnectionString"));
// Parse the connection string and return a reference to the storage account.
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
// Create the blob client.
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve reference to a previously created container.
CloudBlobContainer container = blobClient.GetContainerReference("syslogs");
// Retrieve reference to a previously created container for output logs
CloudBlobContainer containerlogs = blobClient.GetContainerReference("logs");
// Create the logs container if it does not already exist
containerlogs.CreateIfNotExists();
CloudAppendBlob logout = containerlogs.GetAppendBlobReference("ideapoceh04out");
if (!logout.Exists())
{
logout.CreateOrReplace();
}
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Beginning execution {2}", DateTime.UtcNow, assemblyName, Environment.NewLine));
if (traceLevel == 4)
{
// Loop over items within the container and output the length and URI.
foreach (IListBlobItem item in container.ListBlobs(null, false))
{
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
//Console.WriteLine("Block blob of length {0}: {1}", blob.Properties.Length, blob.Uri);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Block blob of length {2}: {3} {4}", DateTime.UtcNow, assemblyName, blob.Properties.Length, blob.Uri, Environment.NewLine));
}
else if (item.GetType() == typeof(CloudPageBlob))
{
CloudPageBlob pageBlob = (CloudPageBlob)item;
//Console.WriteLine("Page blob of length {0}: {1}", pageBlob.Properties.Length, pageBlob.Uri);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Page blob of length {2}: {3} {4}", DateTime.UtcNow, assemblyName, pageBlob.Properties.Length, pageBlob.Uri, Environment.NewLine));
}
else if (item.GetType() == typeof(CloudBlobDirectory))
{
CloudBlobDirectory directory = (CloudBlobDirectory)item;
//Console.WriteLine("Directory: {0}", directory.Uri);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t Directory {2}: {3}", DateTime.UtcNow, assemblyName, directory.Uri, Environment.NewLine));
}
else if (item.GetType() == typeof(CloudAppendBlob))
{
CloudAppendBlob appendBlob = (CloudAppendBlob)item;
//Console.WriteLine("Append blob of length: {0}: {1}", appendBlob.Properties.Length, appendBlob.Uri);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Append blob of length {2}: {3} {4}", DateTime.UtcNow, assemblyName, appendBlob.Properties.Length, appendBlob.Uri, Environment.NewLine));
}
}
}
//Console.ReadLine();
// Retrieve reference to a blob named "myblob.txt"
CloudBlockBlob blockBlob2 = container.GetBlockBlobReference("mls_syslog");
string line;
//string json_eh_message;
DateTime startTime = DateTime.UtcNow;
//using (var memoryStream = new MemoryStream())
using (var stream = blockBlob2.OpenRead())
{
using (StreamReader reader = new StreamReader(stream))
{
// while ( !reader.EndOfStream)
while ((line = reader.ReadLine()) != null)
{
// Console.WriteLine(reader.ReadLine());
//Console.WriteLine(line);
if (traceLevel == 4)
{
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} {3}", DateTime.UtcNow, assemblyName, line, Environment.NewLine));
}
//Console.ReadLine();
string[] items = line.Split('\t');
syslog sl = new syslog();
sl.time = items[0];
sl.facility = items[1];
sl.address = items[2];
sl.message = items[3];
bool testupdown = sl.message.StartsWith("%LINK-3-UPDOWN:");
/* Console.WriteLine("Found UPDOWN? {0}", testupdown); */
if (testupdown)
{
updowncount++;
string[] messageItems = sl.message.Split(' ');
string msgInterface = messageItems[2];
/* Console.WriteLine("Interface is {0}", msgInterface); */
sl.netInterface = msgInterface;
}
// string json_eh_message = JsonConvert.SerializeObject(sl);
json_eh_message = JsonConvert.SerializeObject(sl);
if (event_count > 500)
{
try
{
//eventHubClient.SendBatchAsync(e);
eventHubClient.SendBatch(e);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Submitting a batch of {2} events to EventHub {3}", DateTime.UtcNow, assemblyName, event_count, Environment.NewLine));
//Console.WriteLine("Submitting batch of {0} events", event_count);
}
catch (Exception exception)
{
//Console.ForegroundColor = ConsoleColor.Red;
//Console.WriteLine("{0} > Exception: {1}", DateTime.Now, exception.Message);
//Console.ResetColor();
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Exception raised: {2} {3}", DateTime.UtcNow, assemblyName, exception.Message, Environment.NewLine));
}
total_event_count = total_event_count + event_count;
event_count = 0;
//Console.WriteLine("Total events submitted: {0}", total_event_count);
e.Clear();
e.Add(new EventData(Encoding.UTF8.GetBytes(json_eh_message)));
event_count++;
//Console.WriteLine(json_eh_message);
}
else
{
e.Add(new EventData(Encoding.UTF8.GetBytes(json_eh_message)));
event_count++;
//Console.WriteLine(json_eh_message);
}
} // EOF -- submit final batch of remaining events
try
{
//eventHubClient.SendBatchAsync(e);
eventHubClient.SendBatch(e);
//Console.WriteLine("Submitting final batch of {0}", event_count);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Submitting final batch of {2} events to EventHub {3}", DateTime.UtcNow, assemblyName, event_count, Environment.NewLine));
}
catch (Exception exception)
{
//Console.ForegroundColor = ConsoleColor.Red;
//Console.WriteLine("{0} > Exception: {1}", DateTime.Now, exception.Message);
//Console.ResetColor();
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Exception occurred while submitting batch to EventHub: {2} {3}", DateTime.UtcNow, assemblyName, exception.Message, Environment.NewLine));
}
total_event_count += event_count;
}
}
DateTime endTime = DateTime.UtcNow;
var ts = new TimeSpan(startTime.Ticks - endTime.Ticks);
double delta = Math.Abs(ts.TotalSeconds);
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} link up/down events processed {3}", DateTime.UtcNow, assemblyName, updowncount, Environment.NewLine));
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} total events processed {3}", DateTime.UtcNow, assemblyName, total_event_count, Environment.NewLine));
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} events per second {3}", DateTime.UtcNow, assemblyName, total_event_count / delta, Environment.NewLine));
logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Ending execution {2}", DateTime.UtcNow, assemblyName, Environment.NewLine));
//Console.WriteLine("Port flap event count: {0}", updowncount);
//Console.WriteLine("Total events: {0}", total_event_count);
//Console.ReadLine();
//using (var memoryStream2 = new BufferedStream(memoryStream))
/* {
* blockBlob2.DownloadToStream(memoryStream);
* //while ((text = memoryStream2.Read)
* text = System.Text.Encoding.UTF8.GetString(memoryStream.ToArray());
* text = memoryStream.
* Console.WriteLine(text);
* Console.ReadLine();
* } */
}
static void AzureStorageBlobExamples()
{
// Getting configuration from the App.config file and get access to the CloudStorageAccount.
// Azure access Key should is stored in the App.config
String storageKeyValue = CloudConfigurationManager.GetSetting("AzureStorageAccount");
// Just in case to check whether reading the correct value
Console.WriteLine("Storage Account Key Used" + storageKeyValue);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageKeyValue);
// Getting the Azure Client
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
// Getting reference for the container
CloudBlobContainer cloudContainer = cloudBlobClient.GetContainerReference("images");
// Create the container (folder) if does not exists
cloudContainer.CreateIfNotExists();
// Get the reference for the Blob in Azure
CloudBlockBlob blobCloud = cloudContainer.GetBlockBlobReference("photo.png");
// Learning 1: Copy a file from local path in to Azure Blob storage
//Open a local file and upload to the Azure
using (var fileStream = System.IO.File.OpenRead(@"C:\Users\Administrator\Pictures\photo.jpg"))
{
blobCloud.UploadFromStream(fileStream);
Console.WriteLine("File is uploaded to the cloud");
}
// Learning 2: Download a blob from Azure to the local host
// Just copy a blob from cloud to a local file. Assume the same file which has been uploaded in the previous step
blobCloud.DownloadToFile(@"C:\Users\Administrator\Pictures\photocopy.jpg", FileMode.CreateNew);
// Learning 3: List all blob objects URI from a container
//List all blobs in a container
var blobs = cloudContainer.ListBlobs();
foreach (var blob in blobs)
{
Console.Write(blob.Uri);
}
// Learning 4: List properties of a continer (Folder) and listing metadata of a container
cloudContainer.FetchAttributes();
Console.WriteLine(cloudContainer.Properties.LastModified);
Console.WriteLine(cloudContainer.Properties.ETag);
var metaData = cloudContainer.Metadata;
foreach (var metaDataItem in metaData)
{
Console.Write("Key " + metaDataItem.Key + " & ");
Console.WriteLine("Value" + metaDataItem.Value);
}
// Learning 5: Setting metaData for a container
// Method 1
cloudContainer.Metadata.Add("SampleKey", "SampleValue");
// Method 2
cloudContainer.Metadata["SecondSample"] = "Second Value";
// Dont ask me why FetchAttributes() and SetMetadata() ! why not SetAttributes() or GetMetaData? Microsoft Way!
cloudContainer.SetMetadata();
// Learning 6: Setting permission for a container
BlobContainerPermissions permissions = cloudContainer.GetPermissions();
Console.WriteLine("Container permission " + permissions.PublicAccess.ToString());
foreach (var sharedAccessPolicy in permissions.SharedAccessPolicies)
{
Console.WriteLine(sharedAccessPolicy.Key.ToString() + " = " + sharedAccessPolicy.Value.ToString());
}
// In order to as per the parent container
permissions.PublicAccess = BlobContainerPublicAccessType.Container;
// In order to remove the public access
permissions.PublicAccess = BlobContainerPublicAccessType.Off;
//Finally set the permission
cloudContainer.SetPermissions(permissions);
// Learning 7: Azure copy from one blob to another
// Create a new Block Blog Reference
CloudBlockBlob copyBlob = cloudContainer.GetBlockBlobReference("photo-copy.jpg");
// Copy the original - blobCloud to the copyBlob
copyBlob.StartCopy(new Uri(blobCloud.Uri.AbsoluteUri));
// Learning 8: Copy all blobs from one container to another
CloudBlobContainer sourceContainer = cloudContainer;
CloudBlobContainer targetContainer = cloudBlobClient.GetContainerReference("newimages");
targetContainer.CreateIfNotExists();
foreach (var blob in sourceContainer.ListBlobs())
{
var sourceBlob = blob as CloudBlob;
Console.WriteLine("Source Blob " + sourceBlob.Name);
CloudBlockBlob newBlob = targetContainer.GetBlockBlobReference(sourceBlob.Name);
newBlob.StartCopy(new Uri(blob.Uri.AbsoluteUri));
}
// Learning 9: Rename the blob
CloudBlockBlob sourceBlockBlob = cloudContainer.GetBlockBlobReference("photo-copy.jpg");
CloudBlockBlob targetBlockBlob = cloudContainer.GetBlockBlobReference("copy-photo.jpg");
targetBlockBlob.StartCopy(new Uri(sourceBlockBlob.Uri.AbsoluteUri));
while (targetBlockBlob.CopyState.Status == CopyStatus.Pending)
{
// Sleep for 3 seconds
System.Threading.Thread.Sleep(2000);
}
sourceBlockBlob.Delete();
// Learning 10: Appending to a blob
DateTime date = DateTime.Today;
CloudBlobContainer logContainer = cloudBlobClient.GetContainerReference("logs");
CloudAppendBlob logBlog = logContainer.GetAppendBlobReference(string.Format("{0}{1}", date.ToString("yyyyMMdd"), ".log"));
logContainer.CreateIfNotExists();
// If the append blog does not exists, create one
if (!logBlog.Exists())
{
logBlog.CreateOrReplace();
}
// AppendText
logBlog.AppendText(string.Format("{0} : Azure is rocking in the cloud space at ", date.ToString("HH:MM:ss")));
// Similar to the AppendText, there are
// logBlog.AppendBlock
// logBlog.AppendFromByteArray
// logBlog.AppendFromFile
// logBlog.AppendFromStream
// Finally display the content of the log file.
Console.WriteLine(logBlog.DownloadText());
// Learning 11: Multiple Chunk of file upload to Azure
AzureStorageReference.Program.uploadLargeFiles(cloudBlobClient, @"C:\Users\Administrator\Pictures");
//Learning 12: Upload using Async
AsyncCallback callBack = new AsyncCallback(x => Console.WriteLine("Copy Async Completed"));
CloudBlockBlob copyAsync = cloudContainer.GetBlockBlobReference("newphoto.png");
copyAsync.BeginStartCopy(blobCloud, callBack, null);
Console.WriteLine("Press any key to continue");
Console.ReadKey();
}
public static void Run([BlobTrigger("telemetry-archive/willowtelemetry01/{name}", Connection = "StorageAccount")] Stream myBlob, string name, TraceWriter log)
{
DataSet ds = new DataSet(
new SchemaElement <int>("ObjectID"),
new SchemaElement <string>("Value"),
new SchemaElement <string>("ClientID"),
new SchemaElement <string>("TimeStamp")
);
if (myBlob.Length == 508)
{
return;
} // empty dataset
//log.Info($"C# Blob trigger function Processed blob\n Name:{name} \n Size: {myBlob.Length} Bytes");
var serializer = AvroSerializer.CreateGeneric(Schema);
var jsonSerializer = new JsonSerializer();
using (var reader = AvroContainer.CreateGenericReader(myBlob))
{
using (var streamReader = new SequentialReader <object>(reader))
{
var results = streamReader.Objects;
foreach (var item in results)
{
var body = ((AvroRecord)item)[5]; // ["Body"] is the 5th element
var json = System.Text.Encoding.Default.GetString((byte[])body);
var array = (JArray)JsonConvert.DeserializeObject(json);
var telemetry = jsonSerializer.Deserialize <Telemetry>(array[0].CreateReader());
ds.Add(telemetry.ObjectID, telemetry.Value, telemetry.ClientID, telemetry.TimeStamp.ToString());
}
}
}
//log.Info($"{ds.RowCount} Records processed");
if (ds.RowCount == 0)
{
return;
}
storageAccount = CloudStorageAccount.Parse(storageAcct);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer blobContainer = blobClient.GetContainerReference(parquetContainerName);
blobContainer.CreateIfNotExists();
name = name + ".parquet";
CloudBlob blob = blobContainer.GetBlobReference(blobName);
CloudAppendBlob appendBlob = blobContainer.GetAppendBlobReference(name);
if (appendBlob.Exists())
{
appendBlob.Delete(DeleteSnapshotsOption.IncludeSnapshots);
}
log.Info($"===> {ds.RowCount} Records, {name}");
using (var ms = new MemoryStream())
{
using (var writer = new ParquetWriter(ms))
{
writer.Write(ds);
}
ms.Seek(0, SeekOrigin.Begin);
appendBlob.UploadFromStream(ms);
}
}
