public void UploadToAppendBlob()
        {
            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting("StorageConnectionString"));

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            CloudBlobContainer container = blobClient.GetContainerReference("my-append-blobs");

            container.CreateIfNotExists();

            CloudAppendBlob appendBlob = container.GetAppendBlobReference("append-blob.log");

            if (!appendBlob.Exists())
            {
                appendBlob.CreateOrReplace();
            }

            int numBlocks = 10;

            Random rnd = new Random();

            byte[] bytes = new byte[numBlocks];
            rnd.NextBytes(bytes);

            //Simulate a logging operation by writing text data and byte data to the end of the append blob.
            for (int i = 0; i < numBlocks; i++)
            {
                appendBlob.AppendText(string.Format("Timestamp: {0:u} \tLog Entry: {1}{2}", DateTime.UtcNow, bytes[i], Environment.NewLine));
            }

            Console.WriteLine(appendBlob.DownloadText());
        }
示例#2
0
        private bool AppendToBlob(CloudAppendBlob blob, MemoryStream stream)
        {
            try
            {
                if (!blob.Exists())
                {
                    return(false);
                }

                blob.AppendBlock(stream,
                                 accessCondition: AccessCondition.GenerateIfMaxSizeLessThanOrEqualCondition(_maxBlobSize));
            }
            catch (StorageException ex)
            {
                if (ex.RequestInformation.HttpStatusCode == 412) // HttpStatusCode.PreconditionFailed
                {
                    return(false);
                }
                else
                {
                    throw;
                }
            }

            return(true);
        }
示例#3
0
        public static string GetLogs()
        {
            string connectionString = @"DefaultEndpointsProtocol=https;AccountName=hotelmanagerblob;AccountKey=0rIspWPXwm7U22uvlHZshW6mHeJKLw1W5Tu3OQbi8jUasSRVju61eU0teHvNHwtzuGW68vmtRH9uE/rlhoqAnQ==;EndpointSuffix=core.windows.net";
                                      
            string file = "appendBlop.txt";
            string logs = "No Log or file not found";

            CloudStorageAccount account = CloudStorageAccount.Parse(connectionString);

            if (account == null) return logs;

            CloudBlobClient client = account.CreateCloudBlobClient();

            if (client == null) return logs;

            CloudBlobContainer container = client.GetContainerReference("hotelmanagerlog");

            if (!container.Exists()) container.Create();

            CloudAppendBlob blob = container.GetAppendBlobReference(file);

            if (!blob.Exists()) return logs;

            logs = blob.DownloadText();

            return logs;
        }
示例#4
0
        public static void Read([QueueTrigger("customers", Connection = "AzureWebJobsStorage")] string message, TraceWriter log)
        {
            //Deserialize customer from the queue
            Customer customer = JsonConvert.DeserializeObject <Customer>(message);

#if Debug
            var builder = new ConfigurationBuilder()
                          .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true);
            IConfiguration Configuration = builder.Build();

            string con = Configuration["Values:AzureWebJobsStorage"];
#endif

            string con = System.Environment.GetEnvironmentVariable("AzureWebJobsStorage");



            CloudBlobClient blobClient = CloudStorageAccount.Parse(con).CreateCloudBlobClient();

            var folder = "files";

            CloudBlobContainer container = blobClient.GetContainerReference(folder);
            container.CreateIfNotExists();

            CloudAppendBlob blob = container.GetAppendBlobReference("customers.txt");

            if (!blob.Exists())
            {
                blob.CreateOrReplace();
            }

            blob.AppendText(customer.Name + ";" + customer.Age + ";" + customer.Phone + "\n");

            // blob.UploadText(customer.Name + ";" + customer.Age + ";" + customer.Phone);
        }
示例#5
0
        protected override void Write(LogEventInfo logEvent)
        {
            if (_client == null)
            {
                return;
            }

            string containerName = Container.Render(logEvent);
            string blobName      = BlobName.Render(logEvent);

            if (_container == null || _container.Name != containerName)
            {
                _container = _client.GetContainerReference(containerName);
                _blob      = null;
            }

            if (_blob == null || _blob.Name != blobName)
            {
                _blob = _container.GetAppendBlobReference(blobName);

                if (!_blob.Exists())
                {
                    try
                    {
                        _blob.Properties.ContentType = "text/plain";
                        _blob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition(), null, null);
                    }
                    catch (StorageException)
                    { }
                }
            }
            var logMessage = this.Layout.Render(logEvent);

            _blob.AppendText(logMessage + "\r\n", Encoding.UTF8);
        }
示例#6
0
        private void uploadToBlob(string fileName, string message)
        {
            CloudAppendBlob blob = this.container.GetAppendBlobReference(fileName);

            //make sure the file exsits
            if (!blob.Exists())
            {
                blob.CreateOrReplace();
            }
            try {
                blob.AppendText(message);
            }
            catch (StorageException e1)
            {
                //retry after sleep
                System.Threading.Thread.Sleep(10);
                try
                {
                    blob.AppendText(message);
                }
                catch (StorageException e)
                {
                    writeToLocalFile(fileName, message + "\r\n" + e.ToString() + "\r\n");
                }
            }
        }
示例#7
0
        public static void WriteLogs(string log)
        {
            string connectionString = @"DefaultEndpointsProtocol=https;AccountName=hotelmanagerblob;AccountKey=0rIspWPXwm7U22uvlHZshW6mHeJKLw1W5Tu3OQbi8jUasSRVju61eU0teHvNHwtzuGW68vmtRH9uE/rlhoqAnQ==;EndpointSuffix=core.windows.net";
            string file = "appendBlop.txt";

            CloudStorageAccount account = CloudStorageAccount.Parse(connectionString);

            if (account == null) return;

            CloudBlobClient client = account.CreateCloudBlobClient();

            if (client == null) return;

            CloudBlobContainer container = client.GetContainerReference("hotelmanagerlog");

            try
            {
                container.CreateIfNotExists();
            }
            catch (Exception e)
            {
                Debug.WriteLine(e.Message);
            }


            CloudAppendBlob blob = container.GetAppendBlobReference(file);

            if (!blob.Exists()) blob.CreateOrReplace();

            blob.AppendText($"{log} {Environment.NewLine}");
        }
示例#8
0
        public void CreateAppendBlob(string filePath)
        {
            CloudAppendBlob blockBlob = this.cloudBlobContainer.GetAppendBlobReference(filePath);

            if (!blockBlob.Exists())
            {
                blockBlob.CreateOrReplace();
            }
        }
示例#9
0
        private CloudAppendBlob GetAppendBlob()
        {
            CloudAppendBlob blob = this.container.GetAppendBlobReference($"{DateTimeOffset.Now.ToString("yyyyMMdd")}/{DateTimeOffset.Now.ToString("yyyyMMddTHH")}.nm4");

            if (!blob.Exists())
            {
                blob.CreateOrReplace();
            }

            return(blob);
        }
示例#10
0
        private async Task <CloudAppendBlob> GetAppendBlobAsync()
        {
            CloudAppendBlob blob = this.container.GetAppendBlobReference($"raw/{DateTimeOffset.Now.ToString("yyyy")}/{DateTimeOffset.Now.ToString("MM")}/{DateTimeOffset.Now.ToString("dd")}/{DateTimeOffset.Now.ToString("yyyyMMddTHH")}.nm4");

            if (!blob.Exists())
            {
                await blob.CreateOrReplaceAsync().ConfigureAwait(false);
            }

            return(blob);
        }
示例#11
0
        public WeatherMeasurements GetWeatherMeasurements(string deviceId, string measuredDate, string sensorType)
        {
            try
            {
                CreateStorageAccountFromConnectionString();

                // Create file path from deviceId, sensorType and measuredDate
                var prefix   = string.Join("/", deviceId, sensorType);
                var fileName = string.Join(".", measuredDate, "csv");
                var path     = string.Join("/", prefix, fileName);

                // Create a blob client for interacting with the blob service.
                CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

                // Create a container for organizing blobs within the storage account.
                CloudBlobContainer container = blobClient.GetContainerReference(CONTAINER);

                CloudAppendBlob appendBlob = container.GetAppendBlobReference(path);

                if (appendBlob.Exists())
                {
                    WeatherMeasurements weatherMeasurements = new WeatherMeasurements(deviceId, measuredDate, sensorType);
                    using (var memoryStream = new MemoryStream())
                    {
                        appendBlob.DownloadToStream(memoryStream);
                        memoryStream.Position = 0;
                        var text = Encoding.ASCII.GetString(memoryStream.ToArray());
                        using (var reader = new StreamReader(memoryStream, Encoding.ASCII))
                        {
                            string line = String.Empty;
                            while ((line = reader.ReadLine()) != null)
                            {
                                string[]           val    = line.Split(';');
                                AtomicMeasurements atomic = new AtomicMeasurements(val[0], val[1]);
                                weatherMeasurements.Measurements.Add(atomic);
                            }
                        }
                    }
                    return(weatherMeasurements);
                }
                else
                {
                    throw new FileNotFoundException();
                }
            }
            catch (Exception ex)
            {
                throw ex;
            }
        }
 private CloudAppendBlob NewCloudAppendBlob(string userId)
 {
     lock (cloudAppendBlobDictionary)
     {
         CloudAppendBlob cloudAppendBlob =
             this.container.GetAppendBlobReference($"{userId}_{DateTime.Now.ToString("yyyy-MM-dd-HH-mm", CultureInfo.InvariantCulture)}.log");
         if (!cloudAppendBlob.Exists())
         {
             cloudAppendBlob.CreateOrReplace();
         }
         cloudAppendBlobDictionary[userId] = cloudAppendBlob;
         return(cloudAppendBlobDictionary[userId]);
     }
 }
示例#13
0
        /// <summary>
        /// Sends the events.
        /// </summary>
        /// <param name="events">The events that need to be send.</param>
        /// <remarks>
        /// <para>
        /// The subclass must override this method to process the buffered events.
        /// </para>
        /// </remarks>
        protected override void SendBuffer(LoggingEvent[] events)
        {
            CloudAppendBlob appendBlob = _cloudBlobContainer.GetAppendBlobReference(Filename(_directoryName));

            if (!appendBlob.Exists())
            {
                appendBlob.CreateOrReplace();
            }
            else
            {
                _lineFeed = Environment.NewLine;
            }

            Parallel.ForEach(events, ProcessEvent);
        }
示例#14
0
        private CloudAppendBlob GetBlob(DateTime eventTime)
        {
            string tagName = eventTime.ToString("yyyy-MM-dd");

            if (tagName != _currentTagName)
            {
                _currentTagName = tagName;
                _appendBlob     = _blobContainer.GetAppendBlobReference($"{_blobNamePrefix}{_currentTagName}.txt");
                if (!_appendBlob.Exists())
                {
                    _appendBlob.CreateOrReplace();
                }
            }

            return(_appendBlob);
        }
        protected override void Write(LogEventInfo logEvent)
        {
            var connectionString = ConnectionString.Render(logEvent);

            if (_client == null || !string.Equals(_connectionString, connectionString, System.StringComparison.OrdinalIgnoreCase))
            {
                _client = CloudStorageAccount.Parse(connectionString).CreateCloudBlobClient();
                InternalLogger.Debug("Initialized connection to {0}", connectionString);
            }

            string containerName = Container.Render(logEvent);
            string blobName      = BlobName.Render(logEvent);

            if (_container == null || _container.Name != containerName)
            {
                _container = _client.GetContainerReference(containerName);
                InternalLogger.Debug("Got container reference to {0}", containerName);

                if (_container.CreateIfNotExists())
                {
                    InternalLogger.Debug("Created container {0}", containerName);
                }

                _blob = null;
            }

            if (_blob == null || _blob.Name != blobName)
            {
                _blob = _container.GetAppendBlobReference(blobName);

                if (!_blob.Exists())
                {
                    try
                    {
                        _blob.Properties.ContentType = "text/plain";
                        _blob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition());
                        InternalLogger.Debug("Created blob: {0}", blobName);
                    }
                    catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict)
                    {
                        // to be expected
                    }
                }
            }

            _blob.AppendText(Layout.Render(logEvent) + "\r\n", Encoding.UTF8);
        }
示例#16
0
 public async Task SaveDataToBlobAsync(string data)
 {
     try
     {
         CloudAppendBlob blob = _dataContainer.GetAppendBlobReference(GetBlobName());
         if (!blob.Exists())
         {
             blob.CreateOrReplace();
         }
         await blob.AppendTextAsync(data);
     }
     catch (Exception ex)
     {
         _errorLogger.ErrorFormat("SaveDataToBlobAsync function encounter error : {0}",
                                  ex.ToString().Replace("\r\n", " "));
     }
 }
        public void Log(object data, string source, string message = null)
        {
            try
            {
                string dataString = (data != null) ? data as string : "null";
                if (dataString == null)
                {
                    dataString = JsonConvert.SerializeObject(data);
                }

                var sasUri                     = new Uri(this.configuration.ObjectLogSASUrl);
                var accountName                = sasUri.Host.TrimEndString(".blob.core.windows.net");
                StorageCredentials  creds      = new StorageCredentials(sasUri.Query);
                CloudStorageAccount strAcc     = new CloudStorageAccount(creds, accountName, endpointSuffix: null, useHttps: true);
                CloudBlobClient     blobClient = strAcc.CreateCloudBlobClient();

                //Setup our container we are going to use and create it.
                CloudBlobContainer container = blobClient.GetContainerReference(configuration.ObjectLogBlobContainer);
                container.CreateIfNotExistsAsync();

                // Build my typical log file name.
                DateTime date = DateTime.Now;

                // This creates a reference to the append blob we are going to use.
                CloudAppendBlob appBlob = container.GetAppendBlobReference(
                    $"{date.ToString("yyyy-MM")}/{date.ToString("dd")}/{date.ToString("HH")}-{source}.log");

                // Now we are going to check if todays file exists and if it doesn't we create it.
                if (!appBlob.Exists())
                {
                    appBlob.CreateOrReplace();
                }

                // Add the entry to our log.
                var logMessage = $"{date.ToString("o")}|{message}|{dataString}{Environment.NewLine}";
                using (MemoryStream ms = new MemoryStream(Encoding.UTF8.GetBytes(logMessage)))
                {
                    // Append a block.  AppendText is not safe across multiple threads & servers so use AppendBlock instead.
                    appBlob.AppendBlock(ms);
                }
            }
            catch (Exception ex)
            {
                this.logger.Error("Error logging data to the permanent store", ex);
            }
        }
示例#18
0
        /// <summary>
        /// Sends the events.
        /// </summary>
        /// <param name="events">The events that need to be send.</param>
        /// <remarks>
        /// <para>
        /// The subclass must override this method to process the buffered events.
        /// </para>
        /// </remarks>
        protected override void SendBuffer(LoggingEvent[] events)
        {
            CloudAppendBlob appendBlob = _cloudBlobContainer.GetAppendBlobReference(Filename(_directoryName, _filenamePrefixDateTimeFormat, _filenameSuffix));

            if (!appendBlob.Exists())
            {
                appendBlob.CreateOrReplace();
            }
            else
            {
                _lineFeed = Environment.NewLine;
            }

            foreach (var e in events)
            {
                ProcessEvent(e);
            }
        }
示例#19
0
        protected override void SendBuffer(LoggingEvent[] events)
        {
            CloudAppendBlob appendBlob = _cloudBlobContainer.GetAppendBlobReference(Filename(DirectoryName));

            if (!appendBlob.Exists())
            {
                appendBlob.CreateOrReplace();
            }

            StringBuilder sb = new StringBuilder();

            foreach (var loggingEvent in events)
            {
                sb.Append(RenderLoggingEvent(loggingEvent));
            }

            appendBlob.AppendText(sb.ToString());
        }
示例#20
0
        private void WriteToAppendBlob(string logLevel, string log)
        {
            DateTime date         = DateTime.Today;
            DateTime dateLogEntry = DateTime.UtcNow;

            try
            {
                CloudAppendBlob appBlob = _container.GetAppendBlobReference(string.Format("{0}{1}", date.ToString("yyyy-MM-dd"), ".log"));
                if (!appBlob.Exists())
                {
                    appBlob.CreateOrReplace();
                }
                appBlob.AppendText(string.Format("{0}\t{1}\t{2}\r\n", dateLogEntry.ToString("o"), logLevel, log));
            }
            catch (Exception)
            {
                ;
            }
        }
示例#21
0
        private void InstanciateStorageBlob(string formattedStorageContainerName, string formattedStorageBlobName)
        {
            if ((_appendBlob == null) || (_appendBlob.Name != formattedStorageBlobName))
            {
                _appendBlob = _blobContainer.GetAppendBlobReference(formattedStorageBlobName);

                if (!_appendBlob.Exists())
                {
                    try
                    {
                        _appendBlob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition(), null, null);
                    }
                    catch (StorageException storageException)
                    {
                        WriteDebugError(String.Format("NLog.AzureStorage - Failed to create Azure Storage Append Blob '{0}' in the Container '{1}' - Storage Exception: {2} {3}", formattedStorageBlobName, formattedStorageContainerName, storageException.Message, GetStorageExceptionHttpStatusMessage(storageException)));
                        throw;
                    }
                }
            }
        }
示例#22
0
        /// <summary>
        /// Create the blob storage refrences
        /// </summary>
        /// <param name="containerref">name of the container references
        ///</param>
        /// <param name="blockref">name of the blob reference</param>
        public StorageClient(string containerref, string blockref)
        {
            // Parse the connection string and return a reference to the storage account.
            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
                CloudConfigurationManager.GetSetting("StorageConnectionString"));

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Retrieve a reference to a container.
            CloudBlobContainer container = blobClient.GetContainerReference(containerref);

            // Create the container if it doesn't already exist.
            container.CreateIfNotExists();

            // Retrieve reference to a blob named "myblob".
            _blockBlob = container.GetAppendBlobReference(blockref);
            if (!_blockBlob.Exists())
            {
                _blockBlob.CreateOrReplace();
            }
        }
示例#23
0
        /// <summary>
        /// Initializes the BLOB.
        /// </summary>
        /// <param name="blobName">Name of the BLOB.</param>
        private void InitializeBlob(string blobName)
        {
            if (_appendBlob == null || _appendBlob.Name != blobName)
            {
                _appendBlob = _container.GetAppendBlobReference(blobName);

                if (!_appendBlob.Exists())
                {
                    try
                    {
                        _appendBlob.Properties.ContentType = "text/plain";
                        _appendBlob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition());
                    }
                    catch (StorageException ex)
                    {
                        InternalLogger.Error(ex, "Initialize Blob Exception");
                        throw;
                    }
                }
            }
        }
    public static void AppendToBlob(string fileName, string textToAppend)
    {
        CloudStorageAccount account = CloudStorageAccount.Parse(AmbientConnectionStringProvider.Instance.GetConnectionString("blobConnection"));

        CloudBlobClient    client    = account.CreateCloudBlobClient();
        CloudBlobContainer container = client.GetContainerReference("images");

        CloudAppendBlob appendBlob = container.GetAppendBlobReference(fileName);

        if (!appendBlob.Exists())
        {
            appendBlob.CreateOrReplace();
        }

        MemoryStream stream = new MemoryStream();
        StreamWriter writer = new StreamWriter(stream);

        writer.Write(textToAppend + Environment.NewLine);
        writer.Flush();
        stream.Position = 0;

        appendBlob.AppendBlock(stream);
    }
示例#25
0
        protected override void Write(LogEventInfo logEvent)
        {
            if (_client == null)
            {
                return;
            }

            string containerName = Container.Render(logEvent);
            string blobName      = BlobName.Render(logEvent);

            if (_container == null || _container.Name != containerName)
            {
                _container = _client.GetContainerReference(containerName);
                _blob      = null;
            }

            if (_blob == null || _blob.Name != blobName)
            {
                _blob = _container.GetAppendBlobReference(blobName);

                if (!_blob.Exists())
                {
                    try
                    {
                        _blob.Properties.ContentType = "text/plain";
                        _blob.CreateOrReplace(AccessCondition.GenerateIfNotExistsCondition());
                    }
                    catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict)
                    {
                        // to be expected
                    }
                }
            }

            _blob.AppendText(Layout.Render(logEvent) + "\r\n", Encoding.UTF8);
        }
示例#26
0
        static void Main(string[] args)
        {
            Assembly     execAssembly = Assembly.GetCallingAssembly();
            AssemblyName assemblyName = execAssembly.GetName();

            int traceLevel        = 3;
            int event_count       = 0;
            int total_event_count = 0;
            int updowncount       = 0;

            List <EventData> e = new List <EventData>();
            //EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(connectionString);
            EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(CloudConfigurationManager.GetSetting("EventHubConnectionString"));

            // Parse the connection string and return a reference to the storage account.
            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
                CloudConfigurationManager.GetSetting("StorageConnectionString"));
            // Create the blob client.
            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Retrieve reference to a previously created container.
            CloudBlobContainer container = blobClient.GetContainerReference("syslogs");

            // Retrieve reference to a previously created container for output logs
            CloudBlobContainer containerlogs = blobClient.GetContainerReference("logs");

            // Create the logs container if it does not already exist
            containerlogs.CreateIfNotExists();
            CloudAppendBlob logout = containerlogs.GetAppendBlobReference("ideapoceh04out");

            if (!logout.Exists())
            {
                logout.CreateOrReplace();
            }
            logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Beginning execution {2}", DateTime.UtcNow, assemblyName, Environment.NewLine));

            if (traceLevel == 4)
            {
                // Loop over items within the container and output the length and URI.
                foreach (IListBlobItem item in container.ListBlobs(null, false))
                {
                    if (item.GetType() == typeof(CloudBlockBlob))
                    {
                        CloudBlockBlob blob = (CloudBlockBlob)item;

                        //Console.WriteLine("Block blob of length {0}: {1}", blob.Properties.Length, blob.Uri);
                        logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Block blob of length {2}: {3} {4}", DateTime.UtcNow, assemblyName, blob.Properties.Length, blob.Uri, Environment.NewLine));
                    }
                    else if (item.GetType() == typeof(CloudPageBlob))
                    {
                        CloudPageBlob pageBlob = (CloudPageBlob)item;

                        //Console.WriteLine("Page blob of length {0}: {1}", pageBlob.Properties.Length, pageBlob.Uri);
                        logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Page blob of length {2}: {3} {4}", DateTime.UtcNow, assemblyName, pageBlob.Properties.Length, pageBlob.Uri, Environment.NewLine));
                    }
                    else if (item.GetType() == typeof(CloudBlobDirectory))
                    {
                        CloudBlobDirectory directory = (CloudBlobDirectory)item;

                        //Console.WriteLine("Directory: {0}", directory.Uri);
                        logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t Directory {2}: {3}", DateTime.UtcNow, assemblyName, directory.Uri, Environment.NewLine));
                    }
                    else if (item.GetType() == typeof(CloudAppendBlob))
                    {
                        CloudAppendBlob appendBlob = (CloudAppendBlob)item;

                        //Console.WriteLine("Append blob of length: {0}: {1}", appendBlob.Properties.Length, appendBlob.Uri);
                        logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Append blob of length {2}: {3} {4}", DateTime.UtcNow, assemblyName, appendBlob.Properties.Length, appendBlob.Uri, Environment.NewLine));
                    }
                }
            }
            //Console.ReadLine();

            // Retrieve reference to a blob named "myblob.txt"
            CloudBlockBlob blockBlob2 = container.GetBlockBlobReference("mls_syslog");

            string line;
            //string json_eh_message;

            DateTime startTime = DateTime.UtcNow;

            //using (var memoryStream = new MemoryStream())
            using (var stream = blockBlob2.OpenRead())
            {
                using (StreamReader reader = new StreamReader(stream))
                {
                    //      while ( !reader.EndOfStream)
                    while ((line = reader.ReadLine()) != null)
                    {
                        //   Console.WriteLine(reader.ReadLine());
                        //Console.WriteLine(line);
                        if (traceLevel == 4)
                        {
                            logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} {3}", DateTime.UtcNow, assemblyName, line, Environment.NewLine));
                        }
                        //Console.ReadLine();
                        string[] items = line.Split('\t');
                        syslog   sl    = new syslog();

                        sl.time     = items[0];
                        sl.facility = items[1];
                        sl.address  = items[2];
                        sl.message  = items[3];

                        bool testupdown = sl.message.StartsWith("%LINK-3-UPDOWN:");
                        /* Console.WriteLine("Found UPDOWN? {0}", testupdown); */
                        if (testupdown)
                        {
                            updowncount++;
                            string[] messageItems = sl.message.Split(' ');
                            string   msgInterface = messageItems[2];
                            /* Console.WriteLine("Interface is {0}", msgInterface); */
                            sl.netInterface = msgInterface;
                        }
                        // string json_eh_message = JsonConvert.SerializeObject(sl);
                        json_eh_message = JsonConvert.SerializeObject(sl);

                        if (event_count > 500)
                        {
                            try
                            {
                                //eventHubClient.SendBatchAsync(e);
                                eventHubClient.SendBatch(e);
                                logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Submitting a batch of {2} events to EventHub {3}", DateTime.UtcNow, assemblyName, event_count, Environment.NewLine));
                                //Console.WriteLine("Submitting batch of {0} events", event_count);
                            }
                            catch (Exception exception)
                            {
                                //Console.ForegroundColor = ConsoleColor.Red;
                                //Console.WriteLine("{0} > Exception: {1}", DateTime.Now, exception.Message);
                                //Console.ResetColor();
                                logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Exception raised: {2} {3}", DateTime.UtcNow, assemblyName, exception.Message, Environment.NewLine));
                            }
                            total_event_count = total_event_count + event_count;
                            event_count       = 0;
                            //Console.WriteLine("Total events submitted: {0}", total_event_count);

                            e.Clear();

                            e.Add(new EventData(Encoding.UTF8.GetBytes(json_eh_message)));
                            event_count++;
                            //Console.WriteLine(json_eh_message);
                        }
                        else
                        {
                            e.Add(new EventData(Encoding.UTF8.GetBytes(json_eh_message)));
                            event_count++;
                            //Console.WriteLine(json_eh_message);
                        }
                    }   // EOF -- submit final batch of remaining events
                    try
                    {
                        //eventHubClient.SendBatchAsync(e);
                        eventHubClient.SendBatch(e);
                        //Console.WriteLine("Submitting final batch of {0}", event_count);
                        logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Submitting final batch of {2} events to EventHub {3}", DateTime.UtcNow, assemblyName, event_count, Environment.NewLine));
                    }
                    catch (Exception exception)
                    {
                        //Console.ForegroundColor = ConsoleColor.Red;
                        //Console.WriteLine("{0} > Exception: {1}", DateTime.Now, exception.Message);
                        //Console.ResetColor();
                        logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Exception occurred while submitting batch to EventHub: {2} {3}", DateTime.UtcNow, assemblyName, exception.Message, Environment.NewLine));
                    }
                    total_event_count += event_count;
                }
            }

            DateTime endTime = DateTime.UtcNow;
            var      ts      = new TimeSpan(startTime.Ticks - endTime.Ticks);
            double   delta   = Math.Abs(ts.TotalSeconds);

            logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} link up/down events processed {3}", DateTime.UtcNow, assemblyName, updowncount, Environment.NewLine));
            logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} total events processed {3}", DateTime.UtcNow, assemblyName, total_event_count, Environment.NewLine));
            logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : {2} events per second {3}", DateTime.UtcNow, assemblyName, total_event_count / delta, Environment.NewLine));
            logout.AppendText(String.Format("Time: {0:u} : Assembly: {1} \t : Ending execution {2}", DateTime.UtcNow, assemblyName, Environment.NewLine));

            //Console.WriteLine("Port flap event count: {0}", updowncount);
            //Console.WriteLine("Total events: {0}", total_event_count);
            //Console.ReadLine();
            //using (var memoryStream2 = new BufferedStream(memoryStream))

            /* {
             *   blockBlob2.DownloadToStream(memoryStream);
             *   //while ((text = memoryStream2.Read)
             *   text = System.Text.Encoding.UTF8.GetString(memoryStream.ToArray());
             *   text = memoryStream.
             *   Console.WriteLine(text);
             *   Console.ReadLine();
             * } */
        }
示例#27
0
        static void AzureStorageBlobExamples()
        {
            // Getting configuration from the App.config file and get access to the CloudStorageAccount.
            // Azure access Key should is stored in the App.config
            String storageKeyValue = CloudConfigurationManager.GetSetting("AzureStorageAccount");

            // Just in case to check whether reading the correct value
            Console.WriteLine("Storage Account Key Used" + storageKeyValue);

            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageKeyValue);

            // Getting the Azure Client
            CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();

            // Getting reference for the container
            CloudBlobContainer cloudContainer = cloudBlobClient.GetContainerReference("images");

            // Create the container (folder) if does not exists
            cloudContainer.CreateIfNotExists();

            // Get the reference for the Blob in Azure
            CloudBlockBlob blobCloud = cloudContainer.GetBlockBlobReference("photo.png");


            // Learning 1: Copy a file from local path in to Azure Blob storage

            //Open a local file and upload to the Azure
            using (var fileStream = System.IO.File.OpenRead(@"C:\Users\Administrator\Pictures\photo.jpg"))
            {
                blobCloud.UploadFromStream(fileStream);
                Console.WriteLine("File is uploaded to the cloud");
            }

            // Learning 2: Download a blob from Azure to the local host

            // Just copy a blob from cloud to a local file. Assume the same file which has been uploaded in the previous step
            blobCloud.DownloadToFile(@"C:\Users\Administrator\Pictures\photocopy.jpg", FileMode.CreateNew);

            // Learning 3: List all blob objects URI from a container

            //List all blobs in a container
            var blobs = cloudContainer.ListBlobs();

            foreach (var blob in blobs)
            {
                Console.Write(blob.Uri);
            }

            // Learning 4: List properties of a continer (Folder) and listing metadata of a container

            cloudContainer.FetchAttributes();
            Console.WriteLine(cloudContainer.Properties.LastModified);
            Console.WriteLine(cloudContainer.Properties.ETag);
            var metaData = cloudContainer.Metadata;

            foreach (var metaDataItem in metaData)
            {
                Console.Write("Key " + metaDataItem.Key + " & ");
                Console.WriteLine("Value" + metaDataItem.Value);
            }

            // Learning 5: Setting metaData for a container

            // Method 1
            cloudContainer.Metadata.Add("SampleKey", "SampleValue");
            // Method 2
            cloudContainer.Metadata["SecondSample"] = "Second Value";

            // Dont ask me why FetchAttributes() and SetMetadata() ! why not SetAttributes() or GetMetaData? Microsoft Way!
            cloudContainer.SetMetadata();


            // Learning 6: Setting permission for a container
            BlobContainerPermissions permissions = cloudContainer.GetPermissions();

            Console.WriteLine("Container permission " + permissions.PublicAccess.ToString());
            foreach (var sharedAccessPolicy in permissions.SharedAccessPolicies)
            {
                Console.WriteLine(sharedAccessPolicy.Key.ToString() + " = " + sharedAccessPolicy.Value.ToString());
            }
            // In order to as per the parent container
            permissions.PublicAccess = BlobContainerPublicAccessType.Container;
            // In order to remove the public access
            permissions.PublicAccess = BlobContainerPublicAccessType.Off;
            //Finally set the permission
            cloudContainer.SetPermissions(permissions);



            // Learning 7: Azure copy from one blob to another

            // Create a new Block Blog Reference
            CloudBlockBlob copyBlob = cloudContainer.GetBlockBlobReference("photo-copy.jpg");

            // Copy the original - blobCloud to the copyBlob
            copyBlob.StartCopy(new Uri(blobCloud.Uri.AbsoluteUri));


            // Learning 8: Copy all blobs from one container to another
            CloudBlobContainer sourceContainer = cloudContainer;
            CloudBlobContainer targetContainer = cloudBlobClient.GetContainerReference("newimages");

            targetContainer.CreateIfNotExists();
            foreach (var blob in sourceContainer.ListBlobs())
            {
                var sourceBlob = blob as CloudBlob;
                Console.WriteLine("Source Blob " + sourceBlob.Name);
                CloudBlockBlob newBlob = targetContainer.GetBlockBlobReference(sourceBlob.Name);
                newBlob.StartCopy(new Uri(blob.Uri.AbsoluteUri));
            }


            // Learning 9: Rename the blob
            CloudBlockBlob sourceBlockBlob = cloudContainer.GetBlockBlobReference("photo-copy.jpg");
            CloudBlockBlob targetBlockBlob = cloudContainer.GetBlockBlobReference("copy-photo.jpg");

            targetBlockBlob.StartCopy(new Uri(sourceBlockBlob.Uri.AbsoluteUri));
            while (targetBlockBlob.CopyState.Status == CopyStatus.Pending)
            {
                // Sleep for 3 seconds
                System.Threading.Thread.Sleep(2000);
            }
            sourceBlockBlob.Delete();


            // Learning 10: Appending to a blob
            DateTime           date         = DateTime.Today;
            CloudBlobContainer logContainer = cloudBlobClient.GetContainerReference("logs");
            CloudAppendBlob    logBlog      = logContainer.GetAppendBlobReference(string.Format("{0}{1}", date.ToString("yyyyMMdd"), ".log"));

            logContainer.CreateIfNotExists();

            // If the append blog does not exists, create one
            if (!logBlog.Exists())
            {
                logBlog.CreateOrReplace();
            }

            // AppendText
            logBlog.AppendText(string.Format("{0} : Azure is rocking in the cloud space at ", date.ToString("HH:MM:ss")));
            // Similar to the AppendText, there are
            // logBlog.AppendBlock
            // logBlog.AppendFromByteArray
            // logBlog.AppendFromFile
            // logBlog.AppendFromStream

            // Finally display the content of the log file.
            Console.WriteLine(logBlog.DownloadText());

            // Learning 11: Multiple Chunk of file upload to Azure
            AzureStorageReference.Program.uploadLargeFiles(cloudBlobClient, @"C:\Users\Administrator\Pictures");


            //Learning 12: Upload using Async

            AsyncCallback  callBack  = new AsyncCallback(x => Console.WriteLine("Copy Async Completed"));
            CloudBlockBlob copyAsync = cloudContainer.GetBlockBlobReference("newphoto.png");

            copyAsync.BeginStartCopy(blobCloud, callBack, null);

            Console.WriteLine("Press any key to continue");
            Console.ReadKey();
        }
        public static void Run([BlobTrigger("telemetry-archive/willowtelemetry01/{name}", Connection = "StorageAccount")] Stream myBlob, string name, TraceWriter log)
        {
            DataSet ds = new DataSet(
                new SchemaElement <int>("ObjectID"),
                new SchemaElement <string>("Value"),
                new SchemaElement <string>("ClientID"),
                new SchemaElement <string>("TimeStamp")
                );

            if (myBlob.Length == 508)
            {
                return;
            }                                     // empty dataset

            //log.Info($"C# Blob trigger function Processed blob\n Name:{name} \n Size: {myBlob.Length} Bytes");


            var serializer     = AvroSerializer.CreateGeneric(Schema);
            var jsonSerializer = new JsonSerializer();


            using (var reader = AvroContainer.CreateGenericReader(myBlob))
            {
                using (var streamReader = new SequentialReader <object>(reader))
                {
                    var results = streamReader.Objects;

                    foreach (var item in results)
                    {
                        var body = ((AvroRecord)item)[5];  // ["Body"] is the 5th element

                        var json  = System.Text.Encoding.Default.GetString((byte[])body);
                        var array = (JArray)JsonConvert.DeserializeObject(json);

                        var telemetry = jsonSerializer.Deserialize <Telemetry>(array[0].CreateReader());

                        ds.Add(telemetry.ObjectID, telemetry.Value, telemetry.ClientID, telemetry.TimeStamp.ToString());
                    }
                }
            }

            //log.Info($"{ds.RowCount} Records processed");


            if (ds.RowCount == 0)
            {
                return;
            }

            storageAccount = CloudStorageAccount.Parse(storageAcct);

            CloudBlobClient    blobClient    = storageAccount.CreateCloudBlobClient();
            CloudBlobContainer blobContainer = blobClient.GetContainerReference(parquetContainerName);

            blobContainer.CreateIfNotExists();

            name = name + ".parquet";

            CloudBlob       blob       = blobContainer.GetBlobReference(blobName);
            CloudAppendBlob appendBlob = blobContainer.GetAppendBlobReference(name);

            if (appendBlob.Exists())
            {
                appendBlob.Delete(DeleteSnapshotsOption.IncludeSnapshots);
            }

            log.Info($"===> {ds.RowCount} Records, {name}");


            using (var ms = new MemoryStream())
            {
                using (var writer = new ParquetWriter(ms))
                {
                    writer.Write(ds);
                }

                ms.Seek(0, SeekOrigin.Begin);

                appendBlob.UploadFromStream(ms);
            }
        }