public void ProcessTweetBlob(CloudBlockBlob inputBlob, CloudTable outputAzureTable, string folder) { int count = 0; List <TweetSentimentScore> scores = new List <TweetSentimentScore>(); using (var reader = new CsvReader(new StreamReader(inputBlob.OpenRead()))) { while (reader.Read()) { if (count == 0) { logger.Write(TraceEventType.Information, "First line: [{0}]", string.Join(",", reader.CurrentRecord)); } count++; var tweet = reader.GetField(0); //get the tweet var entity = new TweetSentimentScore() { PartitionKey = "tweetsentimentscore", RowKey = Guid.NewGuid().ToString(), Tweet = tweet, SentimentScore = GetScore(url, email, apikey, tweet) }; scores.Add(entity); outputAzureTable.Execute(TableOperation.InsertOrReplace(entity)); //Do it one row at a time for demo output } } var iter = scores.Count() / 100; for (int i = 0; i <= iter; i++) { var batchOp = new TableBatchOperation(); scores.Skip(100 * i).Take(100).ToList().ForEach(a => { batchOp.Add(TableOperation.InsertOrReplace(a)); }); //outputAzureTable.ExecuteBatch(batchOp); //Removed for demo purposes. } logger.Write(TraceEventType.Information, string.Format(CultureInfo.InvariantCulture, "{0},{1},{2},{3},{4}\n", folder, inputBlob.Name, count, Environment.MachineName, DateTime.UtcNow)); }
public void ProcessTweetBlob(CloudBlockBlob inputBlob, CloudTable outputAzureTable, string folder) { int count = 0; List<TweetSentimentScore> scores = new List<TweetSentimentScore>(); using (var reader = new CsvReader(new StreamReader(inputBlob.OpenRead()))) { while (reader.Read()) { if (count == 0) { logger.Write(TraceEventType.Information, "First line: [{0}]", string.Join(",", reader.CurrentRecord)); } count++; var tweet = reader.GetField(0); //get the tweet var entity = new TweetSentimentScore() { PartitionKey = "tweetsentimentscore", RowKey = Guid.NewGuid().ToString(), Tweet = tweet, SentimentScore = GetScore(url, email, apikey, tweet) }; scores.Add(entity); outputAzureTable.Execute(TableOperation.InsertOrReplace(entity)); //Do it one row at a time for demo output } } var iter = scores.Count() / 100; for (int i = 0; i <= iter; i++) { var batchOp = new TableBatchOperation(); scores.Skip(100 * i).Take(100).ToList().ForEach(a => { batchOp.Add(TableOperation.InsertOrReplace(a)); }); //outputAzureTable.ExecuteBatch(batchOp); //Removed for demo purposes. } logger.Write(TraceEventType.Information, string.Format(CultureInfo.InvariantCulture, "{0},{1},{2},{3},{4}\n", folder, inputBlob.Name, count, Environment.MachineName, DateTime.UtcNow)); }