private void UploadStockDataToAzure(List<StockEntity> stockEntities, string azureTableStockCode)
{
Console.WriteLine("Uploading data to azure table...");
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudTable table = GetAzureTable();
DateTime startingDate = DateTime.FromFileTimeUtc(0);
TableOperation retrieveStockEntityStatus = TableOperation.Retrieve<StockEntityStatus>("status-" + azureTableStockCode, "status");
var stockEntityStatus = (StockEntityStatus)table.Execute(retrieveStockEntityStatus).Result;
if (stockEntityStatus != null)
{
Console.WriteLine("Latest data from azure table is on {0}", stockEntityStatus.LastestRawDataDate.ToString("yyyy-MM-dd"));
startingDate = stockEntityStatus.LastestRawDataDate;
}
var stockEntitiesToUpload = stockEntities.Where(entity => entity.Date > startingDate);
long totalCountToUpload = stockEntitiesToUpload.LongCount();
long currentCountUploaded = 0;
DateTime lastestRawDataDate = DateTime.FromFileTimeUtc(0);
TableBatchOperation tableBatchOperation = new TableBatchOperation();
foreach (var stockEntity in stockEntitiesToUpload)
{
if (stockEntity.Date >= lastestRawDataDate) lastestRawDataDate = stockEntity.Date;
tableBatchOperation.Add(TableOperation.InsertOrMerge(stockEntity));
if (tableBatchOperation.Count == 100)
{
table.ExecuteBatch(tableBatchOperation);
currentCountUploaded += 100;
Console.WriteLine("{0}/{1} entities uploaded...", currentCountUploaded, totalCountToUpload);
tableBatchOperation.Clear();
}
}
if (tableBatchOperation.Count > 0)
{
table.ExecuteBatch(tableBatchOperation);
currentCountUploaded += tableBatchOperation.Count;
Console.WriteLine("{0}/{1} entities uploaded...", currentCountUploaded, totalCountToUpload);
}
if (stockEntitiesToUpload.LongCount() > 0)
{
if (stockEntityStatus == null)
{
stockEntityStatus = new StockEntityStatus(azureTableStockCode);
}
stockEntityStatus.LastestRawDataDate = lastestRawDataDate;
Console.WriteLine("Set latest raw data date to {0}", stockEntityStatus.LastestRawDataDate.ToString("yyyy-MM-dd"));
table.Execute(TableOperation.InsertOrMerge(stockEntityStatus));
}
}
private void CalculateMADataToAzure(CloudTable table, string azureTableStockCode, int MA)
{
DateTime startingDate = DateTime.FromFileTimeUtc(0);
TableOperation retrieveStockEntityStatus = TableOperation.Retrieve<StockEntityStatus>("status-" + azureTableStockCode, "status");
var stockEntityStatus = (StockEntityStatus)table.Execute(retrieveStockEntityStatus).Result;
if (stockEntityStatus != null)
{
startingDate = stockEntityStatus.GetLatestMAStartDate(MA);
Console.WriteLine("Latest starting date for MA{0} is on {1}", MA, startingDate.ToString("yyyy-MM-dd"));
}
string pkFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, azureTableStockCode);
string rkLowerFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, startingDate.ToString("yyyy-MM-dd"));
string combinedFilter = TableQuery.CombineFilters(pkFilter, TableOperators.And, rkLowerFilter);
TableQuery<StockEntity> query = new TableQuery<StockEntity>().Where(combinedFilter);
var sortedStockEntities = table.ExecuteQuery<StockEntity>(query).OrderBy(entity => entity.Date).ToList();
if (sortedStockEntities.LongCount() >= MA)
{
long totalCountToUpload = sortedStockEntities.LongCount();
long currentCountUploaded = 0;
Queue<double> maData = new Queue<double>();
TableBatchOperation tableBatchOperation = new TableBatchOperation();
foreach (var stockEntity in sortedStockEntities)
{
maData.Enqueue(stockEntity.Close);
if (maData.Count == MA)
{
double sum = 0;
foreach (var data in maData)
{
sum += data;
}
stockEntity.SetMA(MA, sum / MA);
tableBatchOperation.Add(TableOperation.InsertOrMerge(stockEntity));
maData.Dequeue();
}
if (tableBatchOperation.Count == 100)
{
table.ExecuteBatch(tableBatchOperation);
currentCountUploaded += 100;
Console.WriteLine("{0}/{1} entities uploaded...", currentCountUploaded, totalCountToUpload);
tableBatchOperation.Clear();
}
}
if (tableBatchOperation.Count > 0)
{
table.ExecuteBatch(tableBatchOperation);
currentCountUploaded += tableBatchOperation.Count;
Console.WriteLine("{0}/{1} entities uploaded...", currentCountUploaded, totalCountToUpload);
}
sortedStockEntities.Reverse();
if (sortedStockEntities == null)
{
stockEntityStatus = new StockEntityStatus(azureTableStockCode);
}
stockEntityStatus.SetLatestMAStartDate(MA, sortedStockEntities[MA - 2].Date);
table.Execute(TableOperation.InsertOrMerge(stockEntityStatus));
}
}