/// <summary>
/// Upgrade the store version number in azure table storage
/// </summary>
/// <param name="tableStoreManager">table store manager</param>
/// <param name="azureConnectionString">azure connection string</param>
/// <returns>true if upgrade is successful</returns>
private static async Task <bool> UpgradeStoreVersionAzureTables(CTStoreManager tableStoreManager, string azureConnectionString)
{
// Get azure table storage with the give connection string
AzureTableStorage azureTableStorage = new AzureTableStorage(azureConnectionString);
// Get CTStore using the azure table storage
CTStore store = new CTStore(azureTableStorage, null);
ObjectTable versionTable = Table.GetObjectTable(
ContainerIdentifier.ServiceConfig.ToString(),
tableStoreManager.ServiceConfigContainerInitials,
tableStoreManager.StoreVersionTableName,
tableStoreManager.StoreVersionTableInitials,
StorageMode.PersistentOnly);
var currentTableVersion = await azureTableStorage.QueryObjectAsync <StoreVersionEntity>(versionTable, tableStoreManager.StoreVersionKey, tableStoreManager.StoreVersionKey);
// if version in store does not match oldVersion, then refuse to upgrade
if (currentTableVersion.Version != oldVersion)
{
Console.WriteLine("Version mismatch in Azure table storage: original version in store is {0}", currentTableVersion.Version);
return(false);
}
currentTableVersion.Version = newVersion;
var operation = Operation.Replace(versionTable, tableStoreManager.StoreVersionKey, tableStoreManager.StoreVersionKey, currentTableVersion);
// perform the insert on Azure table storage
await store.ExecuteOperationAsync(operation, ConsistencyMode.Strong);
return(true);
}
/// <summary>
/// Upgrade the store version number in persistent redis
/// </summary>
/// <param name="ctStoreMananger">ctstore manager</param>
/// <param name="persistentCacheConnectionString">connection string for persistent cache</param>
/// <returns>true if upgrade is successful</returns>
private static async Task <bool> UpgradeStoreVersionRedis(CTStoreManager ctStoreMananger, string persistentCacheConnectionString)
{
RedisCache redisCachePersistent = new RedisCache(persistentCacheConnectionString);
CTStore persistentCacheStore = new CTStore(null, redisCachePersistent);
ObjectTable versionTable = Table.GetObjectTable(
ContainerIdentifier.ServiceConfig.ToString(),
ctStoreMananger.ServiceConfigContainerInitials,
ctStoreMananger.StoreVersionTableName,
ctStoreMananger.StoreVersionTableInitials,
StorageMode.CacheOnly);
var persistentCacheVersion = await persistentCacheStore.QueryObjectAsync <StoreVersionEntity>(versionTable, ctStoreMananger.StoreVersionKey, ctStoreMananger.StoreVersionKey);
// if version in store does not match oldVersion, then refuse to upgrade
if (persistentCacheVersion.Version != oldVersion)
{
Console.WriteLine("Version mismatch in persistent Redis: original version in store is {0}", persistentCacheVersion.Version);
return(false);
}
persistentCacheVersion.Version = newVersion;
var operation = Operation.Replace(versionTable, ctStoreMananger.StoreVersionKey, ctStoreMananger.StoreVersionKey, persistentCacheVersion);
// perform the insert operation on persistent redis
await persistentCacheStore.ExecuteOperationAsync(operation, ConsistencyMode.Strong);
return(true);
}
/// <summary>
/// upgrade the store version number in Azure table storage and persistent redis
/// </summary>
/// <param name="tableStoreManager">ctstore manager</param>
/// <param name="azureConnectionString">azure connection string</param>
/// <param name="persistentCacheConnectionString">persistent redis connection string</param>
/// <returns>upgrade version task</returns>
private static async Task UpgradeStoreVersion(CTStoreManager tableStoreManager, string azureConnectionString, string persistentCacheConnectionString)
{
bool result = await UpgradeStoreVersionAzureTables(tableStoreManager, azureConnectionString);
bool result2 = await UpgradeStoreVersionRedis(tableStoreManager, persistentCacheConnectionString);
if (!result)
{
Console.WriteLine("Version upgrade failed for azure table storage");
}
if (!result2)
{
Console.WriteLine("Version upgrade failed for persistent redis storage");
}
if (result && result2)
{
Console.WriteLine("Store version upgrade succeeded.");
}
return;
}
/// <summary>
/// Provision the contents of the redis caches
/// </summary>
/// <param name="persistentCacheConnectionString">connection string for persistent cache</param>
/// <param name="tableStoreManager">table store manager</param>
/// <returns>provisioning task</returns>
private static async Task ProvisionRedisCaches(string persistentCacheConnectionString, CTStoreManager tableStoreManager)
{
Console.WriteLine("Provisioning Redis Caches...");
// insert the store version number into table storage
var versionEntity = new StoreVersionEntity {
Version = tableStoreManager.StoreVersionString
};
ObjectTable versionTable = Table.GetObjectTable(
ContainerIdentifier.ServiceConfig.ToString(),
tableStoreManager.ServiceConfigContainerInitials,
tableStoreManager.StoreVersionTableName,
tableStoreManager.StoreVersionTableInitials,
StorageMode.CacheOnly);
var operation = Operation.Insert(versionTable, tableStoreManager.StoreVersionKey, tableStoreManager.StoreVersionKey, versionEntity);
RedisCache redisCachePersistent = new RedisCache(persistentCacheConnectionString);
CTStore persistentCache = new CTStore(null, redisCachePersistent);
// perform the insert operation on persistent redis
// note that we only insert the version number into persistent redis,
// because with volatile redis each time the cache restarts the version number will be lost
await persistentCache.ExecuteOperationAsync(operation, ConsistencyMode.Strong);
}
/// <summary>
/// Provision azure table storage with tables
/// </summary>
/// <param name="tableStoreManager">table store manager</param>
/// <param name="azureConnectionString">Azure connection string</param>
/// <returns>provision task</returns>
private static async Task ProvisionAzureStorageTables(CTStoreManager tableStoreManager, string azureConnectionString)
{
// Creates Social Plus containers/tables defined in ContainerIdentifier.cs/TableIdentifier.cs
// Containers map to tables in Azure table storage (container names map to table names)
// We create them (if not exists) through the CTStore interface
Console.WriteLine("Creating all tables in Azure Table Store...");
// Get azure table storage with the give connection string
AzureTableStorage azureTableStorage = new AzureTableStorage(azureConnectionString);
// Get CTStore using the azure table storage
CTStore store = new CTStore(azureTableStorage, null);
// Enumerate all the containers defined
foreach (ContainerIdentifier containerIdentifier in Enum.GetValues(typeof(ContainerIdentifier)))
{
if (!ContainerTableDescriptorProvider.Containers.ContainsKey(containerIdentifier))
{
Console.WriteLine(" " + containerIdentifier.ToString() + " - Descriptor not found");
continue;
}
// in Azure, table deletion can potentially take a long time.
// this may lead to conflict exceptions if you delete a table and then attempt to
// recreate it. Below, we retry once every 30 seconds for up to 5 minutes if needed.
// wait up to 5 minutes before giving up on table creation
int attempts = 0;
int maxAttempts = 10;
while (true)
{
try
{
attempts++;
// create the container
await store.CreateContainerAsync(containerIdentifier.ToString());
// if we reach here, the create was successful
break;
}
catch (ConflictException e)
{
if (attempts < maxAttempts)
{
// sleep for 30 seconds before trying
await Task.Delay(30 * 1000);
}
else
{
// give up after reaching maxAttempts
throw e;
}
}
}
Console.WriteLine(" " + containerIdentifier.ToString() + " - Table Container Provisioned");
}
// insert the store version number into table storage
var versionEntity = new StoreVersionEntity {
Version = tableStoreManager.StoreVersionString
};
// the StoreVersion container has no descriptor, so we need to create it
await store.CreateContainerAsync(ContainerIdentifier.ServiceConfig.ToString());
Console.WriteLine(" " + ContainerIdentifier.ServiceConfig.ToString() + " - Table Container Provisioned");
ObjectTable versionTable = Table.GetObjectTable(
ContainerIdentifier.ServiceConfig.ToString(),
tableStoreManager.ServiceConfigContainerInitials,
tableStoreManager.StoreVersionTableName,
tableStoreManager.StoreVersionTableInitials,
StorageMode.PersistentOnly);
var operation = Operation.Insert(versionTable, tableStoreManager.StoreVersionKey, tableStoreManager.StoreVersionKey, versionEntity);
// perform the insert on Azure table storage
await store.ExecuteOperationAsync(operation, ConsistencyMode.Strong);
Console.WriteLine(" SocialPlusStoreVersion number provisioned");
}
/// <summary>
/// Initialization routine
/// </summary>
private void Initialize()
{
// load the environment configuration file from UtilsInternal
var sr = new FileSettingsReader(ConfigurationManager.AppSettings["ConfigRelativePath"] + Path.DirectorySeparatorChar + environmentName + ".config");
var certThumbprint = sr.ReadValue(SocialPlusCertThumbprint);
var clientID = sr.ReadValue(EmbeddedSocialClientIdSetting);
var storeLocation = StoreLocation.CurrentUser;
var vaultUrl = sr.ReadValue(SocialPlusVaultUrlSetting);
ICertificateHelper cert = new CertificateHelper(certThumbprint, clientID, storeLocation);
IKeyVaultClient client = new AzureKeyVaultClient(cert);
var log = new Log(LogDestination.Console, Log.DefaultCategoryName);
var kv = new KV(log, clientID, vaultUrl, certThumbprint, storeLocation, client);
var kvReader = new KVSettingsReader(sr, kv);
IConnectionStringProvider connectionStringProvider = new ConnectionStringProvider(kvReader);
int queueBatchIntervalMs = int.Parse(sr.ReadValue(ServiceBusBatchIntervalMsSetting));
// Lots of things need to be created to create an appsManager.
ICTStoreManager tableStoreManager = new CTStoreManager(connectionStringProvider);
bool tableInit = false;
Exception exception = null;
try
{
// use Task.Run to ensure that the async Initialize routine runs on a threadpool thread
Task <bool> task = Task <bool> .Run(() => tableStoreManager.Initialize());
// task.Result blocks until the result is ready
tableInit = task.Result;
}
catch (Exception e)
{
exception = e;
}
if (tableInit == false)
{
string errorMessage = "CTstore version number does not match the expected version number." + Environment.NewLine +
"If your tables are empty, then you probably forgot to provision storage." + Environment.NewLine +
"If not, then you need to convert the data format and update the storage version number.";
Console.WriteLine(errorMessage);
if (exception != null)
{
Console.WriteLine("Exception message:" + exception.Message);
}
Environment.Exit(0);
}
ICBStoreManager blobStoreManager = new CBStoreManager(connectionStringProvider);
AppsStore appsStore = new AppsStore(tableStoreManager);
UsersStore usersStore = new UsersStore(tableStoreManager);
ViewsManager viewsManager = new ViewsManager(
log,
appsStore,
usersStore,
new UserRelationshipsStore(tableStoreManager),
new TopicsStore(tableStoreManager),
new TopicRelationshipsStore(tableStoreManager),
new CommentsStore(tableStoreManager),
new RepliesStore(tableStoreManager),
new LikesStore(tableStoreManager),
new PinsStore(tableStoreManager),
new BlobsStore(blobStoreManager));
PushNotificationsManager pushManager = new PushNotificationsManager(log, new PushRegistrationsStore(tableStoreManager), appsStore, viewsManager, connectionStringProvider);
this.appsManager = new AppsManager(appsStore, pushManager);
SearchManager searchManager = new SearchManager(log, connectionStringProvider);
PopularUsersManager popularUsersManager = new PopularUsersManager(usersStore);
QueueManager queueManager = new QueueManager(connectionStringProvider, queueBatchIntervalMs);
SearchQueue searchQueue = new SearchQueue(queueManager);
this.usersManager = new UsersManager(usersStore, pushManager, popularUsersManager, searchQueue);
}
/// <summary>
/// Async version of the Main program
/// </summary>
/// <param name="args">command line args</param>
/// <returns>a task</returns>
public static async Task AsyncMain(string[] args)
{
ParseArgs(args);
var sr = new FileSettingsReader(ConfigurationManager.AppSettings["ConfigRelativePath"] + Path.DirectorySeparatorChar + environmentName + ".config");
var certThumbprint = sr.ReadValue(SocialPlusCertThumbprint);
var clientID = sr.ReadValue(EmbeddedSocialClientIdSetting);
var storeLocation = StoreLocation.CurrentUser;
var vaultUrl = sr.ReadValue(SocialPlusVaultUrlSetting);
ICertificateHelper cert = new CertificateHelper(certThumbprint, clientID, storeLocation);
IKeyVaultClient client = new AzureKeyVaultClient(cert);
log = new Log(LogDestination.Console, Log.DefaultCategoryName);
kv = new KV(log, clientID, vaultUrl, certThumbprint, storeLocation, client);
var kvReader = new KVSettingsReader(sr, kv);
// Create a null connection string provider needed for blobStoreManager and tableStoreManager
NullConnectionStringProvider connectionStringProvider = new NullConnectionStringProvider();
if (doUpgradeStoreVersion)
{
if (!forceOperation)
{
Console.WriteLine("You must specify the -Force option when using -UpgradeStoreVersion");
Console.WriteLine("The -UpgradeStoreVersion option is only intended to be used by our version upgrade scripts");
Console.WriteLine("If you are trying to use this by hand, you're probably doing something wrong.");
return;
}
CTStoreManager tableStoreManager = new CTStoreManager(connectionStringProvider);
string redisPersistentConnectionString = await kvReader.ReadValueAsync("PersistentRedisConnectionString");
string azureTableStorageConnectionString = await kvReader.ReadValueAsync("AzureStorageConnectionString");
await UpgradeStoreVersion(tableStoreManager, azureTableStorageConnectionString, redisPersistentConnectionString);
return;
}
if (doClean)
{
DisplayWarning();
}
// display current configuration
await ValidateAndPrintConfiguration(environmentName, kvReader);
if (forceOperation == false)
{
// get user approval
Console.Write("Are you sure you want to proceed? [y/n] : ");
ConsoleKeyInfo keyInfo = Console.ReadKey(false);
if (keyInfo.KeyChar != 'y')
{
return;
}
}
// Mr Clean!!
Console.WriteLine();
Console.WriteLine();
if (doAll || doSearch)
{
string searchServiceName = await kvReader.ReadValueAsync("SearchServiceName");
string searchServiceAdminKey = await kvReader.ReadValueAsync("SearchServiceAdminKey");
if (doClean)
{
// delete search indices
await DeleteSearch(searchServiceName, searchServiceAdminKey);
}
if (doCreate)
{
// create search indices
await ProvisionSearch(searchServiceName, searchServiceAdminKey);
}
}
if (doAll || doQueues)
{
string serviceBusConnectionString = await kvReader.ReadValueAsync("ServiceBusConnectionString");
if (doClean)
{
// Delete queues
await DeleteServiceBusQueues(serviceBusConnectionString);
}
if (doCreate)
{
// Create queues
await ProvisionServiceBusQueues(serviceBusConnectionString);
}
}
if (doAll || doTables)
{
CTStoreManager tableStoreManager = new CTStoreManager(connectionStringProvider);
string azureTableStorageConnectionString = await kvReader.ReadValueAsync("AzureStorageConnectionString");
if (doClean)
{
// Delete tables
await DeleteAzureTables(azureTableStorageConnectionString);
}
if (doCreate)
{
await ProvisionAzureStorageTables(tableStoreManager, azureTableStorageConnectionString);
}
}
if (doAll || doBlobs)
{
CBStoreManager blobStoreManager = new CBStoreManager(connectionStringProvider);
string azureBlobStorageConnectionString = await kvReader.ReadValueAsync("AzureBlobStorageConnectionString");
if (doClean)
{
// Delete blobs
await DeleteAzureBlobs(blobStoreManager, azureBlobStorageConnectionString);
}
if (doCreate)
{
await ProvisionAzureStorageBlobs(blobStoreManager, azureBlobStorageConnectionString);
}
}
if (doAll || doRedis)
{
if (doClean)
{
// Delete redis cache
string redisVolatileConnectionString = await kvReader.ReadValueAsync("VolatileRedisConnectionString") + ", allowAdmin=1";
string redisPersistentConnectionString = await kvReader.ReadValueAsync("PersistentRedisConnectionString") + ", allowAdmin=1";
DeleteRedisCaches(redisVolatileConnectionString, redisPersistentConnectionString);
}
if (doCreate)
{
string redisPersistentConnectionString = await kvReader.ReadValueAsync("PersistentRedisConnectionString");
CTStoreManager tableStoreManager = new CTStoreManager(connectionStringProvider);
await ProvisionRedisCaches(redisPersistentConnectionString, tableStoreManager);
}
}
if (doAll || doLogs)
{
CBStoreManager blobStoreManager = new CBStoreManager(connectionStringProvider);
string azureBlobStorageConnectionString = await kvReader.ReadValueAsync("AzureBlobStorageConnectionString");
if (doClean)
{
// Delete logs
await DeleteAzureLogs(azureBlobStorageConnectionString);
}
if (doCreate)
{
// No need to create the Azure logs (aka WAD* tables). Azure Diagnostics creates them automatically.
}
}
// bye
Console.WriteLine();
Console.WriteLine("All done! Bye!");
Console.WriteLine();
}
