private string DeleteAzureTableHelper(string TableNameToDelete, int TimeoutSeconds = 30, bool Recreate = false) { try { if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount)) { throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string."); } CosmosTable.CloudTableClient client = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration()); CosmosTable.CloudTable TableToDelete = client.GetTableReference(TableNameToDelete); TableToDelete.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds); bool TableExisted = TableToDelete.DeleteIfExists(); //TableToDelete.Delete(); if (Recreate) { TableToDelete.CreateIfNotExists(); } if (Recreate) { //return String.Format("Table '{0}' did not exist.", TableNameToDelete); return(String.Format("Table '{0}' deleted and recreated.", TableNameToDelete)); } else { return(String.Format("Table '{0}' deleted.", TableNameToDelete)); } } catch (ConnectionException cex) { throw cex; } catch (Exception ex) { throw new DeleteFailedException(String.Format("Table '{0}' delete failed.", TableNameToDelete), ex); } finally { } }
/// <summary> /// Restore file created by BackupAzureTables to the destination table name specified. /// </summary> /// <param name="DestinationTableName">Name of the Azure Table to restore to - may be different than name backed up originally.</param> /// <param name="InFilePathName">Complete file name and path containing the data to be restored.</param> /// <param name="TimeoutSeconds">Set timeout for table client.</param> /// <returns>A string indicating the table restored and record count.</returns> public string RestoreTableFromFile(string DestinationTableName, string InFilePathName, int TimeoutSeconds = 30) { if (String.IsNullOrWhiteSpace(DestinationTableName)) { throw new ParameterSpecException("DestinationTableName is missing."); } if (Path.GetFullPath(InFilePathName) != InFilePathName) { throw new ParameterSpecException(String.Format("Invalid file name/path '{0}' specified.", InFilePathName)); } else { if (!File.Exists(InFilePathName)) { throw new ParameterSpecException(String.Format("File '{0}' does not exist.", InFilePathName)); } } TableSpec footer = null; try { if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount)) { throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string."); } CosmosTable.CloudTableClient client = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration()); CosmosTable.CloudTable TableDest = client.GetTableReference(DestinationTableName); TableDest.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds); TableDest.CreateIfNotExists(); DynamicTableEntityJsonSerializer serializer = new DynamicTableEntityJsonSerializer(); bool BatchWritten = true; string PartitionKey = String.Empty; CosmosTable.TableBatchOperation Batch = new CosmosTable.TableBatchOperation(); int BatchSize = 100; int BatchCount = 0; long TotalRecordCount = 0; using (StreamReader InputFileStream = new StreamReader(InFilePathName)) { string InFileLine = InputFileStream.ReadLine(); while (InFileLine != null) { if (InFileLine.Contains("ProcessingMetaData") && InFileLine.Contains("Header")) { System.Console.WriteLine(String.Format("Header {0}", InFileLine)); } else if (InFileLine.Contains("ProcessingMetaData") && InFileLine.Contains("Footer")) { footer = JsonConvert.DeserializeObject <TableSpec>(InFileLine); System.Console.WriteLine(String.Format("Footer {0}", InFileLine)); } else { CosmosTable.DynamicTableEntity dte2 = serializer.Deserialize(InFileLine); if (String.Empty.Equals(PartitionKey)) { PartitionKey = dte2.PartitionKey; } if (dte2.PartitionKey == PartitionKey) { Batch.InsertOrReplace(dte2); BatchCount++; TotalRecordCount++; BatchWritten = false; } else { try { TableDest.ExecuteBatch(Batch); Batch = new CosmosTable.TableBatchOperation(); PartitionKey = dte2.PartitionKey; Batch.InsertOrReplace(dte2); BatchCount = 1; TotalRecordCount++; BatchWritten = false; } catch (Exception ex) { throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex); } } if (BatchCount >= BatchSize) { try { TableDest.ExecuteBatch(Batch); PartitionKey = String.Empty; Batch = new CosmosTable.TableBatchOperation(); BatchWritten = true; BatchCount = 0; } catch (Exception ex) { throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex); } } } InFileLine = InputFileStream.ReadLine(); } // while (InFileLine != null) //final batch if (!BatchWritten) { try { TableDest.ExecuteBatch(Batch); PartitionKey = String.Empty; } catch (Exception ex) { throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex); } } } // using (StreamReader if (null == footer) { throw new RestoreFailedException(String.Format("Table '{0}' restore failed, no footer record found.", DestinationTableName)); } else if (TotalRecordCount == footer.RecordCount) { //OK, do nothing } else { throw new RestoreFailedException(String.Format("Table '{0}' restore failed, records read {1} does not match expected count {2} in footer record.", DestinationTableName, TotalRecordCount, footer.RecordCount)); } return(String.Format("Restore to table '{0}' Successful; {1} entries.", DestinationTableName, TotalRecordCount)); } catch (ConnectionException cex) { throw cex; } catch (RestoreFailedException rex) { throw rex; } catch (Exception ex) { throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex); } finally { } }
/// <summary> /// Restore file from blob storage by BackupAzureTables to the destination table name specified. /// File will be read directly from blob storage. If the file is compressed, it will be decompressed to blob storage and then read. /// </summary> /// <param name="DestinationTableName">Name of the Azure Table to restore to - may be different than name backed up originally.</param> /// <param name="OriginalTableName">Name of the Azure Table originally backed (required for determining blob directory to use)</param> /// <param name="BlobRoot">Name to use as blob root folder.</param> /// <param name="BlobFileName">Name of the blob file to restore.</param> /// <param name="TimeoutSeconds">Set timeout for table client.</param> /// <returns>A string indicating the table restored and record count.</returns> public string RestoreTableFromBlobDirect(string DestinationTableName, string OriginalTableName, string BlobRoot, string BlobFileName, int TimeoutSeconds = 30) { if (String.IsNullOrWhiteSpace(DestinationTableName)) { throw new ParameterSpecException("DestinationTableName is missing."); } if (String.IsNullOrWhiteSpace(OriginalTableName)) { throw new ParameterSpecException("OriginalTableName is missing."); } if (String.IsNullOrWhiteSpace(BlobFileName)) { throw new ParameterSpecException(String.Format("Invalid BlobFileName '{0}' specified.", BlobFileName)); } bool Decompress = BlobFileName.EndsWith(".7z"); string TempFileName = String.Format("{0}.temp", BlobFileName); if (String.IsNullOrWhiteSpace(BlobRoot)) { throw new ParameterSpecException(String.Format("Invalid BlobRoot '{0}' specified.", BlobRoot)); } try { if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount)) { throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string."); } if (!AZStorage.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureBlobConnectionSpec).Password, out AZStorage.CloudStorageAccount StorageAccountAZ)) { throw new ConnectionException("Can not connect to CloudStorage Account. Verify connection string."); } AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ); var container = ClientBlob.GetContainerReference(BlobRoot); container.CreateIfNotExists(); AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobRoot.ToLower() + "-table-" + OriginalTableName.ToLower()); // If file is compressed, Decompress to a temp file in the blob if (Decompress) { AZBlob.CloudBlockBlob BlobBlockTemp = directory.GetBlockBlobReference(TempFileName); AZBlob.CloudBlockBlob BlobBlockRead = directory.GetBlockBlobReference(BlobFileName); using (AZBlob.CloudBlobStream decompressedStream = BlobBlockTemp.OpenWrite()) { using (Stream readstream = BlobBlockRead.OpenRead()) { using (var zip = new GZipStream(readstream, CompressionMode.Decompress, true)) { zip.CopyTo(decompressedStream); } } } BlobFileName = TempFileName; } AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(BlobFileName); CosmosTable.CloudTableClient client = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration()); CosmosTable.CloudTable TableDest = client.GetTableReference(DestinationTableName); TableDest.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds); TableDest.CreateIfNotExists(); using (Stream BlobStream = BlobBlock.OpenRead()) { using (StreamReader InputFileStream = new StreamReader(BlobStream)) { string result = RestoreFromStream(InputFileStream, TableDest, DestinationTableName); if (Decompress) { AZBlob.CloudBlockBlob BlobBlockTemp = directory.GetBlockBlobReference(TempFileName); BlobBlockTemp.DeleteIfExists(); } return(result); } } } catch (ConnectionException cex) { throw cex; } catch (RestoreFailedException rex) { throw rex; } catch (Exception ex) { throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex); } finally { } } // RestoreTableFromBlobDirect
/// <summary> /// This method will copy entries from SourceTableName to DestinationTableName. The destination table is NOT deleted first. /// </summary> /// <param name="SourceTableName">Name of table to copy from.</param> /// <param name="DestinationTableName">Name of table to copy to.</param> /// <param name="TimeoutSeconds">Set timeout for table client.</param> /// <param name="filters">A list of Filter objects to be applied to table values extracted.</param> /// <returns>A string indicating the result of the operation.</returns> public string CopyTableToTable(string SourceTableName, string DestinationTableName, int TimeoutSeconds = 30, List <Filter> filters = default(List <Filter>)) { if (String.IsNullOrWhiteSpace(SourceTableName)) { throw new ParameterSpecException("SourceTableName is missing."); } if (String.IsNullOrWhiteSpace(DestinationTableName)) { throw new ParameterSpecException("DestinationTableName is missing."); } if (!Filter.AreFiltersValid(filters)) { throw new ParameterSpecException(String.Format("One or more of the supplied filter criteria is invalid.")); } try { if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureSourceTableConnection).Password, out CosmosTable.CloudStorageAccount StorageAccountSource)) { throw new ConnectionException("Can not connect to CloudStorage Account for Source Table. Verify connection string."); } CosmosTable.CloudTableClient clientSource = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccountSource, new CosmosTable.TableClientConfiguration()); CosmosTable.CloudTable TableSource = clientSource.GetTableReference(SourceTableName); TableSource.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds); if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureDestinationTableConnection).Password, out CosmosTable.CloudStorageAccount StorageAccountDest)) { throw new ConnectionException("Can not connect to CloudStorage Account for Destination Table. Verify connection string."); } CosmosTable.CloudTableClient clientDestnation = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccountDest, new CosmosTable.TableClientConfiguration()); CosmosTable.CloudTable TableDest = clientDestnation.GetTableReference(DestinationTableName); TableDest.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds); TableDest.CreateIfNotExists(); bool BatchWritten = true; string PartitionKey = String.Empty; CosmosTable.TableBatchOperation Batch = new CosmosTable.TableBatchOperation(); int BatchSize = 100; int BatchCount = 0; long TotalRecordCountIn = 0; long TotalRecordCountOut = 0; CosmosTable.TableContinuationToken token = null; do { CosmosTable.TableQuery <CosmosTable.DynamicTableEntity> tq; if (default(List <Filter>) == filters) { tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>(); } else { tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>().Where(Filter.BuildFilterSpec(filters)); } var queryResult = TableSource.ExecuteQuerySegmented(tq, token); foreach (CosmosTable.DynamicTableEntity dte in queryResult.Results) { TotalRecordCountIn++; if (String.Empty.Equals(PartitionKey)) { PartitionKey = dte.PartitionKey; } if (dte.PartitionKey == PartitionKey) { Batch.InsertOrReplace(dte); BatchCount++; TotalRecordCountOut++; BatchWritten = false; } else { try { TableDest.ExecuteBatch(Batch); Batch = new CosmosTable.TableBatchOperation(); PartitionKey = dte.PartitionKey; Batch.InsertOrReplace(dte); BatchCount = 1; TotalRecordCountOut++; BatchWritten = false; } catch (Exception ex) { throw new CopyFailedException(String.Format("Table '{0}' copy to '{1}' failed.", SourceTableName, DestinationTableName), ex); } } if (BatchCount >= BatchSize) { try { TableDest.ExecuteBatch(Batch); PartitionKey = String.Empty; Batch = new CosmosTable.TableBatchOperation(); BatchWritten = true; BatchCount = 0; } catch (Exception ex) { throw new CopyFailedException(String.Format("Table '{0}' copy to '{1}' failed.", SourceTableName, DestinationTableName), ex); } } } token = queryResult.ContinuationToken; } while (token != null); if (!BatchWritten) { try { TableDest.ExecuteBatch(Batch); PartitionKey = String.Empty; } catch (Exception ex) { throw new CopyFailedException(String.Format("Table '{0}' copy to '{1}' failed.", SourceTableName, DestinationTableName), ex); } } return(String.Format("Table '{0}' copied to table '{1}', total records {2}.", SourceTableName, DestinationTableName, TotalRecordCountIn)); } catch (ConnectionException cex) { throw cex; } catch (Exception ex) { throw new CopyFailedException(String.Format("Table '{0}' copy to '{1}' failed.", SourceTableName, DestinationTableName), ex); } finally { } }