示例#1
0
 public void Log <TState>(LogLevel logLevel, EventId eventId, TState state,
                          Exception exception, Func <TState, Exception, string> formatter)
 {
     Console.WriteLine("Custom Logger Worked!!!");
     Task.Run(() =>
     {
         //TODO: Add here your customized logic to handle logging
         //e.g. Store exceptions into a database
         Microsoft.Azure.Cosmos.Table.CloudTableClient cloudTableClient =
             new Microsoft.Azure.Cosmos.Table.CloudTableClient
                 (new Uri(this.CustomLoggerConfiguration.AzureStorageUrl),
                 new Microsoft.Azure.Cosmos.Table.StorageCredentials(
                     keyValue: this.CustomLoggerConfiguration.AzureStorageKey,
                     accountName: this.CustomLoggerConfiguration.AzureStorageAccountName
                     ));
         LogData logData = new LogData()
         {
             CreatedTime     = DateTimeOffset.UtcNow,
             ExceptionInfo   = exception,
             ExceptionString = exception.ToString(),
             PartitionKey    = Guid.NewGuid().ToString(),
             Message         = exception.Message,
             Timestamp       = DateTimeOffset.UtcNow,
             RowKey          = Guid.NewGuid().ToString()
         };
         var tableReference  = cloudTableClient.GetTableReference("ExceptionsLog");
         var insertOperation = TableOperation.InsertOrMerge(logData);
         var result          = tableReference.Execute(insertOperation);
     });
 }
        /// <summary>
        /// Return a list of table names for the given azure connection.
        /// </summary>
        /// <param name="AzureTableConnection"></param>
        /// <returns></returns>
        public static List <string> GetTableNames(SecureString AzureTableConnection)
        {
            List <string> TableNames = new List <string>();

            if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnection).Password, out CosmosTable.CloudStorageAccount StorageAccount))
            {
                throw new ConnectionException("Can not connect to CloudStorage Account.  Verify connection string.");
            }

            CosmosTable.CloudTableClient client = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration());
            var result = client.ListTables();

            foreach (var Table in result)
            {
                TableNames.Add(Table.Name);
            }
            return(TableNames);
        }
示例#3
0
        private string DeleteAzureTableHelper(string TableNameToDelete, int TimeoutSeconds = 30, bool Recreate = false)
        {
            try
            {
                if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount))
                {
                    throw new ConnectionException("Can not connect to CloudStorage Account.  Verify connection string.");
                }

                CosmosTable.CloudTableClient client        = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration());
                CosmosTable.CloudTable       TableToDelete = client.GetTableReference(TableNameToDelete);
                TableToDelete.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds);

                bool TableExisted = TableToDelete.DeleteIfExists();
                //TableToDelete.Delete();

                if (Recreate)
                {
                    TableToDelete.CreateIfNotExists();
                }

                if (Recreate)
                {
                    //return String.Format("Table '{0}' did not exist.", TableNameToDelete);
                    return(String.Format("Table '{0}' deleted and recreated.", TableNameToDelete));
                }
                else
                {
                    return(String.Format("Table '{0}' deleted.", TableNameToDelete));
                }
            }
            catch (ConnectionException cex)
            {
                throw cex;
            }
            catch (Exception ex)
            {
                throw new DeleteFailedException(String.Format("Table '{0}' delete failed.", TableNameToDelete), ex);
            }
            finally
            {
            }
        }
示例#4
0
        /// <summary>
        /// This method will delete rows for the table name specified corresponding to the filter criteria.  If no filter criteria is specified, all rows will be deleted.
        /// </summary>
        /// <param name="TableNameToDelete">Name of table from entries will be deleted.</param>
        /// <param name="TimeoutSeconds">Set timeout for table client.</param>
        /// <param name="filters">A list of Filter objects to be applied to table rows to delete.</param>
        /// <returns>A string indicating the result of the operation.</returns>
        public string DeleteAzureTableRows(string TableNameToDelete, int TimeoutSeconds = 30, List <Filter> filters = default(List <Filter>))
        {
            if (String.IsNullOrWhiteSpace(TableNameToDelete))
            {
                throw new ParameterSpecException("TableNameToDelete rows from is missing.");
            }

            if (!Filter.AreFiltersValid(filters))
            {
                throw new ParameterSpecException(String.Format("One or more of the supplied filter criteria is invalid."));
            }

            try
            {
                if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccountSource))
                {
                    throw new ConnectionException("Can not connect to CloudStorage Account for Source Table.  Verify connection string.");
                }

                CosmosTable.CloudTableClient clientSource = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccountSource, new CosmosTable.TableClientConfiguration());
                CosmosTable.CloudTable       TableSource  = clientSource.GetTableReference(TableNameToDelete);
                TableSource.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds);

                //CosmosTable.CloudTable TableDest = TableSource;

                bool   BatchWritten = true;
                string PartitionKey = String.Empty;
                CosmosTable.TableBatchOperation Batch = new CosmosTable.TableBatchOperation();
                int  BatchSize           = 100;
                int  BatchCount          = 0;
                long TotalRecordCountIn  = 0;
                long TotalRecordCountOut = 0;
                CosmosTable.TableContinuationToken token = null;

                do
                {
                    CosmosTable.TableQuery <CosmosTable.DynamicTableEntity> tq;
                    if (default(List <Filter>) == filters)
                    {
                        tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>();
                    }
                    else
                    {
                        tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>().Where(Filter.BuildFilterSpec(filters));
                    }
                    var queryResult = TableSource.ExecuteQuerySegmented(tq, token);

                    foreach (CosmosTable.DynamicTableEntity dte in queryResult.Results)
                    {
                        TotalRecordCountIn++;
                        if (String.Empty.Equals(PartitionKey))
                        {
                            PartitionKey = dte.PartitionKey;
                        }
                        if (dte.PartitionKey == PartitionKey)
                        {
                            Batch.Delete(dte);
                            //Batch.InsertOrReplace(dte);
                            BatchCount++;
                            TotalRecordCountOut++;
                            BatchWritten = false;
                        }
                        else
                        {
                            try
                            {
                                TableSource.ExecuteBatch(Batch);
                                Batch        = new CosmosTable.TableBatchOperation();
                                PartitionKey = dte.PartitionKey;
                                Batch.Delete(dte);
                                //Batch.InsertOrReplace(dte);
                                BatchCount = 1;
                                TotalRecordCountOut++;
                                BatchWritten = false;
                            }
                            catch (Exception ex)
                            {
                                throw new DeleteFailedException(String.Format("Table '{0}' row delete failed.", TableNameToDelete), ex);
                            }
                        }
                        if (BatchCount >= BatchSize)
                        {
                            try
                            {
                                TableSource.ExecuteBatch(Batch);
                                PartitionKey = String.Empty;
                                Batch        = new CosmosTable.TableBatchOperation();
                                BatchWritten = true;
                                BatchCount   = 0;
                            }
                            catch (Exception ex)
                            {
                                throw new DeleteFailedException(String.Format("Table '{0}' row deletion failed.", TableNameToDelete), ex);
                            }
                        }
                    }
                    token = queryResult.ContinuationToken;
                } while (token != null);

                if (!BatchWritten)
                {
                    try
                    {
                        TableSource.ExecuteBatch(Batch);
                        PartitionKey = String.Empty;
                    }
                    catch (Exception ex)
                    {
                        throw new DeleteFailedException(String.Format("Table '{0}' row deletion failed.", TableNameToDelete), ex);
                    }
                }
                return(String.Format("Table '{0}' total rows deleted {1}.", TableNameToDelete, TotalRecordCountIn));
            }
            catch (ConnectionException cex)
            {
                throw cex;
            }
            catch (Exception ex)
            {
                throw new DeleteFailedException(String.Format("Table '{0}' row deletion failed.", TableNameToDelete), ex);
            }
            finally
            {
            }
        }
示例#5
0
        /// <summary>
        /// Create a local file copy of the specified Azure Table.
        /// </summary>
        /// <param name="TableName">Name of Azure Table to backup.</param>
        /// <param name="OutFileDirectory">Local directory (path) with authority to create/write a file.</param>
        /// <param name="Compress">True to compress the file.</param>
        /// <param name="Validate">True to validate the written record count matches what was queried.</param>
        /// <param name="TimeoutSeconds">Set timeout for table client.</param>
        /// <param name="filters">A list of Filter objects to be applied to table values extracted.</param>
        /// <returns>A string containing the name of the file created.</returns>
        public string BackupTableToFile(string TableName, string OutFileDirectory, bool Compress = false, bool Validate = false, int TimeoutSeconds = 30, List <Filter> filters = default(List <Filter>))
        {
            if (String.IsNullOrWhiteSpace(TableName))
            {
                throw new ParameterSpecException("TableName is missing.");
            }

            if (String.IsNullOrWhiteSpace(OutFileDirectory))
            {
                throw new ParameterSpecException("OutFileDirectory is missing.");
            }

            string OutFileName           = String.Format(TableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss") + ".txt");
            string OutFileNameCompressed = String.Format(TableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss") + ".7z");
            int    RecordCount           = 0;

            if (!Filter.AreFiltersValid(filters))
            {
                throw new ParameterSpecException(String.Format("One or more of the supplied filter criteria is invalid."));
            }

            string OutFileCreated            = "";
            string OutFileNamePath           = "";
            string OutFileNamePathCompressed = "";

            if (Path.GetFullPath(OutFileDirectory) != OutFileDirectory)
            {
                throw new ParameterSpecException(String.Format("Invalid output directory '{0}' specified.", OutFileDirectory));
            }
            else
            {
                OutFileNamePath           = Path.Combine(OutFileDirectory, OutFileName);
                OutFileNamePathCompressed = Path.Combine(OutFileDirectory, OutFileNameCompressed);
            }
            if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount))
            {
                throw new ConnectionException("Can not connect to CloudStorage Account.  Verify connection string.");
            }

            try
            {
                CosmosTable.CloudTableClient client = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration());
                CosmosTable.CloudTable       table  = client.GetTableReference(TableName);
                table.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds);

                CosmosTable.TableContinuationToken token = null;
                var entities           = new List <CosmosTable.DynamicTableEntity>();
                var entitiesSerialized = new List <string>();

                DynamicTableEntityJsonSerializer serializer = new DynamicTableEntityJsonSerializer();

                TableSpec TableSpecStart = new TableSpec(TableName);

                using (StreamWriter OutFile = new StreamWriter(OutFileNamePath))
                {
                    OutFile.WriteLine(JsonConvert.SerializeObject(TableSpecStart));
                    CosmosTable.TableQuery <CosmosTable.DynamicTableEntity> tq;
                    if (default(List <Filter>) == filters)
                    {
                        tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>();
                    }
                    else
                    {
                        tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>().Where(Filter.BuildFilterSpec(filters));
                    }
                    do
                    {
                        var queryResult = table.ExecuteQuerySegmented(tq, token);
                        foreach (CosmosTable.DynamicTableEntity dte in queryResult.Results)
                        {
                            OutFile.WriteLine(serializer.Serialize(dte));
                            RecordCount++;
                        }
                        token = queryResult.ContinuationToken;
                    } while (token != null);

                    TableSpec TableSpecEnd = new TableSpec(TableName, RecordCount);
                    OutFile.WriteLine(JsonConvert.SerializeObject(TableSpecEnd));

                    OutFile.Flush();
                    OutFile.Close();
                }

                if (Validate)
                {
                    int InRecords = 0;
                    // Read file/validate footer
                    using (StreamReader InFile = new StreamReader(OutFileNamePath))
                    {
                        string HeaderRec = InFile.ReadLine();
                        string FooterRec = "";
                        string DetailRec = "x";
                        do
                        {
                            DetailRec = InFile.ReadLine();
                            if (DetailRec == null)
                            {
                                InRecords--;
                            }
                            else
                            {
                                InRecords++;
                                FooterRec = DetailRec;
                            }
                        } while (DetailRec != null);
                        InFile.Close();

                        TableSpec footer = JsonConvert.DeserializeObject <TableSpec>(FooterRec);
                        if ((footer.RecordCount == InRecords) && (footer.TableName.Equals(TableName)))
                        {
                            //Do nothing, in count=out count
                        }
                        else
                        {
                            throw new AzureTableBackupException("Backup file validation failed.");
                        }
                    }
                }

                // https://stackoverflow.com/questions/11153542/how-to-compress-files
                if (Compress)
                {
                    FileStream FileToCompress = File.OpenRead(OutFileNamePath);
                    byte[]     buffer         = new byte[FileToCompress.Length];
                    FileToCompress.Read(buffer, 0, buffer.Length);

                    FileStream CompressedFileTarget = File.Create(OutFileNamePathCompressed);
                    using (GZipStream OutFileCompressed = new GZipStream(CompressedFileTarget, CompressionMode.Compress))
                    {
                        OutFileCompressed.Write(buffer, 0, buffer.Length);
                    }
                    FileToCompress.Close();
                    CompressedFileTarget.Close();
                    if (File.Exists(OutFileNamePath))
                    {
                        try
                        {
                            File.Delete(OutFileNamePath);
                        }
                        catch (Exception ex)
                        {
                            throw new AzureTableBackupException(String.Format("Error cleaning up files '{0}'.", OutFileNamePath), ex);
                        }
                    }
                    OutFileCreated = OutFileNameCompressed;
                }
                else
                {
                    OutFileCreated = OutFileName;
                }
            }
            catch (Exception ex)
            {
                throw new BackupFailedException(String.Format("Table '{0}' backup failed.", TableName), ex);
            }
            return(OutFileCreated);
        }
示例#6
0
        /// <summary>
        /// Restore file from blob storage by BackupAzureTables to the destination table name specified.
        /// File will be read directly from blob storage.  If the file is compressed, it will be decompressed to blob storage and then read.
        /// </summary>
        /// <param name="DestinationTableName">Name of the Azure Table to restore to -  may be different than name backed up originally.</param>
        /// <param name="OriginalTableName">Name of the Azure Table originally backed (required for determining blob directory to use)</param>
        /// <param name="BlobRoot">Name to use as blob root folder.</param>
        /// <param name="BlobFileName">Name of the blob file to restore.</param>
        /// <param name="TimeoutSeconds">Set timeout for table client.</param>
        /// <returns>A string indicating the table restored and record count.</returns>
        public string RestoreTableFromBlobDirect(string DestinationTableName, string OriginalTableName, string BlobRoot, string BlobFileName, int TimeoutSeconds = 30)
        {
            if (String.IsNullOrWhiteSpace(DestinationTableName))
            {
                throw new ParameterSpecException("DestinationTableName is missing.");
            }

            if (String.IsNullOrWhiteSpace(OriginalTableName))
            {
                throw new ParameterSpecException("OriginalTableName is missing.");
            }

            if (String.IsNullOrWhiteSpace(BlobFileName))
            {
                throw new ParameterSpecException(String.Format("Invalid BlobFileName '{0}' specified.", BlobFileName));
            }
            bool   Decompress   = BlobFileName.EndsWith(".7z");
            string TempFileName = String.Format("{0}.temp", BlobFileName);

            if (String.IsNullOrWhiteSpace(BlobRoot))
            {
                throw new ParameterSpecException(String.Format("Invalid BlobRoot '{0}' specified.", BlobRoot));
            }

            try
            {
                if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount))
                {
                    throw new ConnectionException("Can not connect to CloudStorage Account.  Verify connection string.");
                }

                if (!AZStorage.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureBlobConnectionSpec).Password, out AZStorage.CloudStorageAccount StorageAccountAZ))
                {
                    throw new ConnectionException("Can not connect to CloudStorage Account.  Verify connection string.");
                }

                AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ);
                var container = ClientBlob.GetContainerReference(BlobRoot);
                container.CreateIfNotExists();
                AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobRoot.ToLower() + "-table-" + OriginalTableName.ToLower());

                // If file is compressed, Decompress to a temp file in the blob
                if (Decompress)
                {
                    AZBlob.CloudBlockBlob BlobBlockTemp = directory.GetBlockBlobReference(TempFileName);
                    AZBlob.CloudBlockBlob BlobBlockRead = directory.GetBlockBlobReference(BlobFileName);

                    using (AZBlob.CloudBlobStream decompressedStream = BlobBlockTemp.OpenWrite())
                    {
                        using (Stream readstream = BlobBlockRead.OpenRead())
                        {
                            using (var zip = new GZipStream(readstream, CompressionMode.Decompress, true))
                            {
                                zip.CopyTo(decompressedStream);
                            }
                        }
                    }
                    BlobFileName = TempFileName;
                }

                AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(BlobFileName);

                CosmosTable.CloudTableClient client    = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration());
                CosmosTable.CloudTable       TableDest = client.GetTableReference(DestinationTableName);
                TableDest.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds);
                TableDest.CreateIfNotExists();

                using (Stream BlobStream = BlobBlock.OpenRead())
                {
                    using (StreamReader InputFileStream = new StreamReader(BlobStream))
                    {
                        string result = RestoreFromStream(InputFileStream, TableDest, DestinationTableName);
                        if (Decompress)
                        {
                            AZBlob.CloudBlockBlob BlobBlockTemp = directory.GetBlockBlobReference(TempFileName);
                            BlobBlockTemp.DeleteIfExists();
                        }
                        return(result);
                    }
                }
            }
            catch (ConnectionException cex)
            {
                throw cex;
            }
            catch (RestoreFailedException rex)
            {
                throw rex;
            }
            catch (Exception ex)
            {
                throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex);
            }
            finally
            {
            }
        } // RestoreTableFromBlobDirect
示例#7
0
        /// <summary>
        /// Restore file created by BackupAzureTables to the destination table name specified.
        /// </summary>
        /// <param name="DestinationTableName">Name of the Azure Table to restore to -  may be different than name backed up originally.</param>
        /// <param name="InFilePathName">Complete file name and path containing the data to be restored.</param>
        /// <param name="TimeoutSeconds">Set timeout for table client.</param>
        /// <returns>A string indicating the table restored and record count.</returns>
        public string RestoreTableFromFile(string DestinationTableName, string InFilePathName, int TimeoutSeconds = 30)
        {
            if (String.IsNullOrWhiteSpace(DestinationTableName))
            {
                throw new ParameterSpecException("DestinationTableName is missing.");
            }

            if (Path.GetFullPath(InFilePathName) != InFilePathName)
            {
                throw new ParameterSpecException(String.Format("Invalid file name/path '{0}' specified.", InFilePathName));
            }
            else
            {
                if (!File.Exists(InFilePathName))
                {
                    throw new ParameterSpecException(String.Format("File '{0}' does not exist.", InFilePathName));
                }
            }

            TableSpec footer = null;

            try
            {
                if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount))
                {
                    throw new ConnectionException("Can not connect to CloudStorage Account.  Verify connection string.");
                }

                CosmosTable.CloudTableClient client    = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration());
                CosmosTable.CloudTable       TableDest = client.GetTableReference(DestinationTableName);
                TableDest.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds);
                TableDest.CreateIfNotExists();

                DynamicTableEntityJsonSerializer serializer = new DynamicTableEntityJsonSerializer();

                bool   BatchWritten = true;
                string PartitionKey = String.Empty;
                CosmosTable.TableBatchOperation Batch = new CosmosTable.TableBatchOperation();
                int  BatchSize        = 100;
                int  BatchCount       = 0;
                long TotalRecordCount = 0;

                using (StreamReader InputFileStream = new StreamReader(InFilePathName))
                {
                    string InFileLine = InputFileStream.ReadLine();
                    while (InFileLine != null)
                    {
                        if (InFileLine.Contains("ProcessingMetaData") && InFileLine.Contains("Header"))
                        {
                            System.Console.WriteLine(String.Format("Header {0}", InFileLine));
                        }
                        else if (InFileLine.Contains("ProcessingMetaData") && InFileLine.Contains("Footer"))
                        {
                            footer = JsonConvert.DeserializeObject <TableSpec>(InFileLine);
                            System.Console.WriteLine(String.Format("Footer {0}", InFileLine));
                        }
                        else
                        {
                            CosmosTable.DynamicTableEntity dte2 = serializer.Deserialize(InFileLine);
                            if (String.Empty.Equals(PartitionKey))
                            {
                                PartitionKey = dte2.PartitionKey;
                            }
                            if (dte2.PartitionKey == PartitionKey)
                            {
                                Batch.InsertOrReplace(dte2);
                                BatchCount++;
                                TotalRecordCount++;
                                BatchWritten = false;
                            }
                            else
                            {
                                try
                                {
                                    TableDest.ExecuteBatch(Batch);
                                    Batch        = new CosmosTable.TableBatchOperation();
                                    PartitionKey = dte2.PartitionKey;
                                    Batch.InsertOrReplace(dte2);
                                    BatchCount = 1;
                                    TotalRecordCount++;
                                    BatchWritten = false;
                                }
                                catch (Exception ex)
                                {
                                    throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex);
                                }
                            }
                            if (BatchCount >= BatchSize)
                            {
                                try
                                {
                                    TableDest.ExecuteBatch(Batch);
                                    PartitionKey = String.Empty;
                                    Batch        = new CosmosTable.TableBatchOperation();
                                    BatchWritten = true;
                                    BatchCount   = 0;
                                }
                                catch (Exception ex)
                                {
                                    throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex);
                                }
                            }
                        }
                        InFileLine = InputFileStream.ReadLine();
                    }  // while (InFileLine != null)

                    //final batch
                    if (!BatchWritten)
                    {
                        try
                        {
                            TableDest.ExecuteBatch(Batch);
                            PartitionKey = String.Empty;
                        }
                        catch (Exception ex) {
                            throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex);
                        }
                    }
                } // using (StreamReader

                if (null == footer)
                {
                    throw new RestoreFailedException(String.Format("Table '{0}' restore failed, no footer record found.", DestinationTableName));
                }
                else if (TotalRecordCount == footer.RecordCount)
                {
                    //OK, do nothing
                }
                else
                {
                    throw new RestoreFailedException(String.Format("Table '{0}' restore failed, records read {1} does not match expected count {2} in footer record.", DestinationTableName, TotalRecordCount, footer.RecordCount));
                }

                return(String.Format("Restore to table '{0}' Successful; {1} entries.", DestinationTableName, TotalRecordCount));
            }
            catch (ConnectionException cex)
            {
                throw cex;
            }
            catch (RestoreFailedException rex)
            {
                throw rex;
            }
            catch (Exception ex)
            {
                throw new RestoreFailedException(String.Format("Table '{0}' restore failed.", DestinationTableName), ex);
            }
            finally
            {
            }
        }
        /// <summary>
        /// Backup table directly to Blob.
        ///
        /// </summary>
        /// <param name="TableName">Name of Azure Table to backup.</param>
        /// <param name="BlobRoot">Name to use as blob root folder.</param>
        /// <param name="Compress">True to compress the file.</param>
        /// <param name="RetentionDays">Process will age files in blob created more than x days ago.</param>
        /// <param name="TimeoutSeconds">Set timeout for table client.</param>
        /// <param name="filters">A list of Filter objects to be applied to table values extracted.</param>
        /// <returns>A string containing the name of the file created.</returns>
        public string BackupTableToBlobDirect(string TableName, string BlobRoot, bool Compress = false, int RetentionDays = 30, int TimeoutSeconds = 30, List <Filter> filters = default(List <Filter>))
        {
            string OutFileName = "";
            int    RecordCount = 0;
            int    BackupsAged = 0;

            if (String.IsNullOrWhiteSpace(TableName))
            {
                throw new ParameterSpecException("TableName is missing.");
            }

            if (String.IsNullOrWhiteSpace(BlobRoot))
            {
                throw new ParameterSpecException("BlobRoot is missing.");
            }

            try
            {
                if (Compress)
                {
                    OutFileName = String.Format(TableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss") + ".txt.7z");
                }
                else
                {
                    OutFileName = String.Format(TableName + "_Backup_" + System.DateTime.Now.ToString("yyyyMMddHHmmss") + ".txt");
                }

                if (!CosmosTable.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureTableConnectionSpec).Password, out CosmosTable.CloudStorageAccount StorageAccount))
                {
                    throw new ConnectionException("Can not connect to CloudStorage Account.  Verify connection string.");
                }

                if (!AZStorage.CloudStorageAccount.TryParse(new System.Net.NetworkCredential("", AzureBlobConnectionSpec).Password, out AZStorage.CloudStorageAccount StorageAccountAZ))
                {
                    throw new ConnectionException("Can not connect to CloudStorage Account.  Verify connection string.");
                }

                AZBlob.CloudBlobClient ClientBlob = AZBlob.BlobAccountExtensions.CreateCloudBlobClient(StorageAccountAZ);
                var container = ClientBlob.GetContainerReference(BlobRoot);
                container.CreateIfNotExists();
                AZBlob.CloudBlobDirectory directory = container.GetDirectoryReference(BlobRoot.ToLower() + "-table-" + TableName.ToLower());

                AZBlob.CloudBlockBlob BlobBlock = directory.GetBlockBlobReference(OutFileName);

                // start upload from stream, iterate through table, possible inline compress
                try
                {
                    CosmosTable.CloudTableClient client = CosmosTable.CloudStorageAccountExtensions.CreateCloudTableClient(StorageAccount, new CosmosTable.TableClientConfiguration());
                    CosmosTable.CloudTable       table  = client.GetTableReference(TableName);
                    table.ServiceClient.DefaultRequestOptions.ServerTimeout = new TimeSpan(0, 0, TimeoutSeconds);

                    CosmosTable.TableContinuationToken token = null;
                    var entities           = new List <CosmosTable.DynamicTableEntity>();
                    var entitiesSerialized = new List <string>();
                    DynamicTableEntityJsonSerializer serializer = new DynamicTableEntityJsonSerializer();

                    TableSpec TableSpecStart = new TableSpec(TableName);
                    var       NewLineAsBytes = Encoding.UTF8.GetBytes("\n");

                    var tempTableSpecStart     = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(TableSpecStart));
                    AZBlob.CloudBlobStream bs2 = BlobBlock.OpenWrite();
                    Stream bs = BlobBlock.OpenWrite();

                    if (Compress)
                    {
                        bs = new GZipStream(bs2, CompressionMode.Compress);
                    }
                    else
                    {
                        bs = bs2;
                    }

                    bs.Write(tempTableSpecStart, 0, tempTableSpecStart.Length);
                    bs.Flush();
                    bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length);
                    bs.Flush();

                    CosmosTable.TableQuery <CosmosTable.DynamicTableEntity> tq;
                    if (default(List <Filter>) == filters)
                    {
                        tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>();
                    }
                    else
                    {
                        tq = new CosmosTable.TableQuery <CosmosTable.DynamicTableEntity>().Where(Filter.BuildFilterSpec(filters));
                    }
                    do
                    {
                        var queryResult = table.ExecuteQuerySegmented(tq, token);
                        foreach (CosmosTable.DynamicTableEntity dte in queryResult.Results)
                        {
                            var tempDTE = Encoding.UTF8.GetBytes(serializer.Serialize(dte));
                            bs.Write(tempDTE, 0, tempDTE.Length);
                            bs.Flush();
                            bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length);
                            bs.Flush();
                            RecordCount++;
                        }
                        token = queryResult.ContinuationToken;
                    } while (token != null);

                    TableSpec TableSpecEnd     = new TableSpec(TableName, RecordCount);
                    var       tempTableSpecEnd = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(TableSpecEnd));
                    bs.Write(tempTableSpecEnd, 0, tempTableSpecEnd.Length);
                    bs.Flush();
                    bs.Write(NewLineAsBytes, 0, NewLineAsBytes.Length);
                    bs.Flush();
                    bs.Close();
                }
                catch (Exception ex)
                {
                    throw new BackupFailedException(String.Format("Table '{0}' backup failed.", TableName), ex);
                }

                DateTimeOffset OffsetTimeNow    = System.DateTimeOffset.Now;
                DateTimeOffset OffsetTimeRetain = System.DateTimeOffset.Now.AddDays(-1 * RetentionDays);

                //Cleanup old versions
                var BlobList = directory.ListBlobs().OfType <AZBlob.CloudBlockBlob>().ToList();;
                foreach (var blob in BlobList)
                {
                    if (blob.Properties.Created < OffsetTimeRetain)
                    {
                        try
                        {
                            blob.Delete();
                            BackupsAged++;
                        }
                        catch (Exception ex)
                        {
                            throw new AgingException(String.Format("Error aging file '{0}'.", blob.Name), ex);
                        }
                    }
                }

                return(String.Format("Table '{0}' backed up as '{2}' under blob '{3}\\{4}'; {1} files aged.", TableName, BackupsAged, OutFileName, BlobRoot, directory.ToString()));
            }
            catch (ConnectionException cex)
            {
                throw cex;
            }
            catch (Exception ex)
            {
                throw new BackupFailedException(String.Format("Table '{0}' backup failed.", TableName), ex);
            }
            finally
            {
            }
        } // BackupTableToBlobDirect
示例#9
0
        internal static TableRequestOptions ApplyDefaults(TableRequestOptions requestOptions, CloudTableClient serviceClient)
        {
            TableRequestOptions tableRequestOptions = new TableRequestOptions(requestOptions);

            if (serviceClient.IsPremiumEndpoint())
            {
                tableRequestOptions.TableQueryMaxItemCount               = (tableRequestOptions.TableQueryMaxItemCount ?? serviceClient.DefaultRequestOptions.TableQueryMaxItemCount ?? BaseDefaultRequestOptions.TableQueryMaxItemCount);
                tableRequestOptions.TableQueryMaxDegreeOfParallelism     = (tableRequestOptions.TableQueryMaxDegreeOfParallelism ?? serviceClient.DefaultRequestOptions.TableQueryMaxDegreeOfParallelism ?? BaseDefaultRequestOptions.TableQueryMaxDegreeOfParallelism);
                tableRequestOptions.TableQueryEnableScan                 = (tableRequestOptions.TableQueryEnableScan ?? serviceClient.DefaultRequestOptions.TableQueryEnableScan ?? BaseDefaultRequestOptions.TableQueryEnableScan);
                tableRequestOptions.TableQueryContinuationTokenLimitInKb = (tableRequestOptions.TableQueryContinuationTokenLimitInKb ?? serviceClient.DefaultRequestOptions.TableQueryContinuationTokenLimitInKb ?? BaseDefaultRequestOptions.TableQueryContinuationTokenLimitInKb);
                tableRequestOptions.ConsistencyLevel = (tableRequestOptions.ConsistencyLevel ?? serviceClient.DefaultRequestOptions.ConsistencyLevel ?? BaseDefaultRequestOptions.ConsistencyLevel);
            }
            else
            {
                tableRequestOptions.LocationMode = (tableRequestOptions.LocationMode ?? serviceClient.DefaultRequestOptions.LocationMode ?? BaseDefaultRequestOptions.LocationMode);
            }
            tableRequestOptions.RetryPolicy          = (tableRequestOptions.RetryPolicy ?? serviceClient.DefaultRequestOptions.RetryPolicy ?? BaseDefaultRequestOptions.RetryPolicy);
            tableRequestOptions.ServerTimeout        = (tableRequestOptions.ServerTimeout ?? serviceClient.DefaultRequestOptions.ServerTimeout ?? BaseDefaultRequestOptions.ServerTimeout);
            tableRequestOptions.MaximumExecutionTime = (tableRequestOptions.MaximumExecutionTime ?? serviceClient.DefaultRequestOptions.MaximumExecutionTime ?? BaseDefaultRequestOptions.MaximumExecutionTime);
            tableRequestOptions.PayloadFormat        = (tableRequestOptions.PayloadFormat ?? serviceClient.DefaultRequestOptions.PayloadFormat ?? BaseDefaultRequestOptions.PayloadFormat);
            if (!tableRequestOptions.OperationExpiryTime.HasValue && tableRequestOptions.MaximumExecutionTime.HasValue)
            {
                tableRequestOptions.OperationExpiryTime = DateTime.Now + tableRequestOptions.MaximumExecutionTime.Value;
            }
            tableRequestOptions.PropertyResolver        = (tableRequestOptions.PropertyResolver ?? serviceClient.DefaultRequestOptions.PropertyResolver ?? BaseDefaultRequestOptions.PropertyResolver);
            tableRequestOptions.ProjectSystemProperties = (tableRequestOptions.ProjectSystemProperties ?? serviceClient.DefaultRequestOptions.ProjectSystemProperties ?? BaseDefaultRequestOptions.ProjectSystemProperties);
            return(tableRequestOptions);
        }