private async Task DeserializeMessage(CloudQueueMessage message) { Exception exception = null; BrokerQueueItem brokerQueueItem = null; if (message.AsBytes != null && message.AsBytes.Length > 0) { // Deserialize message to BrokerQueueItem try { brokerQueueItem = (BrokerQueueItem)this.formatter.Deserialize( await AzureStorageTool.GetMsgBody(this.blobContainer, message.AsBytes)); brokerQueueItem.PersistAsyncToken.AsyncToken = brokerQueueItem.Message.Headers.MessageId.ToString(); BrokerTracing.TraceVerbose( "[AzureQueueRequestFetcher] .DeserializeMessage: deserialize header={0} property={1}", brokerQueueItem.Message.Headers.MessageId, brokerQueueItem.Message.Properties); } catch (Exception e) { BrokerTracing.TraceError( "[AzureQueueRequestFetcher] .DeserializeMessage: deserialize message failed, Exception:{0}", e.ToString()); exception = e; } } if (brokerQueueItem != null && !this.requestDic.TryAdd( brokerQueueItem.PersistAsyncToken.AsyncToken.ToString(), true)) { Interlocked.Increment(ref this.pendingFetchCount); try { await this.requestQueue.DeleteMessageAsync(message); } catch (Exception e) { BrokerTracing.TraceError( "[AzureQueueRequestFetcher] .DeserializeMessage: delete duplicate message in request queue failed, Exception:{0}", e.ToString()); } } else { var copyMessage = new CloudQueueMessage(message.AsBytes); await this.pendingQueue.AddMessageAsync(copyMessage); try { await this.requestQueue.DeleteMessageAsync(message); } catch (Exception e) { BrokerTracing.TraceError( "[AzureQueueRequestFetcher] .DeserializeMessage: delete message in request queue failed, Exception:{0}", e.ToString()); } this.HandleMessageResult(new MessageResult(brokerQueueItem, exception)); } }
internal AzureQueuePersist(string userName, string sessionId, string clientId, string storageConnectString) { BrokerTracing.TraceVerbose( "[AzureQueuePersist] .AzureQueuePersist: constructor. session id = {0}, client id = {1}, user name = {2}", sessionId, clientId, userName); Debug.Write($"[AzureQueuePersist].AzureQueuePersist: {Environment.NewLine}{Environment.StackTrace}"); Debug.Write( $"[AzureQueuePersist].AzureQueuePersist: Current principal: {Thread.CurrentPrincipal.Identity.Name}"); this.sessionIdField = sessionId; this.clientIdField = clientId; this.userNameField = userName; this.storageConnectString = storageConnectString; var requestQueueName = MakeQueuePath(sessionId, clientId, true); var responseTableName = MakeTablePath(sessionId, clientId); var pendingQueueName = MakeQueuePath(sessionId, clientId, false); this.isNewCreatePersistField = true; try { var requestQueueExist = AzureStorageTool.ExistsQueue(storageConnectString, requestQueueName) .GetAwaiter().GetResult(); var responseQueueExist = AzureStorageTool.ExistTable(storageConnectString, responseTableName) .GetAwaiter().GetResult(); if (requestQueueExist && responseQueueExist) { this.isNewCreatePersistField = false; } else if (requestQueueExist != responseQueueExist) { // If there is only request queue but not response queue, it could be caused by: // a. Queue creation operation interrupted // b. Queue deletion operation interrupted if (requestQueueExist) { BrokerTracing.TraceError( "[AzureQueuePersist] .AzureQueuePersist: queue data not integrety. Fix it by deleting queue = {0}", requestQueueName); AzureStorageTool.DeleteQueueAsync(storageConnectString, sessionId, requestQueueName) .GetAwaiter().GetResult(); } else { // There is only response queue but no request queue - this should rarely happen, as we always // create request queue before response queue, and delete request queue after response queue. BrokerTracing.TraceError( "[AzureQueuePersist] .AzureQueuePersist: queue data not integrety. Fix it by deleting queue = {0}", responseTableName); AzureStorageTool.DeleteTableAsync(storageConnectString, sessionId, responseTableName) .GetAwaiter().GetResult(); } } } catch (Exception e) { BrokerTracing.TraceError( "[AzureQueuePersist] .AzureQueuePersist: MessageQueue.Exists raised exception, exception: {0}", e); throw new BrokerQueueException((int)BrokerQueueErrorCode.E_BQ_PERSIST_STORAGE_FAIL, e.ToString()); } if (this.isNewCreatePersistField) { try { this.pendingQueueField = AzureStorageTool.CreateQueueAsync( this.storageConnectString, sessionId, pendingQueueName, clientId, true, this.FormatRequestQueueLabel()).GetAwaiter().GetResult(); this.blobContainer = AzureStorageTool .CreateBlobContainerAsync(this.storageConnectString, requestQueueName).GetAwaiter().GetResult(); BrokerTracing.TraceInfo( "[AzureQueuePersist] .AzureQueuePersist: creating message requests queue {0}", requestQueueName); this.EOMFlag = false; this.requestQueueField = AzureStorageTool.CreateQueueAsync( this.storageConnectString, sessionId, requestQueueName, clientId, true, this.FormatRequestQueueLabel()).GetAwaiter().GetResult(); BrokerTracing.TraceInfo( "[AzureQueuePersist] .AzureQueuePersist: creating message responses queue {0}", responseTableName); this.responseTableField = AzureStorageTool.CreateTableAsync( this.storageConnectString, sessionId, responseTableName, clientId, false, "0").GetAwaiter().GetResult(); this.persistVersion = BrokerQueueItem.PersistVersion; BrokerTracing.TraceVerbose( "[AzureQueuePersist] .AzureQueuePersist: set persist version = {0}", BrokerQueueItem.PersistVersion); // On queue creation completion, mark response queue label with "0", which means failed response count = 0. // Note: label response queue is used as a flag indicating queue creation is completed. So it must be the last step of queue creation. } catch (Exception e) { BrokerTracing.TraceError( "[AzureQueuePersist] .AzureQueuePersist: failed to create AzureQueue queue, {0}, exception: {1}.", responseTableName, e); // delete request queue after response queue if (this.responseTableField != null) { this.responseTableField.DeleteAsync().GetAwaiter().GetResult(); } if (this.requestQueueField != null) { this.requestQueueField.DeleteAsync().GetAwaiter().GetResult(); } throw; } } else { this.requestQueueField = AzureStorageTool.GetQueue(storageConnectString, requestQueueName).GetAwaiter() .GetResult(); this.responseTableField = AzureStorageTool.GetTable(storageConnectString, responseTableName); this.pendingQueueField = AzureStorageTool.GetQueue(storageConnectString, pendingQueueName).GetAwaiter() .GetResult(); this.blobContainer = AzureStorageTool .CreateBlobContainerAsync(this.storageConnectString, requestQueueName).GetAwaiter().GetResult(); try { /*AzureStorageTool.RestoreRequest( * this.requestQueueField, * this.pendingQueueField, * this.responseTableField, * this.blobContainer).GetAwaiter().GetResult();*/ this.requestsCountField = this.requestQueueField.ApproximateMessageCount ?? 0; (this.responsesCountField, this.lastReponseIndex) = AzureStorageTool .CountTableEntity(storageConnectString, responseTableName).GetAwaiter().GetResult(); this.allRequestsCountField = this.requestsCountField + this.responsesCountField; if (this.lastReponseIndex > this.responseIndex) { this.responseIndex = this.lastReponseIndex; } this.persistVersion = BrokerVersion.DefaultPersistVersion; this.EOMFlag = this.allRequestsCountField > 0; } catch (Exception e) { BrokerTracing.TraceError( "[AzureQueuePersist] .AzureQueuePersist: failed to access azure queue, {0}, exception: {1}.", responseTableName, e); throw; } } // Init fetchers BrokerTracing.TraceVerbose("[AzureQueuePersist] .AzureQueuePersist: AzureQueue Transactions Enabled."); this.requestFetcher = new AzureQueueRequestFetcher( this.requestQueueField, this.pendingQueueField, this.requestsCountField, binFormatterField, this.blobContainer); this.responseFetcher = new AzureQueueResponseFetcher( this.responseTableField, this.responsesCountField, binFormatterField, this.blobContainer, this.lastReponseIndex); }
private async Task PeekMessageAsync() { if (this.isDisposedField) { this.RevertFetchCount(); return; } while (true) { Exception exception = null; if (this.pendingFetchCount < 1) { break; } while (this.pendingFetchCount > 0) { byte[] messageBody = null; try { if (responseList == null || responseList.Count <= index) { BrokerTracing.TraceVerbose( "[AzureQueueResponseFetch] .PeekMessageAsync: lastIndex={0}, ackIndex={1}", lastIndex, ackIndex); responseList = await AzureStorageTool.GetBatchEntityAsync( this.responseTable, this.lastIndex, this.ackIndex); index = 0; BrokerTracing.TraceVerbose( "[AzureQueueResponseFetch] .PeekMessageAsync: get batch entity count={0}", responseList.Count); } if (responseList.Count > index) { messageBody = responseList[index].Message; if (long.TryParse(responseList[index].RowKey, out var tempIndex) && tempIndex > this.lastIndex) { this.lastIndex = tempIndex; } index++; } } catch (Exception e) { BrokerTracing.TraceError( "[AzureQueueResponseFetch] .PeekMessageAsync: peek batch messages failed, Exception:{0}", e.ToString()); exception = e; } if (messageBody == null && exception == null) { BrokerTracing.TraceWarning("[AzureQueueResponseFetch] .PeekMessage: null message and exception, lastIndex = {0}, ack = {1}, Count = {2}, index = {3}.", this.lastIndex, this.ackIndex, responseList.Count, index); } else { BrokerQueueItem brokerQueueItem = null; if (messageBody != null && messageBody.Length > 0) { // Deserialize message to BrokerQueueItem try { brokerQueueItem = (BrokerQueueItem)this.formatter.Deserialize( await AzureStorageTool.GetMsgBody(this.blobContainer, messageBody)); brokerQueueItem.PersistAsyncToken.AsyncToken = brokerQueueItem.Message.Headers.RelatesTo.ToString(); BrokerTracing.TraceVerbose( "[AzureQueueResponseFetch] .PeekMessage: deserialize header={0} property={1}", brokerQueueItem.Message.Headers.RelatesTo, brokerQueueItem.Message.Properties); } catch (Exception e) { BrokerTracing.TraceError( "[AzureQueueResponseFetch] .PeekMessage: deserialize message failed, Exception:{0}", e.ToString()); exception = e; } } this.HandleMessageResult(new MessageResult(brokerQueueItem, exception)); Interlocked.Decrement(ref this.pendingFetchCount); } } this.CheckAndGetMoreMessages(); } }