public SendMessageHeaders GetSendMessageHeaders(SendIterationContext context)
{
var info = agentInfoService.Get();
var headers = new SendMessageHeaders
{
Domain = info.Domain,
ComputerName = info.ComputerName,
DbToken = info.DbToken,
ConfigToken = info.ConfigToken,
ConfigVersion = info.ConfigVersion,
OrganizationUnit = info.OrganizationUnit,
Attempt = context.Attempt,
Format = TransportConstants.RequestFormDataFormat,
Hints = new[] { MessageHints.OrderedFormData },
};
return(headers);
}
protected virtual SendResult SendIteration(IList <ITransportPacketInfo> packetInfos, IList <ConfigurationRequestDataItem> configurationsToDownloadInfos, SendIterationContext context)
{
var url = this.GetUri(this.GetSendMessageHeaders(context));
string formDataBoundary = string.Format("----------{0:N}", Guid.NewGuid());
string contentType = "multipart/form-data; boundary=" + formDataBoundary;
#if !NETSTANDARD2_0
try
{
var req = WebRequest.Create(url) as HttpWebRequest;
//req.KeepAlive = true;
//req.KeepAlive = false;
req.AllowWriteStreamBuffering = false;
req.Method = "POST";
req.ContentType = contentType;
TransportSendStats sendededStats = null;
using (Stream requestStream = req.GetRequestStream())
{
sendededStats = this.WriteMessageToBody(formDataBoundary, requestStream, packetInfos, configurationsToDownloadInfos, context);
}
try
{
var response = req.GetResponse();
using (var resStream = response.GetResponseStream())
using (var streamReader = new StreamReader(resStream))
{
// обрабатывает ответ
return(this.SaveRequestResults(resStream, sendededStats));
}
}
catch (WebException ex)
{
var wRespStatusCode = ((HttpWebResponse)ex.Response).StatusCode;
if ((int)wRespStatusCode == 429)
{
var timeout = int.TryParse(ex.Response.Headers.GetValues("Retry-After").FirstOrDefault(), out var retryTm)
? retryTm
: Convert.ToInt64(settings.ErrorRetryTimeout.TotalMilliseconds);
return(SendResult.Retry(timeout));
}
// при общих ошибках
return(SendResult.Retry(settings.ServerErrorRetryTimeout));
}
}
catch (Exception)
{
// при общих ошибках
return(SendResult.Retry(settings.ErrorRetryTimeout));
}
#else
try
{
TransportSendStats sendededStats = null;
var reqContent = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = new WriteToStreamContent((requestStream, ctx) =>
{
sendededStats = this.WriteMessageToBody(formDataBoundary, requestStream, packetInfos, configurationsToDownloadInfos, context);
}),
};
// workaround for net core 2.2 and analogs
// нужно для того чтобы в случаях "отказов" не происходило чтение всего тела запроса, а читались только основные параметры
reqContent.Headers.ExpectContinue = true;
reqContent.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(contentType);
var result = this.httpClient.SendAsync(reqContent, HttpCompletionOption.ResponseContentRead).Result;
if (result.IsSuccessStatusCode)
{
using (var stream = result.Content.ReadAsStreamAsync().Result)
{
// обрабатывает ответ
return(this.SaveRequestResults(stream, sendededStats));
};
}
else
{
if ((int)result.StatusCode == 429)
{
var timeout = int.TryParse(result.Headers.GetValues("Retry-After").FirstOrDefault(), out var retryTm)
? retryTm
: Convert.ToInt64(settings.ErrorRetryTimeout.TotalMilliseconds);
return(SendResult.Retry(timeout));
}
return(SendResult.Retry(settings.ServerErrorRetryTimeout));
}
}
catch (Exception)
{
// при общих ошибках
return(SendResult.Retry(settings.ErrorRetryTimeout));
}
#endif
}
protected virtual TransportSendStats WriteMessageToBody(string formDataBoundary, Stream requestStream, IList <ITransportPacketInfo> packetInfos, IList <ConfigurationRequestDataItem> configurationsToDownloadInfos, SendIterationContext context)
{
var packetsSendStats = new Dictionary <ITransportPacketInfo, SendStats>();
using (var nonClosingStream = new NonClosingStreamWrapper(requestStream))
using (var sr = new StreamWriter(nonClosingStream, this.encoding ?? DefaultEncoding))
using (var formDataWriter = new FormDataWriter(nonClosingStream, formDataBoundary, this.encoding ?? DefaultEncoding))
{
var maxMessageSizeReached = false;
if (context.RequestConfigurations)
{
// записываем данные по конфигурациям
for (int i = 0; i < configurationsToDownloadInfos.Count; i++)
{
var configurationsToDownloadInfo = configurationsToDownloadInfos[i];
// == confs[0].
var itemPrefix = string.Concat(TransportConstants.FormDataConfigurationProp, "[", i, "].");
// == confs[0].ProviderKey
formDataWriter.WriteValue(string.Concat(itemPrefix, nameof(ConfigurationRequestDataItem.ProviderKey)), configurationsToDownloadInfo.ProviderKey);
formDataWriter.WriteValue(string.Concat(itemPrefix, nameof(ConfigurationRequestDataItem.Token)), configurationsToDownloadInfo.Token);
if (configurationsToDownloadInfo.StartPosition != null)
{
formDataWriter.WriteValue(string.Concat(itemPrefix, nameof(ConfigurationRequestDataItem.StartPosition)), configurationsToDownloadInfo.StartPosition);
}
formDataWriter.WriteValue(string.Concat(itemPrefix, nameof(ConfigurationRequestDataItem.IsCompleted)), configurationsToDownloadInfo.IsCompleted);
}
}
if (context.SendPackets)
{
formDataWriter.ResetSize();
// записываем данные по пакетам
for (int i = 0; i < packetInfos.Count; i++)
{
var packetInfo = packetInfos[i];
var sendStats = this.packetManager.ReadSendStats(packetInfo.ProviderKey, packetInfo.Id);
if (sendStats?.TransferCompleted == true)
{
continue;
}
int packetBytesSended = sendStats?.TransferedBytes ?? 0;
int bufferSize = 4096;
bool packetTransferCompleted = false;
string packetBlockHashStr;
// == packets[0].
var itemPrefix = string.Concat(TransportConstants.FormDataPacketsProp, "[", i, "].");
// записываем данные пакета
using (var packetStream = packetInfo.GetReadOnlyStreamOrDefault())
{
// пакет был удалён
if (packetStream == null)
{
continue;
}
// записываем метаданные о пакете
// == packets[0].ProviderKey
formDataWriter.WriteValue(string.Concat(itemPrefix, nameof(PacketFormDataItem.ProviderKey)), packetInfo.ProviderKey);
formDataWriter.WriteValue(string.Concat(itemPrefix, nameof(PacketFormDataItem.PacketId)), packetInfo.Id);
formDataWriter.WriteValue(string.Concat(itemPrefix, nameof(PacketFormDataItem.StartPosition)), packetBytesSended);
packetStream.Seek(packetBytesSended, SeekOrigin.Begin);
using (var hashAlgorithm = this.hashAlgorithmFunc())
{
var packetIdStr = packetInfo.Id.ToString();
formDataWriter.WriteFileHeader(string.Concat(itemPrefix, nameof(PacketFormDataItem.FileKey)), packetIdStr);
byte[] buffer = new byte[bufferSize];
int read = 0;
while (!maxMessageSizeReached && (read = packetStream.Read(buffer, 0, buffer.Length)) > 0)
{
if (formDataWriter.GetWrittenSize() + read > this.settings.PacketSizeLimits.Max)
{
// записываем только до максимального размера
read = this.settings.PacketSizeLimits.Max - formDataWriter.GetWrittenSize();
maxMessageSizeReached = true;
}
formDataWriter.Write(buffer, 0, read);
if (hashAlgorithm != null)
{
// hash contents
hashAlgorithm.TransformBlock(buffer, 0, read, null, 0);
}
packetBytesSended = packetBytesSended + read;
}
if (read <= 0)
{
packetTransferCompleted = true;
}
if (hashAlgorithm != null)
{
hashAlgorithm.TransformFinalBlock(new byte[0], 0, 0);
packetBlockHashStr = hashAlgorithm.GetHashString();
}
}
}
formDataWriter.WriteValue(string.Concat(itemPrefix, nameof(PacketFormDataItem.EndPosition)), packetBytesSended);
//// записываем хеш данные блока пакета
//formDataWriter.WriteValue(nameof(PacketFormDataItem.Hash) + suffix, packetBlockHashStr);
formDataWriter.WriteValue(string.Concat(itemPrefix, nameof(PacketFormDataItem.IsFinal)), packetTransferCompleted);
packetsSendStats.Add(packetInfo, new SendStats
{
TransferedBytes = packetBytesSended,
TransferCompleted = packetTransferCompleted,
});
if (maxMessageSizeReached)
{
break;
}
}
}
}
return(new TransportSendStats
{
SendedPacketsStats = packetsSendStats,
ConfigurationsInfos = configurationsToDownloadInfos,
});
}
public void Process()
{
if (Interlocked.CompareExchange(ref this.processing, 1, 0) == 1)
{
return;
}
try
{
// TODO так отсылка сообщений идёт неравномерно для разных провайдеров
var packetInfos = this.packetManager
.GetTransportPacketInfos()
.OrderBy(x => x.Id)
.ToList();
// TODO implement
var configurationsToDownloadInfos = this.configurationStore.GetRequestItems();
// TODO проверять мин. размер и таймаут (можно ещё проверить что полный размер можно разбить на куски к которым применяется PacketSizeLimits)
var size = packetInfos.Sum(x => x.Length);
var needSendPackets = size > this.settings.PacketSizeLimits.Min || this.NeedSendPacketsByTimeout();
// если нужно cкачать конфигурации
var needUpdateConfigs = this.NeedUpdateConfigs();
if (size > 0 && !needSendPackets)
{
this.sendStateStore.CheckedPacketsSize();
}
if (!needSendPackets && !needUpdateConfigs)
{
Interlocked.Exchange(ref this.processing, 0);
return;
}
SendResult sendResult = null;
SendIterationContext context = new SendIterationContext
{
SendPackets = true,
RequestConfigurations = true,
MessageSize = this.GetMessageSize(packetInfos),
};
do
{
// отправляем и сохраняем данные по ответу
sendResult = this.SendIteration(packetInfos, configurationsToDownloadInfos, context);
if (sendResult.TimeoutToNextTry.HasValue)
{
context.Attempt++;
context.FirstFailTimeUtc = context.FirstFailTimeUtc ?? DateTime.UtcNow;
Delay(sendResult.TimeoutToNextTry.Value)
.Wait();
}
// TODO надо оптимизировать !!! packetInfos будут подтягиваться из провайдера, а они уже есть в памяти
packetInfos = packetManager
.GetTransportPacketInfos()
.OrderBy(x => x.Id)
.ToList();
configurationsToDownloadInfos = this.configurationStore.GetRequestItems();
// нужно ли обновить конфиг
needUpdateConfigs = this.NeedUpdateConfigs();
if (sendResult.IsSended)
{
context = new SendIterationContext
{
SendPackets = true,
RequestConfigurations = true,
MessageSize = this.GetMessageSize(packetInfos),
};
}
} while (packetInfos.Any() && (configurationsToDownloadInfos.Any(x => !x.IsCompleted) || needUpdateConfigs));
}
finally
{
Interlocked.Exchange(ref this.processing, 0);
}
}
