public async Task <AddAddPacketsPartsResult> AddIfNotExistsPacketsPartsAsync(AgentIdentity agentId, IList <AddPacketPartRequest> requests, CancellationToken cancellationToken = default(CancellationToken))
{
if (requests == null)
{
throw new ArgumentException("requests");
}
if (!requests.Any())
{
return(AddAddPacketsPartsResult.EmptyResult());
}
if (!this.initialized)
{
await this.InitAsync(cancellationToken);
}
var stopWatch = Stopwatch.StartNew();
// IsOrdered = false, неупорядоченные операции - при выполнении отдельных операций они выполняются не упорядочено и они не останавливают исполнение остальных операций
try
{
var docs = requests.Select(x => this.Get(agentId, x)).ToList();
await this.chunks.InsertManyAsync(docs, new InsertManyOptions()
{
IsOrdered = false
}, cancellationToken);
stopWatch.Stop();
return(AddAddPacketsPartsResult.CreateResult(this.storageToken, requests.Select((request, i) => {
return AddAddPacketPartResult.SuccessResult(request, this.storageToken, docs[i].Id);
})));
}
catch (MongoBulkWriteOperationException ex)
{
// Instrumentation ???
var results = new List <AddAddPacketPartResult>();
for (int i = 0; i < requests.Count; i++)
{
var request = requests[i];
var rqNonDupKeyErrors = ex.WriteErrors.Where(x => x.Index == i && x.Category != ServerErrorCategory.DuplicateKey);
if (rqNonDupKeyErrors.Any())
{
results.Add(AddAddPacketPartResult.FailResult(request));
}
else
{
// TODO оптимизировать, не делать запрос на каждый fail вариант
var id = await this.chunks.Find(x => x.PacketId == request.PacketId && x.StartPosition == request.StartPosition).Project(x => x.Id).FirstOrDefaultAsync(cancellationToken);
if (id == null)
{
results.Add(AddAddPacketPartResult.FailResult(request));
}
else
{
results.Add(AddAddPacketPartResult.SuccessResult(request, this.storageToken, id));
}
}
}
//var failed = new List<AddAddPacketPartResult>();
//var reqToSelect = new List<AddPacketPartRequest>();
//for (int i = 0; i < requests.Count; i++)
//{
// var request = requests[i];
// var rqNonDupKeyErrors = ex.WriteErrors.Where(x => x.Index == i && x.Category != ServerErrorCategory.DuplicateKey);
// if (rqNonDupKeyErrors.Any())
// {
// failed.Add(AddAddPacketPartResult.FailResult(request));
// }
// else
// {
// reqToSelect.Add(request);
// }
//}
//// select
//Expression<Func<PacketPart, bool>> filter = null;
//var f = Builders<PacketPart>.Filter
// .Or(reqToSelect.Select(r =>))
//FilterDefinition<PacketPart> filterDef =
//var ids = await this.chunks.Find(x => x.PacketId == request.PacketId && x.StartPosition == request.StartPosition).Project(x => x.Id).FirstOrDefaultAsync(cancellationToken);
stopWatch.Stop();
return(AddAddPacketsPartsResult.CreateResult(this.storageToken, results));
}
}
public async Task <ParseBodyAndSavePacketsResult> ParseBodyAndSavePackets(MessageHeaders messageHeaders, HttpRequest request)
{
var cancellationToken = request.HttpContext.RequestAborted;
cancellationToken.ThrowIfCancellationRequested();
var format = messageHeaders.Format ?? TransportConstants.RequestFormDataFormat;
string boundary = null;
if (format == TransportConstants.RequestFormDataFormat)
{
int fileSectionBufferSize = 81920;
// если данные formdata упорядочены в потоке запроса (бинарные данные должны идти после большинства метаданных о пакете)
if (messageHeaders.Hints?.Contains(MessageHints.OrderedFormData) == true)
{
var indexToPacketDataItem = new Dictionary <int, PacketFormDataItem>();
var indexToPacketBytes = new Dictionary <int, byte[]>();
var indexToConfigurationItem = new Dictionary <int, ConfigurationRequestDataItem>();
if (MediaTypeHeaderValue.TryParse(request.ContentType, out var contentType))
{
boundary = HeaderUtilities.GetBoundary(contentType, 70);
}
var multipartReader = new MultipartReader(boundary, request.Body)
{
//ValueCountLimit = _options.ValueCountLimit,
//KeyLengthLimit = _options.KeyLengthLimit,
//ValueLengthLimit = _options.ValueLengthLimit,
HeadersCountLimit = int.MaxValue,
HeadersLengthLimit = int.MaxValue,
BodyLengthLimit = long.MaxValue,
};
//PacketFormDataItem current = null;
//byte[] currentBytes = null;
var agentId = messageHeaders.GetAgentIdData();
var section = await multipartReader.ReadNextSectionAsync(cancellationToken);
while (section != null)
{
// Parse the content disposition here and pass it further to avoid reparsings
if (!ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out var contentDisposition))
{
throw new InvalidDataException("Form section has invalid Content-Disposition value: " + section.ContentDisposition);
}
if (contentDisposition.IsFileDisposition())
{
var fileSection = new FileMultipartSection(section, contentDisposition);
var name = fileSection.Name;
var fileName = fileSection.FileName;
var packetId = fileSection.FileName;
var result = GetFormPathDataFromEntry(name);
if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataPacketsProp)
{
if (int.TryParse(result.Parts[1], out var index))
{
var item = indexToPacketDataItem[index];
var providerKey = item.ProviderKey;
var packetItem = indexToPacketDataItem[index];
if (packetItem.PacketId != packetId)
{
throw new InvalidDataException($"Incorrect format for form-data message. Section {name} has invalid FileName.");
}
var bytes = await ReadToEnd(fileSection.FileStream, fileSectionBufferSize, cancellationToken);
indexToPacketBytes.Add(index, bytes);
}
else
{
throw new InvalidDataException($"Incorrect format for form-data message. Section {name} does not have index suffix.");
}
}
else
{
throw new InvalidDataException($"Incorrect format for form-data message. Section {name} incorrect.");
}
}
else if (contentDisposition.IsFormDisposition())
{
var formDataSection = new FormMultipartSection(section, contentDisposition);
// Content-Disposition: form-data; name="key"
//
// value
// Do not limit the key name length here because the multipart headers length limit is already in effect.
var key = formDataSection.Name;
var value = await formDataSection.GetValueAsync();
var result = GetFormPathDataFromEntry(key);
if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataPacketsProp)
{
if (int.TryParse(result.Parts[1], out var index))
{
if (!indexToPacketDataItem.TryGetValue(index, out var dataItem))
{
//// сохраняем предыдущий
//if (current != null)
//{
// await this.packetsStore.AddIfNotExistsPacketPartAsync(agentId, this.CreateAddPacketRequest(current, currentBytes));
//}
dataItem = new PacketFormDataItem();
indexToPacketDataItem.Add(index, dataItem);
}
if (!dataItem.FillProperty(result.Parts[2], value))
{
throw new InvalidDataException($"Incorrect format for form-data message. Section {key} incorrect.");
}
}
else
{
throw new InvalidDataException($"Incorrect format for form-data message. Section {key} does not have index suffix.");
}
}
else if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataConfigurationProp)
{
if (int.TryParse(result.Parts[1], out var index))
{
if (!indexToConfigurationItem.TryGetValue(index, out var dataItem))
{
dataItem = new ConfigurationRequestDataItem();
indexToConfigurationItem.Add(index, dataItem);
}
if (!dataItem.FillProperty(result.Parts[2], value))
{
throw new InvalidDataException($"Incorrect format for form-data message. Section {key} incorrect.");
}
}
else
{
throw new InvalidDataException($"Incorrect format for form-data message. Section {key} does not have index suffix.");
}
}
else
{
// ignore or throw?
}
//if (formAccumulator.ValueCount > _options.ValueCountLimit)
//{
// throw new InvalidDataException($"Form value count limit {_options.ValueCountLimit} exceeded.");
//}
}
else
{
System.Diagnostics.Debug.Assert(false, "Unrecognized content-disposition for this section: " + section.ContentDisposition);
}
section = await multipartReader.ReadNextSectionAsync(cancellationToken);
}
// сохраняем все
var addResult = indexToPacketDataItem.Any()
? await this.packetsStore.AddIfNotExistsPacketsPartsAsync(agentId, indexToPacketDataItem.Select(x =>
{
var bytes = indexToPacketBytes[x.Key];
return(this.CreateAddPacketRequest(agentId, x.Value, bytes));
}).ToList())
: AddAddPacketsPartsResult.EmptyResult();
return(new ParseBodyAndSavePacketsResult
{
TransferedPackets = indexToPacketDataItem.Values
.Select(x => {
var fromAddResult = addResult.Results.Single(r => r.Request.PacketId == x.PacketId);
return new TransferedPacketResponse
{
PacketId = x.PacketId,
ProviderKey = x.ProviderKey,
//AgentIdData = agentId,
Result = fromAddResult.Success
? TransferedProcessingResult.Saved
: TransferedProcessingResult.Error,
StorageToken = fromAddResult.StorageToken,
Id = fromAddResult.Id,
};
}).ToList(),
ConfigurationsStats = indexToConfigurationItem.Values,
});
}
else
{
var indexToPacketDataItem = new Dictionary <string, PacketFormDataItem>();
var indexToPacketBytes = new Dictionary <string, byte[]>();
var indexToConfigurationItem = new Dictionary <string, ConfigurationRequestDataItem>();
var agentId = messageHeaders.GetAgentIdData();
foreach (var item in request.Form)
{
var key = item.Key;
var value = item.Value;
var result = GetFormPathDataFromEntry(key);
if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataPacketsProp)
{
var index = result.Parts[1];
if (!indexToPacketDataItem.TryGetValue(index, out var dataItem))
{
dataItem = new PacketFormDataItem();
indexToPacketDataItem.Add(index, dataItem);
}
if (!dataItem.FillProperty(result.Parts[2], value))
{
throw new InvalidDataException($"Incorrect format for form-data message. Section {key} incorrect.");
}
}
if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataConfigurationProp)
{
var index = result.Parts[1];
if (!indexToConfigurationItem.TryGetValue(index, out var dataItem))
{
dataItem = new ConfigurationRequestDataItem();
indexToConfigurationItem.Add(index, dataItem);
}
if (!dataItem.FillProperty(result.Parts[2], value))
{
throw new InvalidDataException($"Incorrect format for form-data message. Section {key} incorrect.");
}
}
}
foreach (var file in request.Form.Files)
{
var pair = indexToPacketDataItem
.FirstOrDefault(x => x.Value.PacketId == file.FileName);
if (default(KeyValuePair <string, PacketFormDataItem>).Equals(pair))
{
var item = pair.Value;
var providerKey = item.ProviderKey;
var packetId = item.PacketId;
using (var fileStream = file.OpenReadStream())
{
var currentBytes = await ReadToEnd(fileStream, fileSectionBufferSize, cancellationToken);
indexToPacketBytes.Add(pair.Key, currentBytes);
}
}
}
// сохраняем все
var addResult = indexToPacketDataItem.Any()
? await this.packetsStore.AddIfNotExistsPacketsPartsAsync(agentId, indexToPacketDataItem.Select(x =>
{
var bytes = indexToPacketBytes[x.Key];
return(this.CreateAddPacketRequest(agentId, x.Value, bytes));
}).ToList())
: AddAddPacketsPartsResult.EmptyResult();
return(new ParseBodyAndSavePacketsResult
{
TransferedPackets = indexToPacketDataItem.Values
.Select(x =>
{
var fromAddResult = addResult.Results
.Single(r => r.Request.PacketId == x.PacketId);
return new TransferedPacketResponse
{
PacketId = x.PacketId,
ProviderKey = x.ProviderKey,
//AgentIdData = agentId,
Result = fromAddResult.Success
? TransferedProcessingResult.Saved
: TransferedProcessingResult.Error,
StorageToken = fromAddResult.StorageToken,
Id = fromAddResult.Id,
};
}).ToList(),
ConfigurationsStats = indexToConfigurationItem.Values,
});
}
}
else
{
throw new NotSupportedException($"format {format} is not supported");
}
}