Ejemplo n.º 1
0
        public async Task <AddAddPacketsPartsResult> AddIfNotExistsPacketsPartsAsync(AgentIdentity agentId, IList <AddPacketPartRequest> requests, CancellationToken cancellationToken = default(CancellationToken))
        {
            if (requests == null)
            {
                throw new ArgumentException("requests");
            }

            if (!requests.Any())
            {
                return(AddAddPacketsPartsResult.EmptyResult());
            }

            if (!this.initialized)
            {
                await this.InitAsync(cancellationToken);
            }

            var stopWatch = Stopwatch.StartNew();

            // IsOrdered = false, неупорядоченные операции - при выполнении отдельных операций они выполняются не упорядочено и они не останавливают исполнение остальных операций
            try
            {
                var docs = requests.Select(x => this.Get(agentId, x)).ToList();
                await this.chunks.InsertManyAsync(docs, new InsertManyOptions()
                {
                    IsOrdered = false
                }, cancellationToken);

                stopWatch.Stop();
                return(AddAddPacketsPartsResult.CreateResult(this.storageToken, requests.Select((request, i) => {
                    return AddAddPacketPartResult.SuccessResult(request, this.storageToken, docs[i].Id);
                })));
            }
            catch (MongoBulkWriteOperationException ex)
            {
                // Instrumentation ???
                var results = new List <AddAddPacketPartResult>();
                for (int i = 0; i < requests.Count; i++)
                {
                    var request           = requests[i];
                    var rqNonDupKeyErrors = ex.WriteErrors.Where(x => x.Index == i && x.Category != ServerErrorCategory.DuplicateKey);
                    if (rqNonDupKeyErrors.Any())
                    {
                        results.Add(AddAddPacketPartResult.FailResult(request));
                    }
                    else
                    {
                        // TODO оптимизировать, не делать запрос на каждый fail вариант
                        var id = await this.chunks.Find(x => x.PacketId == request.PacketId && x.StartPosition == request.StartPosition).Project(x => x.Id).FirstOrDefaultAsync(cancellationToken);

                        if (id == null)
                        {
                            results.Add(AddAddPacketPartResult.FailResult(request));
                        }
                        else
                        {
                            results.Add(AddAddPacketPartResult.SuccessResult(request, this.storageToken, id));
                        }
                    }
                }

                //var failed = new List<AddAddPacketPartResult>();
                //var reqToSelect = new List<AddPacketPartRequest>();
                //for (int i = 0; i < requests.Count; i++)
                //{
                //    var request = requests[i];
                //    var rqNonDupKeyErrors = ex.WriteErrors.Where(x => x.Index == i && x.Category != ServerErrorCategory.DuplicateKey);
                //    if (rqNonDupKeyErrors.Any())
                //    {
                //        failed.Add(AddAddPacketPartResult.FailResult(request));
                //    }
                //    else
                //    {
                //        reqToSelect.Add(request);
                //    }
                //}

                //// select
                //Expression<Func<PacketPart, bool>> filter = null;

                //var f = Builders<PacketPart>.Filter
                //    .Or(reqToSelect.Select(r =>))

                //FilterDefinition<PacketPart> filterDef =
                //var ids = await this.chunks.Find(x => x.PacketId == request.PacketId && x.StartPosition == request.StartPosition).Project(x => x.Id).FirstOrDefaultAsync(cancellationToken);

                stopWatch.Stop();
                return(AddAddPacketsPartsResult.CreateResult(this.storageToken, results));
            }
        }
Ejemplo n.º 2
0
        public async Task <ParseBodyAndSavePacketsResult> ParseBodyAndSavePackets(MessageHeaders messageHeaders, HttpRequest request)
        {
            var cancellationToken = request.HttpContext.RequestAborted;

            cancellationToken.ThrowIfCancellationRequested();

            var    format   = messageHeaders.Format ?? TransportConstants.RequestFormDataFormat;
            string boundary = null;

            if (format == TransportConstants.RequestFormDataFormat)
            {
                int fileSectionBufferSize = 81920;

                // если данные formdata упорядочены в потоке запроса (бинарные данные должны идти после большинства метаданных о пакете)
                if (messageHeaders.Hints?.Contains(MessageHints.OrderedFormData) == true)
                {
                    var indexToPacketDataItem    = new Dictionary <int, PacketFormDataItem>();
                    var indexToPacketBytes       = new Dictionary <int, byte[]>();
                    var indexToConfigurationItem = new Dictionary <int, ConfigurationRequestDataItem>();

                    if (MediaTypeHeaderValue.TryParse(request.ContentType, out var contentType))
                    {
                        boundary = HeaderUtilities.GetBoundary(contentType, 70);
                    }

                    var multipartReader = new MultipartReader(boundary, request.Body)
                    {
                        //ValueCountLimit = _options.ValueCountLimit,
                        //KeyLengthLimit = _options.KeyLengthLimit,
                        //ValueLengthLimit = _options.ValueLengthLimit,

                        HeadersCountLimit  = int.MaxValue,
                        HeadersLengthLimit = int.MaxValue,
                        BodyLengthLimit    = long.MaxValue,
                    };

                    //PacketFormDataItem current = null;
                    //byte[] currentBytes = null;
                    var agentId = messageHeaders.GetAgentIdData();
                    var section = await multipartReader.ReadNextSectionAsync(cancellationToken);

                    while (section != null)
                    {
                        // Parse the content disposition here and pass it further to avoid reparsings
                        if (!ContentDispositionHeaderValue.TryParse(section.ContentDisposition, out var contentDisposition))
                        {
                            throw new InvalidDataException("Form section has invalid Content-Disposition value: " + section.ContentDisposition);
                        }

                        if (contentDisposition.IsFileDisposition())
                        {
                            var fileSection = new FileMultipartSection(section, contentDisposition);

                            var name     = fileSection.Name;
                            var fileName = fileSection.FileName;
                            var packetId = fileSection.FileName;
                            var result   = GetFormPathDataFromEntry(name);
                            if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataPacketsProp)
                            {
                                if (int.TryParse(result.Parts[1], out var index))
                                {
                                    var item        = indexToPacketDataItem[index];
                                    var providerKey = item.ProviderKey;

                                    var packetItem = indexToPacketDataItem[index];
                                    if (packetItem.PacketId != packetId)
                                    {
                                        throw new InvalidDataException($"Incorrect format for form-data message. Section {name} has invalid FileName.");
                                    }

                                    var bytes = await ReadToEnd(fileSection.FileStream, fileSectionBufferSize, cancellationToken);

                                    indexToPacketBytes.Add(index, bytes);
                                }
                                else
                                {
                                    throw new InvalidDataException($"Incorrect format for form-data message. Section {name} does not have index suffix.");
                                }
                            }
                            else
                            {
                                throw new InvalidDataException($"Incorrect format for form-data message. Section {name} incorrect.");
                            }
                        }
                        else if (contentDisposition.IsFormDisposition())
                        {
                            var formDataSection = new FormMultipartSection(section, contentDisposition);

                            // Content-Disposition: form-data; name="key"
                            //
                            // value

                            // Do not limit the key name length here because the multipart headers length limit is already in effect.
                            var key   = formDataSection.Name;
                            var value = await formDataSection.GetValueAsync();

                            var result = GetFormPathDataFromEntry(key);
                            if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataPacketsProp)
                            {
                                if (int.TryParse(result.Parts[1], out var index))
                                {
                                    if (!indexToPacketDataItem.TryGetValue(index, out var dataItem))
                                    {
                                        //// сохраняем предыдущий
                                        //if (current != null)
                                        //{
                                        //    await this.packetsStore.AddIfNotExistsPacketPartAsync(agentId, this.CreateAddPacketRequest(current, currentBytes));
                                        //}

                                        dataItem = new PacketFormDataItem();
                                        indexToPacketDataItem.Add(index, dataItem);
                                    }

                                    if (!dataItem.FillProperty(result.Parts[2], value))
                                    {
                                        throw new InvalidDataException($"Incorrect format for form-data message. Section {key} incorrect.");
                                    }
                                }
                                else
                                {
                                    throw new InvalidDataException($"Incorrect format for form-data message. Section {key} does not have index suffix.");
                                }
                            }
                            else if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataConfigurationProp)
                            {
                                if (int.TryParse(result.Parts[1], out var index))
                                {
                                    if (!indexToConfigurationItem.TryGetValue(index, out var dataItem))
                                    {
                                        dataItem = new ConfigurationRequestDataItem();
                                        indexToConfigurationItem.Add(index, dataItem);
                                    }

                                    if (!dataItem.FillProperty(result.Parts[2], value))
                                    {
                                        throw new InvalidDataException($"Incorrect format for form-data message. Section {key} incorrect.");
                                    }
                                }
                                else
                                {
                                    throw new InvalidDataException($"Incorrect format for form-data message. Section {key} does not have index suffix.");
                                }
                            }
                            else
                            {
                                // ignore or throw?
                            }

                            //if (formAccumulator.ValueCount > _options.ValueCountLimit)
                            //{
                            //    throw new InvalidDataException($"Form value count limit {_options.ValueCountLimit} exceeded.");
                            //}
                        }
                        else
                        {
                            System.Diagnostics.Debug.Assert(false, "Unrecognized content-disposition for this section: " + section.ContentDisposition);
                        }

                        section = await multipartReader.ReadNextSectionAsync(cancellationToken);
                    }

                    // сохраняем все
                    var addResult = indexToPacketDataItem.Any()
                                ? await this.packetsStore.AddIfNotExistsPacketsPartsAsync(agentId, indexToPacketDataItem.Select(x =>
                    {
                        var bytes = indexToPacketBytes[x.Key];
                        return(this.CreateAddPacketRequest(agentId, x.Value, bytes));
                    }).ToList())
                                : AddAddPacketsPartsResult.EmptyResult();

                    return(new ParseBodyAndSavePacketsResult
                    {
                        TransferedPackets = indexToPacketDataItem.Values
                                            .Select(x => {
                            var fromAddResult = addResult.Results.Single(r => r.Request.PacketId == x.PacketId);
                            return new TransferedPacketResponse
                            {
                                PacketId = x.PacketId,
                                ProviderKey = x.ProviderKey,
                                //AgentIdData = agentId,
                                Result = fromAddResult.Success
                                                ? TransferedProcessingResult.Saved
                                                : TransferedProcessingResult.Error,
                                StorageToken = fromAddResult.StorageToken,
                                Id = fromAddResult.Id,
                            };
                        }).ToList(),
                        ConfigurationsStats = indexToConfigurationItem.Values,
                    });
                }
                else
                {
                    var indexToPacketDataItem    = new Dictionary <string, PacketFormDataItem>();
                    var indexToPacketBytes       = new Dictionary <string, byte[]>();
                    var indexToConfigurationItem = new Dictionary <string, ConfigurationRequestDataItem>();
                    var agentId = messageHeaders.GetAgentIdData();

                    foreach (var item in request.Form)
                    {
                        var key   = item.Key;
                        var value = item.Value;

                        var result = GetFormPathDataFromEntry(key);
                        if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataPacketsProp)
                        {
                            var index = result.Parts[1];
                            if (!indexToPacketDataItem.TryGetValue(index, out var dataItem))
                            {
                                dataItem = new PacketFormDataItem();
                                indexToPacketDataItem.Add(index, dataItem);
                            }

                            if (!dataItem.FillProperty(result.Parts[2], value))
                            {
                                throw new InvalidDataException($"Incorrect format for form-data message. Section {key} incorrect.");
                            }
                        }

                        if (result != null && result.Parts.Count == 3 && result.Parts[0] == TransportConstants.FormDataConfigurationProp)
                        {
                            var index = result.Parts[1];
                            if (!indexToConfigurationItem.TryGetValue(index, out var dataItem))
                            {
                                dataItem = new ConfigurationRequestDataItem();
                                indexToConfigurationItem.Add(index, dataItem);
                            }

                            if (!dataItem.FillProperty(result.Parts[2], value))
                            {
                                throw new InvalidDataException($"Incorrect format for form-data message. Section {key} incorrect.");
                            }
                        }
                    }

                    foreach (var file in request.Form.Files)
                    {
                        var pair = indexToPacketDataItem
                                   .FirstOrDefault(x => x.Value.PacketId == file.FileName);

                        if (default(KeyValuePair <string, PacketFormDataItem>).Equals(pair))
                        {
                            var item        = pair.Value;
                            var providerKey = item.ProviderKey;
                            var packetId    = item.PacketId;

                            using (var fileStream = file.OpenReadStream())
                            {
                                var currentBytes = await ReadToEnd(fileStream, fileSectionBufferSize, cancellationToken);

                                indexToPacketBytes.Add(pair.Key, currentBytes);
                            }
                        }
                    }

                    // сохраняем все
                    var addResult = indexToPacketDataItem.Any()
                                ? await this.packetsStore.AddIfNotExistsPacketsPartsAsync(agentId, indexToPacketDataItem.Select(x =>
                    {
                        var bytes = indexToPacketBytes[x.Key];
                        return(this.CreateAddPacketRequest(agentId, x.Value, bytes));
                    }).ToList())
                                : AddAddPacketsPartsResult.EmptyResult();

                    return(new ParseBodyAndSavePacketsResult
                    {
                        TransferedPackets = indexToPacketDataItem.Values
                                            .Select(x =>
                        {
                            var fromAddResult = addResult.Results
                                                .Single(r => r.Request.PacketId == x.PacketId);
                            return new TransferedPacketResponse
                            {
                                PacketId = x.PacketId,
                                ProviderKey = x.ProviderKey,
                                //AgentIdData = agentId,
                                Result = fromAddResult.Success
                                                                    ? TransferedProcessingResult.Saved
                                                                    : TransferedProcessingResult.Error,
                                StorageToken = fromAddResult.StorageToken,
                                Id = fromAddResult.Id,
                            };
                        }).ToList(),
                        ConfigurationsStats = indexToConfigurationItem.Values,
                    });
                }
            }
            else
            {
                throw new NotSupportedException($"format {format} is not supported");
            }
        }