Пример #1
0
 private static void PrintBulkCommand(string bulkCommand, string bulkJson, BulkResult bulkResult, string ObjectName)
 {
     Console.WriteLine("Parsed Bulk Results for " + ObjectName + " \r\n *************************************************************************************");
     foreach (var item in bulkResult.items)
     {
         Console.WriteLine(" operation: " + item.ResultType);
         Console.WriteLine(" _index: " + item.Result._index);
         Console.WriteLine(" _type: " + item.Result._type);
         Console.WriteLine(" _id: " + item.Result._id);
         Console.WriteLine();
     }
 }
Пример #2
0
        public static void BulkType(List <Bic_Iban_Codes> ModelData, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index)
        {
            string bulkCommand = new BulkCommand(index: _index, type: "iban_bic").Refresh();

            string bulkJson = new BulkBuilder(serializer)
                              .BuildCollection(ModelData, (builder, pro) => builder.Index(data: pro, id: pro.CodeID.ToString())
                                               );

            string     result     = connection.Post(bulkCommand, bulkJson);
            BulkResult bulkResult = serializer.ToBulkResult(result);

            connection.Post(_index + "/_refresh");
        }
Пример #3
0
        public static void BulkType <T>(List <T> Model, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index, string _type) where T : IElasticMapper
        {
            string bulkCommand = "";

            bulkCommand = new BulkCommand(index: _index, type: _type).Refresh();
            string bulkJson = new BulkBuilder(serializer)
                              .BuildCollection(Model, (builder, pro) => builder.Index(data: pro, id: pro.id)
                                               );
            string     result     = connection.Post(bulkCommand, bulkJson);
            BulkResult bulkResult = serializer.ToBulkResult(result);

            connection.Post(_index + "/_refresh");
        }
Пример #4
0
        private static void PrintBulkCommand(string bulkCommand, string bulkJson, BulkResult bulkResult)
        {
            Console.WriteLine("Executed: \r\nPOST {0} \r\n{1} \r\n".F(bulkCommand, bulkJson));

            Console.WriteLine("Parsed Bulk Results");

            foreach (var item in bulkResult.items)
            {
                Console.WriteLine(" operation: " + item.ResultType);
                Console.WriteLine("     _index: " + item.Result._index);
                Console.WriteLine("     _type: " + item.Result._type);
                Console.WriteLine("     _id: " + item.Result._id);
                Console.WriteLine();
            }
        }
Пример #5
0
        private static void BulkTweetIndex(IEnumerable <Tweet> tweets, ElasticConnection connection, JsonNetSerializer serializer)
        {
            string bulkCommand = new BulkCommand(index: "twitter", type: "tweet").Refresh();

            int    id       = 10; // start adding tweets from id = 10
            string bulkJson = new BulkBuilder(serializer)
                              .BuildCollection(tweets,
                                               (builder, tweet) => builder.Index(data: tweet, id: (id++).AsString())
                                               );

            string result = connection.Post(bulkCommand, bulkJson);

            //Parse bulk result;
            BulkResult bulkResult = serializer.ToBulkResult(result);

            PrintBulkCommand(bulkCommand, bulkJson, bulkResult);
        }
Пример #6
0
        public static List <BulkResult> ToBulkResult(this NestBulkResponse response)
        {
            var result = new List <BulkResult>();

            if (response.Items != null)
            {
                result.AddRange(response.Items.Where(i => i.IsValid)
                                .Select(item => BulkResult.Create(item.Id, StatusCodes.Status200OK, string.Empty)));
            }

            if (response.ItemsWithErrors != null)
            {
                result.AddRange(response.ItemsWithErrors
                                .Select(item => BulkResult.Create(item.Id, StatusCodes.Status406NotAcceptable, item.Error.Reason)));
            }

            return(result);
        }
Пример #7
0
        public BulkResults Bulk(string dataSetName, IEnumerable <object> documents, long requestSize, int parallelLimit)
        {
            var dataSet            = DataSet(dataSetName).DataSet;
            var results            = new BulkResults();
            var validatedDocuments = documents
                                     .Select((document, index) =>
                                             new
            {
                Index    = index,
                Result   = ValidateDocument(dataSetName, document),
                Document = document
            })
                                     .ToList();

            var invalidDocumentResults = validatedDocuments
                                         .Where(document => document.Result.IsFailure)
                                         .Select(document => BulkResult.Create(
                                                     DocumentHelper.GetValue(document.Document, dataSet.IdField)?.ToString(),
                                                     StatusCodes.Status400BadRequest,
                                                     document.Result.Error)
                                                 )
                                         .ToList();

            results.Results.AddRange(invalidDocumentResults);

            var documentElastics = validatedDocuments
                                   .Where(obj => obj.Result.IsSuccess)
                                   .Select(document => new DocumentElastic
            {
                Id             = DocumentHelper.GetValue(document.Document, dataSet.IdField).ToString(),
                DocumentObject = document.Document,
                Text           = DocumentHelper.GetConcatenatedText(document.Document, dataSet.InterpretedFields)
            })
                                   .ToList();

            var bulkResponseStruct = DocumentQuery(dataSetName).ParallelBulkIndex(documentElastics, parallelLimit, requestSize);

            results.Results.AddRange(bulkResponseStruct.ToBulkResult());

            return(results);
        }
        /// <summary>
        /// Parse the JSON and return the bulk request.
        /// </summary>
        /// <exception cref="ArgumentNullException">Thrown when a REQUIRED parameter is null or empty.</exception>
        /// <exception cref="FormatException">Thrown when the 'baseUrlPattern' parameter  is not correctly formatted.</exception>
        /// <param name="jObj">JSON that will be parsed.</param>
        /// <param name="baseUrlPattern">Base url pattern.</param>
        /// <returns>Bulk request or null.</returns>
        public async Task <BulkRequestResponse> Parse(JObject jObj, string baseUrlPattern)
        {
            // 1. Check parameters.
            if (jObj == null)
            {
                throw new ArgumentNullException(nameof(jObj));
            }

            if (string.IsNullOrWhiteSpace(baseUrlPattern))
            {
                throw new ArgumentNullException(nameof(baseUrlPattern));
            }

            if (!baseUrlPattern.Contains("{rootPath}"))
            {
                throw new FormatException("the baseUrlPattern is not correctly formatted");
            }

            // 2. Parse the request.
            var obj = jObj.ToObject <BulkRequest>();

            if (!obj.Schemas.Contains(Common.Constants.Messages.Bulk))
            {
                return(new BulkRequestResponse
                {
                    IsParsed = false,
                    ErrorResponse = _errorResponseFactory.CreateError(
                        ErrorMessages.TheRequestIsNotABulkOperation,
                        HttpStatusCode.BadRequest,
                        Common.Constants.ScimTypeValues.InvalidSyntax)
                });
            }

            if (obj.Operations == null)
            {
                return(new BulkRequestResponse
                {
                    IsParsed = false,
                    ErrorResponse = _errorResponseFactory.CreateError(
                        ErrorMessages.TheOperationsParameterMustBeSpecified,
                        HttpStatusCode.BadRequest,
                        Common.Constants.ScimTypeValues.InvalidSyntax)
                });
            }

            var response = new BulkResult
            {
                FailOnErrors = obj.FailOnErrors
            };

            Func <string, ErrorResponse> getBulkMethodNotSupported = (method) =>
            {
                return(_errorResponseFactory.CreateError(
                           string.Format(ErrorMessages.TheBulkMethodIsNotSupported, method),
                           HttpStatusCode.BadRequest,
                           Common.Constants.ScimTypeValues.InvalidSyntax));
            };
            Func <string, IList <string> > splitPath = (path) =>
            {
                if (string.IsNullOrWhiteSpace(path))
                {
                    return(null);
                }

                path = path.TrimStart('/');
                var subPaths = path.Split('/');
                if (!subPaths.Any() || subPaths.Count() > 2)
                {
                    return(null);
                }

                return(subPaths);
            };
            Func <IList <string>, string> extractRootPath = (subPaths) =>
            {
                if (subPaths == null)
                {
                    return(null);
                }

                return(subPaths[0]);
            };
            Func <IList <string>, string> extractId = (subPaths) =>
            {
                if (subPaths == null || subPaths.Count() < 2)
                {
                    return(null);
                }

                return(subPaths[1]);
            };
            Func <string, string> getResourceType = (subPath) =>
            {
                if (subPath == null)
                {
                    return(null);
                }

                if (!Constants.MappingRoutePathsToResourceTypes.ContainsKey(subPath))
                {
                    return(null);
                }

                return(Constants.MappingRoutePathsToResourceTypes[subPath]);
            };

            var schemas = await _schemaStore.GetSchemas();

            var resourceTypes = schemas.Select(s => s.Name);

            // 3. Check operation parameters are correct.
            var operations = new List <BulkOperationResult>();

            foreach (var operation in obj.Operations)
            {
                try
                {
                    JObject data = operation.Data as JObject;
                    // 3.1. Check data
                    if (data == null)
                    {
                        return(new BulkRequestResponse
                        {
                            IsParsed = false,
                            ErrorResponse = _errorResponseFactory.CreateError(
                                ErrorMessages.TheBulkDataParameterMustBeSpecified,
                                HttpStatusCode.BadRequest,
                                Common.Constants.ScimTypeValues.InvalidSyntax)
                        });
                    }

                    // 3.2. Check method
                    var httpMethod = new HttpMethod(operation.Method);
                    if (!new [] { HttpMethod.Post, HttpMethod.Put, HttpMethod.Delete, new HttpMethod("PATCH") }.Contains(httpMethod))
                    {
                        return(new BulkRequestResponse
                        {
                            IsParsed = false,
                            ErrorResponse = getBulkMethodNotSupported(operation.Method)
                        });
                    }

                    // 3.3. Check path
                    if (string.IsNullOrWhiteSpace(operation.Path))
                    {
                        return(new BulkRequestResponse
                        {
                            IsParsed = false,
                            ErrorResponse = _errorResponseFactory.CreateError(
                                ErrorMessages.TheBulkOperationPathIsRequired,
                                HttpStatusCode.BadRequest,
                                Common.Constants.ScimTypeValues.InvalidSyntax)
                        });
                    }

                    var subPaths     = splitPath(operation.Path);
                    var rootPath     = extractRootPath(subPaths);
                    var resourceId   = extractId(subPaths);
                    var resourceType = getResourceType(rootPath);
                    if (string.IsNullOrWhiteSpace(resourceType) ||
                        !resourceTypes.Contains(resourceType) ||
                        (httpMethod != HttpMethod.Post && string.IsNullOrWhiteSpace(resourceId)))
                    {
                        return(new BulkRequestResponse
                        {
                            IsParsed = false,
                            ErrorResponse = _errorResponseFactory.CreateError(
                                string.Format(ErrorMessages.TheBulkOperationPathIsNotSupported, operation.Path),
                                HttpStatusCode.BadRequest,
                                Common.Constants.ScimTypeValues.InvalidSyntax)
                        });
                    }

                    // 3.4. Check bulkId
                    if (httpMethod == HttpMethod.Post && string.IsNullOrWhiteSpace(operation.BulkId))
                    {
                        return(new BulkRequestResponse
                        {
                            IsParsed = false,
                            ErrorResponse = _errorResponseFactory.CreateError(
                                ErrorMessages.TheBulkIdParameterMustBeSpecified,
                                HttpStatusCode.BadRequest,
                                Common.Constants.ScimTypeValues.InvalidSyntax)
                        });
                    }


                    var schema = schemas.First(s => s.Name == resourceType);
                    operations.Add(new BulkOperationResult
                    {
                        Data            = data,
                        BulkId          = operation.BulkId,
                        Method          = httpMethod,
                        Version         = operation.Version,
                        ResourceId      = resourceId,
                        SchemaId        = schema.Id,
                        ResourceType    = resourceType,
                        LocationPattern = baseUrlPattern.Replace("{rootPath}", rootPath) + "/{id}",
                        Path            = operation.Path
                    });
                }
                catch (Exception)
                {
                    return(new BulkRequestResponse
                    {
                        IsParsed = false,
                        ErrorResponse = getBulkMethodNotSupported(operation.Method)
                    });
                }
            }

            response.Operations = operations;
            return(new BulkRequestResponse
            {
                IsParsed = true,
                BulkResult = response
            });
        }
Пример #9
0
        /// <summary>
        /// Bulk tag creation.
        /// </summary>
        /// <param name="settings">The settings.</param>
        /// <returns></returns>
        public Result <BulkResults> BulkCreate(string dataSetName, List <Tag> tags, int parallelLimit, long requestSize)
        {
            var results = new BulkResults();

            var tagIsInteger = DataSet(dataSetName).TagIsInteger;

            if (tagIsInteger && tags.Any(tag => tag.Id.Any(ch => !char.IsDigit(ch))))
            {
                results.Results = tags
                                  .Where(tag => tag.Id.Any(ch => !char.IsDigit(ch)))
                                  .Select(t => BulkResult.Create(
                                              t.Id,
                                              StatusCodes.Status400BadRequest,
                                              string.Format(TagResources.TagIdShouldBeIntegerType, t.ParentId, t.Id)))
                                  .ToList();

                return(Result.Ok(results));
            }

            var tagIdsByLevel = TagHelper.GetTagIdsByLevel(tags, item => item.ParentId, item => item.Id);
            var validIds      = tagIdsByLevel.SelectMany(l => l.Value).ToList();
            var invalidIds    = tags.Select(t => t.Id).Except(validIds);

            if (invalidIds.Any())
            {
                results.Results = tags
                                  .Where(t => invalidIds.Contains(t.Id))
                                  .Select(t => BulkResult.Create(t.Id, StatusCodes.Status404NotFound, string.Format(TagResources.ParentId_0_NotFoundInTagWithId_1, t.ParentId, t.Id))).ToList();

                // returns with OK status, individual items contain error code
                return(Result.Ok(results));
            }

            var orderedTagElasticList = tagIdsByLevel
                                        .SelectMany(dic => dic.Value)
                                        .Select(id =>
            {
                var tag        = tags.FirstOrDefault(t => t.Id == id);
                var tagElastic = new TagElastic
                {
                    Id           = tag.Id,
                    Name         = tag.Name,
                    ParentIdList = new List <string>()
                };
                if (!string.IsNullOrWhiteSpace(tag.ParentId))
                {
                    tagElastic.ParentIdList.Add(tag.ParentId);
                }
                return(tagElastic);
            })
                                        .ToList();

            TagHelper.AdjustTagElastics(orderedTagElasticList);

            var tagQuery = TagQuery(dataSetName);

            tagQuery.DeleteAll();

            var bulkResponseStruct = tagQuery.ParallelBulkIndex(orderedTagElasticList, parallelLimit, requestSize);

            results.Results.AddRange(bulkResponseStruct.ToBulkResult());

            return(Result.Ok(results));
        }
Пример #10
0
        public async Task <int> LoadFile(
            IEnumerable <FileInfo> files,
            string connectionName,
            ConnectionType connectionType,
            int skip             = 0,
            int take             = 0,
            int maxThreads       = 8,
            CancellationToken ct = default)
        {
            using (var results = new BlockingCollection <IOperationResult>(new ConcurrentQueue <IOperationResult>()))
            {
                var bulkTimer = Stopwatch.StartNew();
                var exec      = await _connectionManager.Open(connectionName, connectionType, ct);

                if (!await exec.TestConnection(ct))
                {
                    _console.WriteLine($"Unable to connect.");
                    return(1);
                }

                var loadTasks = new List <Task>();

                var reporter = Task.Run(() =>
                {
                    while (results.TryTake(out var result, -1))
                    {
                        _console.WriteLine(result.Message);
                    }
                });

                foreach (var file in files)
                {
                    var fileTimer = Stopwatch.StartNew();

                    _console.WriteLine($"Executing queries from {file} on {connectionName} using {connectionType} using {maxThreads} threads.");

                    IEnumerable <string> queries = await File.ReadAllLinesAsync(file.FullName);

                    if (skip > 0)
                    {
                        _console.WriteLine($"skipping {skip} lines.");
                        queries = queries.Skip(skip);
                    }
                    if (take > 0)
                    {
                        _console.WriteLine($"taking {take} lines.");
                        queries = queries.Take(take);
                    }

                    // var lineNumber = skip + 1;
                    var queriesProcessed = 0;
                    var totalQueries     = queries.Count();

                    // Parallel.ForEach(queries, async (queries,loop, i) => )

                    // this looks like it might be better
                    //https://gist.github.com/0xced/94f6c50d620e582e19913742dbd76eb6


                    IProgress <IOperationResult> progress = new Progress <IOperationResult>(
                        op => {
                        queriesProcessed++;
                        _console.WriteLine(op.Message);
                    }
                        );

                    // Parallel.ForEach(Partitioner.Create(queries), () => 0, async (q, loopState, partitionCount) =>
                    // {
                    //     partitionCount ++;
                    //     var result = new BulkResult<dynamic>(
                    //         await exec.ExecuteQuery<dynamic>(q, ct),
                    //         Thread.CurrentThread.ManagedThreadId, file.Name,  skip, queriesProcessed, totalQueries, fileTimer.Elapsed, bulkTimer.Elapsed);
                    //     return partitionCount;
                    // },
                    // (partitionCount) => Interlocked.Add(ref queriesProcessed, partitionCount)
                    // );


                    // await queries.ForEachAsync(maxThreads, async (q) => {
                    //     var result = new BulkResult<dynamic>(
                    //         await exec.ExecuteQuery<dynamic>(q, ct),
                    //         Thread.CurrentThread.ManagedThreadId, file.Name, 0 + skip, queriesProcessed, totalQueries, fileTimer.Elapsed, bulkTimer.Elapsed);
                    //     //results.Add(result);
                    //     return result;
                    // }, progress);

                    // Partitioner.Create(queries).GetPartitions(maxThreads)
                    //     .Select(

                    await queries.ParallelForEachAsync(
                        async (q, queryNumber) =>
                    {
                        Interlocked.Increment(ref queriesProcessed);

                        IOperationResult result;

                        try
                        {
                            var qResult = await exec.ExecuteQuery <dynamic>(q, ct);

                            result = new BulkResult <dynamic>(
                                qResult,
                                Thread.CurrentThread.ManagedThreadId,
                                file.Name,
                                queryNumber + skip,
                                queriesProcessed,
                                totalQueries,
                                fileTimer.Elapsed,
                                bulkTimer.Elapsed);
                        }
                        catch (System.Exception ex)
                        {
                            result = new ErrorResult(ex);
                        }

                        results.Add(result);
                    }, maxDegreeOfParallelism : maxThreads,
                        cancellationToken : ct
                        );
                }

                results.CompleteAdding();

                await reporter;
            }
            return(0);
        }