public bool DeleteBulk(IEnumerable <T> list) { try { string cmd = CreateBulkCommand(); BulkBuilder builder = new BulkBuilder(serializer); StringBuilder str = new StringBuilder(); OperationResult result; foreach (var item in list) { str.Append(builder.Delete(item.id, _indexInfo.IndexName, _indexInfo.IndexType)); } //最后一组 if (str.Length > 0) { result = Client.Post(cmd, str.ToString()); } return(true); } catch (Exception ex) { ESLog.WriteLogException(MethodBase.GetCurrentMethod(), ex); return(false); } }
public static void BulkType(List <Bic_Iban_Codes> ModelData, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index) { string bulkCommand = new BulkCommand(index: _index, type: "iban_bic").Refresh(); string bulkJson = new BulkBuilder(serializer) .BuildCollection(ModelData, (builder, pro) => builder.Index(data: pro, id: pro.CodeID.ToString()) ); string result = connection.Post(bulkCommand, bulkJson); BulkResult bulkResult = serializer.ToBulkResult(result); connection.Post(_index + "/_refresh"); }
public static void BulkType <T>(List <T> Model, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index, string _type) where T : IElasticMapper { string bulkCommand = ""; bulkCommand = new BulkCommand(index: _index, type: _type).Refresh(); string bulkJson = new BulkBuilder(serializer) .BuildCollection(Model, (builder, pro) => builder.Index(data: pro, id: pro.id) ); string result = connection.Post(bulkCommand, bulkJson); BulkResult bulkResult = serializer.ToBulkResult(result); connection.Post(_index + "/_refresh"); }
/// <summary> /// 批量创建 1000-5000个比较好 /// </summary> /// <param name="list"></param> /// <returns></returns> public string CreateBulk(IEnumerable <T> list) { string cmd = CreateBulkCommand(); BulkBuilder builder = new BulkBuilder(serializer); StringBuilder str = new StringBuilder(); foreach (var item in list) { str.Append(builder.Create(item, _indexInfo.IndexName, _indexInfo.IndexType, item.id)); } OperationResult result = Client.Post(cmd, str.ToString()); return(result); }
private static void BulkTweetIndex(IEnumerable <Tweet> tweets, ElasticConnection connection, JsonNetSerializer serializer) { string bulkCommand = new BulkCommand(index: "twitter", type: "tweet").Refresh(); int id = 10; // start adding tweets from id = 10 string bulkJson = new BulkBuilder(serializer) .BuildCollection(tweets, (builder, tweet) => builder.Index(data: tweet, id: (id++).AsString()) ); string result = connection.Post(bulkCommand, bulkJson); //Parse bulk result; BulkResult bulkResult = serializer.ToBulkResult(result); PrintBulkCommand(bulkCommand, bulkJson, bulkResult); }
public BulkResult IndexBulk(BulkCommand bulkCommand, IEnumerable <ESDocument> documents = null) { string bulkJson = new BulkBuilder(serializer) .BuildCollection(documents, (builder, document) => builder.Index(data: document, id: document.Id.ToString()) // You can apply any custom logic here // to generate Indexes, Creates or Deletes. ); string result = connection.Post(bulkCommand, bulkJson); //Parse bulk result; var bulkResult = serializer.ToBulkResult(result); return(bulkResult); }
private static void BulkTweetIndex(IEnumerable<Tweet> tweets, ElasticConnection connection, JsonNetSerializer serializer) { string bulkCommand = new BulkCommand(index: "twitter", type: "tweet").Refresh(); int id = 10; // start adding tweets from id = 10 string bulkJson = new BulkBuilder(serializer) .BuildCollection(tweets, (builder, tweet) => builder.Index(data: tweet, id: (id++).AsString()) ); string result = connection.Post(bulkCommand, bulkJson); //Parse bulk result; BulkResult bulkResult = serializer.ToBulkResult(result); PrintBulkCommand(bulkCommand, bulkJson, bulkResult); }
public override void Import(DateTime @from) { var logQuery = new LogQueryClass(); var inputFormat = new COMW3CInputContextClass(); string strQuery = string.Format(@"SELECT to_timestamp(date, time) as date, s-ip as sourceIP, cs-method as method, cs-uri-stem as uri, cs-uri-query as query, s-port as port, c-ip as clientIP, cs(User-Agent) as userAgent, cs-host as clientToServerHost, sc-status as statusCode, sc-substatus as subStatus, sc-win32-status as win32Status, sc-bytes as serverToClientBytes, cs-bytes as clientToServerBytes, time-taken as duration FROM {0}", "SOME FILE"); ILogRecordset results = logQuery.Execute(strQuery, inputFormat); var tweets = new List<Entry>(); while (!results.atEnd()) { ILogRecord logRecord = results.getRecord(); dynamic date = logRecord.getValue("date"); dynamic sourceIP = logRecord.getValue("sourceIP"); dynamic method = logRecord.getValue("method"); dynamic uri = logRecord.getValue("uri"); dynamic query = logRecord.getValue("query") is DBNull ? string.Empty : logRecord.getValue("query"); dynamic port = logRecord.getValue("port"); dynamic clientIP = logRecord.getValue("clientIP") is DBNull ? string.Empty : logRecord.getValue("clientIP"); dynamic userAgent = logRecord.getValue("userAgent") is DBNull ? string.Empty : logRecord.getValue("userAgent"); dynamic clientToServerHost = logRecord.getValue("clientToServerHost") is DBNull ? string.Empty : logRecord.getValue("clientToServerHost"); dynamic statusCode = logRecord.getValue("statusCode"); dynamic subStatus = logRecord.getValue("subStatus"); dynamic win32Status = logRecord.getValue("win32Status"); dynamic serverToClientBytes = logRecord.getValue("serverToClientBytes"); dynamic clientToServerBytes = logRecord.getValue("clientToServerBytes"); dynamic duration = logRecord.getValue("duration"); tweets.Add(new Entry { Date = date, SourceIP = sourceIP, Method = method, Uri = uri, Query = query, Port = port, ClientIP = clientIP, UserAgent = userAgent, ClientToServerHost = clientToServerHost, StatusCode = statusCode, SubStatus = subStatus, Win32Status = win32Status, ServerToClientBytes = serverToClientBytes, ClientToServerBytes = clientToServerBytes, Duration = duration }); results.moveNext(); } var serializer = new JsonNetSerializer(); string bulkCommand = new BulkCommand(index: "log", type: "iis"); string bulkJson = new BulkBuilder(serializer) .BuildCollection(tweets, (builder, tweet) => builder.Create(tweet) ); _connection.Post(bulkCommand, bulkJson); }
/// <summary> /// Exports the specified entry. /// </summary> /// <param name="entries">The entry.</param> internal void Export(IList<Entry> entries) { if (entries.Any() == false) return; // setup variable to group all error messages together var errorBuilder = new StringBuilder(); // Build a batch based update to avoid memory overhead var bulkCommand = new BulkCommand(GetIndex(), LOG_TYPE); var bulkJsons = new BulkBuilder(serializer) .PipelineCollection(entries, (builder, entity) => builder.Index(entity, id: entity.Id.ToString())) .JoinInBatches(batchSize: 1000); foreach (var bulk in bulkJsons) { string result = connection.Put(bulkCommand, bulk); var bulkResult = serializer.ToBulkResult(result); // Check for errors foreach (var operation in bulkResult.items.Where(a => a.Result.ok == false)) { errorBuilder.AppendFormat("Id: {0} Error: {1} {2}", operation.Result._id, operation.Result.error, Environment.NewLine); } } // Check for any errors that are reported by the ElasticSearch var error = errorBuilder.ToString(); if (string.IsNullOrWhiteSpace(error) == false) throw new ApplicationException(error); }
public override void Import(DateTime @from) { var logQuery = new LogQueryClass(); var inputFormat = new COMW3CInputContextClass(); string strQuery = string.Format(@"SELECT to_timestamp(date, time) as date, s-ip as sourceIP, cs-method as method, cs-uri-stem as uri, cs-uri-query as query, s-port as port, c-ip as clientIP, cs(User-Agent) as userAgent, cs-host as clientToServerHost, sc-status as statusCode, sc-substatus as subStatus, sc-win32-status as win32Status, sc-bytes as serverToClientBytes, cs-bytes as clientToServerBytes, time-taken as duration FROM {0}", "SOME FILE"); ILogRecordset results = logQuery.Execute(strQuery, inputFormat); var tweets = new List <Entry>(); while (!results.atEnd()) { ILogRecord logRecord = results.getRecord(); dynamic date = logRecord.getValue("date"); dynamic sourceIP = logRecord.getValue("sourceIP"); dynamic method = logRecord.getValue("method"); dynamic uri = logRecord.getValue("uri"); dynamic query = logRecord.getValue("query") is DBNull ? string.Empty : logRecord.getValue("query"); dynamic port = logRecord.getValue("port"); dynamic clientIP = logRecord.getValue("clientIP") is DBNull ? string.Empty : logRecord.getValue("clientIP"); dynamic userAgent = logRecord.getValue("userAgent") is DBNull ? string.Empty : logRecord.getValue("userAgent"); dynamic clientToServerHost = logRecord.getValue("clientToServerHost") is DBNull ? string.Empty : logRecord.getValue("clientToServerHost"); dynamic statusCode = logRecord.getValue("statusCode"); dynamic subStatus = logRecord.getValue("subStatus"); dynamic win32Status = logRecord.getValue("win32Status"); dynamic serverToClientBytes = logRecord.getValue("serverToClientBytes"); dynamic clientToServerBytes = logRecord.getValue("clientToServerBytes"); dynamic duration = logRecord.getValue("duration"); tweets.Add(new Entry { Date = date, SourceIP = sourceIP, Method = method, Uri = uri, Query = query, Port = port, ClientIP = clientIP, UserAgent = userAgent, ClientToServerHost = clientToServerHost, StatusCode = statusCode, SubStatus = subStatus, Win32Status = win32Status, ServerToClientBytes = serverToClientBytes, ClientToServerBytes = clientToServerBytes, Duration = duration }); results.moveNext(); } var serializer = new JsonNetSerializer(); string bulkCommand = new BulkCommand(index: "log", type: "iis"); string bulkJson = new BulkBuilder(serializer) .BuildCollection(tweets, (builder, tweet) => builder.Create(tweet) ); _connection.Post(bulkCommand, bulkJson); }