public ViewsFixture()
{
Artists = TestData.Artists.CreateArtists(10);
_client = IntegrationTestsRuntime.CreateClient();
var bulk = new BulkCommand();
bulk.Include(Artists.Select(i => _client.Entities.Serializer.Serialize(i)).ToArray());
var bulkResponse = _client.Documents.BulkAsync(bulk);
foreach (var row in bulkResponse.Result.Rows)
{
var artist = Artists.Single(i => i.ArtistId == row.Id);
_client.Entities.Reflector.RevMember.SetValueTo(artist, row.Rev);
}
_client.Documents.PostAsync(TestData.Views.Artists).Wait();
var touchView1 = new ViewQuery(TestData.Views.ArtistsAlbumsViewId).Configure(q => q.Stale(Stale.UpdateAfter));
var touchView2 = new ViewQuery(TestData.Views.ArtistsNameNoValueViewId).Configure(q => q.Stale(Stale.UpdateAfter));
_client.Views.RunQueryAsync(touchView1).Wait();
_client.Views.RunQueryAsync(touchView2).Wait();
}
public BulkResult Index(IEnumerable objectsToIndex, bool deleteLanguageRoutingDuplicatesOnIndex)
{
objectsToIndex.ValidateNotNullOrEmptyArgument(nameof(objectsToIndex));
List <BulkAction> bulkActionList = new List <BulkAction>();
foreach (object obj in objectsToIndex)
{
string id = this.Conventions.IdConvention.GetId(obj);
if (deleteLanguageRoutingDuplicatesOnIndex)
{
BulkDeleteAction bulkDeleteAction1 = new BulkDeleteAction(this.DefaultIndex, this.GetTypeName(obj), id);
bulkDeleteAction1.ActionAndMeta.LanguageRouting = (LanguageRouting)null;
BulkDeleteAction bulkDeleteAction2 = bulkDeleteAction1;
bulkActionList.Add((BulkAction)bulkDeleteAction2);
}
BulkIndexAction bulkIndexAction = new BulkIndexAction((IndexName)this.DefaultIndex, (TypeName)this.GetTypeName(obj), (DocumentId)id, obj);
bulkIndexAction.ActionAndMeta.TimeToLive = this.Conventions.TimeToLiveConvention.GetTimeToLive(obj);
if (this.Conventions.LanguageRoutingConvention.HasLanguageRouting(obj))
{
LanguageRouting languageRouting = this.Conventions.LanguageRoutingConvention.GetLanguageRouting(obj);
if (languageRouting == null)
{
throw new ArgumentException(string.Format("Language missing for the object with id: {0}.", (object)id));
}
bulkIndexAction.ActionAndMeta.LanguageRouting = this.GetSupportedLanguageRoutingOrDefault(languageRouting);
}
bulkActionList.Add((BulkAction)bulkIndexAction);
}
BulkCommand bulkCommand = this.commands.Bulk((IEnumerable <BulkAction>)bulkActionList);
this.PrepareSerializerUsingConventions(bulkCommand.CommandContext.Serializer);
return(bulkCommand.Execute());
}
protected virtual HttpRequestMessage CreateRequest(BulkCommand cmd)
{
var req = new HttpRequest(HttpMethod.Post, GenerateRequestUrl(cmd));
req.SetContent(cmd.ToJson());
return(req);
}
public virtual async Task<BulkResponse> BulkAsync(BulkCommand cmd)
{
Ensure.That(cmd, "cmd").IsNotNull();
var req = CreateRequest(cmd);
var res = SendAsync(req);
return ProcessBulkResponse(await res.ForAwait());
}
public virtual async Task <BulkResponse> BulkAsync(BulkCommand cmd)
{
Ensure.That(cmd, "cmd").IsNotNull();
var req = CreateRequest(cmd);
var res = SendAsync(req);
return(ProcessBulkResponse(await res.ForAwait()));
}
public static void BulkType <T>(List <T> Model, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index, string _type) where T : IElasticMapper
{
string bulkCommand = "";
bulkCommand = new BulkCommand(index: _index, type: _type).Refresh();
string bulkJson = new BulkBuilder(serializer)
.BuildCollection(Model, (builder, pro) => builder.Index(data: pro, id: pro.id)
);
string result = connection.Post(bulkCommand, bulkJson);
BulkResult bulkResult = serializer.ToBulkResult(result);
connection.Post(_index + "/_refresh");
}
internal static void ClearAllDocuments()
{
var query = new SystemViewQuery("_all_docs");
var response = Client.Views.RunQuery<dynamic>(query);
if (!response.IsEmpty)
{
var bulkCmd = new BulkCommand();
foreach (var row in response.Rows)
bulkCmd.Delete(row.Id, row.Value.rev.ToString());
Client.Documents.Bulk(bulkCmd);
}
}
public static void BulkType(List <Bic_Iban_Codes> ModelData, ElasticConnection connection, PlainElastic.Net.Serialization.JsonNetSerializer serializer, string _index)
{
string bulkCommand = new BulkCommand(index: _index, type: "iban_bic").Refresh();
string bulkJson = new BulkBuilder(serializer)
.BuildCollection(ModelData, (builder, pro) => builder.Index(data: pro, id: pro.CodeID.ToString())
);
string result = connection.Post(bulkCommand, bulkJson);
BulkResult bulkResult = serializer.ToBulkResult(result);
connection.Post(_index + "/_refresh");
}
public BulkResult IndexBulk(BulkCommand bulkCommand, IEnumerable <ESDocument> documents = null)
{
string bulkJson =
new BulkBuilder(serializer)
.BuildCollection(documents,
(builder, document) => builder.Index(data: document, id: document.Id.ToString())
// You can apply any custom logic here
// to generate Indexes, Creates or Deletes.
);
string result = connection.Post(bulkCommand, bulkJson);
//Parse bulk result;
var bulkResult = serializer.ToBulkResult(result);
return(bulkResult);
}
private static void BulkTweetIndex(IEnumerable <Tweet> tweets, ElasticConnection connection, JsonNetSerializer serializer)
{
string bulkCommand = new BulkCommand(index: "twitter", type: "tweet").Refresh();
int id = 10; // start adding tweets from id = 10
string bulkJson = new BulkBuilder(serializer)
.BuildCollection(tweets,
(builder, tweet) => builder.Index(data: tweet, id: (id++).AsString())
);
string result = connection.Post(bulkCommand, bulkJson);
//Parse bulk result;
BulkResult bulkResult = serializer.ToBulkResult(result);
PrintBulkCommand(bulkCommand, bulkJson, bulkResult);
}
internal static void ClearAllDocuments(this IClient client)
{
var query = new SystemViewQuery("_all_docs");
var response = client.Views.RunQueryAsync <dynamic>(query).Result;
if (!response.IsEmpty)
{
var bulkCmd = new BulkCommand();
foreach (var row in response.Rows)
{
bulkCmd.Delete(row.Id, row.Value.rev.ToString());
}
client.Documents.BulkAsync(bulkCmd).Wait();
}
}
protected virtual string GenerateRequestUrl(BulkCommand cmd)
{
return(string.Format("{0}/_bulk_docs", Connection.Address));
}
public override void Import(DateTime @from)
{
var logQuery = new LogQueryClass();
var inputFormat = new COMW3CInputContextClass();
string strQuery = string.Format(@"SELECT to_timestamp(date, time) as date,
s-ip as sourceIP,
cs-method as method,
cs-uri-stem as uri,
cs-uri-query as query,
s-port as port,
c-ip as clientIP,
cs(User-Agent) as userAgent,
cs-host as clientToServerHost,
sc-status as statusCode,
sc-substatus as subStatus,
sc-win32-status as win32Status,
sc-bytes as serverToClientBytes,
cs-bytes as clientToServerBytes,
time-taken as duration FROM {0}", "SOME FILE");
ILogRecordset results = logQuery.Execute(strQuery, inputFormat);
var tweets = new List <Entry>();
while (!results.atEnd())
{
ILogRecord logRecord = results.getRecord();
dynamic date = logRecord.getValue("date");
dynamic sourceIP = logRecord.getValue("sourceIP");
dynamic method = logRecord.getValue("method");
dynamic uri = logRecord.getValue("uri");
dynamic query = logRecord.getValue("query") is DBNull ? string.Empty : logRecord.getValue("query");
dynamic port = logRecord.getValue("port");
dynamic clientIP = logRecord.getValue("clientIP") is DBNull
? string.Empty
: logRecord.getValue("clientIP");
dynamic userAgent = logRecord.getValue("userAgent") is DBNull
? string.Empty
: logRecord.getValue("userAgent");
dynamic clientToServerHost = logRecord.getValue("clientToServerHost") is DBNull
? string.Empty
: logRecord.getValue("clientToServerHost");
dynamic statusCode = logRecord.getValue("statusCode");
dynamic subStatus = logRecord.getValue("subStatus");
dynamic win32Status = logRecord.getValue("win32Status");
dynamic serverToClientBytes = logRecord.getValue("serverToClientBytes");
dynamic clientToServerBytes = logRecord.getValue("clientToServerBytes");
dynamic duration = logRecord.getValue("duration");
tweets.Add(new Entry
{
Date = date,
SourceIP = sourceIP,
Method = method,
Uri = uri,
Query = query,
Port = port,
ClientIP = clientIP,
UserAgent = userAgent,
ClientToServerHost = clientToServerHost,
StatusCode = statusCode,
SubStatus = subStatus,
Win32Status = win32Status,
ServerToClientBytes = serverToClientBytes,
ClientToServerBytes = clientToServerBytes,
Duration = duration
});
results.moveNext();
}
var serializer = new JsonNetSerializer();
string bulkCommand = new BulkCommand(index: "log", type: "iis");
string bulkJson =
new BulkBuilder(serializer)
.BuildCollection(tweets,
(builder, tweet) => builder.Create(tweet)
);
_connection.Post(bulkCommand, bulkJson);
}
/// <summary>
/// Exports the specified entry.
/// </summary>
/// <param name="entries">The entry.</param>
internal void Export(IList<Entry> entries)
{
if (entries.Any() == false) return;
// setup variable to group all error messages together
var errorBuilder = new StringBuilder();
// Build a batch based update to avoid memory overhead
var bulkCommand = new BulkCommand(GetIndex(), LOG_TYPE);
var bulkJsons = new BulkBuilder(serializer)
.PipelineCollection(entries, (builder, entity) => builder.Index(entity, id: entity.Id.ToString()))
.JoinInBatches(batchSize: 1000);
foreach (var bulk in bulkJsons)
{
string result = connection.Put(bulkCommand, bulk);
var bulkResult = serializer.ToBulkResult(result);
// Check for errors
foreach (var operation in bulkResult.items.Where(a => a.Result.ok == false))
{
errorBuilder.AppendFormat("Id: {0} Error: {1} {2}", operation.Result._id, operation.Result.error, Environment.NewLine);
}
}
// Check for any errors that are reported by the ElasticSearch
var error = errorBuilder.ToString();
if (string.IsNullOrWhiteSpace(error) == false) throw new ApplicationException(error);
}
protected virtual string GenerateRequestUrl(BulkCommand cmd)
{
return string.Format("{0}/_bulk_docs", Connection.Address);
}
protected virtual HttpRequestMessage CreateRequest(BulkCommand cmd)
{
var req = new HttpRequest(HttpMethod.Post, GenerateRequestUrl(cmd));
req.SetContent(cmd.ToJson());
return req;
}
private static void BulkTweetIndex(IEnumerable<Tweet> tweets, ElasticConnection connection, JsonNetSerializer serializer)
{
string bulkCommand = new BulkCommand(index: "twitter", type: "tweet").Refresh();
int id = 10; // start adding tweets from id = 10
string bulkJson = new BulkBuilder(serializer)
.BuildCollection(tweets,
(builder, tweet) => builder.Index(data: tweet, id: (id++).AsString())
);
string result = connection.Post(bulkCommand, bulkJson);
//Parse bulk result;
BulkResult bulkResult = serializer.ToBulkResult(result);
PrintBulkCommand(bulkCommand, bulkJson, bulkResult);
}
public override void Import(DateTime @from)
{
var logQuery = new LogQueryClass();
var inputFormat = new COMW3CInputContextClass();
string strQuery = string.Format(@"SELECT to_timestamp(date, time) as date,
s-ip as sourceIP,
cs-method as method,
cs-uri-stem as uri,
cs-uri-query as query,
s-port as port,
c-ip as clientIP,
cs(User-Agent) as userAgent,
cs-host as clientToServerHost,
sc-status as statusCode,
sc-substatus as subStatus,
sc-win32-status as win32Status,
sc-bytes as serverToClientBytes,
cs-bytes as clientToServerBytes,
time-taken as duration FROM {0}", "SOME FILE");
ILogRecordset results = logQuery.Execute(strQuery, inputFormat);
var tweets = new List<Entry>();
while (!results.atEnd())
{
ILogRecord logRecord = results.getRecord();
dynamic date = logRecord.getValue("date");
dynamic sourceIP = logRecord.getValue("sourceIP");
dynamic method = logRecord.getValue("method");
dynamic uri = logRecord.getValue("uri");
dynamic query = logRecord.getValue("query") is DBNull ? string.Empty : logRecord.getValue("query");
dynamic port = logRecord.getValue("port");
dynamic clientIP = logRecord.getValue("clientIP") is DBNull
? string.Empty
: logRecord.getValue("clientIP");
dynamic userAgent = logRecord.getValue("userAgent") is DBNull
? string.Empty
: logRecord.getValue("userAgent");
dynamic clientToServerHost = logRecord.getValue("clientToServerHost") is DBNull
? string.Empty
: logRecord.getValue("clientToServerHost");
dynamic statusCode = logRecord.getValue("statusCode");
dynamic subStatus = logRecord.getValue("subStatus");
dynamic win32Status = logRecord.getValue("win32Status");
dynamic serverToClientBytes = logRecord.getValue("serverToClientBytes");
dynamic clientToServerBytes = logRecord.getValue("clientToServerBytes");
dynamic duration = logRecord.getValue("duration");
tweets.Add(new Entry
{
Date = date,
SourceIP = sourceIP,
Method = method,
Uri = uri,
Query = query,
Port = port,
ClientIP = clientIP,
UserAgent = userAgent,
ClientToServerHost = clientToServerHost,
StatusCode = statusCode,
SubStatus = subStatus,
Win32Status = win32Status,
ServerToClientBytes = serverToClientBytes,
ClientToServerBytes = clientToServerBytes,
Duration = duration
});
results.moveNext();
}
var serializer = new JsonNetSerializer();
string bulkCommand = new BulkCommand(index: "log", type: "iis");
string bulkJson =
new BulkBuilder(serializer)
.BuildCollection(tweets,
(builder, tweet) => builder.Create(tweet)
);
_connection.Post(bulkCommand, bulkJson);
}
public static Task <BulkResponse> ExecuteAsync(this IClient client, BulkCommand cmd)
{
return(client.Documents.BulkAsync(cmd));
}
private void WriteToDatabase()
{
var columns = new Dictionary<int, string>();
for(var i=0; i < columnNumber; i++)
{
columns.Add(i, properties[i]);
}
var bulkInfo = new BulkCommand(currentValues.GetEnumerator(), value.EntityName, columnNumber, schema, database, columns);
executor.Execute(bulkInfo);
this.recordCount = 0;
}
