public void ToBulkString_SimpleContent_OneNewline()
{
var data = MockEventData.CreateSerilogEventData(body: "{ \"message\": \"log message\" }");
var item = BulkItem.FromEventData(data);
var bulkBody = item.ToBulkString();
// BulkString should have exactly one newline when formatted correctly.
Assert.Equal(1, bulkBody.Count(c => c == '\n'));
}
public void FromEventData_NoType_InvalidBulkItem()
{
var data = new EventData();
// Default constructed EventData will not have the Type property set, so should fail.
var item = BulkItem.FromEventData(data);
Assert.False(item.IsValid, "BulkItem with no 'Type' should be invalid.");
Assert.False(string.IsNullOrEmpty(item.InvalidReason), "BulkItem with no 'Type' should have an invalid reason.");
}
public void ToBulkString_MultilineContent_MultipleNewlines()
{
var data = MockEventData.CreateSerilogEventData(body: "{ \"message\": \"log message\n with newline\" }");
var item = BulkItem.FromEventData(data);
var bulkBody = item.ToBulkString();
Assert.False(item.IsValid, "BulkItem with document body containing newlines should be invalid.");
Assert.Equal(2, bulkBody.Count(c => c == '\n'));
}
public BulkLoadItem(BulkLoadAction loadAction, BulkItem item)
: base(item)
{
this.LoadAction = loadAction;
// When creating bucket folder items, we don't want those items to be bucketed again.
if (BucketConfigurationSettings.BucketTemplateId.Guid.Equals(TemplateId))
{
Bucketed = true;
}
}
public void FromEventData_InvalidType_InvalidBulkItem()
{
var data = new EventData();
data.Properties["Type"] = 17;
var item = BulkItem.FromEventData(data);
Assert.False(item.IsValid, "BulkItem with invalid 'Type' should be invalid.");
Assert.False(string.IsNullOrEmpty(item.InvalidReason), "BulkItem with invalid 'Type' should have an invalid reason.");
}
public void ToBulkString_ComplexContent_OneNewline()
{
var data = MockEventData.CreateSerilogEventData(body: JsonConvert.SerializeObject(new { message = "log message\n with newline" }));
var item = BulkItem.FromEventData(data);
var bulkBody = item.ToBulkString();
// BulkString should have exactly one newline when formatted correctly.
Assert.True(item.IsValid, "BulkItem with json encoded newlines should be valid.");
Assert.Equal(1, bulkBody.Count(c => c == '\n'));
}
public void FromEventData_ExternalTelemetryEvent_NoSource_NoMessageId_NoTimestamp()
{
var data = new EventData();
data.Properties["Type"] = "ExternalTelemetry";
var item = BulkItem.FromEventData(data);
Assert.True(item.IsValid);
Assert.Equal("externaltelemetry", item.IndexBaseName);
Assert.Equal("telemetryevent", item.DocumentType); // No 'Source' should provide a default document type.
Assert.False(string.IsNullOrEmpty(item.DocumentId));
Assert.True(ApproximatelyEqual(DateTime.UtcNow, item.Timestamp)); // Timestamp should get generated based on current time.
}
public void FromEventData_SerilogEvent_NoMessageId_NoTimestamp()
{
var data = new EventData();
data.Properties["Type"] = "SerilogEvent";
var item = BulkItem.FromEventData(data);
Assert.True(item.IsValid);
Assert.Equal("logstash", item.IndexBaseName);
Assert.Equal("logevent", item.DocumentType);
Assert.False(string.IsNullOrEmpty(item.DocumentId));
Assert.True(ApproximatelyEqual(DateTime.UtcNow, item.Timestamp)); // Timestamp should get generated based on current time.
}
public void FromEventData_ExternalTelemetryEvent_InvalidTimestamp()
{
var data = new EventData();
data.Properties["Type"] = "ExternalTelemetry";
data.Properties["Source"] = "vmrestarts";
data.Properties["MessageId"] = Guid.NewGuid().ToString();
data.Properties["Timestamp"] = DateTime.UtcNow; // Incorrect type - should be string.
var item = BulkItem.FromEventData(data);
Assert.False(item.IsValid, "BulkItem with invalid 'Timestamp' should be invalid.");
Assert.False(string.IsNullOrEmpty(item.InvalidReason), "BulkItem with invalid 'Timestamp' should have an invalid reason.");
}
public void FromEventData_NoType_InferSerilogEvent()
{
DateTime timestamp = DateTime.Parse("2016-11-02T20:15:21.504Z");
var body = "{ \"@timestamp\": \"2016-11-02T20:15:21.504Z\", \"message\": \"Elapsed time: 100 ms\", \"messageTemplate\": \"Elapsed time: {ElapsedTime} ms\", \"fields.ElapsedTime\": 100 }";
var data = new EventData(Encoding.UTF8.GetBytes(body));
// Default constructed EventData will not have the Type property set, but should infer logstash event from the properties.
var item = BulkItem.FromEventData(data);
Assert.True(item.IsValid, "BulkItem with known logstash properties should be inferred.");
Assert.Equal("logstash", item.IndexBaseName);
Assert.Equal("logevent", item.DocumentType);
Assert.False(string.IsNullOrEmpty(item.DocumentId));
Assert.True(ApproximatelyEqual(timestamp, item.Timestamp)); // Timestamp should be parsed from the message body.
}
public void FromEventData_SerilogEvent_WithProperties()
{
var messageId = Guid.NewGuid().ToString();
var timestamp = DateTime.UtcNow.ToString(CultureInfo.InvariantCulture);
var data = new EventData();
data.Properties["Type"] = "SerilogEvent";
data.Properties["MessageId"] = messageId;
data.Properties["Timestamp"] = timestamp;
var item = BulkItem.FromEventData(data);
Assert.True(item.IsValid);
Assert.Equal("logstash", item.IndexBaseName);
Assert.Equal("logevent", item.DocumentType);
Assert.Equal(messageId, item.DocumentId);
Assert.Equal(timestamp, item.Timestamp.ToString(CultureInfo.InvariantCulture));
}
public void FromEventData_RoboCustosInteraction_WithProperties()
{
var messageId = Guid.NewGuid().ToString();
var timestamp = DateTime.UtcNow.ToString(CultureInfo.InvariantCulture);
var data = new EventData();
data.Properties["Type"] = "RoboCustosInteraction";
data.Properties["MessageId"] = messageId;
data.Properties["Timestamp"] = timestamp;
var item = BulkItem.FromEventData(data);
Assert.True(item.IsValid);
Assert.Equal("robointeractions", item.IndexBaseName);
Assert.Equal("interaction", item.DocumentType);
Assert.Equal(messageId, item.DocumentId);
Assert.Equal(timestamp, item.Timestamp.ToString(CultureInfo.InvariantCulture));
}
public void FromEventData_ExternalTelemetryEvent_WithProperties()
{
var source = "vmrestarts";
var messageId = Guid.NewGuid().ToString();
var timestamp = DateTime.UtcNow.ToString(CultureInfo.InvariantCulture);
var data = new EventData();
data.Properties["Type"] = "ExternalTelemetry";
data.Properties["Source"] = source;
data.Properties["MessageId"] = messageId;
data.Properties["Timestamp"] = timestamp;
var item = BulkItem.FromEventData(data);
Assert.True(item.IsValid);
Assert.Equal("externaltelemetry", item.IndexBaseName);
Assert.Equal(source, item.DocumentType);
Assert.Equal(messageId, item.DocumentId);
Assert.Equal(timestamp, item.Timestamp.ToString(CultureInfo.InvariantCulture));
}
public void FromEventData_AzureResources_WithProperties()
{
var source = "Microsoft.Compute/availabilitySets";
var messageId = Guid.NewGuid().ToString();
var timestamp = DateTime.UtcNow.ToString(CultureInfo.InvariantCulture);
var data = new EventData();
data.Properties["Type"] = "azure-resources";
data.Properties["Source"] = source;
data.Properties["MessageId"] = messageId;
data.Properties["Timestamp"] = timestamp;
var item = BulkItem.FromEventData(data);
Assert.True(item.IsValid);
Assert.Equal("azure-resources", item.IndexBaseName);
Assert.Equal("azure-resources", item.IndexName);
Assert.Equal(source, item.DocumentType);
Assert.Equal(messageId, item.DocumentId);
Assert.Equal(timestamp, item.Timestamp.ToString(CultureInfo.InvariantCulture));
}
/// <summary>
/// Executes a singly bulk item operation
/// </summary>
/// <param name="bulkItem">Bulk item to execute</param>
/// <param name="session">Session to use to execute the bulk item</param>
/// <param name="retries">Number of retries already attempted</param>
protected virtual void ExecuteBulkItem(BulkItem bulkItem, ISession session, int retries)
{
object entry = bulkItem.Item;
if (!_managedEntriesDictionary.ContainsKey(entry.GetType().ToString()))
{
return;
}
switch (bulkItem.Operation)
{
case BulkOperation.Remove:
session.Delete(session.Merge(entry));
break;
case BulkOperation.Write:
case BulkOperation.Update:
if (retries > 0 || _useMerge)
{
session.Merge(entry);
}
else
{
try
{
session.SaveOrUpdate(entry);
}
catch (HibernateException)
{
session.Merge(entry);
}
}
break;
default:
break;
}
}
public void PropertiesTest()
{
var indexBaseName = "logstash";
var timestamp = DateTime.UtcNow;
var enqueueTime = timestamp.Subtract(TimeSpan.FromSeconds(1));
var documentType = "logevent";
var documentId = Guid.NewGuid().ToString();
var documentBody = "{ \"message\": \"log message\", \"data\": 17 }";
var item = new BulkItem(indexBaseName, timestamp, enqueueTime, documentType, documentId, documentBody);
Assert.True(item.IsValid);
Assert.Equal(indexBaseName, item.IndexBaseName);
Assert.NotEqual(indexBaseName, item.IndexName);
Assert.Equal(timestamp, item.Timestamp);
Assert.Equal(enqueueTime, item.EnqueueTime);
Assert.Equal(documentType, item.DocumentType);
Assert.Equal(documentType, item.DocumentType);
Assert.Equal(documentId, item.DocumentId);
Assert.Equal(documentBody, item.DocumentBody);
Assert.Equal(documentBody.Length, item.BodyLength);
Assert.Equal(documentBody, item.DocumentBodyStart); // Since document body is short, these should match.
}
/// <summary>
/// Gets all descendants of an item.
/// </summary>
/// <param name="sqlContext">Sql context to read from.</param>
/// <param name="itemId">Ancestor item id.</param>
/// <param name="itemPath">Ancestor item path.</param>
/// <param name="ofTemplates">Filter descendants by template(s). No inheritance supported.</param>
/// <param name="modifiedSince">Filter descendants by modification timestamp.</param>
/// <param name="onlyPublishable">Filter out descendants that are set to 'never publish'.</param>
/// <returns>Stream of bulk items.</returns>
public virtual IEnumerable <BulkItem> GetDescendants(SqlContext sqlContext, Guid itemId, string itemPath,
Guid[] ofTemplates = null, DateTime?modifiedSince = null, bool onlyPublishable = false)
{
var templateCsv = ofTemplates == null || ofTemplates.Length == 0
? null
: string.Join(",", ofTemplates.Select(x => $"'{x:D}'"));
var sql = sqlContext.GetEmbeddedSqlLines("Sql.GetDescendants.sql", typeof(BulkReader))
.ExpandParameterLineIf(() => ofTemplates?.Length > 1, "@templateIdsCsv", templateCsv)
.RemoveParameterLineIf(() => ofTemplates == null || ofTemplates.Length != 1, "@templateId")
.RemoveParameterLineIf(() => ofTemplates == null || ofTemplates.Length <= 1, "@templateIdsCsv")
.RemoveParameterLineIf(() => modifiedSince == null, "@modifiedSince")
.RemoveParameterLineIf(() => !onlyPublishable, "@neverPublishFieldId")
.RemoveParameterLineIf(() => !onlyPublishable, "@neverPublish");
using (var reader = sqlContext.ExecuteReader(sql, commandProcessor: cmd =>
{
cmd.Parameters.AddWithValue("@rootItemId", itemId);
cmd.Parameters.AddWithValue("@rootItemPath", itemPath);
if (ofTemplates != null && ofTemplates.Length == 1)
{
cmd.Parameters.AddWithValue("templateId", ofTemplates[0]);
}
if (modifiedSince.HasValue)
{
cmd.Parameters.AddWithValue("@modifiedSince", modifiedSince.Value.ToUniversalTime());
}
if (onlyPublishable)
{
cmd.Parameters.AddWithValue("@neverPublishFieldId", Sitecore.FieldIDs.NeverPublish.Guid);
cmd.Parameters.AddWithValue("@neverPublish", "1");
}
}))
{
BulkItem item = null;
while (reader.Read())
{
var recordItemId = reader.GetGuid(0);
if (item != null && item.Id != recordItemId)
{
yield return(item);
item = null;
}
if (item == null)
{
item = new BulkItem(recordItemId,
reader.GetGuid(3), reader.GetGuid(4), reader.GetGuid(5), reader.GetString(2));
}
var language = reader.IsDBNull(10) ? null : reader.GetString(10);
var version = reader.IsDBNull(11) ? null : (int?)reader.GetInt32(11);
if (language != null && version != null)
{
item.AddVersionedField(reader.GetGuid(8), language, version.Value, reader.GetString(9));
}
else if (language != null)
{
item.AddUnversionedField(reader.GetGuid(8), language, reader.GetString(9));
}
else
{
item.AddSharedField(reader.GetGuid(8), reader.GetString(9));
}
}
if (item != null)
{
yield return(item);
}
}
}
/// <summary>
/// Executes a singly bulk item operation
/// </summary>
/// <param name="bulkItem">Bulk item to execute</param>
/// <param name="session">Session to use to execute the bulk item</param>
/// <param name="retries">Number of retries already attempted</param>
protected virtual void ExecuteBulkItem(BulkItem bulkItem, ISession session, int retries)
{
object entry = bulkItem.Item;
if (!_managedEntriesDictionary.ContainsKey(entry.GetType().ToString()))
return;
switch (bulkItem.Operation)
{
case BulkOperation.Remove:
session.Delete(session.Merge(entry));
break;
case BulkOperation.Write:
case BulkOperation.Update:
if (retries > 0 || _useMerge)
session.Merge(entry);
else
{
try
{
session.SaveOrUpdate(entry);
}
catch (HibernateException)
{
session.Merge(entry);
}
}
break;
default:
break;
}
}
