/// <summary>
/// Gets a descending range of events from the database.
/// </summary>
/// <returns>The list of rows within the range</returns>
/// <param name="range">The amount of rows we want</param>
public List <EventRow> GetEvents(int range)
{
try
{
_dbLock.EnterReadLock();
// Get event collection
var events = _db.GetCollection <Event>(COLLECTION_NAME);
var result = events.FindAll()
.OrderBy(x => x.CreatedAt)
.Take(range)
.Select(x => new EventRow(x.Id, TrackerPayload.From(x.Payload)))
.ToList();
Log.Verbose($"Got events: {string.Join(",", result.Select(x => x.GetRowId()))}");
return(result);
}
catch (Exception e)
{
Log.Error($"EventStore: Get Events failed");
Log.Error(e.ToString());
return(new List <EventRow>());
}
finally
{
_dbLock.ExitReadLock();
}
}
// --- Database Functions
/// <summary>
/// Adds an event payload to the database.
/// </summary>
/// <returns><c>true</c>, if event was added, <c>false</c> otherwise.</returns>
/// <param name="payload">An event payload</param>
public bool AddEvent(TrackerPayload payload)
{
try
{
_dbLock.EnterWriteLock();
// Get event collection
var col = _db.GetCollection <Event>(COLLECTION_NAME);
col.Insert(new Event {
Payload = payload.ToString(), Id = Guid.NewGuid(), CreatedAt = DateTime.UtcNow
});
Log.Verbose("EventStore: Event added");
return(true);
}
catch (Exception e)
{
Log.Error("EventStore: Event add failed");
Log.Error(e.ToString());
return(false);
}
finally
{
_dbLock.ExitWriteLock();
}
}
/// <summary>
/// Sends all events as GET requests on background threads
/// </summary>
/// <returns>The get.</returns>
/// <param name="eventRows">Event rows.</param>
protected int HttpGet(List <EventRow> eventRows, ConcurrentQueue <RequestResult> resultQueue)
{
int count = eventRows.Count;
try
{
// Send each row as an individual GET Request
foreach (EventRow eRow in eventRows)
{
TrackerPayload payload = eRow.GetPayload();
long byteSize = payload.GetByteSize() + POST_STM_BYTES;
bool oversize = byteSize > byteLimitGet;
Log.Debug("Emitter: Sending GET with byte-size: " + byteSize);
new ReadyRequest(
GetGETRequest(payload.GetDictionary()),
new List <int> {
eRow.GetRowId()
},
oversize,
resultQueue
).Send();
}
}
catch (Exception e)
{
Log.Debug("Emitter: caught exception in HTTPGet request: " + e.Message);
Log.Debug("Emitter: HTTPGet exception trace: " + e.StackTrace);
}
return(count);
}
public void TestEventStoreFunctions()
{
EventStore es = new EventStore();
Assert.AreEqual(0, es.GetEventCount());
TrackerPayload payload = new TrackerPayload();
payload.Add("hello", "world");
Assert.IsTrue(es.AddEvent(payload));
Assert.IsTrue(es.AddEvent(payload));
Assert.IsTrue(es.AddEvent(payload));
Assert.IsTrue(es.AddEvent(payload));
Assert.AreEqual(4, es.GetEventCount());
Assert.IsTrue(es.DeleteEvent(4));
Assert.AreEqual(3, es.GetEventCount());
Assert.IsTrue(es.DeleteEvents(new List <int> {
3, 2
}));
Assert.AreEqual(1, es.GetEventCount());
Assert.IsTrue(es.DeleteAllEvents());
Assert.AreEqual(0, es.GetEventCount());
Assert.IsTrue(es.AddEvent(payload));
Assert.IsTrue(es.AddEvent(payload));
Assert.IsTrue(es.AddEvent(payload));
Assert.IsTrue(es.AddEvent(payload));
Assert.AreEqual(4, es.GetEventCount());
Assert.AreEqual(4, es.GetAllEvents().Count);
Assert.AreEqual(2, es.GetDescEventRange(2).Count);
Assert.AreEqual(payload.GetDictionary(), es.GetEvent(1).GetPayload().GetDictionary());
Assert.IsTrue(es.DeleteAllEvents());
}
// --- Database Functions
/// <summary>
/// Adds an event payload to the database.
/// </summary>
/// <returns><c>true</c>, if event was added, <c>false</c> otherwise.</returns>
/// <param name="payload">An event payload</param>
public bool AddEvent(TrackerPayload payload)
{
bool result = false;
if (IsDatabaseOpen())
{
byte[] bytes = Utils.SerializeDictionary(payload.GetDictionary());
if (bytes != null)
{
// Build Insert Command
IDbCommand command = dbConnection.CreateCommand();
command.CommandText = INSERT_EVENT + "(@payload)";
SqliteParameter param = new SqliteParameter("@payload", DbType.Binary);
param.Value = bytes;
command.Parameters.Add(param);
// Check whether insert succeeded
result = command.ExecuteNonQuery() > 0;
// Close and dispose of resources
command.Dispose();
command = null;
}
}
Log.Verbose("EventStore: Event add result: " + result);
return(result);
}
/// <summary>
/// Queries the database.
/// </summary>
/// <returns>A list of rows returned from the database</returns>
/// <param name="query">The SQL Query to execute</param>
public List <EventRow> QueryDatabase(string query)
{
List <EventRow> rows = new List <EventRow>();
if (IsDatabaseOpen())
{
try {
IDbCommand command = dbConnection.CreateCommand();
command.CommandText = query;
IDataReader reader = command.ExecuteReader();
// Add the row id and deserialized row data to the list
while (reader.Read())
{
int rowId = (int)reader[COLUMN_ID];
Dictionary <string, object> payloadDict = Utils.DeserializeDictionary((byte[])reader[COLUMN_EVENT_DATA]);
TrackerPayload payload = new TrackerPayload();
payload.AddDict(payloadDict);
rows.Add(new EventRow(rowId, payload));
}
// Close and dispose of resources
reader.Close();
reader = null;
command.Dispose();
command = null;
} catch (Exception e) {
Log.Error("EventStore: Error querying database: " + e.StackTrace);
}
}
return(rows);
}
// --- Database Functions
/// <summary>
/// Adds an event payload to the database.
/// </summary>
/// <returns><c>true</c>, if event was added, <c>false</c> otherwise.</returns>
/// <param name="payload">An event payload</param>
public bool AddEvent(TrackerPayload payload)
{
bool result = false;
if (IsDatabaseOpen())
{
byte[] bytes = Utils.SerializeDictionary(payload.GetDictionary());
if (bytes != null)
{
using (var someCommand = dbConnection.CreateCommand())
{
Log.Debug("Emitter: EventStore.AddEvent preparing SQL");
someCommand.CommandText = INSERT_EVENT + "(@payload)";
Log.Verbose("EventStore: Event add: Checking for null property");
SqliteParameter param = new SqliteParameter("@payload", System.Data.DbType.Binary);
param.Value = bytes;
someCommand.Parameters.Add(param);
// Check whether insert succeeded
result = someCommand.ExecuteNonQuery() > 0;
// Close and dispose of resources
}
}
}
Log.Verbose("EventStore: Event add result: " + result);
return(result);
}
public void TestInit()
{
TrackerPayload payload = new TrackerPayload();
Assert.NotNull(payload);
Assert.AreEqual(0, payload.GetDictionary().Count);
}
/// <summary>
/// Adds the tracker payload to the emitter.
/// </summary>
/// <param name="payload">The base event payload.</param>
/// <param name="contexts">The list of contexts from the event.</param>
private void AddTrackerPayload(TrackerPayload payload, List <IContext> contexts, string eventId)
{
// Add default parameters to the payload
payload.Add(Constants.PLATFORM, this.platform.Value);
payload.Add(Constants.APP_ID, this.appId);
payload.Add(Constants.NAMESPACE, this.trackerNamespace);
payload.Add(Constants.TRACKER_VERSION, Version.VERSION);
// Add the subject data if available
if (subject != null)
{
payload.AddDict(subject.GetPayload().GetDictionary());
}
// Add the session context if available
if (session != null)
{
contexts.Add(session.GetSessionContext(eventId));
}
// Build the final context and add it to the payload
if (contexts != null && contexts.Count > 0)
{
SelfDescribingJson envelope = GetFinalContext(contexts);
payload.AddJson(envelope.GetDictionary(), this.base64Encoded, Constants.CONTEXT_ENCODED, Constants.CONTEXT);
}
Log.Verbose("Tracker: Sending event to the emitter.");
Log.Verbose(" + Event: " + payload.ToString());
// Add the event to the emitter.
emitter.Add(payload);
}
// --- Interface Methods
/// <summary>
/// Gets the event payload.
/// </summary>
/// <returns>The event payload</returns>
public override IPayload GetPayload()
{
TrackerPayload payload = new TrackerPayload();
payload.Add(Constants.SV_NAME, this.name);
payload.Add(Constants.SV_ID, this.id);
return(new SelfDescribingJson(Constants.SCHEMA_SCREEN_VIEW, payload.GetDictionary()));
}
/// <summary>
/// Adds an event payload to the database.
/// </summary>
/// <param name="payload">Payload.</param>
public override void Add(TrackerPayload payload)
{
eventStore.AddEvent(payload);
if (eventStore.GetEventCount() >= sendLimit)
{
EmitLoop();
}
}
// --- Interface Methods
/// <summary>
/// Gets the event payload.
/// </summary>
/// <returns>The event payload</returns>
public override IPayload GetPayload()
{
TrackerPayload payload = new TrackerPayload();
payload.Add(Constants.EVENT, Constants.EVENT_PAGE_VIEW);
payload.Add(Constants.PAGE_URL, pageUrl);
payload.Add(Constants.PAGE_TITLE, pageTitle);
payload.Add(Constants.PAGE_REFR, referrer);
return(AddDefaultPairs(payload));
}
// --- Interface Methods
/// <summary>
/// Gets the event payload.
/// </summary>
/// <returns>The event payload</returns>
public override IPayload GetPayload()
{
TrackerPayload payload = new TrackerPayload();
payload.Add(Constants.EVENT, Constants.EVENT_UNSTRUCTURED);
SelfDescribingJson envelope = new SelfDescribingJson(Constants.SCHEMA_UNSTRUCT_EVENT, this.eventData.GetDictionary());
payload.AddJson(envelope.GetDictionary(), this.base64Encode, Constants.UNSTRUCTURED_ENCODED, Constants.UNSTRUCTURED);
return(AddDefaultPairs(payload));
}
// --- Interface Methods
/// <summary>
/// Gets the event payload.
/// </summary>
/// <returns>The event payload</returns>
public override IPayload GetPayload()
{
TrackerPayload payload = new TrackerPayload();
payload.Add(Constants.EVENT, Constants.EVENT_STRUCTURED);
payload.Add(Constants.SE_CATEGORY, this.category);
payload.Add(Constants.SE_ACTION, this.action);
payload.Add(Constants.SE_LABEL, this.label);
payload.Add(Constants.SE_PROPERTY, this.property);
payload.Add(Constants.SE_VALUE, valueSet ? value.ToString() : null);
return(AddDefaultPairs(payload));
}
public void TestAddDictFunction()
{
TrackerPayload payload = new TrackerPayload();
Dictionary <string, object> dict = new Dictionary <string, object> ();
dict.Add("hello", "world");
dict.Add("demo", 10);
Assert.AreEqual(0, payload.GetDictionary().Count);
payload.AddDict(dict);
Assert.AreEqual(1, payload.GetDictionary().Count);
Assert.AreEqual("world", payload.GetDictionary()["hello"]);
}
// --- Interface Methods
/// <summary>
/// Gets the event payload.
/// </summary>
/// <returns>The event payload</returns>
public override IPayload GetPayload()
{
TrackerPayload payload = new TrackerPayload();
payload.Add(Constants.EVENT, Constants.EVENT_ECOMM_ITEM);
payload.Add(Constants.TI_ITEM_ID, this.itemId);
payload.Add(Constants.TI_ITEM_SKU, this.sku);
payload.Add(Constants.TI_ITEM_NAME, this.name);
payload.Add(Constants.TI_ITEM_CATEGORY, this.category);
payload.Add(Constants.TI_ITEM_PRICE, string.Format("{0:0.00}", this.price));
payload.Add(Constants.TI_ITEM_QUANTITY, this.quantity.ToString());
payload.Add(Constants.TI_ITEM_CURRENCY, this.currency);
return(AddDefaultPairs(payload));
}
public void TestAddFunction()
{
TrackerPayload payload = new TrackerPayload();
payload.Add("demo", "application");
Assert.AreEqual(1, payload.GetDictionary().Count);
Assert.AreEqual("application", payload.GetDictionary()["demo"]);
payload.Add("demo", null);
Assert.AreEqual(1, payload.GetDictionary().Count);
payload.Add(null, "demo");
Assert.AreEqual(1, payload.GetDictionary().Count);
}
public void TestAddJsonFunction()
{
TrackerPayload payload = new TrackerPayload();
Dictionary <string, object> dict = new Dictionary <string, object> ();
dict.Add("hello", "world");
Assert.AreEqual(0, payload.GetDictionary().Count);
payload.AddJson(dict, false, "encoded", "not_encoded");
Assert.AreEqual(1, payload.GetDictionary().Count);
Assert.AreEqual("{\"hello\":\"world\"}", payload.GetDictionary()["not_encoded"]);
Assert.AreEqual(39, payload.GetByteSize());
Assert.AreEqual("{\"not_encoded\":\"{\\\"hello\\\":\\\"world\\\"}\"}", payload.ToString());
}
// --- Interface Methods
/// <summary>
/// Gets the event payload.
/// </summary>
/// <returns>The event payload</returns>
public override IPayload GetPayload()
{
TrackerPayload payload = new TrackerPayload();
payload.Add(Constants.EVENT, Constants.EVENT_ECOMM);
payload.Add(Constants.TR_ID, this.orderId);
payload.Add(Constants.TR_TOTAL, string.Format("{0:0.00}", this.totalValue));
payload.Add(Constants.TR_AFFILIATION, this.affiliation);
payload.Add(Constants.TR_TAX, this.taxValueSet ? (string.Format("{0:0.00}", this.taxValue)) : null);
payload.Add(Constants.TR_SHIPPING, this.shippingSet ? (string.Format("{0:0.00}", this.shipping)) : null);
payload.Add(Constants.TR_CITY, this.city);
payload.Add(Constants.TR_STATE, this.state);
payload.Add(Constants.TR_COUNTRY, this.country);
payload.Add(Constants.TR_CURRENCY, this.currency);
return(AddDefaultPairs(payload));
}
/// <summary>
/// Queries the database.
/// </summary>
/// <returns>A list of rows returned from the database</returns>
/// <param name="query">The SQL Query to execute</param>
public List <EventRow> QueryDatabase(string query)
{
List <EventRow> rows = new List <EventRow>();
if (IsDatabaseOpen())
{
try
{
using (var someCommand = dbConnection.CreateCommand())
{
someCommand.CommandText = query;
using (var reader = someCommand.ExecuteReader())
{
// Add the row id and deserialized row data to the list
if (reader.HasRows)
{
while (reader.Read())
{
if (!reader.IsDBNull(reader.GetOrdinal(COLUMN_ID)) && !reader.IsDBNull(reader.GetOrdinal(COLUMN_EVENT_DATA)))
{
int rowId = (int)reader[COLUMN_ID];
Dictionary <string, object> payloadDict = Utils.DeserializeDictionary((byte[])reader[COLUMN_EVENT_DATA]);
TrackerPayload payload = new TrackerPayload();
payload.AddDict(payloadDict);
rows.Add(new EventRow(rowId, payload));
}
}
}
}
}
}
catch (Exception e)
{
Log.Error("EventStore: Error querying database: " + e.StackTrace);
}
}
return(rows);
}
// --- Consumers & Loopers
/// <summary>
/// Consumes events from the event queue and adds them to the database.
/// </summary>
private void EventConsumer()
{
Log.Debug("Emitter: Event consumer starting up");
while (consuming)
{
TrackerPayload payload = payloadQueue.Dequeue();
// If the consumer was shutdown while waiting
if (!consuming)
{
Log.Debug("Emitter: Event consumer shutting down...");
break;
}
// Signal emit loop
lock (emitLock) {
this.eventStore.AddEvent(payload);
Monitor.Pulse(emitLock);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="SnowplowTracker.Subject"/> class.
/// </summary>
public Subject()
{
standardDict = new TrackerPayload();
}
// --- Helper Methods
/// <summary>
/// Adds the common key-value pairs to an event payload
/// </summary>
/// <param name="payload">The payload to append values to</param>
/// <returns>The complete payload</returns>
protected TrackerPayload AddDefaultPairs(TrackerPayload payload)
{
payload.Add(Constants.EID, eventId);
payload.Add(Constants.TIMESTAMP, timestamp.ToString());
return(payload);
}
/// <summary>
/// Adds an event payload to the database.
/// </summary>
/// <param name="payload">Payload.</param>
public override void Add(TrackerPayload payload)
{
payloadQueue.Enqueue(payload);
}
/// <summary>
/// Initializes a new instance of the <see cref="SnowplowTracker.Storage.EventRow"/> class.
/// </summary>
/// <param name="rowId">The row number of the event in the database</param>
/// <param name="payload">The event payload that needs to be sent</param>
public EventRow(int rowId, TrackerPayload payload)
{
this.rowId = rowId;
this.payload = payload;
}
/// <summary> /// Adds an event payload to the database. /// </summary> /// <param name="payload">Payload.</param> public abstract void Add(TrackerPayload payload);
/// <summary>
/// Send all event rows as POST requests on background threads
/// </summary>
/// <returns>The results of all the requests</returns>
/// <param name="eventRows">Event rows.</param>
protected int HttpPost(List <EventRow> eventRows, ConcurrentQueue <RequestResult> resultQueue)
{
int count = 0;
List <int> rowIds = new List <int>();
List <Dictionary <string, object> > payloadDicts = new List <Dictionary <string, object> >();
long totalByteSize = 0;
try
{
for (int i = 0; i < eventRows.Count; i++)
{
TrackerPayload payload = eventRows[i].GetPayload();
long payloadByteSize = payload.GetByteSize() + POST_STM_BYTES;
if ((payloadByteSize + POST_WRAPPER_BYTES) > byteLimitPost)
{
// A single Payload has exceeded the Byte Limit
Log.Debug("Emitter: Single event exceeds byte limit: " + (payloadByteSize + POST_WRAPPER_BYTES) + " is > " + byteLimitPost);
Log.Debug("Sending POST with byte-size: " + (payloadByteSize + POST_WRAPPER_BYTES));
List <Dictionary <string, object> > singlePayloadPost = new List <Dictionary <string, object> > {
payload.GetDictionary()
};
List <int> singlePayloadId = new List <int> {
eventRows[i].GetRowId()
};
new ReadyRequest(GetPOSTRequest(singlePayloadPost), singlePayloadId, true, resultQueue).Send();
count++;
}
else if ((totalByteSize + payloadByteSize + POST_WRAPPER_BYTES + (payloadDicts.Count - 1)) > byteLimitPost)
{
Log.Debug("Emitter: Byte limit reached: " + (totalByteSize + payloadByteSize + POST_WRAPPER_BYTES + (payloadDicts.Count - 1)) +
" is > " + byteLimitPost);
Log.Debug("Emitter: Sending POST with byte-size: " + (totalByteSize + POST_WRAPPER_BYTES + (payloadDicts.Count - 1)));
new ReadyRequest(GetPOSTRequest(payloadDicts), rowIds, false, resultQueue).Send();
count++;
// Reset collections
payloadDicts = new List <Dictionary <string, object> > {
payload.GetDictionary()
};
rowIds = new List <int> {
eventRows[i].GetRowId()
};
totalByteSize = payloadByteSize;
}
else
{
payloadDicts.Add(payload.GetDictionary());
rowIds.Add(eventRows[i].GetRowId());
totalByteSize += payloadByteSize;
}
}
if (payloadDicts.Count > 0)
{
Log.Debug("Emitter: Sending POST with byte-size: " + (totalByteSize + POST_WRAPPER_BYTES + (payloadDicts.Count - 1)));
new ReadyRequest(GetPOSTRequest(payloadDicts), rowIds, false, resultQueue).Send();
count++;
}
}
catch (Exception e)
{
Log.Debug("Emitter: caught exception in HTTPPost request: " + e.Message);
Log.Debug("Emitter: HTTPPost exception trace: " + e.StackTrace);
}
return(count);
}
public override void Add(TrackerPayload payload)
{
payloads.Add(payload);
}
