public FlowKey CreateKey(Id id)
{
Expect.False(IsSingleInstance && id.IsAssigned, $"Flow {this} is single-instance and cannot have an assigned id of {id}");
Expect.False(IsMultiInstance && id.IsUnassigned, $"Flow {this} is multi-instance and must have an assigned id");
return(FlowKey.From(this, id));
}
public void UnsubscribeFromChanged(Id connectionId, FlowKey key)
{
if (_connectionsById.TryGetValue(connectionId, out var connection))
{
connection.Unsubscribe(key);
}
}
//
// Event data
//
internal static EventData GetAreaEventData(
this EventStoreContext context,
Event e,
TimelinePosition cause,
DateTimeOffset when,
DateTimeOffset?whenOccurs,
Id eventId,
Id commandId,
Id userId,
FlowKey topic,
Many <FlowKey> routes)
{
var type = context.GetEventType(e);
var metadata = new AreaEventMetadata
{
Cause = cause,
When = when,
WhenOccurs = whenOccurs,
CommandId = commandId,
UserId = userId,
Topic = topic
};
metadata.ApplyRoutes(type, routes);
return(new EventData(
eventId.IsUnassigned ? Guid.NewGuid() : Guid.Parse(eventId.ToString()),
type.ToString(),
isJson: true,
data: context.ToJson(e),
metadata: context.ToJson(metadata)));
}
internal ReadFlowToResumeCommand(EventStoreContext context, FlowKey key)
{
_context = context;
_key = key;
_routesStream = key.GetRoutesStream();
}
async Task <TResult> ReadQueryCheckpoint <TResult>(FlowKey key, Func <TResult> getDefault, Func <ResolvedEvent, TResult> getCheckpoint)
{
var stream = key.GetCheckpointStream();
EventReadResult result = default(EventReadResult);
try
{
result = await _context.Connection.ReadEventAsync(stream, StreamPosition.End, resolveLinkTos : false);
}
catch (Exception e)
{
Console.WriteLine(e.Message);
}
switch (result.Status)
{
case EventReadStatus.NoStream:
case EventReadStatus.NotFound:
return(getDefault());
case EventReadStatus.Success:
return(getCheckpoint(result.Event.Value));
default:
throw new Exception($"Unexpected result when reading {stream}: {result.Status}");
}
}
internal ReadFlowWithoutCheckpointCommand(EventStoreContext context, FlowKey key)
{
_context = context;
_key = key;
_stream = key.GetRoutesStream();
}
internal ReadFlowWithCheckpointCommand(EventStoreContext context, FlowKey key, RecordedEvent checkpoint)
{
_context = context;
_key = key;
_checkpoint = checkpoint;
_stream = key.GetRoutesStream();
}
private FlowKey GetConversationKey(FlowKey key)
{
var revKey = key.Reverse();
var sp1 = key.SourcePort;
var sp2 = revKey.SourcePort;
return(sp1 > sp2 ? key : revKey);
}
/// <summary>
/// Generates a new flow id. The id depends on flow key and first seen timestamp.
/// </summary>
/// <param name="flowKey">The key of flow.</param>
/// <param name="firstSeen">Timestemp given as UNIX milliseconds.</param>
/// <returns>A string representation of flow ID. </returns>
public static string NewUid(FlowKey flowKey, long firstSeen)
{
var hashValue = m_hashAlgorithm.ComputeHash(flowKey.Bytes);
var lopart = BitConverter.ToUInt64(hashValue, 0);
var hipart = BitConverter.ToUInt64(hashValue, sizeof(ulong));
return($"{lopart.ToString("X16")}-{hipart.ToString("X16")}");
}
Query GetDefaultContent(FlowKey key)
{
var query = (Query)key.Type.New();
FlowContext.Bind(query, key);
return(query);
}
public async Task <TimelinePosition> WriteNewEvents(TimelinePosition cause, FlowKey topicKey, Many <Event> newEvents)
{
var data = _context.GetNewEventData(cause, topicKey, newEvents);
var result = await _context.AppendToTimeline(data);
return(new TimelinePosition(result.NextExpectedVersion - newEvents.Count + 1));
}
public Task UnsubscribeFromChanged(Id connectionId, FlowKey key)
{
if (_connectionsById.TryGetValue(connectionId, out var connection))
{
connection.Unsubscribe(key);
}
return(Task.CompletedTask);
}
async Task Enqueue(TimelinePoint point, FlowKey route)
{
if (!_flowsByKey.TryGetValue(route, out var flow))
{
flow = AddFlow(route);
await ConnectFlow(flow);
}
flow.Enqueue(point);
}
async Task <IFlowScope> GetOrConnectFlow(FlowKey route)
{
if (!_flowsByKey.TryGetValue(route, out var flow))
{
flow = await ConnectFlow(route);
_flowsByKey[route] = flow;
}
return(flow);
}
QueryETag ReadETag(Type type, Id id)
{
if (TryGetIfNoneMatch(out var ifNoneMatch))
{
return(QueryETag.From(ifNoneMatch, _area));
}
var key = FlowKey.From(_area.Queries.Get(type), id);
return(QueryETag.From(key, TimelinePosition.None));
}
Query GetCheckpointContent(FlowKey key, ResolvedEvent e)
{
var metadata = _context.ReadCheckpointMetadata(e);
if (metadata.ErrorPosition.IsSome)
{
throw new Exception($"Query is stopped at {metadata.ErrorPosition} with the following error: {metadata.ErrorMessage}");
}
return((Query)_context.Json.FromJsonUtf8(e.Event.Data, key.Type.DeclaredType));
}
//
// Flows
//
IFlowScope AddFlow(FlowKey key)
{
var flow = key.Type.IsTopic ?
new TopicScope(key, _db, _services) :
new QueryScope(key, _db) as IFlowScope;
_flowsByKey[key] = flow;
RemoveWhenDone(flow);
return(flow);
}
public static Task <QueryContent> ReadQueryContent(this IQueryDb db, string etag, Type type, Id id) =>
db.ReadQueryContent(area =>
{
if (!string.IsNullOrWhiteSpace(etag))
{
return(QueryETag.From(etag, area));
}
var key = FlowKey.From(area.Queries[type], id);
return(QueryETag.From(key, TimelinePosition.None));
});
public static T Bind <T>(this T query, Id instanceId) where T : Query
{
var queryType = query.GetType();
var areaMap = AreaMap.From(new[] { queryType });
AreaTypeName.TryFrom(queryType.Name, out var areaType);
var flowType = areaMap.GetFlow(areaType);
var subscriptionKey = FlowKey.From(flowType, instanceId);
FlowContext.Bind(query, subscriptionKey);
return(query);
}
async Task <IFlowScope> ConnectFlow(FlowKey route)
{
var flow = CreateFlow(route);
_flowsByKey[route] = flow;
RemoveWhenDone(flow);
await flow.Connect(State.CancellationToken);
return(flow);
}
internal WriteNewEventsCommand(
EventStoreContext context,
TimelinePosition cause,
FlowKey topicKey,
Many <Event> newEvents)
{
_context = context;
_cause = cause;
_topicKey = topicKey;
_newEvents = newEvents;
_newEventTypes = _newEvents.ToMany(e => _context.GetEventType(e));
}
internal void Unsubscribe(FlowKey key)
{
if (_instancesByKey.TryGetValue(key, out var instance))
{
instance.Unsubscribe(this);
_instancesByKey.TryRemove(key, out _);
if (_instancesByKey.Count == 0)
{
_db.RemoveConnection(this);
}
}
}
public void Add(int packetNumber, long packetOffset, FlowKey flowkey)
{
if (m_lastBlock == null)
{
var filterCapacity = m_blockCapacity / m_packetFlowRatio;
m_lastBlock = new FilterBlock(m_blockCapacity, filterCapacity, m_linkType, packetOffset);
m_filterArray.Add(m_lastBlock);
}
m_lastBlock.Add(flowkey);
if (m_lastBlock.IsFull)
{
m_lastBlock = null;
}
}
/// <summary>
/// Write the text representation of <see cref="DnsPacket"/>. It uses Flix assert format, e.g., Frame(DnsPacket(0.1, 1234, Query)).
/// </summary>
/// <param name="packet"></param>
public void WriteRecord(PosixTimeval timestamp, FlowKey flowKey, DnsPacket packet)
{
string answerString(DnsPacket.Answer answer)
{
switch (answer.Rdata)
{
case DnsPacket.ARecord a:
return($"A(\"{answer.Name.DomainNameString}\",\"{new IPAddress(a.Address)}\")");
case DnsPacket.AaaaRecord aaaa:
return($"AAAA(\"{answer.Name.DomainNameString}\",\"{new IPAddress(aaaa.Address)}\")");
case DnsPacket.CnameRecord cname:
return($"CNAME(\"{answer.Name.DomainNameString}\",\"{cname.Hostname.DomainNameString}\")");
case DnsPacket.NsRecord ns:
return($"NS(\"{answer.Name.DomainNameString}\",\"{ns.Hostname.DomainNameString}\")");
case DnsPacket.MxRecord mx:
return($"NS(\"{answer.Name.DomainNameString}\",{mx.Priority},\"{mx.Hostname.DomainNameString}\")");
default:
return("NULL");
}
}
var ts = ((double)timestamp.Seconds + ((double)timestamp.MicroSeconds) / 1000000).ToString("0.000");
var id = packet.TransactionId;
var rcode = packet.Flags.Rcode == 0 ? "NoError" : "NameDoesNotExist";
var queries = String.Join("::", packet.Queries.Select(x => $"{x.Type.ToString().ToUpperInvariant()}(\"{x.Name.DomainNameString}\",\"\")").Append("Nil"));
var answers = String.Join("::", packet.Answers.Select(answerString).Append("Nil"));
var qr = packet.Flags.Qr == 0 ? $"Query({id},{queries})" : $"Response({id},{rcode},{queries},{answers})";
var dns = $"DNS({qr})";
var proto = CultureInfo.InvariantCulture.TextInfo.ToTitleCase(flowKey.Protocol.ToString());
var ipSrc = flowKey.SourceEndpoint.Address.ToString();
var ipDst = flowKey.DestinationEndpoint.Address.ToString();
var portSrc = flowKey.SourceEndpoint.Port;
var portDst = flowKey.DestinationEndpoint.Port;
var flow = $"{proto}(\"{ipSrc}\",\"{ipDst}\",{portSrc},{portDst})";
// Frame(1520329507.498, Udp("192.168.111.100", "147.229.9.43", 1234, 53), DnsPacket(Query(15595,A("api.github.com.", "")::Nil))).
var str = $"Frame({ts}, {flow}, {dns}).";
m_writer.WriteLine(str);
m_writer.Flush();
}
public TTarget Invoke(FlowKey flowKey, IEnumerable <Memory <byte> > frames)
{
var firstTicks = _windowStart.Ticks;
var lastTicks = firstTicks + _duration.Ticks;
bool IsInWindow(Memory <byte> frame)
{
FrameMetadata frameMetadata = default;
FrameMetadata.FromBytes(frame.Span, ref frameMetadata);
return(frameMetadata.Ticks >= firstTicks && frameMetadata.Ticks < lastTicks);
}
return(_processor.Invoke(flowKey, frames.Where(IsInWindow)));
}
IFlowScope CreateFlow(FlowKey key)
{
if (key.Type.IsTopic)
{
return(new TopicScope(key, _db, _services));
}
else if (key.Type.IsQuery)
{
return(new QueryScope(key, _db));
}
else
{
throw new Exception($"Expected topic or query type, received {key.Type}");
}
}
public bool Add(FlowKey flowkey)
{
lock (m_syncObject)
{
if (m_itemCount < m_blockCapacity)
{
m_filter.Add(flowkey.ToString());
m_itemCount++;
return(true);
}
else
{
return(false);
}
}
}
private FlowData PacketFlowFrom(FlowKey flowKey, FrameData frame, string flowUid)
{
return(new FlowData()
{
FlowUid = flowUid,
Protocol = flowKey.Protocol.ToString(),
SourceAddress = flowKey.SourceIpAddress.ToString(),
SourcePort = flowKey.SourcePort,
DestinationAddress = flowKey.DestinationIpAddress.ToString(),
DestinationPort = flowKey.DestinationPort,
FirstSeen = frame.Timestamp,
LastSeen = frame.Timestamp,
Octets = frame.Data.Length,
Packets = 1
});
}
async Task <Stream> AppendAndWaitUntilChanged(FlowKey key, Event e, TestTimeline timeline)
{
var connectionId = Id.FromGuid();
await _queryDb.SubscribeToChanged(connectionId, QueryETag.From(key));
var position = await timeline.Append(e);
var newETag = QueryETag.From(key, position);
await _notifier.WaitUntilChanged(connectionId, newETag);
_queryDb.UnsubscribeFromChanged(connectionId);
return(await _queryDb.ReadContent(newETag));
}
//
// Points
//
bool Ignore(TimelinePoint point, FlowKey route)
{
if (!_ignored.Contains(route))
{
return(false);
}
else if (route.Type.Observations.Get(point.Type).CanBeFirst)
{
_ignored.Remove(route);
return(false);
}
else
{
return(true);
}
}
