private void SendToAll(CTuple tuple)
{
foreach (var neighbor in destinations.Values)
{
neighbor.Send(new CTuple(tuple));
}
}
public override int ChooseReplica(CTuple tuple)
{
int number = random.Next(0, countRep);
//TODO: verify if "crash flag" is on - after checkpoint
return(number);
}
public void StartProcessingFromFile(string path, int startFrom = 0)
{
try
{
using (var f = new StreamReader(path))
{
string line = null;
while ((line = f.ReadLine()) != null)
{
if (line.StartsWith("%"))
{
continue;
}
var tupleData = line.Split(',').Select((x) => x.Trim()).ToList();
var ctuple = new CTuple(tupleData, TupleCounter++, 0, this.OperatorId, 0);
Console.WriteLine($"Read tuple from file: {ctuple}");
if (TupleCounter >= startFrom && routingStrategy.ChooseReplica(ctuple) == ID)
{
ProcessAndForward(ctuple);
}
}
}
}
catch (Exception e)
{
Console.WriteLine($"Unable to read from file {path}. Exception: {e.Message}.");
}
}
public CTuple Next()
{
int chosen = -1;
CTuple chosenTuple = null;
// takes a tuple from each origin and puts it into currentTuple list
for (int i = 0; i < Origins.Count; i++)
{
// Console.WriteLine($"Waiting for {Origins[i].OpId} ({Origins[i].ReplicaId})");
if (currentTuple[i] != null)
{
continue;
}
currentTuple[i] = Origins[i].Take();
}
// chooses one
chosen = Choose(currentTuple);
// non-data tuple (could be from flush)
chosenTuple = currentTuple[chosen];
currentTuple[chosen] = null;
//Console.WriteLine($"Chosen = {chosen}");
return(chosenTuple);
}
public void ProcessAndForward(CTuple tuple)
{
if (tuple.GetFields() != null)
{
Console.WriteLine($"Received {tuple.ID} from {tuple.opName} ({ tuple.repID })");
}
originOperators[tuple.opName][tuple.repID].Insert(tuple);
}
public override int ChooseReplica(CTuple tuple)
{
String s = tuple.GetFieldByIndex(id);
int i = s.GetHashCode();
return(Math.Abs(i) % countRep);
}
override public void Deliver(CTuple tuple)
{
// Console.WriteLine($"Logging deliver: {tuple.ToString()}");
if (Logger != null)
{
Logger.Notify(new Record("tuple", SenderID, tuple.ToString(), DateTime.Now));
}
Console.WriteLine($"Delivering {tuple}");
}
public void Insert(CTuple tuple)
{
try
{
Buffer?.Add(tuple);
}
catch (Exception e)
{
Console.WriteLine($"//DEBUG: The tuple was not inserted due to {e.GetBaseException()}//");
}
}
public override void Deliver(CTuple tuple)
{
// Console.WriteLine($"{GetHashCode()}: Delivering Tuple {tuple.ToString()} to {tuple.destinationId}.");
int id = 0;
lock (this)
{
id = tuple.destinationId == -1 ? RoutingStrategy.ChooseReplica(tuple) : tuple.destinationId;
tuple.destinationId = id;
}
// Console.WriteLine($"{GetHashCode()}: Consulting semantic {Semantic}");
var rep = replicas[id];
somethingSentInRecentPast[id] = true;
//if (tuple.GetFields() == null) //Console.WriteLine($"Delivering flush {tuple.ID} to {id}");
switch (Semantic)
{
case Semantic.AtLeastOnce:
case Semantic.ExactlyOnce:
controlFlag = false;
while (!controlFlag)
{
rep = replicas[id]; //maybe someone updated it
try
{
rep.ProcessAndForward(tuple, id);
controlFlag = true;
}
//FIXME Exceção que faz timeout automatico
catch (Exception e) { Console.WriteLine($"AtLeastOnce : {e.Message}"); };
}
break;
case Semantic.AtMostOnce:
rep.ProcessAndForward(tuple, id);
break;
default:
Console.WriteLine($"The specified semantic ({Semantic}) is not supported within our system");
break;
}
lock (this) {
if (tuple.GetFields() != null)
{
Console.WriteLine($"Delivered tuple {tuple.ID} to {info.ID} ({id})");
}
CachedOutputTuples.Add(tuple);
SentTupleIds[id] = tuple.ID;
}
}
public override void Finish()
{
for (int i = 0; i < replicas.Count; i++)
{
var flushTuple = new CTuple(null, 1000000, 0, master.OperatorId, master.ID);
flushTuple.destinationId = i;
Insert(flushTuple);
}
base.Finish();
}
public void Init()
{
Task.Run(() =>
{
foreach (var path in inputFiles)
{
StartProcessingFromFile(path, StartFrom);
}
var tuple = new CTuple(null, Int32.MaxValue, 0, OperatorId, 0);
originOperators[OperatorId][0].Insert(tuple);
});
}
private int Choose(IList <CTuple> tuples)
{
CTuple currentMin = tuples[0];
int currentMinIndex = 0;
for (int i = 1; i < tuples.Count; i++)
{
var newTuple = tuples[i];
if (newTuple.ID < currentMin.ID || (newTuple.ID == currentMin.ID && i > currentMinIndex))
{
currentMinIndex = i;
currentMin = newTuple;
}
}
return(currentMinIndex);
}
private IEnumerable <CTuple> Process(CTuple tuple)
{
IEnumerable <CTuple> resultTuples = null;
// debug print
var data = tuple.GetFields();
IEnumerable <IList <string> > resultData;
var startSubId = LastProcessedId.GlobalID == tuple.ID.GlobalID ? LastProcessedId.SubID + 1 : 0;
var origin = originOperators[tuple.opName][tuple.repID];
if (origin.LastProcessedId < tuple.ID)
{
if (data == null)
{
origin.LastProcessedId = tuple.ID;
//this.LastProcessedId = new TupleID(tuple.ID.GlobalID, startSubId);
return(new CTuple[0]);
}
else
{
resultData = ProcessFunction.Process(data);
}
origin.LastProcessedId = tuple.ID;
}
else
{
// Console.WriteLine($"Already seen {tuple.ID} from {origin.OpId} ({origin.ReplicaId}). Ignoring.");
return(new CTuple[0]);
}
resultTuples = resultData.Select((tupleData, i) => new CTuple(tupleData.ToList(), tuple.ID.GlobalID, startSubId + i, this.OperatorId, this.ID));
if (resultTuples.Any())
{
this.LastProcessedId = resultTuples.Last().ID;
}
// Console.WriteLine($"Processed {tuple.ToString()}");
return(resultTuples);
}
override public void DoStuff(CTuple tuple)
{
if (Interval > 0)
{
// Console.WriteLine("Interval!");
Thread.Sleep(Interval);
}
lock (this)
{
while (!Processing || FreezeFlag)
{
// Console.WriteLine("Stopped or frozen. Waiting...");
Monitor.Wait(this);
//Console.WriteLine("UnStopped or Unfrozen. Resuming...");
}
}
Deliver(tuple);
}
private void Processor()
{
while (!Buffer.IsCompleted)
{
// Blocks if number.Count == 0
// IOE means that Take() was called on a completed collection.
// Some other thread can call CompleteAdding after we pass the
// IsCompleted check but before we call Take.
// In this example, we can simply catch the exception since the
// loop will break on the next iteration.
try
{
LastTakenTuple = Buffer.Take();
}
catch (InvalidOperationException) { }
if (LastTakenTuple != null)
{
DoStuff(LastTakenTuple);
}
}
}
public override void Flush(TupleID flushId)
{
for (int i = 0; i < replicas.Count; i++)
{
if (!somethingSentInRecentPast[i])
{
// Console.WriteLine($"Checking if need to flush. {SentTupleIds[i]} < {flushId}");
if (flushId >= new TupleID(0, 0) && SentTupleIds[i] < flushId)
{
// Console.WriteLine($"Emitting flush {master.LastSentId} from {master.ID} to {i}");
try
{
var flushTuple = new CTuple(null, flushId.GlobalID, flushId.SubID, master.OperatorId, master.ID);
flushTuple.destinationId = i;
Insert(flushTuple);
}
catch (Exception)
{
Console.WriteLine("Error while flushing");
}
}
}
}
}
public void ProcessAndForward(CTuple tuple, int destinationId)
{ //ExactlyOnce semantic
replicas[destinationId].ProcessAndForward(tuple);
}
public override void DoStuff(CTuple tuple)
{
throw new NotImplementedException();
// Not needed
}
public abstract void DoStuff(CTuple tuple);
public virtual CTuple Take()
{
return(LastTakenTuple = Buffer.Take());
}
abstract public int ChooseReplica(CTuple tuple);
public void Send(CTuple tuple)
{
// Console.WriteLine($"Inserting {tuple.ID}");
Insert(tuple);
// Console.WriteLine($"Inserted {tuple.ID}");
}
public override void Deliver(CTuple tuple)
{
Console.WriteLine($"Writing to file: {tuple}");
OutputStream.WriteLine(tuple);
OutputStream.Flush();
}
public abstract void Deliver(CTuple tuple);
public override int ChooseReplica(CTuple tuple)
{
//TODO: verify if "crash flag" is on - after checkpoint
// verify that it exists at least one element in the list
return(0);
}
