// Handle output of cdr recs
static void outputCDRRecs(CDRrecord r)
{
//Console.WriteLine("RecordType,SystemIdentity,FileNum,SwitchNum,CallingNum,CallingIMSI,CalledNum,CalledIMSI,Date,Time,TimeType,CallPeriod,CallingCellID,CalledCellID,ServiceType");
//Console.WriteLine(r);
try
{
List <Task> tasks = new List <Task>();
var serializedString = JsonConvert.SerializeObject(r);
EventData data = new EventData(Encoding.UTF8.GetBytes(serializedString))
{
PartitionKey = r.CallingIMSI
};
partitionCurrent++;
if (partitionCurrent > 8)
{
partitionCurrent = 0;
}
// Send the metric to Event Hub
tasks.Add(client.SendAsync(data));
Console.WriteLine(r);
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Console.WriteLine("Error on send: " + e.Message);
}
}
// Handle output of cdr recs
static void outputCDRRecs(CDRrecord r, GenConfig config)
{
try
{
List <Task> tasks = new List <Task>();
var serializedString = config.isOutputJson ? r.ToJSON() : r.ToCSV();
EventData data = new EventData(Encoding.UTF8.GetBytes(serializedString))
{
PartitionKey = r.CallingIMSI
};
// Send the metric to Event Hub
if (!config.isPrintOnly)
{
tasks.Add(client.SendAsync(data));
}
Console.WriteLine("CDRrecord:" + serializedString);
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Console.WriteLine("Error on send: " + e.Message);
}
}
// Handle output of cdr recs
static void outputCDRRecs(CDRrecord r)
{
try
{
List <Task> tasks = new List <Task>();
var serializedString = JsonConvert.SerializeObject(r);
EventData data = new EventData(Encoding.UTF8.GetBytes(serializedString))
{
PartitionKey = r.CallingIMSI
};
// Send the metric to Event Hub
tasks.Add(client.SendAsync(data));
Console.WriteLine("CDRrecord:" + r);
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Console.WriteLine("Error on send: " + e.Message);
}
}
// Generate data
static void GenerateData(string[] args)
{
Queue callBackQ = new Queue();
// Parse parameters
// Number of cdr records, probability of fraud, and number of hours
GenConfig config = new GenConfig(Int32.Parse(args[0]), float.Parse(args[1]), Int32.Parse(args[2]));
// print statistics
Console.Error.WriteLine(config);
// Start the generation
// Generate the call nos
CallStore mobileNos = new CallStore(100000);
int numCallbackPerFile = (int) (config.nCallBackPercent * config.nCDRPerFile);
// Data generation always start with the current time
double timeAdvancementPerSet = ( 0.0 + config.nDurationHours )/ config.nSets;
Console.Error.WriteLine("Time Increment Per Set: " + timeAdvancementPerSet);
// Start generating per set
DateTime simulationTime = DateTime.Now;
Random r = new Random();
bool invalidRec = false;
bool genCallback = false;
// TOOD: Update this to number of hours
DateTimeOffset ptTime = DateTimeOffset.Now;
DateTimeOffset endTime = ptTime.AddHours(config.nDurationHours);
while ( endTime.Subtract(ptTime) >= TimeSpan.Zero )
{
DateTimeOffset currentTime = ptTime;
//Console.WriteLine(String.Format("{0:yyyyMMdd HHmmss}", simulationTime));
for (int cdr = 0; cdr < config.nCDRPerFile; cdr++)
{
currentTime = ptTime;
// Determine whether to generate an invalid CDR record
double pvalue = r.NextDouble();
if (pvalue < 0.1)
invalidRec = true;
else
invalidRec = false;
// Determine whether there will be a callback
pvalue = r.NextDouble();
if (pvalue >= config.nCallBackPercent)
genCallback = true;
else
genCallback = false;
// Determine called and calling num
int calledIdx = r.Next(0, mobileNos.CallNos.Length);
int callingIdx = r.Next(0, mobileNos.CallNos.Length);
CDRrecord rec = new CDRrecord();
rec.setData("FileNum", "" + cdr);
int switchIdx = r.Next(0, mobileNos.switchCountries.Length);
int switchAltIdx = r.Next(0, mobileNos.switchCountries.Length);
// Find an alternate switch
while (switchAltIdx == switchIdx)
{
switchAltIdx = r.Next(0, mobileNos.switchCountries.Length);
}
rec.setData("SwitchNum", mobileNos.switchCountries[switchIdx]);
if (invalidRec)
{
rec.setData("Date", "F");
rec.setData("Time", "F");
rec.setData("DateTime", "F F");
}
else
{
String callDate = String.Format("{0:yyyyMMdd}", currentTime);
String callTime = String.Format("{0:HHmmss}", currentTime);
rec.setData("Date", callDate);
rec.setData("Time", callTime);
rec.setData("DateTime", callDate + " " + callTime);
String calledNum = mobileNos.CallNos[calledIdx];
String callingNum = mobileNos.CallNos[callingIdx];
rec.setData("CalledNum", calledNum);
rec.setData("CallingNum", callingNum);
// Sim card fraud record
if (genCallback)
{
// For call back the A->B end has duration 0
rec.setData("CallPeriod", "0");
// need to generate another set of no
calledIdx = callingIdx;
callingIdx = r.Next(0, mobileNos.CallNos.Length);
CDRrecord callbackRec = new CDRrecord();
callbackRec.setData("FileNum", "" + cdr);
callbackRec.setData("SwitchNum", mobileNos.switchCountries[switchAltIdx]);
//callbackRec.setData("SwitchNum", "" + (f + 1));
// Pertub second
int pertubs = r.Next(0, 30);
callDate = String.Format("{0:yyyyMMdd}", currentTime);
callTime = String.Format("{0:HHmmss}", currentTime.AddMinutes(pertubs));
callbackRec.setData("Date", callDate);
callbackRec.setData("Time", callTime);
callbackRec.setData("DateTime", callDate + " " + callTime);
// Set it as the same calling IMSI
callbackRec.setData("CallingIMSI", rec.CallingIMSI);
calledNum = mobileNos.CallNos[calledIdx];
callingNum = mobileNos.CallNos[callingIdx];
callbackRec.setData("CalledNum", calledNum);
callbackRec.setData("CallingNum", callingNum);
// Determine duration of call
int callPeriod = r.Next(1, 1000);
callbackRec.setData("CallPeriod", "" + callPeriod);
// Enqueue the call back rec
callBackQ.Enqueue(callbackRec);
cdr++;
}
else
{
int callPeriod = r.Next(1, 800);
rec.setData("CallPeriod", "" + callPeriod);
}
}
// send cdr rec to output
//if (genCallback) Console.Write("callback A->B ");
outputCDRRecs(rec);
if (callBackQ.Count > 0 && (cdr % 7 == 0))
{
CDRrecord drec;
drec = (CDRrecord)callBackQ.Dequeue();
outputCDRRecs(drec);
//Console.Write("callback C->A!");
//outputCDRRecs(s, f, drec);
}
// Sleep for 1000ms
System.Threading.Thread.Sleep(100);
// get the current time after generation
ptTime = DateTimeOffset.Now;
} // cdr
// Clear the remaining entries in the call back queue
if (callBackQ.Count > 0)
{
// need to empty queue
while (callBackQ.Count > 0)
{
CDRrecord dr = (CDRrecord)callBackQ.Dequeue();
outputCDRRecs(dr);
//outputCDRRecs(s, f, dr);
}
}
// close the file
if (writer != null)
{
writer.Flush();
writer.Close();
writer = null;
}
// Advance Time
if (timeAdvancementPerSet < 1.0)
simulationTime = simulationTime.AddMinutes(timeAdvancementPerSet * 60);
else
simulationTime = simulationTime.AddHours(timeAdvancementPerSet);
// Sleep for 1000ms
System.Threading.Thread.Sleep(1000);
} // while - within duration
}
// Handle output of cdr recs
static void outputCDRRecs(CDRrecord r)
{
//Console.WriteLine("RecordType,SystemIdentity,FileNum,SwitchNum,CallingNum,CallingIMSI,CalledNum,CalledIMSI,Date,Time,TimeType,CallPeriod,CallingCellID,CalledCellID,ServiceType");
//Console.WriteLine(r);
try
{
List<Task> tasks = new List<Task>();
var serializedString = JsonConvert.SerializeObject(r);
EventData data = new EventData(Encoding.UTF8.GetBytes(serializedString))
{
PartitionKey = r.CallingIMSI
};
partitionCurrent++;
if (partitionCurrent >8)
partitionCurrent = 0;
// Send the metric to Event Hub
tasks.Add(client.SendAsync(data));
Console.WriteLine("CDRrecord:" + r);
Console.WriteLine("serialized:" + data);
Task.WaitAll(tasks.ToArray());
}
catch (Exception e)
{
Console.WriteLine("Error on send: " + e.Message);
}
}
// Generate data
static void GenerateData(string[] args)
{
Queue callBackQ = new Queue();
// Parse parameters
// Number of cdr records, probability of fraud, and number of hours
GenConfig config = new GenConfig(Int32.Parse(args[0]), float.Parse(args[1]), Int32.Parse(args[2]));
// print statistics
Console.Error.WriteLine(config);
// Start the generation
// Generate the call nos
CallStore mobileNos = new CallStore(100000);
int numCallbackPerFile = (int)(config.nCallBackPercent * config.nCDRPerFile);
// Data generation always start with the current time
double timeAdvancementPerSet = (0.0 + config.nDurationHours) / config.nSets;
Console.Error.WriteLine("Time Increment Per Set: " + timeAdvancementPerSet);
// Start generating per set
DateTime simulationTime = DateTime.Now;
Random r = new Random();
bool invalidRec = false;
bool genCallback = false;
// TOOD: Update this to number of hours
DateTimeOffset ptTime = DateTimeOffset.Now;
DateTimeOffset endTime = ptTime.AddHours(config.nDurationHours);
while (endTime.Subtract(ptTime) >= TimeSpan.Zero)
{
DateTimeOffset currentTime = ptTime;
Console.WriteLine(String.Format("{0:yyyyMMdd HHmmss}", simulationTime));
for (int cdr = 0; cdr < config.nCDRPerFile; cdr++)
{
currentTime = ptTime;
// Determine whether to generate an invalid CDR record
double pvalue = r.NextDouble();
if (pvalue < 0.1)
{
invalidRec = true;
}
else
{
invalidRec = false;
}
// Determine whether there will be a callback
pvalue = r.NextDouble();
if (pvalue >= config.nCallBackPercent)
{
genCallback = true;
}
else
{
genCallback = false;
}
// Determine called and calling num
int calledIdx = r.Next(0, mobileNos.CallNos.Length);
int callingIdx = r.Next(0, mobileNos.CallNos.Length);
CDRrecord rec = new CDRrecord();
rec.setData("FileNum", "" + cdr);
int switchIdx = r.Next(0, mobileNos.switchCountries.Length);
int switchAltIdx = r.Next(0, mobileNos.switchCountries.Length);
// Find an alternate switch
while (switchAltIdx == switchIdx)
{
switchAltIdx = r.Next(0, mobileNos.switchCountries.Length);
}
rec.setData("SwitchNum", mobileNos.switchCountries[switchIdx]);
if (invalidRec)
{
rec.setData("Date", "F");
rec.setData("Time", "F");
rec.setData("DateTime", "F F");
}
else
{
String callDate = String.Format("{0:yyyyMMdd}", currentTime);
String callTime = String.Format("{0:HHmmss}", currentTime);
rec.setData("Date", callDate);
rec.setData("Time", callTime);
rec.setData("DateTime", callDate + " " + callTime);
String calledNum = mobileNos.CallNos[calledIdx];
String callingNum = mobileNos.CallNos[callingIdx];
rec.setData("CalledNum", calledNum);
rec.setData("CallingNum", callingNum);
// Sim card fraud record
if (genCallback)
{
// For call back the A->B end has duration 0
rec.setData("CallPeriod", "0");
// need to generate another set of no
calledIdx = callingIdx;
callingIdx = r.Next(0, mobileNos.CallNos.Length);
CDRrecord callbackRec = new CDRrecord();
callbackRec.setData("FileNum", "" + cdr);
callbackRec.setData("SwitchNum", mobileNos.switchCountries[switchAltIdx]);
//callbackRec.setData("SwitchNum", "" + (f + 1));
// Pertub second
int pertubs = r.Next(0, 30);
callDate = String.Format("{0:yyyyMMdd}", currentTime);
callTime = String.Format("{0:HHmmss}", currentTime.AddMinutes(pertubs));
callbackRec.setData("Date", callDate);
callbackRec.setData("Time", callTime);
callbackRec.setData("DateTime", callDate + " " + callTime);
// Set it as the same calling IMSI
callbackRec.setData("CallingIMSI", rec.CallingIMSI);
calledNum = mobileNos.CallNos[calledIdx];
callingNum = mobileNos.CallNos[callingIdx];
callbackRec.setData("CalledNum", calledNum);
callbackRec.setData("CallingNum", callingNum);
// Determine duration of call
int callPeriod = r.Next(1, 1000);
callbackRec.setData("CallPeriod", "" + callPeriod);
// Enqueue the call back rec
callBackQ.Enqueue(callbackRec);
cdr++;
}
else
{
int callPeriod = r.Next(1, 800);
rec.setData("CallPeriod", "" + callPeriod);
}
}
// send cdr rec to output
//if (genCallback) Console.Write("callback A->B ");
outputCDRRecs(rec);
if (callBackQ.Count > 0 && (cdr % 7 == 0))
{
CDRrecord drec;
drec = (CDRrecord)callBackQ.Dequeue();
outputCDRRecs(drec);
//Console.Write("callback C->A!");
//outputCDRRecs(s, f, drec);
}
// Sleep for 1000ms
System.Threading.Thread.Sleep(100);
// get the current time after generation
ptTime = DateTimeOffset.Now;
} // cdr
// Clear the remaining entries in the call back queue
if (callBackQ.Count > 0)
{
// need to empty queue
while (callBackQ.Count > 0)
{
CDRrecord dr = (CDRrecord)callBackQ.Dequeue();
outputCDRRecs(dr);
//outputCDRRecs(s, f, dr);
}
}
// close the file
if (writer != null)
{
writer.Flush();
writer.Close();
writer = null;
}
// Advance Time
if (timeAdvancementPerSet < 1.0)
{
simulationTime = simulationTime.AddMinutes(timeAdvancementPerSet * 60);
}
else
{
simulationTime = simulationTime.AddHours(timeAdvancementPerSet);
}
// Sleep for 1000ms
System.Threading.Thread.Sleep(1000);
} // while - within duration
}
static void GenerateData(string[] args)
{
Queue callBackQ = new Queue();
// probability of invalid calls, probability of fraud, and number of hours
GenConfig config = new GenConfig(float.Parse(args[0]), float.Parse(args[1]), Int32.Parse(args[2]), Int32.Parse(args[3]));
Console.Error.WriteLine(config);
CallStore mobileNos = new CallStore(100000);
Random r = new Random();
bool invalidRec = false;
bool genCallback = false;
DateTimeOffset currentTime = DateTimeOffset.UtcNow;
DateTimeOffset endTime = currentTime.AddHours(config.nDurationHours);
// power bi free subscription won't allow more than 10,000 records per hour per dataset, so roughly 3 records per second
// we can generate a cdr per second, plus a fraud cdr, plus aggregation record, so at least 1s (1000ms) per record;
// power bi pro allows for 1,000,000 records per hour, so 100 every sec, 6,000 per min, 360,000 per hour, plus agg;
int milliSecPerCDR = config.isPowerBiPro ? 10 : 1000;
int fraudConsecutiveCallInSec = 5;
int total = 0;
int fraudTotal = 0;
while (endTime.Subtract(currentTime) >= TimeSpan.Zero)
{
for (int cdr = 0; cdr < config.nCDRPerFile; cdr++)
{
#region initRecord
// Determine whether to generate an invalid CDR record
double pvalue = r.NextDouble();
invalidRec = (pvalue < config.nInvalidPercent);
// Determine whether there will be a callback
pvalue = r.NextDouble();
genCallback = (pvalue < config.nCallBackPercent);
CDRrecord rec = new CDRrecord(); ++total;
int switchIdx = r.Next(0, mobileNos.switchCountries.Length);
int switchAltIdx = r.Next(0, mobileNos.switchCountries.Length);
while (switchAltIdx == switchIdx)
{
switchAltIdx = r.Next(0, mobileNos.switchCountries.Length);
}
rec.setData("SwitchNum", mobileNos.switchCountries[switchIdx]);
rec.setData("UnitPrice", "" + mobileNos.dollarPerMin[switchIdx]);
int calledIdx = r.Next(0, mobileNos.CallNos.Length);
int callingIdx = r.Next(0, mobileNos.CallNos.Length);
String calledNum = mobileNos.CallNos[calledIdx];
String callingNum = mobileNos.CallNos[callingIdx];
rec.setData("CalledNum", calledNum);
rec.setData("CallingNum", callingNum);
#endregion
if (invalidRec)
{
rec.setData("DateTime", "F F");
}
else
{
String callDate = String.Format("{0:yyyyMMdd}", currentTime);
String callTime = String.Format("{0:HHmmss}", currentTime);
rec.setData("DateTime", callDate + " " + callTime);
int callPeriod = r.Next(1, 80);
rec.setData("CallPeriod", "" + callPeriod);
#region simFraudRecord
if (genCallback)
{
CDRrecord callbackRec = new CDRrecord(); ++fraudTotal;
// Set it as the same calling IMSI
callbackRec.setData("CallingIMSI", rec.CallingIMSI);
callbackRec.setData("SwitchNum", mobileNos.switchCountries[switchAltIdx]);
callbackRec.setData("UnitPrice", "" + mobileNos.dollarPerMin[switchAltIdx]);
// Pertub second
int pertubs = r.Next(0, fraudConsecutiveCallInSec);
callDate = String.Format("{0:yyyyMMdd}", currentTime);
callTime = String.Format("{0:HHmmss}", currentTime.AddSeconds(pertubs));
callbackRec.setData("DateTime", callDate + " " + callTime);
// called number is different but calling number is same
calledIdx = r.Next(0, mobileNos.CallNos.Length);
calledNum = mobileNos.CallNos[calledIdx];
callbackRec.setData("CalledNum", calledNum);
callbackRec.setData("CallingNum", callingNum);
callPeriod = r.Next(1, 60);
callbackRec.setData("CallPeriod", "" + callPeriod);
callBackQ.Enqueue(callbackRec);
cdr++;
}
#endregion
}
outputCDRRecs(rec);
//output those consecutive fraud calls that happened in the past not future
while (callBackQ.Count > 0)
{
CDRrecord drec, prec;
prec = (CDRrecord)callBackQ.Peek();
DateTimeOffset oldest = DateTimeOffset.Parse(prec.callrecTime, null, System.Globalization.DateTimeStyles.AssumeUniversal);
if (currentTime.Subtract(oldest) >= TimeSpan.Zero)
{
drec = (CDRrecord)callBackQ.Dequeue();
outputCDRRecs(drec);
}
else
{
break;
}
}
System.Threading.Thread.Sleep(milliSecPerCDR);
Console.WriteLine("Total:" + total + ", Fraud Total:" + fraudTotal);
currentTime = DateTimeOffset.Now;
} // cdr
#region close the file
if (writer != null)
{
writer.Flush();
writer.Close();
writer = null;
}
#endregion
System.Threading.Thread.Sleep(milliSecPerCDR);
currentTime = DateTimeOffset.Now;
} // while - within duration
}
