public void Benchmark()
{
var sw = Stopwatch.StartNew();
var count = 10_000_000L;
var ts = new List <Task>();
for (var i = 0; i < Environment.ProcessorCount; i++)
{
ts.Add(Task.Run(() =>
{
var f = new FlowId {
BlockOnSampleTime = false
};
for (var i = 0; i < count; i++)
{
var id = f.NewId();
}
}));
}
Task.WaitAll(ts.ToArray());
sw.Stop();
Assert.True(sw.ElapsedMilliseconds < 10_000);
count *= ts.Count;
XTrace.WriteLine("生成 {0:n0}，耗时 {1}，速度 {2:n0}tps", count, sw.Elapsed, count * 1000 / sw.ElapsedMilliseconds);
}
private static void Test2()
{
var sw = Stopwatch.StartNew();
var count = 100_000_000L;
var ts = new List <Task>();
for (var i = 0; i < Environment.ProcessorCount; i++)
{
ts.Add(Task.Run(() =>
{
var f = new FlowId();
for (var i = 0; i < count; i++)
{
var id = f.NewId();
}
}));
}
Task.WaitAll(ts.ToArray());
sw.Stop();
count *= ts.Count;
XTrace.WriteLine("生成 {0:n0}，耗时 {1}，速度 {2:n0}tps", count, sw.Elapsed, count * 1000 / sw.ElapsedMilliseconds);
}
public void NewId()
{
var f = new FlowId();
var id = f.NewId();
var time = id >> 22;
var tt = f.StartTimestamp.AddMilliseconds(time);
Assert.True(tt <= DateTime.Now);
var wid = (id >> 12) & 0x3FF;
Assert.Equal(f.WorkerId, wid);
var seq = id & 0x0FFF;
Assert.Equal(f.Sequence, seq);
// 时间转编号
var id2 = f.GetId(tt);
Assert.Equal(id >> 22, id2 >> 22);
// 分析
var rs = f.TryParse(id, out var t, out var w, out var s);
Assert.True(rs);
Assert.Equal(tt, t);
Assert.Equal(wid, w);
Assert.Equal(seq, s);
}
protected void GivenAnInitializedFlowConfig()
{
var baseDirectoryName = "Flow2";
var baseDirectory = $@"{Environment.CurrentDirectory}\{baseDirectoryName}";
if (Directory.Exists(baseDirectory))
{
Directory.Delete(baseDirectory, true);
}
_flow = new FlowId("ConfigureFlow");
var flowConfig = new FlowConfiguration(_flow)
{
BasePath = baseDirectory
};
var def1 = flowConfig.AddController <TestCsvFlowController>("ProcessorA");
def1.FieldValidationRules.Add(new ValidationRule
{
FieldName = "Value",
ErrorMessage = "Value must be numeric",
RegEx = "^[0-9]*$"
});
// configuration path
def1.ConfigurationDetails.Add("ValueToConfigure", "http://helplnk.etc");
flowConfig.AddController <TestProcessResultFlowFileController>("ProcessorB");
_flowConfig = flowConfig;
}
/// <summary>
/// Creates a new instance.
/// </summary>
/// <param name="config">Processor configuration settings.</param>
/// <param name="reader">The reader to read incoming data from.</param>
/// <param name="getController">Function to create a new procesor.</param>
public FlowFileController(
ProcessorConfiguration <TIn, TOut> config,
IReader <TIn> reader,
Func <IFlowFileController, Processor <TIn, TOut> > getController = null)
{
Guard.ArgumentNotNull(reader, nameof(reader));
Guard.ArgumentNotNull(config, nameof(config));
_getFlowFileController = getController;
_flowFileLogRepo = new FlowFileLogRepo();
// ReSharper disable once ConvertIfStatementToNullCoalescingExpression
if (_getFlowFileController == null)
{
_getFlowFileController = controller => throw new NotImplementedException();
}
Config = config;
// assign name from type name by default
ControllerName = $"{GetType().Name.Replace("`2", "")}-{typeof(TIn).Name}-{typeof(TOut).Name}";
Flow = new FlowId(typeof(TOut).Name);
_reader = reader;
}
protected void WhenCreatingANewFlow()
{
var files = Directory.GetFiles(Environment.CurrentDirectory, $"*{_flowCode}*.json");
foreach (var file in files)
{
File.Delete(file);
}
_flow1 = _flowService.CreateNew(_flowCode);
}
protected void AndGivenANewFlow()
{
// cleanup any existing file
foreach (var file in Directory.GetFiles(Environment.CurrentDirectory, "*Test*.json"))
{
File.Delete(file);
}
this._flowService = new FlowService(new FlowIdRepo());
this._flow = this._flowService.CreateNew("Test");
this._flowBatch = this._flowService.GetNewFlowBatch("Test");
}
/// <summary>
/// Gets a new instance using a given process configuration operating against a given working folder.
/// </summary>
public EnricherController(ProcessorConfiguration <TTarget, TTarget> config, string dataDirectory = null)
{
var processorConfiguration = config;
// targetDirectory workingFolder default to current workingFolder
DataDir = new DirectoryInfo(dataDirectory ?? Environment.CurrentDirectory);
_logRepo = new EnricherLogRepository()
{
DataDir = DataDir.FullName
};
this._entityRules = processorConfiguration.Rules.ToArray();
this._entityCollection = processorConfiguration.CollectionRules.ToArray();
this._flow = new FlowId("FlowABD");
}
public void ValidRepeat()
{
var sw = Stopwatch.StartNew();
var ws = new ConcurrentBag <Int32>();
var repeat = new ConcurrentBag <Int64>();
var hash = new ConcurrentHashSet <Int64>();
var ts = new List <Task>();
for (var k = 0; k < 10; k++)
{
ts.Add(Task.Run(() =>
{
var f = new FlowId {
StartTimestamp = new DateTime(2020, 1, 1), WorkerId = Rand.Next() & 0x3FF
};
ws.Add(f.WorkerId);
for (var i = 0; i < 100_000; i++)
{
var id = f.NewId();
if (!hash.TryAdd(id))
{
repeat.Add(id);
}
}
}));
}
Task.WaitAll(ts.ToArray());
sw.Stop();
Assert.True(sw.ElapsedMilliseconds < 10_000);
var count = repeat.Count;
Assert.Equal(0, count);
}
public override string ToString()
{
return(string.Format("[{0}] {1}: Id={2}, From=({3})", FlowId == 0 ? "#" : FlowId.ToString(), Type, LabelId, From));
}
public override string ToString()
{
return(string.Format("[{0}] {1}: Hint={2}, Value={3}", FlowId == 0 ? "#" : FlowId.ToString(), Type, Hint, Value));
}
protected void AndGivenAConsistentProcessId()
{
_flow = new FlowId("TestFlow1");
}
/// <summary>时间专用区间函数</summary>
/// <param name="fi"></param>
/// <param name="start">起始时间，大于等于</param>
/// <param name="end">结束时间，小于。如果是日期，则加一天</param>
/// <param name="flow"></param>
/// <returns></returns>
public static Expression Between(this FieldItem fi, DateTime start, DateTime end, FlowId flow)
{
if (fi.Type != typeof(Int64))
{
throw new NotSupportedException($"[{nameof(Between)}]函数仅支持Int64字段！");
}
var exp = new WhereExpression();
if (fi == null)
{
return(exp);
}
if (start <= DateTime.MinValue || start >= DateTime.MaxValue)
{
if (end <= DateTime.MinValue || end >= DateTime.MaxValue)
{
return(exp);
}
// 如果只有日期，则加一天，表示包含这一天
if (end == end.Date)
{
end = end.AddDays(1);
}
return(fi < flow.GetId(end));
}
else
{
exp &= fi >= flow.GetId(start);
if (end <= DateTime.MinValue || end >= DateTime.MaxValue)
{
return(exp);
}
// 如果只有日期，则加一天，表示包含这一天
if (start == start.Date && end == end.Date)
{
end = end.AddDays(1);
}
return(exp & fi < flow.GetId(end));
}
}
protected void GivenAFlowAndABatchProcess()
{
this._flow = new FlowId("Test");
this._flowBatch = new FlowBatch(_flow, 1);
}
public override string ToString()
{
return(string.Format("[{0}] {1}: {2}, Ops={3}", FlowId == 0 ? "#" : FlowId.ToString(), Type, InstructionId, string.Join(", ", Operands.ToList().Select(o => o.ToString()))));
}
