private void button1_Click(object sender, EventArgs e)
{
var time = XXF.Log.TimeWatchLog.Debug(() =>
{
int count = Convert.ToInt32(this.tbcount.Text.Trim());
//System.Threading.Tasks.Parallel.For(0, count, new System.Threading.Tasks.ParallelOptions() { MaxDegreeOfParallelism = Convert.ToInt32(tbbingxing.Text.Trim()) }, (i) =>
//{
List<System.Threading.Tasks.Task> tasks = new List<System.Threading.Tasks.Task>();
for (int i = 0; i < count; i++)
{
var t = new System.Threading.Tasks.TaskFactory().StartNew(() =>
{
if (XXF.ProjectTool.MQHelper.SendMessage<string>(
config, //管理中心数据库
this.tbPath.Text, //队列路径 .分隔,类似类的namespace,是队列的唯一标识，要提前告知运维在消息中心注册，方可使用。
ms + ms) == false)//发送对象
{
throw new Exception("发送失败");
}
});
tasks.Add(t);
}
System.Threading.Tasks.Task.WaitAll(tasks.ToArray());
});
//});
MessageBox.Show("耗时:" + time);
//string timeline = "";
//foreach (var m in ProducterTimeWatchTest.Messages)
//{
// timeline += m + "\r\n";
//}
//System.IO.File.WriteAllText("timewatch.report.txt", timeline);
}
public void TestRecieveEmissionsAndChangeSpeed()
{
//create class
var vehicle = new VehicleSimulator.Vehicle(30, new List <VehicleSimulator.WayPoint>()
{
new WayPoint(33.833769, -117), new WayPoint(33.82652, -117), new WayPoint(33.833769, -117)
});
//attach listener
var listen = new Listener(vehicle);
var factory = new System.Threading.Tasks.TaskFactory();
//start the vehicle
var task = factory.StartNew(() => vehicle.Drive());
//wait 10 seconds
System.Threading.Thread.Sleep(30000);
vehicle.ChangeSpeed(60);
//wait for the vehicle to finish
task.Wait();
Console.WriteLine(listen.CoordinateLog.Split(" ".ToCharArray()).Where(s => s == "Got").Count());
//parse the listener's log to get the number of coordinate events that happened
Assert.IsTrue(listen.CoordinateLog.Split(" ".ToCharArray()).Where(str => str == "Got").Count() == 22);
}
EvaluateAsync(
System.Threading.Tasks.TaskFactory factory)
{
this.EvaluationTask = factory.StartNew(
() => this.EvaluateInternal()
);
}
CheckIfModulesNeedRebuilding(
System.Type metaType)
{
// not all build modes need to determine if modules are up-to-date
var evaluationRequiredAttr =
metaType.GetCustomAttributes(typeof(EvaluationRequiredAttribute), false) as EvaluationRequiredAttribute[];
if (0 == evaluationRequiredAttr.Length)
{
Log.DebugMessage("No Bam.Core.EvaluationRequired attribute on build mode metadata, assume rebuilds necessary");
return(false);
}
if (!evaluationRequiredAttr[0].Enabled)
{
Log.DebugMessage("Module evaluation disabled");
return(false);
}
Log.DebugMessage("Module evaluation enabled");
var cancellationSource = new System.Threading.CancellationTokenSource();
var cancellationToken = cancellationSource.Token;
// LongRunning is absolutely necessary in order to achieve paralleism
var creationOpts = System.Threading.Tasks.TaskCreationOptions.LongRunning;
var continuationOpts = System.Threading.Tasks.TaskContinuationOptions.LongRunning;
var threadCount = 1;
var scheduler = new LimitedConcurrencyLevelTaskScheduler(threadCount);
var factory = new System.Threading.Tasks.TaskFactory(
cancellationToken,
creationOpts,
continuationOpts,
scheduler);
var graph = Graph.Instance;
graph.MetaData = factory;
foreach (var rank in graph.Reverse())
{
foreach (Module module in rank)
{
module.Evaluate();
}
}
return(true);
}
private PictureList FetchPictures(List <Picture> wallResults, bool previewOnly)
{
var result = new PictureList()
{
FetchDate = DateTime.Now
};
try
{
System.Threading.Tasks.TaskFactory tf = new System.Threading.Tasks.TaskFactory();
wallResults
.AsParallel()
.WithDegreeOfParallelism(10)
.ForAll(delegate(Picture p){
try
{
//save original URL as referrer
p.Properties.Add(Picture.StandardProperties.Referrer, p.Url);
p.Properties.Add(Picture.StandardProperties.BanImageKey, Path.GetFileName(p.Url));
p.Properties.Add(Picture.StandardProperties.ProviderLabel, "Wallhaven");
//get actual image URL if this is not a preview
if (!previewOnly)
{
p.Url = GetDirectPictureUrl(p.Url);
}
p.Id = System.IO.Path.GetFileNameWithoutExtension(p.Url);
if (!string.IsNullOrEmpty(p.Url) && !string.IsNullOrEmpty(p.Id))
{
result.Pictures.Add(p);
}
}
catch (Exception ex)
{
Log.Logger.Write(string.Format("Error downloading picture object from '{0}'. Exception details: {0}", ex.ToString()), Log.LoggerLevels.Errors);
}
finally
{
}
});
}
catch (Exception ex) {
Log.Logger.Write(string.Format("Error during multi-threaded wallbase.cc image get. Exception details: {0}", ex.ToString()), Log.LoggerLevels.Errors);
}
finally
{
}
return(result);
}
/// <summary>
/// 异步读
/// </summary>
/// <param name="p_stream"></param>
/// <param name="p_buffer"></param>
/// <param name="p_offset"></param>
/// <param name="p_count"></param>
/// <param name="p_cancellationToken"></param>
/// <returns></returns>
public static Task <int> ReadAsync(this Stream p_stream, byte[] p_buffer, int p_offset, int p_count, CancellationToken p_cancellationToken)
{
if (p_cancellationToken.IsCancellationRequested)
{
TaskCompletionSource <int> source = new TaskCompletionSource <int>();
source.SetCanceled();
return(source.Task);
}
System.Threading.Tasks.TaskFactory factory = System.Threading.Tasks.Task.Factory;
Stream stream2 = p_stream;
Stream stream3 = p_stream;
return(factory.FromAsync(new System.Func <byte[], int, int, AsyncCallback, object, IAsyncResult>(stream2.BeginRead), new Func <IAsyncResult, int>(stream3.EndRead), p_buffer, p_offset, p_count, null));
}
PerformThreadedEvaluation(
Array <Module> modulesNeedEvaluating)
{
var graph = Graph.Instance;
using (var cancellationSource = new System.Threading.CancellationTokenSource())
{
var cancellationToken = cancellationSource.Token;
// LongRunning is absolutely necessary in order to achieve paralleism
var creationOpts = System.Threading.Tasks.TaskCreationOptions.LongRunning;
var continuationOpts = System.Threading.Tasks.TaskContinuationOptions.LongRunning;
var threadCount = 1;
var scheduler = new LimitedConcurrencyLevelTaskScheduler(threadCount);
var factory = new System.Threading.Tasks.TaskFactory(
cancellationToken,
creationOpts,
continuationOpts,
scheduler);
if (null == modulesNeedEvaluating)
{
Log.DebugMessage("Module evaluation enabled for build mode {0}", graph.Mode);
foreach (var rank in graph.Reverse())
{
foreach (Module module in rank)
{
Log.DebugMessage("\tEvaluation for module {0}", module.GetType().ToString());
module.EvaluateAsync(factory);
}
}
}
else
{
Log.DebugMessage("Module evaluation disabled for build mode {0}, but enabled for {1} individual modules:", graph.Mode, modulesNeedEvaluating.Count);
foreach (var module in modulesNeedEvaluating)
{
Log.DebugMessage("\tEvaluation for module {0}", module.GetType().ToString());
module.EvaluateAsync(factory);
}
}
}
}
public void run()
{
// 主线线程
System.Console.WriteLine("A #{0}: {1}", thid, count++);
var facory = new System.Threading.Tasks.TaskFactory();
facory.StartNew(() => {
// System.Threading.Thread.Sleep(1 * 1000);
// 子线程
System.Console.WriteLine("B #{0}: {1}", thid, count);
});
facory.StartNew(() => {
// 子线程
System.Console.WriteLine("C #{0}: {1}", thid, count++);
});
System.Threading.Thread.Sleep(10 * 1000);
}
private void button1_Click(object sender, EventArgs e)
{
var time = XXF.Log.TimeWatchLog.Debug(() =>
{
int count = Convert.ToInt32(this.tbcount.Text.Trim());
//System.Threading.Tasks.Parallel.For(0, count, new System.Threading.Tasks.ParallelOptions() { MaxDegreeOfParallelism = Convert.ToInt32(tbbingxing.Text.Trim()) }, (i) =>
//{
List <System.Threading.Tasks.Task> tasks = new List <System.Threading.Tasks.Task>();
for (int i = 0; i < count; i++)
{
var t = new System.Threading.Tasks.TaskFactory().StartNew(() =>
{
if (XXF.ProjectTool.MQHelper.SendMessage <string>(
config, //管理中心数据库
this.tbPath.Text, //队列路径 .分隔,类似类的namespace,是队列的唯一标识，要提前告知运维在消息中心注册，方可使用。
ms + ms) == false) //发送对象
{
throw new Exception("发送失败");
}
});
tasks.Add(t);
}
System.Threading.Tasks.Task.WaitAll(tasks.ToArray());
});
//});
MessageBox.Show("耗时:" + time);
//string timeline = "";
//foreach (var m in ProducterTimeWatchTest.Messages)
//{
// timeline += m + "\r\n";
//}
//System.IO.File.WriteAllText("timewatch.report.txt", timeline);
}
public void ExecuteCandidate(DateTime nowDate)
{
m_currentTestHour = nowDate.Hour;
foreach (var kvp in m_classifierInfos)
{
kvp.Value.Initialized = false;
}
if (TestParameters.EnnableLoadTestData)
{
if (!TestParameters.OnlyNewestTestDataSaved)
{
CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
}
else
{
DateTime?maxTestDate = CCScoreData.Instance.GetNewestTestData();
if (maxTestDate.HasValue && maxTestDate.Value > nowDate)
{
return;
}
if (maxTestDate.HasValue && maxTestDate.Value == nowDate)
{
CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
}
}
}
WekaData.GenerateArffTemplate(true, true, this.CandidateParameter);
string candidateClsInfoSummary = "CC:N={0},TrN={1},TeN={4},NC={7},NTP={8},NFP={9},NDA={14},NDS={15},TD={10},TV={11},Cls={12},MM={13}";
//Instances testInstancesWithoutClassValue = null; / wrong, should use closeTime
System.Threading.Tasks.Task[] tasks;
System.Threading.Tasks.TaskFactory taskFactory = null;
if (TestParameters.EnableMultiThread)
{
int cpuCount = Environment.ProcessorCount - 2;
cpuCount = Math.Max(cpuCount, 1);
LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(1);
taskFactory = new System.Threading.Tasks.TaskFactory(lcts);
}
if (!TestParameters.EnablePerhourTrain)
{
tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count];
}
else
{
tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count / Parameters.AllHour];
}
int taskIdx = 0;
foreach (var kvp in m_classifierInfos)
{
if (TestParameters.EnablePerhourTrain)
{
if (kvp.Value.Hour != m_currentTestHour)
{
continue;
}
}
var clsInfo = kvp.Value;
Action action = () =>
{
if (clsInfo.Initialized || nowDate > System.DateTime.Now)
{
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
-1, 0, 0,
clsInfo.CurrentClassValue.Length, 0, 0,
clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
}
}
else
{
weka.core.Instances trainInstances;
weka.core.Instances testInstances;
bool noData = false;
if (TestParameters.UseTrain)
{
clsInfo.WekaData.GenerateData(true, true);
trainInstances = clsInfo.WekaData.CurrentTrainInstances;
testInstances = clsInfo.WekaData.CurrentTestInstances;
if (trainInstances.numInstances() == 0)
{
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format("{0} - No Train Data", clsInfo.Name));
}
noData = true;
}
}
else
{
clsInfo.WekaData.GenerateData(false, true);
trainInstances = WekaData.GetTrainInstancesTemplate(this.CandidateParameter.Name);
testInstances = clsInfo.WekaData.CurrentTestInstances;
}
if (testInstances.numInstances() == 0)
{
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format("{0} - No Test Data", clsInfo.Name));
}
noData = true;
}
if (!noData)
{
clsInfo.WekaData.TrainandTest(clsInfo, this.CandidateParameter);
clsInfo.Deals.Now(nowDate, WekaUtils.GetValueFromInstance(testInstances, "mainClose", 0));
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
trainInstances.numInstances(), WekaUtils.GetDateValueFromInstances(trainInstances, 0, 0), WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1),
testInstances.numInstances(), WekaUtils.GetDateValueFromInstances(testInstances, 0, 0), WekaUtils.GetDateValueFromInstances(testInstances, 0, testInstances.numInstances() - 1),
clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
}
}
else
{
clsInfo.CurrentClassValue = Parameters.DoubleArrayEmpty;
clsInfo.CurrentTestRet = Parameters.DoubleArrayEmpty;
clsInfo.Deals.Now(nowDate, null);
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
trainInstances.numInstances(), 0, 0,
testInstances.numInstances(), 0, 0,
clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
}
}
if (TestParameters.EnnableLoadTestData)
{
CCScoreData.Instance.SaveTestData(clsInfo, nowDate);
}
if (TestParameters.EnableDetailLog)
{
// Check TestInstance Result
SortedDictionary <int, int> closeNums = new SortedDictionary <int, int>();
for (int i = 0; i < testInstances.numInstances(); ++i)
{
if (clsInfo.CurrentTestRet[i] != 2)
{
continue;
}
DateTime openDate = WekaUtils.GetDateValueFromInstances(testInstances, 0, i);
DateTime closeDate = WekaUtils.GetDateValueFromInstances(testInstances, 1, i);
WekaUtils.DebugAssert(openDate >= nowDate, "openDate >= nowDate");
int n = (int)(closeDate - nowDate).TotalHours / (TestParameters.BatchTestMinutes / 60) + 1;
if (!closeNums.ContainsKey(n))
{
closeNums[n] = 1;
}
else
{
closeNums[n]++;
}
}
foreach (var cn in closeNums)
{
WekaUtils.Instance.WriteLog(string.Format(" Next {0} Period-{1} has {2} deals",
cn.Key, nowDate.AddHours(TestParameters.BatchTestMinutes / 60 * cn.Key), cn.Value));
}
}
}
clsInfo.WekaData.Clear();
};
if (TestParameters.EnableMultiThread)
{
tasks[taskIdx] = taskFactory.StartNew(action);
taskIdx++;
}
else
{
action();
}
}
if (TestParameters.EnableMultiThread)
{
try
{
System.Threading.Tasks.Task.WaitAll(tasks);
}
catch (AggregateException ex)
{
foreach (var i in ex.InnerExceptions)
{
WekaUtils.Instance.WriteLog(i.Message);
WekaUtils.Instance.WriteLog(i.StackTrace);
}
}
}
if (TestParameters.SaveCCScoresToDb)
{
CCScoreData.SaveCCScoresToDb(nowDate, this.CandidateParameter, this);
}
}
/// <summary>
/// Creates an object that schedules units of work using the provided TaskFactory.
/// </summary>
public TaskPoolScheduler(System.Threading.Tasks.TaskFactory taskFactory)
{
this.taskFactory = taskFactory;
}
/// <inheritdoc />
public TaskFactory()
{
_taskFactory = new System.Threading.Tasks.TaskFactory();
}
private void WriteLog(string message, LogLevel level)
{
if (!Active)
return;
if (level > _level)
return;
System.Threading.Tasks.TaskFactory factory = new System.Threading.Tasks.TaskFactory();
factory.StartNew(() =>
{
lock (_syncObject)
{
// TEMP Log solution
using (var sw = new System.IO.StreamWriter(@"c:\temp\mr.log", true))
{
sw.WriteLine(string.Format("{0} [{1}] {2}", DateTime.Now, level, message));
sw.Close();
}
}
});
}
/// <summary>
/// Launches the target process async on a thread under the control of this
/// app domain.
/// </summary>
public void BeginProc()
{
try
{
var launcher = new System.Threading.Tasks.TaskFactory();
var hostProc = launcher.StartNew(() => StartProc());
}
catch (Exception ex)
{
WriteLogEntry(ex);
}
}
/// <summary>
/// Initializes a TaskFactory<TResult> instance with the specified configuration.
/// </summary>
/// <param name="p_scheduler"></param>
/// <param name="p_cancellationToken"></param>
internal TaskFactory(TaskScheduler p_scheduler, CancellationToken p_cancellationToken)
{
this.m_factory = new TaskFactory(p_scheduler, p_cancellationToken);
}
CheckIfModulesNeedRebuilding(
System.Type metaType)
{
// not all build modes need to determine if modules are up-to-date
var evaluationRequiredAttr =
metaType.GetCustomAttributes(typeof(EvaluationRequiredAttribute), false) as EvaluationRequiredAttribute[];
if (0 == evaluationRequiredAttr.Length)
{
Log.DebugMessage("No Bam.Core.EvaluationRequired attribute on build mode metadata, assume rebuilds necessary");
return false;
}
if (!evaluationRequiredAttr[0].Enabled)
{
Log.DebugMessage("Module evaluation disabled");
return false;
}
Log.DebugMessage("Module evaluation enabled");
var cancellationSource = new System.Threading.CancellationTokenSource();
var cancellationToken = cancellationSource.Token;
// LongRunning is absolutely necessary in order to achieve paralleism
var creationOpts = System.Threading.Tasks.TaskCreationOptions.LongRunning;
var continuationOpts = System.Threading.Tasks.TaskContinuationOptions.LongRunning;
var threadCount = 1;
var scheduler = new LimitedConcurrencyLevelTaskScheduler(threadCount);
var factory = new System.Threading.Tasks.TaskFactory(
cancellationToken,
creationOpts,
continuationOpts,
scheduler);
var graph = Graph.Instance;
graph.MetaData = factory;
foreach (var rank in graph.Reverse())
{
foreach (Module module in rank)
{
module.Evaluate();
}
}
return true;
}
Run()
{
Log.Detail("Running build...");
// TODO: should the rank collections be sorted, so that modules with fewest dependencies are first?
var graph = Graph.Instance;
var metaDataType = graph.BuildModeMetaData.GetType();
var useEvaluation = CheckIfModulesNeedRebuilding(metaDataType);
var explainRebuild = CommandLineProcessor.Evaluate(new Options.ExplainBuildReason());
var immediateOutput = CommandLineProcessor.Evaluate(new Options.ImmediateOutput());
ExecutePreBuild(metaDataType);
// necessary if built with debug symbols
IOWrapper.CreateDirectoryIfNotExists(graph.BuildRoot);
var threadCount = CommandLineProcessor.Evaluate(new Options.MultiThreaded());
if (0 == threadCount)
{
threadCount = System.Environment.ProcessorCount;
}
System.Exception abortException = null;
if (threadCount > 1)
{
using (var cancellationSource = new System.Threading.CancellationTokenSource())
{
var cancellationToken = cancellationSource.Token;
// LongRunning is absolutely necessary in order to achieve paralleism
var creationOpts = System.Threading.Tasks.TaskCreationOptions.LongRunning;
var continuationOpts = System.Threading.Tasks.TaskContinuationOptions.LongRunning;
var scheduler = new LimitedConcurrencyLevelTaskScheduler(threadCount);
var factory = new System.Threading.Tasks.TaskFactory(
cancellationToken,
creationOpts,
continuationOpts,
scheduler);
var tasks = new Array <System.Threading.Tasks.Task>();
foreach (var rank in graph.Reverse())
{
foreach (var module in rank)
{
var context = new ExecutionContext(useEvaluation, explainRebuild, immediateOutput);
var task = factory.StartNew(() =>
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var depTasks = new Array <System.Threading.Tasks.Task>();
foreach (var dep in module.Dependents)
{
if (null == dep.ExecutionTask)
{
continue;
}
depTasks.Add(dep.ExecutionTask);
}
foreach (var dep in module.Requirements)
{
if (null == dep.ExecutionTask)
{
continue;
}
depTasks.Add(dep.ExecutionTask);
}
System.Threading.Tasks.Task.WaitAll(depTasks.ToArray());
if (cancellationToken.IsCancellationRequested)
{
return;
}
try
{
(module as IModuleExecution).Execute(context);
}
catch (Exception ex)
{
abortException = ex;
cancellationSource.Cancel();
}
finally
{
if (context.OutputStringBuilder != null && context.OutputStringBuilder.Length > 0)
{
Log.Info(context.OutputStringBuilder.ToString());
}
if (context.ErrorStringBuilder != null && context.ErrorStringBuilder.Length > 0)
{
Log.Info(context.ErrorStringBuilder.ToString());
}
}
});
tasks.Add(task);
module.ExecutionTask = task;
}
}
try
{
System.Threading.Tasks.Task.WaitAll(tasks.ToArray());
}
catch (System.AggregateException exception)
{
if (!(exception.InnerException is System.Threading.Tasks.TaskCanceledException))
{
throw new Exception(exception, "Error during threaded build");
}
}
}
}
else
{
foreach (var rank in graph.Reverse())
{
if (null != abortException)
{
break;
}
foreach (IModuleExecution module in rank)
{
var context = new ExecutionContext(useEvaluation, explainRebuild, immediateOutput);
try
{
module.Execute(context);
}
catch (Exception ex)
{
abortException = ex;
break;
}
finally
{
if (context.OutputStringBuilder != null && context.OutputStringBuilder.Length > 0)
{
Log.Info(context.OutputStringBuilder.ToString());
}
if (context.ErrorStringBuilder != null && context.ErrorStringBuilder.Length > 0)
{
Log.Info(context.ErrorStringBuilder.ToString());
}
}
}
}
}
if (null != abortException)
{
throw new Exception(abortException, "Error during {0}threaded build", (threadCount > 1) ? string.Empty : "non-");
}
ExecutePostBuild(metaDataType);
}
CheckIfModulesNeedRebuilding(
System.Type metaType)
{
var graph = Graph.Instance;
var modulesNeedEvaluating = new Array <Module>();
// not all build modes need to determine if modules are up-to-date
var evaluationRequiredAttr =
metaType.GetCustomAttributes(typeof(EvaluationRequiredAttribute), false) as EvaluationRequiredAttribute[];
if (0 == evaluationRequiredAttr.Length)
{
// query if any individual modules override this
foreach (var rank in graph.Reverse())
{
foreach (Module module in rank)
{
var moduleEvaluationRequiredAttr = module.GetType().GetCustomAttributes(typeof(EvaluationRequiredAttribute), true) as EvaluationRequiredAttribute[];
if (moduleEvaluationRequiredAttr.Length > 0 && moduleEvaluationRequiredAttr[0].Enabled)
{
modulesNeedEvaluating.Add(module);
}
}
}
if (0 == modulesNeedEvaluating.Count)
{
Log.DebugMessage("No Bam.Core.EvaluationRequired attribute on build mode metadata, assume rebuilds necessary");
return(false);
}
}
if ((evaluationRequiredAttr.Length > 0) && !evaluationRequiredAttr[0].Enabled && 0 == modulesNeedEvaluating.Count)
{
Log.DebugMessage("Module evaluation disabled");
return(false);
}
using (var cancellationSource = new System.Threading.CancellationTokenSource())
{
var cancellationToken = cancellationSource.Token;
// LongRunning is absolutely necessary in order to achieve paralleism
var creationOpts = System.Threading.Tasks.TaskCreationOptions.LongRunning;
var continuationOpts = System.Threading.Tasks.TaskContinuationOptions.LongRunning;
var threadCount = 1;
var scheduler = new LimitedConcurrencyLevelTaskScheduler(threadCount);
var factory = new System.Threading.Tasks.TaskFactory(
cancellationToken,
creationOpts,
continuationOpts,
scheduler);
graph.MetaData = factory;
if (0 == modulesNeedEvaluating.Count)
{
Log.DebugMessage("Module evaluation enabled for build mode {0}", graph.Mode);
foreach (var rank in graph.Reverse())
{
foreach (Module module in rank)
{
module.Evaluate();
}
}
}
else
{
Log.DebugMessage("Module evaluation disabled for build mode {0}, but enabled for individual modules:", graph.Mode);
foreach (var module in modulesNeedEvaluating)
{
Log.DebugMessage("\tEvaluation for module {0}", module.GetType().ToString());
module.Evaluate();
}
}
}
return(true);
}
public void ExecuteCandidate(DateTime nowDate)
{
m_currentTestHour = nowDate.Hour;
foreach (var kvp in m_classifierInfos)
{
kvp.Value.Initialized = false;
}
if (TestParameters.EnnableLoadTestData)
{
if (!TestParameters.OnlyNewestTestDataSaved)
{
CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
}
else
{
DateTime? maxTestDate = CCScoreData.Instance.GetNewestTestData();
if (maxTestDate.HasValue && maxTestDate.Value > nowDate)
return;
if (maxTestDate.HasValue && maxTestDate.Value == nowDate)
CCScoreData.Instance.LoadTestData(nowDate, m_classifierInfos);
}
}
WekaData.GenerateArffTemplate(true, true, this.CandidateParameter);
string candidateClsInfoSummary = "CC:N={0},TrN={1},TeN={4},NC={7},NTP={8},NFP={9},NDA={14},NDS={15},TD={10},TV={11},Cls={12},MM={13}";
//Instances testInstancesWithoutClassValue = null; / wrong, should use closeTime
System.Threading.Tasks.Task[] tasks;
System.Threading.Tasks.TaskFactory taskFactory = null;
if (TestParameters.EnableMultiThread)
{
int cpuCount = Environment.ProcessorCount - 2;
cpuCount = Math.Max(cpuCount, 1);
LimitedConcurrencyLevelTaskScheduler lcts = new LimitedConcurrencyLevelTaskScheduler(1);
taskFactory = new System.Threading.Tasks.TaskFactory(lcts);
}
if (!TestParameters.EnablePerhourTrain)
{
tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count];
}
else
{
tasks = new System.Threading.Tasks.Task[m_classifierInfos.Count / Parameters.AllHour];
}
int taskIdx = 0;
foreach (var kvp in m_classifierInfos)
{
if (TestParameters.EnablePerhourTrain)
{
if (kvp.Value.Hour != m_currentTestHour)
continue;
}
var clsInfo = kvp.Value;
Action action = () =>
{
if (clsInfo.Initialized || nowDate > System.DateTime.Now)
{
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
-1, 0, 0,
clsInfo.CurrentClassValue.Length, 0, 0,
clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
}
}
else
{
weka.core.Instances trainInstances;
weka.core.Instances testInstances;
bool noData = false;
if (TestParameters.UseTrain)
{
clsInfo.WekaData.GenerateData(true, true);
trainInstances = clsInfo.WekaData.CurrentTrainInstances;
testInstances = clsInfo.WekaData.CurrentTestInstances;
if (trainInstances.numInstances() == 0)
{
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format("{0} - No Train Data", clsInfo.Name));
}
noData = true;
}
}
else
{
clsInfo.WekaData.GenerateData(false, true);
trainInstances = WekaData.GetTrainInstancesTemplate(this.CandidateParameter.Name);
testInstances = clsInfo.WekaData.CurrentTestInstances;
}
if (testInstances.numInstances() == 0)
{
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format("{0} - No Test Data", clsInfo.Name));
}
noData = true;
}
if (!noData)
{
clsInfo.WekaData.TrainandTest(clsInfo, this.CandidateParameter);
clsInfo.Deals.Now(nowDate, WekaUtils.GetValueFromInstance(testInstances, "mainClose", 0));
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
trainInstances.numInstances(), WekaUtils.GetDateValueFromInstances(trainInstances, 0, 0), WekaUtils.GetDateValueFromInstances(trainInstances, 0, trainInstances.numInstances() - 1),
testInstances.numInstances(), WekaUtils.GetDateValueFromInstances(testInstances, 0, 0), WekaUtils.GetDateValueFromInstances(testInstances, 0, testInstances.numInstances() - 1),
clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
}
}
else
{
clsInfo.CurrentClassValue = Parameters.DoubleArrayEmpty;
clsInfo.CurrentTestRet = Parameters.DoubleArrayEmpty;
clsInfo.Deals.Now(nowDate, null);
if (m_enableDetailLogLevel2)
{
WekaUtils.Instance.WriteLog(string.Format(candidateClsInfoSummary, clsInfo.Name,
trainInstances.numInstances(), 0, 0,
testInstances.numInstances(), 0, 0,
clsInfo.Deals.NowScore.ToString(Parameters.DoubleFormatString),
clsInfo.Deals.NowTp, clsInfo.Deals.NowFp,
clsInfo.Deals.TotalDeal, clsInfo.Deals.TotalVolume.ToString("N2"),
clsInfo.Classifier == null ? string.Empty : clsInfo.Classifier.ToString(),
clsInfo.MoneyManagement == null ? string.Empty : clsInfo.MoneyManagement.ToString(),
clsInfo.Deals.DealLastTimeAvg, clsInfo.Deals.DealLastTimeStd));
}
}
if (TestParameters.EnnableLoadTestData)
{
CCScoreData.Instance.SaveTestData(clsInfo, nowDate);
}
if (TestParameters.EnableDetailLog)
{
// Check TestInstance Result
SortedDictionary<int, int> closeNums = new SortedDictionary<int, int>();
for (int i = 0; i < testInstances.numInstances(); ++i)
{
if (clsInfo.CurrentTestRet[i] != 2)
continue;
DateTime openDate = WekaUtils.GetDateValueFromInstances(testInstances, 0, i);
DateTime closeDate = WekaUtils.GetDateValueFromInstances(testInstances, 1, i);
WekaUtils.DebugAssert(openDate >= nowDate, "openDate >= nowDate");
int n = (int)(closeDate - nowDate).TotalHours / (TestParameters.BatchTestMinutes / 60) + 1;
if (!closeNums.ContainsKey(n))
{
closeNums[n] = 1;
}
else
{
closeNums[n]++;
}
}
foreach (var cn in closeNums)
{
WekaUtils.Instance.WriteLog(string.Format(" Next {0} Period-{1} has {2} deals",
cn.Key, nowDate.AddHours(TestParameters.BatchTestMinutes / 60 * cn.Key), cn.Value));
}
}
}
clsInfo.WekaData.Clear();
};
if (TestParameters.EnableMultiThread)
{
tasks[taskIdx] = taskFactory.StartNew(action);
taskIdx++;
}
else
{
action();
}
}
if (TestParameters.EnableMultiThread)
{
try
{
System.Threading.Tasks.Task.WaitAll(tasks);
}
catch (AggregateException ex)
{
foreach (var i in ex.InnerExceptions)
{
WekaUtils.Instance.WriteLog(i.Message);
WekaUtils.Instance.WriteLog(i.StackTrace);
}
}
}
if (TestParameters.SaveCCScoresToDb)
{
CCScoreData.SaveCCScoresToDb(nowDate, this.CandidateParameter, this);
}
}
private void btnTest_Click(object sender, EventArgs e)
{
this.txtLogView.Clear();
this.btnTest.Enabled = false;
var model = new ModelTaskLabel();
model.TestViewUrl = this.txtTestUrl.Text;
model.TaskID = TaskID;
model.LabelName = this.txtLabelName.Text;
model.LabelNameCutRegex = this.txtLabelNameCutRegex.Text.Replace("'", "''");
model.LabelRemove = string.Empty;
model.LabelReplace = string.Empty;
#region Html¹ýÂË
model.LabelHtmlRemove = string.Empty;
if (this.chkAllHtml.Checked)
{
model.LabelHtmlRemove += "all||||";
}
if (this.chkTable.Checked)
{
model.LabelHtmlRemove += "table||||";
}
if (this.chkFont.Checked)
{
model.LabelHtmlRemove += "font<span>||||";
}
if (this.chkhref.Checked)
{
model.LabelHtmlRemove += "a||||";
}
if (this.chkScript.Checked)
{
model.LabelHtmlRemove += "script||||";
}
if (model.LabelHtmlRemove.Trim() != "")
{
model.LabelHtmlRemove = model.LabelHtmlRemove.Remove(model.LabelHtmlRemove.Length - 4);
}
#endregion
#region ÄÚÈÝÅųý
foreach (ListViewItem item in this.listViewContentRemove.Items)
{
model.LabelRemove += item.SubItems[0].Text + "||" + item.SubItems[1].Text + "$$$$";
}
if (model.LabelRemove.Trim() != "")
{
model.LabelRemove = model.LabelRemove.Remove(model.LabelRemove.Length - 4);
}
#endregion
#region ÄÚÈÝÌæ»»
foreach (ListViewItem item in this.listViewContentReplace.Items)
{
model.LabelReplace += item.SubItems[0].Text + "||" + item.SubItems[1].Text + "$$$$";
}
if (model.LabelReplace.Trim() != "")
{
model.LabelReplace = model.LabelReplace.Remove(model.LabelReplace.Length - 4);
}
#endregion
model.IsLoop = this.chkLabelIsLoop.Checked ? 1 : 0;
model.SpiderLabelPlugin = this.cmbSpiderPlugin.Text;
model.IsDownResource = this.chkDownResource.Checked ? 1 : 0;
model.DownResourceExts = this.txtDownResourceExt.Text;
var task = new System.Threading.Tasks.TaskFactory().StartNew(() => {
var pageContent = CommonHelper.getPageContent(model.TestViewUrl, PageEncode);
var spider = new SpiderViewHelper();
var s = spider.TestSingleLabel(model, pageContent);
this.Invoke(new MethodInvoker(() => {
this.btnTest.Enabled = true;
this.txtLogView.AppendText(s);
this.txtLogView.AppendText("\r\n");
}));
});
}
void Foo()
{
System.Threading.Tasks.TaskFactory m = null;
m.StartNew(() => delegate { });
}
Run()
{
Log.Detail("Running build");
// TODO: should the rank collections be sorted, so that modules with fewest dependencies are first?
var graph = Graph.Instance;
var metaDataType = graph.BuildModeMetaData.GetType();
var useEvaluation = CheckIfModulesNeedRebuilding(metaDataType);
var explainRebuild = CommandLineProcessor.Evaluate(new Options.ExplainBuildReason());
var immediateOutput = CommandLineProcessor.Evaluate(new Options.ImmediateOutput());
ExecutePreBuild(metaDataType);
if (!System.IO.Directory.Exists(graph.BuildRoot))
{
System.IO.Directory.CreateDirectory(graph.BuildRoot);
}
var threadCount = CommandLineProcessor.Evaluate(new Options.MultiThreaded());
if (0 == threadCount)
{
threadCount = System.Environment.ProcessorCount;
}
System.Exception abortException = null;
if (threadCount > 1)
{
var cancellationSource = new System.Threading.CancellationTokenSource();
var cancellationToken = cancellationSource.Token;
// LongRunning is absolutely necessary in order to achieve paralleism
var creationOpts = System.Threading.Tasks.TaskCreationOptions.LongRunning;
var continuationOpts = System.Threading.Tasks.TaskContinuationOptions.LongRunning;
var scheduler = new LimitedConcurrencyLevelTaskScheduler(threadCount);
var factory = new System.Threading.Tasks.TaskFactory(
cancellationToken,
creationOpts,
continuationOpts,
scheduler);
var tasks = new Array<System.Threading.Tasks.Task>();
foreach (var rank in graph.Reverse())
{
foreach (var module in rank)
{
var context = new ExecutionContext(useEvaluation, explainRebuild, immediateOutput);
var task = factory.StartNew(() =>
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var depTasks = new Array<System.Threading.Tasks.Task>();
foreach (var dep in module.Dependents)
{
if (null == dep.ExecutionTask)
{
continue;
}
depTasks.Add(dep.ExecutionTask);
}
foreach (var dep in module.Requirements)
{
if (null == dep.ExecutionTask)
{
continue;
}
depTasks.Add(dep.ExecutionTask);
}
System.Threading.Tasks.Task.WaitAll(depTasks.ToArray());
if (cancellationToken.IsCancellationRequested)
{
return;
}
try
{
(module as IModuleExecution).Execute(context);
}
catch (Exception ex)
{
abortException = ex;
cancellationSource.Cancel();
}
finally
{
if (context.OutputStringBuilder != null && context.OutputStringBuilder.Length > 0)
{
Log.Info(context.OutputStringBuilder.ToString());
}
if (context.ErrorStringBuilder != null && context.ErrorStringBuilder.Length > 0)
{
Log.Info(context.ErrorStringBuilder.ToString());
}
}
});
tasks.Add(task);
module.ExecutionTask = task;
}
}
try
{
System.Threading.Tasks.Task.WaitAll(tasks.ToArray());
}
catch (System.AggregateException exception)
{
if (!(exception.InnerException is System.Threading.Tasks.TaskCanceledException))
{
throw new Exception(exception, "Error during threaded build");
}
}
}
else
{
foreach (var rank in graph.Reverse())
{
if (null != abortException)
{
break;
}
foreach (IModuleExecution module in rank)
{
var context = new ExecutionContext(useEvaluation, explainRebuild, immediateOutput);
try
{
module.Execute(context);
}
catch (Exception ex)
{
abortException = ex;
break;
}
finally
{
if (context.OutputStringBuilder != null && context.OutputStringBuilder.Length > 0)
{
Log.Info(context.OutputStringBuilder.ToString());
}
if (context.ErrorStringBuilder != null && context.ErrorStringBuilder.Length > 0)
{
Log.Info(context.ErrorStringBuilder.ToString());
}
}
}
}
}
if (null != abortException)
{
throw new Exception(abortException, "Error during {0}threaded build", (threadCount > 1) ? string.Empty : "non-");
}
ExecutePostBuild(metaDataType);
}
/// <summary>
/// Creates an object that schedules units of work using the provided TaskFactory.
/// </summary>
public TaskPoolScheduler(System.Threading.Tasks.TaskFactory taskFactory)
{
this.taskFactory = taskFactory;
}
