/// <summary>
/// Get
/// </summary>
/// <param name="requestUri"></param>
/// <param name="json"></param>
/// <returns></returns>
public static IRestResponse PostComplexData(string requestUri, RestRequest request, int time = 7000)
{
if (time == 0 || time > timeout)
{
time = timeout;
}
request.Timeout = time;
try
{
ThreadLocalData entity = (ThreadLocalData)ThreadSlot.LogicalGetData().Clone();
if (entity != null)
{
string clientID = ThreadSlot.GetClientID();//System.Configuration.ConfigurationSettings.AppSettings["ClientID"];
if (!string.IsNullOrEmpty(clientID))
{
if (clientID.Split('|')[1] == "1")//终端(后台)
{
entity.TraceSecondId = clientID.Split('|')[0];
entity.TraceId = Guid.NewGuid().ToString("N");
}
}
request.AddHeader("BTProcessInfo", JsonHelper.SerializeObject(entity));
}
}
catch (Exception ex)
{
LogService.Default.Fatal(ex, "post请求添加追踪日志异常：" + ex.Message + ex.StackTrace);
}
var restClient = new RestClient {
BaseUrl = new Uri(requestUri)
};
IRestResponse r = restClient.Execute(request);
return(r);
}
/// <summary>
/// Creates a local data object.
/// </summary>
/// <returns></returns>
private ThreadLocalData CreateLocalData()
{
var threadLocalData = new ThreadLocalData();
threadLocalData.MatchesArrayThreadLocal =
new List <FilterHandle>(100);
threadLocalData.ScheduleArrayThreadLocal =
new ArrayBackedCollection <ScheduleHandle>(100);
if (_isPrioritized)
{
threadLocalData.MatchesPerStmtThreadLocal =
new OrderedDictionary <EPStatementAgentInstanceHandle, LinkedList <FilterHandleCallback> >(new EPStatementAgentInstanceHandlePrioritySort());
threadLocalData.SchedulePerStmtThreadLocal =
new OrderedDictionary <EPStatementAgentInstanceHandle, Object>(new EPStatementAgentInstanceHandlePrioritySort(true));
}
else
{
threadLocalData.MatchesPerStmtThreadLocal =
new Dictionary <EPStatementAgentInstanceHandle, LinkedList <FilterHandleCallback> >(10000);
threadLocalData.SchedulePerStmtThreadLocal =
new Dictionary <EPStatementAgentInstanceHandle, Object>(10000);
}
return(threadLocalData);
}
/// <summary>
/// GET接口方法
/// </summary>
/// <param name="requestUri"></param>
/// <param name="time"></param>
/// <returns></returns>
public static string GetHttpRequest(string requestUri, Int32 time = 7000)
{
try
{
HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create(requestUri);
myRequest.UseDefaultCredentials = true;
myRequest.ContentType = "application/json; charset=utf-8";
myRequest.Accept = "application/json";
myRequest.Method = "GET";
if (time == 0 || time > timeout)
{
time = timeout;
}
myRequest.Timeout = time;
//线程信息传递
//BaseThreadEntity entity = BaseThread.GetTraceInfo();
//if (entity.TraceId != null && entity.TraceId != "")
// myRequest.Headers.Add("MyProcessInfo", JsonHelper.ObjectToJson(entity));
try
{
ThreadLocalData entity = (ThreadLocalData)ThreadSlot.LogicalGetData().Clone();
if (entity != null)
{
string clientID = ThreadSlot.GetClientID();//System.Configuration.ConfigurationSettings.AppSettings["ClientID"];
if (!string.IsNullOrEmpty(clientID))
{
if (clientID.Split('|')[1] == "1")//终端(后台)
{
entity.TraceSecondId = clientID.Split('|')[0];
entity.TraceId = Guid.NewGuid().ToString("N");
}
}
myRequest.Headers.Add("BTProcessInfo", JsonHelper.SerializeObject(entity));
}
}
catch (Exception ex)
{
LogService.Default.Fatal(ex, "post请求添加追踪日志异常：" + ex.Message + ex.StackTrace);
}
string ReqResult = string.Empty;
HttpWebResponse myResponse = null;
myResponse = (HttpWebResponse)myRequest.GetResponse();
StreamReader reader = new StreamReader(myResponse.GetResponseStream(), Encoding.UTF8);
ReqResult = reader.ReadToEnd();
reader.Close();
myResponse.Close();
return(ReqResult);
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// Post
/// </summary>
/// <param name="requestUri"></param>
/// <param name="json"></param>
/// <returns></returns>
public static IRestResponse PostComplexData(string requestUri, RestRequest request, string medieaURI = "",
string fileName = "", string jsonBody = "", Int32 time = 7000, string version = "1.0")
{
request.Method = Method.POST;
if (time == 0 || time > timeout)
{
time = timeout;
}
request.Timeout = time;
try
{
ThreadLocalData entity = (ThreadLocalData)ThreadSlot.LogicalGetData().Clone();
if (entity != null)
{
string clientID = ThreadSlot.GetClientID();//System.Configuration.ConfigurationSettings.AppSettings["ClientID"];
if (!string.IsNullOrEmpty(clientID))
{
if (clientID.Split('|')[1] == "1")//终端(后台)
{
entity.TraceSecondId = clientID.Split('|')[0];
entity.TraceId = Guid.NewGuid().ToString("N");
}
}
request.AddHeader("BTProcessInfo", JsonHelper.SerializeObject(entity));
}
}
catch (Exception ex)
{
LogService.Default.Fatal(ex, "post请求添加追踪日志异常：" + ex.Message + ex.StackTrace);
}
if (!string.IsNullOrEmpty(medieaURI))
{
byte[] bufferBest = GetImage(medieaURI);
if (string.IsNullOrEmpty(fileName))
{
fileName = new Guid().ToString("N");
}
request.AddFileBytes(fileName, bufferBest, fileName);
}
if (!string.IsNullOrEmpty(jsonBody))
{
request.AddBody(jsonBody);
}
request.Timeout = time;
var restClient = new RestClient {
BaseUrl = new Uri(requestUri)
};
IRestResponse r = restClient.Execute(request);
return(r);
}
public static void SimpleThreads()
{
IRun tc = new ThreadingClass();
//tc.ThreadMethod();
IRun st = new StopingThread();
//st.Run();
IRun tsat = new ThreadStaticAttributeTest();
//tsat.Run();
IRun tld = new ThreadLocalData();
tld.Run();
IRun tpt = new ThreadPoolTests();
tpt.Run();
}
public ParallelTreeTrainingOperation(
Random random,
ITrainingContext <F, S> trainingContext,
TrainingParameters parameters,
int maxThreads,
IDataPointCollection data,
ProgressWriter progress)
{
data_ = data;
trainingContext_ = trainingContext;
parameters_ = parameters;
maxThreads_ = maxThreads;
random_ = random;
progress_ = progress;
parentStatistics_ = trainingContext_.GetStatisticsAggregator();
leftChildStatistics_ = trainingContext_.GetStatisticsAggregator();
rightChildStatistics_ = trainingContext_.GetStatisticsAggregator();
responses_ = new float[data.Count()];
indices_ = new int[data.Count()];
for (int i = 0; i < indices_.Length; i++)
{
indices_[i] = i;
}
threadLocals_ = new ThreadLocalData[maxThreads_];
for (int threadIndex = 0; threadIndex < maxThreads_; threadIndex++)
{
threadLocals_[threadIndex] = new ThreadLocalData(random_, trainingContext_, parameters_, data_);
}
}
public override Object invoke(ThreadLocalData __tld, Object AMP__rest, Object args)
{
return args;
}
public override Object invoke(ThreadLocalData __tld, Object list)
{
return clojure__last.fn.invoke(__tld, list, Num.ONE);
}
public override Object invoke(ThreadLocalData __tld, Object x)
{
return x;
}
public override Object invoke(ThreadLocalData __tld, Object X)
{
if(__tld == null) __tld = ThreadLocalData.get();
return COMMON_LISP__T.getValue(__tld);
}
public override Object invoke(ThreadLocalData __tld, Object x)
{
if(x != null)
{
return clojure__first.fn.invoke(__tld, clojure__rest.fn.invoke(__tld, x));
}
else
{
return null;
}
}
public override Object invoke(ThreadLocalData __tld, Object x, Object y)
{
return Reflector.invokeStaticMethod("equiv","org.clojure.runtime.Num",new Object[]{x,y});
}
public override Object invoke(ThreadLocalData __tld, Object x)
{
return ((clojure__consQMARK__.fn.invoke(__tld, x))==null?RT.T:null);
}
public override Object invoke(ThreadLocalData __tld, Object fun, Object alist, Object keys)
{
if(clojure__atomQMARK__.fn.invoke(__tld, alist) != null)
{
return null;
}
else
{
if((((clojure__consQMARK__.fn.invoke(__tld, clojure__first.fn.invoke(__tld, alist)) != null))?((IFn)fun).invoke(__tld, (((IFn)KEY__key).invoke(__tld, keys) != null?((IFn)((IFn)KEY__key).invoke(__tld, keys)).invoke(__tld, clojure__ffirst.fn.invoke(__tld, alist)):clojure__ffirst.fn.invoke(__tld, alist))):null) != null)
{
return clojure__first.fn.invoke(__tld, alist);
}
else
{
return clojure__assoc_if.fn.invoke(__tld, fun, clojure__rest.fn.invoke(__tld, alist), keys);
}
}
}
public override Object invoke(ThreadLocalData __tld, Object fun, Object alist)
{
return clojure__assoc_if.fn.invoke(__tld, fun, alist, null);
}
public override Object invoke(ThreadLocalData __tld, Object item, Object alist, Object keys)
{
if(__tld == null) __tld = ThreadLocalData.get();
Object G2165__1;
return clojure__assoc_if.fn.invoke(__tld, clojure__fn.fn.invoke(__tld, clojure__y.fn.invoke(__tld), ((IFn)((((G2165__1 = ((IFn)KEY__test).invoke(__tld, keys)) != null)||((G2165__1 = clojure__eql.getValue(__tld)) != null))?G2165__1:null)).invoke(__tld, item, clojure__y.getValue(__tld))), alist, keys);
}
public override Object invoke(ThreadLocalData __tld, Object item, Object alist)
{
return clojure__assoc.fn.invoke(__tld, item, alist, null);
}
public override Object doInvoke(ThreadLocalData __tld, Object fn, Cons argsPLUS__)
{
if(__tld == null) __tld = ThreadLocalData.get();
return ((IFn)ACC__clojure__applyTo).invoke(__tld, fn, clojure____tld.getValue(__tld), clojure__spreadSTAR__.fn.invoke(__tld, argsPLUS__));
}
public override Object invoke(ThreadLocalData __tld, Object list, Object n)
{
return clojure__nreverse.fn.invoke(__tld, clojure__nthcdr.fn.invoke(__tld, n, clojure__reverse.fn.invoke(__tld, list)));
}
public override Object invoke(ThreadLocalData __tld, Object x)
{
if(x != null)
{
return ((IFn)ACC__clojure__first).invoke(__tld, x);
}
else
{
return null;
}
}
public override Object invoke(ThreadLocalData __tld, Object n)
{
if(COMMON_LISP__EQ__.fn.invoke(__tld, n, Num.ONE) != null)
{
return Num.ONE;
}
else
{
return COMMON_LISP__STAR__.fn.invoke(__tld, n, clojure__fact.fn.invoke(__tld, COMMON_LISP__NUM__1_.fn.invoke(__tld, n)));
}
}
public override Object invoke(ThreadLocalData __tld, Object fn)
{
return (new FN__2163(fn));
}
public override Object invoke(ThreadLocalData __tld, Object args)
{
return clojure__prn.fn.invoke(__tld, clojure__fact.fn.invoke(__tld, Num.from(50)));
}
public override Object doInvoke(ThreadLocalData __tld, Cons args)
{
return ((clojure__apply.fn.invoke(__tld, fn, args))==null?RT.T:null);
}
public override Object doInvoke(ThreadLocalData __tld, Object X, Object Y, Cons REST)
{
if(__tld == null) __tld = ThreadLocalData.get();
return (((Reflector.invokeStaticMethod("gte","org.clojure.runtime.Num",new Object[]{X,Y}) != null))?clojure__apply.fn.invoke(__tld, COMMON_LISP__GT__EQ__.getValue(__tld), Y, REST):null);
}
public override Object invoke(ThreadLocalData __tld, Object x)
{
return (new FN__2164(x));
}
public override Object invoke(ThreadLocalData __tld, Object X, Object Y)
{
return Reflector.invokeStaticMethod("gte","org.clojure.runtime.Num",new Object[]{X,Y});
}
public override Object doInvoke(ThreadLocalData __tld, Cons args)
{
return x;
}
public override Object invoke(ThreadLocalData __tld, Object x)
{
return (x is org.clojure.runtime.IntegerNum?RT.T:null);
}
public override Object invoke(ThreadLocalData __tld, Object list)
{
Box cl__1 = new Box(null);
cl__1.val = (new FN__2166(cl__1));
return clojure__cons.fn.invoke(__tld, clojure__first.fn.invoke(__tld, list), ((IFn)cl__1.val).invoke(__tld, clojure__rest.fn.invoke(__tld, list)));
}
public override Object invoke(ThreadLocalData __tld, Object list, Object n)
{
Object G2169__1;
Object G2168__2;
Object G2167__3;
Object i__4;
Object r__5;
Object l__6;
l__6 = list;
r__5 = list;
i__4 = Num.ZERO;
for(;;)
{
if(clojure__nullQMARK__.fn.invoke(__tld, l__6) != null)
{
return r__5;
}
if(COMMON_LISP__GT__EQ__.fn.invoke(__tld, i__4, n) != null)
{
clojure__pop.fn.invoke(__tld, r__5);
}
G2167__3 = clojure__rest.fn.invoke(__tld, l__6);
G2168__2 = null;
G2169__1 = COMMON_LISP__NUM__1PLUS__.fn.invoke(__tld, i__4);
l__6 = G2167__3;
r__5 = G2168__2;
i__4 = G2169__1;
}
}
public override Object invoke(ThreadLocalData __tld, Object x)
{
if(clojure__atomQMARK__.fn.invoke(__tld, x) != null)
{
return x;
}
else
{
return clojure__cons.fn.invoke(__tld, clojure__first.fn.invoke(__tld, x), ((IFn)cl__1.val).invoke(__tld, clojure__rest.fn.invoke(__tld, x)));
}
}
public override Object doInvoke(ThreadLocalData __tld, Cons args)
{
return clojure__spreadSTAR__.fn.invoke(__tld, args);
}
public override Object invoke(ThreadLocalData __tld, Object tree)
{
if(clojure__atomQMARK__.fn.invoke(__tld, tree) != null)
{
return tree;
}
else
{
return clojure__cons.fn.invoke(__tld, clojure__copy_tree.fn.invoke(__tld, clojure__first.fn.invoke(__tld, tree)), clojure__copy_tree.fn.invoke(__tld, clojure__rest.fn.invoke(__tld, tree)));
}
}
public override Object invoke(ThreadLocalData __tld, Object x, Object y)
{
return ((IFn)ACC__clojure__Equals).invoke(__tld, x, y);
}
public override Object doInvoke(ThreadLocalData __tld, Object x, Object y, Cons rest)
{
if(__tld == null) __tld = ThreadLocalData.get();
return (((Reflector.invokeStaticMethod("equiv","org.clojure.runtime.Num",new Object[]{x,y}) != null))?clojure__apply.fn.invoke(__tld, COMMON_LISP__EQ__.getValue(__tld), y, rest):null);
}
public void TrainNodesRecurse(Node <F, S>[] nodes, int nodeIndex, int i0, int i1, int recurseDepth)
{
System.Diagnostics.Debug.Assert(nodeIndex < nodes.Length);
nodes[nodeIndex] = new Node <F, S>();
progress_.Write(Verbosity.Verbose, "{0}{1}: ", Tree <F, S> .GetPrettyPrintPrefix(nodeIndex), i1 - i0);
// First aggregate statistics over the samples at the parent node
parentStatistics_.Clear();
for (int i = i0; i < i1; i++)
{
parentStatistics_.Aggregate(data_, indices_[i]);
}
// Copy parent statistics to thread local storage in case client
// IStatisticsAggregator implementations are not reentrant.
for (int t = 0; t < maxThreads_; t++)
{
threadLocals_[t].parentStatistics_ = parentStatistics_.DeepClone();
}
if (nodeIndex >= nodes.Length / 2) // this is a leaf node, nothing else to do
{
nodes[nodeIndex].InitializeLeaf(parentStatistics_);
progress_.WriteLine(Verbosity.Verbose, "Terminating at max depth.");
return;
}
// Iterate over threads
Parallel.For(0, maxThreads_, new Action <int>(threadIndex =>
{
ThreadLocalData tl = threadLocals_[threadIndex]; // shorthand
tl.Clear();
// Range of indices of candidate feature evaluated in this thread
// (if the number of candidate features is not a multiple of the
// number of threads, some threads may evaluate one more feature
// than others).
int f1 = (int)(Math.Round(parameters_.NumberOfCandidateFeatures * (double)threadIndex / (double)maxThreads_));
int f2 = (int)(Math.Round(parameters_.NumberOfCandidateFeatures * (double)(threadIndex + 1) / (double)maxThreads_));
// Iterate over candidate features
for (int f = f1; f < f2; f++)
{
F feature = trainingContext_.GetRandomFeature(tl.random_);
for (int b = 0; b < parameters_.NumberOfCandidateThresholdsPerFeature + 1; b++)
{
threadLocals_[threadIndex].partitionStatistics_[b].Clear(); // reset statistics
}
// Compute feature response per sample at this node
for (int i = i0; i < i1; i++)
{
tl.responses_[i] = feature.GetResponse(data_, indices_[i]);
}
int nThresholds;
if ((nThresholds = ParallelTreeTrainingOperation <F, S> .ChooseCandidateThresholds(
tl.random_,
indices_, i0, i1,
tl.responses_,
parameters_.NumberOfCandidateThresholdsPerFeature,
ref tl.thresholds)) == 0)
{
continue; // failed to find meaningful candidate thresholds for this feature
}
// Aggregate statistics over sample partitions
for (int i = i0; i < i1; i++)
{
// Slightly faster than List<float>.BinarySearch() for < O(100) thresholds
int b = 0;
while (b < nThresholds && tl.responses_[i] >= tl.thresholds[b])
{
b++;
}
tl.partitionStatistics_[b].Aggregate(data_, indices_[i]);
}
for (int t = 0; t < nThresholds; t++)
{
tl.leftChildStatistics_.Clear();
tl.rightChildStatistics_.Clear();
for (int p = 0; p < nThresholds + 1 /*i.e. nBins*/; p++)
{
if (p <= t)
{
tl.leftChildStatistics_.Aggregate(tl.partitionStatistics_[p]);
}
else
{
tl.rightChildStatistics_.Aggregate(tl.partitionStatistics_[p]);
}
}
// Compute gain over sample partitions
double gain = trainingContext_.ComputeInformationGain(tl.parentStatistics_, tl.leftChildStatistics_, tl.rightChildStatistics_);
if (gain >= tl.maxGain)
{
tl.maxGain = gain;
tl.bestFeature = feature;
tl.bestThreshold = tl.thresholds[t];
}
}
}
}));
double maxGain = 0.0;
F bestFeature = default(F);
float bestThreshold = 0.0f;
// Now merge over threads
for (int threadIndex = 0; threadIndex < maxThreads_; threadIndex++)
{
ThreadLocalData tl = threadLocals_[threadIndex];
if (tl.maxGain > maxGain)
{
maxGain = tl.maxGain;
bestFeature = tl.bestFeature;
bestThreshold = tl.bestThreshold;
}
}
if (maxGain == 0.0)
{
nodes[nodeIndex].InitializeLeaf(parentStatistics_);
progress_.WriteLine(Verbosity.Verbose, "Terminating with no beneficial split found.");
return;
}
// Now reorder the data point indices using the winning feature and thresholds.
// Also recompute child node statistics so the client can decide whether
// to terminate training of this branch.
leftChildStatistics_.Clear();
rightChildStatistics_.Clear();
for (int i = i0; i < i1; i++)
{
responses_[i] = bestFeature.GetResponse(data_, indices_[i]);
if (responses_[i] < bestThreshold)
{
leftChildStatistics_.Aggregate(data_, indices_[i]);
}
else
{
rightChildStatistics_.Aggregate(data_, indices_[i]);
}
}
if (trainingContext_.ShouldTerminate(parentStatistics_, leftChildStatistics_, rightChildStatistics_, maxGain))
{
nodes[nodeIndex].InitializeLeaf(parentStatistics_);
progress_.WriteLine(Verbosity.Verbose, "Terminating because supplied termination criterion is satisfied.");
return;
}
// Otherwise this is a new split node, recurse for children.
nodes[nodeIndex].InitializeSplit(
bestFeature,
bestThreshold,
parentStatistics_.DeepClone());
// Now do partition sort - any sample with response greater goes left, otherwise right
int ii = Tree <F, S> .Partition(responses_, indices_, i0, i1, bestThreshold);
System.Diagnostics.Debug.Assert(ii >= i0 && i1 >= ii);
progress_.WriteLine(Verbosity.Verbose, "{0} (threshold = {1:G3}, gain = {2:G3}).", bestFeature.ToString(), bestThreshold, maxGain);
TrainNodesRecurse(nodes, nodeIndex * 2 + 1, i0, ii, recurseDepth + 1);
TrainNodesRecurse(nodes, nodeIndex * 2 + 2, ii, i1, recurseDepth + 1);
}
/// <summary>
/// Post接口方法
/// </summary>
/// <param name="requestUri"></param>
/// <param name="json"></param>
/// <returns></returns>
public static string GetHttpRequest(string requestUri, string json, Int32 time = 7000, string version = "1.0")
{
//json格式请求数据
if (time == 0 || time > timeout)
{
time = timeout;
}
string requestData = json;
HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create(requestUri);
//utf-8编码
byte[] buf = System.Text.Encoding.GetEncoding("utf-8").GetBytes(requestData);
//post请求
myRequest.Method = "GET";
myRequest.ContentLength = buf.Length;
myRequest.MaximumAutomaticRedirections = 1;
myRequest.Headers.Add("Api-Version", version);
myRequest.AllowAutoRedirect = true;
myRequest.Timeout = time;
myRequest.ContentType = "application/json; charset=utf-8";
myRequest.Accept = "application/json";
//线程信息传递
//BaseThreadEntity entity = BaseThread.GetTraceInfo();
//if (entity.TraceId != null && entity.TraceId != "")
// myRequest.Headers.Add("MyProcessInfo", JsonHelper.ObjectToJson(entity));
try
{
ThreadLocalData entity = (ThreadLocalData)ThreadSlot.LogicalGetData().Clone();
if (entity != null)
{
string clientID = ThreadSlot.GetClientID();//System.Configuration.ConfigurationSettings.AppSettings["ClientID"];
if (!string.IsNullOrEmpty(clientID))
{
if (clientID.Split('|')[1] == "1")//终端(后台)
{
entity.TraceSecondId = clientID.Split('|')[0];
entity.TraceId = Guid.NewGuid().ToString("N");
}
}
myRequest.Headers.Add("BTProcessInfo", JsonHelper.SerializeObject(entity));
}
}
catch (Exception ex)
{
LogService.Default.Fatal(ex, "post请求添加追踪日志异常：" + ex.Message + ex.StackTrace);
}
string ReqResult = string.Empty;
HttpWebResponse myResponse = null;
StreamReader reader = null;
try
{
myResponse = (HttpWebResponse)myRequest.GetResponse();
reader = new StreamReader(myResponse.GetResponseStream(), Encoding.UTF8);
ReqResult = reader.ReadToEnd();
reader.Close();
myResponse.Close();
}
catch (Exception e)
{
throw e;
}
finally
{
if (reader != null)
{
reader.Close();
}
if (myResponse != null)
{
myResponse.Close();
}
}
return(ReqResult);
}
