private double autocorrelation_coefficient(List <double> data1, List <double> data2)
{
if (data1.Count != data2.Count)
{
throw new System.InvalidOperationException("Autocorrelation just has been calculated by two list of same length.");
}
double mean1 = MathFuncs.CalcMean(data1);
double mean2 = MathFuncs.CalcMean(data2);
double std1 = MathFuncs.CalcStd(data1, mean1);
double std2 = MathFuncs.CalcStd(data2, mean2);
double result = 0;
for (int i = 0; i < data1.Count; i++)
{
result += (data1[i] - mean1) * (data2[i] - mean2);
}
return(result / (data1.Count * std1 * std2));
}
private double K_LOWER_PERCENT = (5 / 100.0); // note: double type
private void doStream(String file_name, String algorithm, List <double> result, List <double> norm_buffer, List <double> data_to_calc_w, List <double> stream_data, int N_LENGTH, int W_LENGTH, List <double> raw_buffer, double threshold_mean, double threshold_std, double threshold_sim, int R, int maxEntry, int minEntry, int period)
{
bool is_the_first_time = true;//control running HOTSAX for the first time
int raw_buffer_len = raw_buffer.Count;
int norm_buffer_len = norm_buffer.Count;
//get last raw segment:
//List<double> last_raw_segment = raw_buffer.GetRange(raw_buffer_len - N_LENGTH, N_LENGTH);
//store index of the limited searching space (in case (b)):
List <int> candidate_list = new List <int>();
int index_stream = 0; //control the stream data point
int index_table = 0;
int i_b = 0; //calc number of cases (b).
int data_calc_w_len = data_to_calc_w.Count;
//store result from HOTSAX algorithm:
//List<double> result
//result[0]: dist
//result[1]: loc
double next_data_point;
double currDist;
System.Diagnostics.Stopwatch watch2;; ///calc execution time
long elapsedMs2 = 0;
double small_match_dist = 0;
List <double> first_segment;
double new_norm_point;
AugmentedTrie tree;
Dictionary <string, int> count_table;
Dictionary <int, string> total_table;
count_table = new Dictionary <string, int>();
total_table = new Dictionary <int, string>();
//Making the root node:
HOTSAX.TreeNode root = new HOTSAX.TreeNode('R');
//init the path (to print the tree later)
List <char> path = new List <char>();
// Making the augmented tree:
tree = new AugmentedTrie(root, path);
// Squeezer
List <int> cluster_belong = new List <int>();
List <Cluster_Squeezer> b_cluster = new List <Cluster_Squeezer>();
List <int> lCluster_NonMember = new List <int>();
// Bounding_Box
int this_id_item = int.MinValue;
List <int> this_id_itemList = new List <int>();
RTree <int> this_RTree = new RTree <int>(maxEntry, minEntry);
List <BoundingBox.Rectangle> this_recList = new List <BoundingBox.Rectangle>();
double this_best_so_far_dist = -Constants.INFINITE;
double this_best_so_far_loc = -1;
BoundingBox.Rectangle new_rec;
// Useless variable to pass parameter
int dumb = 0;
List <int> dumb_list = new List <int>();
List <BoundingBox.Rectangle> dumb_rectlist = new List <BoundingBox.Rectangle>();
RTree <int> dumb_rtree = new RTree <int>(maxEntry, minEntry);
// Sanchez_method
double this_best_so_far_dist_TheMostDiscord = -Constants.INFINITE;
double old_mean = MathFuncs.CalcMean(raw_buffer);
double old_std = MathFuncs.CalcStd(raw_buffer, old_mean);
double new_mean, new_std;
List <double> result_dist = new List <double>();
List <double> result_loc = new List <double>();
var watch = System.Diagnostics.Stopwatch.StartNew();///calc execution time
int num_nor = 0;
while (true)
{
manualResetEvent.WaitOne(Timeout.Infinite);//help for PAUSE, RESUME button
if (is_the_first_time)
{
watch2 = System.Diagnostics.Stopwatch.StartNew();///calc execution time
switch (algorithm)
{
case "HOTSAX_Squeezer_Stream":
result = BitClusterDiscord.squeezer(ref lCluster_NonMember, ref cluster_belong, ref b_cluster, norm_buffer, N_LENGTH, W_LENGTH, threshold_sim, BitClusterDiscord.PAA); Console.WriteLine("Case 1");
break;
case "BFHS":
case "HOTSAX_Stream":
result = HOTSAX.HOTSAX.originalHOTSAX(0, norm_buffer, N_LENGTH, W_LENGTH,
ref tree, ref count_table, ref total_table, true);
break;
case "Bounding_Box":
result = BoundingBox.BoundingBoxDiscordDiscovery.RunOfflineMinDist(raw_buffer, N_LENGTH, maxEntry, minEntry, R, W_LENGTH, ref this_id_item, ref this_id_itemList, ref this_recList, ref this_RTree, true);
this_best_so_far_dist = result[0];
this_best_so_far_loc = result[1];
break;
case "Sanchez_Method":
result = BoundingBox.BoundingBoxDiscordDiscovery.RunOfflineMinDist(raw_buffer, N_LENGTH, maxEntry, minEntry, R, W_LENGTH, ref this_id_item, ref this_id_itemList, ref this_recList, ref this_RTree, true);
this_best_so_far_dist_TheMostDiscord = result[0];
result = new List <double> {
-1, -1
};
break;
}
is_the_first_time = false;
watch2.Stop();
elapsedMs2 = watch2.ElapsedMilliseconds;
//print through console
Console.WriteLine("\n\t best_so_far_dist = " + result.ElementAt(0) +
"\n\t best_so_far_loc = " + result.ElementAt(1) +
"\n\t execution_time = " + elapsedMs2);
Console.WriteLine("----------------- finished i = {0}--------------- \n\n", index_stream - 1);
Console.WriteLine("Finished the first call.");
if (chart_timeSeries.IsHandleCreated)
{
try
{
// call updateChart fucntion in GUI thread by chart thread
if (algorithm == "Bounding_Box" || algorithm == "Sanchez_Method")
{
this.Invoke((MethodInvoker) delegate { updateChart(raw_buffer, (int)(result[1]), N_LENGTH, index_stream, elapsedMs2); });
}
else
{
this.Invoke((MethodInvoker) delegate { updateChart(norm_buffer, (int)(result[1]), N_LENGTH, index_stream, elapsedMs2); });
}
}
catch
{ }
}
}
else
{
if (index_stream >= stream_data.Count)
{
watch.Stop();//stop timer
var elapsedMs = watch.ElapsedMilliseconds;
Console.WriteLine("Streaming is Done in " + elapsedMs + ".\nKeep going...Please wait");
// Write File
string[] all_dist = result_dist.Select(dist => dist.ToString()).ToArray();
string[] all_loc = result_loc.Select(loc => loc.ToString()).ToArray();
var data_name = System.IO.Path.GetFileNameWithoutExtension(file_name);
var extension = System.IO.Path.GetExtension(file_name);
string newPath = "Output\\" + data_name + "\\" + algorithm;
System.IO.Directory.CreateDirectory(newPath);
System.IO.File.WriteAllLines(newPath + "\\dist" + extension, all_dist);
System.IO.File.WriteAllLines(newPath + "\\loc" + extension, all_loc);
using (System.IO.StreamWriter file =
new System.IO.StreamWriter(newPath + "\\time" + extension, false))
{
file.WriteLine(elapsedMs);
}
this.txt_speed.Text = "0";
Statistical_Form statistical_form = new Statistical_Form(algorithm, file_name, result_loc, result_dist, elapsedMs);
System.Windows.Forms.MessageBox.Show("stream_data ran out of points");
Console.WriteLine("num norm: " + num_nor);
return;
}
//get the next data point:
next_data_point = stream_data[index_stream];
new_mean = MathFuncs.CalcNewMean(old_mean, raw_buffer_len, raw_buffer[0], next_data_point);
//store the last subsequence of the buffer for Bounding Box and Sanchez algorithms:
List <double> last_sub = raw_buffer.GetRange(raw_buffer.Count - N_LENGTH, N_LENGTH);
//get the first sub before update the buffer (help to find the small match in Liu's method)
List <double> first_sub = raw_buffer.GetRange(0, N_LENGTH);
//update raw_buffer:
raw_buffer.Add(next_data_point);
raw_buffer.RemoveAt(0);
new_std = MathFuncs.CalcStd(raw_buffer, new_mean);
watch2 = System.Diagnostics.Stopwatch.StartNew();///calc execution time
switch (algorithm)
{
case "HOTSAX_Squeezer_Stream":
case "BFHS":
case "HOTSAX_Stream":
if ((Math.Abs(new_mean - old_mean) <= threshold_mean) && (Math.Abs(new_std - old_std) <= threshold_std))
{
new_norm_point = (next_data_point - old_mean) / old_std;
index_table++;
norm_buffer.Add(new_norm_point);
//calc 'currDist':
currDist = MathFuncs.EuDistance(norm_buffer.GetRange((int)result[1], N_LENGTH), norm_buffer.GetRange(norm_buffer.Count - N_LENGTH, N_LENGTH));
//if the case (a): Modify the Tree, Tables:
if (currDist < result[0] || (int)(result[1]) == 0 || algorithm == "BFHS")
{
Console.WriteLine("--- Running case (a)... ---");
if (algorithm == "HOTSAX_Squeezer_Stream")
{
result = BitClusterDiscord.squeezer(ref lCluster_NonMember, ref cluster_belong, ref b_cluster, norm_buffer.Select(point => point).ToList(), N_LENGTH, W_LENGTH, threshold_sim, BitClusterDiscord.PAA, index_table, false);
//update buffer:
norm_buffer.RemoveAt(0);
Console.WriteLine("B_cluster.Count: " + b_cluster.Count.ToString());
}
else
{
//call the original HOTSAX ver3:
result = HOTSAX.HOTSAX.originalHOTSAX(index_table, norm_buffer,
N_LENGTH, W_LENGTH, ref tree, ref count_table, ref total_table);
}
}
else // case (b), we can limit candidate_list:
{
Console.WriteLine("--------------- Running case (b)... -----------------");
/* Find the candidate_list: */
candidate_list.Clear(); //reset the list
//The local discord at time t:
candidate_list.Add((int)(result[1]) - 1);
//The subsequence (m-n+1, n)(t+1):
candidate_list.Add(norm_buffer_len - N_LENGTH);
//The small match of subsequence (1, n)(t):
first_segment = norm_buffer.GetRange(0, N_LENGTH);
for (int j = N_LENGTH; j <= norm_buffer_len - N_LENGTH; j++)
{
small_match_dist = MathFuncs.EuDistance(norm_buffer.GetRange(j, N_LENGTH), first_segment);
if (small_match_dist < result[0])
{
if ((int)(result[1]) != j)
{
candidate_list.Add(j - 1);
}
}
}
if (algorithm == "HOTSAX_Squeezer_Stream")
{
result = BitClusterDiscord.squeezerAgain(ref lCluster_NonMember, candidate_list, index_table, ref cluster_belong, ref b_cluster, norm_buffer.Select(point => point).ToList(), N_LENGTH, W_LENGTH, threshold_sim, BitClusterDiscord.PAA);
//update buffer:
norm_buffer.RemoveAt(0);
Console.WriteLine("B_cluster.Count: " + b_cluster.Count.ToString());
}
else
{
//update buffer:
norm_buffer.RemoveAt(0);
//searching on candidates;
result = HOTSAX.HOTSAX.candidateHOTSAX(candidate_list, index_table, norm_buffer,
N_LENGTH, W_LENGTH, ref tree, ref count_table, ref total_table);
}
i_b++; //count number of cases (b) - just for testing
Console.WriteLine("len(candidate_list) = {0}, Number of cases (b) is {1}/{2}", candidate_list.Count, i_b, index_stream);
} //end else - case (b)
}
else
{
Console.WriteLine("--------------- Normalizing buffer ... -----------------");
num_nor++;
index_table = 0;
norm_buffer = MathFuncs.zScoreNorm(raw_buffer, raw_buffer_len);
count_table = new Dictionary <string, int>();
total_table = new Dictionary <int, string>();
cluster_belong = new List <int>();
b_cluster = new List <Cluster_Squeezer>();
//Making the root node:
root = new HOTSAX.TreeNode('R');
//init the path (to print the tree later)
path = new List <char>();
// Making the augmented tree:
tree = new AugmentedTrie(root, path);
if (algorithm == "HOTSAX_Squeezer_Stream")
{
result = BitClusterDiscord.squeezer(ref lCluster_NonMember, ref cluster_belong, ref b_cluster, norm_buffer, N_LENGTH, W_LENGTH, threshold_sim, BitClusterDiscord.PAA);
}
else
{
result = HOTSAX.HOTSAX.originalHOTSAX(0, norm_buffer, N_LENGTH, W_LENGTH,
ref tree, ref count_table, ref total_table, true);
}
old_std = new_std;
old_mean = new_mean;
}
break;
case "Bounding_Box":
//update last_sub at time t to get new_sub at time (t+1):
last_sub.Add(next_data_point);
last_sub.RemoveAt(0);
List <double> new_sub = last_sub; // the same object
// Insert the new entry into the tree:
this_id_item++;
// Add the new rec to the tree:
new_rec = new BoundingBox.Rectangle(Utils.MathFuncs.PAA_Lower(new_sub, W_LENGTH, R).ToArray(), Utils.MathFuncs.PAA_Upper(new_sub, W_LENGTH, R).ToArray(), raw_buffer.Count - N_LENGTH + 1 + index_stream);
this_RTree.Add(new_rec, this_id_item);
this_recList.Add(new_rec);
this_id_itemList.Add(this_id_item);
//remove the oldest entry:
this_RTree.Delete(this_recList[index_stream], this_id_itemList[index_stream]);
result = BoundingBoxDiscordDiscovery.RunOnline_LiuMethod_edited(raw_buffer, index_stream, first_sub, this_RTree, this_best_so_far_dist, (int)this_best_so_far_loc, N_LENGTH, W_LENGTH);
this_best_so_far_dist = result[0];
this_best_so_far_loc = result[1];
break;
case "Sanchez_Method":
//update last_sub at time t to get new_sub at time (t+1):
last_sub.Add(next_data_point);
last_sub.RemoveAt(0);
new_sub = last_sub; // the same object
// Insert the new entry into the tree:
this_id_item++;
// Add the new rec to the tree:
new_rec = new BoundingBox.Rectangle(Utils.MathFuncs.PAA_Lower(new_sub, W_LENGTH, R).ToArray(), Utils.MathFuncs.PAA_Upper(new_sub, W_LENGTH, R).ToArray(), raw_buffer.Count - N_LENGTH + 1 + index_stream);
this_RTree.Add(new_rec, this_id_item);
this_recList.Add(new_rec);
this_id_itemList.Add(this_id_item);
//remove the oldest entry:
this_RTree.Delete(this_recList[index_stream], this_id_itemList[index_stream]);
result = BoundingBoxDiscordDiscovery.NewOnlineAlgorithm(raw_buffer, 2 * period, index_stream, period, new_sub, this_RTree, N_LENGTH, W_LENGTH, R, maxEntry, minEntry, ref this_best_so_far_dist_TheMostDiscord);
break;
}
watch2.Stop(); //stop timer
elapsedMs2 = watch2.ElapsedMilliseconds;
//print through console
Console.WriteLine("\n\t best_so_far_dist = " + result.ElementAt(0) +
"\n\t best_so_far_loc = " + result.ElementAt(1) +
"\n\t execution_time = " + elapsedMs2);
Console.WriteLine("----------------- finished i = {0}--------------- \n\n", index_stream);
index_stream++;// make 'index' increase by 1 to get the next data point
}//end else
if (chart_timeSeries.IsHandleCreated)
{
try
{
// call updateChart fucntion in GUI thread by chart thread
if (algorithm == "Bounding_Box" || algorithm == "Sanchez_Method")
{
this.Invoke((MethodInvoker) delegate { updateChart(raw_buffer, (int)(result[1]), N_LENGTH, index_stream, elapsedMs2); });
}
else
{
this.Invoke((MethodInvoker) delegate { updateChart(norm_buffer, (int)(result[1]), N_LENGTH, index_stream, elapsedMs2); });
}
}
catch
{ }
}
// Store result
result_dist.Add(result.ElementAt(0));
result_loc.Add(result.ElementAt(1));
//Thread.Sleep(2000);
}
}