/// <summary>
/// init the node
/// </summary>
/// <param name="compute_node"></param>
public void Init(IComputeNode compute_node)
{
ComputeNode = compute_node;
CatalogDataTables = new Dictionary<string, IDataTable>();
Buckets = new Dictionary<long, IBucket>();
Initialized = false;
}
public IComputeNode GetComputeNode()
{
IComputeNode compute_node = null;
try
{
//amazon client
using (var client = new AmazonS3Client())
{
//download request
using (var response = client.GetObject(new GetObjectRequest()
.WithBucketName(AmazonBucket)
.WithKey(BermudaConfig)))
{
using (StreamReader reader = new StreamReader(response.ResponseStream))
{
//read the file
string data = reader.ReadToEnd();
//deserialize
compute_node = new ComputeNode().DeserializeComputeNode(data);
compute_node.Init(CurrentInstanceIndex, AllNodeEndpoints.Count());
}
}
}
}
catch (Exception ex)
{
Trace.WriteLine(ex.ToString());
}
return(compute_node);
}
/// <summary>
/// init the node
/// </summary>
/// <param name="compute_node"></param>
public void Init(IComputeNode compute_node)
{
ComputeNode = compute_node;
CatalogDataTables = new Dictionary <string, IDataTable>();
Buckets = new Dictionary <long, IBucket>();
Initialized = false;
}
public void SetStream(MaterialShaderStage stage, string stream, IComputeNode computeNode, ObjectParameterKey<Texture> defaultTexturingKey, ParameterKey defaultValueKey, Color? defaultTextureValue)
{
if (defaultValueKey == null) throw new ArgumentNullException(nameof(defaultValueKey));
if (computeNode == null)
{
return;
}
var streamType = MaterialStreamType.Float;
if (defaultValueKey.PropertyType == typeof(Vector4) || defaultValueKey.PropertyType == typeof(Color4))
{
streamType = MaterialStreamType.Float4;
}
else if (defaultValueKey.PropertyType == typeof(Vector3) || defaultValueKey.PropertyType == typeof(Color3))
{
streamType = MaterialStreamType.Float3;
}
else if (defaultValueKey.PropertyType == typeof(Vector2) || defaultValueKey.PropertyType == typeof(Half2))
{
streamType = MaterialStreamType.Float2;
}
else if (defaultValueKey.PropertyType == typeof(float))
{
streamType = MaterialStreamType.Float;
}
else
{
throw new NotSupportedException("ParameterKey type [{0}] is not supported by SetStream".ToFormat(defaultValueKey.PropertyType));
}
var classSource = computeNode.GenerateShaderSource(Context, new MaterialComputeColorKeys(defaultTexturingKey, defaultValueKey, defaultTextureValue));
SetStream(stage, stream, streamType, classSource);
}
public IEnumerable <IDataProcessor> GetProcessors(IComputeNode compute_node)
{
List <IDataProcessor> list = new List <IDataProcessor>();
switch (NodeType)
{
case ComputeNodeType.MX:
list.Add(new DatabaseSaturator.DatabaseSaturator(compute_node));
list.Add(new DataPurge.PurgeProcessor(compute_node));
break;
case ComputeNodeType.WeatherData:
var weather_filesroot = @"c:\code\EvoApp\Bermuda\Bermuda.FileSaturator\weatherdata";
IFileProcessor weather_proc = new FileSaturator.FileSystemFileProcessor()
{
LineProcessor = new FileSaturator.WeatherLineProcessor(),
FilesRootPath = weather_filesroot,
};
list.Add(new FileSaturator.FileSaturator(compute_node, weather_proc, "Weather", "Weather"));
break;
case ComputeNodeType.WikiData:
list.Add(new S3Saturator.S3Saturator(compute_node));
break;
case ComputeNodeType.Net:
list.Add(new NetSaturator.NetSaturator(compute_node,
GetInstanceAddressMap(),
"UDPTest",
"UDPTest",
new NetComUDP(1234, 12345)));
break;
}
return(list);
}
public IComputeNode GetComputeNode()
{
IComputeNode compute_node = null;
try
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(RoleEnvironment.GetConfigurationSettingValue("StorageAccount.ConnectionString"));
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(AzureContainer);
container.CreateIfNotExist();
CloudBlob blob = container.GetBlobReference(BermudaConfig);
string data = blob.DownloadText();
//deserialize
compute_node = new ComputeNode().DeserializeComputeNode(data);
if (compute_node.Catalogs.Values.Cast <ICatalog>().FirstOrDefault().CatalogMetadata.Tables.FirstOrDefault().Value.DataType == null)
{
compute_node.Catalogs.Values.Cast <ICatalog>().FirstOrDefault().CatalogMetadata.Tables.FirstOrDefault().Value.DataType = typeof(UDPTestDataItems);
}
compute_node.Init(CurrentInstanceIndex, AllNodeEndpoints.Count());
}
catch (Exception ex)
{
Trace.WriteLine(ex.ToString());
}
return(compute_node);
}
public IEnumerable <IDataProcessor> GetProcessors(IComputeNode compute_node)
{
List <IDataProcessor> list = new List <IDataProcessor>();
switch (NodeType)
{
case ComputeNodeType.MX:
list.Add(new DatabaseSaturator.DatabaseSaturator(compute_node));
list.Add(new DataPurge.PurgeProcessor(compute_node));
break;
case ComputeNodeType.WeatherData:
var weather_filesroot = @"c:\code\EvoApp\Bermuda\Bermuda.FileSaturator\weatherdata";
IFileProcessor weather_proc = new FileSaturator.FileSystemFileProcessor()
{
LineProcessor = new FileSaturator.WeatherLineProcessor(),
FilesRootPath = weather_filesroot,
};
list.Add(new FileSaturator.FileSaturator(compute_node, weather_proc, "Weather", "Weather"));
break;
case ComputeNodeType.WikiData:
list.Add(new S3Saturator.S3Saturator(compute_node));
//IFileProcessor wiki_proc = new FileSaturator.S3FileProcessor()
// {
// //LineProcessor = null,
// AmazonBucket = "PublicDataSets",
// AmazonPrefix = "wikistats/pagecounts/",
// };
//list.Add(new FileSaturator.FileSaturator(compute_node, wiki_proc, "WikipediaData", "PageStats"));
break;
case ComputeNodeType.Combined:
list.Add(new DatabaseSaturator.DatabaseSaturator(compute_node));
list.Add(new DataPurge.PurgeProcessor(compute_node));
//list.Add(new FileSaturator.FileSaturator(compute_node));
list.Add(new S3Saturator.S3Saturator(compute_node));
break;
case ComputeNodeType.Net:
list.Add(new NetSaturator.NetSaturator(
compute_node,
GetInstanceAddressMap(),
"UDPTest",
"UDPTest",
new NetComUDP(12345)));
break;
case ComputeNodeType.NetGateway:
list.Add(new NetSaturator.NetGateway(
compute_node,
GetInstanceAddressMap(),
"UDPTest",
"UDPTest",
new NetComUDP(1234, 12345)));
break;
}
return(list);
}
/// <summary>
/// default constructor with compute node
/// </summary>
/// <param name="computeNode"></param>
public NetGateway(
IComputeNode computeNode,
IDictionary <long, string> address_mappings,
string catalog_name,
string table_name,
INetCom com)
: base(computeNode, address_mappings, catalog_name, table_name, com)
{
}
/// <summary>
/// the constructor with catalog
/// </summary>
public PurgeProcessor(IComputeNode computeNode)
{
ComputeNode = computeNode;
#if DEBUG
PurgeThreadCount = Environment.ProcessorCount;
#else
PurgeThreadCount = Math.Min(Environment.ProcessorCount * 4, 64);
#endif
}
/// <summary>
/// default constructor with compute node
/// </summary>
/// <param name="computeNode"></param>
public NetSaturator(
IComputeNode computeNode,
IDictionary<long, string> address_mappings,
string catalog_name,
string table_name,
INetCom com)
: base(computeNode, address_mappings, catalog_name, table_name, com)
{
}
/// <summary>
/// the constructor with catalog
/// </summary>
public PurgeProcessor(IComputeNode computeNode)
{
ComputeNode = computeNode;
#if DEBUG
PurgeThreadCount = Environment.ProcessorCount;
#else
PurgeThreadCount = Math.Min(Environment.ProcessorCount * 4, 64);
#endif
}
public void Initialize(IHostEnvironmentConfiguration hostEnvConfig)
{
ServicePointManager.DefaultConnectionLimit = 1000;
ServicePointManager.Expect100Continue = false;
ServicePointManager.UseNagleAlgorithm = false;
this.HostEnvironmentConfiguration = hostEnvConfig;
this.StartInternalService(hostEnvConfig.InternalServiceEndPoint);
this.StartExternalService(hostEnvConfig.ExternalServiceEndPoint);
int bucket_count = BUCKET_COUNT;
#if DEBUG
Thread.Sleep(2000);
bucket_count = BUCKET_COUNT_DEBUG;
#else
Thread.Sleep(10000);
#endif
//ComputeNodeType NodeType = ComputeNodeType.MX;
//switch (NodeType)
//{
// case ComputeNodeType.MX:
// computeNode = new ComputeNodeMX.ComputeNodeMX(this.HostEnvironmentConfiguration.CurrentInstanceIndex, bucket_count, this.HostEnvironmentConfiguration.AllNodeEndpoints.Count());
// computeNode.MinAvailableMemoryPercent = 20;
// computeNode.MaxAvailableMemoryPercent = 30;
// processors.Add(new DatabaseSaturator.DatabaseSaturator(computeNode));
// processors.Add(new PurgeProcessor(computeNode));
// break;
// case ComputeNodeType.WeatherData:
// //Weather data compute node
// computeNode = new ComputeNodeWX.ComputeNodeWX(this.HostEnvironmentConfiguration.CurrentInstanceIndex, bucket_count, this.HostEnvironmentConfiguration.AllNodeEndpoints.Count());
// computeNode.MinAvailableMemoryPercent = 20;
// computeNode.MaxAvailableMemoryPercent = 30;
// processors.Add(new FileSaturator.FileSaturator(computeNode));
// break;
// case ComputeNodeType.WikiData:
// //Wikipedia data compute node
// computeNode = new ComputeNodeWikiData.ComputeNodeWikiData(this.HostEnvironmentConfiguration.CurrentInstanceIndex, bucket_count, this.HostEnvironmentConfiguration.AllNodeEndpoints.Count());
// computeNode.MinAvailableMemoryPercent = 20;
// computeNode.MaxAvailableMemoryPercent = 30;
// processors.Add(new S3Saturator.S3Saturator(computeNode));
// break;
//}
//string json = computeNode.SerializeComputeNode();
//IComputeNode temp = computeNode.DeserializeComputeNode(json);
//File.WriteAllText("c:\\temp\\BermudaMX.Config", json);
computeNode = this.HostEnvironmentConfiguration.GetComputeNode();
processors.AddRange(this.HostEnvironmentConfiguration.GetProcessors(computeNode));
processors.ForEach(p => p.StartProcessor());
}
/// <summary>
/// the constructor with catalog
/// </summary>
public DatabaseSaturator(IComputeNode computeNode)
{
ComputeNode = computeNode;
#if DEBUG
//SaturationThreadCount = Environment.ProcessorCount;
SaturationThreadCount = Math.Min(Environment.ProcessorCount * 4, 64);
//SaturationThreadCount = 1;
//SaturationThreadCount = 64;
#else
SaturationThreadCount = Math.Min(Environment.ProcessorCount * 4, 64);
//SaturationThreadCount = Environment.ProcessorCount;
#endif
LastCatalogRefresh = DateTime.UtcNow;
}
/// <summary>
/// default constructor with compute node
/// </summary>
/// <param name="computeNode"></param>
public NetProcessor(
IComputeNode computeNode,
IDictionary <long, string> address_mappings,
string catalog_name,
string table_name,
INetCom com)
{
Client = com;
ComputeNode = computeNode;
UdpMessageToDecode = new ConcurrentQueue <string>();
AddressMap = address_mappings;
CatalogName = catalog_name;
TableName = table_name;
}
/// <summary>
/// constructor with compute node
/// </summary>
/// <param name="compute_node"></param>
//public FileSaturator(IComputeNode compute_node)
//{
// ComputeNode = compute_node;
// SaturationThreadCount = Math.Min(64,Environment.ProcessorCount * 4);
// //SaturationThreadCount = 1;
// SaturationFiles = new List<FileData>();
// LastFileDataRefresh = DateTime.UtcNow.AddDays(-1);
//}
public FileSaturator(IComputeNode compute_node, IFileProcessor file_processor, string catalog_name, string table_name)
{
ComputeNode = compute_node;
CatalogName = catalog_name;
TableName = table_name;
FileProcessor = file_processor;
//SaturationThreadCount = Math.Min(64,Environment.ProcessorCount * 4);
SaturationThreadCount = 1;
SaturationFiles = new List <FileData>();
LastFileDataRefresh = DateTime.UtcNow.AddDays(-1);
}
/// <summary>
/// the constructor with catalog
/// </summary>
public DatabaseSaturator(IComputeNode computeNode)
{
ComputeNode = computeNode;
#if DEBUG
//SaturationThreadCount = Environment.ProcessorCount;
SaturationThreadCount = Math.Min(Environment.ProcessorCount * 4, 64);
//SaturationThreadCount = 1;
//SaturationThreadCount = 64;
#else
SaturationThreadCount = Math.Min(Environment.ProcessorCount * 4, 64);
//SaturationThreadCount = Environment.ProcessorCount;
#endif
LastCatalogRefresh = DateTime.UtcNow;
}
private void CombineComputeNodes()
{
NodeType = ComputeNodeType.MX;
IComputeNode mx = GetComputeNode();
NodeType = ComputeNodeType.WeatherData;
IComputeNode weather = GetComputeNode();
NodeType = ComputeNodeType.WikiData;
IComputeNode wiki = GetComputeNode();
weather.Catalogs.Values.ToList().ForEach(c => mx.Catalogs.Add(c.Name, c));
wiki.Catalogs.Values.ToList().ForEach(c => mx.Catalogs.Add(c.Name, c));
string json = mx.SerializeComputeNode();
File.WriteAllText("c:\\Temp\\BermudaCombined.Config", json);
}
/// <summary>
/// default constructor
/// </summary>
public ComputeNodeConfig(IComputeNode compute_node)
{
//init properties
Catalogs = new ObservableCollection <ICatalog>();
//read the config file
//string json = File.ReadAllText("c:\\temp\\Bermuda.Config");
//ComputeNode = new ComputeNode().DeserializeComputeNode(json);
ComputeNode = compute_node;
//set the catalogs
ComputeNode.Catalogs.Values.ToList().ForEach(c => Catalogs.Add((ICatalog)c));
//FixCollections();
//init gui
InitializeComponent();
}
/// <summary>
/// default constructor
/// </summary>
public ComputeNodeConfig(IComputeNode compute_node)
{
//init properties
Catalogs = new ObservableCollection<ICatalog>();
//read the config file
//string json = File.ReadAllText("c:\\temp\\Bermuda.Config");
//ComputeNode = new ComputeNode().DeserializeComputeNode(json);
ComputeNode = compute_node;
//set the catalogs
ComputeNode.Catalogs.Values.ToList().ForEach(c => Catalogs.Add((ICatalog)c));
//FixCollections();
//init gui
InitializeComponent();
}
public void SetStream(MaterialShaderStage stage, string stream, IComputeNode computeNode, ObjectParameterKey <Texture> defaultTexturingKey, ParameterKey defaultValueKey, Color?defaultTextureValue)
{
if (defaultValueKey == null)
{
throw new ArgumentNullException(nameof(defaultValueKey));
}
if (computeNode == null)
{
return;
}
var streamType = MaterialStreamType.Float;
if (defaultValueKey.PropertyType == typeof(Vector4) || defaultValueKey.PropertyType == typeof(Color4))
{
streamType = MaterialStreamType.Float4;
}
else if (defaultValueKey.PropertyType == typeof(Vector3) || defaultValueKey.PropertyType == typeof(Color3))
{
streamType = MaterialStreamType.Float3;
}
else if (defaultValueKey.PropertyType == typeof(Vector2) || defaultValueKey.PropertyType == typeof(Half2))
{
streamType = MaterialStreamType.Float2;
}
else if (defaultValueKey.PropertyType == typeof(float))
{
streamType = MaterialStreamType.Float;
}
else
{
throw new NotSupportedException("ParameterKey type [{0}] is not supported by SetStream".ToFormat(defaultValueKey.PropertyType));
}
var classSource = computeNode.GenerateShaderSource(Context, new MaterialComputeColorKeys(defaultTexturingKey, defaultValueKey, defaultTextureValue));
SetStream(stage, stream, streamType, classSource);
}
public IComputeNode GetComputeNode()
{
IComputeNode compute_node = null;
try
{
//amazon client
using (var client = new AmazonS3Client())
{
//download request
using (var response = client.GetObject(new GetObjectRequest()
.WithBucketName(AmazonBucket)
.WithKey(BermudaConfig)))
{
using (StreamReader reader = new StreamReader(response.ResponseStream))
{
//read the file
string data = reader.ReadToEnd();
//deserialize
compute_node = new ComputeNode().DeserializeComputeNode(data);
if (compute_node.Catalogs.Values.Cast <ICatalog>().FirstOrDefault().CatalogMetadata.Tables.FirstOrDefault().Value.DataType == null)
{
compute_node.Catalogs.Values.Cast <ICatalog>().FirstOrDefault().CatalogMetadata.Tables.FirstOrDefault().Value.DataType = typeof(UDPTestDataItems);
}
compute_node.Init(CurrentInstanceIndex, AllNodeEndpoints.Count());
}
}
}
}
catch (Exception ex)
{
Trace.WriteLine(ex.ToString());
}
return(compute_node);
}
public IEnumerable<IDataProcessor> GetProcessors(IComputeNode compute_node)
{
List<IDataProcessor> list = new List<IDataProcessor>();
switch(NodeType)
{
case ComputeNodeType.MX:
list.Add(new DatabaseSaturator.DatabaseSaturator(compute_node));
list.Add(new DataPurge.PurgeProcessor(compute_node));
break;
case ComputeNodeType.WeatherData:
var weather_filesroot = @"c:\code\EvoApp\Bermuda\Bermuda.FileSaturator\weatherdata";
IFileProcessor weather_proc = new FileSaturator.FileSystemFileProcessor()
{
LineProcessor = new FileSaturator.WeatherLineProcessor(),
FilesRootPath = weather_filesroot,
};
list.Add(new FileSaturator.FileSaturator(compute_node, weather_proc, "Weather", "Weather"));
break;
case ComputeNodeType.WikiData:
list.Add(new S3Saturator.S3Saturator(compute_node));
break;
}
return list;
}
public void Initialize(IHostEnvironmentConfiguration hostEnvConfig)
{
ServicePointManager.DefaultConnectionLimit = 1000;
ServicePointManager.Expect100Continue = false;
ServicePointManager.UseNagleAlgorithm = false;
this.HostEnvironmentConfiguration = hostEnvConfig;
this.StartInternalService(hostEnvConfig.InternalServiceEndPoint);
this.StartExternalService(hostEnvConfig.ExternalServiceEndPoint);
int bucket_count = BUCKET_COUNT;
#if DEBUG
Thread.Sleep(2000);
bucket_count = BUCKET_COUNT_DEBUG;
#else
Thread.Sleep(10000);
#endif
//ComputeNodeType NodeType = ComputeNodeType.MX;
//switch (NodeType)
//{
// case ComputeNodeType.MX:
// computeNode = new ComputeNodeMX.ComputeNodeMX(this.HostEnvironmentConfiguration.CurrentInstanceIndex, bucket_count, this.HostEnvironmentConfiguration.AllNodeEndpoints.Count());
// computeNode.MinAvailableMemoryPercent = 20;
// computeNode.MaxAvailableMemoryPercent = 30;
// processors.Add(new DatabaseSaturator.DatabaseSaturator(computeNode));
// processors.Add(new PurgeProcessor(computeNode));
// break;
// case ComputeNodeType.WeatherData:
// //Weather data compute node
// computeNode = new ComputeNodeWX.ComputeNodeWX(this.HostEnvironmentConfiguration.CurrentInstanceIndex, bucket_count, this.HostEnvironmentConfiguration.AllNodeEndpoints.Count());
// computeNode.MinAvailableMemoryPercent = 20;
// computeNode.MaxAvailableMemoryPercent = 30;
// processors.Add(new FileSaturator.FileSaturator(computeNode));
// break;
// case ComputeNodeType.WikiData:
// //Wikipedia data compute node
// computeNode = new ComputeNodeWikiData.ComputeNodeWikiData(this.HostEnvironmentConfiguration.CurrentInstanceIndex, bucket_count, this.HostEnvironmentConfiguration.AllNodeEndpoints.Count());
// computeNode.MinAvailableMemoryPercent = 20;
// computeNode.MaxAvailableMemoryPercent = 30;
// processors.Add(new S3Saturator.S3Saturator(computeNode));
// break;
//}
//string json = computeNode.SerializeComputeNode();
//IComputeNode temp = computeNode.DeserializeComputeNode(json);
//File.WriteAllText("c:\\temp\\BermudaMX.Config", json);
computeNode = this.HostEnvironmentConfiguration.GetComputeNode();
processors.AddRange(this.HostEnvironmentConfiguration.GetProcessors(computeNode));
processors.ForEach(p => p.StartProcessor());
}
/// <summary>
/// the constructor with parent
/// </summary>
public Catalog(IComputeNode compute_node)
{
Init(compute_node);
}
public IEnumerable<IDataProcessor> GetProcessors(IComputeNode compute_node)
{
List<IDataProcessor> list = new List<IDataProcessor>();
switch (NodeType)
{
case ComputeNodeType.MX:
list.Add(new DatabaseSaturator.DatabaseSaturator(compute_node));
list.Add(new DataPurge.PurgeProcessor(compute_node));
break;
case ComputeNodeType.WeatherData:
var weather_filesroot = @"c:\code\EvoApp\Bermuda\Bermuda.FileSaturator\weatherdata";
IFileProcessor weather_proc = new FileSaturator.FileSystemFileProcessor()
{
LineProcessor = new FileSaturator.WeatherLineProcessor(),
FilesRootPath = weather_filesroot,
};
list.Add(new FileSaturator.FileSaturator(compute_node, weather_proc, "Weather", "Weather"));
break;
case ComputeNodeType.WikiData:
list.Add(new S3Saturator.S3Saturator(compute_node));
//IFileProcessor wiki_proc = new FileSaturator.S3FileProcessor()
// {
// //LineProcessor = null,
// AmazonBucket = "PublicDataSets",
// AmazonPrefix = "wikistats/pagecounts/",
// };
//list.Add(new FileSaturator.FileSaturator(compute_node, wiki_proc, "WikipediaData", "PageStats"));
break;
case ComputeNodeType.Combined:
list.Add(new DatabaseSaturator.DatabaseSaturator(compute_node));
list.Add(new DataPurge.PurgeProcessor(compute_node));
//list.Add(new FileSaturator.FileSaturator(compute_node));
list.Add(new S3Saturator.S3Saturator(compute_node));
break;
case ComputeNodeType.Net:
list.Add(new NetSaturator.NetSaturator(
compute_node,
GetInstanceAddressMap(),
"UDPTest",
"UDPTest",
new NetComUDP(12345)));
break;
case ComputeNodeType.NetGateway:
list.Add(new NetSaturator.NetGateway(
compute_node,
GetInstanceAddressMap(),
"UDPTest",
"UDPTest",
new NetComUDP(1234, 12345)));
break;
}
return list;
}
public void SetStream(string stream, IComputeNode computeNode, ObjectParameterKey <Texture> defaultTexturingKey, ParameterKey defaultValueKey, Color?defaultTextureValue = null)
{
SetStream(MaterialShaderStage.Pixel, stream, computeNode, defaultTexturingKey, defaultValueKey, defaultTextureValue);
}
public void SetStream(MaterialShaderStage stage, string stream, IComputeNode computeNode, ObjectParameterKey <Texture> defaultTexturingKey, ParameterKey defaultValueKey, Color?defaultTextureValue = null)
{
currentLayerContext.SetStream(stage, stream, computeNode, defaultTexturingKey, defaultValueKey, defaultTextureValue);
}
/// <summary>
/// the constructor for save mode
/// </summary>
/// <param name="compute_node"></param>
public OpenConfig(IComputeNode compute_node)
{
OpenMode = false;
ComputeNode = compute_node;
InitializeComponent();
}
/// <summary>
/// constructor with compute node
/// </summary>
/// <param name="compute_node"></param>
//public FileSaturator(IComputeNode compute_node)
//{
// ComputeNode = compute_node;
// SaturationThreadCount = Math.Min(64,Environment.ProcessorCount * 4);
// //SaturationThreadCount = 1;
// SaturationFiles = new List<FileData>();
// LastFileDataRefresh = DateTime.UtcNow.AddDays(-1);
//}
public FileSaturator(IComputeNode compute_node, IFileProcessor file_processor, string catalog_name, string table_name)
{
ComputeNode = compute_node;
CatalogName = catalog_name;
TableName = table_name;
FileProcessor = file_processor;
//SaturationThreadCount = Math.Min(64,Environment.ProcessorCount * 4);
SaturationThreadCount = 1;
SaturationFiles = new List<FileData>();
LastFileDataRefresh = DateTime.UtcNow.AddDays(-1);
}
/// <summary>
/// the constructor with parent
/// </summary>
public Catalog(IComputeNode compute_node)
{
Init(compute_node);
}
public S3Saturator(IComputeNode computeNode)
{
this.ComputeNode = computeNode;
}
/// <summary>
/// default constructor with compute node
/// </summary>
/// <param name="computeNode"></param>
public NetProcessor(
IComputeNode computeNode,
IDictionary<long, string> address_mappings,
string catalog_name,
string table_name,
INetCom com)
{
Client = com;
ComputeNode = computeNode;
UdpMessageToDecode = new ConcurrentQueue<string>();
AddressMap = address_mappings;
CatalogName = catalog_name;
TableName = table_name;
}
/// <summary>
/// the constructor for save mode
/// </summary>
/// <param name="compute_node"></param>
public OpenConfig(IComputeNode compute_node)
{
OpenMode = false;
ComputeNode = compute_node;
InitializeComponent();
}
public S3Saturator(IComputeNode computeNode)
{
this.ComputeNode = computeNode;
}
