private void LoadUsers(XmlElement root)
{
var list = root.GetElementsByTagName("user");
XmlNode node = null;
XmlElement e = null;
for (int i = 0, n = list.Count; i < n; i++)
{
node = list.Item(i);
if (node.NodeType != XmlNodeType.Element)
{
continue;
}
e = (XmlElement)node;
var name = e.GetAttribute("name");
var user = new UserConfig();
user.Name = name;
var props = ConfigUtil.LoadElements(e);
user.Password = (string)props.GetValue("password");
var schemas = (string)props.GetValue("schemas");
if (schemas != null)
{
var strArray = SplitUtil.Split(schemas, ',', true);
user.Schemas = new HashSet <string>(strArray);
}
if (users.ContainsKey(name))
{
throw new ConfigException("user " + name + " duplicated!");
}
users[name] = user;
}
}
public void AutoFitImageClassificationTrainTest()
{
var context = new MLContext();
var datasetPath = DatasetUtil.GetFlowersDataset();
var columnInference = context.Auto().InferColumns(datasetPath, "Label");
var textLoader = context.Data.CreateTextLoader(columnInference.TextLoaderOptions);
var trainData = context.Data.ShuffleRows(textLoader.Load(datasetPath), seed: 1);
var originalColumnNames = trainData.Schema.Select(c => c.Name);
TrainTestData trainTestData = context.Data.TrainTestSplit(trainData, testFraction: 0.2, seed: 1);
IDataView trainDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TrainSet, originalColumnNames);
IDataView testDataset = SplitUtil.DropAllColumnsExcept(context, trainTestData.TestSet, originalColumnNames);
var result = context.Auto()
.CreateMulticlassClassificationExperiment(0)
.Execute(trainDataset, testDataset, columnInference.ColumnInformation);
//Known issue, where on Ubuntu there is degradation in accuracy.
if (!(RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ||
RuntimeInformation.IsOSPlatform(OSPlatform.OSX)))
{
Assert.Equal(0.778, result.BestRun.ValidationMetrics.MicroAccuracy, 3);
}
else
{
Assert.Equal(1, result.BestRun.ValidationMetrics.MicroAccuracy, 3);
}
var scoredData = result.BestRun.Model.Transform(trainData);
Assert.Equal(TextDataViewType.Instance, scoredData.Schema[DefaultColumnNames.PredictedLabel].Type);
}
public override Boolean IsAbnormalExceptionTimeout(Exception ex)
{
if (ex == null)
{
return(false);
}
var exception = ex as DbException;
if (exception == null)
{
return(false);
}
Int32 errorCode = ExceptionUtil.GetDbExceptionErrorCode(exception);
String errorCodes = timeoutMarkDownBean.MySqlErrorCodes;
if (!String.IsNullOrEmpty(errorCodes))
{
var result = SplitUtil.SplitAsInt32(errorCodes);
if (result.Contains(errorCode))
{
return(true);
}
}
return(ExceptionUtil.IsTimeoutException(ex));
}
public void CrossValSplitThrowsWhenNotEnoughData()
{
var mlContext = new MLContext(1);
var dataViewBuilder = new ArrayDataViewBuilder(mlContext);
dataViewBuilder.AddColumn("Number", NumberDataViewType.Single, 0f);
dataViewBuilder.AddColumn("Label", NumberDataViewType.Single, 0f);
var dataView = dataViewBuilder.GetDataView();
Assert.Throws <InvalidOperationException>(() => SplitUtil.CrossValSplit(mlContext, dataView, 10, null));
}
private static int[] ToIntArray(string @string)
{
var strs = SplitUtil.Split(@string, ',', true);
var ints = new int[strs.Length];
for (var i = 0; i < strs.Length; ++i)
{
ints[i] = Convert.ToInt32(strs[i]);
}
return(ints);
}
public TableConfig(string name, string dataNode, TableRuleConfig rule, bool ruleRequired)
{
Name = name;
DataNodes = SplitUtil.Split(dataNode, ',', '$', '-', '[', ']');
if (DataNodes == null || DataNodes.Length <= 0)
{
throw new ArgumentException("invalid table _dataNodes: " + dataNode);
}
Rule = rule;
_columnIndex = BuildColumnIndex(rule);
IsRuleRequired = ruleRequired;
}
private static IDictionary <string, IList <string> > LoadClusterGroup(XmlElement root,
IDictionary <string, CobarNodeConfig> nodes)
{
var groups = new Dictionary <string, IList <string> >();
var list = root.GetElementsByTagName("group");
XmlNode node = null;
XmlElement e = null;
for (int i = 0, n = list.Count; i < n; i++)
{
node = list.Item(i);
if (node.NodeType != XmlNodeType.Element)
{
continue;
}
e = (XmlElement)node;
var groupName = e.GetAttribute("name").Trim();
if (groups.ContainsKey(groupName))
{
throw new ConfigException("group duplicated : " + groupName);
}
var props = ConfigUtil.LoadElements(e);
var value = (string)props.GetValue("nodeList");
if (value.IsEmpty())
{
throw new ConfigException("group should contain 'nodeList'");
}
var sList = SplitUtil.Split(value, ',', true);
if (null == sList || sList.Length == 0)
{
throw new ConfigException("group should contain 'nodeList'");
}
foreach (var s in sList)
{
if (!nodes.ContainsKey(s))
{
throw new ConfigException("[ node :" + s + "] in [ group:" + groupName + "] doesn't exist!");
}
}
groups[groupName] = sList;
}
if (!groups.ContainsKey("default"))
{
groups["default"] = new List <string>(nodes.Keys);
}
return(groups);
}
private RuleConfig LoadRule(XmlElement element)
{
var columnsEle = ConfigUtil.LoadElement(element, "columns");
var columns = SplitUtil.Split(columnsEle.InnerText, ',', true);
for (var i = 0; i < columns.Length; ++i)
{
columns[i] = columns[i].ToUpper();
}
var algorithmEle = ConfigUtil.LoadElement(element, "algorithm");
var algorithm = algorithmEle.InnerText;
return(new RuleConfig(columns, algorithm));
}
public void CrossValSplitLargeDataView()
{
var mlContext = new MLContext(seed: 0);
var dataViewBuilder = new ArrayDataViewBuilder(mlContext);
dataViewBuilder.AddColumn("Number", NumberDataViewType.Single, new float[10000]);
dataViewBuilder.AddColumn("Label", NumberDataViewType.Single, new float[10000]);
var dataView = dataViewBuilder.GetDataView();
const int requestedNumSplits = 10;
var splits = SplitUtil.CrossValSplit(mlContext, dataView, requestedNumSplits, null);
Assert.True(splits.trainDatasets.Any());
Assert.Equal(requestedNumSplits, splits.trainDatasets.Count());
Assert.Equal(requestedNumSplits, splits.validationDatasets.Count());
}
private void LoadQuarantine(XmlElement root)
{
var list = root.GetElementsByTagName("host");
XmlNode node = null;
XmlElement e = null;
for (int i = 0, n = list.Count; i < n; i++)
{
node = list.Item(i);
if (node.NodeType != XmlNodeType.Element)
{
continue;
}
e = (XmlElement)node;
var host = e.GetAttribute("name").Trim();
if (Quarantine.Hosts.ContainsKey(host))
{
throw new ConfigException("host duplicated : " + host);
}
var props = ConfigUtil.LoadElements(e);
var users = SplitUtil.Split((string)props["user"], ',', true);
var set = new HashSet <string>();
if (null != users)
{
foreach (var user in users)
{
var uc = this.users.GetValue(user);
if (null == uc)
{
throw new ConfigException("[user: "******"] doesn't exist in [host: " + host + "]");
}
if (null == uc.Schemas || uc.Schemas.Count == 0)
{
throw new ConfigException("[host: " + host + "] contains one root privileges user: "******"[host: " + host + "] contains duplicate user: " + user);
}
set.Add(user);
}
}
Quarantine.Hosts[host] = set;
}
}
public TableConfig(string name, string dataNode, TableRuleConfig rule, bool ruleRequired)
{
if (name == null)
{
throw new ArgumentException("table name is null");
}
Name = name.ToUpper();
var dataNodes = SplitUtil.Split(dataNode, ',', '$', '-', '[', ']');
if (dataNodes == null || dataNodes.Length <= 0)
{
throw new ArgumentException("invalid table dataNodes: " + dataNode);
}
DataNodes = dataNodes;
Rule = rule;
columnIndex = BuildColumnIndex(rule);
IsRuleRequired = ruleRequired;
}
private IDictionary <string, TableConfig> LoadTables(XmlElement node)
{
var tables = new Dictionary <string, TableConfig>();
var nodeList = node.GetElementsByTagName("table");
XmlElement tableElement = null;
TableRuleConfig tableRule = null;
for (var i = 0; i < nodeList.Count; i++)
{
tableElement = (XmlElement)nodeList.Item(i);
var name = tableElement.GetAttribute("name").ToUpper();
var dataNode = tableElement.GetAttribute("dataNode");
tableRule = null;
if (tableElement.HasAttribute("rule"))
{
var ruleName = tableElement.GetAttribute("rule");
tableRule = tableRules[ruleName];
if (tableRule == null)
{
throw new ConfigException("rule " + ruleName + " is not found!");
}
}
var ruleRequired = false;
if (tableElement.HasAttribute("ruleRequired"))
{
ruleRequired = bool.Parse(tableElement.GetAttribute("ruleRequired"));
}
var tableNames = SplitUtil.Split(name, ',', true);
foreach (var tableName in tableNames)
{
var table = new TableConfig(tableName, dataNode, tableRule, ruleRequired);
CheckDataNodeExists(table.DataNodes);
if (tables.ContainsKey(table.Name))
{
throw new ConfigException("table " + tableName + " duplicated!");
}
tables[table.Name] = table;
}
}
return(tables);
}
/// <summary>
/// 自动mark down,怎么区分？
/// </summary>
/// <param name="allInOneKey"></param>
public static void AutoMarkDownADatabase(String allInOneKey)
{
lock (lockObj)
{
markDownDatabases[allInOneKey] = markdownBean.EnableAutoMarkDown ? MarkDownEnums.AutoMarkDown : MarkDownEnums.FakeAutoMarkDown;
var schedules = SplitUtil.SplitAsInt32(markdownBean.AutoMarkUpSchedule).ToArray();
//排序
Array.Sort <Int32>(schedules);
markingUpDatabases[allInOneKey] = new MarkUpInfo()
{
PreMarkUp = false,
MarkDownTime = DateTime.Now,
CurrentMarkUpIndex = 0,
MarkUpSchedules = schedules,
MarkUpSuccess = new Int32[schedules.Length],
MarkUpFail = new Int32[schedules.Length],
CurrentMarkUpSchedule = 0,
CurrentBatch = 1,
MarkUpArray = MarkUpInfo.InitMarkUpArray(schedules[0])
};
}
}
private void LoadDataNodes(XmlElement root)
{
var list = root.GetElementsByTagName("dataNode");
XmlElement element = null;
XmlElement dsElement = null;
XmlNodeList dataSourceList = null;
for (int i = 0, n = list.Count; i < n; i++)
{
element = (XmlElement)list.Item(i);
var dnNamePrefix = element.GetAttribute("name");
IList <DataNodeConfig> confList = new List <DataNodeConfig>();
try
{
dsElement = FindPropertyByName(element, "dataSource");
if (dsElement == null)
{
throw new ArgumentNullException("dataNode xml XmlElement with name of " + dnNamePrefix
+ " has no dataSource Element");
}
dataSourceList = dsElement.GetElementsByTagName("dataSourceRef");
var dataSources = new string[dataSourceList.Count][];
for (int j = 0, m = dataSourceList.Count; j < m; ++j)
{
var @ref = (XmlElement)dataSourceList.Item(j);
var dsString = @ref.InnerText;
dataSources[j] = SplitUtil.Split(dsString, ',', '$', '-', '[', ']');
}
if (dataSources.Length <= 0)
{
throw new ConfigException("no dataSourceRef defined!");
}
foreach (var dss in dataSources)
{
if (dss.Length != dataSources[0].Length)
{
throw new ConfigException("dataSource number not equals!");
}
}
for (int k = 0, limit = dataSources[0].Length; k < limit; ++k)
{
var dsString = new StringBuilder();
for (var dsIndex = 0; dsIndex < dataSources.Length; ++dsIndex)
{
if (dsIndex > 0)
{
dsString.Append(',');
}
dsString.Append(dataSources[dsIndex][k]);
}
var conf = new DataNodeConfig();
ParameterMapping.Mapping(conf, ConfigUtil.LoadElements(element));
confList.Add(conf);
switch (k)
{
case 0:
{
conf.Name = limit == 1 ? dnNamePrefix : dnNamePrefix + "[" + k + "]";
break;
}
default:
{
conf.Name = dnNamePrefix + "[" + k + "]";
break;
}
}
conf.DataSource = dsString.ToString();
}
}
catch (Exception e)
{
throw new ConfigException("dataNode " + dnNamePrefix + " define error", e);
}
foreach (var conf_1 in confList)
{
if (dataNodes.ContainsKey(conf_1.Name))
{
throw new ConfigException("dataNode " + conf_1.Name + " duplicated!");
}
dataNodes[conf_1.Name] = conf_1;
}
}
}
private void LoadDataSources(XmlElement root)
{
var list = root.GetElementsByTagName("dataSource");
XmlElement element = null;
XmlElement locElement = null;
XmlNodeList locationList = null;
for (int i = 0, n = list.Count; i < n; i++)
{
element = (XmlElement)list.Item(i);
var dscList = new List <DataSourceConfig>();
var dsNamePrefix = element.GetAttribute("name");
try
{
var dsType = element.GetAttribute("type");
locElement = FindPropertyByName(element, "location");
if (locElement == null)
{
throw new ArgumentNullException("dataSource xml XmlElement with name of " + dsNamePrefix
+ " has no location Element");
}
locationList = locElement.GetElementsByTagName("location");
var dsIndex = 0;
for (int j = 0, m = locationList.Count; j < m; ++j)
{
var locStr = ((XmlElement)locationList.Item(j)).InnerText;
var colonIndex = locStr.IndexOf(':');
var slashIndex = locStr.IndexOf('/');
var dsHost = Runtime.Substring(locStr, 0, colonIndex).Trim();
var dsPort = Convert.ToInt32(Runtime.Substring(locStr, colonIndex + 1, slashIndex).Trim());
var schemas = SplitUtil.Split(Runtime.Substring(locStr, slashIndex + 1).Trim(), ',', '$', '-');
foreach (var dsSchema in schemas)
{
var dsConf = new DataSourceConfig();
ParameterMapping.Mapping(dsConf, ConfigUtil.LoadElements(element));
dscList.Add(dsConf);
switch (dsIndex)
{
case 0:
{
dsConf.Name = dsNamePrefix;
break;
}
case 1:
{
dscList[0].Name = dsNamePrefix + "[0]";
goto default;
}
default:
{
dsConf.Name = dsNamePrefix + "[" + dsIndex + "]";
break;
}
}
dsConf.Type = dsType;
dsConf.Database = dsSchema;
dsConf.Host = dsHost;
dsConf.Port = dsPort;
++dsIndex;
}
}
}
catch (Exception e)
{
throw new ConfigException("dataSource " + dsNamePrefix + " define error", e);
}
foreach (var dsConf_1 in dscList)
{
if (dataSources.ContainsKey(dsConf_1.Name))
{
throw new ConfigException("dataSource name " + dsConf_1.Name + "duplicated!");
}
dataSources[dsConf_1.Name] = dsConf_1;
}
}
}
