/// <summary>
/// Export result for specific dataset based on trained model in the mlconfig file
/// </summary>
/// <param name="mlConfigPath"></param>
/// <param name="resultPath"></param>
public static void ExportResult(string mlConfigPath, string resultPath, DataSetType dsType)
{
var device = DeviceDescriptor.UseDefaultDevice();
var task = MLExport.ExportToCSV(mlConfigPath, device, resultPath, dsType);
task.Wait();
}
/// <summary>
/// Return path for specified dataset
/// </summary>
/// <param name="dicPath"></param>
/// <param name="dsType"></param>
/// <returns></returns>
private static string GetDataPath(Dictionary <string, string> dicPath, DataSetType dsType)
{
var dataPaths = MLFactory.GetMLConfigComponentPaths(dicPath["paths"]);
//
if (dsType == DataSetType.Training)
{
//
var strPath = $"{dicPath["root"]}\\{dataPaths["Training"]}";
return(strPath);
}
else if (dsType == DataSetType.Validation)
{
var strPath = $"{dicPath["root"]}\\{dataPaths["Validation"]}";
return(strPath);
}
else if (dsType == DataSetType.Testing)
{
var strPath = $"{dicPath["root"]}\\{dataPaths["Test"]}";
return(strPath);
}
else
{
return(null);
}
}
public static void Build()
{
ReportBuilder rptbuild = new ReportBuilder();
ReportType vReport = rptbuild.Report;
BodyType vBody = vReport.Body;
// double x = 0, y = 0, wd = 0 , ht = 0;
vReport.Width.Value = RPT_WIDTH;
vBody.Height.Value = RPT_HEIGHT;
DataSetType dataSet = rptbuild.Report.DataSets[0];
dataSet.Name = "MyDataSource_Items";
ReportUtil.AddFields(dataSet.Fields, new string[] {
POUTWARDITEMS.SLNO,
POUTWARDITEMS.OUTWARD_ID,
POUTWARDITEMS.STYLE_ID,
POUTWARDITEMS.ARTICLE_ID,
POUTWARDITEMS.PRODUCT_ID,
POUTWARDITEMS.COLOURS_ID,
POUTWARDITEMS.SIZES_ID,
POUTWARDITEMS.QTY
});
BuildDetail(vBody);
///== END OF REPORT ===========================================================================///
rptbuild.SaveAs(PRINT_FOLDER + @"\P_OutwardItem.rdlc");
}
public DataSetContext(DataSetType dataSetType, string dataRootFolder, string clrNamespacePrefixBase, string designTimeClrNamespacePrefix)
{
this.DataSetType = dataSetType;
this.DataRootFolder = dataRootFolder;
this.ClrNamespacePrefixBase = clrNamespacePrefixBase;
this.DesignTimeClrNamespacePrefix = designTimeClrNamespacePrefix;
}
private DataSetType CreateDataSet()
{
DataSetType dataSet = new DataSetType();
dataSet.Name = m_dataSetName;
dataSet.Items = new object[] { CreateQuery(), CreateFields() };
return(dataSet);
}
private DataSetType CreateDataSet()
{
var dataSet = new DataSetType();
dataSet.Name = "DataSet1";
dataSet.Items = new object[] { CreateQuery(), CreateFields() };
return(dataSet);
}
public static string GetClrNamespacePrefix(DataSetType dataSetType, string rootNamespace)
{
if (string.IsNullOrEmpty(rootNamespace))
{
return(DataSetContext.ClrNamespacePrefixBaseFromDataSetType(dataSetType));
}
return(rootNamespace + "." + DataSetContext.ClrNamespacePrefixBaseFromDataSetType(dataSetType));
}
public Places(string name, string url, bool sky, string thumbUrl, bool isBrowseable, DataSetType type)
{
browseable = isBrowseable;
thumbnailUrl = thumbUrl;
this.sky = sky;
this.name = name;
this.url = url;
this.dataSetType = type;
}
public Places(string name, string url, bool sky, string thumbUrl, bool isBrowseable, DataSetType type)
{
browseable = isBrowseable;
thumbnailUrl = thumbUrl;
this.sky = sky;
this.name = name;
this.url = url;
this.dataSetType = type;
}
/// <summary>
/// Обучающая выборка.
/// </summary>
/// <param name="type">Тип.</param>
/// <param name="data">Данные.</param>
public DataSet(DataSetType type)
{
if (!type.Equals(DataSetType.ForNumberRecognizing))
{
throw new NotImplementedException();
}
Type = type;
}
public static void Build()
{
ReportBuilder rptbuild = new ReportBuilder();
ReportType vReport = rptbuild.Report;
BodyType vBody = vReport.Body;
double x = 0, y = 0, wd = 0, ht = 0;
vReport.Width.Value = RPT_WIDTH;
vBody.Height.Value = RPT_HEIGHT;
DataSetType dataSet = rptbuild.Report.DataSets[0];
ReportUtil.AddFields(dataSet.Fields, new string[] {
POUTWARD.COPIES
, POUTWARD.COMPANY_NAME
, POUTWARD.ADDRESS1
, POUTWARD.ADDRESS2
, POUTWARD.COMPANY_GSTTIN
, POUTWARD.OUTWARD_ID
, POUTWARD.OUTWARD_NO
, POUTWARD.OUTWARD_DATE
, POUTWARD.PARTY_NAME
, POUTWARD.STREET1
, POUTWARD.STREET2
, POUTWARD.CITY
, POUTWARD.STATE
, POUTWARD.COUNTRY
, POUTWARD.PINCODE
, POUTWARD.GSTIN
, POUTWARD.TOTAL_QTY
, POUTWARD.TOTAL_BUNDLE
, POUTWARD.DELIVREDTHROUGH
});
x = 0; y = 0; wd = RPT_WIDTH; ht = RPT_HEIGHT;
ListType Reportlist = ReportUtil.AddList("Reportlist", vBody.ReportItems, x, y, wd, ht);
Reportlist.Grouping = new GroupingType("group_" + POUTWARD.OUTWARD_ID + "");
Reportlist.Grouping.GroupExpressions.Add("=Fields!" + POUTWARD.OUTWARD_ID + ".Value");
Reportlist.Grouping.GroupExpressions.Add("=Fields!" + POUTWARD.COPIES + ".Value");
Reportlist.Grouping.PageBreakAtEnd = true;
x = 0; y = 0; wd = RPT_WIDTH; ht = RPT_HEIGHT;
RectangleType rectWarper = ReportUtil.AddRectangle("rectWarper", Reportlist.ReportItems, x, y, wd, ht);
rectWarper.Style.BorderStyle.Default = BorderStyleEnum.None;
BuildDetail(rectWarper);
//ReportUtil.AttachRulers(Reportlist.ReportItems, RPT_WIDTH, RPT_HEIGHT);
rptbuild.SaveAs(PRINT_FOLDER + @"\P_Outward.rdlc");
}
public ProcessDataSet(string publisherId, string name, ushort writerId, DataSetType dataSetType)
{
PublisherId = publisherId;
m_ProcessValues = new Dictionary <string, DataPointEntry>();
m_Name = name;
m_WriterId = writerId;
m_MetaFrame = null;
SetDataSetType(dataSetType);
Description = new LocalizedText();
}
private static DataSetType CreateDataSet(string name, IEnumerable <RdlColumnHeader> fields)
{
var dataSet = new DataSetType {
Name = name,
Items = new object[] {
CreateFields(fields),
CreateQuery()
}
};
return(dataSet);
}
public static string ClrNamespacePrefixBaseFromDataSetType(DataSetType dataSetType)
{
if (dataSetType == DataSetType.SampleDataSet)
{
return(DataSetContext.SampleData.ClrNamespacePrefixBase);
}
if (dataSetType == DataSetType.DataStoreSet)
{
return(DataSetContext.DataStore.ClrNamespacePrefixBase);
}
return((string)null);
}
public static List<NaturalEarthRow> GetNaturalEarthTableRows(DataSetType dataSet, string tableName)
{
if (dataSet == DataSetType.Cultural)
{
return GetNaturalEarthTableRows(tableName, naturalEarthCulturalconnectionString);
}
else if (dataSet == DataSetType.Physical)
{
return GetNaturalEarthTableRows(tableName, naturalEarthPhysicalconnectionString);
}
else
return null;
}
public static List<string> GetNaturalEarthTables(DataSetType dataSet)
{
if (dataSet == DataSetType.Cultural)
{
return GetNaturalEarthTables(naturalEarthCulturalconnectionString);
}
else if (dataSet == DataSetType.Physical)
{
return GetNaturalEarthTables(naturalEarthPhysicalconnectionString);
}
else
return null;
}
private (AdfItemType type, object value) DeserializeDataSet()
{
var jo = JObject.Parse(_json);
var typeValue = jo["properties"]?["type"]?.Value <string>();
DataSetType dataSetType = typeValue.ToEnum <DataSetType>();
DataSet dataset = null;
try
{
switch (dataSetType)
{
case DataSetType.AzureBlob:
dataset = jo.ToObject <AzureBlob>();
break;
case DataSetType.AzureDataLakeStore:
dataset = jo.ToObject <AzureDataLakeStore>();
break;
case DataSetType.AzureSqlTable:
dataset = jo.ToObject <AzureSqlTable>();
break;
case DataSetType.AzureSqlDwTable:
dataset = jo.ToObject <AzureSqlDwTable>();
break;
case DataSetType.CosmosDbCollection:
dataset = jo.ToObject <AzureCosmosDbCollection>();
break;
case DataSetType.AzureTable:
dataset = jo.ToObject <AzureTable>();
break;
case DataSetType.AzureSearchIndex:
dataset = jo.ToObject <AzureSearchIndex>();
break;
}
return(AdfItemType.DataSet, dataset);
}
catch (JsonSerializationException ex)
{
Logger.Instance.Error($"DataSet {typeValue}. \"{ex.Message}\" was handled");
throw new AdfParseException($"DataSet {typeValue}", ex);
}
}
void InitDataSetsDefault()
{
numDataSets = 1;
numInputs = 2;
dataSets = new DataSetsClass(numDataSets, numInputs);
DataSetType dataSet = new DataSetType(numInputs);
dataSet.inputData[0] = 3;
dataSet.inputData[1] = 2;
dataSet.answer = 1;
dataSets.LoadInputDataSet(dataSet, 0);
// dataSets.dataSets[0] = dataSet;
dataSets.Graph(scaleData);
Debug.Log("initDataSetsDefault\n");
}
public static List <NaturalEarthRow> GetNaturalEarthTableRows(DataSetType dataSet, string tableName)
{
if (dataSet == DataSetType.Cultural)
{
return(GetNaturalEarthTableRows(tableName, naturalEarthCulturalconnectionString));
}
else if (dataSet == DataSetType.Physical)
{
return(GetNaturalEarthTableRows(tableName, naturalEarthPhysicalconnectionString));
}
else
{
return(null);
}
}
public static List <string> GetNaturalEarthTables(DataSetType dataSet)
{
if (dataSet == DataSetType.Cultural)
{
return(GetNaturalEarthTables(naturalEarthCulturalconnectionString));
}
else if (dataSet == DataSetType.Physical)
{
return(GetNaturalEarthTables(naturalEarthPhysicalconnectionString));
}
else
{
return(null);
}
}
public DataSet(string name, string url, bool sky, DataSetType type)
{
this.dataSetType = type;
this.Sky = sky;
this.name = name;
this.url = url;
DataSetManager.DownloadFile(url, Properties.Settings.Default.CahceDirectory + @"data\places\" + name + ".xml", false, true);
XmlDocument doc = new XmlDocument();
doc.Load(Properties.Settings.Default.CahceDirectory + @"data\places\" + name + ".xml");
dataSets = new Dictionary <string, Places>();
XmlNode root = doc["root"];
XmlNode dataSetsNode = root.SelectSingleNode("dataset");
this.groupingName = dataSetsNode.Attributes["Groups"].InnerXml;
DataSetType dst = DataSetType.Place;
if (dataSetsNode.Attributes["type"].Value != "place")
{
dst = DataSetType.Imageset;
}
foreach (XmlNode dataset in dataSetsNode.ChildNodes)
{
bool browsable = false;
if (dataset.Attributes["Browseable"] != null)
{
browsable = Convert.ToBoolean(dataset.Attributes["Browseable"].Value);
}
string thumbnailUrl = "";
if (dataset.Attributes["Thumbnail"] != null)
{
thumbnailUrl = dataset.Attributes["Thumbnail"].Value;
}
Places places = new Places(dataset.Attributes["name"].InnerXml, dataset.Attributes["url"].InnerXml, sky, thumbnailUrl, browsable, dst);
dataSets.Add(places.Name, places);
}
}
/// <summary>
/// Prints the performance analysis on the console
/// </summary>
/// <param name="mlConfigPath"></param>
public static void PrintPerformance(string mlConfigPath, DataSetType dsType = DataSetType.Validation)
{
try
{
//print evaluation result on console
var performanceData = MLExport.PrintPerformance(mlConfigPath, dsType, DeviceDescriptor.UseDefaultDevice());
performanceData.Wait();
foreach (var s in performanceData.Result)
{
Console.WriteLine(s);
}
}
catch (Exception)
{
throw;
}
}
public TObject GetDataAsObject <TObject>(string arg, DataSetType type)
{
var json = GetDataAsJson(arg, type);
if (json == null)
{
return(default(TObject)); // return null
}
JsonSerializerSettings jsonSettings = new JsonSerializerSettings
{
ContractResolver = new JsonCamelCaseContractResolver(),
NullValueHandling = NullValueHandling.Ignore
};
var obj = JsonConvert.DeserializeObject <TObject>(json, jsonSettings);
return(obj);
}
void InitDataSetsRandom()
{
numDataSets = 10;
numInputs = 10;
dataSets = new DataSetsClass(numDataSets, numInputs);
for (int d = 0; d < numDataSets; d++)
{
DataSetType dataSet = new DataSetType(numInputs);
for (int i = 0; i < dataSet.inputData.Length; i++)
{
dataSet.inputData[i] = Random.Range(1, 10);
}
dataSet.answer = Random.Range(1, 3);
dataSets.dataSets[d] = dataSet;
}
dataSets.Graph(scaleData);
Debug.Log("initDataSetsRandom\n");
}
public static void Build()
{
ReportBuilder rptbuild = new ReportBuilder();
ReportType vReport = rptbuild.Report;
BodyType vBody = vReport.Body;
// double x = 0, y = 0, wd = 0 , ht = 0;
vReport.Width.Value = RPT_WIDTH;
vBody.Height.Value = RPT_HEIGHT;
DataSetType dataSet = rptbuild.Report.DataSets[0];
dataSet.Name = "MyDataSource_Items";
ReportUtil.AddFields(dataSet.Fields, new string[] {
PInvoiceitems.INVOICE_ITEM_ID
, PInvoiceitems.INVOICE_ID
, PInvoiceitems.SL_NO
, PInvoiceitems.PO_NO
, PInvoiceitems.DC_NO
, PInvoiceitems.PRODUCT_ID
, PInvoiceitems.HSN_CODE_ID
, PInvoiceitems.SIZES
, PInvoiceitems.COLOURS
, PInvoiceitems.QTY
, PInvoiceitems.PRICE
, PInvoiceitems.SUB_TOTAL
, PInvoiceitems.TAXABLEVALUE
, PInvoiceitems.CGST_PERCENT
, PInvoiceitems.CGST_AMOUNT
, PInvoiceitems.SGST_PERCENT
, PInvoiceitems.SGST_AMOUNT
, PInvoiceitems.IGST_PERCENT
, PInvoiceitems.IGST_AMOUNT
});
BuildDetail(vBody);
///== END OF REPORT ===========================================================================///
rptbuild.SaveAs(PRINT_FOLDER + @"\P_InvoiceItem.rdlc");
}
public DataSetHarness(DataSetType type)
{
JsonSerializer = new JsonSerializer().UseEgad();
DataSet = CreateDataSet(type);
XmlSchema = new MemoryStream();
XmlData = new MemoryStream();
DataSet.WriteXmlSchema(XmlSchema);
DataSet.WriteXml(XmlData, XmlWriteMode.DiffGram);
XmlSchema.Position = 0;
XmlData.Position = 0;
Json = new MemoryStream();
var writer = new StreamWriter(Json);
JsonSerializer.Serialize(writer, DataSet);
writer.Flush();
}
public string GetDataAsJson(string arg, DataSetType type)
{
if (arg == null)
{
throw new ArgumentNullException(nameof(arg));
}
arg = WebUtility.UrlEncode(arg.Replace(" ", "+"));
var url = "";
url = type == DataSetType.Person
? MyApiFilmsUrlGenerator.CreateImdbPersonUrl(arg, _token)
: MyApiFilmsUrlGenerator.CreateImdbMovieUrl(arg, _token);
var json = "";
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 10000;
request.Method = WebRequestMethods.Http.Get;
response = (HttpWebResponse)request.GetResponse();
var streamReader = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
json = streamReader.ReadToEnd();
}
catch (Exception e)
{
if (e.Message == "The operation has timed out")
{
throw new MyApiFilmsTimeoutException(url);
}
return(null);
}
if (json.Contains("\"error\":"))
{
throw new NoMyApiFilmsResponseException(url);
}
return(json);
}
public void OnData(MessageType control, Message message)
{
// TODO put a lock on the pdu array
// TODO interpret the commands
Logging.Log("CFindServiceSCU({0}).OnData", Reflection.GetName(typeof(SOPClass), this.SOPClassUId));
DataSet dicom = message.Dicom;
if (MessageControl.IsCommand(control))
{
ushort status = (ushort)dicom[t.Status].Value;
bool last = !(status == 0xFF00 || status == 0xFF01);
//if (!last && status == 0xFF01)
//{
// Logging.Log("Warning that one or more Optional Keys were not supported for existence and/or matching for this Identifier.");
//}
DataSetType present = (DataSetType)dicom[t.CommandDataSetType].Value;
Logging.Log("<< C-FIND-RSP {0},{1}", (last) ? "LAST FRAGMENT" : "NOT-LAST FRAGMENT", present.ToString());
//dicom.Dump();
if (last)
{
// TODO do we reset the state to Open here
completeEvent.Set();
}
}
else
{
Logging.Log("<< P-DATA-TF");
if (records == null)
{
records = new RecordCollection();
}
records.Add(dicom.Elements);
}
}
public DataSet(string name, string url, bool sky, DataSetType type)
{
this.dataSetType = type;
this.Sky = sky;
this.name = name;
this.url = url;
DataSetManager.DownloadFile(url, Properties.Settings.Default.CahceDirectory + @"data\places\" + name + ".xml", false, true);
XmlDocument doc = new XmlDocument();
doc.Load(Properties.Settings.Default.CahceDirectory + @"data\places\" + name + ".xml");
dataSets = new Dictionary<string, Places>();
XmlNode root = doc["root"];
XmlNode dataSetsNode = root.SelectSingleNode("dataset");
this.groupingName = dataSetsNode.Attributes["Groups"].InnerXml;
DataSetType dst = DataSetType.Place;
if (dataSetsNode.Attributes["type"].Value != "place")
{
dst = DataSetType.Imageset;
}
foreach (XmlNode dataset in dataSetsNode.ChildNodes)
{
bool browsable = false;
if (dataset.Attributes["Browseable"] != null)
{
browsable = Convert.ToBoolean(dataset.Attributes["Browseable"].Value);
}
string thumbnailUrl = "";
if (dataset.Attributes["Thumbnail"] != null)
{
thumbnailUrl = dataset.Attributes["Thumbnail"].Value;
}
Places places = new Places(dataset.Attributes["name"].InnerXml, dataset.Attributes["url"].InnerXml, sky, thumbnailUrl, browsable, dst);
dataSets.Add(places.Name, places);
}
}
/// <summary>TBD</summary>
/// <param name="type">TBD</param>
/// <returns>TBD</returns>
public static DataSet GetCachedDataSet(DataSetType type)
{
//return GetCachedDataSet(Enum.GetName(typeof(DataSetType), type));
/*
ICacheManager appCache = CacheFactory.GetCacheManager();
if (appCache.Contains(Enum.GetName(typeof(DataSetType), type)))
{
if ((DataSet)appCache.GetData(Enum.GetName(typeof(DataSetType), type)) == null)
{
appCache.Remove(Enum.GetName(typeof(DataSetType), type));
AbsoluteTime expiry = new AbsoluteTime(new TimeSpan(0, 3, 30, 0));
appCache.Add(Enum.GetName(typeof(DataSetType), type), GetDataSetFromKey(type), CacheItemPriority.High, null, new ICacheItemExpiration[] { expiry });
return GetDataSetFromKey(type);
}
return (DataSet)appCache.GetData(Enum.GetName(typeof(DataSetType), type));
//return appCache[Enum.GetName(typeof(DataSetType), type)] as DataSet;
}
else
{
*/
return GetDataSetFromKey(type);
//}
}
internal static DataPointValue GetDataPointValue(DeltaEntry keyEntry, DataSetType dataSetType)
{
ProcessDataPointValue dpv = ProcessValueFactory.CreateValue(keyEntry.DataType);
if (dpv == null)
{
Logger.Error($"Unable to create DataPoint for entry: {keyEntry}");
return(null);
}
dpv.Orcat = keyEntry.Orcat;
dpv.Quality = keyEntry.Quality;
dpv.Timestamp = keyEntry.TimeStamp;
dpv.Value = keyEntry.Value;
if (keyEntry is KeyEntry key)
{
dpv.Prefix = key.Prefix;
dpv.Unit = key.Unit;
dpv.FieldID = key.FieldID;
}
if (dpv is CounterValue)
{
dpv.SetAttributeValue(CounterValue.QuantityAttributeName, keyEntry.Value2);
}
if (dpv is StepPosValue || dpv is StepPosEvent)
{
dpv.SetAttributeValue(StepPosValue.TransientAttributeName, keyEntry.Value2);
}
if (dpv is ComplexMeasuredValue)
{
dpv.SetAttributeValue(ComplexMeasuredValue.AngleAttributeName, keyEntry.Value2);
}
if (dpv.Timestamp == 0)
{
dpv.Timestamp = DateTime.Now.ToFileTimeUtc();
}
return(dpv);
}
/// <summary>TBD</summary>
/// <param name="type">TBD</param>
/// <returns>TBD</returns>
public static DataSet GetCachedDataSet(DataSetType type)
{
//return GetCachedDataSet(Enum.GetName(typeof(DataSetType), type));
/*
* ICacheManager appCache = CacheFactory.GetCacheManager();
* if (appCache.Contains(Enum.GetName(typeof(DataSetType), type)))
* {
* if ((DataSet)appCache.GetData(Enum.GetName(typeof(DataSetType), type)) == null)
* {
* appCache.Remove(Enum.GetName(typeof(DataSetType), type));
* AbsoluteTime expiry = new AbsoluteTime(new TimeSpan(0, 3, 30, 0));
* appCache.Add(Enum.GetName(typeof(DataSetType), type), GetDataSetFromKey(type), CacheItemPriority.High, null, new ICacheItemExpiration[] { expiry });
* return GetDataSetFromKey(type);
* }
* return (DataSet)appCache.GetData(Enum.GetName(typeof(DataSetType), type));
* //return appCache[Enum.GetName(typeof(DataSetType), type)] as DataSet;
* }
* else
* {
*/
return(GetDataSetFromKey(type));
//}
}
void InitDataSetsSlope()
{
numDataSets = 10;
numInputs = 10;
dataSets = new DataSetsClass(numDataSets, numInputs);
for (int d = 0; d < numDataSets; d++)
{
DataSetType dataSet = new DataSetType(numInputs);
float slope = Random.Range(.1f, 9f);
for (int i = 0; i < dataSet.inputData.Length; i++)
{
dataSet.inputData[i] = i * slope;
}
dataSet.isAnswered = false;
if (d < numDataSets * .75f)
{
dataSet.isAnswered = true;
dataSet.answer = slope;
}
dataSets.dataSets[d] = dataSet;
}
dataSets.Graph(scaleData);
Debug.Log("initDataSetsRandom\n");
}
private void SetDataSetType(DataSetType value)
{
m_DataSetType = value;
}
/// <summary>TBD</summary>
/// <param name="type">TBD</param>
/// <returns>TBD</returns>
public static DataSet GetDataSetFromKey(DataSetType type)
{
switch (type)
{
case DataSetType.AdvertiserDataSetType:
return GetAdvertisers();
case DataSetType.ADDepartments:
return GetADDepartments();
case DataSetType.ADOffices:
return GetADOffices();
case DataSetType.AEDataSetType:
return GetAEs();
case DataSetType.AgencyDataSetType:
return GetAgencies();
case DataSetType.AppDefaultDataSetType:
return GetAppDefaults();
case DataSetType.ConsolidatedAdvertiserType:
return GetConsolidatedAdvertisers();
case DataSetType.ContractLineYearsDataSetType:
return GetContractLineItemYears();
case DataSetType.MarketDataSetType:
return GetMarkets();
case DataSetType.MediaFormDataSetType:
return GetMediaForms();
case DataSetType.MediaTypeDataSetType:
return GetMediaTypes();
case DataSetType.NewMediaFormDataSetType:
return GetNewMediaForms();
case DataSetType.NewMediaTypeDataSetType:
return GetNewMediaTypes();
case DataSetType.ParentProductClassDataSetType:
return GetParentProductClasses();
case DataSetType.ProductClassDataSetType:
return GetProductClasses();
case DataSetType.ProductClassDLDataSetType:
return GetProductClassesDL();
case DataSetType.ProfitCenterDataSetType:
return GetProfitCenters();
case DataSetType.StationDataSetType:
return GetStations();
case DataSetType.UploadedByDataSetType:
return GetUploadedBy();
default:
return null;
}
}
/// <summary>
/// Evaluate model defined in the mlconfig file
/// </summary>
/// <param name="mlconfigPath"></param>
/// <param name="device"></param>
/// <returns></returns>
public static async Task <EvaluationResult> EvaluateMLConfig(string mlconfigPath, DeviceDescriptor device, DataSetType dsType, EvaluationType evType)
{
try
{
//define eval result
var er = new EvaluationResult();
er.OutputClasses = new List <string>()
{
""
};
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.Header = new List <string>();
//Load ML configuration file
var dicMParameters = MLFactory.LoadMLConfiguration(mlconfigPath);
//add full path of model folder since model file doesn't contains any absolute path
dicMParameters.Add("root", MLFactory.GetMLConfigFolder(mlconfigPath));
// get model data paths
var dicPath = MLFactory.GetMLConfigComponentPaths(dicMParameters["paths"]);
//parse feature variables
var projectValues = dicMParameters["training"].Split(MLFactory.m_cntkSpearator, StringSplitOptions.RemoveEmptyEntries);
var modelName = MLFactory.GetParameterValue(projectValues, "TrainedModel");
var nnModelPath = Path.Combine(dicMParameters["root"], modelName);
//check if model exists
if (!MLFactory.IsFileExist(nnModelPath))
{
return(er);
}
//
var dataset = MLFactory.GetDataPath(dicMParameters, dsType);
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
if (dsType == DataSetType.Testing)
{
dataset = MLFactory.GetDataPath(dicMParameters, DataSetType.Validation);
}
if (string.IsNullOrEmpty(dataset) || string.IsNullOrEmpty(dataset) || dataset == " ")
{
return(er);
}
}
//get output classes in case the ml problem is classification
var strCls = dicMParameters.ContainsKey("metadata") ? dicMParameters["metadata"] : "";
var oc = MLFactory.GetOutputClasses(strCls);
if (oc != null)
{
er.OutputClasses = oc;
}
//MInibatch
var mbTypestr = MLFactory.GetParameterValue(projectValues, "Type");
MinibatchType mbType = (MinibatchType)Enum.Parse(typeof(MinibatchType), mbTypestr, true);
var mbSizetr = MLFactory.GetParameterValue(projectValues, "BatchSize");
var mf = MLFactory.CreateMLFactory(dicMParameters);
//perform evaluation
var evParams = new EvaluationParameters()
{
MinibatchSize = uint.Parse(mbSizetr),
MBSource = new MinibatchSourceEx(mbType, mf.StreamConfigurations.ToArray(), mf.InputVariables, mf.OutputVariables, dataset, null, MinibatchSource.FullDataSweep, false, 0),
Input = mf.InputVariables,
Ouptut = mf.OutputVariables,
};
//evaluate model
if (evType == EvaluationType.FeaturesOnly)
{
if (!dicMParameters.ContainsKey("metadata"))
{
throw new Exception("The result cannot be exported to Excel, since no metadata is stored in mlconfig file.");
}
var desc = MLFactory.ParseRawDataSet(dicMParameters["metadata"]);
er.Header = MLFactory.GenerateHeader(desc);
var fun = Function.Load(nnModelPath, device);
//
er.DataSet = await Task.Run(() => MLEvaluator.FeaturesAndLabels(fun, evParams, device));
return(er);
}
else if (evType == EvaluationType.Results)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
var fun = Function.Load(nnModelPath, device);
//
var result = await Task.Run(() => MLEvaluator.EvaluateFunction(fun, evParams, device));
er.Actual = result.actual.ToList();
er.Predicted = result.predicted.ToList();
if (er.OutputClasses.Count < 2 && evParams.Ouptut.First().Shape.Dimensions.Last() > 1)
{
var result1 = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
er.ActualEx = result1.actual;
er.PredictedEx = result1.predicted;
}
return(er);
}
else if (evType == EvaluationType.ResultExtended)
{
//define header
er.Header.Add(evParams.Ouptut.First().Name + "_actual");
er.Header.Add(evParams.Ouptut.First().Name + "_predicted");
er.Actual = new List <float>();
er.Predicted = new List <float>();
er.ActualEx = new List <List <float> >();
er.PredictedEx = new List <List <float> >();
//
var fun = Function.Load(nnModelPath, device);
var resultEx = await Task.Run(() => MLEvaluator.EvaluateFunctionEx(fun, evParams, device));
//var resultEx = EvaluateFunctionEx(nnModelPath, dataPath, evParams, device);
for (int i = 0; i < resultEx.actual.Count(); i++)
{
var res1 = MLValue.GetResult(resultEx.actual[i]);
er.Actual.Add(res1);
var res2 = MLValue.GetResult(resultEx.predicted[i]);
er.Predicted.Add(res2);
}
er.ActualEx = resultEx.actual;
er.PredictedEx = resultEx.predicted;
return(er);
}
else
{
throw new Exception("Unknown evaluation type!");
}
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// <para>
/// DataSetType acts as a structural base, which is extended through the addition of attributes to reflect the particular needs of a specific key family using the xs:extends element.
/// </para>
/// </summary>
public DataSet(DataSetType content) {
SetInnerType(content);
}
private void SetInnerType(DataSetType ContentField) {
this.ContentField = ((DataSetType)(XTypedServices.GetCloneIfRooted(ContentField)));
XTypedServices.SetName(this, this.ContentField);
}
/// <summary>
///
/// </summary>
/// <param name="filePath"></param>
/// <param name="GTAIV"></param>
/// <returns></returns>
private bool doImport(string filePath, DataSetType data_type )
{
// Import the file from the specified file path. For now the GTAIV flag is
// ignored.
m_log.InfoFormat("[CITY BUILDER]: Importing from {0}", filePath);
return (false);
}
/// <summary>
///
/// </summary>
/// <param name="filePath"></param>
/// <param name="GTAIV"></param>
/// <returns></returns>
private bool doExport(string filePath, DataSetType data_type )
{
// Initial stage for the export/import functionality is the ability to export/import
// the data correctly for the current city, uses OSDMap. For now the GTAIV flag is
// ignored, this needs to be changed to allow for internal, GTA and GIS data sets to
// be used for cities.
if (filePath == null || data_type == DataSetType.DATASET_TYPE_NULL)
{
return (false);
}
// Make sure the file is not present on the destination.
// ???
// First stage use a TarArchiveWriter in conjunction with OSDMap to write a file
// containing all the information needed about the city and regions to be able to
// import it in when the server is shutdown and restarted.
System.IO.MemoryStream data = new System.IO.MemoryStream(1024);
TarArchiveWriter tarArchive = null;
// Construct the archive before writing it to the destination.
tarArchive = new TarArchiveWriter(data);
tarArchive.WriteDir("citybuilder/");
tarArchive.WriteFile(filePath, data.GetBuffer());
return (false);
}
