public void TestMultipleObjects()
{
var a = DataSourceFactory.Create(new float[] { 1, 2, 3 }, new int[] { 3, 1, 1 });
var dss = new DataSourceSet();
dss.Add("a", a);
var dss2 = new DataSourceSet();
var b = DataSourceFactory.Create(new float[] { 4, 5, 6, 7 }, new int[] { 2, 2, 1 });
dss2.Add("b", b);
var stream = new MemoryStream();
MsgPackSerializer.Serialize(dss, stream);
MsgPackSerializer.Serialize(dss2, stream);
stream.Position = 0;
var result = MsgPackSerializer.Deserialize(stream);
var result2 = MsgPackSerializer.Deserialize(stream);
Assert.AreEqual(1, result.Features.Count);
var x = result["a"];
CollectionAssert.AreEqual(new int[] { 3, 1, 1 }, x.Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 1, 2, 3 }, x.Data.ToArray());
Assert.AreEqual(1, result2.Features.Count);
var y = result2["b"];
CollectionAssert.AreEqual(new int[] { 2, 2, 1 }, y.Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 4, 5, 6, 7 }, y.Data.ToArray());
}
public async Task OnGet()
{
var datasource = DataSourceFactory.Create(Program.DataSource);
var databases = await datasource.GetDatabases("*");
Databases = databases.Select(x => new SelectListItem(x, x)).ToList();
}
public async Task TestGetDatabases_Filter_NoMatch()
{
IDataSource ds = DataSourceFactory.Create(dsi);
var databases = await ds.GetDatabases("²»¿ÉÄÜÆ¥Åä");
Assert.AreEqual(0, databases.Count);
}
public void TestConvolutionTranspose2()
{
var inputData = DataSourceFactory.Create(new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { 2, 2, 3 });
// var inputData = DataSourceFactory.Create(new float[] { 1, 2, 3, 4 }, new int[] { 2, 2, 1 });
var input = CNTKLib.InputVariable(new int[] { 2, 2, 3 }, DataType.Float);
var convolutionMap = new Parameter(new int[] { 2, 2, 6, 3 }, DataType.Float, CNTKLib.ConstantInitializer(1));
var f = CNTKLib.ConvolutionTranspose(convolutionMap, input, new int[] { 2, 2, 3 }, new BoolVector(new bool[] { true }), new BoolVector(new bool[] { true }), new int[] { 0 }, new int[] { 1 }, 1, 0, "");
var inputs = new Dictionary <Variable, Value>()
{
{ input, inputData.ToValue() }
};
var outputs = new Dictionary <Variable, Value>()
{
{ f.Output, null }
};
f.Evaluate(inputs, outputs, DeviceDescriptor.UseDefaultDevice());
var result = DataSourceFactory.FromValue(outputs[f.Output]);
CollectionAssert.AreEqual(new int[] { 3, 3, 6, 1, 1 }, result.Shape.Dimensions);
Assert.AreEqual(15, result.Data[0]);
Assert.AreEqual(15, result.Data[1]);
Assert.AreEqual(18, result.Data[2]);
Assert.AreEqual(15, result.Data[3]);
}
/// <summary>
/// Get count of container by query
/// </summary>
/// <typeparam name="T">The type of the resulting containers</typeparam>
/// <param name="types">The allowed types of contained content items returned</param>
/// <param name="queryBody">An operator on an IQueryable of the container type to filter the ones to count</param>
/// <returns>The count of containers</returns>
public virtual int GetCount <T>(IEnumerable <Type> types, Func <IQueryable <T>, IQueryable <T> > queryBody) where T : class
{
if (types == null || !types.Any())
{
return(0);
}
if (types.Count() == 1)
{
using (var dataSource = DataSourceFactory.Create(true))
{
var qed = new QueryEventData <IQueryable>
{
Source = dataSource.GetSource(typeof(T).UnextendedType()),
QueryBody = iq => queryBody(iq.AsFacade <T>())
};
qed = System.Events.ProcessEvent("Repository.Get.Count", this, qed).Data as QueryEventData <IQueryable>;
int ct = qed.QueryBody(qed.Source).AsFacade <T>().Count();
return(ct);
}
}
else
{
int count = 0;
foreach (Type t in types)
{
count += this.GetCount <T>(new Type[] { t }, queryBody);
}
return(count);
}
}
public static List <Record> LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord)
{
// (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>)
var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content));
try
{
if (multiRecord)
{
return(dataSource.ReadMulti(recordType));
}
else
{
Record record = dataSource.ReadOne(recordType);
return(record != null ? new List <Record> {
record
} : new List <Record>());
}
}
catch (DataCreateException dce)
{
if (string.IsNullOrWhiteSpace(dce.OriginDataLocation))
{
dce.OriginDataLocation = originFile;
}
throw;
}
catch (Exception e)
{
throw new Exception($"配置文件:{originFile} 生成失败.", e);
}
}
public void TestConvTrans()
{
var inputData = DataSourceFactory.Create(new float[] { 1, 2, 3, 4 }, new int[] { 2, 2, 1 });
var input = CNTKLib.InputVariable(new int[] { 2, 2, 1 }, DataType.Float);
var f = Composite.ConvolutionTranspose(input, new int[] { 2, 2 }, 1, null, CNTKLib.ConstantInitializer(1), new bool[] { true }, new int[] { 2, 2, 1 }, false, null, new int[] { 0 }, new int[] { 1 }, 1, 0, "");
var inputs = new Dictionary <Variable, Value>()
{
{ input, inputData.ToValue() }
};
var outputs = new Dictionary <Variable, Value>()
{
{ f.Output, null }
};
f.Evaluate(inputs, outputs, DeviceDescriptor.UseDefaultDevice());
var result = DataSourceFactory.FromValue(outputs[f.Output]);
CollectionAssert.AreEqual(new int[] { 3, 3, 1, 1, 1 }, result.Shape.Dimensions);
Assert.AreEqual(1, result.Data[0]);
Assert.AreEqual(1, result.Data[1]);
Assert.AreEqual(2, result.Data[2]);
Assert.AreEqual(1, result.Data[3]);
}
public void Create_CreatingExistingProvider_ReturnsDataSource()
{
var providerFactory = A.Fake <DbProviderFactory>();
var dataSource = DataSourceFactory.Create(providerFactory, string.Empty);
Assert.IsNotNull(dataSource);
}
public void TestDataSourceCTFBuilider()
{
var writer = new StringWriter();
var ds1 = DataSourceFactory.Create(new float[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 }, new int[] { 2, 3, 3 });
var ds2 = DataSourceFactory.Create(new float[] { 0, 1, 2, 3, 4, 5 }, new int[] { 2, 1, 3 });
var dss = new DataSourceSet();
dss.Features.Add("data", ds1);
dss.Features.Add("label", ds2);
DataSourceSetCTFBuilder.Write(writer, dss, true);
var s = writer.ToString();
var expected =
"0\t|data 0 1\t|label 0 1\r\n" +
"0\t|data 2 3\r\n" +
"0\t|data 4 5\r\n" +
"1\t|data 6 7\t|label 2 3\r\n" +
"1\t|data 8 9\r\n" +
"1\t|data 10 11\r\n" +
"2\t|data 12 13\t|label 4 5\r\n" +
"2\t|data 14 15\r\n" +
"2\t|data 16 17";
Assert.AreEqual(expected, s);
}
public void TestMsgPackSamplerMultipleFiles()
{
var files = new string[2];
for (var i = 0; i < 2; ++i)
{
files[i] = Path.GetTempFileName();
var a = DataSourceFactory.Create(new float[] { i, i + 1, i + 2 }, new int[] { 3, 1 });
var dss = new DataSourceSet();
dss.Add("a", a);
using (var stream = new FileStream(files[i], FileMode.Create, FileAccess.Write))
{
MsgPackSerializer.Serialize(dss, stream);
}
}
using (var sampler = new MsgPackSampler(files, 3, false, -1, 10, false, 10))
{
Assert.AreEqual(2, sampler.SampleCountPerEpoch);
var batch = sampler.GetNextMinibatch();
CollectionAssert.AreEqual(new int[] { 3, 3 }, batch.Features["a"].Shape.Dimensions.ToArray());
var values = DataSourceFactory.FromValue(batch.Features["a"]);
CollectionAssert.AreEqual(new float[] { 0, 1, 2, 1, 2, 3, 0, 1, 2 }, values.TypedData);
}
}
/// <summary>
/// Set (create or update) a list of containers to the data source
/// </summary>
/// <param name="items">The list of containers</param>
/// <param name="setOptions">Options for setting</param>
/// <returns>List of flags for whether the corresponding by position item was created (rather than updated)</returns>
public virtual List <bool> Set(List <object> items, Dictionary <string, object> setOptions)
{
if (!(items.All(i => i is ContentItem)))
{
throw new ArgumentException("Content repository can only set ContentItem type");
}
var createds = new List <bool>();
bool?create = setOptions.Get <bool?>("create");
bool bypassChecks = setOptions.Get <bool>("bypassChecks", false);
using (var dataSource = DataSourceFactory.Create(false))
{
var ciSaved = new List <Tuple <ContentItem, bool> >();
bool anyUnhandled = false;
foreach (ContentItem ci in items)
{
if (ChangeProblems.Any(cp => cp.TypeName == ci.DataType) && !BypassChangeProblems)
{
throw new ProhibitedActionException("Changes in the structure of the data may cause data loss, please advise an administrator");
}
bool isAdd = (create == null ? ci.Id == Guid.Empty : create.Value);
var eventData = new RepositoryEventData(ci, setOptions);
var eventResult = System.Events.ProcessEvent("Repository.Set." + (isAdd ? "Add" : "Update"), this, eventData);
var ciSave = (ContentItem)((RepositoryEventData)eventResult.Data).Container;
bool wasHandled = ((RepositoryEventData)eventResult.Data).WasHandled;
if (!wasHandled)
{
anyUnhandled = true;
}
isAdd = eventResult.EventName.EndsWith("Add");
createds.Add(isAdd);
if (isAdd)
{
DoAdd(dataSource, ciSave, wasHandled);
}
else if (!wasHandled)
{
dataSource.Update(ciSave);
}
ciSaved.Add(Tuple.Create(ciSave, isAdd));
}
if (ciSaved.Count > 0 && anyUnhandled)
{
dataSource.SaveChanges();
}
foreach (var sv in ciSaved)
{
var savedEventData = new RepositoryEventData(sv.Item1, setOptions);
System.Events.ProcessEvent("Repository.Saved." + (sv.Item2 ? "Add" : "Update"), this, savedEventData);
}
}
return(createds);
}
public void TestTranspose3()
{
var a = DataSourceFactory.Create(new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { 3, 2, 2 });
var b = a.Transpose(1, 2, 0);
CollectionAssert.AreEqual(new int[] { 1, 4, 7, 10, 2, 5, 8, 11, 3, 6, 9, 12 }, b.TypedData);
}
public void TestTranspose()
{
var a = DataSourceFactory.Create(new int[] { 1, 2, 3, 4, 5, 6 }, new int[] { 2, 3 });
var b = a.Transpose(1, 0);
CollectionAssert.AreEqual(new int[] { 1, 3, 5, 2, 4, 6 }, b.TypedData);
}
public async Task TestGetDatabases_Filter_ExactMatch()
{
IDataSource ds = DataSourceFactory.Create(dsi);
var databases = await ds.GetDatabases("mysql");
Assert.AreEqual(1, databases.Count);
CollectionAssert.Contains(databases, "mysql");
}
public void TestSubset()
{
var a = DataSourceFactory.Create(new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { 2, 2, 3 });
var b = a.Subset(1, 2);
CollectionAssert.AreEqual(new int[] { 2, 2, 2 }, b.Shape.Dimensions);
CollectionAssert.AreEqual(new int[] { 5, 6, 7, 8, 9, 10, 11, 12 }, b.TypedData.ToArray());
}
public async Task TestGetDatabases_Filter_PartialMatch()
{
IDataSource ds = DataSourceFactory.Create(dsi);
var databases2 = await ds.GetDatabases("*_schema");
Assert.AreEqual(2, databases2.Count);
CollectionAssert.Contains(databases2, "information_schema");
CollectionAssert.Contains(databases2, "performance_schema");
}
public void TestToString2()
{
var a = DataSourceFactory.Create(new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { 2, 2, 3 });
var s = a.ToString();
var expected = "DataSource [2 x 2 x 3] [1 2 3 4 5...]";
Assert.AreEqual(expected, s);
}
public async Task TestGetDatabases_Filter_NoMatch()
{
DataSourceInfo dsi = new DataSourceInfo();
dsi.ConnectionString = ConnectionString;
IDataSource ds = DataSourceFactory.Create(dsi);
var databases = await ds.GetDatabases("²»¿ÉÄÜÆ¥Åä");
Assert.AreEqual(0, databases.Count);
}
public void TestCompression()
{
var a = DataSourceFactory.Create(new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { 2, 2, 3 });
var bytes = a.Serialize(true);
var b = DataSourceFactory.Load <float>(bytes, true);
Assert.AreEqual(a.Shape, b.Shape);
CollectionAssert.AreEqual(a.TypedData, b.TypedData);
}
public async Task TestGetDatabases_Filter_ExactMatch()
{
DataSourceInfo dsi = new DataSourceInfo();
dsi.ConnectionString = ConnectionString;
IDataSource ds = DataSourceFactory.Create(dsi);
var databases = await ds.GetDatabases("postgres");
Assert.AreEqual(1, databases.Count);
CollectionAssert.Contains(databases, "postgres");
}
public void TestMinibatch()
{
var features = DataSourceFactory.Create(new float[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, new int[] { 2, 1, -1 });
var dss = new Dictionary <string, IDataSource <float> >()
{
{ "input", features }
};
var sampler = new DataSourceSampler(dss, 2, false, true);
{
var batch = sampler.GetNextMinibatch();
// GC.Collect();
// GC.Collect();
Assert.AreEqual(2, batch.SampleCount);
Assert.AreEqual(false, batch.SweepEnd);
var data = batch.Features["input"];
// var c1 = SharedPtrMethods.GetUseCountOf(data);
// var c2 = SharedPtrMethods.GetUseCountOf(data.data);
// var c3 = SharedPtrMethods.GetUseCountOf(data.data.Data);
var ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(new float[] { 0, 1, 2, 3 }, ds.TypedData);
CollectionAssert.AreEqual(new int[] { 2, 1, 2 }, data.Shape.Dimensions.ToArray());
}
{
var batch = sampler.GetNextMinibatch();
// GC.Collect();
Assert.AreEqual(2, batch.SampleCount);
Assert.AreEqual(true, batch.SweepEnd);
var data = batch.Features["input"];
// var c1 = SharedPtrMethods.GetUseCountOf(data);
// var c2 = SharedPtrMethods.GetUseCountOf(data.data);
// var c3 = SharedPtrMethods.GetUseCountOf(data.data.Data);
var ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(new float[] { 4, 5, 6, 7 }, ds.TypedData);
CollectionAssert.AreEqual(new int[] { 2, 1, 2 }, data.Shape.Dimensions.ToArray());
}
// When not randomized, remnant data that is smaller than the minibatch size is ignored.
{
var batch = sampler.GetNextMinibatch();
// GC.Collect();
Assert.AreEqual(2, batch.SampleCount);
Assert.AreEqual(false, batch.SweepEnd);
var data = batch.Features["input"];
var ds = DataSourceFactory.FromValue(data);
CollectionAssert.AreEqual(new float[] { 0, 1, 2, 3 }, ds.TypedData);
CollectionAssert.AreEqual(new int[] { 2, 1, 2 }, data.Shape.Dimensions.ToArray());
}
}
public async Task <ActionResult <ListResult <string> > > GetDatabases()
{
var dsi = new DataSourceInfo();
dsi.ConnectionString = "Server=127.0.0.1;Database=test;Uid=root;Pwd=tonyqus;";
var datasource = DataSourceFactory.Create(dsi);
var databases = await datasource.GetDatabases("*");
var result = new ListResult <string>(databases);
return(Ok(result));
}
public async Task TestGetDatabases_NoFilter()
{
IDataSource ds = DataSourceFactory.Create(dsi);
var databases = await ds.GetDatabases("*");
Assert.IsTrue(databases.Count > 0);
CollectionAssert.Contains(databases, "mysql");
var databases2 = await ds.GetDatabases(" ");
CollectionAssert.AreEqual(databases, databases2);
}
public void SendSelectedDataSource(DataSourceType dataSource)
{
try
{
notebookManager.DataSource = DataSourceFactory.Create(dataSource);
}
catch (Exception e)
{
InformationDispatcher.Default.Dispatch(e, context);
}
//ExceptionUtils.LogExceptions(() => notebookManager.DataSource = DataSourceFactory.Create(dataSource), context);
}
public void TestTranspose2()
{
var a = DataSourceFactory.Create(new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { 2, 3, 2 });
var b = a.Transpose(1, 0, 2);
CollectionAssert.AreEqual(new int[] { 1, 3, 5, 2, 4, 6, 7, 9, 11, 8, 10, 12 }, b.TypedData);
b = a.Transpose(1, 2, 0);
CollectionAssert.AreEqual(new int[] { 1, 3, 5, 7, 9, 11, 2, 4, 6, 8, 10, 12 }, b.TypedData);
b = a.Transpose(2, 1, 0);
CollectionAssert.AreEqual(new int[] { 1, 7, 3, 9, 5, 11, 2, 8, 4, 10, 6, 12 }, b.TypedData);
}
public void TestAsString1()
{
var a = DataSourceFactory.Create(new float[] { 1, 2, 3, 4, 5, 6 }, new int[] { 2, 3 });
var s = a.AsString();
var expected =
"DataSource [2 x 3]\r\n" +
" [ [1 2]\r\n" +
" [3 4]\r\n" +
" [5 6] ]";
Assert.AreEqual(expected, s);
}
public void TestAsString2()
{
var a = DataSourceFactory.Create(new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { 2, 2, 3 });
var s = a.AsString();
var expected =
"DataSource [2 x 2 x 3]\r\n" +
" [ [ [1 2] [3 4] ]\r\n" +
" [ [5 6] [7 8] ]\r\n" +
" [ [9 10] [11 12] ] ]";
Assert.AreEqual(expected, s);
}
public void TestReadDataSourceSet()
{
const int NUM_SAMPLES = 10;
DataSourceSet dss;
var file = Path.GetTempFileName();
using (var stream = new FileStream(file, FileMode.Create, FileAccess.Write))
{
for (var i = 0; i < NUM_SAMPLES; ++i)
{
var a = DataSourceFactory.Create(new float[] { i, i * 10, i * 100 }, new int[] { 3, 1, 1 });
dss = new DataSourceSet();
dss.Add("a", a);
MsgPackSerializer.Serialize(dss, stream);
}
}
var total = MsgPackTools.GetTotalSampleCount(file);
Assert.AreEqual(NUM_SAMPLES, total);
var reader = MsgPackTools.ReadDataSourceSet(file, total, 4).GetEnumerator();
var hasNext = reader.MoveNext();
Assert.AreEqual(true, hasNext);
dss = reader.Current;
Assert.AreEqual(4, dss.SampleCount);
CollectionAssert.AreEqual(new int[] { 3, 1, 4 }, dss["a"].Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 0, 0, 0, 1, 10, 100, 2, 20, 200, 3, 30, 300 }, dss["a"].Data.ToArray());
hasNext = reader.MoveNext();
Assert.AreEqual(true, hasNext);
dss = reader.Current;
Assert.AreEqual(4, dss.SampleCount);
CollectionAssert.AreEqual(new int[] { 3, 1, 4 }, dss["a"].Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 4, 40, 400, 5, 50, 500, 6, 60, 600, 7, 70, 700 }, dss["a"].Data.ToArray());
hasNext = reader.MoveNext();
Assert.AreEqual(true, hasNext);
dss = reader.Current;
Assert.AreEqual(2, dss.SampleCount);
CollectionAssert.AreEqual(new int[] { 3, 1, 2 }, dss["a"].Shape.Dimensions);
CollectionAssert.AreEqual(new float[] { 8, 80, 800, 9, 90, 900 }, dss["a"].Data.ToArray());
hasNext = reader.MoveNext();
Assert.AreEqual(false, hasNext);
}
public void TestSubsequence1()
{
var a = DataSourceFactory.Create(new float[] { 1, 2, 3, 4 }, new int[] { 1, 4, 1 });
var result = a.GetSubsequences(3);
var newShape = new int[] { 1, 3, 2 };
var newData = new float[] {
1, 2, 3,
2, 3, 4
};
CollectionAssert.AreEqual(newShape, result.Shape.Dimensions);
CollectionAssert.AreEqual(newData, (float[])result.Data);
}
public void TestSubsequence5()
{
var a = DataSourceFactory.Create(new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }, new int[] { 1, 2, 6, 1 });
var result = a.GetSubsequences(2, 2);
var newShape = new int[] { 1, 2, 2, 3 };
var newData = new float[] {
1, 2, 3, 4, 5, 6, 7, 8,
9, 10, 11, 12
};
CollectionAssert.AreEqual(newShape, result.Shape.Dimensions);
CollectionAssert.AreEqual(newData, (float[])result.Data);
}
