public void TestRdd() { const string jsonSchema = @" { ""type"" : ""struct"", ""fields"" : [{ ""name"" : ""age"", ""type"" : ""long"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""id"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } }, { ""name"" : ""name"", ""type"" : ""string"", ""nullable"" : true, ""metadata"" : { } } ] }"; Mock <IStructTypeProxy> mockStructTypeProxy = new Mock <IStructTypeProxy>(); mockStructTypeProxy.Setup(m => m.ToJson()).Returns(jsonSchema); mockDataFrameProxy.Setup(m => m.GetSchema()).Returns(mockStructTypeProxy.Object); var rows = new object[] { new RowImpl(new object[] { 34, "123", "Bill" }, RowSchema.ParseRowSchemaFromJson(jsonSchema)) }; mockDataFrameProxy.Setup(m => m.JavaToCSharp()).Returns(new MockRddProxy(rows)); var sc = new SparkContext(null); var dataFrame = new DataFrame(mockDataFrameProxy.Object, sc); // act var rdd = dataFrame.Rdd; Assert.IsNotNull(rdd); mockDataFrameProxy.Verify(m => m.JavaToCSharp(), Times.Once); mockStructTypeProxy.Verify(m => m.ToJson(), Times.Once); mockDataFrameProxy.Reset(); mockStructTypeProxy.Reset(); rdd = dataFrame.Rdd; Assert.IsNotNull(rdd); mockDataFrameProxy.Verify(m => m.JavaToCSharp(), Times.Never); mockStructTypeProxy.Verify(m => m.ToJson(), Times.Never); }
public async Task RunAsync(CancellationToken token) { var sourceRows = await _tableStore.GetRowsAsync(_sourceTableName, CancellationToken.None); var targetRows = new List <DataRow>(); var targetSchema = new RowSchema( new FieldDefinition("ContactId", FieldKind.Key, FieldDataType.Guid), new FieldDefinition("MovieId", FieldKind.Key, FieldDataType.String), new FieldDefinition("Title", FieldKind.Attribute, FieldDataType.String), new FieldDefinition("Overview", FieldKind.Attribute, FieldDataType.String), new FieldDefinition("Image", FieldKind.Attribute, FieldDataType.String), new FieldDefinition("ImdbRating", FieldKind.Attribute, FieldDataType.String) ); while (await sourceRows.MoveNext()) { foreach (var row in sourceRows.Current) { var movieIds = row["LastMoviesId"].ToString().Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries); //grabbing only unique movie ID's movieIds = movieIds.Distinct().ToArray(); if (!movieIds.Any()) { continue; } var recommendations = _movieRecommendationsProvider.GetRecommendations(movieIds); //clean recommendations to not have any dups recommendations = recommendations.GroupBy(m => m.netflixid).Select(g => g.FirstOrDefault()).ToList(); if (recommendations.Any()) { for (var i = 0; i < _limit; i++) { if (i < recommendations.Count) { var targetRow = new DataRow(targetSchema); targetRow.SetGuid(0, row.GetGuid(0)); targetRow.SetString(1, recommendations[i].netflixid); targetRow.SetString(2, recommendations[i].title); targetRow.SetString(3, recommendations[i].synopsis); targetRow.SetString(4, recommendations[i].image); targetRow.SetString(5, recommendations[i].rating); targetRows.Add(targetRow); } } } } } var tableDefinition = new TableDefinition(_targetTableName, targetSchema); var targetTable = new InMemoryTableData(tableDefinition, targetRows); await _tableStore.PutTableAsync(targetTable, TimeSpan.FromMinutes(30), CancellationToken.None); }
public async Task RunAsync(CancellationToken token) { var sourceRows = await _tableStore.GetRowsAsync(_sourceTableName, CancellationToken.None); var targetRows = new List <DataRow>(); var targetSchema = new RowSchema( new FieldDefinition("ContactId", FieldKind.Key, FieldDataType.Guid), new FieldDefinition("MovieId", FieldKind.Key, FieldDataType.Int64), new FieldDefinition("Title", FieldKind.Attribute, FieldDataType.String), new FieldDefinition("Overview", FieldKind.Attribute, FieldDataType.String), new FieldDefinition("PosterPath", FieldKind.Attribute, FieldDataType.String) ); //ML context MLContext mlContext = new MLContext(); ITransformer mlModel = mlContext.Model.Load(GetAbsolutePath(MODEL_FILEPATH), out DataViewSchema inputSchema); var predEngine = mlContext.Model.CreatePredictionEngine <TacoFlixML.Model.DataModels.ModelInput, ModelOutput>(mlModel); while (await sourceRows.MoveNext()) { foreach (var row in sourceRows.Current) { var movieId = int.Parse(row["LastMovieId"].ToString()); var recommendations = _movieRecommendationsProvider.GetRecommendations(movieId); if (recommendations.Any()) { for (var i = 0; i < _limit; i++) { if (i < recommendations.Count) { //ML ModelInput input = new ModelInput(); input.Movieid = recommendations[i].Id; ModelOutput predictionResult = predEngine.Predict(input); var targetRow = new DataRow(targetSchema); targetRow.SetGuid(0, row.GetGuid(0)); targetRow.SetInt64(1, recommendations[i].Id); targetRow.SetString(2, recommendations[i].Title + "score=" + predictionResult.Score); targetRow.SetString(3, recommendations[i].Overview); targetRow.SetString(4, recommendations[i].PosterPath); targetRows.Add(targetRow); } } } } } var tableDefinition = new TableDefinition(_targetTableName, targetSchema); var targetTable = new InMemoryTableData(tableDefinition, targetRows); await _tableStore.PutTableAsync(targetTable, TimeSpan.FromMinutes(30), CancellationToken.None); }
public TableObserver(Database db, SqlObjectName sqlObjectName, string[] primaryKeyColumns, bool memoryTrigger) { _db = db; SqlObjectName = sqlObjectName; Schema = new RowSchema(db.Config.DatabaseConnectionString, sqlObjectName, primaryKeyColumns); _insertTriggerName = string.Format(db.Config.TriggerNameTemplate, sqlObjectName.Schema, "insert", sqlObjectName.Name); _deleteTriggerName = string.Format(db.Config.TriggerNameTemplate, sqlObjectName.Schema, "delete", sqlObjectName.Name); _updateTriggerName = string.Format(db.Config.TriggerNameTemplate, sqlObjectName.Schema, "update", sqlObjectName.Name); SetupDatabaseSchema(memoryTrigger); }
public override IReadOnlyCollection <Row> Initialize() { Db.Config.DatabaseConnectionString.WithReader(Query, reader => { Schema = new RowSchema(reader, PrimaryKeyColumns); _data = new ReducableDictionary <object[], Row>(Schema.RowKeyEqualityComparer); while (reader.Read()) { var row = Schema.ReadRow(reader); _data.Add(row.Key, row); } }); return(_data.Values.ToArray()); }
private RowSchema <TCellType> CreateRowSchema <TCellType>(Func <DataTableColumn, TCellType> cellTemplateSelector) where TCellType : TableCellBase <TCellType> { var schema = new RowSchema <TCellType>(); var newRow = true; List <List <DataTableColumn> > rows = new List <List <DataTableColumn> >(); List <DataTableColumn> row = null; foreach (var column in _columns) { column.ItemTemplate.RowSpan = 0; column.ItemTemplate.ColSpan = 0; column.HeaderTemplate.RowSpan = 0; column.HeaderTemplate.ColSpan = 0; if (newRow) { newRow = false; row = new List <DataTableColumn>(); rows.Add(row); } if (column.ColSpanBehavior == ColSpanBehavior.Row) { foreach (var col in row) { col.HeaderTemplate.RowSpan = Math.Max(2, col.HeaderTemplate.RowSpan + 1); col.ItemTemplate.RowSpan = Math.Max(2, col.ItemTemplate.RowSpan + 1); } var colSpan = _columns.Count - row.Count - 1; column.HeaderTemplate.ColSpan = colSpan; column.ItemTemplate.ColSpan = colSpan; newRow = true; } row.Add(column); } foreach (var rowItems in rows) { var tableRow = new List <TCellType>(); schema.Add(tableRow); foreach (var col in rowItems) { var cell = cellTemplateSelector(col); tableRow.Add(cell); } } return(schema); }
public async Task RunAsync(CancellationToken token) { // Retrieve the source data from the projection. var sourceRows = await _tableStore.GetRowsAsync(_sourceTableName, CancellationToken.None); // Define the target table shcema we'll be populating into. var targetRows = new List <DataRow>(); var targetSchema = new RowSchema( new FieldDefinition("ContactID", FieldKind.Key, FieldDataType.Guid), new FieldDefinition("AlbumID", FieldKind.Key, FieldDataType.String), new FieldDefinition("AlbumName", FieldKind.Attribute, FieldDataType.String), new FieldDefinition("ArtistID", FieldKind.Attribute, FieldDataType.String), new FieldDefinition("ArtistName", FieldKind.Attribute, FieldDataType.String) ); // Iterate the source data. while (await sourceRows.MoveNext()) { foreach (var row in sourceRows.Current) { // Retrieve the IDs of the album artists from the projected data. var artistIds = row["Artists"].ToString().Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries); // Call the recomender service. var recommendations = await _musicRecommender.GetRecommendationsAsync(artistIds, _limit); // Add a target data row for each recommendation. foreach (var album in recommendations) { var targetRow = new DataRow(targetSchema); targetRow.SetGuid(0, row.GetGuid(0)); targetRow.SetString(1, album.AlbumId); targetRow.SetString(2, album.AlbumName); targetRow.SetString(3, album.ArtistId); targetRow.SetString(4, album.ArtistName); targetRows.Add(targetRow); } } } // Populate the rows into the target table. var tableDefinition = new TableDefinition(_targetTableName, targetSchema); var targetTable = new InMemoryTableData(tableDefinition, targetRows); await _tableStore.PutTableAsync(targetTable, TimeSpan.FromMinutes(30), CancellationToken.None); }
/// <summary> /// Verify the schema of people dataframe. /// </summary> /// <param name="schema"> RowSchema of people DataFrame </param> internal static void VerifySchemaOfPeopleDataFrame(RowSchema schema) { Assert.IsNotNull(schema); Assert.AreEqual("struct", schema.type); Assert.IsNotNull(schema.columns); Assert.AreEqual(4, schema.columns.Count); // name var nameColSchema = schema.columns.Find(c => c.name.Equals("name")); Assert.IsNotNull(nameColSchema); Assert.AreEqual("name", nameColSchema.name); Assert.IsTrue(nameColSchema.nullable); Assert.AreEqual("string", nameColSchema.type.ToString()); // id var idColSchema = schema.columns.Find(c => c.name.Equals("id")); Assert.IsNotNull(idColSchema); Assert.AreEqual("id", idColSchema.name); Assert.IsTrue(idColSchema.nullable); Assert.AreEqual("string", nameColSchema.type.ToString()); // age var ageColSchema = schema.columns.Find(c => c.name.Equals("age")); Assert.IsNotNull(ageColSchema); Assert.AreEqual("age", ageColSchema.name); Assert.IsTrue(ageColSchema.nullable); Assert.AreEqual("long", ageColSchema.type.ToString()); // address var addressColSchema = schema.columns.Find(c => c.name.Equals("address")); Assert.IsNotNull(addressColSchema); Assert.AreEqual("address", addressColSchema.name); Assert.IsTrue(addressColSchema.nullable); Assert.IsNotNull(addressColSchema.type); Assert.AreEqual("struct", addressColSchema.type.type); Assert.IsNotNull(addressColSchema.type.columns.Find(c => c.name.Equals("state"))); Assert.IsNotNull(addressColSchema.type.columns.Find(c => c.name.Equals("city"))); }
/// <summary> /// Print given rows from people dataframe, and verify schema and contents if validation is enabled. /// </summary> /// <param name="rows"> Rows from people DataFrame </param> /// <param name="num"> Expected number of rows from people DataFrame </param> internal static void PrintAndVerifyPeopleDataFrameRows(IEnumerable <Row> rows, int num) { Console.WriteLine("peopleDataFrame:"); var count = 0; RowSchema schema = null; Row firstRow = null; foreach (var row in rows) { if (count == 0) { firstRow = row; schema = row.GetSchema(); Console.WriteLine("schema: {0}", schema); } // output each row Console.WriteLine(row); Console.Write("id: {0}, name: {1}, age: {2}", row.GetAs <string>("id"), row.GetAs <string>("name"), row.GetAs <int>("age")); var address = row.GetAs <Row>("address"); if (address != null) { Console.WriteLine(", state: {0}, city: {1}", address.GetAs <string>("state"), address.GetAs <string>("city")); } else { Console.WriteLine(); } count++; } if (SparkCLRSamples.Configuration.IsValidationEnabled) { Assert.AreEqual(count, num); VerifySchemaOfPeopleDataFrame(schema); VerifyFirstRowOfPeopleDataFrame(firstRow); } }
public async Task RunAsync(CancellationToken token) { var sourceRows = await _tableStore.GetRowsAsync(_sourceTableName, CancellationToken.None); var targetRows = new List <DataRow>(); var targetSchema = new RowSchema( new FieldDefinition("ContactId", FieldKind.Key, FieldDataType.Guid), new FieldDefinition("ProductId", FieldKind.Key, FieldDataType.Int64), new FieldDefinition("Score", FieldKind.Attribute, FieldDataType.Double) ); while (await sourceRows.MoveNext()) { foreach (var row in sourceRows.Current) { var purchasedProductId = uint.Parse(row["ProductId"].ToString()); var recommendations = GetRecommendList(purchasedProductId, _limit); if (recommendations.Any()) { for (var i = 0; i < _limit; i++) { var targetRow = new DataRow(targetSchema); targetRow.SetGuid(0, row.GetGuid(0)); targetRow.SetInt64(1, recommendations[i].RecommendedItemId); targetRow.SetDouble(2, recommendations[i].Score); targetRows.Add(targetRow); } } } } var tableDefinition = new TableDefinition(_targetTableName, targetSchema); var targetTable = new InMemoryTableData(tableDefinition, targetRows); await _tableStore.PutTableAsync(targetTable, TimeSpan.FromMinutes(30), CancellationToken.None); }
public override IReadOnlyCollection <Row> Initialize() { var connectionString = Db.Config.DatabaseConnectionString; Schema = new RowSchema(connectionString, View.ViewName, PrimaryKeyColumns); CreateCacheTable(connectionString, View.ViewName, _cacheTableName, PrimaryKeyColumns); //Db.Config.DatabaseConnectionString.CreateMemoryViewCache(_view.ViewName,_cacheTableName,PrimaryKeyColumns); DisposeHelper.Attach(() => connectionString.WithConnection(con => con.ExecuteNonQuery($"DROP TABLE {_cacheTableName}"))); _mergeSelectViewChanges = new MergeSelectViewChanges(_cacheTableName, View.ViewName, Schema.ColumnNames.Values.ToArray(), PrimaryKeyColumns, PrimaryKeyColumns); List <Row> rows = new List <Row>(); connectionString.WithReader($"SELECT * FROM {_cacheTableName}", r => { while (r.Read()) { rows.Add(Schema.ReadRow(r)); } }); return(rows); }
/// <summary> /// Initialize a new instance of <see cref="TableObserver"/> /// </summary> /// <param name="db">The database that contains table</param> /// <param name="sqlObjectName">The name of the table</param> public TableObserver(Database db, SqlObjectName sqlObjectName) { _db = db; SqlObjectName = sqlObjectName; Schema = new RowSchema(db.Config.DatabaseConnectionString, sqlObjectName); _insertTriggerName = string.Format(db.Config.TriggerNameTemplate, sqlObjectName.Schema, "insert", sqlObjectName.Name); _deleteTriggerName = string.Format(db.Config.TriggerNameTemplate, sqlObjectName.Schema, "delete", sqlObjectName.Name); _updateTriggerName = string.Format(db.Config.TriggerNameTemplate, sqlObjectName.Schema, "update", sqlObjectName.Name); bool isMemoryTable = false; db.Config.DatabaseConnectionString.WithConnection(con => { using (var command = con.CreateCommand()) { command.CommandText = "SELECT is_memory_optimized FROM sys.tables WHERE name = @name And schema_id = schema_id(@schema)"; command.Parameters.AddWithValue("name", sqlObjectName.Name); command.Parameters.AddWithValue("schema", sqlObjectName.Schema); isMemoryTable = (bool)command.ExecuteScalar(); } }); SetupDatabaseSchema(isMemoryTable); }
/// <summary> /// Verify the schema of people dataframe. /// </summary> /// <param name="schema"> RowSchema of people DataFrame </param> internal static void VerifySchemaOfPeopleDataFrame(RowSchema schema) { Assert.IsNotNull(schema); Assert.AreEqual("struct", schema.type); Assert.IsNotNull(schema.columns); Assert.AreEqual(4, schema.columns.Count); // name var nameColSchema = schema.columns.Find(c => c.name.Equals("name")); Assert.IsNotNull(nameColSchema); Assert.AreEqual("name", nameColSchema.name); Assert.IsTrue(nameColSchema.nullable); Assert.AreEqual("string", nameColSchema.type.ToString()); // id var idColSchema = schema.columns.Find(c => c.name.Equals("id")); Assert.IsNotNull(idColSchema); Assert.AreEqual("id", idColSchema.name); Assert.IsTrue(idColSchema.nullable); Assert.AreEqual("string", nameColSchema.type.ToString()); // age var ageColSchema = schema.columns.Find(c => c.name.Equals("age")); Assert.IsNotNull(ageColSchema); Assert.AreEqual("age", ageColSchema.name); Assert.IsTrue(ageColSchema.nullable); Assert.AreEqual("long", ageColSchema.type.ToString()); // address var addressColSchema = schema.columns.Find(c => c.name.Equals("address")); Assert.IsNotNull(addressColSchema); Assert.AreEqual("address", addressColSchema.name); Assert.IsTrue(addressColSchema.nullable); Assert.IsNotNull(addressColSchema.type); Assert.AreEqual("struct", addressColSchema.type.type); Assert.IsNotNull(addressColSchema.type.columns.Find(c => c.name.Equals("state"))); Assert.IsNotNull(addressColSchema.type.columns.Find(c => c.name.Equals("city"))); }
private RowSchema GetRowSchema() { if (rowSchema == null) { string json = Schema.ToJson(); rowSchema = RowSchema.ParseRowSchemaFromJson(json); } return rowSchema; }
private List<Row> Collect(int port, RowSchema dataType) { List<Row> items = new List<Row>(); IFormatter formatter = new BinaryFormatter(); Unpickler unpickler = new Unpickler(); Socket sock = new Socket(SocketType.Stream, ProtocolType.Tcp); sock.Connect("127.0.0.1", port); NetworkStream ns = new NetworkStream(sock); using (BinaryReader br = new BinaryReader(ns)) { byte[] buffer = null; do { buffer = br.ReadBytes(4); if (buffer != null && buffer.Length > 0) { //In JVM, Multibyte data items are always stored in big-endian order, where the high bytes come first //http://docs.oracle.com/javase/specs/jvms/se7/html/jvms-4.html //So we should handle revert buffer if our system is little-endian //https://msdn.microsoft.com/en-us/library/system.bitconverter(v=vs.110).aspx if (BitConverter.IsLittleEndian) { Array.Reverse(buffer); } int len = BitConverter.ToInt32(buffer, 0); byte[] data = br.ReadBytes(len); foreach (var item in (unpickler.loads(data) as object[])) { RowImpl row = new RowImpl(item, dataType); items.Add(row); } } } while (buffer != null && buffer.Length > 0); } return items; }