/// <summary> /// Process the fetched cells sequential. /// </summary> /// <param name="tableItem"> /// The table item. /// </param> /// <param name="cells"> /// The cells. /// </param> /// <param name="entityFetched"> /// The entity fetched delegate. /// </param> private static void SequentialProcessFetchedCells(KeyValuePair <Pair <string>, ITableScan> tableItem, IEnumerable <Cell> cells, EntityFetched entityFetched) { foreach (var cell in cells) { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(cell.Key, out entityScanTarget)) { var fetchedCell = new FetchedCell(cell, entityScanTarget); entityFetched(ref fetchedCell); } } }
/// <summary> /// Process the fetched cells in parallel. /// </summary> /// <param name="tableItem"> /// The table item. /// </param> /// <param name="cells"> /// The cells. /// </param> /// <param name="entityFetched"> /// The entity fetched delegate. /// </param> private static void ParallelProcessFetchedCells(KeyValuePair <Pair <string>, ITableScan> tableItem, IEnumerable <Cell> cells, EntityFetched entityFetched) { Parallel.ForEach( cells, cell => { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(cell.Key, out entityScanTarget)) { var fetchedCell = new FetchedCell(cell, entityScanTarget); entityFetched(ref fetchedCell); } }); }
/// <summary> /// Fetches all the scan targets. /// </summary> /// <param name="tryGetFetchedEntity"> /// The try Get Fetched Entity. /// </param> /// <param name="entityFetched"> /// The entity fetched delegate. /// </param> internal void Fetch(TryGetFetchedEntity tryGetFetchedEntity, EntityFetched entityFetched) { IEnumerable <KeyValuePair <Pair <string>, ITableScan> > tablesToFetch; lock (this.syncRoot) { tablesToFetch = this.tables; this.tables = null; } foreach (var tableItem in tablesToFetch) { var tableScan = tableItem.Value; foreach (var entityScanTarget in tableScan.EntityScanTargets.ToList()) { object entity; if (tryGetFetchedEntity(entityScanTarget, out entity)) { if (entity == null || entityScanTarget.EntityType.IsAssignableFrom(entity.GetType())) { entityScanTarget.SetValue(entity); } EntityScanTarget removedEntityScanTarget; tableScan.TryRemoveScanTarget(entityScanTarget.Key, out removedEntityScanTarget); } } } tablesToFetch = tablesToFetch.Where(kv => kv.Value.IsEmpty.IsNullOrFalse()).ToList(); var reviewScanSpec = this.entityContext.Configuration.ReviewScanSpec; ////TODO compare async vs sync on multi-core if (this.useAsyncTableScanner) { using (var asynResult = new AsyncResult( (ctx, cells) => { try { var tableItem = (KeyValuePair <Pair <string>, ITableScan>)ctx.Param; ////TODO check what's the fasted way to process the fetched cells on multi-core if (cells.Count > 256) { ParallelProcessFetchedCells(tableItem, cells, entityFetched); } else { SequentialProcessFetchedCells(tableItem, cells, entityFetched); } } catch (AggregateException aggregateException) { foreach (var exception in aggregateException.Flatten().InnerExceptions) { Logging.TraceException(exception); } throw; } catch (Exception exception) { Logging.TraceException(exception); throw; } return(AsyncCallbackResult.Continue); })) { foreach (var tableItem in tablesToFetch) { var table = this.entityContext.GetTable(tableItem.Key.First, tableItem.Key.Second); if (table == null) { throw new PersistenceException( string.Format(CultureInfo.InvariantCulture, @"Table {0}/{1} does not exists", tableItem.Key.First.TrimEnd('/'), tableItem.Key.Second)); } var scanSpec = tableItem.Value.CreateScanSpec(); if (reviewScanSpec != null) { reviewScanSpec(table, scanSpec); } //// TODO add more infos + time, other places??? Logging.TraceEvent( TraceEventType.Verbose, () => string.Format(CultureInfo.InvariantCulture, @"Begin scan {0} on table {1}", scanSpec, table.Name)); table.BeginScan(asynResult, scanSpec, tableItem); } asynResult.Join(); if (asynResult.Error != null) { throw asynResult.Error; } foreach (var tableItem in tablesToFetch) { if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } } } else { foreach (var tableItem in tablesToFetch) { var table = this.entityContext.GetTable(tableItem.Key.First, tableItem.Key.Second); if (table == null) { throw new PersistenceException( string.Format(CultureInfo.InvariantCulture, @"Table {0}/{1} does not exists", tableItem.Key.First.TrimEnd('/'), tableItem.Key.Second)); } var scanSpec = tableItem.Value.CreateScanSpec(); if (reviewScanSpec != null) { reviewScanSpec(table, scanSpec); } //// TODO add/remove more infos + time, other places??? Logging.TraceEvent( TraceEventType.Verbose, () => string.Format(CultureInfo.InvariantCulture, @"Scan {0} on table {1}", scanSpec, table.Name)); using (var scanner = table.CreateScanner(scanSpec)) { Cell cell; while (scanner.Next(out cell)) { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(cell.Key, out entityScanTarget)) { var fetchedCell = new FetchedCell(cell, entityScanTarget); entityFetched(ref fetchedCell); } } if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } } } }
/// <summary> /// Process the fetched cells sequential. /// </summary> /// <param name="tableItem"> /// The table item. /// </param> /// <param name="cells"> /// The cells. /// </param> /// <param name="entityFetched"> /// The entity fetched delegate. /// </param> private static void SequentialProcessFetchedCells(KeyValuePair<Pair<string>, ITableScan> tableItem, IEnumerable<Cell> cells, EntityFetched entityFetched) { foreach (var cell in cells) { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(cell.Key, out entityScanTarget)) { var fetchedCell = new FetchedCell(cell, entityScanTarget); entityFetched(ref fetchedCell); } } }
/// <summary> /// Process the fetched cells in parallel. /// </summary> /// <param name="tableItem"> /// The table item. /// </param> /// <param name="cells"> /// The cells. /// </param> /// <param name="entityFetched"> /// The entity fetched delegate. /// </param> private static void ParallelProcessFetchedCells(KeyValuePair<Pair<string>, ITableScan> tableItem, IEnumerable<Cell> cells, EntityFetched entityFetched) { Parallel.ForEach( cells, cell => { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(cell.Key, out entityScanTarget)) { var fetchedCell = new FetchedCell(cell, entityScanTarget); entityFetched(ref fetchedCell); } }); }
/// <summary> /// Fetches all the scan targets. /// </summary> /// <param name="tryGetFetchedEntity"> /// The try Get Fetched Entity. /// </param> /// <param name="entityFetched"> /// The entity fetched delegate. /// </param> internal void Fetch(TryGetFetchedEntity tryGetFetchedEntity, EntityFetched entityFetched) { IEnumerable<KeyValuePair<Pair<string>, ITableScan>> tablesToFetch; lock (this.syncRoot) { tablesToFetch = this.tables; this.tables = null; } foreach (var tableItem in tablesToFetch) { var tableScan = tableItem.Value; foreach (var entityScanTarget in tableScan.EntityScanTargets.ToList()) { object entity; if (tryGetFetchedEntity(entityScanTarget, out entity)) { if (entity == null || entityScanTarget.EntityType.IsAssignableFrom(entity.GetType())) { entityScanTarget.SetValue(entity); } EntityScanTarget removedEntityScanTarget; tableScan.TryRemoveScanTarget(entityScanTarget.Key, out removedEntityScanTarget); } } } tablesToFetch = tablesToFetch.Where(kv => kv.Value.IsEmpty.IsNullOrFalse()).ToList(); var reviewScanSpec = this.entityContext.Configuration.ReviewScanSpec; ////TODO compare async vs sync on multi-core if (this.useAsyncTableScanner) { using (var asynResult = new AsyncResult( (ctx, cells) => { try { var tableItem = (KeyValuePair<Pair<string>, ITableScan>)ctx.Param; ////TODO check what's the fasted way to process the fetched cells on multi-core if (cells.Count > 256) { ParallelProcessFetchedCells(tableItem, cells, entityFetched); } else { SequentialProcessFetchedCells(tableItem, cells, entityFetched); } } catch (AggregateException aggregateException) { foreach (var exception in aggregateException.Flatten().InnerExceptions) { Logging.TraceException(exception); } throw; } catch (Exception exception) { Logging.TraceException(exception); throw; } return AsyncCallbackResult.Continue; })) { foreach (var tableItem in tablesToFetch) { var table = this.entityContext.GetTable(tableItem.Key.First, tableItem.Key.Second); if (table == null) { throw new PersistenceException( string.Format(CultureInfo.InvariantCulture, @"Table {0}/{1} does not exists", tableItem.Key.First.TrimEnd('/'), tableItem.Key.Second)); } var scanSpec = tableItem.Value.CreateScanSpec(); if (reviewScanSpec != null) { reviewScanSpec(table, scanSpec); } //// TODO add more infos + time, other places??? Logging.TraceEvent( TraceEventType.Verbose, () => string.Format(CultureInfo.InvariantCulture, @"Begin scan {0} on table {1}", scanSpec, table.Name)); table.BeginScan(asynResult, scanSpec, tableItem); } asynResult.Join(); if (asynResult.Error != null) { throw asynResult.Error; } foreach (var tableItem in tablesToFetch) { if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } } } else { foreach (var tableItem in tablesToFetch) { var table = this.entityContext.GetTable(tableItem.Key.First, tableItem.Key.Second); if (table == null) { throw new PersistenceException( string.Format(CultureInfo.InvariantCulture, @"Table {0}/{1} does not exists", tableItem.Key.First.TrimEnd('/'), tableItem.Key.Second)); } var scanSpec = tableItem.Value.CreateScanSpec(); if (reviewScanSpec != null) { reviewScanSpec(table, scanSpec); } //// TODO add/remove more infos + time, other places??? Logging.TraceEvent( TraceEventType.Verbose, () => string.Format(CultureInfo.InvariantCulture, @"Scan {0} on table {1}", scanSpec, table.Name)); using (var scanner = table.CreateScanner(scanSpec)) { Cell cell; while (scanner.Next(out cell)) { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(cell.Key, out entityScanTarget)) { var fetchedCell = new FetchedCell(cell, entityScanTarget); entityFetched(ref fetchedCell); } } if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } } } }
/// <summary> /// The entity fetched callback. /// </summary> /// <param name="fc"> /// The fetched cell. /// </param> private void EntityFetched(ref FetchedCell fc) { this.fetchedCell = fc; var entityScanTarget = fc.EntityScanTarget; var entity = EntityDeserializer.Deserialize(this, typeof(object)/*TODO REMOVE ?? entityScanTarget.EntityType*/, fc.Cell.Value, this.DeserializingEntity); if (!this.behaviors.DoNotCache()) { this.entitySpecsFetched.Add(new EntitySpec(entityScanTarget)); } if (entity == null || entityScanTarget.EntityType.IsAssignableFrom(entity.GetType())) { entityScanTarget.SetValue(entity); } }
/// <summary> /// Fetches all the scan targets. /// </summary> /// <param name="tryGetFetchedEntity"> /// The try Get Fetched Entity. /// </param> /// <param name="entityFetched"> /// The entity fetched delegate. /// </param> internal void Fetch(TryGetFetchedEntity tryGetFetchedEntity, EntityFetched entityFetched) { IEnumerable <KeyValuePair <Pair <string>, ITableScan> > tablesToFetch; lock (this.syncRoot) { tablesToFetch = this.tables; this.tables = null; } foreach (var tableItem in tablesToFetch) { var tableScan = tableItem.Value; foreach (var entityScanTarget in tableScan.EntityScanTargets.ToList()) { object entity; if (tryGetFetchedEntity(entityScanTarget, out entity)) { if (entity == null || entityScanTarget.EntityType.IsAssignableFrom(entity.GetType())) { entityScanTarget.SetValue(entity); } EntityScanTarget removedEntityScanTarget; tableScan.TryRemoveScanTarget(entityScanTarget.Key, out removedEntityScanTarget); } } } tablesToFetch = tablesToFetch.Where(kv => kv.Value.IsEmpty.IsNullOrFalse()).ToList(); var reviewScanSpec = this.entityContext.Configuration.ReviewScanSpec; ////TODO compare async vs sync on multi-core if (this.useAsyncTableScanner) { using (var asynResult = new AsyncResult( (ctx, cells) => { try { var tableItem = (KeyValuePair <Pair <string>, ITableScan>)ctx.Param; ////TODO check what's the fasted way to process the fetched cells on multi-core if (cells.Count > 256) { ParallelProcessFetchedCells(tableItem, cells, entityFetched); } else { SequentialProcessFetchedCells(tableItem, cells, entityFetched); } } catch (AggregateException aggregateException) { foreach (var exception in aggregateException.Flatten().InnerExceptions) { Logging.TraceException(exception); } throw; } catch (Exception exception) { Logging.TraceException(exception); throw; } return(AsyncCallbackResult.Continue); })) { foreach (var tableItem in tablesToFetch) { var table = this.entityContext.GetTable(tableItem.Key.First, tableItem.Key.Second); if (table == null) { throw new PersistenceException( string.Format(CultureInfo.InvariantCulture, @"Table {0}/{1} does not exists", tableItem.Key.First.TrimEnd('/'), tableItem.Key.Second)); } var scanSpec = tableItem.Value.CreateScanSpec(); if (reviewScanSpec != null) { reviewScanSpec(table, scanSpec); } //// TODO add more infos + time, other places??? Logging.TraceEvent( TraceEventType.Verbose, () => string.Format(CultureInfo.InvariantCulture, @"Begin scan {0} on table {1}", scanSpec, table.Name)); table.BeginScan(asynResult, scanSpec, tableItem); } asynResult.Join(); if (asynResult.Error != null) { throw asynResult.Error; } foreach (var tableItem in tablesToFetch) { if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } } } else if (this.useParallelDeserialization) { foreach (var tableItem in tablesToFetch) { var table = this.entityContext.GetTable(tableItem.Key.First, tableItem.Key.Second); if (table == null) { throw new PersistenceException( string.Format(CultureInfo.InvariantCulture, @"Table {0}/{1} does not exists", tableItem.Key.First.TrimEnd('/'), tableItem.Key.Second)); } var scanSpec = tableItem.Value.CreateScanSpec(); if (reviewScanSpec != null) { reviewScanSpec(table, scanSpec); } //// TODO add/remove more infos + time, other places??? Logging.TraceEvent( TraceEventType.Verbose, () => string.Format(CultureInfo.InvariantCulture, @"Scan {0} on table {1}", scanSpec, table.Name)); if (scanSpec.RowCount == 0 || scanSpec.RowCount > 32) { var fetchedCells = new BlockingQueue <FetchedCell>(); Action ProcessFetchedCells = () => { while (true) { var fetchedCell = fetchedCells.Dequeue(); if (fetchedCell == null) { break; } try { entityFetched(fetchedCell, null, IntPtr.Zero, 0, fetchedCell.EntityScanTarget); } finally { PooledCell.Return(fetchedCell.Value); } } }; var tasks = new Task[FetchTaskCount]; for (var i = 0; i < tasks.Length; ++i) { tasks[i] = Task.Run(ProcessFetchedCells); } using (var scanner = table.CreateScanner(scanSpec)) { var fetchedCell = new FetchedCell(); while (scanner.Move(fetchedCell)) { if (tableItem.Value.TryRemoveScanTarget(fetchedCell.Key, out fetchedCell.EntityScanTarget)) { fetchedCells.Enqueue(fetchedCell); fetchedCell = new FetchedCell(); } else { PooledCell.Return(fetchedCell.Value); } } if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } for (var i = 0; i < tasks.Length; ++i) { fetchedCells.Enqueue(null); } Task.WaitAll(tasks); } else if (this.useCellBuffer) { Func <Key, IntPtr, int, bool> callback = (key, buffer, length) => { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(key, out entityScanTarget)) { entityFetched(null, key, buffer, length, entityScanTarget); } return(true); }; using (var scanner = table.CreateScanner(scanSpec)) { while (scanner.Next(callback)) { ; } if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } } else { var bufferedCell = new BufferedCell(0); using (var scanner = table.CreateScanner(scanSpec)) { while (scanner.Move(bufferedCell)) { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(bufferedCell.Key, out entityScanTarget)) { entityFetched(bufferedCell, null, IntPtr.Zero, 0, entityScanTarget); } } if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } } } } else if (this.useCellBuffer) { foreach (var tableItem in tablesToFetch) { var table = this.entityContext.GetTable(tableItem.Key.First, tableItem.Key.Second); if (table == null) { throw new PersistenceException( string.Format(CultureInfo.InvariantCulture, @"Table {0}/{1} does not exists", tableItem.Key.First.TrimEnd('/'), tableItem.Key.Second)); } var scanSpec = tableItem.Value.CreateScanSpec(); if (reviewScanSpec != null) { reviewScanSpec(table, scanSpec); } //// TODO add/remove more infos + time, other places??? Logging.TraceEvent( TraceEventType.Verbose, () => string.Format(CultureInfo.InvariantCulture, @"Scan {0} on table {1}", scanSpec, table.Name)); Func <Key, IntPtr, int, bool> callback = (key, buffer, length) => { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(key, out entityScanTarget)) { entityFetched(null, key, buffer, length, entityScanTarget); } return(true); }; using (var scanner = table.CreateScanner(scanSpec)) { while (scanner.Next(callback)) { ; } if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } } } else { var bufferedCell = new BufferedCell(0); foreach (var tableItem in tablesToFetch) { var table = this.entityContext.GetTable(tableItem.Key.First, tableItem.Key.Second); if (table == null) { throw new PersistenceException( string.Format(CultureInfo.InvariantCulture, @"Table {0}/{1} does not exists", tableItem.Key.First.TrimEnd('/'), tableItem.Key.Second)); } var scanSpec = tableItem.Value.CreateScanSpec(); if (reviewScanSpec != null) { reviewScanSpec(table, scanSpec); } //// TODO add/remove more infos + time, other places??? Logging.TraceEvent( TraceEventType.Verbose, () => string.Format(CultureInfo.InvariantCulture, @"Scan {0} on table {1}", scanSpec, table.Name)); using (var scanner = table.CreateScanner(scanSpec)) { while (scanner.Move(bufferedCell)) { EntityScanTarget entityScanTarget; if (tableItem.Value.TryRemoveScanTarget(bufferedCell.Key, out entityScanTarget)) { entityFetched(bufferedCell, null, IntPtr.Zero, 0, entityScanTarget); } } if (tableItem.Value.IsEmpty.IsFalse()) { //// TODO remaining cells should be deleted } } } } }