Example #1
0
        /// <summary>
        /// Get all parts containing this table
        /// Could be multiple parts, since the table may be spread across multiples files
        /// </summary>
        public IEnumerable <SyncTable> GetTable(string tableName, string schemaName)
        {
            var tableInfo = new BatchPartTableInfo(tableName, schemaName);

            if (InMemory)
            {
                if (this.InMemoryData.HasTables)
                {
                    yield return(this.InMemoryData.Tables[tableName, schemaName]);
                }
            }
            else
            {
                foreach (var batchPartinInfo in this.BatchPartsInfo.OrderBy(bpi => bpi.Index))
                {
                    if (batchPartinInfo.Tables != null && batchPartinInfo.Tables.Any(t => t == tableInfo))
                    {
                        batchPartinInfo.LoadBatchAsync(schema, GetDirectoryFullPath()).ConfigureAwait(false).GetAwaiter().GetResult();

                        // Get the table from the batchPartInfo
                        // generate a tmp SyncTable for
                        var batchTable = batchPartinInfo.Data.Tables.FirstOrDefault(bt => bt == new SyncTable(tableName, schemaName));

                        if (batchTable != null)
                        {
                            yield return(batchTable);

                            // Once read, clear it
                            batchPartinInfo.Data.Clear();
                            batchPartinInfo.Data = null;
                        }
                    }
                }
            }
        }
Example #2
0
        /// <summary>
        /// Check if this batchinfo has some data (in memory or not)
        /// </summary>
        public bool HasData(string tableName, string schemaName)
        {
            if (this.SanitizedSchema == null)
            {
                throw new NullReferenceException("Batch info schema should not be null");
            }

            if (InMemory && InMemoryData != null && InMemoryData.HasTables)
            {
                var table = InMemoryData.Tables[tableName, schemaName];
                if (table == null)
                {
                    return(false);
                }

                return(table.HasRows);
            }

            if (!InMemory && BatchPartsInfo != null && BatchPartsInfo.Count > 0)
            {
                var tableInfo = new BatchPartTableInfo(tableName, schemaName);

                var bptis = BatchPartsInfo.SelectMany(bpi => bpi.Tables.Where(t => t.EqualsByName(tableInfo)));

                if (bptis == null)
                {
                    return(false);
                }


                return(bptis.Sum(bpti => bpti.RowsCount) > 0);
            }

            return(false);
        }
Example #3
0
        public async IAsyncEnumerable <SyncTable> GetTableAsync(string tableName, string schemaName, ISerializerFactory serializerFactory = default, BaseOrchestrator orchestrator = null)
        {
            if (this.SanitizedSchema == null)
            {
                throw new NullReferenceException("Batch info schema should not be null");
            }

            var tableInfo = new BatchPartTableInfo(tableName, schemaName);

            if (InMemory)
            {
                this.SerializerFactoryKey = null;

                if (this.InMemoryData != null && this.InMemoryData.HasTables)
                {
                    yield return(this.InMemoryData.Tables[tableName, schemaName]);
                }
            }
            else
            {
                this.SerializerFactoryKey = serializerFactory.Key;

                var bpiTables = BatchPartsInfo.Where(bpi => bpi.RowsCount > 0 && bpi.Tables.Any(t => t.EqualsByName(tableInfo))).OrderBy(t => t.Index);

                if (bpiTables != null)
                {
                    foreach (var batchPartinInfo in bpiTables)
                    {
                        // load only if not already loaded in memory
                        if (batchPartinInfo.Data == null)
                        {
                            await batchPartinInfo.LoadBatchAsync(this.SanitizedSchema, GetDirectoryFullPath(), serializerFactory, orchestrator).ConfigureAwait(false);
                        }

                        // Get the table from the batchPartInfo
                        // generate a tmp SyncTable for
                        var batchTable = batchPartinInfo.Data.Tables.FirstOrDefault(bt => bt.EqualsByName(new SyncTable(tableName, schemaName)));

                        if (batchTable != null)
                        {
                            yield return(batchTable);

                            // We may need this same BatchPartInfo for another table,
                            // but we dispose it anyway, because memory can be quickly a bottleneck
                            // if batchpartinfos are resident in memory
                            batchPartinInfo.Data.Dispose();
                            batchPartinInfo.Data = null;
                        }
                    }
                }
            }
        }
Example #4
0
        /// <summary>
        /// Get all parts containing this table
        /// Could be multiple parts, since the table may be spread across multiples files
        /// </summary>
        public IEnumerable <SyncTable> GetTable(string tableName, string schemaName, BaseOrchestrator orchestrator = null)
        {
            if (this.SanitizedSchema == null)
            {
                throw new NullReferenceException("Batch info schema should not be null");
            }

            var tableInfo = new BatchPartTableInfo(tableName, schemaName);

            if (InMemory)
            {
                if (this.InMemoryData != null && this.InMemoryData.HasTables)
                {
                    yield return(this.InMemoryData.Tables[tableName, schemaName]);
                }
            }
            else
            {
                foreach (var batchPartinInfo in this.BatchPartsInfo.OrderBy(bpi => bpi.Index))
                {
                    if (batchPartinInfo.Tables != null && batchPartinInfo.Tables.Any(t => t.EqualsByName(tableInfo)))
                    {
                        // TODO : Need to implement IAsyncEnumerable
                        // Need to use GetAwaiter().GetResult() until we have await in foreach yield
                        // IAsyncEnumerable is part of .Net Standard 2.1
                        // But .Net Standard 2.1 is not compatible with .Net Framework 4.8
                        // So far, we can't use it, until we decide to abandon .Net FX 4.8
                        // For sure, it will be replaced when .Net 5 will appears ...
                        batchPartinInfo.LoadBatchAsync(this.SanitizedSchema, GetDirectoryFullPath(), orchestrator).ConfigureAwait(false)
                        .GetAwaiter().GetResult();

                        // Get the table from the batchPartInfo
                        // generate a tmp SyncTable for
                        var batchTable = batchPartinInfo.Data.Tables.FirstOrDefault(bt => bt.EqualsByName(new SyncTable(tableName, schemaName)));

                        if (batchTable != null)
                        {
                            yield return(batchTable);

                            //// Once read, clear it
                            //batchPartinInfo.Data.Clear();
                            //batchPartinInfo.Data = null;
                        }
                    }
                }
            }
        }
Example #5
0
        public async IAsyncEnumerable <SyncTable> GetTableAsync(string tableName, string schemaName, BaseOrchestrator orchestrator = null)
        {
            if (this.SanitizedSchema == null)
            {
                throw new NullReferenceException("Batch info schema should not be null");
            }

            var tableInfo = new BatchPartTableInfo(tableName, schemaName);

            if (InMemory)
            {
                if (this.InMemoryData != null && this.InMemoryData.HasTables)
                {
                    yield return(this.InMemoryData.Tables[tableName, schemaName]);
                }
            }
            else
            {
                var bpiTables = BatchPartsInfo.Where(bpi => bpi.RowsCount > 0 && bpi.Tables.Any(t => t.EqualsByName(tableInfo))).OrderBy(t => t.Index);

                if (bpiTables != null)
                {
                    foreach (var batchPartinInfo in bpiTables)
                    {
                        // load only if not already loaded in memory
                        if (batchPartinInfo.Data == null)
                        {
                            await batchPartinInfo.LoadBatchAsync(this.SanitizedSchema, GetDirectoryFullPath(), orchestrator).ConfigureAwait(false);
                        }

                        // Get the table from the batchPartInfo
                        // generate a tmp SyncTable for
                        var batchTable = batchPartinInfo.Data.Tables.FirstOrDefault(bt => bt.EqualsByName(new SyncTable(tableName, schemaName)));

                        if (batchTable != null)
                        {
                            yield return(batchTable);

                            // once loaded and yield, can dispose
                            batchPartinInfo.Data.Dispose();
                            batchPartinInfo.Data = null;
                        }
                    }
                }
            }
        }
Example #6
0
        public static BatchPartInfo NewBatchPartInfo(string batchPartFileName, string tableName, string schemaName, int rowsCount, int batchIndex)
        {
            var bpi = new BatchPartInfo {
                FileName = batchPartFileName
            };

            // Create the info on the batch part
            BatchPartTableInfo tableInfo = new BatchPartTableInfo
            {
                TableName  = tableName,
                SchemaName = schemaName,
                RowsCount  = rowsCount
            };

            bpi.Tables      = new BatchPartTableInfo[] { tableInfo };
            bpi.RowsCount   = rowsCount;
            bpi.IsLastBatch = false;
            bpi.Index       = batchIndex;
            return(bpi);
        }