Пример #1
0
        public static void XForm(string xformCommand, int expectedExitCode = 0, XDatabaseContext context = null)
        {
            if (context == null)
            {
                context = SampleDatabase.XDatabaseContext;

                // Ensure the as-of DateTime is reset for each operation
                context.RequestedAsOfDateTime = DateTime.MaxValue;
            }

            List <string> args    = new List <string>();
            XqlScanner    scanner = new XqlScanner(xformCommand);

            while (scanner.Current.Type != TokenType.End)
            {
                if (scanner.Current.Type == TokenType.Newline)
                {
                    break;
                }
                args.Add(scanner.Current.Value);
                scanner.Next();
            }

            int result = Program.Run(args.ToArray(), context);

            Assert.AreEqual(expectedExitCode, result, $"Unexpected Exit Code for XForm {xformCommand}");
        }
Пример #2
0
        public void Database_RunAll()
        {
            SampleDatabase.EnsureBuilt();

            // XForm build each source
            foreach (string sourceName in SampleDatabase.XDatabaseContext.Runner.SourceNames)
            {
                XForm($"build {XqlScanner.Escape(sourceName, TokenType.Value)}", ExpectedResult(sourceName));
            }

            // When one fails, put it by itself in the test below to debug
        }
Пример #3
0
        private static string TestSplitAndJoin(string xqlLine)
        {
            XqlScanner scanner = new XqlScanner(xqlLine);

            List <string> parts = new List <string>();

            while (scanner.Current.Type != TokenType.End)
            {
                parts.Add(scanner.Current.Value);
                scanner.Next();
            }

            return(string.Join("|", parts));
        }
Пример #4
0
 public override string ToString()
 {
     if (_defaultValue != null && _changeToDefaultKinds != ValueKinds.InvalidOrNull)
     {
         return($"Cast({_column}, {_targetType.Name}, {_errorOnKinds}, {XqlScanner.Escape(_defaultValue, TokenType.Value)}, {_changeToDefaultKinds})");
     }
     else if (_errorOnKinds != ValueKinds.None)
     {
         return($"Cast({_column}, {_targetType.Name}, {_errorOnKinds})");
     }
     else
     {
         return($"Cast({_column}, {_targetType.Name})");
     }
 }
        public void Current()
        {
            ColumnCache.IsEnabled = true;

            string query;

            // Still want to tune schema requests. Current bottleneck: LatestBeforeCutoff
            //query = @"
            //    read WebRequest
            //    schema
            //";

            //query = @"
            //    read Asset.Extended.Release.Typed
            //    where [Asset_SourceID] = 29 AND [IsBaseline] = 0";

            //query = @"
            //    read Identity.StandingAdmins.AssetAdminSummary.V0
            //    join [Name] Asset.Extended.Release [Name] Asset. ";

            //query = @"
            //    read WebRequestHuge
            //    peek [DataCenter]";

            //query = @"
            //    read WebRequestHuge.Typed
            //    where [EventTime] = null
            //    count";

            query = @"
                read WebRequest
                where [ClientBrowser]: ""Edge""
                choose Max [ResponseBytes] [ClientOS]
                limit 10
            ";

            string singleLineQuery = XqlScanner.QueryToSingleLineStyle(query);

            using (Benchmarker b = new Benchmarker(singleLineQuery, 10000))
            {
                b.Measure($"XForm", 1, () =>
                {
                    return(Context.Query(query).Count());
                });
            }
        }
Пример #6
0
 public override string ToString()
 {
     return(XqlScanner.Escape(Value.ToString(), TokenType.Value, (IsNull || ColumnDetails.Type != typeof(String8))));
 }
Пример #7
0
 public override string ToString()
 {
     return(XqlScanner.Escape((_array.GetValue(0) ?? "null").ToString(), TokenType.Value, (_isNull || ColumnDetails.Type != typeof(String8))));
 }
Пример #8
0
        public IXTable Build(string tableName, XDatabaseContext outerContext, bool deferred)
        {
            // Validate the source name is recognized
            if (!Sources.Contains(tableName))
            {
                // If it wasn't in cache, check individually for it live
                if (!XDatabaseContext.StreamProvider.ContainsTable(tableName))
                {
                    throw new UsageException(tableName, "Table", Sources);
                }

                // If found, update the cache
                UpdateSources();
            }

            // If only a Date was passed for AsOfDate, look for the last version as of that day
            if (outerContext.RequestedAsOfDateTime.TimeOfDay == TimeSpan.Zero)
            {
                outerContext.RequestedAsOfDateTime = outerContext.RequestedAsOfDateTime.AddDays(1).AddSeconds(-1);
            }

            // If we previously found the latest for this table, just return it again
            LatestTableForCutoff previousLatest;

            if (_currentTableVersions.TryGet(tableName, out previousLatest) &&
                previousLatest.Cutoff >= outerContext.RequestedAsOfDateTime &&
                previousLatest.TableVersion.AsOfDate <= outerContext.RequestedAsOfDateTime &&
                TableMetadataSerializer.UncachedExists(outerContext.StreamProvider, previousLatest.TableVersion.Path))
            {
                outerContext.NewestDependency = previousLatest.TableVersion.AsOfDate;
                return(BinaryTableReader.Build(outerContext.StreamProvider, previousLatest.TableVersion.Path));
            }

            // Create a context to track what we're building now
            XDatabaseContext innerContext = XDatabaseContext.Push(outerContext);

            innerContext.CurrentTable = tableName;

            // If this is a query, there won't be a cached table - just build a pipeline to make it
            StreamAttributes queryAttributes = innerContext.StreamProvider.Attributes(innerContext.StreamProvider.Path(LocationType.Query, tableName, ".xql"));

            if (queryAttributes.Exists)
            {
                IXTable queryPipeline = innerContext.Query(innerContext.StreamProvider.ReadAllText(queryAttributes.Path));
                innerContext.Pop(outerContext);
                return(queryPipeline);
            }

            // Find the latest already built result, and associated query
            ItemVersions tableVersions    = innerContext.StreamProvider.ItemVersions(LocationType.Table, tableName);
            ItemVersion  latestTable      = tableVersions.LatestBeforeCutoff(CrawlType.Full, outerContext.RequestedAsOfDateTime);
            string       latestTableQuery = null;

            if (latestTable != null)
            {
                latestTableQuery = TableMetadataSerializer.Read(outerContext.StreamProvider, latestTable.Path).Query;
            }

            // Set the dependency date to the latest table we've already built (if any)
            innerContext.NewestDependency = (latestTable == null ? DateTime.MinValue : latestTable.AsOfDate);

            // Determine the XQL to build the table and construct a builder which can do so
            string  xql;
            IXTable builder;

            // Find the config to build the table and scan dependency versions to determine whether table is out-of-date
            StreamAttributes configAttributes = innerContext.StreamProvider.Attributes(innerContext.StreamProvider.Path(LocationType.Config, tableName, ".xql"));

            if (!configAttributes.Exists)
            {
                // If this is a simple source, just reading it is how to build it
                xql = $"read {XqlScanner.Escape(tableName, TokenType.Value)}";

                // Build a reader concatenating all needed pieces
                builder = ReadSource(tableName, innerContext);
            }
            else
            {
                // If there is a config, the config is how to build it
                xql = innerContext.StreamProvider.ReadAllText(configAttributes.Path);

                // Build a pipeline for the query, recursively creating dependencies
                builder = innerContext.Query(xql);
            }

            // If we don't have the table or the source, we have to throw
            if (latestTable == null && builder == null)
            {
                throw new UsageException(tableName, "Table", innerContext.StreamProvider.Tables());
            }

            // Get the path we're either reading or building
            string tablePath = innerContext.StreamProvider.Path(LocationType.Table, tableName, CrawlType.Full, innerContext.NewestDependency);

            // If we can rebuild this table and we need to (sources rebuilt, query changed, out-of-date, deleted), rebuild it
            if (builder != null)
            {
                if (latestTable == null ||
                    innerContext.RebuiltSomething ||
                    (latestTableQuery != null && xql != latestTableQuery) ||
                    IsOutOfDate(latestTable.AsOfDate, innerContext.NewestDependency) ||
                    !TableMetadataSerializer.UncachedExists(outerContext.StreamProvider, latestTable.Path))
                {
                    // If we're not running now, just return how to build it
                    if (deferred)
                    {
                        return(builder);
                    }

                    // Otherwise, build it now; we'll return the query to read the output
                    innerContext.CurrentQuery = xql;
                    Trace.WriteLine($"COMPUTE: [{innerContext.NewestDependency.ToString(StreamProviderExtensions.DateTimeFolderFormat)}] {tableName}");
                    BinaryTableWriter.Build(builder, innerContext, tablePath).RunAndDispose();
                    innerContext.RebuiltSomething = true;
                }
            }

            // Report the newest dependency in this chain to the components above
            innerContext.Pop(outerContext);

            _currentTableVersions.Add(tableName, new LatestTableForCutoff(outerContext.RequestedAsOfDateTime, new ItemVersion(LocationType.Table, tableName, CrawlType.Full, innerContext.NewestDependency)));
            return(BinaryTableReader.Build(innerContext.StreamProvider, tablePath));
        }
Пример #9
0
 public override string ToString()
 {
     return(XqlScanner.Escape(this.ColumnDetails.Name, TokenType.ColumnName));
 }
Пример #10
0
 public override string ToString()
 {
     return($"{_column} AS {XqlScanner.Escape(ColumnDetails.Name, TokenType.ColumnName)}");
 }