Beispiel #1
0
        public void TestSignaturesV2_3_X()
        {
            Assert.IsType <SparkContext>(_spark.SparkContext);

            Assert.IsType <Builder>(SparkSession.Builder());

            SparkSession.ClearDefaultSession();
            SparkSession.SetDefaultSession(_spark);
            Assert.IsType <SparkSession>(SparkSession.GetDefaultSession());

            Assert.IsType <RuntimeConfig>(_spark.Conf());

            Assert.IsType <SparkSession>(_spark.NewSession());

            Assert.IsType <DataFrameReader>(_spark.Read());

            Assert.IsType <DataFrame>(_spark.Range(10));
            Assert.IsType <DataFrame>(_spark.Range(10, 100));
            Assert.IsType <DataFrame>(_spark.Range(10, 100, 10));
            Assert.IsType <DataFrame>(_spark.Range(10, 100, 10, 5));

            _spark.Range(10).CreateOrReplaceTempView("testView");
            Assert.IsType <DataFrame>(_spark.Table("testView"));

            Assert.IsType <DataStreamReader>(_spark.ReadStream());

            Assert.IsType <UdfRegistration>(_spark.Udf());

            Assert.IsType <Catalog>(_spark.Catalog());
        }
Beispiel #2
0
        /// Tests for the Catclog Functions - returned from SparkSession.Catalog
        public void CatalogFunctions()
        {
            Catalog catalog = _spark.Catalog();

            Assert.IsType <DataFrame>(catalog.ListDatabases());
            Assert.IsType <DataFrame>(catalog.ListFunctions());
            Assert.IsType <DataFrame>(catalog.ListFunctions("default"));

            DataFrame table = catalog.CreateTable("users",
                                                  Path.Combine(TestEnvironment.ResourceDirectory, "users.parquet"));

            Assert.IsType <DataFrame>(table);

            Assert.IsType <string>(catalog.CurrentDatabase());
            Assert.IsType <bool>(catalog.DatabaseExists("default"));

            Assert.IsType <bool>(catalog.DropGlobalTempView("no-view"));
            Assert.IsType <bool>(catalog.DropTempView("no-view"));
            Assert.IsType <bool>(catalog.FunctionExists("default", "functionname"));
            Assert.IsType <bool>(catalog.FunctionExists("functionname"));
            Assert.IsType <Database>(catalog.GetDatabase("default"));
            Assert.IsType <Function>(catalog.GetFunction("abs"));
            Assert.IsType <Function>(catalog.GetFunction(null, "abs"));
            Assert.IsType <Table>(catalog.GetTable("users"));
            Assert.IsType <Table>(catalog.GetTable("default", "users"));
            Assert.IsType <bool>(catalog.IsCached("users"));
            Assert.IsType <DataFrame>(catalog.ListColumns("users"));
            Assert.IsType <DataFrame>(catalog.ListColumns("default", "users"));
            Assert.IsType <DataFrame>(catalog.ListDatabases());
            Assert.IsType <DataFrame>(catalog.ListFunctions());
            Assert.IsType <DataFrame>(catalog.ListFunctions("default"));
            Assert.IsType <DataFrame>(catalog.ListTables());
            Assert.IsType <DataFrame>(catalog.ListTables("default"));

            catalog.RefreshByPath("/");
            catalog.RefreshTable("users");
            catalog.SetCurrentDatabase("default");
            catalog.CacheTable("users");
            catalog.UncacheTable("users");
            catalog.ClearCache();

            Assert.IsType <bool>(catalog.TableExists("users"));
            Assert.IsType <bool>(catalog.TableExists("default", "users"));

            _spark.Sql(@"CREATE TABLE IF NOT EXISTS usersp USING PARQUET PARTITIONED BY (name)  
                            AS SELECT * FROM users");
            catalog.RecoverPartitions("usersp");
        }