/// <name>Structure</name> /// <type>Constructor</type> /// <summary> /// Creates a new structure object with initialization. /// </summary> /// <param name="type">Type of the structure</param> public Structure(StructType type) : base() { this.type = type; }
public AbstractType TryDeduce(DSymbol ds, IEnumerable<ISemantic> templateArguments) { TemplateTypeParameter tp; var t = ds as TemplateType; if (t == null) return null; var orig = ds.Definition; var tupleStruct = new DClassLike(DTokens.Struct) { NameHash = ds.NameHash, Parent = orig.Parent, Location = orig.Location, EndLocation = orig.EndLocation, NameLocation = orig.NameLocation }; var ded = new Templates.DeducedTypeDictionary(tupleStruct); if (templateArguments != null) { var typeList = new List<AbstractType>(); var en = templateArguments.GetEnumerator(); if(en.MoveNext()) { var next = en.Current; int i = 0; for (; ; i++) { var fieldType = AbstractType.Get(next); if (fieldType == null) break; fieldType.NonStaticAccess = true; typeList.Add(fieldType); if (!en.MoveNext()) break; next = en.Current; if (next is ArrayValue && (next as ArrayValue).IsString) { var name = (next as ArrayValue).StringValue; var templateParamName = "_" + i.ToString(); tp = new TemplateTypeParameter(templateParamName, CodeLocation.Empty, tupleStruct); ded[tp] = new TemplateParameterSymbol(tp, fieldType); tupleStruct.Add(new DVariable { Name = name, Type = new IdentifierDeclaration(templateParamName) }); if (!en.MoveNext()) break; next = en.Current; } } } var tupleName = "Types"; tp = new TemplateTypeParameter(tupleName, CodeLocation.Empty, tupleStruct); ded[tp] = new TemplateParameterSymbol(tp, new DTuple(null, typeList)); tupleStruct.Add(new DVariable { NameHash = DVariable.AliasThisIdentifierHash, IsAlias = true, IsAliasThis = true, Type = new IdentifierDeclaration(tupleName) }); } var res = new StructType(tupleStruct, ds.DeclarationOrExpressionBase, ded.Count != 0 ? ded.Values : null); resultStore.Add(res, tupleStruct); //TODO: Ensure renaming and other AST-based things run properly return res; }
public void Test() { StructType s2 = new StructType(5); s2.Increment(); AssertEqual(new StackFrame(), 6, s2.Value); StructType.StaticValue = 5; StructType.StaticIncrement(); AssertEqual(new StackFrame(), 6, StructType.StaticValue); d = new D(DMethod); d+=new D(DMethod); d-=new D(DMethod); d = ReturnDMethod(d); s2 = d(d(new StructType[]{s2}))[0]; AssertEqual(new StackFrame(), 6, StructType.StaticValue); uint i = this; AssertEqual(new StackFrame(), 1u, i); i = this + 2u; AssertEqual(new StackFrame(), 3u, i); try{ throw new MyException();} catch{} int[] results = DoubleValues(new int[]{1, 2, 3}); AssertEqual(new StackFrame(), 4, results[1]); results = Sums(new int[,]{{1,2}, {3,4}}); AssertEqual(new StackFrame(), 7, results[1]); int[,] otherResults = DoubleValues(new int[,]{{1,2}, {3,4}}); AssertEqual(new StackFrame(), 4, otherResults[0, 1]); AssertEqual(new StackFrame(), Values.One, Echo (Values.One)); int first = 3, second = 4; Swap(ref first, ref second); AssertEqual(new StackFrame(), 3, second); BaseType bt = new ChildType(); AssertEqual(new StackFrame(), 4, bt.Compute(1)); AssertEqual(new StackFrame(), 4, LockingMethod()); try{ LockingThrowingMethod(); AssertTrue(new StackFrame(), false); } catch{} }
/// <summary> /// Specifies the schema by using <see cref="StructType"/>. /// </summary> /// <remarks> /// Some data sources (e.g. JSON) can infer the input schema automatically /// from data. By specifying the schema here, the underlying data source can /// skip the schema inference step, and thus speed up data loading. /// </remarks> /// <param name="schema">The input schema</param> /// <returns>This DataFrameReader object</returns> public DataFrameReader Schema(StructType schema) { Reference.Invoke("schema", DataType.FromJson(Reference.Jvm, schema.Json)); return(this); }
public void TestCreateDataFrame() { // Calling CreateDataFrame with schema { var data = new List <GenericRow> { new GenericRow(new object[] { "Alice", 20, new Date(2020, 1, 1) }), new GenericRow(new object[] { "Bob", 30, new Date(2020, 1, 2) }) }; var schema = new StructType(new List <StructField>() { new StructField("Name", new StringType()), new StructField("Age", new IntegerType()), new StructField("Date", new DateType()) }); DataFrame df = _spark.CreateDataFrame(data, schema); ValidateDataFrame(df, data.Select(a => a.Values), schema); } // Calling CreateDataFrame(IEnumerable<string> _) without schema { var data = new string[] { "Alice", "Bob", null }; StructType schema = SchemaWithSingleColumn(new StringType()); DataFrame df = _spark.CreateDataFrame(data); ValidateDataFrame(df, data.Select(a => new object[] { a }), schema); } // Calling CreateDataFrame(IEnumerable<int> _) without schema { var data = new int[] { 1, 2 }; StructType schema = SchemaWithSingleColumn(new IntegerType(), false); DataFrame df = _spark.CreateDataFrame(data); ValidateDataFrame(df, data.Select(a => new object[] { a }), schema); } // Calling CreateDataFrame(IEnumerable<int?> _) without schema { var data = new int?[] { 1, 2, null }; StructType schema = SchemaWithSingleColumn(new IntegerType()); DataFrame df = _spark.CreateDataFrame(data); ValidateDataFrame(df, data.Select(a => new object[] { a }), schema); } // Calling CreateDataFrame(IEnumerable<double> _) without schema { var data = new double[] { 1.2, 2.3 }; StructType schema = SchemaWithSingleColumn(new DoubleType(), false); DataFrame df = _spark.CreateDataFrame(data); ValidateDataFrame(df, data.Select(a => new object[] { a }), schema); } // Calling CreateDataFrame(IEnumerable<double?> _) without schema { var data = new double?[] { 1.2, 2.3, null }; StructType schema = SchemaWithSingleColumn(new DoubleType()); DataFrame df = _spark.CreateDataFrame(data); ValidateDataFrame(df, data.Select(a => new object[] { a }), schema); } // Calling CreateDataFrame(IEnumerable<bool> _) without schema { var data = new bool[] { true, false }; StructType schema = SchemaWithSingleColumn(new BooleanType(), false); DataFrame df = _spark.CreateDataFrame(data); ValidateDataFrame(df, data.Select(a => new object[] { a }), schema); } // Calling CreateDataFrame(IEnumerable<bool?> _) without schema { var data = new bool?[] { true, false, null }; StructType schema = SchemaWithSingleColumn(new BooleanType()); DataFrame df = _spark.CreateDataFrame(data); ValidateDataFrame(df, data.Select(a => new object[] { a }), schema); } // Calling CreateDataFrame(IEnumerable<Date> _) without schema { var data = new Date[] { new Date(2020, 1, 1), new Date(2020, 1, 2), null }; StructType schema = SchemaWithSingleColumn(new DateType()); DataFrame df = _spark.CreateDataFrame(data); ValidateDataFrame(df, data.Select(a => new object[] { a }), schema); } }
/// <summary> /// Write instruction operands into bytecode stream. /// </summary> /// <param name="writer">Bytecode writer.</param> public override void WriteOperands(WordWriter writer) { StructType.Write(writer); Member.Write(writer); Decoration.Write(writer); }
public ISymbolValue VisitStructType(StructType t) { return(new TypeValue(t)); }
public void Run(string[] args) { string kafkaBrokers = args[0]; double maxSpeed = double.Parse(args[1]); // Obtém a referência ao contexto de execução do Spark SparkSession spark = SparkSession .Builder() .AppName("Credit Card Fraud") .GetOrCreate(); spark.Conf().Set("spark.sql.shuffle.partitions", "1"); // sem essa configuração, cada stage ficou com 200 tasks, o que levou uns 4 minutos pra cada batch executar // Criando um dataframe pra receber dados do Kafka DataFrame df = spark .ReadStream() .Format("kafka") .Option("kafka.bootstrap.servers", kafkaBrokers) .Option("subscribe", "transactions") .Load() .SelectExpr("CAST(value AS STRING)"); /* Criando schema pra validar o JSON que virá nas mensagens do Kafka * Exemplo do JSON: * { * "transaction":"431", * "number":"0015-0000-0000-0000", * "lat":-23.1618, * "lng":-46.47201, * "amount":91.01487, * "category":"pets", * "eventTime":"2021-01-05T19:07:19.3888" * } */ var schema = new StructType(new[] { new StructField("transaction", new StringType()), new StructField("number", new StringType()), new StructField("lat", new DoubleType()), new StructField("lng", new DoubleType()), new StructField("amount", new DoubleType()), new StructField("category", new StringType()), new StructField("eventTime", new TimestampType()) }); // Fazendo o parse do JSON pra um array ... df = df.WithColumn("json", FromJson( df.Col("value"), schema.SimpleString) ) .Select("json.*"); // ... e retornando todas as colunas do array como um novo dataframe // Gerando dois dataframes distintos para poder fazer o join e analisar a correção entre as transações DataFrame df1 = df .WithWatermark("eventTime", "7 minutes"); DataFrame df2 = df .WithColumnRenamed("transaction", "transaction2") .WithColumnRenamed("lat", "lat2") .WithColumnRenamed("lng", "lng2") .WithColumnRenamed("eventTime", "eventTime2") .WithWatermark("eventTime2", "7 minutes"); // Efetuando o join para verificar a correlação de transações dos cartões de crédito DataFrame dfJoin = df1.Join(df2, df1.Col("number").EqualTo(df2.Col("number")) .And(Col("transaction").NotEqual(Col("transaction2"))) .And(Col("eventTime2").Between(Col("eventTime"), Col("eventTime") + Expr("interval 5 minutes"))) ); //Registrando uma função personalizada pra ser usada no dataframe spark.Udf().Register <double, double, double, double, double>("CalculateDistance", (lat1, lng1, lat2, lng2) => CalculateDistance(lat1, lng1, lat2, lng2)); spark.Udf().Register <double, Timestamp, Timestamp, double>("CalculateSpeed", (dist, eventTime, eventTime2) => CalculateSpeed(dist, eventTime, eventTime2)); // Criando novas colunas para armazenar a execução do código da UDF dfJoin = dfJoin.WithColumn("dist", CallUDF("CalculateDistance", Col("lat"), Col("lng"), Col("lat2"), Col("lng2"))); dfJoin = dfJoin.WithColumn("speed", CallUDF("CalculateSpeed", Col("dist"), Col("eventTime"), Col("eventTime2"))); // Filtrando as transações que tiverem a velocidade acima do esperado (parâmetro "maxSpeed") dfJoin = dfJoin.Where(Col("speed").Gt(maxSpeed)); // Colocando o streaming pra funcionar StreamingQuery query = dfJoin .WriteStream() .Format("console") .Option("truncate", "false") .OutputMode(OutputMode.Append) .Start(); query.AwaitTermination(); }
private CheckApplicability IsCheckApplicableToData(Check check, StructType schema, SparkSession sparkSession) => new Applicability(sparkSession).IsApplicable(check, schema);
private AnalyzersApplicability AreCheckApplicableToData(IEnumerable <IAnalyzer <IMetric> > analyzers, StructType schema, SparkSession sparkSession) => new Applicability(sparkSession).IsApplicable(analyzers, schema);
public void TestComplex() { /* * def kju() : Unit { * struct P { * x : [Int]; * }; * * struct S { * x : Int; * y : P; * }; * * var s : [S] = new(S, 10); * var arr : [[Bool]] = new([Bool], 20); * } */ var dummyRange = new Core.Lexer.Range(new StringLocation(0), new StringLocation(0)); var pFields = new List <StructField>() { new StructField(dummyRange, "x", new ArrayType(IntType.Instance)) }; var pDeclaration = new StructDeclaration(dummyRange, "P", pFields); var pType = StructType.GetInstance(pDeclaration); var sFields = new List <StructField>() { new StructField(dummyRange, "x", IntType.Instance), new StructField(dummyRange, "y", pType) }; var sDeclaration = new StructDeclaration(dummyRange, "S", sFields); var sType = StructType.GetInstance(sDeclaration); var sAlloc = new ArrayAlloc( dummyRange, sType, new IntegerLiteral(dummyRange, 10)); var sVarDeclaration = new VariableDeclaration( dummyRange, new ArrayType(sType), "s", sAlloc); var arrAlloc = new ArrayAlloc( dummyRange, new ArrayType(BoolType.Instance), new IntegerLiteral(dummyRange, 20)); var arrVarDeclaration = new VariableDeclaration( dummyRange, new ArrayType(new ArrayType(BoolType.Instance)), "arr", arrAlloc); var kjuInstructions = new List <Expression> { pDeclaration, sDeclaration, sVarDeclaration, arrVarDeclaration }; var kjuDeclaration = new FunctionDeclaration( dummyRange, "kju", new ArrayType(UnitType.Instance), new List <VariableDeclaration>(), new InstructionBlock(dummyRange, kjuInstructions), false); var root = new Program(dummyRange, new List <StructDeclaration>(), new List <FunctionDeclaration> { kjuDeclaration }); var expectedTypes = new HashSet <DataType>() { pType, sType, new ArrayType(BoolType.Instance), new ArrayType(IntType.Instance), new ArrayType(sType), new ArrayType(new ArrayType(BoolType.Instance)) }; this.CheckAnswer(root, expectedTypes); }
public Context Enter(StructType type) { return(new Context(Level + 1, type)); }
public Context(int level, StructType current) { Level = level; Current = current; }
public StructDecoder(StructType structType) { _names = structType.Fields.Select(x => x.Name).ToArray(); _decoders = SchemaToDecoder.FieldsToDecoders(structType.Fields); }
/// <name>Structure</name> /// <type>Constructor</type> /// <summary> /// Creates a new structure object with initialization. /// </summary> /// <param name="type">Type of the structure</param> public Structure(StructType type) : base() { this.type = type; }
public void CreateClass(AstClass astClass, Namescope parent, IEnumerable <AstBlockMember> parentItems) { var sources = new List <string>(); if (astClass.Modifiers.HasFlag(Modifiers.Partial)) { astClass = FlattenClass(astClass, sources, parentItems); } DataType result; if (_cachedClasses.TryGetValue(astClass, out result)) { return; } var src = astClass.Name.Source; var modifiers = GetTypeModifiers(parent, astClass.Modifiers); switch (astClass.Type) { case AstClassType.Struct: result = new StructType(src, parent, astClass.DocComment, modifiers, astClass.Name.Symbol); break; case AstClassType.Class: result = new ClassType(src, parent, astClass.DocComment, modifiers, astClass.Name.Symbol); break; case AstClassType.Interface: if (modifiers.HasFlag(Modifiers.Abstract)) { Log.Error(src, ErrorCode.E0000, "'abstract' is not valid for interface"); } result = new InterfaceType(src, parent, astClass.DocComment, modifiers | Modifiers.Abstract, astClass.Name.Symbol); break; default: Log.Error(src, ErrorCode.I3045, "<" + astClass.Type + "> is not a class, struct or interface type"); return; } if (parent is DataType) { (parent as DataType).NestedTypes.Add(result); } else if (parent is Namespace) { (parent as Namespace).Types.Add(result); } else { Log.Error(result.Source, ErrorCode.I3046, "<" + astClass.Type + "> is not allowed in this context"); } _cachedClasses.Add(astClass, result); if (astClass.OptionalGeneric != null) { CreateGenericSignature(result, astClass.OptionalGeneric, false); } result.SetBlock(new Block(src, result, null, result.Modifiers & Modifiers.ProtectionModifiers, ".block")); foreach (var s in sources) { result.SourceFiles.Add(s); } foreach (var b in astClass.Members) { if (b is AstBlockBase) { _compiler.AstProcessor.CreateBlock(b as AstBlockBase, result, astClass.Members); } } if (astClass.Attributes.Count > 0) { EnqueueAttributes(result, x => { result.SetAttributes(_compiler.CompileAttributes(result.Parent, astClass.Attributes)); // Remove default constructor if TargetSpecificType if (result.HasAttribute(_ilf.Essentials.TargetSpecificTypeAttribute) && result.Constructors.Count == 1 && result.Constructors[0].IsGenerated) { result.Constructors.Clear(); } }); } EnqueueType(result, x => CompileBaseTypes(x, astClass.Bases), x => PopulateClass(astClass, x)); }
/// <summary> /// Check transform validity and derive the output schema from the input schema. /// /// This checks for validity of interactions between parameters during Transform and /// raises an exception if any parameter value is invalid. /// /// Typical implementation should first conduct verification on schema change and parameter /// validity, including complex parameter interaction checks. /// </summary> /// <param name="value"> /// The <see cref="StructType"/> of the <see cref="DataFrame"/> which will be transformed. /// </param> /// <returns> /// The <see cref="StructType"/> of the output schema that would have been derived from the /// input schema, if Transform had been called. /// </returns> public override StructType TransformSchema(StructType value) => new StructType( (JvmObjectReference)Reference.Invoke( "transformSchema", DataType.FromJson(Reference.Jvm, value.Json)));
/// <summary> /// Specifies the schema by using <see cref="StructType"/>. /// </summary> /// <remarks> /// Some data sources (e.g. JSON) can infer the input schema automatically /// from data. By specifying the schema here, the underlying data source can /// skip the schema inference step, and thus speed up data loading. /// </remarks> /// <param name="schema">The input schema</param> /// <returns>This DataFrameReader object</returns> public DataFrameReader Schema(StructType schema) { _jvmObject.Invoke("schema", DataType.FromJson(_jvmObject.Jvm, schema.Json)); return(this); }
/// <summary> /// Create an external table from the given path based on a data source, a schema and /// a set of options.Then, returns the corresponding DataFrame. /// </summary> /// <param name="tableName">Name of the table</param> /// <param name="source">Data source</param> /// <param name="schema">Schema of the table</param> /// <param name="options">Options to create table</param> /// <returns></returns> public DataFrame CreateExternalTable(string tableName, string source, StructType schema, Dictionary<string, string> options) { return catalogProxy.CreateExternalTable(tableName, source, schema, options); }
public void Initialize(bool canBeEdit, AgentType agent, StructType structType, PropertyDef prop, bool canBePar) { Debug.Check(agent != null || structType != null); _initialized = false; _isModified = false; _shouldCheckMembersInWorkspace = false; _isNew = (prop == null); _agent = agent; _structType = structType; _originalProperty = prop; setTypes(); if (_isNew) { this.Text = canBeEdit ? Resources.AddProperty : Resources.ViewProperty; if (_structType == null) { if (agent != null) { _property = new PropertyDef(agent, null, agent.Name, "", "", ""); } } else { _property = new PropertyDef(null, null, _structType.Name, "", "", ""); } _property.IsPublic = false; resetProperty(_property, _property.IsPar); } else { this.Text = canBeEdit ? Resources.EditProperty : Resources.ViewProperty; resetProperty(prop, prop.IsPar); } //this.customizedCheckBox.Visible = canBeEdit && !_property.IsInherited && agent != null; this.customizedCheckBox.Visible = false; this.isLocalCheckBox.Checked = _structType == null && _property.IsPar; this.isLocalCheckBox.Visible = canBePar && _structType == null && !_property.IsMember; this.isLocalCheckBox.Enabled = canBeEdit; this.nameTextBox.Enabled = canBeEdit; this.arrayCheckBox.Enabled = canBeEdit || (_structType == null || _structType.IsCustomized) && _property.IsChangeableType; this.typeComboBox.Enabled = canBeEdit || (_structType == null || _structType.IsCustomized) && _property.IsChangeableType; this.isStaticCheckBox.Enabled = canBeEdit; this.isPublicCheckBox.Enabled = canBeEdit; this.isConstCheckBox.Enabled = canBeEdit; this.dispTextBox.Enabled = canBeEdit; this.descTextBox.Enabled = canBeEdit; this.nameTextBox.Focus(); if (this.nameTextBox.TextLength > 0) { this.nameTextBox.SelectionStart = this.nameTextBox.TextLength; } else { this.nameTextBox.Select(); } _initialized = true; }
/// <summary> /// Creates a <see cref="DataFrame"/> from an <see cref="IEnumerable"/> containing /// <see cref="GenericRow"/>s using the given schema. /// It is important to make sure that the structure of every <see cref="GenericRow"/> of /// the provided <see cref="IEnumerable"/> matches /// the provided schema. Otherwise, there will be runtime exception. /// </summary> /// <param name="data">List of Row objects</param> /// <param name="schema">Schema as StructType</param> /// <returns>DataFrame object</returns> public DataFrame CreateDataFrame(IEnumerable <GenericRow> data, StructType schema) => new DataFrame((JvmObjectReference)_jvmObject.Invoke( "createDataFrame", data, DataType.FromJson(_jvmObject.Jvm, schema.Json)));
public void TestSignaturesV2_3_X() { Assert.IsType <Column>(_df["name"]); Assert.IsType <Column>(_df["age"]); Assert.IsType <DataFrame>(_df.ToDF()); Assert.IsType <DataFrame>(_df.ToDF("name2", "age2")); StructType schema = _df.Schema(); Assert.NotNull(schema); _df.PrintSchema(); _df.Explain(); _df.Explain(true); _df.Explain(false); Assert.Equal(2, _df.Columns().ToArray().Length); var expected = new List <Tuple <string, string> > { new Tuple <string, string>("age", "integer"), new Tuple <string, string>("name", "string") }; Assert.Equal(expected, _df.DTypes()); Assert.IsType <bool>(_df.IsLocal()); Assert.IsType <bool>(_df.IsStreaming()); using (var tempDir = new TemporaryDirectory()) { // The following is required for *CheckPoint(). _spark.SparkContext.SetCheckpointDir(tempDir.Path); Assert.IsType <DataFrame>(_df.Checkpoint()); Assert.IsType <DataFrame>(_df.Checkpoint(false)); Assert.IsType <DataFrame>(_df.LocalCheckpoint()); Assert.IsType <DataFrame>(_df.LocalCheckpoint(false)); } Assert.IsType <DataFrame>(_df.WithWatermark("time", "10 minutes")); _df.Show(); _df.Show(10); _df.Show(10, 10); _df.Show(10, 10, true); Assert.IsType <DataFrame>(_df.Join(_df)); Assert.IsType <DataFrame>(_df.Join(_df, "name")); Assert.IsType <DataFrame>(_df.Join(_df, new[] { "name" })); Assert.IsType <DataFrame>(_df.Join(_df, new[] { "name" }, "outer")); Assert.IsType <DataFrame>(_df.Join(_df, _df["age"] == _df["age"])); Assert.IsType <DataFrame>(_df.Join(_df, _df["age"] == _df["age"], "outer")); Assert.IsType <DataFrame>(_df.CrossJoin(_df)); Assert.IsType <DataFrame>(_df.SortWithinPartitions("age")); Assert.IsType <DataFrame>(_df.SortWithinPartitions("age", "name")); Assert.IsType <DataFrame>(_df.SortWithinPartitions()); Assert.IsType <DataFrame>(_df.SortWithinPartitions(_df["age"])); Assert.IsType <DataFrame>(_df.SortWithinPartitions(_df["age"], _df["name"])); Assert.IsType <DataFrame>(_df.Sort("age")); Assert.IsType <DataFrame>(_df.Sort("age", "name")); Assert.IsType <DataFrame>(_df.Sort()); Assert.IsType <DataFrame>(_df.Sort(_df["age"])); Assert.IsType <DataFrame>(_df.Sort(_df["age"], _df["name"])); Assert.IsType <DataFrame>(_df.OrderBy("age")); Assert.IsType <DataFrame>(_df.OrderBy("age", "name")); Assert.IsType <DataFrame>(_df.OrderBy()); Assert.IsType <DataFrame>(_df.OrderBy(_df["age"])); Assert.IsType <DataFrame>(_df.OrderBy(_df["age"], _df["name"])); Assert.IsType <DataFrame>(_df.Hint("broadcast")); Assert.IsType <DataFrame>(_df.Hint("broadcast", new[] { "hello", "world" })); Assert.IsType <Column>(_df.Col("age")); Assert.IsType <Column>(_df.ColRegex("age")); Assert.IsType <DataFrame>(_df.As("alias")); Assert.IsType <DataFrame>(_df.Alias("alias")); Assert.IsType <DataFrame>(_df.Select("age")); Assert.IsType <DataFrame>(_df.Select("age", "name")); Assert.IsType <DataFrame>(_df.Select()); Assert.IsType <DataFrame>(_df.Select(_df["age"])); Assert.IsType <DataFrame>(_df.Select(_df["age"], _df["name"])); Assert.IsType <DataFrame>(_df.SelectExpr()); Assert.IsType <DataFrame>(_df.SelectExpr("age * 2")); Assert.IsType <DataFrame>(_df.SelectExpr("age * 2", "abs(age)")); Assert.IsType <DataFrame>(_df.Filter(_df["age"] > 21)); Assert.IsType <DataFrame>(_df.Filter("age > 21")); Assert.IsType <DataFrame>(_df.Where(_df["age"] > 21)); Assert.IsType <DataFrame>(_df.Where("age > 21")); Assert.IsType <RelationalGroupedDataset>(_df.GroupBy("age")); Assert.IsType <RelationalGroupedDataset>(_df.GroupBy("age", "name")); Assert.IsType <RelationalGroupedDataset>(_df.GroupBy()); Assert.IsType <RelationalGroupedDataset>(_df.GroupBy(_df["age"])); Assert.IsType <RelationalGroupedDataset>(_df.GroupBy(_df["age"], _df["name"])); { RelationalGroupedDataset df = _df.WithColumn("tempAge", _df["age"]).GroupBy("name"); Assert.IsType <DataFrame>(df.Mean("age")); Assert.IsType <DataFrame>(df.Mean("age", "tempAge")); Assert.IsType <DataFrame>(df.Max("age")); Assert.IsType <DataFrame>(df.Max("age", "tempAge")); Assert.IsType <DataFrame>(df.Avg("age")); Assert.IsType <DataFrame>(df.Avg("age", "tempAge")); Assert.IsType <DataFrame>(df.Min("age")); Assert.IsType <DataFrame>(df.Min("age", "tempAge")); Assert.IsType <DataFrame>(df.Sum("age")); Assert.IsType <DataFrame>(df.Sum("age", "tempAge")); } Assert.IsType <RelationalGroupedDataset>(_df.Rollup("age")); Assert.IsType <RelationalGroupedDataset>(_df.Rollup("age", "name")); Assert.IsType <RelationalGroupedDataset>(_df.Rollup()); Assert.IsType <RelationalGroupedDataset>(_df.Rollup(_df["age"])); Assert.IsType <RelationalGroupedDataset>(_df.Rollup(_df["age"], _df["name"])); Assert.IsType <RelationalGroupedDataset>(_df.Cube("age")); Assert.IsType <RelationalGroupedDataset>(_df.Cube("age", "name")); Assert.IsType <RelationalGroupedDataset>(_df.Cube()); Assert.IsType <RelationalGroupedDataset>(_df.Cube(_df["age"])); Assert.IsType <RelationalGroupedDataset>(_df.Cube(_df["age"], _df["name"])); Assert.IsType <DataFrame>(_df.Agg(Avg(_df["age"]))); Assert.IsType <DataFrame>(_df.Agg(Avg(_df["age"]), Avg(_df["name"]))); Assert.IsType <DataFrame>(_df.Limit(10)); Assert.IsType <DataFrame>(_df.Union(_df)); Assert.IsType <DataFrame>(_df.UnionByName(_df)); Assert.IsType <DataFrame>(_df.Intersect(_df)); Assert.IsType <DataFrame>(_df.Except(_df)); Assert.IsType <DataFrame>(_df.Sample(0.5)); Assert.IsType <DataFrame>(_df.Sample(0.5, true)); Assert.IsType <DataFrame>(_df.Sample(0.5, false, 12345)); Assert.IsType <DataFrame[]>(_df.RandomSplit(new[] { 0.2, 0.8 })); Assert.IsType <DataFrame[]>(_df.RandomSplit(new[] { 0.2, 0.8 }, 12345)); Assert.IsType <DataFrame>(_df.WithColumn("age2", _df["age"])); Assert.IsType <DataFrame>(_df.WithColumnRenamed("age", "age2")); Assert.IsType <DataFrame>(_df.Drop()); Assert.IsType <DataFrame>(_df.Drop("age")); Assert.IsType <DataFrame>(_df.Drop("age", "name")); Assert.IsType <DataFrame>(_df.Drop(_df["age"])); Assert.IsType <DataFrame>(_df.DropDuplicates()); Assert.IsType <DataFrame>(_df.DropDuplicates("age")); Assert.IsType <DataFrame>(_df.DropDuplicates("age", "name")); Assert.IsType <DataFrame>(_df.Describe()); Assert.IsType <DataFrame>(_df.Describe("age")); Assert.IsType <DataFrame>(_df.Describe("age", "name")); Assert.IsType <DataFrame>(_df.Summary()); Assert.IsType <DataFrame>(_df.Summary("count")); Assert.IsType <DataFrame>(_df.Summary("count", "mean")); Assert.IsType <Row[]>(_df.Head(2).ToArray()); Assert.IsType <Row>(_df.Head()); Assert.IsType <Row>(_df.First()); Assert.IsType <Row[]>(_df.Take(3).ToArray()); Assert.IsType <Row[]>(_df.Collect().ToArray()); Assert.IsType <Row[]>(_df.ToLocalIterator().ToArray()); Assert.IsType <long>(_df.Count()); Assert.IsType <DataFrame>(_df.Repartition(2)); Assert.IsType <DataFrame>(_df.Repartition(2, _df["age"])); Assert.IsType <DataFrame>(_df.Repartition(_df["age"])); Assert.IsType <DataFrame>(_df.Repartition()); Assert.IsType <DataFrame>(_df.RepartitionByRange(2, _df["age"])); Assert.IsType <DataFrame>(_df.RepartitionByRange(_df["age"])); Assert.IsType <DataFrame>(_df.Coalesce(1)); Assert.IsType <DataFrame>(_df.Distinct()); Assert.IsType <DataFrame>(_df.Persist()); Assert.IsType <DataFrame>(_df.Persist(StorageLevel.DISK_ONLY)); Assert.IsType <DataFrame>(_df.Cache()); Assert.IsType <StorageLevel>(_df.StorageLevel()); Assert.IsType <DataFrame>(_df.Unpersist()); _df.CreateTempView("view"); _df.CreateOrReplaceTempView("view"); _df.CreateGlobalTempView("global_view"); _df.CreateOrReplaceGlobalTempView("global_view"); }
public IDataFrameProxy ReadDataFrame(string path, StructType schema, System.Collections.Generic.Dictionary <string, string> options) { throw new NotImplementedException(); }
public StructType[] DMethod(StructType[] s) { return s; }
public IDataFrameProxy TextFile(string path, StructType schema, string delimiter) { return(new MockDataFrameProxy(new object[] { path, schema, delimiter }, this)); }
public LanguageStructModel(StructType intIDType, LanguageStructModel objParent, CompilationUnitModel objCompilationUnit = null) { IDType = intIDType; Parent = objParent; CompilationUnit = objCompilationUnit; }
public void TestSignaturesV2_3_X() { Column col = _df["name"]; col = _df["age"]; DataFrame df = _df.ToDF(); df = df.ToDF("name2", "age2"); StructType schema = _df.Schema(); Assert.NotNull(schema); _df.PrintSchema(); _df.Explain(); _df.Explain(true); _df.Explain(false); Assert.Equal(2, _df.Columns().ToArray().Length); _df.IsLocal(); _df.IsStreaming(); using (var tempDir = new TemporaryDirectory()) { // The following is required for *CheckPoint(). _spark.SparkContext.SetCheckpointDir(tempDir.Path); _df.Checkpoint(); _df.Checkpoint(false); _df.LocalCheckpoint(); _df.LocalCheckpoint(false); } _df.WithWatermark("time", "10 minutes"); _df.Show(); _df.Show(10); _df.Show(10, 10); _df.Show(10, 10, true); _df.Join(_df); _df.Join(_df, "name"); _df.Join(_df, new[] { "name" }); _df.Join(_df, new[] { "name" }, "outer"); _df.Join(_df, _df["age"] == _df["age"]); _df.Join(_df, _df["age"] == _df["age"], "outer"); _df.CrossJoin(_df); _df.SortWithinPartitions("age"); _df.SortWithinPartitions("age", "name"); _df.SortWithinPartitions(); _df.SortWithinPartitions(_df["age"]); _df.SortWithinPartitions(_df["age"], _df["name"]); _df.Sort("age"); _df.Sort("age", "name"); _df.Sort(); _df.Sort(_df["age"]); _df.Sort(_df["age"], _df["name"]); _df.OrderBy("age"); _df.OrderBy("age", "name"); _df.OrderBy(); _df.OrderBy(_df["age"]); _df.OrderBy(_df["age"], _df["name"]); _df.Hint("broadcast"); _df.Hint("broadcast", new[] { "hello", "world" }); _df.Col("age"); _df.ColRegex("age"); _df.As("alias"); _df.Alias("alias"); _df.Select("age"); _df.Select("age", "name"); _df.Select(); _df.Select(_df["age"]); _df.Select(_df["age"], _df["name"]); _df.SelectExpr(); _df.SelectExpr("age * 2"); _df.SelectExpr("age * 2", "abs(age)"); _df.Filter(_df["age"] > 21); _df.Filter("age > 21"); _df.Where(_df["age"] > 21); _df.Where("age > 21"); _df.GroupBy("age"); _df.GroupBy("age", "name"); _df.GroupBy(); _df.GroupBy(_df["age"]); _df.GroupBy(_df["age"], _df["name"]); _df.Rollup("age"); _df.Rollup("age", "name"); _df.Rollup(); _df.Rollup(_df["age"]); _df.Rollup(_df["age"], _df["name"]); _df.Cube("age"); _df.Cube("age", "name"); _df.Cube(); _df.Cube(_df["age"]); _df.Cube(_df["age"], _df["name"]); _df.Agg(Avg(_df["age"])); _df.Agg(Avg(_df["age"]), Avg(_df["name"])); _df.Limit(10); _df.Union(_df); _df.UnionByName(_df); _df.Intersect(_df); _df.Except(_df); _df.Sample(0.5); _df.Sample(0.5, true); _df.Sample(0.5, false, 12345); _df.RandomSplit(new[] { 0.2, 0.8 }); _df.RandomSplit(new[] { 0.2, 0.8 }, 12345); _df.WithColumn("age2", _df["age"]); _df.WithColumnRenamed("age", "age2"); _df.Drop(); _df.Drop("age"); _df.Drop("age", "name"); _df.Drop(_df["age"]); _df.DropDuplicates(); _df.DropDuplicates("age"); _df.DropDuplicates("age", "name"); _df.Describe(); _df.Describe("age"); _df.Describe("age", "name"); _df.Summary(); _df.Summary("count"); _df.Summary("count", "mean"); _df.Head(2); _df.Head(); _df.First(); _df.Take(3).ToArray(); _df.Collect().ToArray(); _df.ToLocalIterator().ToArray(); _df.Count(); _df.Repartition(2); _df.Repartition(2, _df["age"]); _df.Repartition(_df["age"]); _df.Repartition(); _df.RepartitionByRange(2, _df["age"]); _df.RepartitionByRange(_df["age"]); _df.Coalesce(1); _df.Distinct(); _df.Persist(); _df.Cache(); _df.Unpersist(); _df.CreateTempView("view"); _df.CreateOrReplaceTempView("view"); _df.CreateGlobalTempView("global_view"); _df.CreateOrReplaceGlobalTempView("global_view"); }
/// <summary> /// The variable's or method's base type will be resolved (if auto type, the intializer's type will be taken). /// A class' base class will be searched. /// etc.. /// </summary> public static AbstractType HandleNodeMatch( INode m, ResolverContextStack ctxt, AbstractType resultBase = null, object typeBase = null) { stackNum_HandleNodeMatch++; bool popAfterwards = m.Parent != ctxt.ScopedBlock && m.Parent is IBlockNode; if (popAfterwards) ctxt.PushNewScope((IBlockNode)m.Parent); //HACK: Really dirty stack overflow prevention via manually counting call depth var canResolveBaseGenerally = stackNum_HandleNodeMatch < 6; var DoResolveBaseType = canResolveBaseGenerally && !ctxt.Options.HasFlag(ResolutionOptions.DontResolveBaseClasses) && (m.Type == null || m.Type.ToString(false) != m.Name); AbstractType ret = null; // To support resolving type parameters to concrete types if the context allows this, introduce all deduced parameters to the current context if (canResolveBaseGenerally && resultBase is DSymbol) ctxt.CurrentContext.IntroduceTemplateParameterTypes((DSymbol)resultBase); // Only import symbol aliases are allowed to search in the parse cache if (m is ImportSymbolAlias) { var isa = (ImportSymbolAlias)m; if (isa.IsModuleAlias ? isa.Type != null : isa.Type.InnerDeclaration != null) { var mods = new List<DModule>(); var td=isa.IsModuleAlias ? isa.Type : isa.Type.InnerDeclaration; foreach (var mod in ctxt.ParseCache.LookupModuleName(td.ToString())) mods.Add(mod as DModule); if(mods.Count == 0) ctxt.LogError(new NothingFoundError(isa.Type)); else if(mods.Count > 1) { var m__=new List<ISemantic>(); foreach(var mod in mods) m__.Add(new ModuleSymbol(mod, isa.Type)); ctxt.LogError(new AmbiguityError(isa.Type,m__)); } var bt=mods.Count != 0 ? (AbstractType)new ModuleSymbol(mods[0], td) : null; //TODO: Is this correct behaviour? if (!isa.IsModuleAlias){ var furtherId = ResolveFurtherTypeIdentifier(isa.Type.ToString(false), new[]{ bt }, ctxt, isa.Type); ctxt.CheckForSingleResult(furtherId, isa.Type); if (furtherId != null && furtherId.Length != 0) bt = furtherId[0]; else bt = null; } ret = new AliasedType(isa, bt, isa.Type); } } else if (m is DVariable) { var v = (DVariable)m; AbstractType bt = null; if (DoResolveBaseType) { var bts = TypeDeclarationResolver.Resolve(v.Type, ctxt); if (bts != null && bts.Length != 0 && ctxt.CheckForSingleResult(bts, v.Type)) bt = bts[0]; // For auto variables, use the initializer to get its type else if (v.Initializer != null) bt = ExpressionSemantics.Evaluation.EvaluateType(v.Initializer, ctxt); // Check if inside an foreach statement header if (bt == null && ctxt.ScopedStatement != null) bt = GetForeachIteratorType(v, ctxt); } // Note: Also works for aliases! In this case, we simply try to resolve the aliased type, otherwise the variable's base type ret=v.IsAlias ? (DSymbol)new AliasedType(v, bt, typeBase as ISyntaxRegion) : new MemberSymbol(v, bt, typeBase as ISyntaxRegion); } else if (m is DMethod) { ret = new MemberSymbol((DNode)m, DoResolveBaseType ? GetMethodReturnType((DMethod)m, ctxt) : null , typeBase as ISyntaxRegion); } else if (m is DClassLike) { UserDefinedType udt = null; var dc=(DClassLike)m; switch (dc.ClassType) { case DTokens.Struct: udt = new StructType(dc, typeBase as ISyntaxRegion); break; case DTokens.Union: udt = new UnionType(dc, typeBase as ISyntaxRegion); break; case DTokens.Class: udt = new ClassType(dc, typeBase as ISyntaxRegion, null); break; case DTokens.Template: udt = new TemplateType(dc, typeBase as ISyntaxRegion); break; case DTokens.Interface: udt = new InterfaceType(dc, typeBase as ISyntaxRegion); break; default: ctxt.LogError(new ResolutionError(m, "Unknown type ("+DTokens.GetTokenString(dc.ClassType)+")")); break; } if (canResolveBaseGenerally && !ctxt.Options.HasFlag(ResolutionOptions.DontResolveBaseClasses)) ret = DResolver.ResolveBaseClasses(udt, ctxt); else ret = udt; } else if (m is IAbstractSyntaxTree) { var mod = (IAbstractSyntaxTree)m; if (typeBase != null && typeBase.ToString() != mod.ModuleName) { var pack = ctxt.ParseCache.LookupPackage(typeBase.ToString()).First(); if (pack != null) ret = new PackageSymbol(pack, typeBase as ISyntaxRegion); } else ret = new ModuleSymbol(m as DModule, typeBase as ISyntaxRegion); } else if (m is DEnum) ret = new EnumType((DEnum)m, typeBase as ISyntaxRegion); else if (m is TemplateParameterNode) { var tmp = ((TemplateParameterNode)m).TemplateParameter; //ResolveResult[] templateParameterType = null; //TODO: Resolve the specialization type //var templateParameterType = TemplateInstanceHandler.ResolveTypeSpecialization(tmp, ctxt); ret = new MemberSymbol((DNode)m, null, typeBase as ISyntaxRegion); } if (canResolveBaseGenerally && resultBase is DSymbol) ctxt.CurrentContext.RemoveParamTypesFromPreferredLocals((DSymbol)resultBase); if (popAfterwards) ctxt.Pop(); stackNum_HandleNodeMatch--; return ret; }
public ISymbolValue VisitStructType(StructType t) { throw new NotImplementedException(); }
static AbstractType HandleClassLikeMatch (DClassLike dc, ResolutionContext ctxt, object typeBase, bool canResolveBase) { AbstractType ret; UserDefinedType udt = null; var invisibleTypeParams = GetInvisibleTypeParameters (dc, ctxt); switch (dc.ClassType) { case DTokens.Struct: ret = new StructType (dc, typeBase as ISyntaxRegion, invisibleTypeParams); break; case DTokens.Union: ret = new UnionType (dc, typeBase as ISyntaxRegion, invisibleTypeParams); break; case DTokens.Class: udt = new ClassType (dc, typeBase as ISyntaxRegion, null, null, invisibleTypeParams); ret = null; break; case DTokens.Interface: udt = new InterfaceType (dc, typeBase as ISyntaxRegion, null, invisibleTypeParams); ret = null; break; case DTokens.Template: if (dc.ContainsAttribute (DTokens.Mixin)) ret = new MixinTemplateType (dc, typeBase as ISyntaxRegion, invisibleTypeParams); else ret = new TemplateType (dc, typeBase as ISyntaxRegion, invisibleTypeParams); break; default: ret = null; ctxt.LogError (new ResolutionError (dc, "Unknown type (" + DTokens.GetTokenString (dc.ClassType) + ")")); break; } if (dc.ClassType == DTokens.Class || dc.ClassType == DTokens.Interface) ret = canResolveBase ? DResolver.ResolveBaseClasses (udt, ctxt) : udt; return ret; }
/// <summary> /// Check transform validity and derive the output schema from the input schema. /// /// This checks for validity of interactions between parameters during Transform and /// raises an exception if any parameter value is invalid. /// /// Typical implementation should first conduct verification on schema change and parameter /// validity, including complex parameter interaction checks. /// </summary> /// <param name="value"> /// The <see cref="StructType"/> of the <see cref="DataFrame"/> which will be transformed. /// </param> /// <returns> /// The <see cref="StructType"/> of the output schema that would have been derived from the /// input schema, if Transform had been called. /// </returns> public StructType TransformSchema(StructType value) => new StructType( (JvmObjectReference)_jvmObject.Invoke( "transformSchema", DataType.FromJson(_jvmObject.Jvm, value.Json)));