public TransposeLoader(IHostEnvironment env, Arguments args, IMultiStreamSource file) { Contracts.CheckValue(env, nameof(env)); _host = env.Register(LoadName); _host.CheckValue(args, nameof(args)); _host.CheckValue(file, nameof(file)); _host.Check(file.Count == 1, "Transposed loader accepts a single file only"); _threads = args.Threads ?? 0; if (_threads < 0) { _threads = 0; } _file = file; using (Stream stream = _file.Open(0)) using (BinaryReader reader = new BinaryReader(stream)) { _header = InitHeader(reader); reader.Seek(_header.SubIdvTableOffset); _schemaEntry = new SubIdvEntry.SchemaSubIdv(this, reader); _entries = new SubIdvEntry.TransposedSubIdv[_header.ColumnCount]; for (int c = 0; c < _entries.Length; ++c) { _entries[c] = new SubIdvEntry.TransposedSubIdv(this, reader, c); } _schema = new SchemaImpl(this); if (!HasRowData) { _colTransposers = new Transposer[_header.ColumnCount]; _colTransposersLock = new object(); } } }
internal static IDictionary <string, ISchema> ParseSchemas(JsonDictionary js) { js.ThrowIfNull("values"); RemoveAnnotations(js, 0); var working = new Dictionary <string, ISchema>(); var resolver = new FutureJsonSchemaResolver(); foreach (KeyValuePair <string, object> kvp in js) { logger.Debug("Found schema {0}", kvp.Key); var serilizer = new JsonSerializer(); var textWriter = new StringWriter(); serilizer.Serialize(textWriter, kvp.Value); string result = textWriter.ToString(); ISchema schema = new SchemaImpl(kvp.Key, result, resolver); working.Add(schema.Name, schema); } resolver.ResolveAndVerify(); return(working.AsReadOnly()); }
/// <summary> /// Create a SchemaObject from a sql snippet, attempting to detect the type and name of the object /// </summary> /// <param name="sql">The sql to parse</param> /// <exception cref="SchemaParsingException">If the SQL cannot be parsed</exception> public SchemaObject(string sql) { _sql = sql; var name = ParseSql(); _implementation = SchemaImpl.GetImplementation(_type, name, _sql); }
/// <summary> /// This is the constructor called for the initial wrapping. /// </summary> public Bound(IHostEnvironment env, ISchemaBoundRowMapper mapper, VectorType type, ValueGetter <VBuffer <T> > getter, string metadataKind, Func <ISchemaBoundMapper, ColumnType, bool> canWrap) { Contracts.CheckValue(env, nameof(env)); _host = env.Register(LoaderSignature); _host.CheckValue(mapper, nameof(mapper)); _host.CheckValue(type, nameof(type)); _host.CheckValue(getter, nameof(getter)); _host.CheckNonEmpty(metadataKind, nameof(metadataKind)); _host.CheckValueOrNull(canWrap); _mapper = mapper; int scoreIdx; bool result = mapper.Schema.TryGetColumnIndex(MetadataUtils.Const.ScoreValueKind.Score, out scoreIdx); if (!result) { throw env.ExceptParam(nameof(mapper), "Mapper did not have a '{0}' column", MetadataUtils.Const.ScoreValueKind.Score); } _labelNameType = type; _labelNameGetter = getter; _metadataKind = metadataKind; _outSchema = new SchemaImpl(mapper.Schema, scoreIdx, _labelNameType, _labelNameGetter, _metadataKind); _canWrap = canWrap; }
public ISchema CreateSchema(string schemaId, string name) { Assert.EmptyString(schemaId, "schemaId"); Assert.EmptyString(name, "name"); ISchema schema = null; Sync(() => { schema = GetSchemaInternal(schemaId, false); if (schema == null) { var model = new SchemaModel() { SchemaId = schemaId, Name = name }; if (_store.Save(model)) { schema = new SchemaImpl(model); _schemas[schemaId.ToLower()] = schema; } else { throw new Exception("Não foi possivel criar o esquema de usuários - erro ao persistir as informações"); } } else { throw new Exception("Já existe um esquema de usuários com o id " + schemaId); } }); return(schema); }
private TransposeLoader(IHost host, ModelLoadContext ctx, IMultiStreamSource file) { Contracts.CheckValue(host, nameof(host)); _host = host; _host.CheckValue(file, nameof(file)); _host.Check(file.Count == 1, "Transposed loader accepts a single file only"); // *** Binary format ** // int: Number of threads if explicitly defined, or 0 if the // number of threads was automatically determined _threads = ctx.Reader.ReadInt32(); _host.CheckDecode(_threads >= 0); // Dedupe code somehow? _file = file; using (Stream stream = _file.Open(0)) using (BinaryReader reader = new BinaryReader(stream)) { _header = InitHeader(reader); reader.Seek(_header.SubIdvTableOffset); _schemaEntry = new SubIdvEntry.SchemaSubIdv(this, reader); _entries = new SubIdvEntry.TransposedSubIdv[_header.ColumnCount]; for (int c = 0; c < _entries.Length; ++c) { _entries[c] = new SubIdvEntry.TransposedSubIdv(this, reader, c); } _schema = new SchemaImpl(this); if (!HasRowData) { _colTransposers = new Transposer[_header.ColumnCount]; _colTransposersLock = new object(); } } }
public ISchema[] GetSchemas() { if (!_allSchemasLoaded) { Sync(() => { if (_allSchemasLoaded) { return; } _allSchemasLoaded = true; var qry = _store.CreateQuery <SchemaModel>(); var schemas = qry.Find(); ISchema schema = null; foreach (var schemaModel in schemas) { if (!_schemas.TryGetValue(schemaModel.SchemaId.ToLower(), out schema)) { schema = new SchemaImpl(schemaModel); _schemas[schema.Id.ToLower()] = schema; } } }); } return(_schemas.Values.OrderBy(s => s.Name).ToArray()); }
public RowColumnRow(ICounted counted, IColumn[] columns) { Contracts.AssertValueOrNull(counted); Contracts.AssertValue(columns); _counted = counted ?? _defCount; _columns = columns; _schema = new SchemaImpl(this); }
private UngroupTransform(IHost host, ModelLoadContext ctx, IDataView input) : base(host, input) { Host.AssertValue(ctx); // *** Binary format *** // (schema) _schemaImpl = SchemaImpl.Create(ctx, host, input.Schema); }
public UngroupTransform(IHostEnvironment env, Arguments args, IDataView input) : base(env, LoaderSignature, input) { Host.CheckValue(args, nameof(args)); Host.CheckUserArg(Utils.Size(args.Column) > 0, nameof(args.Column), "There must be at least one pivot column"); Host.CheckUserArg(args.Column.Distinct().Count() == args.Column.Length, nameof(args.Column), "Duplicate pivot columns are not allowed"); _schemaImpl = new SchemaImpl(Host, Source.Schema, args.Mode, args.Column); }
public MetadataRow(ISchema schema, int col) { Contracts.AssertValue(schema); Contracts.Assert(0 <= col && col < schema.ColumnCount); _metaSchema = schema; _col = col; _map = _metaSchema.GetMetadataTypes(_col).ToArray(); _schema = new SchemaImpl(this); }
public MetadataRow(ISchema schema, int col) { Contracts.CheckValue(schema, nameof(schema)); Contracts.CheckParam(0 <= col && col < schema.ColumnCount, nameof(col)); _metaSchema = schema; _col = col; _map = _metaSchema.GetMetadataTypes(_col).ToArray(); _schema = new SchemaImpl(this); }
/// <summary> /// Constructs a SchemaObject of the given type, name, and sql script /// </summary> /// <param name="type">The type of the SchemaObject</param> /// <param name="name">The name of the SchemaObject</param> /// <param name="sql">The SQL script for the SchemaObject</param> /// <remarks>The type and name must match the SQL script.</remarks> /// <exception cref="ArgumentNullException">If name or sql is null</exception> internal SchemaObject(SchemaObjectType type, string name, string sql) { if (name == null) { throw new ArgumentNullException("name"); } _type = type; _sql = sql; _implementation = SchemaImpl.GetImplementation(_type, name, _sql); }
public MetadataRow(ISchema schema, int col, Func <string, bool> takeMetadata) { Contracts.CheckValue(schema, nameof(schema)); Contracts.CheckParam(0 <= col && col < schema.ColumnCount, nameof(col)); Contracts.CheckValue(takeMetadata, nameof(takeMetadata)); _metaSchema = schema; _col = col; _map = _metaSchema.GetMetadataTypes(_col).Where(x => takeMetadata(x.Key)).ToArray(); _schema = new SchemaImpl(this); }
public DataView(IHostEnvironment env, ArrayDataViewBuilder builder, int rowCount) { Contracts.AssertValue(env, "env"); _host = env.Register("ArrayDataView"); _host.AssertValue(builder); _host.Assert(rowCount >= 0); _host.Assert(builder._names.Count == builder._columns.Count); _columns = builder._columns.ToArray(); _schema = new SchemaImpl(_host, _columns.Select(c => c.Type).ToArray(), builder._names.ToArray(), builder); _rowCount = rowCount; }
public void SimpleCreateTest() { FutureJsonSchemaResolver resolver = new FutureJsonSchemaResolver(); SchemaImpl impl = new SchemaImpl("Activitiylist", JsonSchemaElement, resolver); Assert.AreEqual("Activitiylist", impl.Id); Assert.AreEqual("Activitiylist", impl.Name); Assert.IsNotNull(impl.SchemaDetails); Assert.AreEqual(JsonSchemaType.Object, impl.SchemaDetails.Type); Assert.AreEqual(6, impl.SchemaDetails.Properties.Count); Assert.IsNotNull(impl.SchemaDetails.Properties["id"]); Assert.IsNotNull(impl.SchemaDetails.Properties["title"]); Assert.IsNotNull(impl.SchemaDetails.Properties["items"]); Assert.IsNotNull(impl.SchemaDetails.Properties["updated"]); Assert.IsNotNull(impl.SchemaDetails.Properties["links"]); Assert.IsNotNull(impl.SchemaDetails.Properties["kind"]); }
public Cursor(IChannelProvider provider, IRowCursor input, SchemaImpl schema, Func <int, bool> predicate) : base(provider, input) { _schema = schema; _active = Utils.BuildArray(_schema.ColumnCount, predicate); _cachedGetters = new Delegate[_schema.ColumnCount]; _colSizes = new int[_schema.ColumnCount]; int sizeColumnsLim = _schema.Mode == UngroupMode.First ? 1 : _schema.PivotColumnCount; _fixedSize = 0; var needed = new List <Func <int> >(); for (int i = 0; i < sizeColumnsLim; i++) { var info = _schema.GetPivotColumnInfo(i); if (info.Size > 0) { if (_fixedSize == 0) { _fixedSize = info.Size; } else if (_schema.Mode == UngroupMode.Inner && _fixedSize > info.Size) { _fixedSize = info.Size; } else if (_schema.Mode == UngroupMode.Outer && _fixedSize < info.Size) { _fixedSize = info.Size; } } else { // This will also create and cache a getter for the pivot column. // That's why MakeSizeGetter is an instance method. var rawItemType = info.ItemType.RawType; Func <int, Func <int> > del = MakeSizeGetter <int>; var mi = del.GetMethodInfo().GetGenericMethodDefinition().MakeGenericMethod(rawItemType); var sizeGetter = (Func <int>)mi.Invoke(this, new object[] { info.Index }); needed.Add(sizeGetter); } } _sizeGetters = needed.ToArray(); Ch.Assert(_fixedSize > 0 || _sizeGetters.Length > 0); }
protected SequentialAnomalyDetectionTransformBase(IHostEnvironment env, ModelLoadContext ctx, string name, IDataView input) : base(env, ctx, name, input) { // *** Binary format *** // <base> // byte: _martingale // byte: _alertingScore // byte: _anomalySide // Double: _powerMartingaleEpsilon // Double: _alertThreshold byte temp; temp = ctx.Reader.ReadByte(); Host.CheckDecode(Enum.IsDefined(typeof(MartingaleType), temp)); Martingale = (MartingaleType)temp; temp = ctx.Reader.ReadByte(); Host.CheckDecode(Enum.IsDefined(typeof(AlertingScore), temp)); ThresholdScore = (AlertingScore)temp; Host.CheckDecode(Martingale != MartingaleType.None || ThresholdScore != AlertingScore.MartingaleScore); Host.CheckDecode(WindowSize > 0 || ThresholdScore == AlertingScore.RawScore); temp = ctx.Reader.ReadByte(); Host.CheckDecode(Enum.IsDefined(typeof(AnomalySide), temp)); Side = (AnomalySide)temp; PowerMartingaleEpsilon = ctx.Reader.ReadDouble(); Host.CheckDecode(0 < PowerMartingaleEpsilon && PowerMartingaleEpsilon < 1); AlertThreshold = ctx.Reader.ReadDouble(); Host.CheckDecode(AlertThreshold >= 0); Host.CheckDecode(ThresholdScore != AlertingScore.PValueScore || (0 <= AlertThreshold && AlertThreshold <= 1)); _outputLength = GetOutputLength(ThresholdScore, Host); _wrappedSchema = CreateSchema(base.Schema, OutputColumnName, _outputLength); }
private ISchema GetSchemaInternal(String schemaId, bool throwIfNotExists) { Assert.EmptyString(schemaId, "schemaId"); ISchema result = null; schemaId = schemaId.ToLower(); if (!_schemas.TryGetValue(schemaId, out result)) { var qry = _store.CreateQuery <SchemaModel>(); qry.AddFilterEqual("SchemaId", schemaId); var model = qry.FindFirst(); if (model != null) { result = new SchemaImpl(model); _schemas[schemaId] = result; } } if (result == null && throwIfNotExists) { throw new InvalidOperationException("Não existe um esquema de usuários com o id " + schemaId); } return(result); }
protected SequentialAnomalyDetectionTransformBase(int windowSize, int initialWindowSize, string inputColumnName, string outputColumnName, string name, IHostEnvironment env, IDataView input, AnomalySide anomalySide, MartingaleType martingale, AlertingScore alertingScore, Double powerMartingaleEpsilon, Double alertThreshold) : base(windowSize, initialWindowSize, inputColumnName, outputColumnName, name, env, input, new VectorType(NumberType.R8, GetOutputLength(alertingScore, env))) { Host.CheckUserArg(Enum.IsDefined(typeof(MartingaleType), martingale), nameof(ArgumentsBase.Martingale), "Value is undefined."); Host.CheckUserArg(Enum.IsDefined(typeof(AnomalySide), anomalySide), nameof(ArgumentsBase.Side), "Value is undefined."); Host.CheckUserArg(Enum.IsDefined(typeof(AlertingScore), alertingScore), nameof(ArgumentsBase.AlertOn), "Value is undefined."); Host.CheckUserArg(martingale != MartingaleType.None || alertingScore != AlertingScore.MartingaleScore, nameof(ArgumentsBase.Martingale), "A martingale type should be specified if alerting is based on the martingale score."); Host.CheckUserArg(windowSize > 0 || alertingScore == AlertingScore.RawScore, nameof(ArgumentsBase.AlertOn), "When there is no windowed buffering (i.e., " + nameof(ArgumentsBase.WindowSize) + " = 0), the alert can be generated only based on the raw score (i.e., " + nameof(ArgumentsBase.AlertOn) + " = " + nameof(AlertingScore.RawScore) + ")"); Host.CheckUserArg(0 < powerMartingaleEpsilon && powerMartingaleEpsilon < 1, nameof(ArgumentsBase.PowerMartingaleEpsilon), "Should be in (0,1)."); Host.CheckUserArg(alertThreshold >= 0, nameof(ArgumentsBase.AlertThreshold), "Must be non-negative."); Host.CheckUserArg(alertingScore != AlertingScore.PValueScore || (0 <= alertThreshold && alertThreshold <= 1), nameof(ArgumentsBase.AlertThreshold), "Must be in [0,1]."); ThresholdScore = alertingScore; Side = anomalySide; Martingale = martingale; PowerMartingaleEpsilon = powerMartingaleEpsilon; AlertThreshold = alertThreshold; _outputLength = GetOutputLength(ThresholdScore, Host); _wrappedSchema = CreateSchema(base.Schema, outputColumnName, _outputLength); }
/// <summary> /// Drop an object from the database /// </summary> /// <param name="connection">The Sql connection to use</param> /// <param name="type">The type of the object</param> /// <param name="objectName">The name of the object</param> internal static void Drop(IDbConnection connection, SchemaObjectType type, string objectName) { var implementation = SchemaImpl.GetImplementation(type, objectName, null); implementation.Drop(connection); }