public TransposeLoader(IHostEnvironment env, Arguments args, IMultiStreamSource file)
{
Contracts.CheckValue(env, nameof(env));
_host = env.Register(LoadName);
_host.CheckValue(args, nameof(args));
_host.CheckValue(file, nameof(file));
_host.Check(file.Count == 1, "Transposed loader accepts a single file only");
_threads = args.Threads ?? 0;
if (_threads < 0)
{
_threads = 0;
}
_file = file;
using (Stream stream = _file.Open(0))
using (BinaryReader reader = new BinaryReader(stream))
{
_header = InitHeader(reader);
reader.Seek(_header.SubIdvTableOffset);
_schemaEntry = new SubIdvEntry.SchemaSubIdv(this, reader);
_entries = new SubIdvEntry.TransposedSubIdv[_header.ColumnCount];
for (int c = 0; c < _entries.Length; ++c)
{
_entries[c] = new SubIdvEntry.TransposedSubIdv(this, reader, c);
}
_schema = new SchemaImpl(this);
if (!HasRowData)
{
_colTransposers = new Transposer[_header.ColumnCount];
_colTransposersLock = new object();
}
}
}
internal static IDictionary <string, ISchema> ParseSchemas(JsonDictionary js)
{
js.ThrowIfNull("values");
RemoveAnnotations(js, 0);
var working = new Dictionary <string, ISchema>();
var resolver = new FutureJsonSchemaResolver();
foreach (KeyValuePair <string, object> kvp in js)
{
logger.Debug("Found schema {0}", kvp.Key);
var serilizer = new JsonSerializer();
var textWriter = new StringWriter();
serilizer.Serialize(textWriter, kvp.Value);
string result = textWriter.ToString();
ISchema schema = new SchemaImpl(kvp.Key, result, resolver);
working.Add(schema.Name, schema);
}
resolver.ResolveAndVerify();
return(working.AsReadOnly());
}
/// <summary>
/// Create a SchemaObject from a sql snippet, attempting to detect the type and name of the object
/// </summary>
/// <param name="sql">The sql to parse</param>
/// <exception cref="SchemaParsingException">If the SQL cannot be parsed</exception>
public SchemaObject(string sql)
{
_sql = sql;
var name = ParseSql();
_implementation = SchemaImpl.GetImplementation(_type, name, _sql);
}
/// <summary>
/// This is the constructor called for the initial wrapping.
/// </summary>
public Bound(IHostEnvironment env, ISchemaBoundRowMapper mapper, VectorType type, ValueGetter <VBuffer <T> > getter,
string metadataKind, Func <ISchemaBoundMapper, ColumnType, bool> canWrap)
{
Contracts.CheckValue(env, nameof(env));
_host = env.Register(LoaderSignature);
_host.CheckValue(mapper, nameof(mapper));
_host.CheckValue(type, nameof(type));
_host.CheckValue(getter, nameof(getter));
_host.CheckNonEmpty(metadataKind, nameof(metadataKind));
_host.CheckValueOrNull(canWrap);
_mapper = mapper;
int scoreIdx;
bool result = mapper.Schema.TryGetColumnIndex(MetadataUtils.Const.ScoreValueKind.Score, out scoreIdx);
if (!result)
{
throw env.ExceptParam(nameof(mapper), "Mapper did not have a '{0}' column", MetadataUtils.Const.ScoreValueKind.Score);
}
_labelNameType = type;
_labelNameGetter = getter;
_metadataKind = metadataKind;
_outSchema = new SchemaImpl(mapper.Schema, scoreIdx, _labelNameType, _labelNameGetter, _metadataKind);
_canWrap = canWrap;
}
public ISchema CreateSchema(string schemaId, string name)
{
Assert.EmptyString(schemaId, "schemaId");
Assert.EmptyString(name, "name");
ISchema schema = null;
Sync(() =>
{
schema = GetSchemaInternal(schemaId, false);
if (schema == null)
{
var model = new SchemaModel()
{
SchemaId = schemaId,
Name = name
};
if (_store.Save(model))
{
schema = new SchemaImpl(model);
_schemas[schemaId.ToLower()] = schema;
}
else
{
throw new Exception("Não foi possivel criar o esquema de usuários - erro ao persistir as informações");
}
}
else
{
throw new Exception("Já existe um esquema de usuários com o id " + schemaId);
}
});
return(schema);
}
private TransposeLoader(IHost host, ModelLoadContext ctx, IMultiStreamSource file)
{
Contracts.CheckValue(host, nameof(host));
_host = host;
_host.CheckValue(file, nameof(file));
_host.Check(file.Count == 1, "Transposed loader accepts a single file only");
// *** Binary format **
// int: Number of threads if explicitly defined, or 0 if the
// number of threads was automatically determined
_threads = ctx.Reader.ReadInt32();
_host.CheckDecode(_threads >= 0);
// Dedupe code somehow?
_file = file;
using (Stream stream = _file.Open(0))
using (BinaryReader reader = new BinaryReader(stream))
{
_header = InitHeader(reader);
reader.Seek(_header.SubIdvTableOffset);
_schemaEntry = new SubIdvEntry.SchemaSubIdv(this, reader);
_entries = new SubIdvEntry.TransposedSubIdv[_header.ColumnCount];
for (int c = 0; c < _entries.Length; ++c)
{
_entries[c] = new SubIdvEntry.TransposedSubIdv(this, reader, c);
}
_schema = new SchemaImpl(this);
if (!HasRowData)
{
_colTransposers = new Transposer[_header.ColumnCount];
_colTransposersLock = new object();
}
}
}
public ISchema[] GetSchemas()
{
if (!_allSchemasLoaded)
{
Sync(() =>
{
if (_allSchemasLoaded)
{
return;
}
_allSchemasLoaded = true;
var qry = _store.CreateQuery <SchemaModel>();
var schemas = qry.Find();
ISchema schema = null;
foreach (var schemaModel in schemas)
{
if (!_schemas.TryGetValue(schemaModel.SchemaId.ToLower(), out schema))
{
schema = new SchemaImpl(schemaModel);
_schemas[schema.Id.ToLower()] = schema;
}
}
});
}
return(_schemas.Values.OrderBy(s => s.Name).ToArray());
}
public RowColumnRow(ICounted counted, IColumn[] columns)
{
Contracts.AssertValueOrNull(counted);
Contracts.AssertValue(columns);
_counted = counted ?? _defCount;
_columns = columns;
_schema = new SchemaImpl(this);
}
private UngroupTransform(IHost host, ModelLoadContext ctx, IDataView input)
: base(host, input)
{
Host.AssertValue(ctx);
// *** Binary format ***
// (schema)
_schemaImpl = SchemaImpl.Create(ctx, host, input.Schema);
}
public UngroupTransform(IHostEnvironment env, Arguments args, IDataView input)
: base(env, LoaderSignature, input)
{
Host.CheckValue(args, nameof(args));
Host.CheckUserArg(Utils.Size(args.Column) > 0, nameof(args.Column), "There must be at least one pivot column");
Host.CheckUserArg(args.Column.Distinct().Count() == args.Column.Length, nameof(args.Column),
"Duplicate pivot columns are not allowed");
_schemaImpl = new SchemaImpl(Host, Source.Schema, args.Mode, args.Column);
}
public MetadataRow(ISchema schema, int col)
{
Contracts.AssertValue(schema);
Contracts.Assert(0 <= col && col < schema.ColumnCount);
_metaSchema = schema;
_col = col;
_map = _metaSchema.GetMetadataTypes(_col).ToArray();
_schema = new SchemaImpl(this);
}
public MetadataRow(ISchema schema, int col)
{
Contracts.CheckValue(schema, nameof(schema));
Contracts.CheckParam(0 <= col && col < schema.ColumnCount, nameof(col));
_metaSchema = schema;
_col = col;
_map = _metaSchema.GetMetadataTypes(_col).ToArray();
_schema = new SchemaImpl(this);
}
/// <summary>
/// Constructs a SchemaObject of the given type, name, and sql script
/// </summary>
/// <param name="type">The type of the SchemaObject</param>
/// <param name="name">The name of the SchemaObject</param>
/// <param name="sql">The SQL script for the SchemaObject</param>
/// <remarks>The type and name must match the SQL script.</remarks>
/// <exception cref="ArgumentNullException">If name or sql is null</exception>
internal SchemaObject(SchemaObjectType type, string name, string sql)
{
if (name == null)
{
throw new ArgumentNullException("name");
}
_type = type;
_sql = sql;
_implementation = SchemaImpl.GetImplementation(_type, name, _sql);
}
public MetadataRow(ISchema schema, int col, Func <string, bool> takeMetadata)
{
Contracts.CheckValue(schema, nameof(schema));
Contracts.CheckParam(0 <= col && col < schema.ColumnCount, nameof(col));
Contracts.CheckValue(takeMetadata, nameof(takeMetadata));
_metaSchema = schema;
_col = col;
_map = _metaSchema.GetMetadataTypes(_col).Where(x => takeMetadata(x.Key)).ToArray();
_schema = new SchemaImpl(this);
}
public DataView(IHostEnvironment env, ArrayDataViewBuilder builder, int rowCount)
{
Contracts.AssertValue(env, "env");
_host = env.Register("ArrayDataView");
_host.AssertValue(builder);
_host.Assert(rowCount >= 0);
_host.Assert(builder._names.Count == builder._columns.Count);
_columns = builder._columns.ToArray();
_schema = new SchemaImpl(_host, _columns.Select(c => c.Type).ToArray(), builder._names.ToArray(), builder);
_rowCount = rowCount;
}
public void SimpleCreateTest()
{
FutureJsonSchemaResolver resolver = new FutureJsonSchemaResolver();
SchemaImpl impl = new SchemaImpl("Activitiylist", JsonSchemaElement, resolver);
Assert.AreEqual("Activitiylist", impl.Id);
Assert.AreEqual("Activitiylist", impl.Name);
Assert.IsNotNull(impl.SchemaDetails);
Assert.AreEqual(JsonSchemaType.Object, impl.SchemaDetails.Type);
Assert.AreEqual(6, impl.SchemaDetails.Properties.Count);
Assert.IsNotNull(impl.SchemaDetails.Properties["id"]);
Assert.IsNotNull(impl.SchemaDetails.Properties["title"]);
Assert.IsNotNull(impl.SchemaDetails.Properties["items"]);
Assert.IsNotNull(impl.SchemaDetails.Properties["updated"]);
Assert.IsNotNull(impl.SchemaDetails.Properties["links"]);
Assert.IsNotNull(impl.SchemaDetails.Properties["kind"]);
}
public Cursor(IChannelProvider provider, IRowCursor input, SchemaImpl schema, Func <int, bool> predicate)
: base(provider, input)
{
_schema = schema;
_active = Utils.BuildArray(_schema.ColumnCount, predicate);
_cachedGetters = new Delegate[_schema.ColumnCount];
_colSizes = new int[_schema.ColumnCount];
int sizeColumnsLim = _schema.Mode == UngroupMode.First ? 1 : _schema.PivotColumnCount;
_fixedSize = 0;
var needed = new List <Func <int> >();
for (int i = 0; i < sizeColumnsLim; i++)
{
var info = _schema.GetPivotColumnInfo(i);
if (info.Size > 0)
{
if (_fixedSize == 0)
{
_fixedSize = info.Size;
}
else if (_schema.Mode == UngroupMode.Inner && _fixedSize > info.Size)
{
_fixedSize = info.Size;
}
else if (_schema.Mode == UngroupMode.Outer && _fixedSize < info.Size)
{
_fixedSize = info.Size;
}
}
else
{
// This will also create and cache a getter for the pivot column.
// That's why MakeSizeGetter is an instance method.
var rawItemType = info.ItemType.RawType;
Func <int, Func <int> > del = MakeSizeGetter <int>;
var mi = del.GetMethodInfo().GetGenericMethodDefinition().MakeGenericMethod(rawItemType);
var sizeGetter = (Func <int>)mi.Invoke(this, new object[] { info.Index });
needed.Add(sizeGetter);
}
}
_sizeGetters = needed.ToArray();
Ch.Assert(_fixedSize > 0 || _sizeGetters.Length > 0);
}
protected SequentialAnomalyDetectionTransformBase(IHostEnvironment env, ModelLoadContext ctx, string name, IDataView input)
: base(env, ctx, name, input)
{
// *** Binary format ***
// <base>
// byte: _martingale
// byte: _alertingScore
// byte: _anomalySide
// Double: _powerMartingaleEpsilon
// Double: _alertThreshold
byte temp;
temp = ctx.Reader.ReadByte();
Host.CheckDecode(Enum.IsDefined(typeof(MartingaleType), temp));
Martingale = (MartingaleType)temp;
temp = ctx.Reader.ReadByte();
Host.CheckDecode(Enum.IsDefined(typeof(AlertingScore), temp));
ThresholdScore = (AlertingScore)temp;
Host.CheckDecode(Martingale != MartingaleType.None || ThresholdScore != AlertingScore.MartingaleScore);
Host.CheckDecode(WindowSize > 0 || ThresholdScore == AlertingScore.RawScore);
temp = ctx.Reader.ReadByte();
Host.CheckDecode(Enum.IsDefined(typeof(AnomalySide), temp));
Side = (AnomalySide)temp;
PowerMartingaleEpsilon = ctx.Reader.ReadDouble();
Host.CheckDecode(0 < PowerMartingaleEpsilon && PowerMartingaleEpsilon < 1);
AlertThreshold = ctx.Reader.ReadDouble();
Host.CheckDecode(AlertThreshold >= 0);
Host.CheckDecode(ThresholdScore != AlertingScore.PValueScore || (0 <= AlertThreshold && AlertThreshold <= 1));
_outputLength = GetOutputLength(ThresholdScore, Host);
_wrappedSchema = CreateSchema(base.Schema, OutputColumnName, _outputLength);
}
private ISchema GetSchemaInternal(String schemaId, bool throwIfNotExists)
{
Assert.EmptyString(schemaId, "schemaId");
ISchema result = null;
schemaId = schemaId.ToLower();
if (!_schemas.TryGetValue(schemaId, out result))
{
var qry = _store.CreateQuery <SchemaModel>();
qry.AddFilterEqual("SchemaId", schemaId);
var model = qry.FindFirst();
if (model != null)
{
result = new SchemaImpl(model);
_schemas[schemaId] = result;
}
}
if (result == null && throwIfNotExists)
{
throw new InvalidOperationException("Não existe um esquema de usuários com o id " + schemaId);
}
return(result);
}
protected SequentialAnomalyDetectionTransformBase(int windowSize, int initialWindowSize, string inputColumnName, string outputColumnName, string name, IHostEnvironment env, IDataView input,
AnomalySide anomalySide, MartingaleType martingale, AlertingScore alertingScore, Double powerMartingaleEpsilon,
Double alertThreshold)
: base(windowSize, initialWindowSize, inputColumnName, outputColumnName, name, env, input, new VectorType(NumberType.R8, GetOutputLength(alertingScore, env)))
{
Host.CheckUserArg(Enum.IsDefined(typeof(MartingaleType), martingale), nameof(ArgumentsBase.Martingale), "Value is undefined.");
Host.CheckUserArg(Enum.IsDefined(typeof(AnomalySide), anomalySide), nameof(ArgumentsBase.Side), "Value is undefined.");
Host.CheckUserArg(Enum.IsDefined(typeof(AlertingScore), alertingScore), nameof(ArgumentsBase.AlertOn), "Value is undefined.");
Host.CheckUserArg(martingale != MartingaleType.None || alertingScore != AlertingScore.MartingaleScore, nameof(ArgumentsBase.Martingale), "A martingale type should be specified if alerting is based on the martingale score.");
Host.CheckUserArg(windowSize > 0 || alertingScore == AlertingScore.RawScore, nameof(ArgumentsBase.AlertOn),
"When there is no windowed buffering (i.e., " + nameof(ArgumentsBase.WindowSize) + " = 0), the alert can be generated only based on the raw score (i.e., "
+ nameof(ArgumentsBase.AlertOn) + " = " + nameof(AlertingScore.RawScore) + ")");
Host.CheckUserArg(0 < powerMartingaleEpsilon && powerMartingaleEpsilon < 1, nameof(ArgumentsBase.PowerMartingaleEpsilon), "Should be in (0,1).");
Host.CheckUserArg(alertThreshold >= 0, nameof(ArgumentsBase.AlertThreshold), "Must be non-negative.");
Host.CheckUserArg(alertingScore != AlertingScore.PValueScore || (0 <= alertThreshold && alertThreshold <= 1), nameof(ArgumentsBase.AlertThreshold), "Must be in [0,1].");
ThresholdScore = alertingScore;
Side = anomalySide;
Martingale = martingale;
PowerMartingaleEpsilon = powerMartingaleEpsilon;
AlertThreshold = alertThreshold;
_outputLength = GetOutputLength(ThresholdScore, Host);
_wrappedSchema = CreateSchema(base.Schema, outputColumnName, _outputLength);
}
/// <summary>
/// Drop an object from the database
/// </summary>
/// <param name="connection">The Sql connection to use</param>
/// <param name="type">The type of the object</param>
/// <param name="objectName">The name of the object</param>
internal static void Drop(IDbConnection connection, SchemaObjectType type, string objectName)
{
var implementation = SchemaImpl.GetImplementation(type, objectName, null);
implementation.Drop(connection);
}
