/// <summary>
/// Gets the output <see cref="SchemaShape"/> of the <see cref="IDataView"/> after fitting the calibrator.
/// Fitting the calibrator will add a column named "Probability" to the schema. If you already had such a column, a new one will be added.
/// </summary>
/// <param name="inputSchema">The input <see cref="SchemaShape"/>.</param>
SchemaShape IEstimator <CalibratorTransformer <TICalibrator> > .GetOutputSchema(SchemaShape inputSchema)
{
Action <SchemaShape.Column, string> checkColumnValid = (SchemaShape.Column column, string expected) =>
{
if (column.IsValid)
{
if (!inputSchema.TryFindColumn(column.Name, out var outCol))
{
throw Host.Except($"{expected} column '{column.Name}' is not found");
}
if (!column.IsCompatibleWith(outCol))
{
throw Host.Except($"{expected} column '{column.Name}' is not compatible");
}
}
};
// check the input schema
checkColumnValid(ScoreColumn, DefaultColumnNames.Score);
checkColumnValid(WeightColumn, DefaultColumnNames.Weight);
checkColumnValid(LabelColumn, DefaultColumnNames.Label);
checkColumnValid(FeatureColumn, DefaultColumnNames.Features);
checkColumnValid(PredictedLabel, DefaultColumnNames.PredictedLabel);
//create the new Probability column
var outColumns = inputSchema.ToDictionary(x => x.Name);
outColumns[DefaultColumnNames.Probability] = new SchemaShape.Column(DefaultColumnNames.Probability,
SchemaShape.Column.VectorKind.Scalar,
NumberType.R4,
false,
new SchemaShape(MetadataUtils.GetTrainerOutputMetadata(true)));
return(new SchemaShape(outColumns.Values));
}
/// <summary>
/// Writes an instance annotation.
/// </summary>
/// <param name="instanceAnnotation">The instance annotation to write.</param>
/// <param name="ignoreFilter">Whether to ignore the filter in settings.</param>
/// <param name="propertyName">The name of the property this instance annotation applies to</param>
internal void WriteInstanceAnnotation(ODataInstanceAnnotation instanceAnnotation, bool ignoreFilter = false, string propertyName = null)
{
string name = instanceAnnotation.Name;
ODataValue value = instanceAnnotation.Value;
Debug.Assert(!string.IsNullOrEmpty(name), "name should not be null or empty");
Debug.Assert(value != null, "value should not be null because we use ODataNullValue for null instead");
Debug.Assert(!(value is ODataStreamReferenceValue), "!(value is ODataStreamReferenceValue) -- ODataInstanceAnnotation and InstanceAnnotationCollection will throw if the value is a stream value.");
Debug.Assert(this.valueSerializer.Model != null, "this.valueSerializer.Model != null");
if (!ignoreFilter && this.valueSerializer.MessageWriterSettings.ShouldSkipAnnotation(name))
{
return;
}
IEdmTypeReference expectedType = MetadataUtils.LookupTypeOfTerm(name, this.valueSerializer.Model);
if (value is ODataNullValue)
{
if (expectedType != null && !expectedType.IsNullable)
{
throw new ODataException(
ODataErrorStrings.JsonLightInstanceAnnotationWriter_NullValueNotAllowedForInstanceAnnotation(
instanceAnnotation.Name, expectedType.FullName()));
}
this.WriteInstanceAnnotationName(propertyName, name);
this.valueSerializer.WriteNullValue();
return;
}
// If we didn't find an expected type from looking up the term in the model, treat this value the same way we would for open property values.
// That is, write the type name (unless its a primitive value with a JSON-native type). If we did find an expected type, treat the annotation value like a
// declared property with an expected type. This will still write out the type if the value type is more derived than the declared type, for example.
bool treatLikeOpenProperty = expectedType == null;
ODataCollectionValue collectionValue = value as ODataCollectionValue;
if (collectionValue != null)
{
IEdmTypeReference typeFromCollectionValue = (IEdmCollectionTypeReference)TypeNameOracle.ResolveAndValidateTypeForCollectionValue(
this.valueSerializer.Model, expectedType, collectionValue, treatLikeOpenProperty, this.writerValidator);
string collectionTypeNameToWrite = this.typeNameOracle.GetValueTypeNameForWriting(collectionValue, expectedType, typeFromCollectionValue, treatLikeOpenProperty);
if (collectionTypeNameToWrite != null)
{
this.odataAnnotationWriter.WriteODataTypePropertyAnnotation(name, collectionTypeNameToWrite);
}
this.WriteInstanceAnnotationName(propertyName, name);
this.valueSerializer.WriteCollectionValue(collectionValue, expectedType, typeFromCollectionValue, false /*isTopLevelProperty*/, false /*isInUri*/, treatLikeOpenProperty);
return;
}
ODataUntypedValue untypedValue = value as ODataUntypedValue;
if (untypedValue != null)
{
this.WriteInstanceAnnotationName(propertyName, name);
this.valueSerializer.WriteUntypedValue(untypedValue);
return;
}
ODataEnumValue enumValue = value as ODataEnumValue;
if (enumValue != null)
{
this.WriteInstanceAnnotationName(propertyName, name);
this.valueSerializer.WriteEnumValue(enumValue, expectedType);
return;
}
ODataPrimitiveValue primitiveValue = value as ODataPrimitiveValue;
Debug.Assert(primitiveValue != null, "Did we add a new subclass of ODataValue?");
IEdmTypeReference typeFromPrimitiveValue = TypeNameOracle.ResolveAndValidateTypeForPrimitiveValue(primitiveValue);
string primitiveTypeNameToWrite = this.typeNameOracle.GetValueTypeNameForWriting(primitiveValue, expectedType, typeFromPrimitiveValue, treatLikeOpenProperty);
if (primitiveTypeNameToWrite != null)
{
this.odataAnnotationWriter.WriteODataTypePropertyAnnotation(name, primitiveTypeNameToWrite);
}
this.WriteInstanceAnnotationName(propertyName, name);
this.valueSerializer.WritePrimitiveValue(primitiveValue.Value, typeFromPrimitiveValue, expectedType);
}
private SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
{
if (LabelColumn.IsValid)
{
bool success = inputSchema.TryFindColumn(LabelColumn.Name, out var labelCol);
Contracts.Assert(success);
var metadata = new SchemaShape(labelCol.Metadata.Where(x => x.Name == MetadataUtils.Kinds.KeyValues)
.Concat(MetadataForScoreColumn()));
return(new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Vector, NumberType.R4, false, new SchemaShape(MetadataUtils.MetadataForMulticlassScoreColumn(labelCol))),
new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, NumberType.U4, true, metadata)
});
}
else
{
return new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Vector, NumberType.R4, false, new SchemaShape(MetadataForScoreColumn())),
new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, NumberType.U4, true, new SchemaShape(MetadataForScoreColumn()))
}
};
}
private protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
{
bool success = inputSchema.TryFindColumn(LabelColumn.Name, out var labelCol);
Contracts.Assert(success);
var metadata = new SchemaShape(labelCol.Metadata.Where(x => x.Name == MetadataUtils.Kinds.KeyValues)
.Concat(MetadataUtils.GetTrainerOutputMetadata()));
return(new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Vector, NumberDataViewType.Single, false, new SchemaShape(MetadataUtils.MetadataForMulticlassScoreColumn(labelCol))),
new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, NumberDataViewType.UInt32, true, metadata)
});
}
private protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
{
return(new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberDataViewType.Single, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())),
new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, BooleanDataViewType.Instance, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
});
}
private async Task UpdateDatabaseMetadataAsync(IFileMetadata fileMetadata, bool updateAlbumArtwork)
{
Track track = await this.trackRepository.GetTrackAsync(fileMetadata.SafePath);
if (track == null)
{
return;
}
// Track
if (fileMetadata.Title.IsValueChanged)
{
track.TrackTitle = fileMetadata.Title.Value;
}
if (fileMetadata.Year.IsValueChanged)
{
track.Year = fileMetadata.Year.Value.SafeConvertToLong();
}
if (fileMetadata.TrackNumber.IsValueChanged)
{
track.TrackNumber = fileMetadata.TrackNumber.Value.SafeConvertToLong();
}
if (fileMetadata.TrackCount.IsValueChanged)
{
track.TrackCount = fileMetadata.TrackCount.Value.SafeConvertToLong();
}
if (fileMetadata.DiscNumber.IsValueChanged)
{
track.DiscNumber = fileMetadata.DiscNumber.Value.SafeConvertToLong();
}
if (fileMetadata.DiscCount.IsValueChanged)
{
track.DiscCount = fileMetadata.DiscCount.Value.SafeConvertToLong();
}
if (fileMetadata.Lyrics.IsValueChanged)
{
track.HasLyrics = string.IsNullOrWhiteSpace(fileMetadata.Lyrics.Value) ? 0 : 1;
}
// Artist
if (fileMetadata.Artists.IsValueChanged)
{
string newArtistName = fileMetadata.Artists.Values != null && !string.IsNullOrEmpty(fileMetadata.Artists.Values.FirstOrDefault()) ? fileMetadata.Artists.Values.FirstOrDefault() : Defaults.UnknownArtistText;
Artist artist = await this.artistRepository.GetArtistAsync(newArtistName);
if (artist == null)
{
artist = await this.artistRepository.AddArtistAsync(new Artist { ArtistName = newArtistName });
}
if (artist != null)
{
track.ArtistID = artist.ArtistID;
}
}
// Genre
if (fileMetadata.Genres.IsValueChanged)
{
string newGenreName = fileMetadata.Genres.Values != null && !string.IsNullOrEmpty(fileMetadata.Genres.Values.FirstOrDefault()) ? fileMetadata.Genres.Values.FirstOrDefault() : Defaults.UnknownGenreText;
Genre genre = await this.genreRepository.GetGenreAsync(newGenreName);
if (genre == null)
{
genre = await this.genreRepository.AddGenreAsync(new Genre { GenreName = newGenreName });
}
if (genre != null)
{
track.GenreID = genre.GenreID;
}
}
// Album
if (fileMetadata.Album.IsValueChanged || fileMetadata.AlbumArtists.IsValueChanged || fileMetadata.Year.IsValueChanged)
{
string newAlbumTitle = !string.IsNullOrWhiteSpace(fileMetadata.Album.Value) ? fileMetadata.Album.Value : Defaults.UnknownAlbumText;
string newAlbumArtist = fileMetadata.AlbumArtists.Values != null && !string.IsNullOrEmpty(fileMetadata.AlbumArtists.Values.FirstOrDefault()) ? fileMetadata.AlbumArtists.Values.FirstOrDefault() : Defaults.UnknownArtistText;
Album album = await this.albumRepository.GetAlbumAsync(newAlbumTitle, newAlbumArtist);
if (album == null)
{
album = new Album {
AlbumTitle = newAlbumTitle, AlbumArtist = newAlbumArtist, DateLastSynced = DateTime.Now.Ticks
};
album.ArtworkID = await this.cacheService.CacheArtworkAsync(IndexerUtils.GetArtwork(album, this.metadataFactory.Create(track.Path)));
album = await this.albumRepository.AddAlbumAsync(album);
}
if (album != null)
{
track.AlbumID = album.AlbumID;
}
await Task.Run(() => MetadataUtils.UpdateAlbumYear(album, fileMetadata.Year.Value.SafeConvertToLong())); // Update Album year
await this.albumRepository.UpdateAlbumAsync(album);
}
await this.trackRepository.UpdateTrackAsync(track); // Update Track in the database
if (updateAlbumArtwork)
{
// Get album artist
string albumArtist = fileMetadata.AlbumArtists.Values != null && !string.IsNullOrEmpty(fileMetadata.AlbumArtists.Values.FirstOrDefault()) ? fileMetadata.AlbumArtists.Values.FirstOrDefault() : string.Empty;
// If no album artist is found, use the artist name. The album was probably saved using the artist name.
if (string.IsNullOrEmpty(albumArtist))
{
albumArtist = fileMetadata.Artists.Values != null && !string.IsNullOrEmpty(fileMetadata.Artists.Values.FirstOrDefault()) ? fileMetadata.Artists.Values.FirstOrDefault() : Defaults.UnknownArtistText;
}
// Get the album title
string albumTitle = !string.IsNullOrWhiteSpace(fileMetadata.Album.Value) ? fileMetadata.Album.Value : Defaults.UnknownAlbumText;
// Cache the new artwork
string artworkID = await this.cacheService.CacheArtworkAsync(fileMetadata.ArtworkData.Value);
// Update the album artwork in the database
await this.albumRepository.UpdateAlbumArtworkAsync(albumTitle, albumArtist, artworkID);
}
}
public SchemaShape GetOutputSchema(SchemaShape inputSchema)
{
Host.CheckValue(inputSchema, nameof(inputSchema));
var outColumns = inputSchema.ToDictionary(x => x.Name);
var newColumns = new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())),
new SchemaShape.Column(DefaultColumnNames.Probability, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata(true))),
new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, BoolType.Instance, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
};
foreach (SchemaShape.Column column in newColumns)
{
outColumns[column.Name] = column;
}
return(new SchemaShape(outColumns.Values));
}
public static TrackInfo Path2TrackInfo(string path, string artworkPrefix)
{
var ti = new TrackInfo();
try
{
var fmd = new FileMetadata(path);
var fi = new FileInformation(path);
ti.Path = path;
ti.FileName = fi.NameWithoutExtension;
ti.MimeType = fmd.MimeType;
ti.FileSize = fi.SizeInBytes;
ti.BitRate = fmd.BitRate;
ti.SampleRate = fmd.SampleRate;
ti.TrackTitle = MetadataUtils.SanitizeTag(fmd.Title.Value);
ti.TrackNumber = MetadataUtils.SafeConvertToLong(fmd.TrackNumber.Value);
ti.TrackCount = MetadataUtils.SafeConvertToLong(fmd.TrackCount.Value);
ti.DiscNumber = MetadataUtils.SafeConvertToLong(fmd.DiscNumber.Value);
ti.DiscCount = MetadataUtils.SafeConvertToLong(fmd.DiscCount.Value);
ti.Duration = Convert.ToInt64(fmd.Duration.TotalMilliseconds);
ti.Year = MetadataUtils.SafeConvertToLong(fmd.Year.Value);
ti.Rating = fmd.Rating.Value;
ti.ArtistName = GetFirstArtist(fmd);
ti.GenreName = GetFirstGenre(fmd);
ti.AlbumTitle = string.IsNullOrWhiteSpace(fmd.Album.Value) ? Defaults.UnknownAlbumString : MetadataUtils.SanitizeTag(fmd.Album.Value);
ti.AlbumArtist = GetFirstAlbumArtist(fmd);
var dummyAlbum = new Album
{
AlbumTitle = ti.AlbumTitle,
AlbumArtist = ti.AlbumArtist
};
IndexerUtils.UpdateAlbumYear(dummyAlbum, MetadataUtils.SafeConvertToLong(fmd.Year.Value));
IndexerUtils.CacheArtwork(dummyAlbum, ti.Path);
ti.AlbumArtworkID = dummyAlbum.ArtworkID;
ti.AlbumArtist = dummyAlbum.AlbumArtist;
ti.AlbumTitle = dummyAlbum.AlbumTitle;
ti.AlbumYear = dummyAlbum.Year;
}
catch (Exception ex)
{
LogClient.Instance.Logger.Error("Error while creating TrackInfo from file '{0}'. Exception: {1}", path, ex.Message);
// Make sure the file can be opened by creating a TrackInfo with some default values
ti = new TrackInfo();
ti.Path = path;
ti.FileName = System.IO.Path.GetFileNameWithoutExtension(path);
ti.ArtistName = Defaults.UnknownArtistString;
ti.GenreName = Defaults.UnknownGenreString;
ti.AlbumTitle = Defaults.UnknownAlbumString;
ti.AlbumArtist = Defaults.UnknownAlbumArtistString;
}
return(ti);
}
private void GetSlotNames(int iinfo, ref VBuffer <ReadOnlyMemory <char> > dst)
{
Host.Assert(0 <= iinfo && iinfo < Infos.Length);
int size = _types[iinfo].VectorSize;
if (size == 0)
{
throw MetadataUtils.ExceptGetMetadata();
}
var editor = VBufferEditor.Create(ref dst, size);
var type = Infos[iinfo].TypeSrc;
if (!type.IsVector)
{
Host.Assert(_types[iinfo].VectorSize == 2);
var columnName = Source.Schema.GetColumnName(Infos[iinfo].Source);
editor.Values[0] = columnName.AsMemory();
editor.Values[1] = (columnName + IndicatorSuffix).AsMemory();
}
else
{
Host.Assert(type.IsKnownSizeVector);
Host.Assert(size == 2 * type.VectorSize);
// REVIEW: Do we need to verify that there is metadata or should we just call GetMetadata?
var typeNames = Source.Schema.GetMetadataTypeOrNull(MetadataUtils.Kinds.SlotNames, Infos[iinfo].Source);
if (typeNames == null || typeNames.VectorSize != type.VectorSize || !typeNames.ItemType.IsText)
{
throw MetadataUtils.ExceptGetMetadata();
}
var names = default(VBuffer <ReadOnlyMemory <char> >);
Source.Schema.GetMetadata(MetadataUtils.Kinds.SlotNames, Infos[iinfo].Source, ref names);
// We both assert and check. If this fails, there is a bug somewhere (possibly in this code
// but more likely in the implementation of Base. On the other hand, we don't want to proceed
// if we've received garbage.
Host.Check(names.Length == type.VectorSize, "Unexpected slot name vector size");
var sb = new StringBuilder();
int slot = 0;
foreach (var kvp in names.Items(all: true))
{
Host.Assert(0 <= slot && slot < size);
Host.Assert(slot % 2 == 0);
sb.Clear();
if (kvp.Value.IsEmpty)
{
sb.Append('[').Append(slot / 2).Append(']');
}
else
{
sb.AppendMemory(kvp.Value);
}
int len = sb.Length;
sb.Append(IndicatorSuffix);
var str = sb.ToString();
editor.Values[slot++] = str.AsMemory().Slice(0, len);
editor.Values[slot++] = str.AsMemory();
}
Host.Assert(slot == size);
}
dst = editor.Commit();
}
public static string GetFirstGenre(FileMetadata fmd)
{
return(string.IsNullOrWhiteSpace(fmd.Genres.Value) ? Defaults.UnknownGenreString : MetadataUtils.PatchID3v23Enumeration(fmd.Genres.Values).FirstNonEmpty(Defaults.UnknownGenreString));
}
public static string GetFirstAlbumArtist(FileMetadata iFileMetadata)
{
return(string.IsNullOrWhiteSpace(iFileMetadata.AlbumArtists.Value) ? Defaults.UnknownAlbumArtistString : MetadataUtils.SanitizeTag(MetadataUtils.PatchID3v23Enumeration(iFileMetadata.AlbumArtists.Values).FirstNonEmpty(Defaults.UnknownAlbumArtistString)));
}
public static void SplitMetadata(string path, ref Track track, ref Album album, ref Artist artist, ref Genre genre)
{
if (!string.IsNullOrEmpty(path))
{
var fmd = new FileMetadata(path);
var fi = new FileInformation(path);
// Track information
track.Path = path;
track.FileName = fi.NameWithoutExtension;
track.Duration = Convert.ToInt64(fmd.Duration.TotalMilliseconds);
track.MimeType = fmd.MimeType;
track.BitRate = fmd.BitRate;
track.SampleRate = fmd.SampleRate;
track.TrackTitle = MetadataUtils.SanitizeTag(fmd.Title.Value);
track.TrackNumber = MetadataUtils.SafeConvertToLong(fmd.TrackNumber.Value);
track.TrackCount = MetadataUtils.SafeConvertToLong(fmd.TrackCount.Value);
track.DiscNumber = MetadataUtils.SafeConvertToLong(fmd.DiscNumber.Value);
track.DiscCount = MetadataUtils.SafeConvertToLong(fmd.DiscCount.Value);
track.Year = MetadataUtils.SafeConvertToLong(fmd.Year.Value);
track.Rating = fmd.Rating.Value;
// Before proceeding, get the available artists
string albumArtist = GetFirstAlbumArtist(fmd);
string trackArtist = GetFirstArtist(fmd); // will be used for the album if no album artist is found
// Album information
album.AlbumTitle = string.IsNullOrWhiteSpace(fmd.Album.Value) ? Defaults.UnknownAlbumString : MetadataUtils.SanitizeTag(fmd.Album.Value);
album.AlbumArtist = (albumArtist == Defaults.UnknownAlbumArtistString ? trackArtist : albumArtist);
album.DateAdded = FileOperations.GetDateCreated(path);
IndexerUtils.UpdateAlbumYear(album, MetadataUtils.SafeConvertToLong(fmd.Year.Value));
// Artist information
artist.ArtistName = trackArtist;
// Genre information
genre.GenreName = GetFirstGenre(fmd);
// Metadata hash
System.Text.StringBuilder sb = new System.Text.StringBuilder();
sb.Append(album.AlbumTitle);
sb.Append(artist.ArtistName);
sb.Append(genre.GenreName);
sb.Append(track.TrackTitle);
sb.Append(track.TrackNumber);
sb.Append(track.Year);
track.MetaDataHash = CryptographyUtils.MD5Hash(sb.ToString());
// File information
track.FileSize = fi.SizeInBytes;
track.DateFileModified = fi.DateModifiedTicks;
track.DateLastSynced = DateTime.Now.Ticks;
}
}
private void updateReadOnlyState()
{
RomanisedArtistTextBox.ReadOnly = MetadataUtils.IsRomanised(ArtistTextBox.Current.Value);
RomanisedTitleTextBox.ReadOnly = MetadataUtils.IsRomanised(TitleTextBox.Current.Value);
}
private void load()
{
var metadata = Beatmap.Metadata;
Children = new[]
{
ArtistTextBox = createTextBox <LabelledTextBox>("Artist",
!string.IsNullOrEmpty(metadata.ArtistUnicode) ? metadata.ArtistUnicode : metadata.Artist),
RomanisedArtistTextBox = createTextBox <LabelledRomanisedTextBox>("Romanised Artist",
!string.IsNullOrEmpty(metadata.Artist) ? metadata.Artist : MetadataUtils.StripNonRomanisedCharacters(metadata.ArtistUnicode)),
Empty(),
TitleTextBox = createTextBox <LabelledTextBox>("Title",
!string.IsNullOrEmpty(metadata.TitleUnicode) ? metadata.TitleUnicode : metadata.Title),
RomanisedTitleTextBox = createTextBox <LabelledRomanisedTextBox>("Romanised Title",
!string.IsNullOrEmpty(metadata.Title) ? metadata.Title : MetadataUtils.StripNonRomanisedCharacters(metadata.ArtistUnicode)),
Empty(),
creatorTextBox = createTextBox <LabelledTextBox>("Creator", metadata.Author.Username),
difficultyTextBox = createTextBox <LabelledTextBox>("Difficulty Name", Beatmap.BeatmapInfo.DifficultyName),
sourceTextBox = createTextBox <LabelledTextBox>("Source", metadata.Source),
tagsTextBox = createTextBox <LabelledTextBox>("Tags", metadata.Tags)
};
foreach (var item in Children.OfType <LabelledTextBox>())
{
item.OnCommit += onCommit;
}
}
static void Main(string[] args)
{
Console.SetWindowSize(150, 40);
PrintWelcomeMessage();
DoOptions();
if (!Directory.Exists(path))
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("osu's song directory was not found. Do you even have osu installed?");
Console.ReadKey();
Environment.Exit(0);
}
// Create the destination directory if it doesn't exist
if (!Directory.Exists(destination))
{
Directory.CreateDirectory(destination);
}
// Goes through every beatmap's directory, gets the mp3 file from it and copies it to the destination folder.
foreach (string dir in GetSongDirectories(path))
{
if (!(File.Exists(GetSongFile(dir, "*.osu")) || File.Exists(GetSongFile(dir, ".mp3"))))
{
continue;
}
reader = new BeatmapFileReader(GetSongFile(dir, "*.osu"));
string file = destination + reader.GetTitle() + ".mp3";
if (!File.Exists(destination + reader.GetTitle() + ".mp3"))
{
if (unicodeTitle)
{
Console.WriteLine("Copied: " + reader.GetArtistUnicode() + " - " + reader.GetTitle());
}
else
{
Console.WriteLine("Copied: " + reader.GetArtist() + " - " + reader.GetTitle());
}
File.Copy(GetSongFile(dir, "*.mp3"), destination + reader.GetTitle() + ".mp3", true);
if (addMetadata)
{
if (metadataUnicode)
{
MetadataUtils.AddMetadata(file, reader.GetArtistUnicode(), reader.GetTitleUnicode());
}
else
{
MetadataUtils.AddMetadata(file, reader.GetArtist(), reader.GetTitle());
}
}
}
}
MessageBox.Show("Finished exporting.", "osu! song exporter", MessageBoxButtons.OK, MessageBoxIcon.Information);
// Opens the folder with the exported songs.
Process.Start(destination);
}
/// <summary>
/// Gets the output <see cref="SchemaShape"/> of the <see cref="IDataView"/> after fitting the calibrator.
/// Fitting the calibrator will add a column named "Probability" to the schema. If you already had such a column, a new one will be added.
/// </summary>
/// <param name="inputSchema">The input <see cref="SchemaShape"/>.</param>
SchemaShape IEstimator <CalibratorTransformer <TICalibrator> > .GetOutputSchema(SchemaShape inputSchema)
{
Action <SchemaShape.Column, string> checkColumnValid = (SchemaShape.Column column, string columnRole) =>
{
if (column.IsValid)
{
if (!inputSchema.TryFindColumn(column.Name, out var outCol))
{
throw Host.ExceptSchemaMismatch(nameof(inputSchema), columnRole, column.Name);
}
if (!column.IsCompatibleWith(outCol))
{
throw Host.ExceptSchemaMismatch(nameof(inputSchema), columnRole, column.Name, column.GetTypeString(), outCol.GetTypeString());
}
}
};
// Check the input schema.
checkColumnValid(ScoreColumn, "score");
checkColumnValid(WeightColumn, "weight");
checkColumnValid(LabelColumn, "label");
// Create the new Probability column.
var outColumns = inputSchema.ToDictionary(x => x.Name);
outColumns[DefaultColumnNames.Probability] = new SchemaShape.Column(DefaultColumnNames.Probability,
SchemaShape.Column.VectorKind.Scalar,
NumberType.R4,
false,
new SchemaShape(MetadataUtils.GetTrainerOutputMetadata(true)));
return(new SchemaShape(outColumns.Values));
}
/// <summary>
/// Gets the edm model. Constructs it if it doesn't exist
/// </summary>
/// <returns>Edm model</param>
public IEdmModel GetModel()
{
if (this.model == null)
{
ConstructableMetadata metadata = new ConstructableMetadata("InMemoryEntities", "Microsoft.Test.Taupo.OData.WCFService");
IEdmComplexType addressType = metadata.AddComplexType("Address", typeof(Address), null, false);
metadata.AddPrimitiveProperty(addressType, "Street", typeof(string));
metadata.AddPrimitiveProperty(addressType, "City", typeof(string));
metadata.AddPrimitiveProperty(addressType, "PostalCode", typeof(string));
IEdmComplexType homeAddressType = metadata.AddComplexType("HomeAddress", typeof(HomeAddress), addressType, false);
metadata.AddPrimitiveProperty(homeAddressType, "HomeNO", typeof(string));
IEdmEntityType personType = metadata.AddEntityType("Person", typeof(Person), null, false, "Microsoft.Test.Taupo.OData.WCFService");
metadata.AddKeyProperty(personType, "PersonID", typeof(Int32));
metadata.AddPrimitiveProperty(personType, "FirstName", typeof(string));
metadata.AddPrimitiveProperty(personType, "LastName", typeof(string));
metadata.AddComplexProperty(personType, "HomeAddress", addressType);
metadata.AddPrimitiveProperty(personType, "Home", typeof(GeographyPoint));
metadata.AddMultiValueProperty(personType, "Numbers", typeof(string));
metadata.AddContainedResourceSetReferenceProperty(personType, "Brother", personType);
metadata.AddContainedResourceReferenceProperty(personType, "Child", personType);
var peopleset = metadata.AddEntitySet("People", personType);
var specialPerson = metadata.AddSingleton("SpecialPerson", personType);
IEdmEntityType schoolType = metadata.AddEntityType("School", typeof(School), null, false);
metadata.AddKeyProperty(schoolType, "SchoolID", typeof(Int32));
var schoolSet = metadata.AddEntitySet("Schools", schoolType);
var studentNavigation = metadata.AddResourceSetReferenceProperty(schoolType, "Student", peopleset, null);
((EdmEntitySet)schoolSet).AddNavigationTarget(studentNavigation, peopleset);
IEdmEntityType customerType = metadata.AddEntityType("Customer", typeof(Customer), personType, false, "Microsoft.Test.Taupo.OData.WCFService");
metadata.AddPrimitiveProperty(customerType, "City", typeof(string));
metadata.AddPrimitiveProperty(customerType, "Birthday", typeof(DateTimeOffset));
metadata.AddPrimitiveProperty(customerType, "TimeBetweenLastTwoOrders", typeof(TimeSpan));
var customerset = metadata.AddEntitySet("Customers", customerType);
var vipCustomer = metadata.AddSingleton("VipCustomer", customerType);
IEdmEntityType employeeType = metadata.AddEntityType("Employee", typeof(Employee), personType, false, "Microsoft.Test.Taupo.OData.WCFService", true);
metadata.AddPrimitiveProperty(employeeType, "DateHired", typeof(DateTimeOffset));
metadata.AddPrimitiveProperty(employeeType, "Office", typeof(GeographyPoint));
var employeeset = metadata.AddEntitySet("Employees", employeeType);
var boss = metadata.AddSingleton("Boss", employeeType);
IEdmEntityType productType = metadata.AddEntityType("Product", typeof(Product), null, false, "Microsoft.Test.Taupo.OData.WCFService");
metadata.AddKeyProperty(productType, "ProductID", typeof(Int32));
metadata.AddPrimitiveProperty(productType, "Name", typeof(string));
metadata.AddPrimitiveProperty(productType, "QuantityPerUnit", typeof(string));
metadata.AddPrimitiveProperty(productType, "UnitPrice", typeof(float));
metadata.AddPrimitiveProperty(productType, "QuantityInStock", typeof(Int32));
metadata.AddPrimitiveProperty(productType, "Discontinued", typeof(bool));
metadata.AddComplexProperty(productType, "ManufactureAddresss", addressType, true);
var productset = metadata.AddEntitySet("Products", productType);
var specialProduct = metadata.AddSingleton("SpecialProduct", productType);
IEdmEntityType orderType = metadata.AddEntityType("Order", typeof(Order), null, false, "Microsoft.Test.Taupo.OData.WCFService");
var orderset = metadata.AddEntitySet("Orders", orderType);
metadata.AddKeyProperty(orderType, "OrderID", typeof(Int32));
metadata.AddPrimitiveProperty(orderType, "CustomerID", typeof(Int32));
metadata.AddPrimitiveProperty(orderType, "EmployeeID", typeof(Int32?));
metadata.AddPrimitiveProperty(orderType, "OrderDate", typeof(DateTimeOffset));
metadata.AddResourceReferenceProperty(orderType, "LoggedInEmployee", employeeset, null);
metadata.AddResourceReferenceProperty(orderType, "CustomerForOrder", customerset, null);
var specialOrder = metadata.AddSingleton("SpecialOrder", orderType);
metadata.AddContainedResourceReferenceProperty(personType, "FirstOrder", orderType);
IEdmEntityType orderDetailType = metadata.AddEntityType("OrderDetail", typeof(OrderDetail), null, false, "Microsoft.Test.Taupo.OData.WCFService");
metadata.AddKeyProperty(orderDetailType, "OrderID", typeof(Int32));
metadata.AddKeyProperty(orderDetailType, "ProductID", typeof(Int32));
metadata.AddPrimitiveProperty(orderDetailType, "OrderPlaced", typeof(DateTimeOffset));
metadata.AddPrimitiveProperty(orderDetailType, "Quantity", typeof(Int32));
metadata.AddPrimitiveProperty(orderDetailType, "UnitPrice", typeof(float));
var productOrderedNavigation = metadata.AddResourceReferenceProperty(orderDetailType, "ProductOrdered", productset, null);
var associatedOrderNavigation = metadata.AddResourceReferenceProperty(orderDetailType, "AssociatedOrder", orderset, null);
var orderdetailsSet = metadata.AddEntitySet("OrderDetails", orderDetailType);
// Edm.Duration
IEdmEntityType durationInKeyType = metadata.AddEntityType("DurationInKey", typeof(DurationInKey), null, false, "Microsoft.Test.Taupo.OData.WCFService");
metadata.AddKeyProperty(durationInKeyType, "Id", typeof(TimeSpan));
metadata.AddEntitySet("DurationInKeys", durationInKeyType);
// FUNCTIONS
// Function that binds to single order
metadata.AddFunctionAndFunctionImport("GetOrderRate", orderType.ToTypeReference(), MetadataUtils.GetPrimitiveTypeReference(typeof(Int32)), null, true);
//Function that binds to a single order and returns a single order
metadata.AddFunction("GetNextOrder", orderType.ToTypeReference(), orderType.ToTypeReference(), true, new EdmPathExpression("bindingparameter"), true);
// Function that returns a set of orders
var collectionOrders = new EdmCollectionType(orderType.ToTypeReference()).ToTypeReference();
metadata.AddFunction("OrdersWithMoreThanTwoItems", collectionOrders, collectionOrders, true, new EdmPathExpression("bindingparameter"), true /*iscomposable*/);
var overload1Function = metadata.AddFunction("OrdersWithMoreThanTwoItems", collectionOrders, collectionOrders, true, new EdmPathExpression("bindingparameter"), true /*iscomposable*/);
overload1Function.AddParameter("IntParameter", MetadataUtils.GetPrimitiveTypeReference(typeof(Int32)));
var overload2Function = metadata.AddFunction("OrdersWithMoreThanTwoItems", collectionOrders, collectionOrders, true, new EdmPathExpression("bindingparameter"), true /*iscomposable*/);
overload2Function.AddParameter("IntParameter", MetadataUtils.GetPrimitiveTypeReference(typeof(Int32)));
overload2Function.AddParameter("EntityParameter", productType.ToTypeReference());
var collectionCustomers = new EdmCollectionType(customerType.ToTypeReference()).ToTypeReference();
var customersInCityFunction = metadata.AddFunction("InCity", collectionCustomers, collectionCustomers, true, new EdmPathExpression("bindingparameter"), true /*iscomposable*/);
customersInCityFunction.AddParameter("City", MetadataUtils.GetPrimitiveTypeReference(typeof(String)));
var customersWithinFunction = metadata.AddFunction("Within", collectionCustomers, collectionCustomers, true, new EdmPathExpression("bindingparameter"), true /*iscomposable*/);
customersWithinFunction.AddParameter("Location", MetadataUtils.GetPrimitiveTypeReference(typeof(GeographyPoint)));
customersWithinFunction.AddParameter("Address", addressType.ToTypeReference(true /*nullable*/));
customersWithinFunction.AddParameter("Distance", MetadataUtils.GetPrimitiveTypeReference(typeof(Double)));
customersWithinFunction.AddParameter("ArbitraryInt", MetadataUtils.GetPrimitiveTypeReference(typeof(Int32)));
customersWithinFunction.AddParameter("DateTimeOffset", MetadataUtils.GetPrimitiveTypeReference(typeof(DateTimeOffset?)));
customersWithinFunction.AddParameter("Byte", MetadataUtils.GetPrimitiveTypeReference(typeof(Byte)));
customersWithinFunction.AddParameter("LineString", MetadataUtils.GetPrimitiveTypeReference(typeof(GeometryLineString)));
var withinFunction = metadata.AddFunction("Within", customerType.ToTypeReference(), MetadataUtils.GetPrimitiveTypeReference(typeof(bool)), true, null, true /*iscomposable*/);
withinFunction.AddParameter("Location", addressType.ToTypeReference());
withinFunction.AddParameter("Distance", MetadataUtils.GetPrimitiveTypeReference(typeof(Int32)));
var withinFunction2 = metadata.AddFunction("Within", customerType.ToTypeReference(), MetadataUtils.GetPrimitiveTypeReference(typeof(bool)), true, null, true /*iscomposable*/);
withinFunction2.AddParameter("Distance", MetadataUtils.GetPrimitiveTypeReference(typeof(Int32)));
metadata.AddFunction("GetChild", personType.ToTypeReference(), personType.ToTypeReference(), true, new EdmPathExpression("bindingparameter/Child"), true /*iscomposable*/);
metadata.AddAction("GetBrothers", personType.ToTypeReference(), new EdmCollectionTypeReference(new EdmCollectionType(personType.ToTypeReference())), true, new EdmPathExpression("bindingparameter/Child"));
//Unbound Functions
var lotsofOrders = metadata.AddFunctionAndFunctionImport("HasLotsOfOrders",
null,
MetadataUtils.GetPrimitiveTypeReference(typeof(bool)),
null,
false /*isBindable*/);
lotsofOrders.Function.AsEdmFunction().AddParameter("Person", personType.ToTypeReference());
metadata.AddFunctionAndFunctionImport("HowManyPotatoesEaten", null, MetadataUtils.GetPrimitiveTypeReference(typeof(Int32)), null, false);
metadata.AddFunctionAndFunctionImport("QuoteOfTheDay", null, MetadataUtils.GetPrimitiveTypeReference(typeof(string)), null, false);
// ACTIONS
var action1 = metadata.AddAction("ChangeAddress", personType.ToTypeReference(), null /*returnType*/, true /*isbound*/, null /*entitySetPathExpression*/);
action1.AddParameter(new EdmOperationParameter(action1, "Street", MetadataUtils.GetPrimitiveTypeReference(typeof(string))));
action1.AddParameter(new EdmOperationParameter(action1, "City", MetadataUtils.GetPrimitiveTypeReference(typeof(string))));
action1.AddParameter(new EdmOperationParameter(action1, "PostalCode", MetadataUtils.GetPrimitiveTypeReference(typeof(string))));
metadata.AddActionImport("ChangeAddress", action1, null /*entitySet*/);
// Unbound action with no parameters
var getRecentCustomersAction = metadata.AddAction("GetRecentCustomers", null /*boundType*/, new EdmCollectionTypeReference(new EdmCollectionType(orderType.ToTypeReference())), false /*isbound*/, null /*entitySetPathExpression*/);
metadata.AddActionImport("GetRecentCustomers", getRecentCustomersAction, orderset);
//Adding order details navigation property to order.
var orderDetailNavigation = metadata.AddResourceSetReferenceProperty(orderType, "OrderDetails", orderdetailsSet, null);
//Adding orders navigation to Customer.
var ordersNavigation = metadata.AddResourceSetReferenceProperty(customerType, "Orders", orderset, null);
((EdmEntitySet)customerset).AddNavigationTarget(ordersNavigation, orderset);
//Adding parent navigation to person
metadata.AddResourceSetReferenceProperty(personType, "Parent", null, personType);
//Since the people set can contain a customer we need to include the target for that navigation in the people set.
((EdmEntitySet)peopleset).AddNavigationTarget(ordersNavigation, orderset);
//Since the OrderSet can contain a OrderDetail we need to include the target for that navigation in the order set.
((EdmEntitySet)orderset).AddNavigationTarget(orderDetailNavigation, orderdetailsSet);
//Since the OrderDetailSet can contain a AssociatedOrder we need to include the target for that navigation in the orderdetail set.
((EdmEntitySet)orderdetailsSet).AddNavigationTarget(associatedOrderNavigation, orderset);
//Since the OrderDetailSet can contain a ProductOrdered we need to include the target for that navigation in the orderdetail set.
((EdmEntitySet)orderdetailsSet).AddNavigationTarget(productOrderedNavigation, productset);
((EdmSingleton)specialOrder).AddNavigationTarget(orderDetailNavigation, orderdetailsSet);
((EdmSingleton)specialPerson).AddNavigationTarget(ordersNavigation, orderset);
this.model = metadata;
}
return(this.model);
}
private void HashTestCore <T>(T val, PrimitiveType type, uint expected, uint expectedOrdered, uint expectedOrdered3)
{
const int bits = 10;
var builder = new MetadataBuilder();
builder.AddPrimitiveValue("Foo", type, val);
var inRow = MetadataUtils.MetadataAsRow(builder.GetMetadata());
// First do an unordered hash.
var info = new HashingTransformer.ColumnInfo("Foo", "Bar", hashBits: bits);
var xf = new HashingTransformer(Env, new[] { info });
var mapper = xf.GetRowToRowMapper(inRow.Schema);
mapper.OutputSchema.TryGetColumnIndex("Bar", out int outCol);
var outRow = mapper.GetRow(inRow, c => c == outCol);
var getter = outRow.GetGetter <uint>(outCol);
uint result = 0;
getter(ref result);
Assert.Equal(expected, result);
// Next do an ordered hash.
info = new HashingTransformer.ColumnInfo("Foo", "Bar", hashBits: bits, ordered: true);
xf = new HashingTransformer(Env, new[] { info });
mapper = xf.GetRowToRowMapper(inRow.Schema);
mapper.OutputSchema.TryGetColumnIndex("Bar", out outCol);
outRow = mapper.GetRow(inRow, c => c == outCol);
getter = outRow.GetGetter <uint>(outCol);
getter(ref result);
Assert.Equal(expectedOrdered, result);
// Next build up a vector to make sure that hashing is consistent between scalar values
// at least in the first position, and in the unordered case, the last position.
const int vecLen = 5;
var denseVec = new VBuffer <T>(vecLen, Utils.CreateArray(vecLen, val));
builder = new MetadataBuilder();
builder.Add("Foo", new VectorType(type, vecLen), (ref VBuffer <T> dst) => denseVec.CopyTo(ref dst));
inRow = MetadataUtils.MetadataAsRow(builder.GetMetadata());
info = new HashingTransformer.ColumnInfo("Foo", "Bar", hashBits: bits, ordered: false);
xf = new HashingTransformer(Env, new[] { info });
mapper = xf.GetRowToRowMapper(inRow.Schema);
mapper.OutputSchema.TryGetColumnIndex("Bar", out outCol);
outRow = mapper.GetRow(inRow, c => c == outCol);
var vecGetter = outRow.GetGetter <VBuffer <uint> >(outCol);
VBuffer <uint> vecResult = default;
vecGetter(ref vecResult);
Assert.Equal(vecLen, vecResult.Length);
// They all should equal this in this case.
Assert.All(vecResult.DenseValues(), v => Assert.Equal(expected, v));
// Now do ordered with the dense vector.
info = new HashingTransformer.ColumnInfo("Foo", "Bar", hashBits: bits, ordered: true);
xf = new HashingTransformer(Env, new[] { info });
mapper = xf.GetRowToRowMapper(inRow.Schema);
mapper.OutputSchema.TryGetColumnIndex("Bar", out outCol);
outRow = mapper.GetRow(inRow, c => c == outCol);
vecGetter = outRow.GetGetter <VBuffer <uint> >(outCol);
vecGetter(ref vecResult);
Assert.Equal(vecLen, vecResult.Length);
Assert.Equal(expectedOrdered, vecResult.GetItemOrDefault(0));
Assert.Equal(expectedOrdered3, vecResult.GetItemOrDefault(3));
Assert.All(vecResult.DenseValues(), v => Assert.True((v == 0) == (expectedOrdered == 0)));
// Let's now do a sparse vector.
var sparseVec = new VBuffer <T>(10, 3, Utils.CreateArray(3, val), new[] { 0, 3, 7 });
builder = new MetadataBuilder();
builder.Add("Foo", new VectorType(type, vecLen), (ref VBuffer <T> dst) => sparseVec.CopyTo(ref dst));
inRow = MetadataUtils.MetadataAsRow(builder.GetMetadata());
info = new HashingTransformer.ColumnInfo("Foo", "Bar", hashBits: bits, ordered: false);
xf = new HashingTransformer(Env, new[] { info });
mapper = xf.GetRowToRowMapper(inRow.Schema);
mapper.OutputSchema.TryGetColumnIndex("Bar", out outCol);
outRow = mapper.GetRow(inRow, c => c == outCol);
vecGetter = outRow.GetGetter <VBuffer <uint> >(outCol);
vecGetter(ref vecResult);
Assert.Equal(10, vecResult.Length);
Assert.Equal(expected, vecResult.GetItemOrDefault(0));
Assert.Equal(expected, vecResult.GetItemOrDefault(3));
Assert.Equal(expected, vecResult.GetItemOrDefault(7));
info = new HashingTransformer.ColumnInfo("Foo", "Bar", hashBits: bits, ordered: true);
xf = new HashingTransformer(Env, new[] { info });
mapper = xf.GetRowToRowMapper(inRow.Schema);
mapper.OutputSchema.TryGetColumnIndex("Bar", out outCol);
outRow = mapper.GetRow(inRow, c => c == outCol);
vecGetter = outRow.GetGetter <VBuffer <uint> >(outCol);
vecGetter(ref vecResult);
Assert.Equal(10, vecResult.Length);
Assert.Equal(expectedOrdered, vecResult.GetItemOrDefault(0));
Assert.Equal(expectedOrdered3, vecResult.GetItemOrDefault(3));
}
private bool ProcessTrack(Track track, SQLiteConnection conn)
{
bool processingSuccessful = false;
var newTrackStatistic = new TrackStatistic();
var newAlbum = new Album();
var newArtist = new Artist();
var newGenre = new Genre();
try
{
MetadataUtils.SplitMetadata(track.Path, ref track, ref newTrackStatistic, ref newAlbum, ref newArtist, ref newGenre);
processingSuccessful = true;
}
catch (Exception ex)
{
processingSuccessful = false;
LogClient.Error("Error while retrieving tag information for file {0}. File not added to the database. Exception: {1}", track.Path, ex.Message);
}
if (processingSuccessful)
{
// Check if such TrackStatistic already exists in the database
if (!this.cache.HasCachedTrackStatistic(newTrackStatistic))
{
// If not, add it.
conn.Insert(newTrackStatistic);
}
// Check if such Artist already exists in the database
if (!this.cache.HasCachedArtist(ref newArtist))
{
// If not, add it.
conn.Insert(newArtist);
}
// Check if such Genre already exists in the database
if (!this.cache.HasCachedGenre(ref newGenre))
{
// If not, add it.
conn.Insert(newGenre);
}
// Check if such Album already exists in the database
if (!this.cache.HasCachedAlbum(ref newAlbum))
{
// If Not, add it.
conn.Insert(newAlbum);
}
else
{
// Make sure the Year of the existing album is updated
Album dbAlbum = conn.Table <Album>().Where((a) => a.AlbumID.Equals(newAlbum.AlbumID)).FirstOrDefault();
if (dbAlbum != null)
{
dbAlbum.Year = newAlbum.Year;
conn.Update(dbAlbum);
}
}
track.AlbumID = newAlbum.AlbumID;
track.ArtistID = newArtist.ArtistID;
track.GenreID = newGenre.GenreID;
}
return(processingSuccessful);
}
public OnlineGradientDescentTrainer(IHostEnvironment env, Arguments args)
: base(args, env, UserNameValue, MakeLabelColumn(args.LabelColumn))
{
LossFunction = args.LossFunction.CreateComponent(env);
_outputColumns = new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
};
}
private BoundColumn MakeColumn(Schema inputSchema, int iinfo)
{
Contracts.AssertValue(inputSchema);
Contracts.Assert(0 <= iinfo && iinfo < _parent._columns.Length);
ColumnType itemType = null;
int[] sources = new int[_parent._columns[iinfo].Inputs.Count];
// Go through the columns, and establish the following:
// - indices of input columns in the input schema. Throw if they are not there.
// - output type. Throw if the types of inputs are not the same.
// - how many slots are there in the output vector (or variable). Denoted by totalSize.
// - total size of CategoricalSlotRanges metadata, if present. Denoted by catCount.
// - whether the column is normalized.
// It is true when ALL inputs are normalized (and of numeric type).
// - whether the column has slot names.
// It is true if ANY input is a scalar, or has slot names.
// - whether the column has categorical slot ranges.
// It is true if ANY input has this metadata.
int totalSize = 0;
int catCount = 0;
bool isNormalized = true;
bool hasSlotNames = false;
bool hasCategoricals = false;
for (int i = 0; i < _parent._columns[iinfo].Inputs.Count; i++)
{
var(srcName, srcAlias) = _parent._columns[iinfo].Inputs[i];
if (!inputSchema.TryGetColumnIndex(srcName, out int srcCol))
{
throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", srcName);
}
sources[i] = srcCol;
var curType = inputSchema[srcCol].Type;
VectorType curVectorType = curType as VectorType;
ColumnType currentItemType = curVectorType?.ItemType ?? curType;
int currentValueCount = curVectorType?.Size ?? 1;
if (itemType == null)
{
itemType = currentItemType;
totalSize = currentValueCount;
}
else if (currentItemType.Equals(itemType))
{
// If any one input is variable length, then the output is variable length.
if (totalSize == 0 || currentValueCount == 0)
{
totalSize = 0;
}
else
{
totalSize += currentValueCount;
}
}
else
{
throw Host.ExceptSchemaMismatch(nameof(inputSchema), "input", srcName, itemType.ToString(), curType.ToString());
}
if (isNormalized && !inputSchema[srcCol].IsNormalized())
{
isNormalized = false;
}
if (MetadataUtils.TryGetCategoricalFeatureIndices(inputSchema, srcCol, out int[] typeCat))
/// <summary>
/// Write the items in a MultiValue in ATOM format.
/// </summary>
/// <param name="writer">The <see cref="XmlWriter"/> to write to.</param>
/// <param name="metadata">The metadata provider to use or null if no metadata is available.</param>
/// <param name="multiValue">The MultiValue to write.</param>
/// <param name="resourcePropertyType">The resource type of the multi value (or null if not metadata is available).</param>
/// <param name="isOpenPropertyType">True if the type name belongs to an open property.</param>
/// <param name="isWritingCollection">True if we are writing a collection instead of an entry.</param>
/// <param name="version">The protocol version used for writing.</param>
/// <param name="epmValueCache">Cache of values used in EPM so that we avoid multiple enumerations of properties/items. (can be null)</param>
/// <param name="epmSourcePathSegment">The EPM source path segment which points to the multivalue property we're writing. (can be null)</param>
private static void WriteMultiValue(
XmlWriter writer,
DataServiceMetadataProviderWrapper metadata,
ODataMultiValue multiValue,
ResourceType resourcePropertyType,
bool isOpenPropertyType,
bool isWritingCollection,
ODataVersion version,
EpmValueCache epmValueCache,
EpmSourcePathSegment epmSourcePathSegment)
{
Debug.Assert(multiValue != null, "multiValue != null");
string typeName = multiValue.TypeName;
// resolve the type name to the resource type; if no type name is specified we will use the
// type inferred from metadata
MultiValueResourceType multiValueType = (MultiValueResourceType)MetadataUtils.ResolveTypeName(metadata, resourcePropertyType, ref typeName, ResourceTypeKind.MultiValue, isOpenPropertyType);
if (typeName != null)
{
WritePropertyTypeAttribute(writer, typeName);
}
ResourceType expectedItemType = multiValueType == null ? null : multiValueType.ItemType;
IEnumerable items = EpmValueCache.GetMultiValueItems(epmValueCache, epmSourcePathSegment, multiValue, true);
if (items != null)
{
foreach (object itemValue in items)
{
object item;
EpmMultiValueItemCache epmItemCache = itemValue as EpmMultiValueItemCache;
if (epmItemCache != null)
{
item = epmItemCache.ItemValue;
}
else
{
item = itemValue;
}
ValidationUtils.ValidateMultiValueItem(item);
writer.WriteStartElement(AtomConstants.ODataNamespacePrefix, AtomConstants.ODataMultiValueItemElementName, AtomConstants.ODataNamespace);
ODataComplexValue complexValue = item as ODataComplexValue;
if (complexValue != null)
{
WriteComplexValue(writer, metadata, complexValue, expectedItemType, false, isWritingCollection, version, epmItemCache, epmSourcePathSegment);
}
else
{
ODataMultiValue multiValueItem = item as ODataMultiValue;
if (multiValueItem != null)
{
throw new ODataException(Strings.ODataWriter_NestedMultiValuesAreNotSupported);
}
else
{
AtomValueUtils.WritePrimitiveValue(writer, item, expectedItemType);
}
}
writer.WriteEndElement();
}
}
}
private SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
{
bool success = inputSchema.TryFindColumn(LabelColumn.Name, out var labelCol);
Contracts.Assert(success);
return(new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())),
new SchemaShape.Column(DefaultColumnNames.Probability, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata(true))),
new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, BoolType.Instance, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
});
}
public AveragedPerceptronTrainer(IHostEnvironment env, Arguments args)
: base(args, env, UserNameValue, TrainerUtils.MakeBoolScalarLabel(args.LabelColumn))
{
_args = args;
LossFunction = _args.LossFunction.CreateComponent(env);
_outputColumns = new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())),
new SchemaShape.Column(DefaultColumnNames.Probability, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata(true))),
new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, BoolType.Instance, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
};
}
protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
{
return(new[]
{
// REVIEW AP is currently not calibrating. Add the probability column after fixing the behavior.
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())),
new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, BoolType.Instance, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
});
}
/// <summary>
/// Writes a single instance annotation in ATOM format.
/// </summary>
/// <param name="instanceAnnotation">The instance annotation to write.</param>
internal void WriteInstanceAnnotation(AtomInstanceAnnotation instanceAnnotation)
{
Debug.Assert(instanceAnnotation != null, "instanceAnnotation != null");
Debug.Assert(!string.IsNullOrEmpty(instanceAnnotation.TermName), "!string.IsNullOrEmpty(instanceAnnotation.TermName)");
if (this.MessageWriterSettings.ShouldSkipAnnotation(instanceAnnotation.TermName))
{
return;
}
IEdmTypeReference expectedType = MetadataUtils.LookupTypeOfValueTerm(instanceAnnotation.TermName, this.Model);
this.WriteInstanceAnnotationStart(instanceAnnotation);
ODataPrimitiveValue primitiveValue = instanceAnnotation.Value as ODataPrimitiveValue;
if (primitiveValue != null)
{
this.WritePrimitiveInstanceAnnotationValue(primitiveValue, expectedType);
}
else
{
ODataComplexValue complexValue = instanceAnnotation.Value as ODataComplexValue;
if (complexValue != null)
{
this.WriteComplexValue(
complexValue,
expectedType,
/*isOpenPropertyType*/ false,
/*isWritingCollection*/ false,
/*beforeValueAction*/ null,
/*afterValueAction*/ null,
this.CreateDuplicatePropertyNamesChecker(),
/*collectionValidator*/ null,
/*projectedProperties*/ null);
}
else
{
ODataCollectionValue collectionValue = instanceAnnotation.Value as ODataCollectionValue;
if (collectionValue != null)
{
this.WriteCollectionValue(
collectionValue,
expectedType,
/*isOpenPropertyType*/ false,
/*isWritingCollection*/ false);
}
else
{
// Note that the ODataInstanceAnnotation constructor validates that the value is never an ODataStreamReferenceValue
Debug.Assert(instanceAnnotation.Value is ODataNullValue, "instanceAnnotation.Value is ODataNullValue");
if (expectedType != null && !expectedType.IsNullable)
{
throw new ODataException(ODataErrorStrings.ODataAtomPropertyAndValueSerializer_NullValueNotAllowedForInstanceAnnotation(instanceAnnotation.TermName, expectedType.FullName()));
}
this.WriteNullAttribute();
}
}
}
this.WriteInstanceAnnotationEnd();
}
protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
{
return(new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())),
new SchemaShape.Column(DefaultColumnNames.Probability, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata(true))),
new SchemaShape.Column(DefaultColumnNames.PredictedLabel, SchemaShape.Column.VectorKind.Scalar, BoolType.Instance, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
});
}
private void ProcessTrack(Track track, SQLiteConnection conn)
{
var newTrackStatistic = new TrackStatistic();
var newAlbum = new Album()
{
NeedsIndexing = 1
}; // Make sure album art gets indexed for this album
var newArtist = new Artist();
var newGenre = new Genre();
try
{
MetadataUtils.SplitMetadata(track.Path, ref track, ref newTrackStatistic, ref newAlbum, ref newArtist, ref newGenre);
// Check if such TrackStatistic already exists in the database
if (!this.cache.HasCachedTrackStatistic(newTrackStatistic))
{
// If not, add it.
conn.Insert(newTrackStatistic);
}
// Check if such Artist already exists in the database
if (!this.cache.HasCachedArtist(ref newArtist))
{
// If not, add it.
conn.Insert(newArtist);
}
// Check if such Genre already exists in the database
if (!this.cache.HasCachedGenre(ref newGenre))
{
// If not, add it.
conn.Insert(newGenre);
}
// Check if such Album already exists in the database
if (!this.cache.HasCachedAlbum(ref newAlbum))
{
// If Not, add it.
conn.Insert(newAlbum);
}
else
{
// Make sure the Year of the existing album is updated
Album dbAlbum = conn.Table <Album>().Where((a) => a.AlbumID.Equals(newAlbum.AlbumID)).FirstOrDefault();
if (dbAlbum != null)
{
dbAlbum.Year = newAlbum.Year;
// A track from this album has changed, so make sure album art gets re-indexed.
dbAlbum.NeedsIndexing = 1;
conn.Update(dbAlbum);
}
}
track.IndexingSuccess = 1;
track.AlbumID = newAlbum.AlbumID;
track.ArtistID = newArtist.ArtistID;
track.GenreID = newGenre.GenreID;
}
catch (Exception ex)
{
// When updating tracks: for tracks that were indexed successfully in the past and had IndexingSuccess = 1
track.IndexingSuccess = 0;
track.AlbumID = 0;
track.ArtistID = 0;
track.GenreID = 0;
track.IndexingFailureReason = ex.Message;
LogClient.Error("Error while retrieving tag information for file {0}. Exception: {1}", track.Path, ex.Message);
}
return;
}
protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
{
return(new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberType.R4, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata()))
});
}
private SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema)
{
bool success = inputSchema.TryFindColumn(LabelName, out var labelCol);
Contracts.Assert(success);
return(new[]
{
new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberDataViewType.Single, false, new SchemaShape(MetadataUtils.GetTrainerOutputMetadata())),
});
}
