protected override BeatmapSetInfo CreateModel(ArchiveReader reader)
{
// let's make sure there are actually .osu files to import.
string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu"));
if (string.IsNullOrEmpty(mapName))
{
// Todo: This is temporary for debugging purposes
var files = reader.Filenames.ToList();
throw new InvalidOperationException($"No beatmap files found in this beatmap archive. Files ({files.Count}): {string.Join(", ", files)}");
}
Beatmap beatmap;
using (var stream = new StreamReader(reader.GetStream(mapName)))
beatmap = Decoder.GetDecoder <Beatmap>(stream).Decode(stream);
return(new BeatmapSetInfo
{
OnlineBeatmapSetID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID,
Beatmaps = new List <BeatmapInfo>(),
Hash = computeBeatmapSetHash(reader),
Metadata = beatmap.Metadata
});
}
protected override BeatmapSetInfo?CreateModel(ArchiveReader reader)
{
// let's make sure there are actually .osu files to import.
string?mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrEmpty(mapName))
{
Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database);
return(null);
}
Beatmap beatmap;
using (var stream = new LineBufferedReader(reader.GetStream(mapName)))
{
if (stream.PeekLine() == null)
{
Logger.Log($"No content found in first .osu file of beatmap archive ({reader.Name} / {mapName})", LoggingTarget.Database);
return(null);
}
beatmap = Decoder.GetDecoder <Beatmap>(stream).Decode(stream);
}
return(new BeatmapSetInfo
{
OnlineID = beatmap.BeatmapInfo.BeatmapSet?.OnlineID ?? -1,
// Metadata = beatmap.Metadata,
DateAdded = DateTimeOffset.UtcNow
});
}
/// <summary>
/// Create all required <see cref="BeatmapInfo"/>s for the provided archive.
/// </summary>
private List <BeatmapInfo> createBeatmapDifficulties(ArchiveReader reader)
{
var beatmapInfos = new List <BeatmapInfo>();
foreach (var name in reader.Filenames.Where(f => f.EndsWith(".osu")))
{
using (var raw = reader.GetStream(name))
using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit
using (var sr = new StreamReader(ms))
{
raw.CopyTo(ms);
ms.Position = 0;
var decoder = Decoder.GetDecoder <Beatmap>(sr);
IBeatmap beatmap = decoder.Decode(sr);
beatmap.BeatmapInfo.Path = name;
beatmap.BeatmapInfo.Hash = ms.ComputeSHA2Hash();
beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash();
var ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID);
beatmap.BeatmapInfo.Ruleset = ruleset;
// TODO: this should be done in a better place once we actually need to dynamically update it.
beatmap.BeatmapInfo.StarDifficulty = ruleset?.CreateInstance().CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating ?? 0;
beatmapInfos.Add(beatmap.BeatmapInfo);
}
}
return(beatmapInfos);
}
/// <summary>
/// Create a SHA-2 hash from the provided archive based on file content of all files matching <see cref="HashableFileTypes"/>.
/// </summary>
private string computeHash(ArchiveReader reader)
{
// for now, concatenate all .osu files in the set to create a unique hash.
MemoryStream hashable = new MemoryStream();
foreach (string file in reader.Filenames.Where(f => HashableFileTypes.Any(f.EndsWith)))
{
using (Stream s = reader.GetStream(file))
s.CopyTo(hashable);
}
return(hashable.Length > 0 ? hashable.ComputeSHA2Hash() : null);
}
/// <summary>
/// Create a SHA-2 hash from the provided archive based on contained beatmap (.osu) file content.
/// </summary>
private string computeBeatmapSetHash(ArchiveReader reader)
{
// for now, concatenate all .osu files in the set to create a unique hash.
MemoryStream hashable = new MemoryStream();
foreach (string file in reader.Filenames.Where(f => f.EndsWith(".osu")))
{
using (Stream s = reader.GetStream(file))
s.CopyTo(hashable);
}
return(hashable.ComputeSHA2Hash());
}
/// <summary>
/// Create all required <see cref="BeatmapInfo"/>s for the provided archive.
/// </summary>
private List <BeatmapInfo> createBeatmapDifficulties(BeatmapSetInfo model, ArchiveReader reader)
{
var beatmapInfos = new List <BeatmapInfo>();
foreach (var name in reader.Filenames.Where(f => f.EndsWith(".osu")))
{
using (var raw = reader.GetStream(name))
using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit
using (var sr = new StreamReader(ms))
{
raw.CopyTo(ms);
ms.Position = 0;
var decoder = Decoder.GetDecoder <Beatmap>(sr);
IBeatmap beatmap = decoder.Decode(sr);
beatmap.BeatmapInfo.Path = name;
beatmap.BeatmapInfo.Hash = ms.ComputeSHA2Hash();
beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash();
// ensure we have the same online set ID as the set itself.
beatmap.BeatmapInfo.OnlineBeatmapSetID = model.OnlineBeatmapSetID;
beatmap.BeatmapInfo.Metadata.OnlineBeatmapSetID = model.OnlineBeatmapSetID;
// check that no existing beatmap exists that is imported with the same online beatmap ID. if so, give it precedence.
if (beatmap.BeatmapInfo.OnlineBeatmapID.HasValue && QueryBeatmap(b => b.OnlineBeatmapID.Value == beatmap.BeatmapInfo.OnlineBeatmapID.Value) != null)
{
beatmap.BeatmapInfo.OnlineBeatmapID = null;
}
RulesetInfo ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID);
beatmap.BeatmapInfo.Ruleset = ruleset;
if (ruleset != null)
{
// TODO: this should be done in a better place once we actually need to dynamically update it.
var converted = new DummyConversionBeatmap(beatmap).GetPlayableBeatmap(ruleset);
beatmap.BeatmapInfo.StarDifficulty = ruleset.CreateInstance().CreateDifficultyCalculator(converted).Calculate();
}
else
{
beatmap.BeatmapInfo.StarDifficulty = 0;
}
beatmapInfos.Add(beatmap.BeatmapInfo);
}
}
return(beatmapInfos);
}
protected override ScoreInfo?CreateModel(ArchiveReader archive)
{
using (var stream = archive.GetStream(archive.Filenames.First(f => f.EndsWith(".osr", StringComparison.OrdinalIgnoreCase))))
{
try
{
return(new DatabasedLegacyScoreDecoder(rulesets, beatmaps()).Parse(stream).ScoreInfo);
}
catch (LegacyScoreDecoder.BeatmapNotFoundException e)
{
Logger.Log(e.Message, LoggingTarget.Information, LogLevel.Error);
return(null);
}
}
}
private List <RealmNamedFileUsage> createFileInfos(ArchiveReader reader, RealmFileStore files, Realm realm)
{
var fileInfos = new List <RealmNamedFileUsage>();
// import files to manager
foreach (var filenames in getShortenedFilenames(reader))
{
using (Stream s = reader.GetStream(filenames.original))
{
var item = new RealmNamedFileUsage(files.Add(s, realm), filenames.shortened);
fileInfos.Add(item);
}
}
return(fileInfos);
}
private string computeHashFast(ArchiveReader reader)
{
MemoryStream hashable = new MemoryStream();
foreach (string?file in reader.Filenames.Where(f => HashableFileTypes.Any(ext => f.EndsWith(ext, StringComparison.OrdinalIgnoreCase))).OrderBy(f => f))
{
using (Stream s = reader.GetStream(file))
s.CopyTo(hashable);
}
if (hashable.Length > 0)
{
return(hashable.ComputeSHA2Hash());
}
return(reader.Name.ComputeSHA2Hash());
}
/// <summary>
/// Create all required <see cref="FileInfo"/>s for the provided archive, adding them to the global file store.
/// </summary>
private List <BeatmapSetFileInfo> createFileInfos(ArchiveReader reader, FileStore files)
{
List <BeatmapSetFileInfo> fileInfos = new List <BeatmapSetFileInfo>();
// import files to manager
foreach (string file in reader.Filenames)
{
using (Stream s = reader.GetStream(file))
fileInfos.Add(new BeatmapSetFileInfo
{
Filename = file,
FileInfo = files.Add(s)
});
}
return(fileInfos);
}
/// <summary>
/// Create all required <see cref="FileInfo"/>s for the provided archive, adding them to the global file store.
/// </summary>
private List <TFileModel> createFileInfos(ArchiveReader reader, FileStore files)
{
var fileInfos = new List <TFileModel>();
// import files to manager
foreach (string file in reader.Filenames)
{
using (Stream s = reader.GetStream(file))
fileInfos.Add(new TFileModel
{
Filename = FileSafety.PathStandardise(file),
FileInfo = files.Add(s)
});
}
return(fileInfos);
}
protected override ScoreInfo CreateModel(ArchiveReader archive)
{
if (archive == null)
{
return(null);
}
using (var stream = archive.GetStream(archive.Filenames.First(f => f.EndsWith(".osr"))))
{
try
{
return(new DatabasedLegacyScoreParser(rulesets, beatmaps).Parse(stream).ScoreInfo);
}
catch (LegacyScoreParser.BeatmapNotFoundException e)
{
Logger.Log(e.Message, LoggingTarget.Information, LogLevel.Error);
return(null);
}
}
}
protected override BeatmapSetInfo CreateModel(ArchiveReader reader)
{
// let's make sure there are actually .osu files to import.
string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu"));
if (string.IsNullOrEmpty(mapName))
{
throw new InvalidOperationException("No beatmap files found in this beatmap archive.");
}
BeatmapMetadata metadata;
using (var stream = new StreamReader(reader.GetStream(mapName)))
metadata = Decoder.GetDecoder <Beatmap>(stream).Decode(stream).Metadata;
return(new BeatmapSetInfo
{
OnlineBeatmapSetID = metadata.OnlineBeatmapSetID,
Beatmaps = new List <BeatmapInfo>(),
Hash = computeBeatmapSetHash(reader),
Metadata = metadata
});
}
protected override BeatmapSetInfo CreateModel(ArchiveReader reader)
{
// let's make sure there are actually .osu files to import.
string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu"));
if (string.IsNullOrEmpty(mapName))
{
Logger.Log($"No beatmap files found in the beatmap archive ({reader.Name}).", LoggingTarget.Database);
return(null);
}
Beatmap beatmap;
using (var stream = new StreamReader(reader.GetStream(mapName)))
beatmap = Decoder.GetDecoder <Beatmap>(stream).Decode(stream);
return(new BeatmapSetInfo
{
OnlineBeatmapSetID = beatmap.BeatmapInfo.BeatmapSet?.OnlineBeatmapSetID,
Beatmaps = new List <BeatmapInfo>(),
Metadata = beatmap.Metadata,
});
}
private BeatmapSetInfo getBeatmapSet(ArchiveReader archiveReader)
{
BeatmapMetadata metadata;
using (var stream = new StreamReader(archiveReader.GetStream(archiveReader.BeatmapFilenames[0])))
metadata = BeatmapDecoder.GetDecoder(stream).Decode(stream).Metadata;
string hash;
string path;
using (var input = archiveReader.GetUnderlyingStream())
{
hash = input.GetMd5Hash();
input.Seek(0, SeekOrigin.Begin);
path = Path.Combine(@"beatmaps", hash.Remove(1), hash.Remove(2), hash);
if (!Storage.Exists(path))
{
using (var output = Storage.GetStream(path, FileAccess.Write))
input.CopyTo(output);
}
}
var existing = Connection.Table <BeatmapSetInfo>().FirstOrDefault(b => b.Hash == hash);
if (existing != null)
{
GetChildren(existing);
if (existing.DeletePending)
{
existing.DeletePending = false;
Update(existing, false);
BeatmapSetAdded?.Invoke(existing);
}
return(existing);
}
var beatmapSet = new BeatmapSetInfo
{
OnlineBeatmapSetID = metadata.OnlineBeatmapSetID,
Beatmaps = new List <BeatmapInfo>(),
Path = path,
Hash = hash,
Metadata = metadata
};
using (var archive = ArchiveReader.GetReader(Storage, path))
{
string[] mapNames = archive.BeatmapFilenames;
foreach (var name in mapNames)
{
using (var raw = archive.GetStream(name))
using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit
using (var sr = new StreamReader(ms))
{
raw.CopyTo(ms);
ms.Position = 0;
var decoder = BeatmapDecoder.GetDecoder(sr);
Beatmap beatmap = decoder.Decode(sr);
beatmap.BeatmapInfo.Path = name;
beatmap.BeatmapInfo.Hash = ms.GetMd5Hash();
// TODO: Diff beatmap metadata with set metadata and leave it here if necessary
beatmap.BeatmapInfo.Metadata = null;
// TODO: this should be done in a better place once we actually need to dynamically update it.
beatmap.BeatmapInfo.Ruleset = rulesets.Query <RulesetInfo>().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID);
beatmap.BeatmapInfo.StarDifficulty = rulesets.Query <RulesetInfo>().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID)?.CreateInstance()?.CreateDifficultyCalculator(beatmap).Calculate() ?? 0;
beatmapSet.Beatmaps.Add(beatmap.BeatmapInfo);
}
}
beatmapSet.StoryboardFile = archive.StoryboardFilename;
}
return(beatmapSet);
}
/// <summary>
/// Create all required <see cref="BeatmapInfo"/>s for the provided archive.
/// </summary>
private List <BeatmapInfo> createBeatmapDifficulties(ArchiveReader reader)
{
var beatmapInfos = new List <BeatmapInfo>();
bool invalidateOnlineIDs = false;
foreach (var name in reader.Filenames.Where(f => f.EndsWith(".osu")))
{
using (var raw = reader.GetStream(name))
using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit
using (var sr = new StreamReader(ms))
{
raw.CopyTo(ms);
ms.Position = 0;
var decoder = Decoder.GetDecoder <Beatmap>(sr);
IBeatmap beatmap = decoder.Decode(sr);
beatmap.BeatmapInfo.Path = name;
beatmap.BeatmapInfo.Hash = ms.ComputeSHA2Hash();
beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash();
if (beatmap.BeatmapInfo.OnlineBeatmapID.HasValue)
{
var ourId = beatmap.BeatmapInfo.OnlineBeatmapID;
// check that no existing beatmap in database exists that is imported with the same online beatmap ID. if so, give it precedence.
if (QueryBeatmap(b => b.OnlineBeatmapID.Value == ourId) != null)
{
beatmap.BeatmapInfo.OnlineBeatmapID = null;
}
// check that no other beatmap in this imported set has a conflicting online beatmap ID. If so, presume *all* are incorrect.
if (beatmapInfos.Any(b => b.OnlineBeatmapID == ourId))
{
invalidateOnlineIDs = true;
}
}
RulesetInfo ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID);
beatmap.BeatmapInfo.Ruleset = ruleset;
if (ruleset != null)
{
// TODO: this should be done in a better place once we actually need to dynamically update it.
beatmap.BeatmapInfo.StarDifficulty = ruleset.CreateInstance().CreateDifficultyCalculator(new DummyConversionBeatmap(beatmap)).Calculate().StarRating;
}
else
{
beatmap.BeatmapInfo.StarDifficulty = 0;
}
beatmapInfos.Add(beatmap.BeatmapInfo);
}
}
if (invalidateOnlineIDs)
{
beatmapInfos.ForEach(b => b.OnlineBeatmapID = null);
}
return(beatmapInfos);
}
/// <summary>
/// Import a beamap into our local <see cref="FileStore"/> storage.
/// If the beatmap is already imported, the existing instance will be returned.
/// </summary>
/// <param name="files">The store to import beatmap files to.</param>
/// <param name="beatmaps">The store to import beatmaps to.</param>
/// <param name="reader">The beatmap archive to be read.</param>
/// <returns>The imported beatmap, or an existing instance if it is already present.</returns>
private BeatmapSetInfo importToStorage(FileStore files, BeatmapStore beatmaps, ArchiveReader reader)
{
// let's make sure there are actually .osu files to import.
string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu"));
if (string.IsNullOrEmpty(mapName))
{
throw new InvalidOperationException("No beatmap files found in the map folder.");
}
// for now, concatenate all .osu files in the set to create a unique hash.
MemoryStream hashable = new MemoryStream();
foreach (string file in reader.Filenames.Where(f => f.EndsWith(".osu")))
{
using (Stream s = reader.GetStream(file))
s.CopyTo(hashable);
}
var hash = hashable.ComputeSHA2Hash();
// check if this beatmap has already been imported and exit early if so.
var beatmapSet = beatmaps.BeatmapSets.FirstOrDefault(b => b.Hash == hash);
if (beatmapSet != null)
{
undelete(beatmaps, files, beatmapSet);
// ensure all files are present and accessible
foreach (var f in beatmapSet.Files)
{
if (!storage.Exists(f.FileInfo.StoragePath))
{
using (Stream s = reader.GetStream(f.Filename))
files.Add(s, false);
}
}
// todo: delete any files which shouldn't exist any more.
return(beatmapSet);
}
List <BeatmapSetFileInfo> fileInfos = new List <BeatmapSetFileInfo>();
// import files to manager
foreach (string file in reader.Filenames)
{
using (Stream s = reader.GetStream(file))
fileInfos.Add(new BeatmapSetFileInfo
{
Filename = file,
FileInfo = files.Add(s)
});
}
BeatmapMetadata metadata;
using (var stream = new StreamReader(reader.GetStream(mapName)))
metadata = Decoder.GetDecoder(stream).DecodeBeatmap(stream).Metadata;
// check if a set already exists with the same online id.
if (metadata.OnlineBeatmapSetID != null)
{
beatmapSet = beatmaps.BeatmapSets.FirstOrDefault(b => b.OnlineBeatmapSetID == metadata.OnlineBeatmapSetID);
}
if (beatmapSet == null)
{
beatmapSet = new BeatmapSetInfo
{
OnlineBeatmapSetID = metadata.OnlineBeatmapSetID,
Beatmaps = new List <BeatmapInfo>(),
Hash = hash,
Files = fileInfos,
Metadata = metadata
}
}
;
var mapNames = reader.Filenames.Where(f => f.EndsWith(".osu"));
foreach (var name in mapNames)
{
using (var raw = reader.GetStream(name))
using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit
using (var sr = new StreamReader(ms))
{
raw.CopyTo(ms);
ms.Position = 0;
var decoder = Decoder.GetDecoder(sr);
Beatmap beatmap = decoder.DecodeBeatmap(sr);
beatmap.BeatmapInfo.Path = name;
beatmap.BeatmapInfo.Hash = ms.ComputeSHA2Hash();
beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash();
var existing = beatmaps.Beatmaps.FirstOrDefault(b => b.Hash == beatmap.BeatmapInfo.Hash || beatmap.BeatmapInfo.OnlineBeatmapID != null && b.OnlineBeatmapID == beatmap.BeatmapInfo.OnlineBeatmapID);
if (existing == null)
{
// Exclude beatmap-metadata if it's equal to beatmapset-metadata
if (metadata.Equals(beatmap.Metadata))
{
beatmap.BeatmapInfo.Metadata = null;
}
RulesetInfo ruleset = rulesets.GetRuleset(beatmap.BeatmapInfo.RulesetID);
// TODO: this should be done in a better place once we actually need to dynamically update it.
beatmap.BeatmapInfo.Ruleset = ruleset;
beatmap.BeatmapInfo.StarDifficulty = ruleset?.CreateInstance()?.CreateDifficultyCalculator(beatmap).Calculate() ?? 0;
beatmapSet.Beatmaps.Add(beatmap.BeatmapInfo);
}
}
}
return(beatmapSet);
}
/// <summary>
/// Import a beamap into our local <see cref="FileStore"/> storage.
/// If the beatmap is already imported, the existing instance will be returned.
/// </summary>
/// <param name="reader">The beatmap archive to be read.</param>
/// <returns>The imported beatmap, or an existing instance if it is already present.</returns>
private BeatmapSetInfo importToStorage(ArchiveReader reader)
{
// let's make sure there are actually .osu files to import.
string mapName = reader.Filenames.FirstOrDefault(f => f.EndsWith(".osu"));
if (string.IsNullOrEmpty(mapName))
{
throw new InvalidOperationException("No beatmap files found in the map folder.");
}
// for now, concatenate all .osu files in the set to create a unique hash.
MemoryStream hashable = new MemoryStream();
foreach (string file in reader.Filenames.Where(f => f.EndsWith(".osu")))
{
using (Stream s = reader.GetStream(file))
s.CopyTo(hashable);
}
var hash = hashable.ComputeSHA2Hash();
// check if this beatmap has already been imported and exit early if so.
BeatmapSetInfo beatmapSet;
lock (beatmaps)
beatmapSet = beatmaps.QueryAndPopulate <BeatmapSetInfo>(b => b.Hash == hash).FirstOrDefault();
if (beatmapSet != null)
{
Undelete(beatmapSet);
return(beatmapSet);
}
List <BeatmapSetFileInfo> fileInfos = new List <BeatmapSetFileInfo>();
// import files to manager
foreach (string file in reader.Filenames)
{
using (Stream s = reader.GetStream(file))
fileInfos.Add(new BeatmapSetFileInfo
{
Filename = file,
FileInfo = files.Add(s)
});
}
BeatmapMetadata metadata;
using (var stream = new StreamReader(reader.GetStream(mapName)))
metadata = BeatmapDecoder.GetDecoder(stream).Decode(stream).Metadata;
beatmapSet = new BeatmapSetInfo
{
OnlineBeatmapSetID = metadata.OnlineBeatmapSetID,
Beatmaps = new List <BeatmapInfo>(),
Hash = hash,
Files = fileInfos,
Metadata = metadata
};
var mapNames = reader.Filenames.Where(f => f.EndsWith(".osu"));
foreach (var name in mapNames)
{
using (var raw = reader.GetStream(name))
using (var ms = new MemoryStream()) //we need a memory stream so we can seek and shit
using (var sr = new StreamReader(ms))
{
raw.CopyTo(ms);
ms.Position = 0;
var decoder = BeatmapDecoder.GetDecoder(sr);
Beatmap beatmap = decoder.Decode(sr);
beatmap.BeatmapInfo.Path = name;
beatmap.BeatmapInfo.Hash = ms.ComputeSHA2Hash();
beatmap.BeatmapInfo.MD5Hash = ms.ComputeMD5Hash();
// TODO: Diff beatmap metadata with set metadata and leave it here if necessary
beatmap.BeatmapInfo.Metadata = null;
// TODO: this should be done in a better place once we actually need to dynamically update it.
beatmap.BeatmapInfo.Ruleset = rulesets.Query <RulesetInfo>().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID);
beatmap.BeatmapInfo.StarDifficulty = rulesets.Query <RulesetInfo>().FirstOrDefault(r => r.ID == beatmap.BeatmapInfo.RulesetID)?.CreateInstance()?.CreateDifficultyCalculator(beatmap)
.Calculate() ?? 0;
beatmapSet.Beatmaps.Add(beatmap.BeatmapInfo);
}
}
return(beatmapSet);
}
