private void HandleErrOutput(DataReceivedEventArgs arguments, Track track)
{
LoggerBundle.Trace($"Processing error response of track '{track}'...");
try
{
String output = arguments?.Data?.Trim();
if (String.IsNullOrEmpty(output))
{
return;
}
LoggerBundle.Warn(new CalculationException(output));
track.FingerprintError = output;
track.LastFingerprintCalculation = DateTime.Now;
using (DataContext dataContext = DataContextFactory.GetInstance())
{
LoggerBundle.Trace($"Saving track '{track}'...");
dataContext.SetTracks.Attach(track);
dataContext.Entry(track).State = EntityState.Modified;
dataContext.SaveChanges();
LoggerBundle.Debug($"Successfully updated track '{track}'...");
}
}
catch (Exception ex)
{
LoggerBundle.Error(ex);
}
finally
{
_buffer.Remove(track);
}
}
private Program()
{
using (DataContext dc = DataContextFactory.GetInstance())
{
LoggerBundle.Debug("Trying to load data of every model type...");
foreach (Type t in Types)
{
LoggerBundle.Trace($"Loading data for set of type '{t.Name}'...");
MethodInfo setMethod = dc.GetType().GetMethods().First(x => x.Name.Equals("Set"));
MethodInfo setMethodGeneric = setMethod.MakeGenericMethod(t);
Object set = setMethodGeneric.Invoke(dc, new Object[] { });
MethodInfo methodLoad = typeof(EntityFrameworkQueryableExtensions).GetMethods()
.First(x => x.Name.Equals("Load"));
MethodInfo methodLoadGeneric = methodLoad.MakeGenericMethod(t);
methodLoadGeneric.Invoke(set
, new[]
{
set
});
dc.SetTracks.Load();
LoggerBundle.Trace("Loading data done.");
}
LoggerBundle.Debug("Done.");
}
}
protected override void Process(String[] args)
{
OnProcessStarting();
TriggerActions(args.ToList());
List <Track> tracks;
do
{
using (DataContext dataContext = DataContextFactory.GetInstance())
{
LoggerBundle.Debug("Preloading data...");
Stopwatch sw = new Stopwatch();
sw.Start();
tracks = _includeFailed
? dataContext.SetTracks.Where(x => !x.LastFingerprintCalculation.HasValue || null != x.FingerprintError)
.Take(_config.BufferSize)
.ToList()
: dataContext.SetTracks.Where(x => !x.LastFingerprintCalculation.HasValue)
.Take(_config.BufferSize)
.ToList();
sw.Stop();
LoggerBundle.Debug($"Getting data finished in {sw.ElapsedMilliseconds}ms");
}
LoggerBundle.Inform($"Batch contains {tracks.Count} record(s).");
foreach (Track track in tracks)
{
LoggerBundle.Trace($"Initializing process for track '{track}'...");
while (_buffer.Count >= _config.ParallelProcesses)
{
Thread.Sleep(1);
}
LoggerBundle.Trace($"Starting process for track '{track}'...");
Process p = new Process
{
StartInfo =
{
FileName = FingerprintCalculationExecutablePath
, Arguments = $"-json \"{track.Path}\""
, CreateNoWindow = true
, RedirectStandardError = true
, RedirectStandardInput = true
, RedirectStandardOutput = true
, UseShellExecute = false
}
};
p.OutputDataReceived += (_, arguments) => HandleStdOutput(arguments, track);
p.ErrorDataReceived += (_, arguments) => HandleErrOutput(arguments, track);
_buffer.Add(track);
LoggerBundle.Trace($"Starting computation process for file '{track}'...");
p.Start();
p.BeginOutputReadLine();
p.BeginErrorReadLine();
LoggerBundle.Debug($"Computation process started for file '{track}'");
}
}while (tracks.Count > 0);
}
private static Dictionary <String, PluginBase> InitializePlugin(List <PluginBase> pluginsToLoad)
{
Dictionary <String, PluginBase> plugins = new Dictionary <String, PluginBase>();
pluginsToLoad.ForEach(x =>
{
// initialize
Boolean initialized = x.Initialize();
if (!initialized)
{
LoggerBundle.Warn($"Plugin '{x.Name}' cannot be initialized. This plugin will be deactivated.");
return;
}
LoggerBundle.Debug($"Plugin '{x.Name}' initialized successfully. Validating...");
// validate
String pcName = x.Name.ToLower().Trim();
if (plugins.ContainsKey(pcName))
{
LoggerBundle.Warn($"Plugin '{x.Name}' does not pass validation because a plugin with the same name has already been registered. This plugin will be deactivated.");
}
LoggerBundle.Debug($"Plugin '{x.Name}' passed validation");
// add to plugin registry
plugins.Add(pcName, x);
LoggerBundle.Inform($"Plugin '{x.Name}' activated");
});
return(plugins);
}
public static T Request <T>(String path) where T : class
{
LoggerBundle.Debug($"Requested configuration '{path}'");
if (!File.Exists(path))
{
LoggerBundle.Trace($"File '{path}' not found. Trying to create it...");
FileInterface.Save(Activator.CreateInstance <T>(), path);
LoggerBundle.Trace($"Successfully created file '{path}'");
LoggerBundle.Inform(
$"Changes to the newly created file '{path}' will take effect after restarting the executable. Adjust default value as needed and restart the application.");
}
LoggerBundle.Trace($"Trying to read file '{path}'...");
(T result, Boolean success) = FileInterface.Read <T>(path);
if (!success)
{
LoggerBundle.Warn(new ProcessAbortedException());
return(null);
}
LoggerBundle.Debug($"Successfully read configuration file '{path}'");
return(result);
}
protected override void OnInitialize()
{
LoggerBundle.Debug($"Initializing plugin '{Name}'...");
LoggerBundle.Trace("Requesting config...");
_config = RequestConfig <Config>();
LoggerBundle.Trace("Done.");
}
protected override void Process(String[] args)
{
OnProcessStarting();
TriggerActions(args.ToList());
List <String> paths = args.Distinct().Select(x => x.Trim()).ToList();
if (paths.Count.Equals(0))
{
LoggerBundle.Fatal(new ArgumentException("no argument given"));
Environment.Exit(1);
}
foreach (String path in paths)
{
LoggerBundle.Inform($"Processing path '{path}'...");
if (!Directory.Exists(path))
{
LoggerBundle.Warn($"Path '{path}' not found. Skipping.");
continue;
}
LoggerBundle.Debug("Preloading data...");
List <String> tracks;
Stopwatch sw = new Stopwatch();
sw.Start();
using (DataContext dataContext = DataContextFactory.GetInstance())
{
tracks = dataContext.SetTracks.AsNoTracking().Select(x => x.Path).ToList();
}
sw.Stop();
LoggerBundle.Debug($"Getting data finished in {sw.ElapsedMilliseconds}ms");
List <String> buffer = new List <String>();
DataSource <String> ds = new PathDataSource(path, _config.Extensions);
LoggerBundle.Inform($"Start to crawl path '{path}'...");
foreach (String file in ds.Get())
{
buffer.Add(file);
Int32 bufferCount = buffer.Count;
if (bufferCount < _config.BufferSize)
{
if (bufferCount % (_config.BufferSize < 1337 ? _config.BufferSize : 1337) == 0)
{
LoggerBundle.Trace($"Adding files to buffer [{bufferCount}/{_config.BufferSize}] ...");
}
continue;
}
ProcessBuffer(ref buffer, ref tracks);
}
ProcessBuffer(ref buffer, ref tracks);
}
}
protected override void OnInitialize()
{
LoggerBundle.Debug($"Initializing plugin '{Name}'...");
LoggerBundle.Trace("Requesting config...");
_config = RequestConfig <Config>();
LoggerBundle.Trace("Done");
RegisterAction("include-failed", () => _includeFailed = true);
}
private static void Main(String[] args)
{
// mainly for debugging
LoggerBundle.Trace("TEST");
LoggerBundle.Debug("TEST");
LoggerBundle.Inform("TEST");
LoggerBundle.Warn(new Exception());
LoggerBundle.Error(new Exception());
LoggerBundle.Fatal(new Exception());
}
private void HandleStdOutput(DataReceivedEventArgs arguments, Track track)
{
LoggerBundle.Trace($"Processing response of track '{track}'...");
try
{
String output = arguments?.Data?.Trim();
if (String.IsNullOrEmpty(output))
{
return;
}
LoggerBundle.Debug(Logger.DefaultLogFlags & ~LogFlags.SuffixNewLine
, $"Trying to serialize computation output of file '{track}'...");
JsonFingerprint jfp = JsonConvert.DeserializeObject <JsonFingerprint>(output);
LoggerBundle.Debug(Logger.DefaultLogFlags & ~LogFlags.PrefixLoggerType & ~LogFlags.PrefixTimeStamp, "Ok.");
track.LastFingerprintCalculation = DateTime.Now;
track.FingerprintHash = _hasher.Compute(jfp.Fingerprint);
track.FingerprintError = null;
LoggerBundle.Trace($"Fingerprint hash: {track.FingerprintHash} for fingerprint {jfp.Fingerprint}");
using (DataContext dataContext = DataContextFactory.GetInstance())
{
LoggerBundle.Trace($"Checking for duplicates for file '{track}'...");
if (dataContext.SetTracks.AsNoTracking().Any(x => x.FingerprintHash.Equals(track.FingerprintHash)))
{
LoggerBundle.Debug($"File with same fingerprint already in database. Path '{track}' will be skipped");
track.FingerprintError = "duplicate";
}
else
{
LoggerBundle.Trace($"No duplicate found for file '{track}'");
track.Duration = jfp.Duration;
track.Fingerprint = jfp.Fingerprint;
LoggerBundle.Trace($"New meta data duration '{track.Duration}' and fingerprint '{jfp.Fingerprint}'");
}
LoggerBundle.Trace($"Saving file '{track.Path}'...");
dataContext.SetTracks.Attach(track);
dataContext.Entry(track).State = EntityState.Modified;
dataContext.SaveChanges();
LoggerBundle.Debug($"Successfully saved file '{track}'...");
}
}
catch (Exception ex)
{
LoggerBundle.Error(ex);
}
finally
{
_buffer.Remove(track);
}
}
public DataContext(DbContextOptions <DataContext> options) : base(options)
{
LoggerBundle.Debug("Initializing data context...");
try
{
_config = Configurator.Request <DatabaseConfig>(DatabaseSettingsFilePath);
}
catch (Exception ex)
{
LoggerBundle.Fatal("Could not load database config file", ex);
Environment.Exit(1);
}
LoggerBundle.Debug("Context initialized.");
}
public void TriggerAction(String key)
{
if (key == null)
{
LoggerBundle.Warn(new ArgumentNullException(nameof(key)));
return;
}
if (!_actions.ContainsKey(key))
{
LoggerBundle.Debug(new KeyNotFoundException($"No action with name '{key}' found"));
return;
}
_actions[key].Invoke();
}
public Object Get(String id)
{
LoggerBundle.Debug($"Get request for id '{id}'...");
while ((DateTime.Now - _lastRequest).TotalMilliseconds < DELAY_BETWEEN_REQUESTS)
{
Thread.Sleep(1);
}
String url = String.Format(URI_API_MUSICBRAINZ, id ?? String.Empty);
LoggerBundle.Trace($"Posting to '{url}'");
Assembly coreAssembly = typeof(PluginBase).Assembly;
Version coreVersion = coreAssembly.GetName().Version;
HttpRequestMessage req = new HttpRequestMessage(HttpMethod.Get, url);
String userAgent = $"Mux/{coreVersion} ( mail @fooo.ooo ) - Instance {_guid}";
req.Headers.Add("User-Agent", userAgent);
LoggerBundle.Trace($"User-Agent: {userAgent}");
_lastRequest = DateTime.Now;
LoggerBundle.Trace(Logger.DefaultLogFlags & ~LogFlags.SuffixNewLine, "Sending async request...");
Task <HttpResponseMessage> response = _client.SendAsync(req);
LoggerBundle.Trace(Logger.DefaultLogFlags & ~LogFlags.PrefixTimeStamp & ~LogFlags.PrefixLoggerType, "Ok.");
Task <String> responseString = response.Result.Content.ReadAsStringAsync();
String responseBody = responseString.Result.Trim();
LoggerBundle.Trace($"Response: {responseBody}");
LoggerBundle.Trace(Logger.DefaultLogFlags & ~LogFlags.SuffixNewLine, "Trying to deserialize object...");
JsonErrorMusicBrainz status = JsonConvert.DeserializeObject <JsonErrorMusicBrainz>(responseBody);
LoggerBundle.Trace(Logger.DefaultLogFlags & ~LogFlags.PrefixTimeStamp & ~LogFlags.PrefixLoggerType, "Ok.");
if (null == status.Error)
{
// no error found
return(JsonConvert.DeserializeObject <JsonMusicBrainzRequest>(responseBody));
}
return(status);
}
private static void CreateGlobalDirectories()
{
List <String> paths = new List <String>
{
Location.ApplicationDataDirectoryPath
, Location.PluginsDirectoryPath
, Location.LogsDirectoryPath
};
paths.Where(x => !Directory.Exists(x))
.ToList()
.ForEach(x =>
{
LoggerBundle.Debug(Logger.DefaultLogFlags & ~LogFlags.SuffixNewLine, $"Trying to create directory '{x}'...");
Directory.CreateDirectory(x);
LoggerBundle.Debug(Logger.DefaultLogFlags & ~LogFlags.PrefixLoggerType & ~LogFlags.PrefixTimeStamp, "Ok.");
});
}
protected override void OnInitialize()
{
LoggerBundle.Debug($"Initializing plugin '{Name}'...");
if (!File.Exists(FingerprintCalculationExecutablePath))
{
LoggerBundle.Fatal(new FileNotFoundException(
$"File '{FingerprintCalculationExecutablePath}' not found. Visit https://github.com/acoustid/chromaprint/releases to download the latest version."
, FingerprintCalculationExecutablePath));
Environment.Exit(1);
}
LoggerBundle.Trace("Requesting config...");
_config = RequestConfig <Config>();
LoggerBundle.Trace("Done");
RegisterAction("include-failed", () => _includeFailed = true);
}
private void ProcessBuffer(ref List <String> buffer, ref List <String> tracks)
{
LoggerBundle.Debug("Buffer full. Searching new entries...");
List <String> newPaths = buffer.Except(tracks).ToList();
Int32 newPathsCount = newPaths.Count;
LoggerBundle.Debug($"{newPathsCount} new files found");
LoggerBundle.Debug("Saving to database...");
Stopwatch sw = new Stopwatch();
sw.Start();
using (DataContext dataContext = DataContextFactory.GetInstance())
{
dataContext.ChangeTracker.AutoDetectChangesEnabled = false;
// todo disable validation on save
for (Int32 i = 0; i < newPathsCount; i++)
{
dataContext.SetTracks.Add(new Track
{
Path = newPaths[i]
});
if (i % 1337 != 0)
{
continue;
}
dataContext.SaveChanges();
LoggerBundle.Trace($"Saved {i + 1}/{newPathsCount}...");
}
dataContext.SaveChanges();
}
sw.Stop();
Int64 elms = sw.ElapsedMilliseconds;
LoggerBundle.Debug($"Saved {newPathsCount} items in {elms}ms ({(Double) elms / newPathsCount}ms per item average)");
tracks.AddRange(newPaths);
buffer.Clear();
LoggerBundle.Debug("Finished processing buffer. Returning.");
}
private void CleanInvisible()
{
List <Track> data;
do
{
LoggerBundle.Inform("Removing invisible data");
LoggerBundle.Debug("Getting data...");
using (DataContext dataContext = DataContextFactory.GetInstance())
{
data = dataContext.SetTracks.Where(x => null != x.LastAcoustIdApiCall).Where(x => 1 > x.AcoustIdResults.Count).OrderBy(x => x.UniqueId).Take(_config.BufferSize).ToList();
LoggerBundle.Inform($"Batch containing: {data.Count} entries");
foreach (Track track in data)
{
try
{
LoggerBundle.Debug($"Processing track '{track}'...");
if (File.Exists(track.Path))
{
File.Delete(track.Path);
}
dataContext.SetTracks.Remove(track);
dataContext.SaveChanges();
LoggerBundle.Trace("Processing done");
}
catch (Exception ex)
{
LoggerBundle.Error(ex);
}
}
}
}while (data.Count > 0);
}
protected override void OnInitialize()
{
LoggerBundle.Debug($"Initializing plugin '{Name}'...");
RegisterAction("add", AddUser);
}
protected override void Process(String[] args)
{
base.OnProcessStarting();
TriggerActions(args.ToList());
List <Track> data;
do
{
using (DataContext dataContext = DataContextFactory.GetInstance())
{
LoggerBundle.Debug("Loading batch...");
data = _includeFailed
? dataContext.SetTracks
.Where(x => null != x.LastFingerprintCalculation &&
null == x.FingerprintError &&
null == x.LastAcoustIdApiCall ||
x.LastAcoustIdApiCall.HasValue && null != x.AcoustIdApiError)
.Take(_config.BufferSize)
.ToList()
: dataContext.SetTracks
.Where(x => null != x.LastFingerprintCalculation &&
null == x.FingerprintError &&
null == x.LastAcoustIdApiCall)
.Take(_config.BufferSize)
.ToList();
LoggerBundle.Inform($"Batch containing {data.Count} entries");
foreach (Track track in data)
{
LoggerBundle.Debug($"Posting metadata of track '{track}'...");
track.LastAcoustIdApiCall = DateTime.Now;
Object response = _apiHandler.Post(track.Duration ?? 0d, track.Fingerprint);
LoggerBundle.Trace($"Response: {response}");
switch (response)
{
case JsonErrorAcoustId jea:
{
LoggerBundle.Warn(new AcoustIdApiException($"Error {jea.Error.Code}: {jea.Error.Message}"));
track.AcoustIdApiError = jea.Error.Message;
track.AcoustIdApiErrorCode = jea.Error.Code;
break;
}
case JsonAcoustIdRequest air:
{
HandleResponse(dataContext, track, air);
break;
}
default:
{
LoggerBundle.Trace(Logger.DefaultLogFlags & ~LogFlags.SuffixNewLine
, "Trying to serialize unknown response object...");
String serializedResponse = "<unknown>";
try
{
serializedResponse = JsonConvert.SerializeObject(response);
}
catch (Exception ex)
{
LoggerBundle.Error(ex);
}
LoggerBundle.Trace(Logger.DefaultLogFlags
& ~LogFlags.PrefixTimeStamp
& ~LogFlags.PrefixLoggerType
, "Ok.");
LoggerBundle.Warn(new AcoustIdApiException($"Unknown response: {serializedResponse}"));
track.AcoustIdApiError = serializedResponse;
break;
}
}
dataContext.SaveChanges();
}
}
}while (data.Count > 0);
}
protected override void Process(String[] args)
{
OnProcessStarting();
TriggerActions(args.ToList());
List <MusicBrainzRecord> data;
do
{
LoggerBundle.Debug("Getting data...");
using (DataContext dataContext = DataContextFactory.GetInstance())
{
data = dataContext.SetMusicBrainzRecords.Where(x => null == x.LastMusicBrainzApiCall)
.Include(x => x.MusicBrainzAliasMusicBrainzRecords)
.ThenInclude(x => x.MusicBrainzAlias)
.Include(x => x.MusicBrainzArtistCreditMusicBrainzRecords)
.ThenInclude(x => x.MusicBrainzArtistCredit)
.Include(x => x.MusicBrainzReleaseMusicBrainzRecords)
.ThenInclude(x => x.MusicBrainzRelease)
.Include(x => x.MusicBrainzTagMusicBrainzRecords)
.ThenInclude(x => x.MusicBrainzTag)
.OrderBy(x => x.UniqueId)
.Take(_config.BatchSize)
.ToList();
LoggerBundle.Inform($"Batch containing: {data.Count} entries");
foreach (MusicBrainzRecord mbr in data)
{
try
{
LoggerBundle.Debug($"Processing record '{mbr}'...");
DateTime requestTime = DateTime.Now;
Object o = _api.Get(mbr.MusicbrainzId);
Stopwatch sw = new Stopwatch();
sw.Start();
switch (o)
{
case JsonMusicBrainzRequest req:
HandleResponse(mbr, req, dataContext);
break;
case JsonErrorMusicBrainz err:
mbr.MusicBrainzApiCallError = err.Error?.Trim() ?? "<unknown>";
LoggerBundle.Warn(new MusicBrainzApiException($"Error: {mbr.MusicBrainzApiCallError}"));
break;
}
mbr.LastMusicBrainzApiCall = requestTime;
dataContext.SaveChanges();
sw.Stop();
LoggerBundle.Debug($"Processing done in {sw.ElapsedMilliseconds}ms");
}
catch (Exception ex)
{
LoggerBundle.Error(ex);
}
}
}
}while (data.Count > 0);
}
private void AddUser()
{
LoggerBundle.Debug("Starting process to add new user...");
try
{
// read username
LoggerBundle.Inform(Logger.DefaultLogFlags & ~LogFlags.SuffixNewLine, "Enter a username: "******"";
if (String.IsNullOrWhiteSpace(username))
{
LoggerBundle.Fatal(new ArgumentException("Username cannot be empty"));
Environment.Exit(1);
}
// check existance
LoggerBundle.Debug("Checking if user already exists...");
Boolean exists;
using (DataContext dataContext = DataContextFactory.GetInstance())
{
exists = dataContext.SetUsers.Any(x => x.Username.ToLower().Equals(username.ToLower()));
}
if (exists)
{
LoggerBundle.Fatal(new ArgumentException("Username already exists"));
Environment.Exit(1);
}
LoggerBundle.Trace("User not found database. Allowed to proceed forward");
// get password
LoggerBundle.Inform(Logger.DefaultLogFlags & ~LogFlags.SuffixNewLine, "Enter a password: "******"Confirm password: "******"Passwords do not match"));
Environment.Exit(1);
}
// hash password
Sha512HashPipe hashPipe = new Sha512HashPipe();
String hashedPw = hashPipe.Process(pw1);
// save model
User user = new User
{
Username = username
, Password = hashedPw
};
using (DataContext dataContext = DataContextFactory.GetInstance())
{
dataContext.SetUsers.Add(user);
dataContext.SaveChanges();
}
LoggerBundle.Inform(
$"Successfully created user '{user.Username}' created with unique identifier '{user.UniqueId}'");
}
catch (Exception ex)
{
LoggerBundle.Error(ex);
}
}
