private static void LoadUsersToDatabase(string usersFilePath)
{
using StreamReader streamReader = new StreamReader(usersFilePath);
using CsvReader csvReader = new CsvReader(streamReader, CultureInfo.InvariantCulture);
using DataLoaderContext context = new DataLoaderContext();
context.Database.EnsureCreated();
csvReader.Configuration.HasHeaderRecord = true;
csvReader.Configuration.AllowComments = true;
csvReader.Configuration.Delimiter = "\t";
while (csvReader.Read())
{
string id = csvReader.GetField(0);
Gender gender = ParseGender(csvReader.GetField(1));
bool ageParseSucceeded = int.TryParse(csvReader.GetField(2), out int age);
string country = csvReader.GetField(3);
bool registrationDateParseSuccess = DateTime.TryParseExact(csvReader.GetField(4), "MMM d, yyyy", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTime registrationDate);
User user = new User
{
Id = id,
Gender = gender,
Age = ageParseSucceeded && age != 0 ? age : (int?)null,
Country = country,
RegistrationDate = registrationDate,
};
Console.WriteLine($"READ: {user.Id} - Gender: {user.Gender} Age: {user.Age} Country: {user.Country} RegistrationDate: {user.RegistrationDate}");
context.Users.Add(user);
}
context.SaveChanges();
}
public DataLoaderContextSwitcher(DataLoaderContext loadCtx)
{
_prevLoadCtx = DataLoaderContext.Current;
DataLoaderContext.SetLoaderContext(loadCtx);
_prevSyncCtx = SynchronizationContext.Current;
SynchronizationContext.SetSynchronizationContext(loadCtx.SyncContext);
}
/// <summary>
/// Binds an instance to a particular loading context.
/// </summary>
internal void SetContext(DataLoaderContext context)
{
lock (_lock)
if (_queue.Count > 0)
{
throw new InvalidOperationException("Cannot set context while a load is pending or executing");
}
_boundContext = context;
}
/// <summary>
/// Creates a scope for the given <see cref="DataLoaderContext"/>.
/// </summary>
/// <param name="context"></param>
/// <param name="completeOnDisposal">Configures whether to complete the context when the scope is disposed of. </param>
internal DataLoaderScope(DataLoaderContext context, bool completeOnDisposal)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
_completeOnDisposal = completeOnDisposal;
_loadCtx = context;
_prevLoadCtx = DataLoaderContext.Current;
DataLoaderContext.SetCurrentContext(_loadCtx);
}
private static void LoadUserArtistsPlaysCsvToDatabase(string userArtistPlaysFilePath)
{
using StreamReader streamReader = new StreamReader(userArtistPlaysFilePath);
using CsvReader csvReader = new CsvReader(streamReader, CultureInfo.InvariantCulture);
using DataLoaderContext context = new DataLoaderContext();
context.Database.EnsureCreated();
csvReader.Configuration.HasHeaderRecord = true;
csvReader.Configuration.AllowComments = true;
csvReader.Configuration.Delimiter = "\t";
csvReader.Configuration.BadDataFound = (context) =>
{
Console.WriteLine($"BAD RECORD - {context.RawRecord}");
};
uint i = 0, createdRegistries = 0;
User user = context.Users.FirstOrDefault();
while (csvReader.Read())
{
// Increase the read registiries counter.
i++;
string userId = csvReader.GetField(0);
long.TryParse(csvReader.GetField(3), out long userArtistPlaysNumber);
string artistId = csvReader.GetField(1);
string artistName = csvReader.GetField(2);
UserPlaysCsv userPlaysCsv = new UserPlaysCsv
{
Id = i,
UserId = userId,
ArtistId = artistId,
ArtistName = artistName,
Plays = userArtistPlaysNumber,
};
// Console.WriteLine($"READ: {userArtistPlays.User.Id} - ArtistName: {userArtistPlays.Artist.Name} Plays: {userArtistPlays.PlaysNumber}");
context.UserPlaysCsvs.Add(userPlaysCsv);
createdRegistries++;
if (createdRegistries % 10000 == 0)
{
Console.WriteLine($"Saving Changes... {(i / 17559530.0) * 100:0.00}% {i}/{17559530.0} rows. ");
context.SaveChanges();
}
}
context.SaveChanges();
}
/// <summary>
/// Runs a user-specified delegate inside a new loader context, which keeps track of loaders called by the delegate
/// and defers query execution until after the delegate has returned.
/// </summary>
/// <remarks>
/// Continuations are handled in the same way - loaders called in continuation code will fire after all the
/// continuations for that particular result have run.
/// </remarks>
public static async Task Run(Func <CancellationToken, Task> func, CancellationToken cancellationToken)
{
if (func == null)
{
throw new ArgumentNullException();
}
using (var loadCtx = new DataLoaderContext())
{
await loadCtx._taskFactory
.StartNew(() => func(cancellationToken), cancellationToken)
.Unwrap().ConfigureAwait(false);
}
}
/// <summary>
/// Runs a user-specified delegate inside a new loader context, which keeps track of loaders called by the delegate
/// and defers query execution until after the delegate has returned.
/// </summary>
/// <remarks>
/// Continuations are handled in the same way - loaders called in continuation code will fire after all the
/// continuations for that particular result have run.
/// </remarks>
public static async Task <T> Run <T>(Func <DataLoaderContext, Task <T> > func)
{
if (func == null)
{
throw new ArgumentNullException();
}
using (var loadCtx = new DataLoaderContext())
{
return(await loadCtx._taskFactory
.StartNew(() => func(loadCtx))
.Unwrap().ConfigureAwait(false));
}
}
/// <summary>
/// Marks the end of this scope and the point at which pending loaders will be fired.
/// </summary>
public void Dispose()
{
#if NETSTANDARD1_3
if (_loadCtx != DataLoaderContext.Current)
{
throw new InvalidOperationException("This context for this scope does not match the current context");
}
#endif
if (_completeOnDisposal)
{
_loadCtx.Complete();
}
DataLoaderContext.SetCurrentContext(_prevLoadCtx);
}
/// <summary>
/// Runs code within a new loader context before firing any pending loaders.
/// </summary>
public static async Task Run(Func <DataLoaderContext, Task> func)
{
if (func == null)
{
throw new ArgumentNullException();
}
using (var loadCtx = new DataLoaderContext())
using (new DataLoaderContextSwitcher(loadCtx))
{
var task = Task.Factory.StartNew(() => func(loadCtx), CancellationToken.None, TaskCreationOptions.LongRunning, loadCtx.TaskScheduler).Unwrap();
// loadCtx.FlushLoadersOnThread();
await task.ConfigureAwait(false);
}
}
/// <summary>
/// Runs code within a new loader context before firing any pending loaders.
/// </summary>
public static async Task Run(Action <DataLoaderContext> action)
{
if (action == null)
{
throw new ArgumentNullException(nameof(action));
}
var loadCtx = new DataLoaderContext();
var syncCtx = new DataLoaderSynchronizationContext(loadCtx);
using (new DataLoaderContextSwitcher(loadCtx))
using (new SynchronizationContextSwitcher(syncCtx))
{
action(loadCtx);
await loadCtx.CompleteAsync().ConfigureAwait(false);
}
}
/// <summary>
/// Runs code within a new loader context before firing any pending loaders.
/// </summary>
public static async Task Run(Func <DataLoaderContext, Task> func)
{
if (func == null)
{
throw new ArgumentNullException(nameof(func));
}
var loadCtx = new DataLoaderContext();
var syncCtx = new DataLoaderSynchronizationContext(loadCtx);
using (new DataLoaderContextSwitcher(loadCtx))
using (new SynchronizationContextSwitcher(syncCtx))
{
var result = func(loadCtx);
await loadCtx.CompleteAsync().ConfigureAwait(false);
await result.ConfigureAwait(false);
}
}
private static void LoadUserPlays()
{
using DataLoaderContext context = new DataLoaderContext();
int i = 0, createdRegistries = 0;
Dictionary <string, User> users = context.Users.ToDictionary(item => item.Id, item => item);
Dictionary <string, Artist> artists = context.Artists.ToDictionary(item => item.Id, item => item);
var userPlays = context.UserPlaysCsvs.AsNoTracking()
.Where(upc => !string.IsNullOrEmpty(upc.UserId) && !string.IsNullOrEmpty(upc.ArtistId))
.OrderByDescending(upc => upc.Plays)
.AsEnumerable()
.GroupBy(upc => upc.UserId);
foreach (var up in userPlays)
{
// Increase the read registries counter.
i += up.Count();
// Take only the 10 most played artists.
foreach (var userPlay in up.Take(10))
{
context.UserArtistPlays.Add(new UserArtistPlays
{
User = users[userPlay.UserId],
Artist = artists[userPlay.ArtistId],
PlaysNumber = userPlay.Plays,
});
createdRegistries++;
}
if (createdRegistries % 1000 == 0)
{
context.SaveChanges();
Console.WriteLine($"Saving Changes... {(i / 11500000.0) * 100:0.000}% {i}/{11500000.0} rows. ");
}
}
context.SaveChanges();
}
private static void LoadArtists()
{
using DataLoaderContext context = new DataLoaderContext();
int i = 0, createdRegistries = 0;
IQueryable <UserPlaysCsv> userPlaysCsvs = context.UserPlaysCsvs.AsNoTracking();
var uniqueArtistRecords = userPlaysCsvs
.Where(a => a.ArtistName != null)
.AsEnumerable()
.GroupBy(upc => upc.ArtistId);
foreach (var uar in uniqueArtistRecords)
{
// Increase the read registries counter.
i += uar.Count();
var artistRecord = uar.FirstOrDefault();
Artist artist = new Artist
{
Id = artistRecord.ArtistId,
Name = artistRecord.ArtistName,
};
context.Artists.Add(artist);
createdRegistries++;
if (createdRegistries % 10000 == 0)
{
Console.WriteLine($"Saving Changes... {(i / 11500000.0) * 100:0.000}% {i}/{11500000.0} rows. ");
context.SaveChanges();
}
}
context.SaveChanges();
}
internal static void SetCurrentContext(DataLoaderContext context)
{
}
/// <summary>
/// Sets the <see cref="DataLoaderContext"/> visible from the <see cref="DataLoaderContext.Current"/> property.
/// </summary>
/// <param name="context"></param>
internal static void SetCurrentContext(DataLoaderContext context)
{
LocalContext.Value = context;
}
/// <summary> /// Sets the currently visible ambient loader context. /// </summary> /// <remarks> /// If available, <see cref="DataLoader"/> instances that are not explicitly bound to a context /// will register themselves with the ambient context when the load method is called and /// the loader is not yet scheduled for execution. /// </remarks> internal static void SetLoaderContext(DataLoaderContext context) => _localContext.Value = context;
/// <summary>
/// Creates a new <see cref="ObjectDataLoader{TKey,TReturn}"/> bound to a specific context.
/// </summary>
internal ObjectDataLoader(Func <IEnumerable <TKey>, Task <Dictionary <TKey, TReturn> > > fetchDelegate, DataLoaderContext context) : base(context)
{
_fetchDelegate = fetchDelegate;
}
/// <summary>
/// Creates a new <see cref="DataLoader{TKey,TReturn}"/> bound to the specified context.
/// </summary>
internal DataLoader(Func <IEnumerable <TKey>, Task <ILookup <TKey, TReturn> > > fetch, DataLoaderContext context) : this(fetch)
{
SetContext(context);
}
/// <summary>
/// Creates a new <see cref="RootDataLoader{T}"/> bound to a specific context.
/// </summary>
internal RootDataLoader(Func <Task <T> > fetchDelegate, DataLoaderContext context)
: base(context)
{
_fetchDelegate = fetchDelegate;
}
/// <summary>
/// Creates a new <see cref="DataLoaderFactory"/> that produces loaders for the given <see cref="DataLoaderContext"/>.
/// </summary>
public DataLoaderFactory(DataLoaderContext loaderContext)
{
_loaderContext = loaderContext;
}
public DataLoaderSynchronizationContext(DataLoaderContext loadCtx)
{
_loadCtx = loadCtx;
}
/// <summary>
/// Creates a new loader.
/// </summary>
/// <param name="context">Context the loader will be bound to.</param>
protected internal DataLoaderBase(DataLoaderContext context)
{
_completionSource = CreateNewLazyCompletionSource();
_context = context;
}
public DataLoaderContextSwitcher(DataLoaderContext loadCtx)
{
_prevLoadCtx = DataLoaderContext.Current;
DataLoaderContext.SetLoaderContext(loadCtx);
}
private static void LoadArtistGenreXml(string artistsFilePath)
{
using DataLoaderContext context = new DataLoaderContext();
// Get the artists ordered by their play number.
IEnumerable <Artist> artists = context.Artists
.FromSqlInterpolated(
$@"SELECT A.Id, A.Name, A.Genre
FROM Artists A, UserArtistPlays UAP
WHERE A.Id = UAP.ArtistId
GROUP BY UAP.ArtistId
ORDER BY SUM(UAP.PlaysNumber) DESC")
.ToList();
Dictionary <long, (string artistName, Dictionary <string, int> genresFrequency)> mappedArtistGenres = new Dictionary <long, (string artistName, Dictionary <string, int> genresFrequency)> ();
using XmlReader xmlReader = XmlReader.Create(artistsFilePath);
xmlReader.MoveToContent();
int updatedArtists = 0;
Regex duplicateArtistNamePattern = new Regex(@".*\(\d+\)$", RegexOptions.Compiled);
while (xmlReader.Read())
{
if (!(xmlReader.NodeType == XmlNodeType.Element &&
xmlReader.Name == "release"))
{
continue;
}
XElement element = XNode.ReadFrom(xmlReader) as XElement;
var children = element.Descendants();
XElement artistElement = children.FirstOrDefault(n => n.Name == "artist");
string artistId = artistElement.Element("id")?.Value;
string artistName = artistElement.Element("name")?.Value;
if (duplicateArtistNamePattern.IsMatch(artistName))
{
int startOfDuplicate = artistName.LastIndexOf('(');
artistName = artistName.Substring(0, startOfDuplicate).Trim();
}
mappedArtistGenres.TryGetValue(long.Parse(artistId), out (string artistName, Dictionary <string, int> genresFrequency)artistMapping);
if (artistMapping == default)
{
artistMapping = (artistName, new Dictionary <string, int>());
mappedArtistGenres[long.Parse(artistId)] = artistMapping;
}
XElement genres = children?.FirstOrDefault(n => n.Name == "genres");
// Some releases might not have genre.
if (genres == null)
{
continue;
}
foreach (XElement genre in genres.Descendants())
{
string genreName = genre.Value;
artistMapping.genresFrequency.TryGetValue(genreName, out int freq);
artistMapping.genresFrequency[genreName] = ++freq;
}
}
// We assume that the artists that have more releases are the ones most listened to in Last.Fm.
var orderedMappedArtistsByFrequency = mappedArtistGenres
.OrderByDescending(mag => mag.Value.genresFrequency
.Sum(gf => gf.Value));
foreach (var artistGenresMapping in orderedMappedArtistsByFrequency)
{
Artist artist = artists
.Where(a => a.Name.Equals(artistGenresMapping.Value.artistName, StringComparison.InvariantCultureIgnoreCase))
.FirstOrDefault(a => a.Genre == null);
if (artist == null)
{
continue;
}
// Get the most frequent genre of the artist.
var genresOrdered = artistGenresMapping.Value.genresFrequency
.OrderByDescending(gf => gf.Value);
KeyValuePair <string, int> mostFrequentGenre = genresOrdered.FirstOrDefault();
artist.Genre = mostFrequentGenre.Key;
updatedArtists++;
if (updatedArtists % 100 == 0)
{
Console.WriteLine($"Saving changes... Updated Artist entries: {updatedArtists}");
context.SaveChanges();
}
}
context.SaveChanges();
}
private static void LoadArtistGenreJson(string artistsFilePath)
{
using DataLoaderContext context = new DataLoaderContext();
// Get the artists ordered by their play number.
IEnumerable <Artist> artists = context.Artists
.FromSqlInterpolated(
$@"SELECT A.Id, A.Name, A.Genre
FROM Artists A, UserArtistPlays UAP
WHERE A.Id = UAP.ArtistId
GROUP BY UAP.ArtistId
ORDER BY SUM(UAP.PlaysNumber) DESC")
.ToList();
using StreamReader streamReader = new StreamReader(artistsFilePath);
using var jsonReader = new JsonTextReader(streamReader);
jsonReader.SupportMultipleContent = true;
JsonSerializer serializer = new JsonSerializer();
int updatedArtists = 0;
while (jsonReader.Read())
{
JObject jObject = (JObject)serializer.Deserialize(jsonReader);
string artistName = jObject.GetValue("name")
.Value <string>()
.ToUpperInvariant();
IEnumerable <JToken> genres = jObject.GetValue("genres")?.Values();
if (genres?.Any() != true)
{
continue;
}
string genreName = genres.FirstOrDefault()
.Children()
.FirstOrDefault()
.Value <string>();
Artist artist = artists
.Where(a => artistName.Equals(a.Name.ToUpperInvariant()))
.FirstOrDefault(a => a.Genre == null);
if (artist == null)
{
continue;
}
artist.Genre = genreName;
Console.WriteLine($"Artist: {artistName}\t Genre: {genreName}");
updatedArtists++;
if (updatedArtists % 100 == 0)
{
Console.WriteLine($"Saving changes... Updated Artist entries: {updatedArtists}");
context.SaveChanges();
}
}
context.SaveChanges();
}
public void Dispose()
{
DataLoaderContext.SetLoaderContext(_prevLoadCtx);
}
public void Dispose()
{
DataLoaderContext.SetLoaderContext(_prevLoadCtx);
SynchronizationContext.SetSynchronizationContext(_prevSyncCtx);
}
internal static void SetLoaderContext(DataLoaderContext context)
{
}
public DataLoaderTaskScheduler(DataLoaderContext context)
{
_context = context;
}
/// <summary>
/// Creates a new <see cref="CollectionDataLoader{TKey,TReturn}"/> bound to a specific context.
/// </summary>
internal CollectionDataLoader(Func <IEnumerable <TKey>, Task <ILookup <TKey, TReturn> > > fetchDelegate, DataLoaderContext context)
: base(context)
{
_fetchDelegate = fetchDelegate;
}