public async Task <ImportStats> ProcessEras(Artist artist, PerformContext ctx)
{
var stats = new ImportStats();
var order = 0;
foreach (var era in (await PhishinApiRequest <IDictionary <string, IList <string> > >("eras", ctx)).data)
{
var dbEra = existingEras.GetValue(era.Key);
if (dbEra == null)
{
dbEra = await _eraService.Save(new Era()
{
artist_id = artist.id,
name = era.Key,
order = order,
updated_at = DateTime.Now
});
existingEras[dbEra.name] = dbEra;
stats.Created++;
}
foreach (var year in era.Value)
{
yearToEraMapping[year] = dbEra;
}
order++;
}
return(stats);
}
public async Task <ImportStats> ProcessSongs(Artist artist, PerformContext ctx)
{
var stats = new ImportStats();
var songsToSave = new List <SetlistSong>();
foreach (var song in (await PhishinApiRequest <IEnumerable <PhishinSmallSong> >("songs", ctx)).data)
{
var dbSong = existingSetlistSongs.GetValue(song.id.ToString());
// skip aliases for now
if (dbSong == null && song.alias_for.HasValue == false)
{
songsToSave.Add(new SetlistSong()
{
updated_at = song.updated_at,
artist_id = artist.id,
name = song.title,
slug = song.slug,
upstream_identifier = song.id.ToString()
});
}
}
var newSongs = await _setlistSongService.InsertAll(artist, songsToSave);
foreach (var s in newSongs)
{
existingSetlistSongs[s.upstream_identifier] = s;
}
stats.Created += newSongs.Count();
return(stats);
}
public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
{
await PreloadData(artist);
var stats = new ImportStats();
ctx?.WriteLine("Processing Eras");
stats += await ProcessEras(artist, ctx);
ctx?.WriteLine("Processing Tours");
stats += await ProcessTours(artist, ctx);
ctx?.WriteLine("Processing Songs");
stats += await ProcessSongs(artist, ctx);
ctx?.WriteLine("Processing Venues");
stats += await ProcessVenues(artist, ctx);
ctx?.WriteLine("Processing Shows");
stats += await ProcessShows(artist, src, ctx);
ctx?.WriteLine("Rebuilding");
await RebuildShows(artist);
await RebuildYears(artist);
return(stats);
//return await ProcessIdentifiers(artist, await this.http.GetAsync(SearchUrlForArtist(artist)));
}
public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
{
uint page = 1;
var stats = new ImportStats();
await PreloadData(artist);
ctx?.WriteLine($"Requesting page #{page}");
while (await ImportPage(artist, stats, ctx, await http.GetAsync(UrlForArtist(src, page))))
{
page++;
await Task.Delay(100);
ctx?.WriteLine($"Requesting page #{page}");
}
ctx?.WriteLine("Updating tour start/end dates");
await UpdateTourStartEndDates(artist);
// update shows
await RebuildShows(artist);
// update years
await RebuildYears(artist);
return(stats);
}
public async Task <ImportStats> Import(Artist artist, Func <ArtistUpstreamSource, bool> filterUpstreamSources, string showIdentifier, PerformContext ctx)
{
var stats = new ImportStats();
var srcs = artist.upstream_sources.ToList();
ctx?.WriteLine($"Found {srcs.Count} valid importers.");
var prog = ctx?.WriteProgressBar();
await srcs.AsyncForEachWithProgress(prog, async item =>
{
if (filterUpstreamSources != null && !filterUpstreamSources(item))
{
ctx?.WriteLine($"Skipping (rejected by filter): {item.upstream_source_id}, {item.upstream_identifier}");
return;
}
ctx?.WriteLine($"Importing with {item.upstream_source_id}, {item.upstream_identifier}");
if (showIdentifier != null)
{
stats += await item.upstream_source.importer.ImportSpecificShowDataForArtist(artist, item, showIdentifier, ctx);
}
else
{
stats += await item.upstream_source.importer.ImportDataForArtist(artist, item, ctx);
}
});
return(stats);
}
public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
{
var stats = new ImportStats();
await PreloadData(artist);
var contents = await FetchUrl(PanicIndexUrl(), ctx);
var matches = ShowDirMatcher.Matches(contents);
ctx?.WriteLine($"Check {matches.Count} subdirectories");
var prog = ctx?.WriteProgressBar();
var counter = 1;
foreach (Match match in ShowDirMatcher.Matches(contents))
{
var panicDate = match.Groups[1].Value;
var panicRecLetter = match.Groups[2].Value;
// 27-Jul-2016 19:14
var panicUpdatedAt = DateTime.ParseExact(match.Groups[3].Value.Trim(), "dd-MMM-yyyy HH:mm", CultureInfo.InvariantCulture);
await ProcessShow(stats, artist, panicDate, panicRecLetter, panicUpdatedAt, ctx);
prog.SetValue(100.0 * counter / matches.Count);
counter++;
}
await RebuildShows(artist);
await RebuildYears(artist);
return(stats);
}
async Task <bool> ImportPage(Artist artist, ImportStats stats, PerformContext ctx, HttpResponseMessage res)
{
var body = await res.Content.ReadAsStringAsync();
var json = JsonConvert.DeserializeObject <IList <PhantasyTourShowListing> >(body);
var prog = ctx?.WriteProgressBar();
await json.AsyncForEachWithProgress(prog, async show =>
{
if (show.showDate.ToUniversalTime() > DateTime.UtcNow)
{
// future shows can't have recordings
return;
}
var showId = UpstreamIdentifierForPhantasyTourId(show.showId);
var venueId = UpstreamIdentifierForPhantasyTourId(show.venueId);
// we have no way to tell if things get updated, so just pull once
if (!existingSetlistShows.ContainsKey(showId))
{
await ImportSingle(artist, stats, ctx, show.showId);
}
});
// once we aren't at the full items per page this is the last page and we should stop
var shouldContinue = json.Count == ITEMS_PER_PAGE;
return(shouldContinue);
}
async Task <Tuple <bool, ImportStats> > ProcessSetlistPage(Artist artist, HttpResponseMessage res, PerformContext ctx, IProgressBar prog)
{
var body = await res.Content.ReadAsStringAsync();
Relisten.Vendor.SetlistFm.SetlistsRootObject root = null;
try
{
root = JsonConvert.DeserializeObject <Relisten.Vendor.SetlistFm.SetlistsRootObject>(
body,
new Vendor.SetlistFm.TolerantListConverter <Vendor.SetlistFm.Song>(),
new Vendor.SetlistFm.TolerantListConverter <Vendor.SetlistFm.Set>(),
new Vendor.SetlistFm.TolerantSetsConverter()
);
}
catch (JsonReaderException e)
{
ctx?.WriteLine("Failed to parse {0}:\n{1}", res.RequestMessage.RequestUri.ToString(), body);
throw e;
}
var stats = new ImportStats();
var count = 1;
foreach (var setlist in root.setlists.setlist)
{
if (setlist.sets.set.Count > 0)
{
Stopwatch s = new Stopwatch();
s.Start();
// ctx?.WriteLine("Indexing setlist: {0}/{1}...", artist.name, setlist.eventDate);
try
{
var thisStats = await ProcessSetlist(artist, setlist);
s.Stop();
// ctx?.WriteLine("...success in {0}! Stats: {1}", s.Elapsed, thisStats);
stats += thisStats;
}
catch (Exception e)
{
s.Stop();
ctx?.WriteLine("{0}/{1}...failed in {2}! Stats: {3}", artist.name, setlist.eventDate, s.Elapsed, e.Message);
throw e;
}
prog.SetValue(((100.0 * (root.setlists.page - 1) * root.setlists.itemsPerPage) + count * 1.0) / root.setlists.total);
count++;
}
}
var hasMorePages = root.setlists.page < Math.Ceiling(1.0 * root.setlists.total / root.setlists.itemsPerPage);
return(new Tuple <bool, ImportStats>(hasMorePages, stats));
}
public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
{
var stats = new ImportStats();
await PreloadData(artist);
var resp = await http.GetAsync(ShowPagesListingUrl(src));
var showFilesResponse = await resp.Content.ReadAsStringAsync();
var showFiles = JsonConvert.DeserializeObject <List <string> >(showFilesResponse);
var files = showFiles
.Select(f =>
{
var fileName = Path.GetFileName(f);
return(new FileMetaObject
{
DisplayDate = fileName.Substring(0, 10),
Date = DateTime.Parse(fileName.Substring(0, 10)),
FilePath = f,
Identifier = fileName.Remove(fileName.LastIndexOf(".html", StringComparison.OrdinalIgnoreCase))
});
})
.ToList()
;
ctx?.WriteLine($"Checking {files.Count} html files");
var prog = ctx?.WriteProgressBar();
await files.AsyncForEachWithProgress(prog, async f =>
{
if (existingSetlistShows.ContainsKey(f.Identifier))
{
return;
}
var url = ShowPageUrl(src, f.FilePath);
var pageResp = await http.GetAsync(url);
var pageContents = await pageResp.Content.ReadAsStringAsync();
await ProcessPage(stats, artist, f, pageContents, pageResp.Content.Headers.LastModified?.UtcDateTime ?? DateTime.UtcNow, ctx);
});
if (artist.features.tours)
{
await UpdateTourStartEndDates(artist);
}
ctx.WriteLine("Rebuilding shows and years");
// update shows
await RebuildShows(artist);
// update years
await RebuildYears(artist);
return(stats);
}
async Task ImportSingle(Artist artist, ImportStats stats, PerformContext ctx, int showId)
{
var policy = Policy
.Handle <JsonReaderException>()
.WaitAndRetryAsync(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt) / 2.0));
await policy.ExecuteAsync(() => _ImportSingle(artist, stats, ctx, showId));
}
public async Task <ImportStats> ProcessShows(Artist artist, PerformContext ctx)
{
var stats = new ImportStats();
var shows = (await PhishinApiRequest <IEnumerable <PhishinSmallShow> >("shows", ctx, "date")).ToList();
var prog = ctx?.WriteProgressBar();
await shows.AsyncForEachWithProgress(prog, async show =>
{
var dbSource = existingSources.GetValue(show.id.ToString());
if (dbSource == null)
{
dbSource = await ProcessShow(stats, artist, new Source()
{
updated_at = show.updated_at,
artist_id = artist.id,
venue_id = existingVenues[show.venue_id.ToString()].id,
display_date = show.date,
upstream_identifier = show.id.ToString(),
is_soundboard = show.sbd,
is_remaster = show.remastered,
description = "",
taper_notes = show.taper_notes
}, ctx);
existingSources[dbSource.upstream_identifier] = dbSource;
stats.Created++;
}
else if (show.updated_at > dbSource.updated_at)
{
dbSource.updated_at = show.updated_at;
dbSource.venue_id = existingVenues[show.venue_id.ToString()].id;
dbSource.display_date = show.date;
dbSource.upstream_identifier = show.id.ToString();
dbSource.is_soundboard = show.sbd;
dbSource.is_remaster = show.remastered;
dbSource.description = "";
dbSource.taper_notes = show.taper_notes;
stats.Removed += await _sourceService.DropAllSetsAndTracksForSource(dbSource);
dbSource = await ProcessShow(stats, artist, dbSource, ctx);
existingSources[dbSource.upstream_identifier] = dbSource;
stats.Updated++;
}
});
return(stats);
}
public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
{
var stats = new ImportStats();
await PreloadData(artist);
var contents = await FetchUrl(PanicIndexUrl(), ctx);
var tracks = JsonConvert.DeserializeObject <List <PanicStream.PanicStreamTrack> >(contents);
var tracksByShow = tracks
.Where(t => t.SourceName != null)
.GroupBy(t => t.ShowDate)
.Select(g => new
{
ShowDate = g.Key,
Sources = g
.GroupBy(subg => subg.SourceName)
.Select(subg => new
{
SourceName = subg.Key,
Tracks = subg.ToList()
})
})
.ToList();
ctx?.WriteLine($"Found {tracksByShow.Count} shows");
var prog = ctx?.WriteProgressBar();
await tracksByShow.AsyncForEachWithProgress(prog, async grp =>
{
foreach (var source in grp.Sources)
{
try {
await ProcessShow(stats, artist, src, grp.ShowDate, source.SourceName, source.Tracks, ctx);
}
catch (Exception e) {
ctx?.WriteLine("EXCEPTION: " + e.Message);
ctx?.WriteLine("Source name: " + source.SourceName);
ctx?.WriteLine(e.ToString());
ctx?.WriteLine(JsonConvert.SerializeObject(source));
}
}
});
ctx?.WriteLine("Rebuilding shows...");
await RebuildShows(artist);
ctx?.WriteLine("Rebuilding years...");
await RebuildYears(artist);
return(stats);
}
public async Task <ImportStats> ProcessVenues(Artist artist, PerformContext ctx)
{
var stats = new ImportStats();
foreach (var venue in (await PhishinApiRequest <IEnumerable <PhishinSmallVenue> >("venues", ctx)).data)
{
var dbVenue = existingVenues.GetValue(venue.id.ToString());
if (dbVenue == null)
{
var sc = new VenueWithShowCount()
{
updated_at = venue.updated_at,
artist_id = artist.id,
name = venue.name,
location = venue.location,
slug = Slugify(venue.name),
latitude = venue.latitude,
longitude = venue.longitude,
past_names = venue.past_names,
upstream_identifier = venue.id.ToString()
};
var createdDb = await _venueService.Save(sc);
sc.id = createdDb.id;
existingVenues[sc.upstream_identifier] = sc;
stats.Created++;
dbVenue = sc;
}
else if (venue.updated_at > dbVenue.updated_at)
{
dbVenue.name = venue.name;
dbVenue.location = venue.location;
dbVenue.longitude = venue.longitude;
dbVenue.latitude = venue.latitude;
dbVenue.past_names = venue.past_names;
dbVenue.updated_at = venue.updated_at;
await _venueService.Save(dbVenue);
existingVenues[dbVenue.upstream_identifier] = dbVenue;
stats.Updated++;
}
}
return(stats);
}
private async Task ProcessSetlistShow(ImportStats stats, PhishinShow show, Artist artist, Source dbSource, IDictionary <string, SourceSet> sets)
{
var dbShow = existingSetlistShows.GetValue(show.date);
var addSongs = false;
if (dbShow == null)
{
dbShow = await _setlistShowService.Save(new SetlistShow()
{
artist_id = artist.id,
upstream_identifier = show.date,
date = DateTime.Parse(show.date),
venue_id = existingVenues[show.venue.id.ToString()].id,
tour_id = existingTours[show.tour_id.ToString()].id,
era_id = yearToEraMapping.GetValue(show.date.Substring(0, 4), yearToEraMapping["1983-1987"]).id,
updated_at = dbSource.updated_at
});
stats.Created++;
addSongs = true;
}
else if (show.updated_at > dbShow.updated_at)
{
dbShow.date = DateTime.Parse(show.date);
dbShow.venue_id = existingVenues[show.venue.id.ToString()].id;
dbShow.tour_id = existingTours[show.tour_id.ToString()].id;
dbShow.era_id = yearToEraMapping.GetValue(show.date.Substring(0, 4), yearToEraMapping["1983-1987"]).id;
dbShow.updated_at = dbSource.updated_at;
dbShow = await _setlistShowService.Save(dbShow);
stats.Updated++;
stats.Removed += await _setlistShowService.RemoveSongPlays(dbShow);
addSongs = true;
}
if (addSongs)
{
var dbSongs = show.tracks.
SelectMany(phishinTrack => phishinTrack.song_ids.Select(song_id => existingSetlistSongs.GetValue(song_id.ToString()))).
Where(t => t != null).
GroupBy(t => t.upstream_identifier).
Select(g => g.First()).
ToList()
;
stats.Created += await _setlistShowService.AddSongPlays(dbShow, dbSongs);
}
}
private async Task <ImportStats> ProcessSource(Artist artist, Source dbSource, PerformContext ctx)
{
var stats = new ImportStats();
var ratings = await ScrapePhishNetForSource(dbSource, ctx);
var dirty = false;
if (dbSource.num_ratings != ratings.RatingVotesCast)
{
dbSource.num_ratings = ratings.RatingVotesCast;
dbSource.avg_rating = ratings.RatingAverage;
dirty = true;
}
if (dbSource.num_reviews != ratings.NumberOfReviewsWritten)
{
var reviewsTask = GetPhishNetApiReviews(dbSource, ctx);
var setlistTask = GetPhishNetApiSetlist(dbSource, ctx);
await Task.WhenAll(reviewsTask, setlistTask);
var dbReviews = reviewsTask.Result.Select(rev =>
{
return(new SourceReview()
{
rating = null,
title = null,
review = rev.review,
author = rev.author,
updated_at = DateTimeOffset.FromUnixTimeSeconds(rev.tstamp).UtcDateTime
});
}).ToList();
dbSource.num_reviews = dbReviews.Count();
dbSource.description = setlistTask.Result.setlistnotes + "\n\n\n" + setlistTask.Result.setlistdata;
dirty = true;
await ReplaceSourceReviews(stats, dbSource, dbReviews);
}
if (dirty)
{
await _sourceService.Save(dbSource);
stats.Updated++;
}
return(stats);
}
private async Task <IEnumerable <SourceReview> > ReplaceSourceReviews(ImportStats stats, Source source, IEnumerable <SourceReview> reviews)
{
foreach (var review in reviews)
{
review.source_id = source.id;
}
var res = await _sourceReviewService.UpdateAll(source, reviews);
stats.Created += res.Count();
return(res);
}
private async Task <Source> ProcessShow(ImportStats stats, Artist artist, Source dbSource, PerformContext ctx)
{
var fullShow = await PhishinApiRequest <PhishinShow>("shows/" + dbSource.upstream_identifier, ctx);
dbSource.has_jamcharts = fullShow.tags.Contains("Jamcharts");
dbSource = await _sourceService.Save(dbSource);
var sets = new Dictionary <string, SourceSet>();
foreach (var track in fullShow.tracks)
{
var set = sets.GetValue(track.set);
if (set == null)
{
set = await _sourceSetService.Insert(new SourceSet()
{
source_id = dbSource.id,
index = SetIndexForIdentifier(track.set),
name = track.set_name,
is_encore = track.set[0] == 'E',
updated_at = dbSource.updated_at
});
// this needs to be set after loading from the db
set.tracks = new List <SourceTrack>();
stats.Created++;
sets[track.set] = set;
}
set.tracks.Add(new SourceTrack()
{
source_set_id = set.id,
source_id = dbSource.id,
title = track.title,
duration = track.duration,
track_position = track.position,
slug = Slugify(track.title),
mp3_url = track.mp3,
updated_at = dbSource.updated_at
});
}
stats.Created += (await _sourceTrackService.InsertAll(sets.SelectMany(kvp => kvp.Value.tracks))).Count();
await ProcessSetlistShow(stats, fullShow, artist, dbSource, sets);
return(dbSource);
}
private async Task <IEnumerable <SourceReview> > ReplaceSourceReviews(ImportStats stats, Source source, IEnumerable <SourceReview> reviews)
{
stats.Removed += await _sourceReviewService.RemoveAllForSource(source);
foreach (var review in reviews)
{
review.source_id = source.id;
}
var res = await _sourceReviewService.InsertAll(reviews);
stats.Created += res.Count();
return(res);
}
public async Task <ImportStats> Import(Artist artist, PerformContext ctx)
{
var stats = new ImportStats();
var srcs = artist.upstream_sources.ToList();
ctx?.WriteLine($"Found {srcs.Count} valid importers.");
var prog = ctx?.WriteProgressBar();
await srcs.AsyncForEachWithProgress(prog, async item =>
{
ctx?.WriteLine($"Importing with {item.GetType()}");
stats += await item.upstream_source.importer.ImportDataForArtist(artist, item, ctx);
});
return(stats);
}
private async Task <ImportStats> ProcessIdentifiers(Artist artist, HttpResponseMessage res, PerformContext ctx)
{
var stats = new ImportStats();
var json = await res.Content.ReadAsStringAsync();
var root = JsonConvert.DeserializeObject <Relisten.Vendor.ArchiveOrg.SearchRootObject>(
json,
new Relisten.Vendor.ArchiveOrg.TolerantArchiveDateTimeConverter()
);
ctx?.WriteLine($"Checking {root.response.docs.Count} archive.org results");
var prog = ctx?.WriteProgressBar();
await root.response.docs.AsyncForEachWithProgress(prog, async doc =>
{
var dbShow = existingSources.GetValue(doc.identifier);
if (dbShow == null ||
doc._iguana_updated_at > dbShow.updated_at)
{
ctx?.WriteLine("Pulling https://archive.org/metadata/{0}", doc.identifier);
var detailRes = await http.GetAsync(DetailsUrlForIdentifier(doc.identifier));
var detailsJson = await detailRes.Content.ReadAsStringAsync();
var detailsRoot = JsonConvert.DeserializeObject <Relisten.Vendor.ArchiveOrg.Metadata.RootObject>(
detailsJson,
new Vendor.ArchiveOrg.TolerantStringConverter()
);
stats += await ImportSingleIdentifier(artist, dbShow, doc, detailsRoot, ctx);
}
});
// update shows
await RebuildShows(artist);
// update years
await RebuildYears(artist);
return(stats);
}
public async Task <ImportStats> ProcessTours(Artist artist, PerformContext ctx)
{
var stats = new ImportStats();
foreach (var tour in (await PhishinApiRequest <IEnumerable <PhishinSmallTour> >("tours", ctx)).data)
{
var dbTour = existingTours.GetValue(tour.id.ToString());
if (dbTour == null)
{
dbTour = await _tourService.Save(new Tour
{
updated_at = tour.updated_at,
artist_id = artist.id,
start_date = DateTime.Parse(tour.starts_on),
end_date = DateTime.Parse(tour.ends_on),
name = tour.name,
slug = Slugify(tour.name),
upstream_identifier = tour.id.ToString()
});
existingTours[dbTour.upstream_identifier] = dbTour;
stats.Created++;
}
else if (tour.updated_at > dbTour.updated_at)
{
dbTour.start_date = DateTime.Parse(tour.starts_on);
dbTour.end_date = DateTime.Parse(tour.ends_on);
dbTour.name = tour.name;
dbTour = await _tourService.Save(dbTour);
existingTours[dbTour.upstream_identifier] = dbTour;
stats.Updated++;
}
}
return(stats);
}
async Task <bool> ImportPage(Artist artist, ImportStats stats, PerformContext ctx, HttpResponseMessage res)
{
var body = await res.Content.ReadAsStringAsync();
var json = JsonConvert.DeserializeObject <IList <PhantasyTourShowListing> >(body);
if (json == null)
{
ctx?.WriteLine("Improper response from phantasytour.com: " + body);
ctx?.WriteLine($"Status code: {res.StatusCode}. Headers: {string.Join("\n", res.Headers.Select(h => h.Key + ": " + string.Join(" || ", h.Value)))}");
ctx?.WriteLine($"Request url: {res.RequestMessage.RequestUri}. Headers: {string.Join("\n", res.RequestMessage.Headers.Select(h => h.Key + ": " + string.Join(" || ", h.Value)))}");
return(false);
}
var prog = ctx?.WriteProgressBar();
await json.AsyncForEachWithProgress(prog, async show =>
{
if (show.dateTime.ToUniversalTime() > DateTime.UtcNow)
{
// future shows can't have recordings
return;
}
var showId = UpstreamIdentifierForPhantasyTourId(show.id);
var venueId = UpstreamIdentifierForPhantasyTourId(show.venue.id);
// we have no way to tell if things get updated, so just pull once
if (!existingSetlistShows.ContainsKey(showId))
{
await ImportSingle(artist, stats, ctx, show.id);
}
});
// once we aren't at the full items per page this is the last page and we should stop
var shouldContinue = json.Count == ITEMS_PER_PAGE;
return(shouldContinue);
}
public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
{
var stats = new ImportStats();
var shows = (await _sourceService.AllForArtist(artist)).OrderBy(s => s.display_date).ToList();
var prog = ctx?.WriteProgressBar();
ctx?.WriteLine($"Processing {shows.Count} shows");
await shows.ForEachAsync(async dbSource =>
{
stats += await ProcessSource(artist, src, dbSource, ctx);
}, prog, 1);
ctx?.WriteLine("Rebuilding...");
await RebuildShows(artist);
await RebuildYears(artist);
return(stats);
}
public override async Task <ImportStats> ImportDataForArtist(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
{
var stats = new ImportStats();
await PreloadData(artist);
var files = Directory.EnumerateFiles(ShowFilesDirectory)
.Where(f => f.EndsWith(".html", StringComparison.OrdinalIgnoreCase))
.Select(f =>
{
var fileName = Path.GetFileName(f);
return(new FileMetaObject
{
DisplayDate = fileName.Substring(0, 10),
Date = DateTime.Parse(fileName.Substring(0, 10)),
FilePath = f,
Identifier = fileName.Remove(fileName.LastIndexOf(".html", StringComparison.OrdinalIgnoreCase))
});
})
.ToList()
;
ctx?.WriteLine($"Checking {files.Count} html files");
var prog = ctx?.WriteProgressBar();
await files.AsyncForEachWithProgress(prog, async f =>
{
await ProcessPage(stats, artist, f, File.ReadAllText(f.FilePath), ctx);
});
if (artist.features.tours)
{
await UpdateTourStartEndDates(artist);
}
return(stats);
}
private async Task ProcessShow(ImportStats stats, Artist artist, ArtistUpstreamSource upstreamSrc, string showDate, string sourceName, IList <PanicStream.PanicStreamTrack> sourceTracks, PerformContext ctx)
{
var upstreamId = sourceName;
var dbSource = existingSources.GetValue(upstreamId);
var panicUpdatedAt = sourceTracks
.Where(t => t.System.ParsedModificationTime.HasValue)
.Max(t => t.System.ParsedModificationTime.Value);
if (dbSource != null && dbSource.updated_at <= panicUpdatedAt)
{
return;
}
var isUpdate = dbSource != null;
var src = new Source
{
artist_id = artist.id,
display_date = showDate,
is_soundboard = false,
is_remaster = false,
has_jamcharts = false,
avg_rating = 0,
num_reviews = 0,
avg_rating_weighted = 0,
upstream_identifier = upstreamId,
taper_notes = "",
updated_at = panicUpdatedAt
};
if (isUpdate)
{
src.id = dbSource.id;
}
dbSource = await _sourceService.Save(src);
existingSources[dbSource.upstream_identifier] = dbSource;
if (isUpdate)
{
stats.Updated++;
}
else
{
stats.Created++;
stats.Created += (await linkService.AddLinksForSource(dbSource, new[]
{
new Link
{
source_id = dbSource.id,
for_ratings = false,
for_source = true,
for_reviews = false,
upstream_source_id = upstreamSrc.upstream_source_id,
url = $"https://www.panicstream.com/vault/widespread-panic/{dbSource.display_date.Substring(0, 4)}-streams/",
label = "View show page on panicstream.com"
}
})).Count();
}
var dbSet = await _sourceSetService.Update(dbSource, new SourceSet
{
source_id = dbSource.id,
index = 0,
is_encore = false,
name = "Default Set",
updated_at = panicUpdatedAt
});
stats.Created++;
var trackIndex = 0;
var mp3s = sourceTracks
.OrderBy(t => t.FileName)
.Select(t =>
{
var trackName = t.FileName
.Replace(".mp3", "")
.Replace(".MP3", "")
.Replace(".M4A", "")
.Replace(".m4a", "")
.Trim();
var cleanedTrackName = Regex.Replace(trackName, @"(wsp[0-9-]+d\d+t\d+\.)|(^\d+ ?-? ?)", "").Trim();
if (cleanedTrackName.Length != 0)
{
trackName = cleanedTrackName;
}
trackIndex++;
return(new SourceTrack
{
source_id = dbSource.id,
source_set_id = dbSet.id,
track_position = trackIndex,
duration = ((int?)t.Composite?.CalculatedDuration.TotalSeconds ?? (int?)0).Value,
title = trackName,
slug = SlugifyTrack(trackName),
mp3_url = t.AbsoluteUrl(_configuration["PANIC_KEY"]),
updated_at = panicUpdatedAt,
artist_id = artist.id
});
});
ResetTrackSlugCounts();
await _sourceTrackService.InsertAll(dbSource, mp3s);
stats.Created += mp3s.Count();
}
async Task _ImportSingle(Artist artist, ImportStats stats, PerformContext ctx, int showId)
{
ctx?.WriteLine($"Requesting page for show id {showId}");
var res = await http.GetAsync(UrlForShow(showId));
var body = await res.Content.ReadAsStringAsync();
ctx?.WriteLine($"Result: [{res.StatusCode}]: {body.Length}");
var json = JsonConvert.DeserializeObject <PhantasyTourEnvelope>(body).data;
var now = DateTime.UtcNow;
Venue dbVenue = existingVenues.GetValue(UpstreamIdentifierForPhantasyTourId(json.venue.id));
if (dbVenue == null)
{
var sc = new VenueWithShowCount()
{
updated_at = now,
artist_id = artist.id,
name = json.venue.name,
latitude = null,
longitude = null,
location = $"{json.venue.city}, {json.venue.state}, {json.venue.country}",
upstream_identifier = UpstreamIdentifierForPhantasyTourId(json.venue.id),
slug = Slugify($"{json.venue.name}, {json.venue.city}, {json.venue.state}")
};
dbVenue = await _venueService.Save(sc);
sc.id = dbVenue.id;
existingVenues[dbVenue.upstream_identifier] = sc;
stats.Created++;
}
ctx?.WriteLine($"Importing show id {showId}");
// tour
Tour dbTour = null;
if (artist.features.tours)
{
var tour_name = json.tour?.name ?? "Not Part of a Tour";
var tour_upstream = UpstreamIdentifierForPhantasyTourId(json.tour?.id ?? -1);
dbTour = existingTours.GetValue(tour_upstream);
if (dbTour == null)
{
dbTour = await _tourService.Save(new Tour()
{
updated_at = now,
artist_id = artist.id,
start_date = null,
end_date = null,
name = tour_name,
slug = Slugify(tour_name),
upstream_identifier = tour_upstream
});
existingTours[dbTour.upstream_identifier] = dbTour;
stats.Created++;
}
}
var dbShow = existingSetlistShows.GetValue(UpstreamIdentifierForPhantasyTourId(json.id));
var date = json.dateTimeUtc.Date;
if (dbShow == null)
{
dbShow = await _setlistShowService.Save(new SetlistShow()
{
artist_id = artist.id,
updated_at = now,
date = date,
upstream_identifier = UpstreamIdentifierForPhantasyTourId(json.id),
venue_id = dbVenue.id,
tour_id = artist.features.tours ? dbTour?.id : null
});
existingSetlistShows[dbShow.upstream_identifier] = dbShow;
stats.Created++;
}
// phantasy tour doesn't provide much info about tours so we need to find the start
// and end date ourselves.
if (artist.features.tours &&
(dbTour.start_date == null ||
dbTour.end_date == null ||
dbShow.date < dbTour.start_date ||
dbShow.date > dbTour.end_date))
{
if (!tourToStartDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date < tourToStartDate[dbTour.upstream_identifier])
{
tourToStartDate[dbTour.upstream_identifier] = dbShow.date;
}
if (!tourToEndDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date > tourToEndDate[dbTour.upstream_identifier])
{
tourToEndDate[dbTour.upstream_identifier] = dbShow.date;
}
}
var songs = json.sets.
SelectMany(set => set.songs).
Select(song => new { Name = song.name, Slug = Slugify(song.name) }).
GroupBy(song => song.Slug).
Select(grp => grp.First()).
ToList();
var dbSongs = existingSetlistSongs.
Where(kvp => songs.Select(song => song.Slug).Contains(kvp.Key)).
Select(kvp => kvp.Value).
ToList();
if (songs.Count != dbSongs.Count)
{
var newSongs = songs.
Where(song => dbSongs.Find(dbSong => dbSong.slug == song.Slug) == null).
Select(song => new SetlistSong()
{
artist_id = artist.id,
name = song.Name,
slug = song.Slug,
updated_at = now,
upstream_identifier = song.Slug
}).
ToList();
var justAdded = await _setlistSongService.InsertAll(artist, newSongs);
dbSongs.AddRange(justAdded);
stats.Created += newSongs.Count;
foreach (var justAddedSong in justAdded)
{
existingSetlistSongs[justAddedSong.upstream_identifier] = justAddedSong;
}
}
stats += await _setlistShowService.UpdateSongPlays(dbShow, dbSongs);
}
private async Task ProcessPage(ImportStats stats, Artist artist, FileMetaObject meta, string pageContents, PerformContext ctx)
{
var dbShow = existingSetlistShows.GetValue(meta.Identifier);
if (dbShow != null)
{
return;
}
var html = new HtmlDocument();
html.LoadHtml(pageContents);
var root = html.DocumentNode;
var ps = root.DescendantsWithClass("venue-name").ToList();
var bandName = ps[1].InnerText.CollapseSpacesAndTrim();
var venueName = ps[0].InnerText.CollapseSpacesAndTrim();
var venueCityOrCityState = root.DescendantsWithClass("venue-address").Single().InnerText.CollapseSpacesAndTrim().TrimEnd(',');
var venueCountry = root.DescendantsWithClass("venue-country").Single().InnerText.CollapseSpacesAndTrim();
string tourName = ps.Count > 2 ? ps[2].InnerText.CollapseSpacesAndTrim() : "Not Part of a Tour";
var venueUpstreamId = "jerrygarcia.com_" + venueName;
var dbVenue = existingVenues.GetValue(venueUpstreamId);
if (dbVenue == null)
{
dbVenue = await _venueService.Save(new Venue
{
artist_id = artist.id,
name = venueName,
location = venueCityOrCityState + ", " + venueCountry,
upstream_identifier = venueUpstreamId,
slug = Slugify(venueName)
});
existingVenues[dbVenue.upstream_identifier] = dbVenue;
stats.Created++;
}
var dbTour = existingTours.GetValue(tourName);
if (dbTour == null && artist.features.tours)
{
dbTour = await _tourService.Save(new Tour
{
artist_id = artist.id,
name = tourName,
slug = Slugify(tourName),
upstream_identifier = tourName
});
existingTours[dbTour.upstream_identifier] = dbTour;
stats.Created++;
}
dbShow = await _setlistShowService.Save(new SetlistShow
{
artist_id = artist.id,
tour_id = dbTour?.id,
venue_id = dbVenue.id,
date = meta.Date,
upstream_identifier = meta.Identifier
});
existingSetlistShows[dbShow.upstream_identifier] = dbShow;
stats.Created++;
var dbSongs = root.Descendants("ol")
.SelectMany(node => node.Descendants("li"))
.Select(node =>
{
var trackName = node.InnerText.Trim().TrimEnd('>', '*', ' ');
var slug = Slugify(trackName);
return(new SetlistSong
{
artist_id = artist.id,
name = trackName,
slug = slug,
upstream_identifier = slug
});
})
.GroupBy(s => s.upstream_identifier)
.Select(g => g.First())
.ToList()
;
var dbSongsToAdd = dbSongs.Where(song => !existingSetlistSongs.ContainsKey(song.upstream_identifier));
dbSongs = dbSongs.Where(song => existingSetlistSongs.ContainsKey(song.upstream_identifier))
.Select(song => existingSetlistSongs[song.upstream_identifier])
.ToList()
;
var added = await _setlistSongService.InsertAll(artist, dbSongsToAdd);
foreach (var s in added)
{
existingSetlistSongs[s.upstream_identifier] = s;
}
stats.Created += added.Count();
dbSongs.AddRange(added);
if (artist.features.tours &&
(dbTour.start_date == null ||
dbTour.end_date == null ||
dbShow.date < dbTour.start_date ||
dbShow.date > dbTour.end_date))
{
if (!tourToStartDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date < tourToStartDate[dbTour.upstream_identifier])
{
tourToStartDate[dbTour.upstream_identifier] = dbShow.date;
}
if (!tourToEndDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date > tourToEndDate[dbTour.upstream_identifier])
{
tourToEndDate[dbTour.upstream_identifier] = dbShow.date;
}
}
await _setlistShowService.AddSongPlays(dbShow, dbSongs);
}
public async Task <ImportStats> ProcessShows(Artist artist, ArtistUpstreamSource src, PerformContext ctx)
{
var stats = new ImportStats();
var pages = 80;
var prog = ctx?.WriteProgressBar();
var pageSize = 20;
for (var currentPage = 1; currentPage <= pages; currentPage++)
{
var apiShows = await PhishinApiRequest <IEnumerable <PhishinShow> >("shows", ctx, "date", per_page : pageSize, page : currentPage);
pages = apiShows.total_pages;
var shows = apiShows.data.ToList();
foreach (var(idx, show) in shows.Select((s, i) => (i, s)))
{
try
{
await processShow(show);
}
catch (Exception e)
{
ctx?.WriteLine($"Error processing show (but continuing): {show.date} (id: {show.id})");
ctx?.LogException(e);
}
prog?.SetValue(100.0 * ((currentPage - 1) * pageSize + idx + 1) / apiShows.total_entries);
}
}
async Task processShow(PhishinShow show)
{
using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
var dbSource = existingSources.GetValue(show.id.ToString());
if (dbSource == null)
{
dbSource = await ProcessShow(stats, artist, show, src, new Source()
{
updated_at = show.updated_at,
artist_id = artist.id,
venue_id = existingVenues[show.venue.id.ToString()].id,
display_date = show.date,
upstream_identifier = show.id.ToString(),
is_soundboard = show.sbd,
is_remaster = show.remastered,
description = "",
taper_notes = show.taper_notes
}, ctx);
existingSources[dbSource.upstream_identifier] = dbSource;
stats.Created++;
stats.Created += (await linkService.AddLinksForSource(dbSource, new[] {
new Link
{
source_id = dbSource.id,
for_ratings = false,
for_source = true,
for_reviews = false,
upstream_source_id = src.upstream_source_id,
url = $"http://phish.in/{dbSource.display_date}",
label = "View on phish.in"
}
})).Count();
}
else if (show.updated_at > dbSource.updated_at)
{
dbSource.updated_at = show.updated_at;
dbSource.venue_id = existingVenues[show.venue.id.ToString()].id;
dbSource.display_date = show.date;
dbSource.upstream_identifier = show.id.ToString();
dbSource.is_soundboard = show.sbd;
dbSource.is_remaster = show.remastered;
dbSource.description = "";
dbSource.taper_notes = show.taper_notes;
dbSource = await ProcessShow(stats, artist, show, src, dbSource, ctx);
existingSources[dbSource.upstream_identifier] = dbSource;
stats.Updated++;
}
scope.Complete();
}
}
return(stats);
}
async Task <ImportStats> ProcessSetlist(Artist artist, Relisten.Vendor.SetlistFm.Setlist setlist)
{
var stats = new ImportStats();
var now = DateTime.UtcNow;
// venue
Venue dbVenue = existingVenues.GetValue(setlist.venue._iguanaUpstreamId);
if (dbVenue == null)
{
var sc = new VenueWithShowCount()
{
updated_at = now,
artist_id = artist.id,
name = setlist.venue.name,
latitude = setlist.venue.city.coords?.lat,
longitude = setlist.venue.city.coords?.@long,
location = $"{setlist.venue.city.name}, {setlist.venue.city.state}",
upstream_identifier = setlist.venue._iguanaUpstreamId,
slug = Slugify(setlist.venue.name)
};
dbVenue = await _venueService.Save(sc);
sc.id = dbVenue.id;
existingVenues[dbVenue.upstream_identifier] = sc;
stats.Created++;
}
// tour
Tour dbTour = null;
if (artist.features.tours)
{
var tour_upstream = setlist.tour?.name ?? "Not Part of a Tour";
dbTour = existingTours.GetValue(tour_upstream);
if (dbTour == null)
{
dbTour = await _tourService.Save(new Tour()
{
updated_at = now,
artist_id = artist.id,
start_date = null,
end_date = null,
name = tour_upstream,
slug = Slugify(tour_upstream),
upstream_identifier = tour_upstream
});
existingTours[dbTour.upstream_identifier] = dbTour;
stats.Created++;
}
}
// show
var dbShow = existingSetlistShows.GetValue(setlist.id);
var date = DateTime.ParseExact(setlist.eventDate, "dd-MM-yyyy", null);
var setlistLastUpdated = setlist.lastUpdated;
var shouldAddSongs = false;
if (dbShow == null)
{
dbShow = await _setlistShowService.Save(new SetlistShow()
{
artist_id = artist.id,
updated_at = setlistLastUpdated,
date = date,
upstream_identifier = setlist.id,
venue_id = dbVenue.id,
tour_id = artist.features.tours ? dbTour?.id : null
});
existingSetlistShows[dbShow.upstream_identifier] = dbShow;
stats.Created++;
shouldAddSongs = true;
}
else if (setlistLastUpdated > dbShow.updated_at)
{
dbShow.artist_id = artist.id;
dbShow.updated_at = setlistLastUpdated;
dbShow.date = date;
dbShow.venue_id = dbVenue.id;
dbShow.tour_id = dbTour?.id;
dbShow = await _setlistShowService.Save(dbShow);
existingSetlistShows[dbShow.upstream_identifier] = dbShow;
stats.Updated++;
shouldAddSongs = true;
}
// setlist.fm doesn't provide much info about tours so we need to find the start
// and end date ourselves.
if (artist.features.tours &&
(dbTour.start_date == null ||
dbTour.end_date == null ||
dbShow.date < dbTour.start_date ||
dbShow.date > dbTour.end_date))
{
if (!tourToStartDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date < tourToStartDate[dbTour.upstream_identifier])
{
tourToStartDate[dbTour.upstream_identifier] = dbShow.date;
}
if (!tourToEndDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date > tourToEndDate[dbTour.upstream_identifier])
{
tourToEndDate[dbTour.upstream_identifier] = dbShow.date;
}
}
if (shouldAddSongs)
{
var songs = setlist.sets.set.
SelectMany(set => set.song).
Select(song => new { Name = song.name, Slug = Slugify(song.name) }).
GroupBy(song => song.Slug).
Select(grp => grp.First()).
ToList();
var dbSongs = existingSetlistSongs.
Where(kvp => songs.Select(song => song.Slug).Contains(kvp.Key)).
Select(kvp => kvp.Value).
ToList();
if (songs.Count != dbSongs.Count)
{
var newSongs = songs.
Where(song => dbSongs.Find(dbSong => dbSong.slug == song.Slug) == null).
Select(song => new SetlistSong()
{
artist_id = artist.id,
name = song.Name,
slug = song.Slug,
updated_at = now,
upstream_identifier = song.Slug
}).
ToList();
var justAdded = await _setlistSongService.InsertAll(artist, newSongs);
dbSongs.AddRange(justAdded);
stats.Created += newSongs.Count;
foreach (var justAddedSong in justAdded)
{
existingSetlistSongs[justAddedSong.upstream_identifier] = justAddedSong;
}
}
stats += await _setlistShowService.UpdateSongPlays(dbShow, dbSongs);
}
return(stats);
}
private async Task <ImportStats> ProcessSource(Artist artist, ArtistUpstreamSource src, Source dbSource, PerformContext ctx)
{
var stats = new ImportStats();
var ratings = await ScrapePhishNetForSource(dbSource, ctx);
var dirty = false;
if (dbSource.num_ratings != ratings.RatingVotesCast)
{
dbSource.num_ratings = ratings.RatingVotesCast;
dbSource.avg_rating = ratings.RatingAverage * 2.0;
dirty = true;
}
if (dbSource.num_reviews != ratings.NumberOfReviewsWritten)
{
var reviewsTask = GetPhishNetApiReviews(dbSource, ctx);
var setlistTask = GetPhishNetApiSetlist(dbSource, ctx);
await Task.WhenAll(reviewsTask, setlistTask);
var dbReviews = reviewsTask.Result.Select(rev =>
{
return(new SourceReview()
{
rating = null,
title = null,
review = rev.review,
author = rev.author,
updated_at = DateTimeOffset.FromUnixTimeSeconds(rev.tstamp).UtcDateTime
});
}).ToList();
dbSource.num_reviews = dbReviews.Count();
dbSource.description = setlistTask.Result.setlistnotes + "\n\n\n" + setlistTask.Result.setlistdata;
dirty = true;
await ReplaceSourceReviews(stats, dbSource, dbReviews);
}
if (dirty)
{
await _sourceService.Save(dbSource);
stats.Updated++;
}
stats.Created += (await linkService.AddLinksForSource(dbSource, new[]
{
new Link
{
source_id = dbSource.id,
for_ratings = true,
for_source = false,
for_reviews = true,
upstream_source_id = src.upstream_source_id,
url = PhishNetUrlForSource(dbSource),
label = "View on phish.net"
}
})).Count();
return(stats);
}
