public async Task <ImportStats> ProcessVenues(Artist artist, PerformContext ctx)
{
var stats = new ImportStats();
foreach (var venue in (await PhishinApiRequest <IEnumerable <PhishinSmallVenue> >("venues", ctx)).data)
{
var dbVenue = existingVenues.GetValue(venue.id.ToString());
if (dbVenue == null)
{
var sc = new VenueWithShowCount()
{
updated_at = venue.updated_at,
artist_id = artist.id,
name = venue.name,
location = venue.location,
slug = Slugify(venue.name),
latitude = venue.latitude,
longitude = venue.longitude,
past_names = venue.past_names,
upstream_identifier = venue.id.ToString()
};
var createdDb = await _venueService.Save(sc);
sc.id = createdDb.id;
existingVenues[sc.upstream_identifier] = sc;
stats.Created++;
dbVenue = sc;
}
else if (venue.updated_at > dbVenue.updated_at)
{
dbVenue.name = venue.name;
dbVenue.location = venue.location;
dbVenue.longitude = venue.longitude;
dbVenue.latitude = venue.latitude;
dbVenue.past_names = venue.past_names;
dbVenue.updated_at = venue.updated_at;
await _venueService.Save(dbVenue);
existingVenues[dbVenue.upstream_identifier] = dbVenue;
stats.Updated++;
}
}
return(stats);
}
async Task _ImportSingle(Artist artist, ImportStats stats, PerformContext ctx, int showId)
{
ctx?.WriteLine($"Requesting page for show id {showId}");
var res = await http.GetAsync(UrlForShow(showId));
var body = await res.Content.ReadAsStringAsync();
ctx?.WriteLine($"Result: [{res.StatusCode}]: {body.Length}");
var json = JsonConvert.DeserializeObject <PhantasyTourEnvelope>(body).data;
var now = DateTime.UtcNow;
Venue dbVenue = existingVenues.GetValue(UpstreamIdentifierForPhantasyTourId(json.venue.id));
if (dbVenue == null)
{
var sc = new VenueWithShowCount()
{
updated_at = now,
artist_id = artist.id,
name = json.venue.name,
latitude = null,
longitude = null,
location = $"{json.venue.city}, {json.venue.state}, {json.venue.country}",
upstream_identifier = UpstreamIdentifierForPhantasyTourId(json.venue.id),
slug = Slugify($"{json.venue.name}, {json.venue.city}, {json.venue.state}")
};
dbVenue = await _venueService.Save(sc);
sc.id = dbVenue.id;
existingVenues[dbVenue.upstream_identifier] = sc;
stats.Created++;
}
ctx?.WriteLine($"Importing show id {showId}");
// tour
Tour dbTour = null;
if (artist.features.tours)
{
var tour_name = json.tour?.name ?? "Not Part of a Tour";
var tour_upstream = UpstreamIdentifierForPhantasyTourId(json.tour?.id ?? -1);
dbTour = existingTours.GetValue(tour_upstream);
if (dbTour == null)
{
dbTour = await _tourService.Save(new Tour()
{
updated_at = now,
artist_id = artist.id,
start_date = null,
end_date = null,
name = tour_name,
slug = Slugify(tour_name),
upstream_identifier = tour_upstream
});
existingTours[dbTour.upstream_identifier] = dbTour;
stats.Created++;
}
}
var dbShow = existingSetlistShows.GetValue(UpstreamIdentifierForPhantasyTourId(json.id));
var date = json.dateTimeUtc.Date;
if (dbShow == null)
{
dbShow = await _setlistShowService.Save(new SetlistShow()
{
artist_id = artist.id,
updated_at = now,
date = date,
upstream_identifier = UpstreamIdentifierForPhantasyTourId(json.id),
venue_id = dbVenue.id,
tour_id = artist.features.tours ? dbTour?.id : null
});
existingSetlistShows[dbShow.upstream_identifier] = dbShow;
stats.Created++;
}
// phantasy tour doesn't provide much info about tours so we need to find the start
// and end date ourselves.
if (artist.features.tours &&
(dbTour.start_date == null ||
dbTour.end_date == null ||
dbShow.date < dbTour.start_date ||
dbShow.date > dbTour.end_date))
{
if (!tourToStartDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date < tourToStartDate[dbTour.upstream_identifier])
{
tourToStartDate[dbTour.upstream_identifier] = dbShow.date;
}
if (!tourToEndDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date > tourToEndDate[dbTour.upstream_identifier])
{
tourToEndDate[dbTour.upstream_identifier] = dbShow.date;
}
}
var songs = json.sets.
SelectMany(set => set.songs).
Select(song => new { Name = song.name, Slug = Slugify(song.name) }).
GroupBy(song => song.Slug).
Select(grp => grp.First()).
ToList();
var dbSongs = existingSetlistSongs.
Where(kvp => songs.Select(song => song.Slug).Contains(kvp.Key)).
Select(kvp => kvp.Value).
ToList();
if (songs.Count != dbSongs.Count)
{
var newSongs = songs.
Where(song => dbSongs.Find(dbSong => dbSong.slug == song.Slug) == null).
Select(song => new SetlistSong()
{
artist_id = artist.id,
name = song.Name,
slug = song.Slug,
updated_at = now,
upstream_identifier = song.Slug
}).
ToList();
var justAdded = await _setlistSongService.InsertAll(artist, newSongs);
dbSongs.AddRange(justAdded);
stats.Created += newSongs.Count;
foreach (var justAddedSong in justAdded)
{
existingSetlistSongs[justAddedSong.upstream_identifier] = justAddedSong;
}
}
stats += await _setlistShowService.UpdateSongPlays(dbShow, dbSongs);
}
async Task <ImportStats> ProcessSetlist(Artist artist, Relisten.Vendor.SetlistFm.Setlist setlist)
{
var stats = new ImportStats();
var now = DateTime.UtcNow;
// venue
Venue dbVenue = existingVenues.GetValue(setlist.venue._iguanaUpstreamId);
if (dbVenue == null)
{
var sc = new VenueWithShowCount()
{
updated_at = now,
artist_id = artist.id,
name = setlist.venue.name,
latitude = setlist.venue.city.coords?.lat,
longitude = setlist.venue.city.coords?.@long,
location = $"{setlist.venue.city.name}, {setlist.venue.city.state}",
upstream_identifier = setlist.venue._iguanaUpstreamId,
slug = Slugify(setlist.venue.name)
};
dbVenue = await _venueService.Save(sc);
sc.id = dbVenue.id;
existingVenues[dbVenue.upstream_identifier] = sc;
stats.Created++;
}
// tour
Tour dbTour = null;
if (artist.features.tours)
{
var tour_upstream = setlist.tour?.name ?? "Not Part of a Tour";
dbTour = existingTours.GetValue(tour_upstream);
if (dbTour == null)
{
dbTour = await _tourService.Save(new Tour()
{
updated_at = now,
artist_id = artist.id,
start_date = null,
end_date = null,
name = tour_upstream,
slug = Slugify(tour_upstream),
upstream_identifier = tour_upstream
});
existingTours[dbTour.upstream_identifier] = dbTour;
stats.Created++;
}
}
// show
var dbShow = existingSetlistShows.GetValue(setlist.id);
var date = DateTime.ParseExact(setlist.eventDate, "dd-MM-yyyy", null);
var setlistLastUpdated = setlist.lastUpdated;
var shouldAddSongs = false;
if (dbShow == null)
{
dbShow = await _setlistShowService.Save(new SetlistShow()
{
artist_id = artist.id,
updated_at = setlistLastUpdated,
date = date,
upstream_identifier = setlist.id,
venue_id = dbVenue.id,
tour_id = artist.features.tours ? dbTour?.id : null
});
existingSetlistShows[dbShow.upstream_identifier] = dbShow;
stats.Created++;
shouldAddSongs = true;
}
else if (setlistLastUpdated > dbShow.updated_at)
{
dbShow.artist_id = artist.id;
dbShow.updated_at = setlistLastUpdated;
dbShow.date = date;
dbShow.venue_id = dbVenue.id;
dbShow.tour_id = dbTour?.id;
dbShow = await _setlistShowService.Save(dbShow);
existingSetlistShows[dbShow.upstream_identifier] = dbShow;
stats.Updated++;
shouldAddSongs = true;
}
// setlist.fm doesn't provide much info about tours so we need to find the start
// and end date ourselves.
if (artist.features.tours &&
(dbTour.start_date == null ||
dbTour.end_date == null ||
dbShow.date < dbTour.start_date ||
dbShow.date > dbTour.end_date))
{
if (!tourToStartDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date < tourToStartDate[dbTour.upstream_identifier])
{
tourToStartDate[dbTour.upstream_identifier] = dbShow.date;
}
if (!tourToEndDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date > tourToEndDate[dbTour.upstream_identifier])
{
tourToEndDate[dbTour.upstream_identifier] = dbShow.date;
}
}
if (shouldAddSongs)
{
var songs = setlist.sets.set.
SelectMany(set => set.song).
Select(song => new { Name = song.name, Slug = Slugify(song.name) }).
GroupBy(song => song.Slug).
Select(grp => grp.First()).
ToList();
var dbSongs = existingSetlistSongs.
Where(kvp => songs.Select(song => song.Slug).Contains(kvp.Key)).
Select(kvp => kvp.Value).
ToList();
if (songs.Count != dbSongs.Count)
{
var newSongs = songs.
Where(song => dbSongs.Find(dbSong => dbSong.slug == song.Slug) == null).
Select(song => new SetlistSong()
{
artist_id = artist.id,
name = song.Name,
slug = song.Slug,
updated_at = now,
upstream_identifier = song.Slug
}).
ToList();
var justAdded = await _setlistSongService.InsertAll(artist, newSongs);
dbSongs.AddRange(justAdded);
stats.Created += newSongs.Count;
foreach (var justAddedSong in justAdded)
{
existingSetlistSongs[justAddedSong.upstream_identifier] = justAddedSong;
}
}
stats += await _setlistShowService.UpdateSongPlays(dbShow, dbSongs);
}
return(stats);
}
private async Task ProcessPage(ImportStats stats, Artist artist, FileMetaObject meta, string pageContents, DateTime updated_at, PerformContext ctx)
{
var dbShow = existingSetlistShows.GetValue(meta.Identifier);
if (dbShow != null)
{
return;
}
ctx.WriteLine($"Processing: {meta.FilePath}");
var html = new HtmlDocument();
html.LoadHtml(pageContents);
var root = html.DocumentNode;
var ps = root.DescendantsWithClass("venue-name").ToList();
var bandName = ps[1].InnerText.CollapseSpacesAndTrim();
var venueName = ps[0].InnerText.CollapseSpacesAndTrim();
var venueCityOrCityState = root.DescendantsWithClass("venue-address").Single().InnerText.CollapseSpacesAndTrim().TrimEnd(',');
var venueCountry = root.DescendantsWithClass("venue-country").Single().InnerText.CollapseSpacesAndTrim();
string tourName = ps.Count > 2 ? ps[2].InnerText.CollapseSpacesAndTrim() : "Not Part of a Tour";
var venueUpstreamId = "jerrygarcia.com_" + venueName;
Venue dbVenue = existingVenues.GetValue(venueUpstreamId);
if (dbVenue == null)
{
var sc = new VenueWithShowCount
{
artist_id = artist.id,
name = venueName,
location = venueCityOrCityState + ", " + venueCountry,
upstream_identifier = venueUpstreamId,
slug = Slugify(venueName),
updated_at = updated_at
};
dbVenue = await _venueService.Save(sc);
sc.id = dbVenue.id;
existingVenues[dbVenue.upstream_identifier] = sc;
stats.Created++;
}
var dbTour = existingTours.GetValue(tourName);
if (dbTour == null && artist.features.tours)
{
dbTour = await _tourService.Save(new Tour
{
artist_id = artist.id,
name = tourName,
slug = Slugify(tourName),
upstream_identifier = tourName,
updated_at = updated_at
});
existingTours[dbTour.upstream_identifier] = dbTour;
stats.Created++;
}
dbShow = await _setlistShowService.Save(new SetlistShow
{
artist_id = artist.id,
tour_id = dbTour?.id,
venue_id = dbVenue.id,
date = meta.Date,
upstream_identifier = meta.Identifier,
updated_at = updated_at
});
existingSetlistShows[dbShow.upstream_identifier] = dbShow;
stats.Created++;
var dbSongs = root.Descendants("ol")
.SelectMany(node => node.Descendants("li"))
.Select(node =>
{
var trackName = node.InnerText.Trim().TrimEnd('>', '*', ' ');
var slug = SlugifyTrack(trackName);
return(new SetlistSong
{
artist_id = artist.id,
name = trackName,
slug = slug,
upstream_identifier = slug,
updated_at = updated_at
});
})
.GroupBy(s => s.upstream_identifier)
.Select(g => g.First())
.ToList()
;
ResetTrackSlugCounts();
var dbSongsToAdd = dbSongs.Where(song => !existingSetlistSongs.ContainsKey(song.upstream_identifier));
dbSongs = dbSongs.Where(song => existingSetlistSongs.ContainsKey(song.upstream_identifier))
.Select(song => existingSetlistSongs[song.upstream_identifier])
.ToList()
;
var added = await _setlistSongService.InsertAll(artist, dbSongsToAdd);
foreach (var s in added)
{
existingSetlistSongs[s.upstream_identifier] = s;
}
stats.Created += added.Count();
dbSongs.AddRange(added);
if (artist.features.tours &&
(dbTour.start_date == null ||
dbTour.end_date == null ||
dbShow.date < dbTour.start_date ||
dbShow.date > dbTour.end_date))
{
if (!tourToStartDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date < tourToStartDate[dbTour.upstream_identifier])
{
tourToStartDate[dbTour.upstream_identifier] = dbShow.date;
}
if (!tourToEndDate.ContainsKey(dbTour.upstream_identifier) ||
dbShow.date > tourToEndDate[dbTour.upstream_identifier])
{
tourToEndDate[dbTour.upstream_identifier] = dbShow.date;
}
}
await _setlistShowService.UpdateSongPlays(dbShow, dbSongs);
}
