private int Replicate(string source, string destination, int batchSize, int expectedSourceMaxQueryTime, int pauseDuration)
{
Log.Trace("Check for data to replicate.");
int total = 0;
bool hasWork;
do
{
int originalKey = GetLastOriginalKey(destination);
Log.Trace("replicating records since {0}", originalKey);
DateTime before = DateTime.Now;
DownloadBatch batch = GetDownloadRecords(source, originalKey, batchSize);
DateTime after = DateTime.Now;
TimeSpan duration = after - before;
Log.Trace(string.Format("source query duration: {0} seconds", duration.TotalSeconds));
if (batch.Rows.Count > 0)
{
hasWork = true;
if (!WhatIf)
{
PutDownloadRecords(destination, batch, CancellationToken);
}
if (CancellationToken.IsCancellationRequested)
{
return(total);
}
Log.Trace(string.Format("replicated {0} records", batch.Rows.Count));
total += batch.Rows.Count;
if (duration.TotalSeconds > expectedSourceMaxQueryTime)
{
Log.Trace("previous source query exceeded threshold so pausing");
Thread.Sleep(pauseDuration * 1000);
}
}
else
{
hasWork = false;
}
if (CancellationToken.IsCancellationRequested)
{
return(total);
}
}while (hasWork);
return(total);
}
private static void PutDownloadRecords(string connectionString, DownloadBatch batch, CancellationToken ct)
{
try
{
using (SqlConnection connection = new SqlConnection(connectionString))
{
connection.Open();
foreach (DownloadBatchRow row in batch.Rows)
{
if (ct.IsCancellationRequested)
{
return;
}
SqlCommand command = new SqlCommand("AddDownloadFact", connection);
command.CommandType = CommandType.StoredProcedure;
command.CommandTimeout = 120;
command.Parameters.AddWithValue("@OriginalKey", row.OriginalKey);
command.Parameters.AddWithValue("@PackageId", row.PackageId);
command.Parameters.AddWithValue("@PackageVersion", row.PackageVersion);
command.Parameters.AddWithValue("@PackageListed", row.PackageListed ? 1 : 0);
command.Parameters.AddWithValue("@PackageTitle", AddNullableString(row.PackageTitle));
command.Parameters.AddWithValue("@PackageDescription", AddNullableString(row.PackageDescription));
command.Parameters.AddWithValue("@PackageIconUrl", AddNullableString(row.PackageIconUrl));
command.Parameters.AddWithValue("@DownloadUserAgent", row.DownloadUserAgent);
command.Parameters.AddWithValue("@DownloadOperation", row.DownloadOperation);
command.Parameters.AddWithValue("@DownloadTimestamp", row.DownloadTimestamp);
command.Parameters.AddWithValue("@DownloadProjectTypes", AddNullableString(row.DownloadProjectTypes));
command.Parameters.AddWithValue("@DownloadDependentPackageId", AddNullableString(row.DownloadDependentPackageId));
command.ExecuteNonQuery();
}
}
}
catch (Exception e)
{
string msg = string.Format("(DEBUG) Exception in PutDownloadRecords (warehouse side): {0}", e.Message);
throw new ApplicationException(msg, e);
}
}
/// <summary>
/// Downloads the songs in the provided Dictionary.
/// </summary>
/// <param name="queuedSongs"></param>
/// <param name="skippedsongs"></param>
/// <param name="useSongKeyAsOutputFolder"></param>
/// <returns></returns>
public List <SongInfo> DownloadSongs(Dictionary <string, SongInfo> queuedSongs, out int skippedsongs, bool useSongKeyAsOutputFolder)
{
//var existingSongs = Directory.GetDirectories(CustomSongsPath);
string tempPath = "";
string outputPath = "";
List <SongInfo> matchedSongs = new List <SongInfo>();
skippedsongs = 0;
DownloadBatch jobs = new DownloadBatch();
jobs.JobCompleted += OnJobFinished;
foreach (var song in queuedSongs.Values)
{
tempPath = Path.Combine(Path.GetTempPath(), song.key + zipExtension);
if (useSongKeyAsOutputFolder)
{
outputPath = Path.Combine(CustomSongsPath, song.key);
}
else
{
outputPath = CustomSongsPath;
}
bool songExists = Directory.Exists(outputPath);
bool songInHistory = _songDownloadHistory.Contains(song.key);
if ((songExists && songInHistory) || !songInHistory)
{
matchedSongs.Add(song);
}
if (songExists || songInHistory)
{
skippedsongs++;
//Logger.Debug($"Skipping song - SongExists: {songExists}, SongInHistory: {songInHistory}");
continue; // We already have the song or don't want it, skip
}
DownloadJob job = new DownloadJob(song, tempPath, outputPath);
jobs.AddJob(job);
}
jobs.RunJobs().Wait();
return(matchedSongs);
}
private static void PutDownloadRecords(string connectionString, DownloadBatch batch, CancellationToken ct)
{
try
{
using (SqlConnection connection = new SqlConnection(connectionString))
{
connection.Open();
foreach (DownloadBatchRow row in batch.Rows)
{
if (ct.IsCancellationRequested)
{
return;
}
SqlCommand command = new SqlCommand("AddDownloadFact", connection);
command.CommandType = CommandType.StoredProcedure;
command.CommandTimeout = 120;
command.Parameters.AddWithValue("@OriginalKey", row.OriginalKey);
command.Parameters.AddWithValue("@PackageId", row.PackageId);
command.Parameters.AddWithValue("@PackageVersion", row.PackageVersion);
command.Parameters.AddWithValue("@PackageListed", row.PackageListed ? 1 : 0);
command.Parameters.AddWithValue("@PackageTitle", AddNullableString(row.PackageTitle));
command.Parameters.AddWithValue("@PackageDescription", AddNullableString(row.PackageDescription));
command.Parameters.AddWithValue("@PackageIconUrl", AddNullableString(row.PackageIconUrl));
command.Parameters.AddWithValue("@DownloadUserAgent", row.DownloadUserAgent);
command.Parameters.AddWithValue("@DownloadOperation", row.DownloadOperation);
command.Parameters.AddWithValue("@DownloadTimestamp", row.DownloadTimestamp);
command.Parameters.AddWithValue("@DownloadProjectTypes", AddNullableString(row.DownloadProjectTypes));
command.Parameters.AddWithValue("@DownloadDependentPackageId", AddNullableString(row.DownloadDependentPackageId));
command.ExecuteNonQuery();
}
}
}
catch (Exception e)
{
string msg = $"(DEBUG) Exception in PutDownloadRecords (warehouse side): {e.Message}";
throw new ApplicationException(msg, e);
}
}
private static void PutDownloadRecords(string connectionString, DownloadBatch batch, CancellationToken ct)
{
try
{
using (SqlConnection connection = new SqlConnection(connectionString))
{
connection.Open();
foreach (DownloadBatchRow row in batch.Rows)
{
if (ct.IsCancellationRequested)
{
return;
}
SqlCommand command = new SqlCommand("AddDownloadFact", connection);
command.CommandType = CommandType.StoredProcedure;
command.CommandTimeout = 120;
command.Parameters.AddWithValue("@OriginalKey", row.OriginalKey);
command.Parameters.AddWithValue("@PackageId", row.PackageId);
command.Parameters.AddWithValue("@PackageVersion", row.PackageVersion);
command.Parameters.AddWithValue("@DownloadUserAgent", row.DownloadUserAgent);
command.Parameters.AddWithValue("@DownloadOperation", row.DownloadOperation);
command.Parameters.AddWithValue("@DownloadTimestamp", row.DownloadTimestamp);
command.ExecuteNonQuery();
}
}
}
catch (Exception e)
{
string msg = string.Format("Exception in PutDownloadRecords (warehouse side): {0}", e.Message);
throw new ApplicationException(msg, e);
}
}
