public Tasker.Conclusion ScrapeGames(Tasker tasker, Object syncObject = null)
{
if (Program.TheGamesDBAPI == null)
{
return(Tasker.Conclusion.Success);
}
try
{
var idList = new Dictionary <UInt32, int>();
foreach (var app in NewGames)
{
if (app.Metadata.OriginalCrc32 != 0 &&
data.GamesDB.HashLookup.ContainsKey(app.Metadata.OriginalCrc32) &&
data.GamesDB.HashLookup[app.Metadata.OriginalCrc32].TgdbId.Count > 0)
{
if (!idList.ContainsKey(app.Metadata.OriginalCrc32))
{
idList.Add(app.Metadata.OriginalCrc32, data.GamesDB.HashLookup[app.Metadata.OriginalCrc32].TgdbId.First());
}
}
}
var api = Program.TheGamesDBAPI;
var infoList = new Dictionary <int, IScraperData>();
foreach (var chunk in idList.Values.Distinct().ToList().ChunkBy(20))
{
var apiResponse = api.GetInfoByID(chunk.ToArray());
while (true)
{
apiResponse.Wait();
foreach (var result in apiResponse.Result.Items)
{
infoList.Add(int.Parse(result.ID), result);
}
if (apiResponse.Result.HasNextPage)
{
apiResponse = apiResponse.Result.GetNextPage();
}
else
{
break;
}
}
}
var counter = 0;
foreach (var app in NewGames)
{
tasker.SetProgress(++counter, NewGames.Count);
tasker.SetStatus(string.Format(Resources.Scraping0, app.Name));
int tgdbId;
if (app.Metadata.OriginalCrc32 != 0 &&
data.GamesDB.HashLookup.ContainsKey(app.Metadata.OriginalCrc32) &&
data.GamesDB.HashLookup[app.Metadata.OriginalCrc32].TgdbId.Count > 0 &&
infoList.ContainsKey(tgdbId = data.GamesDB.HashLookup[app.Metadata.OriginalCrc32].TgdbId.First()))
{
try
{
var apiResult = infoList[tgdbId];
if (apiResult.Name != null)
{
var name = GamesDB.HashLookup[app.Metadata.OriginalCrc32].Name;
if (name != null)
{
name = Shared.CleanName(name, true);
}
app.Desktop.Name = name ?? apiResult.Name;
app.Desktop.SortName = Shared.GetSortName(app.Desktop.Name);
}
if (apiResult.Publishers != null && apiResult.Publishers.Length > 0)
{
app.Desktop.Publisher = String.Join(", ", apiResult.Publishers).ToUpper();
}
else if (apiResult.Developers != null && apiResult.Developers.Length > 0)
{
if (apiResult.ReleaseDate != null)
{
app.Desktop.Copyright = $"© {apiResult.ReleaseDate.Year} {String.Join(", ", apiResult.Developers)}";
}
else
{
app.Desktop.Copyright = $"© {String.Join(", ", apiResult.Developers)}";
}
}
if (apiResult.Description != null)
{
app.Desktop.Description = apiResult.Description;
}
if (apiResult.ReleaseDate != null)
{
app.Desktop.ReleaseDate = apiResult.ReleaseDate.ToString("yyyy-MM-dd");
}
if (apiResult.PlayerCount > 0)
{
app.Desktop.Players = Convert.ToByte(apiResult.PlayerCount);
app.Desktop.Simultaneous = apiResult.PlayerCount == 2;
}
if (apiResult.Genres != null && apiResult.Genres.Length > 0)
{
foreach (var genre in apiResult.Genres)
{
var match = Genre.GenreList.Where(g => g.GamesDbId.Contains(genre.ID)).Select(g => g);
if (match.Count() > 0)
{
var firstGenre = match.First();
app.Desktop.Genre = firstGenre.DesktopName;
break;
}
}
}
using (var wc = new HakchiWebClient())
{
try
{
var front = apiResult.Images.Where(i => i.Type == TeamShinkansen.Scrapers.Enums.ArtType.Front).ToArray();
if (front.Length > 0 && !app.CoverArtMatchSuccess)
{
tasker.SetStatus(string.Format(Resources.DownloadingFrontArtFor0, apiResult.Name));
var data = wc.DownloadData(front[0].Url);
using (var ms = new MemoryStream(data))
using (var bm = new Bitmap(ms))
{
app.SetImage(bm);
}
}
}
catch (WebException ex) { }
try
{
tasker.SetStatus(string.Format(Resources.DownloadingClearLogoFor0, apiResult.Name));
var imageData = wc.DownloadData($"https://cdn.thegamesdb.net/images/original/clearlogo/{apiResult.ID}.png");
using (var ms = new MemoryStream(imageData))
using (var clearLogo = File.OpenWrite(Path.Combine(app.BasePath, $"{app.Code}_logo.png")))
{
ms.Seek(0, SeekOrigin.Begin);
ms.CopyTo(clearLogo);
ms.Seek(0, SeekOrigin.Begin);
tasker.SetStatus(string.Format(Resources.GeneratingSpineFor0, apiResult.Name));
using (var bm = new Bitmap(ms))
using (var cl = new SystemDrawingBitmap(bm))
using (var spineImage = (Program.SpineTemplates.Where(e => e.Value.Name == "Custom (Black Grid)").First().Value as SpineTemplate <Bitmap>).Process(cl).Bitmap)
{
app.SetMdMini(spineImage, NesMenuElementBase.GameImageType.MdSpine);
}
}
}
catch (WebException ex) { }
}
}
catch (Exception innerEx) { }
}
}
}
catch (Exception ex) { }
return(Tasker.Conclusion.Success);
}
public static TaskFunc DownloadFile(string url, string fileName, bool successOnError = false, bool onlyLatest = false, DateTime?comparisonDate = null)
{
return((Tasker tasker, Object sync) =>
{
Conclusion result = Conclusion.Success;
try
{
var wc = new HakchiWebClient();
wc.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
wc.Method = "HEAD";
wc.DownloadData(url);
if (comparisonDate == null && File.Exists(fileName))
{
comparisonDate = File.GetLastWriteTime(fileName);
}
if (onlyLatest && comparisonDate != null)
{
var date = DateTime.ParseExact(wc.ResponseHeaders.Get("Last-Modified"),
"ddd, dd MMM yyyy HH:mm:ss 'GMT'",
CultureInfo.InvariantCulture.DateTimeFormat,
DateTimeStyles.AssumeUniversal);
if (comparisonDate >= date)
{
return Conclusion.Success;
}
}
wc.Method = "GET";
Debug.WriteLine($"Downloading: {url} to {fileName}");
wc.DownloadProgressChanged += new DownloadProgressChangedEventHandler(async(object sender, DownloadProgressChangedEventArgs e) =>
{
tasker.SetProgress(e.BytesReceived, e.TotalBytesToReceive);
tasker.SetStatus(String.Format(Resources.DownloadingProgress, Shared.SizeSuffix(e.BytesReceived), Shared.SizeSuffix(e.TotalBytesToReceive)));
});
wc.DownloadFileCompleted += (object sender, System.ComponentModel.AsyncCompletedEventArgs e) =>
{
if (e.Error != null)
{
File.Delete(fileName);
result = Conclusion.Error;
}
else
{
try
{
var date = DateTime.ParseExact(wc.ResponseHeaders.Get("Last-Modified"),
"ddd, dd MMM yyyy HH:mm:ss 'GMT'",
CultureInfo.InvariantCulture.DateTimeFormat,
DateTimeStyles.AssumeUniversal);
File.SetLastWriteTimeUtc(fileName, date);
}
catch (Exception) { }
}
};
var downloadTask = wc.DownloadFileTaskAsync(new Uri(url), fileName);
new Thread(() =>
{
while (true)
{
if (tasker.TaskConclusion == Conclusion.Abort)
{
Debug.WriteLine("Download Aborted");
wc.CancelAsync();
break;
}
if (downloadTask.IsCanceled || downloadTask.IsCompleted || downloadTask.IsFaulted)
{
break;
}
Thread.Sleep(100);
}
}).Start();
downloadTask.Wait();
}
catch (WebException e)
{
if (!successOnError)
{
throw e;
}
}
return result;
});
}
public Tasker.Conclusion AddGames(Tasker tasker, Object syncObject = null)
{
tasker.SetProgress(-1, -1, Tasker.State.Running, Resources.AddingGames);
tasker.SetTitle(Resources.AddingGames);
tasker.SetStatusImage(Resources.sign_cogs);
// static presets
NesApplication.ParentForm = tasker.HostForm;
NesApplication.NeedPatch = null;
NesApplication.Need3rdPartyEmulator = null;
NesApplication.CachedCoverFiles = null;
NesGame.IgnoreMapper = null;
SnesGame.NeedAutoDownloadCover = null;
int total = files.Count();
int count = 0;
var gamesWithMultipleArt = new List <NesApplication>();
foreach (var sourceFileName in files)
{
NesApplication app = null;
try
{
tasker.SetStatus(string.Format(Resources.AddingGame, Path.GetFileName(sourceFileName)));
var fileName = sourceFileName;
var ext = Path.GetExtension(sourceFileName).ToLower();
byte[] rawData = null;
string tmp = null;
if (!asIs && (ext == ".7z" || ext == ".zip" || ext == ".rar" || ext == ".clvg"))
{
if (ext == ".clvg")
{
tmp = TempHelpers.getUniqueTempPath();
Directory.CreateDirectory(tmp);
using (var file = File.OpenRead(sourceFileName))
using (var reader = ReaderFactory.Open(file))
{
reader.WriteAllToDirectory(tmp, new ExtractionOptions()
{
ExtractFullPath = true, PreserveFileTime = true
});
}
var gameFilesInArchive = Directory.GetFiles(tmp, "*.desktop").Select(o => new DirectoryInfo(o)).Cast <DirectoryInfo>().ToArray();
switch (gameFilesInArchive.LongLength)
{
case 0:
// no files found
break;
case 1:
// one file found
fileName = gameFilesInArchive[0].FullName;
break;
default:
// multiple files found
var r = SelectFile(tasker, gameFilesInArchive.Select(o => o.FullName).ToArray());
if (r == DialogResult.OK)
{
fileName = selectedFile;
}
else if (r == DialogResult.Ignore)
{
fileName = sourceFileName;
}
else
{
continue;
}
break;
}
}
else
{
using (var extractor = ArchiveFactory.Open(sourceFileName))
{
var filesInArchive = extractor.Entries;
var gameFilesInArchive = new List <string>();
foreach (var f in extractor.Entries)
{
if (!f.IsDirectory)
{
var e = Path.GetExtension(f.Key).ToLower();
if (e == ".desktop")
{
gameFilesInArchive.Clear();
gameFilesInArchive.Add(f.Key);
break;
}
else if (CoreCollection.Extensions.Contains(e))
{
gameFilesInArchive.Add(f.Key);
}
}
}
if (gameFilesInArchive.Count == 1) // Only one known file (or app)
{
fileName = gameFilesInArchive[0];
}
else if (gameFilesInArchive.Count > 1) // Many known files, need to select
{
var r = SelectFile(tasker, gameFilesInArchive.ToArray());
if (r == DialogResult.OK)
{
fileName = selectedFile;
}
else if (r == DialogResult.Ignore)
{
fileName = sourceFileName;
}
else
{
continue;
}
}
else if (filesInArchive.Count() == 1) // No known files but only one another file
{
fileName = filesInArchive.First().Key;
}
else // Need to select
{
var r = SelectFile(tasker, filesInArchive.Select(f => f.Key).ToArray());
if (r == DialogResult.OK)
{
fileName = selectedFile;
}
else if (r == DialogResult.Ignore)
{
fileName = sourceFileName;
}
else
{
continue;
}
}
if (fileName != sourceFileName)
{
var o = new MemoryStream();
if (Path.GetExtension(fileName).ToLower() == ".desktop" || // App in archive, need the whole directory
filesInArchive.Select(f => f.Key).Contains(Path.GetFileNameWithoutExtension(fileName) + ".jpg") || // Or it has cover in archive
filesInArchive.Select(f => f.Key).Contains(Path.GetFileNameWithoutExtension(fileName) + ".png") ||
filesInArchive.Select(f => f.Key).Contains(Path.GetFileNameWithoutExtension(fileName) + ".ips") // Or IPS file
)
{
tmp = Path.Combine(tempDirectory, fileName);
Directory.CreateDirectory(tmp);
extractor.WriteToDirectory(tmp, new ExtractionOptions()
{
ExtractFullPath = true, Overwrite = true
});
fileName = Path.Combine(tmp, fileName);
}
else
{
extractor.Entries.Where(f => f.Key == fileName).First().WriteTo(o);
rawData = new byte[o.Length];
o.Seek(0, SeekOrigin.Begin);
o.Read(rawData, 0, (int)o.Length);
}
}
}
}
}
app = NesApplication.Import(fileName, sourceFileName, rawData, asIs);
if (ext == ".clvg")
{
app.SkipCoreSelect = true;
}
else
{
if (app.CoverArtMatches != null && app.CoverArtMatches.Count() > 1)
{
gamesWithMultipleArt.Add(app);
}
if (ConfigIni.Instance.EnableImportScraper && Program.TheGamesDBAPI != null &&
app.Metadata.OriginalCrc32 != 0 &&
data.GamesDB.HashLookup.ContainsKey(app.Metadata.OriginalCrc32) &&
data.GamesDB.HashLookup[app.Metadata.OriginalCrc32].Length > 0)
{
var api = Program.TheGamesDBAPI;
var task = api.GetInfoByID(data.GamesDB.HashLookup[app.Metadata.OriginalCrc32]);
try
{
task.Wait();
var result = task.Result;
if (result.Items.Count() > 0)
{
var first = result.Items.First();
if (first.Name != null)
{
app.Desktop.Name = first.Name;
app.Desktop.SortName = Shared.GetSortName(first.Name);
}
if (first.Publishers != null && first.Publishers.Length > 0)
{
app.Desktop.Publisher = String.Join(", ", first.Publishers).ToUpper();
}
else if (first.Developers != null && first.Developers.Length > 0)
{
if (first.ReleaseDate != null)
{
app.Desktop.Copyright = $"© {first.ReleaseDate.Year} {String.Join(", ", first.Developers)}";
}
else
{
app.Desktop.Copyright = $"© {String.Join(", ", first.Developers)}";
}
}
if (first.Description != null)
{
app.Desktop.Description = first.Description;
}
if (first.ReleaseDate != null)
{
app.Desktop.ReleaseDate = first.ReleaseDate.ToString("yyyy-MM-dd");
}
if (first.PlayerCount > 0)
{
app.Desktop.Players = Convert.ToByte(first.PlayerCount);
app.Desktop.Simultaneous = first.PlayerCount == 2;
}
if (first.Genres != null && first.Genres.Length > 0)
{
foreach (var genre in first.Genres)
{
var match = ScraperForm.TheGamesDBGenreLookup.Where(g => g.Value.Contains(genre.ID)).Select(g => g.Key);
if (match.Count() > 0)
{
var firstGenre = match.First();
app.Desktop.Genre = firstGenre;
break;
}
}
}
using (var wc = new HakchiWebClient())
{
try
{
var front = first.Images.Where(i => i.Type == TeamShinkansen.Scrapers.Enums.ArtType.Front).ToArray();
if (front.Length > 0 && !app.CoverArtMatchSuccess)
{
var data = wc.DownloadData(front[0].Url);
using (var ms = new MemoryStream(data))
using (var bm = new Bitmap(ms))
{
app.SetImage(bm);
}
}
}
catch (WebException ex) { }
try
{
var imageData = wc.DownloadData($"https://cdn.thegamesdb.net/images/original/clearlogo/{first.ID}.png");
using (var ms = new MemoryStream(imageData))
using (var clearLogo = File.OpenWrite(Path.Combine(app.BasePath, $"{app.Code}_logo.png")))
{
ms.Seek(0, SeekOrigin.Begin);
ms.CopyTo(clearLogo);
}
}
catch (WebException ex) { }
}
}
}
catch (Exception) { }
}
}
if (app is ISupportsGameGenie && Path.GetExtension(fileName).ToLower() == ".nes")
{
var lGameGeniePath = Path.Combine(Path.GetDirectoryName(fileName), Path.GetFileNameWithoutExtension(fileName) + ".xml");
if (File.Exists(lGameGeniePath))
{
GameGenieDataBase lGameGenieDataBase = new GameGenieDataBase(app);
lGameGenieDataBase.ImportCodes(lGameGeniePath, true);
lGameGenieDataBase.Save();
}
}
if (!string.IsNullOrEmpty(tmp) && Directory.Exists(tmp))
{
Directory.Delete(tmp, true);
}
}
catch (Exception ex)
{
if (ex is ThreadAbortException)
{
return(Tasker.Conclusion.Abort);
}
if (ex.InnerException != null && !string.IsNullOrEmpty(ex.InnerException.Message))
{
Trace.WriteLine(ex.InnerException.Message + ex.InnerException.StackTrace);
tasker.ShowError(ex.InnerException, Path.GetFileName(sourceFileName));
}
else
{
Trace.WriteLine(ex.Message + ex.StackTrace, Path.GetFileName(sourceFileName));
tasker.ShowError(ex);
}
return(Tasker.Conclusion.Error);
}
if (app != null)
{
addedApps.Add(app);
}
tasker.SetProgress(++count, total);
}
if (gamesWithMultipleArt.Count > 0)
{
tasker.HostForm.Invoke(new Action(() => {
using (SelectCoverDialog selectCoverDialog = new SelectCoverDialog())
{
selectCoverDialog.Games.AddRange(gamesWithMultipleArt);
selectCoverDialog.ShowDialog(tasker.HostForm);
}
}));
}
return(Tasker.Conclusion.Success);
}