public void AddDataFromXmlTest()
{
XElement xml;
EshopTitle title;
// check samurai data
xml = XElement.Parse("<title id=\"1234\">" +
"<product_code>CTR-P-WXYZ</product_code>" +
"<name>TestTitle <br>Newline <br>Newline2</name>" +
"<icon_url>https://icon.url/test.jpg</icon_url>" +
"<platform id=\"124\" device=\"CTR\"><name>Nintendo 3DS (Card/Download)</name></platform>" +
"</title>");
title = new EshopTitle();
title.AddDataFromXml(xml);
Assert.AreEqual("1234", title.EshopId);
Assert.AreEqual("WXYZ", title.ProductCode);
Assert.AreEqual("TestTitle Newline Newline2", title.Name);
Assert.AreEqual("https://icon.url/test.jpg", title.IconUrl);
Assert.AreEqual((int)124, title.Platform);
// check ninja data
xml = XElement.Parse("<title_ec_info>" +
"<title_id>00050000101C9500</title_id>" +
"<content_size>10551265752</content_size>" +
"<title_version>0</title_version>" +
"<disable_download>false</disable_download>" +
"</title_ec_info>");
title = new EshopTitle();
title.AddDataFromXml(xml);
Assert.AreEqual("00050000101C9500", title.TitleIdString);
Assert.AreEqual((ulong)10551265752, title.Size);
Assert.AreEqual("", title.VersionString);
}
/// <summary>
/// Retrieves a list of DLCs for the specified titles
/// </summary>
/// <param name="titles">The titles for which DLC info is downloaded for</param>
/// <returns>A list of DLC titles</returns>
public async Task <List <EshopTitle> > GetAllDLCsForTitles(ICollection <EshopTitle> titles)
{
int gameCount = titles.Count(t => (t.JsonType == DatabaseJsonType.Games || t.JsonType == DatabaseJsonType.Games3DS) && t.IsNativeTitle);
progressManager.Reset(gameCount);
Console.WriteLine("Downloading DLC info for {0} titles ...", gameCount);
progressManager.SetTitle(string.Format("Downloading DLC info for {0} titles ...", gameCount));
List <EshopTitle> dlcList = new List <EshopTitle>();
foreach (EshopTitle title in titles.Where(t => (t.JsonType == DatabaseJsonType.Games || t.JsonType == DatabaseJsonType.Games3DS) && t.IsNativeTitle))
{
progressManager.Step("Downloading DLC info ...");
EshopTitle dlc = new EshopTitle();
dlc.TitleId = title.TitleId.DLCID;
await Retry(async() =>
{
dlc.Size = await GetContentSizeForTitle(dlc);
dlcList.Add(dlc); // if no exception is thrown, the title does have a DLC
},
shouldStopRetrying : (e) =>
{
// 404 -> no DLC
return(e is WebException we && we.Response is HttpWebResponse resp && resp.StatusCode == HttpStatusCode.NotFound);
});
}
return(dlcList);
}
public void EqualsTest()
{
string idGame1 = "00050000101C9500", id2 = "0004000EA1B2C3D4", id3 = "0004000EC3D4E5F6";
EshopTitle title1, title2;
// test game equality (make sure that the version is not included in the checks)
title1 = new EshopTitle()
{
TitleIdString = idGame1
};
title2 = new EshopTitle()
{
TitleIdString = idGame1,
Version = 42
};
Assert.AreEqual(title1, title2);
// test other title equality
title1 = new EshopTitle()
{
TitleIdString = id2,
Version = 42
};
title2 = new EshopTitle()
{
TitleIdString = id2,
Version = 42
};
Assert.AreEqual(title1, title2);
// test inequality when titleIDs are different
title1 = new EshopTitle()
{
TitleIdString = id2,
Version = 42
};
title2 = new EshopTitle()
{
TitleIdString = id3,
Version = 42
};
Assert.AreNotEqual(title1, title2);
// test inequality when versions are different
title1 = new EshopTitle()
{
TitleIdString = id2,
Version = 24
};
title2 = new EshopTitle()
{
TitleIdString = id2,
Version = 42
};
Assert.AreNotEqual(title1, title2);
}
public void JsonTypeTest()
{
EshopTitle title = new EshopTitle();
Assert.AreEqual(DatabaseJsonType.None, title.JsonType);
// make sure that titleID updates are reflected in the json type (if previous type is '.None')
title.TitleId = new TitleID("0004000EA1B2C3D4");
Assert.AreEqual(DatabaseJsonType.Updates3DS, title.JsonType);
// make sure that the json type doesn't change once it was set to a value other than '.None'
// (this enforces that existing titles are saved back into the same file they were read from)
title.TitleId = new TitleID("0005000EA1B2C3D4");
Assert.AreEqual(DatabaseJsonType.Updates3DS, title.JsonType);
}
/// <summary>
/// Downloads the tmd for the title and calculates the size from the tmd's contents
/// </summary>
/// <param name="title">Title for which the size is calculated</param>
/// <returns>The size of the given title</returns>
/// <exception cref="InvalidDataException">Thrown if response from server is invalid</exception>
public async Task <ulong> GetContentSizeForTitle(EshopTitle title)
{
if (title.JsonType == DatabaseJsonType.None)
{
return(0);
}
bool isUpdate = title.TitleId.IsUpdate;
string tmdPath = isUpdate
? String.Format(TMDUpdatePath, title.TitleId, title.VersionString)
: String.Format(TMDGamePath, title.TitleId);
byte[] tmd = await webClient.DownloadDataTaskAsync(TMDUrl + tmdPath);
// see: https://3dbrew.org/wiki/Title_metadata
// sanity checks
if (!(new TitleID(BitConverter.ToUInt64(tmd, 0x18c).SwapEndianness().ToString("X16")).Equals(title.TitleId)))
{
throw new InvalidDataException("TMD's titleID does not match");
}
if (isUpdate && BitConverter.ToUInt16(tmd, 0x1dc).SwapEndianness().ToString() != title.VersionString)
{
throw new InvalidDataException("TMD's version does not match");
}
int contentCount = BitConverter.ToUInt16(tmd, 0x1de).SwapEndianness();
ulong totalSize = 0;
using (MemoryStream memoryStream = new MemoryStream(tmd))
using (BinaryReader binaryReader = new BinaryReader(memoryStream))
{
// go to start of content entries
memoryStream.Seek(0xb04, SeekOrigin.Begin);
for (int i = 0; i < contentCount; i++)
{
binaryReader.ReadBytes(0x8); // skip contentID, index, type
totalSize += binaryReader.ReadUInt64().SwapEndianness();
binaryReader.ReadBytes(0x20); // sha256
}
}
return(totalSize);
}
/// <summary>
/// Retrieves a list of all 3DS updates
/// </summary>
/// <returns>A list of update titles</returns>
/// <exception cref="InvalidDataException">Thrown if response from server is invalid</exception>
public async Task <List <EshopTitle> > GetAll3DSUpdates()
{
Console.WriteLine("Downloading 3DS update list ...");
progressManager.SetTitle("Downloading 3DS update list ...");
byte[] versionListData = null;
await Retry(async() => {
versionListData = await Tagaya3DS.GetVersionListData(webClient);
});
// see: http://3dbrew.org/wiki/Home_Menu#VersionList
// first byte should always be 1
if (versionListData[0] != 0x01)
{
throw new InvalidDataException("3DS versionlist response is invalid.");
}
List <EshopTitle> updateList = new List <EshopTitle>();
using (MemoryStream memoryStream = new MemoryStream(versionListData))
using (BinaryReader binaryReader = new BinaryReader(memoryStream))
{
// go to start of version entries
memoryStream.Seek(0x10, SeekOrigin.Begin);
while (memoryStream.Position != memoryStream.Length)
{
EshopTitle title = new EshopTitle();
title.TitleId = new TitleID(binaryReader.ReadUInt64().ToString("X16"));
title.VersionString = binaryReader.ReadUInt32().ToString();
if (title.JsonType != DatabaseJsonType.Games3DS && title.JsonType != DatabaseJsonType.None) // for some reason there are a few games in the versionlist, skip them
{
updateList.Add(title);
}
binaryReader.ReadBytes(4); // skip 4 bytes, unknown data
}
}
await AddSizesToTitles(updateList);
return(updateList);
}
public void ToJSONTest()
{
EshopTitle title = new EshopTitle();
title.EshopId = "20010000000026";
title.IconUrl = "https://icon.url/test.jpg";
title.Name = "TestTitle\u00ae Wii U";
title.Platform = 124;
title.ProductCode = "WAHJ";
title.Region = Region.JPN;
title.Size = 391053332;
title.TitleId = new TitleID("0005000E10100D00");
title.Version = 42;
JObject jobj = JObject.FromObject(title);
JArray jarr = new JArray(jobj);
string jsonString = DatabaseJsonIO.JsonArrayToString(jarr, Formatting.None);
Assert.AreEqual("[{\"EshopId\":\"20010000000026\",\"IconUrl\":\"https:\\/\\/icon.url\\/test.jpg\",\"Name\":\"TestTitle\\u00ae Wii U\",\"Platform\":124,\"ProductCode\":\"WAHJ\",\"Region\":\"JPN\",\"Size\":\"391053332\",\"TitleId\":\"0005000E10100D00\",\"PreLoad\":false,\"Version\":\"42\",\"DiscOnly\":false}]", jsonString);
}
public void CompareToTest()
{
string id1 = "0001000100000059", id2 = "000100010000005A";
EshopTitle titleI1V1 = new EshopTitle()
{
TitleIdString = id1,
VersionString = "96"
};
EshopTitle titleI2V1 = new EshopTitle()
{
TitleIdString = id2,
VersionString = "96"
};
EshopTitle titleI1V2 = new EshopTitle()
{
TitleIdString = id1,
VersionString = "112"
};
EshopTitle titleI2V2 = new EshopTitle()
{
TitleIdString = id2,
VersionString = "112"
};
Assert.IsTrue(titleI1V1.CompareTo(titleI1V1) == 0);
// test titleID only
Assert.IsTrue(titleI1V1.CompareTo(titleI2V1) < 0);
Assert.IsTrue(titleI2V1.CompareTo(titleI1V1) > 0);
// test version only
Assert.IsTrue(titleI1V1.CompareTo(titleI1V2) < 0);
Assert.IsTrue(titleI1V2.CompareTo(titleI1V1) > 0);
// test sorting
EshopTitle[] expected = { titleI1V1, titleI1V2, titleI2V1, titleI2V2 };
EshopTitle[] arr = { titleI2V1, titleI2V2, titleI1V2, titleI1V1 };
Array.Sort(arr);
CollectionAssert.AreEqual(expected, arr);
}
public void VersionStringTest()
{
EshopTitle title = new EshopTitle();
Assert.AreEqual("", title.VersionString);
foreach (TitleID id in new TitleID[] { TitleIDTests.id3DSGames, TitleIDTests.idWiiUGames })
{
title = new EshopTitle();
title.Version = 42;
title.TitleId = id;
Assert.AreEqual("", title.VersionString);
}
foreach (TitleID id in TitleIDTests.updates.Concat(TitleIDTests.dlcs))
{
title = new EshopTitle();
title.Version = 42;
title.TitleId = id;
Assert.AreEqual("42", title.VersionString);
}
}
public void FromJSONTest()
{
string jsonString = "{\"EshopId\":\"20010000000026\",\"IconUrl\":\"https:\\/\\/icon.url\\/test.jpg\",\"Name\":\"TestTitle\\u00ae Wii U\",\"Platform\":124,\"ProductCode\":\"WAHJ\",\"Region\":\"JPN\",\"Size\":\"391053332\",\"TitleId\":\"0005000E10100D00\",\"PreLoad\":false,\"Version\":\"42\",\"DiscOnly\":false}";
JObject jobj = JObject.Parse(jsonString);
EshopTitle title = jobj.ToObject <EshopTitle>();
Assert.AreEqual("20010000000026", title.EshopId);
Assert.AreEqual("https://icon.url/test.jpg", title.IconUrl);
Assert.AreEqual("TestTitle\u00ae Wii U", title.Name);
Assert.AreEqual((int)124, title.Platform);
Assert.AreEqual("WAHJ", title.ProductCode);
Assert.AreEqual("JPN", title.RegionString);
Assert.AreEqual(Region.JPN, title.Region);
Assert.AreEqual("391053332", title.SizeString);
Assert.AreEqual((ulong)391053332, title.Size);
Assert.AreEqual("0005000E10100D00", title.TitleIdString);
Assert.AreEqual(new TitleID("0005000E10100D00"), title.TitleId);
Assert.AreEqual("42", title.VersionString);
Assert.AreEqual((int)42, title.Version);
}
/// <summary>
/// Retrieves a list of all titles from all available regions.
/// </summary>
/// <param name="shopID">Use 1 for the 3DS eShop, 2 for the WiiU eShop.</param>
/// <returns>A list of titles</returns>
public async Task <List <EshopTitle> > GetAllTitles(int shopID)
{
// get title counts for all regions
Console.WriteLine("Getting title count for {0} regions ...", Enum.GetValues(typeof(Region)).Length - 2);
Dictionary <Region, int> titleCounts = new Dictionary <Region, int>();
foreach (Region region in Enum.GetValues(typeof(Region)))
{
if (region == Region.None || region == Region.ALL)
{
continue;
}
await Retry(async() => {
titleCounts.Add(region, await Samurai.GetTitleCountForRegion(webClient, region, shopID));
});
}
int totalTitleCount = titleCounts.Values.Sum();
Console.WriteLine("Downloading metadata for {0} titles ...", totalTitleCount);
progressManager.SetTitle(string.Format("Downloading metadata for {0} titles ...", totalTitleCount));
progressManager.Reset(totalTitleCount);
DateTime currentDate = DateTime.Today;
List <EshopTitle> titleList = new List <EshopTitle>();
// loop through regions
foreach (KeyValuePair <Region, int> pair in titleCounts)
{
Region region = pair.Key;
int titleCount = pair.Value;
// get titles from samurai
for (int offset = 0; offset < titleCount; offset += TitleRequestLimit)
{
XDocument titlesXml = null;
await Retry(async() => {
titlesXml = await Samurai.GetTitlesXmlForRegion(webClient, region, shopID, TitleRequestLimit, offset);
});
/* structure:
* <eshop><contents ...>
* <content index=1><title ...>[title info]</title></content>
* <content index=2><title ...>[title info]</title></content>
* </contents></eshop>
*/
XElement contentsElement = titlesXml.Root.Element("contents");
// iterate over titles in xml
foreach (XElement titleElement in contentsElement.Elements().Select(e => e.Element("title")))
{
// check release date
string releaseDateString = titleElement.Element("release_date_on_eshop")?.Value;
if (!String.IsNullOrEmpty(releaseDateString))
{
DateTime releaseDate;
if (!DateTime.TryParseExact(releaseDateString, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.AllowWhiteSpaces, out releaseDate))
{
continue;
}
if (releaseDate > currentDate)
{
continue;
}
}
// create title and add data
EshopTitle title = new EshopTitle();
title.Region = region;
// set title fields from xml (incomplete, some fields must be retrieved from ninja)
title.AddDataFromXml(titleElement);
progressManager.Step(string.Format("{0}-{1}", region.ToCountryCode(), title.EshopId));
// some exceptions:
if (title.Platform == 63 || title.Platform == 1003) // ignore 3DS software updates
{
continue;
}
if (title.Platform == 143) // ignore some unknown platform with just 1 title
{
continue;
}
// get remaining data from ninja
XElement titleECInfoElement;
await Retry(async() => {
titleECInfoElement = await Ninja.GetECInfoForRegionAndTitleID(certWebClient, region, title.EshopId);
title.AddDataFromXml(titleECInfoElement);
if (title.JsonType != DatabaseJsonType.None)
{
titleList.Add(title);
}
});
}
}
}
// merge titles that occur in all regions into one title with the 'ALL' region
IEnumerable <EshopTitle> titleListEnumerable = titleList;
var allRegions = Enum.GetValues(typeof(Region)).OfType <Region>().Where(r => r != Region.None && r != Region.ALL);
var groups = titleListEnumerable.GroupBy(t => t.TitleId).Select(g => g.ToList()).ToList();
foreach (var group in groups)
{
var groupRegions = group.Select(t => t.Region);
if (allRegions.All(groupRegions.Contains))
{
titleListEnumerable = titleListEnumerable.Except(group.Skip(1));
group[0].Region = Region.ALL;
}
}
titleList = titleListEnumerable.ToList();
return(titleList);
}
/// <summary>
/// Retrieves a list of all WiiU updates
/// </summary>
/// <param name="currentListVersion">The update list version to start from. Use 1 to download everything.</param>
/// <returns>A list of update titles</returns>
public async Task <List <EshopTitle> > GetAllWiiUUpdates(int currentListVersion = 1)
{
if (currentListVersion < 1)
{
currentListVersion = 1;
}
Console.Write("Getting latest update list version ...");
int listVersion = -1;
await Retry(async() => {
listVersion = await Tagaya.GetLatestListVersion(webClient);
});
NewestWiiUUpdateListVersion = listVersion;
Console.WriteLine(" {0}.", listVersion);
progressManager.Reset(listVersion - currentListVersion);
Console.WriteLine("Downloading {0} update lists ...", listVersion - currentListVersion);
progressManager.SetTitle(string.Format("Downloading {0} update lists ...", listVersion - currentListVersion));
HashSet <EshopTitle> updateSet = new HashSet <EshopTitle>();
// download all update lists
for (int i = currentListVersion + 1; i <= listVersion; i++)
{
progressManager.Step("Downloading WiiU update list ...");
/* structure:
* <version_list ...><titles>
* <title><id>[titleID]</id><version>[updateVersion]</version></title>
* ...
* </titles></version_list>
*/
XDocument updatesXml = null;
bool success = await Retry(async() =>
{
updatesXml = await Tagaya.GetUpdatesXmlForListVersion(webClient, i);
},
shouldStopRetrying : (e) =>
{
// for some list versions a 403 error is received, ignore
return(e is WebException we && we.Response is HttpWebResponse resp && resp.StatusCode == HttpStatusCode.Forbidden);
});
if (!success)
{
continue;
}
// iterate over update version in xml
XElement titlesElement = updatesXml.Root.Element("titles");
foreach (XElement updateElement in titlesElement.Elements())
{
EshopTitle update = new EshopTitle();
update.TitleId = new TitleID(updateElement.Element("id").Value);
update.VersionString = updateElement.Element("version").Value;
update.JsonType = DatabaseJsonType.Updates;
updateSet.Add(update);
}
}
await AddSizesToTitles(updateSet);
return(updateSet.ToList());
}