internal void Synchronize(string url, DownloadMode mode, string directory, bool deleteRemovedSongs, string clientId)
{
verifyParameters(
new Dictionary<string, string>()
{
{"URL", url},
{"Directory", directory},
{"Client ID", clientId}
}
);
ResetProgress();
string apiURL = null;
switch (mode)
{
case DownloadMode.Playlist:
// determine whether it is an api url or a normal url. if it is a normal url, get the api url from it
// and then call SynchronizeFromPlaylistAPIUrl. Otherwise just call that method directly
if (!url.Contains("api.soundcloud.com"))
{
apiURL = determineAPIUrlForNormalUrl(url, clientId,"playlists");
}
else
{
apiURL = url;
}
SynchronizeFromPlaylistAPIUrl(apiURL, clientId, directory, deleteRemovedSongs);
break;
case DownloadMode.Favorites:
// get the username from the url and then call SynchronizeFromProfile
string username = parseUserIdFromProfileUrl(url);
SynchronizeFromProfile(username, clientId, directory, deleteRemovedSongs);
break;
case DownloadMode.Artist:
if (!url.Contains("api.soundcloud.com"))
{
apiURL = determineAPIUrlForNormalUrl(url, clientId,"tracks");
}
else
{
apiURL = url;
}
SynchronizeFromArtistUrl(apiURL, clientId, directory, deleteRemovedSongs);
break;
default:
IsError = true;
throw new NotImplementedException("Unknown download mode");
}
}
public static void Main(string[] args)
{
Program program = new Program();
while (true)
{
try
{
Console.WriteLine("Operation mode :");
Console.WriteLine("\t 1. Normal");
Console.WriteLine("\t 2. AsyncAndWait");
Console.WriteLine("\t 3. AsyncParalel");
Console.WriteLine("\t 0. Quit");
Console.Write("Input operation mode (1/2/3/0): ");
string inputText = Console.ReadLine();
int inputInt = int.Parse(inputText);
if (inputInt == 0)
{
break;
}
DownloadMode mode = (DownloadMode)inputInt;
program.Execute(mode);
}
catch
{
Console.WriteLine("Error process, try again");
}
Console.WriteLine();
}
Console.WriteLine("Finish");
}
public static AssetBundleInfo FromString(string str)
{
if (string.IsNullOrEmpty(str))
{
return(null);
}
try
{
JSONNode node = JSON.Parse(str);
string name = (string)node[FileNamePattern];
long asInt = node[FileCompressedSizePattern].AsInt;
string crc = (string)node[FileCrcPattern];
UnloadMode unloadMode = (UnloadMode)node[FileUnloadModePattern].AsInt;
DownloadMode downloadMode = (DownloadMode)node[FileDownloadModePattern].AsInt;
BundleType bundleType = (BundleType)node[FileBundleTypePattern].AsInt;
bool asBool = node[RemainPattern].AsBool;
string remoteDir = (string)node[RemoteDirPattern];
HashSet <string> assetPathSet = new HashSet <string>();
JSONArray asArray = node[AssetPathSetPattern].AsArray;
for (int i = 0; i < asArray.Count; i++)
{
assetPathSet.Add((string)asArray[i]);
}
HashSet <string> parentFileNameSet = new HashSet <string>();
JSONArray array2 = node[ParentFileNamePattern].AsArray;
for (int j = 0; j < array2.Count; j++)
{
parentFileNameSet.Add((string)array2[j]);
}
return(new AssetBundleInfo(name, asInt, crc, parentFileNameSet, assetPathSet, unloadMode, downloadMode, bundleType, asBool, remoteDir));
}
catch (Exception)
{
return(null);
}
}
internal static async Task <FileInfo> DownloadNupkgV3Async(
CatalogEntry entry,
IEnumerable <DirectoryInfo> storagePaths,
DownloadMode mode,
ILogger log,
ILogger deepLog,
CancellationToken token)
{
FileInfo result = null;
DirectoryInfo rootDir = null;
var lastCreated = DateTimeOffset.MinValue;
// Check if the nupkg already exists on another drive.
foreach (var storagePath in storagePaths)
{
var currentResolver = new VersionFolderPathResolver(storagePath.FullName);
var checkNupkgPath = currentResolver.GetPackageFilePath(entry.Id, entry.Version);
if (File.Exists(checkNupkgPath))
{
// Use the existing path
lastCreated = File.GetCreationTimeUtc(checkNupkgPath);
rootDir = storagePath;
break;
}
}
if (rootDir == null)
{
// Not found, use the path with the most space
rootDir = GetPathWithTheMostFreeSpace(storagePaths);
}
// id/version/id.version.nupkg
var versionFolderResolver = new VersionFolderPathResolver(rootDir.FullName);
var outputDir = versionFolderResolver.GetInstallPath(entry.Id, entry.Version);
var hashPath = versionFolderResolver.GetHashPath(entry.Id, entry.Version);
var nuspecPath = versionFolderResolver.GetManifestFilePath(entry.Id, entry.Version);
var nupkgPath = versionFolderResolver.GetPackageFilePath(entry.Id, entry.Version);
// Download
var nupkgFile = await entry.DownloadNupkgAsync(outputDir, mode, token);
if (File.Exists(nupkgPath))
{
var currentCreated = File.GetCreationTimeUtc(nupkgPath);
// Clean up nuspec and hash if the file changed
if (lastCreated < currentCreated || !File.Exists(hashPath) || !File.Exists(nuspecPath))
{
result = nupkgFile;
log.LogInformation(nupkgFile.FullName);
FileUtility.Delete(hashPath);
FileUtility.Delete(nuspecPath);
using (var fileStream = File.OpenRead(nupkgFile.FullName))
{
var packageHash = Convert.ToBase64String(new CryptoHashProvider("SHA512").CalculateHash(fileStream));
fileStream.Seek(0, SeekOrigin.Begin);
// Write nuspec
using (var reader = new PackageArchiveReader(fileStream))
{
var nuspecFile = reader.GetNuspecFile();
reader.ExtractFile(nuspecFile, nuspecPath, deepLog);
}
// Write package hash
File.WriteAllText(hashPath, packageHash);
}
}
}
return(result);
}
internal static async Task <FileInfo> DownloadNupkgV2Async(
CatalogEntry entry,
IEnumerable <DirectoryInfo> storagePaths,
DownloadMode mode,
ILogger log,
CancellationToken token)
{
FileInfo result = null;
DirectoryInfo rootDir = null;
var lastCreated = DateTimeOffset.MinValue;
// Check if the nupkg already exists on another drive.
foreach (var storagePath in storagePaths)
{
var checkOutputDir = Path.Combine(storagePath.FullName, entry.Id.ToLowerInvariant());
var checkNupkgPath = Path.Combine(checkOutputDir, $"{entry.FileBaseName}.nupkg");
if (File.Exists(checkNupkgPath))
{
// Use the existing path
lastCreated = File.GetCreationTimeUtc(checkNupkgPath);
rootDir = storagePath;
break;
}
}
// Does not exist, use the path with the most free space.
if (rootDir == null)
{
rootDir = GetPathWithTheMostFreeSpace(storagePaths);
}
// id/id.version.nupkg
var outputDir = Path.Combine(rootDir.FullName, entry.Id.ToLowerInvariant());
var nupkgPath = Path.Combine(outputDir, $"{entry.FileBaseName}.nupkg");
// Download
var nupkgFile = await entry.DownloadNupkgAsync(outputDir, mode, token);
if (File.Exists(nupkgPath))
{
var currentCreated = File.GetCreationTimeUtc(nupkgPath);
if (lastCreated < currentCreated)
{
result = nupkgFile;
log.LogInformation(nupkgFile.FullName);
}
else
{
log.LogDebug($"Skipping. Current file is the same or newer. {lastCreated.ToString("o")} {currentCreated.ToString("o")}" + nupkgFile.FullName);
}
}
else
{
log.LogDebug($"Nupkg skipped. " + nupkgFile.FullName);
}
return(result);
}
static public void ImportHistoricalData(DataClientBase dcb, string Username, string Password, DateTime StartTime, DateTime EndTime, DownloadMode DownloadMode)
{
if (FuncMsg == null)
{
FuncMsg = "Importing historical data, Data Feed = " + dcb + ";Mode=" + DownloadMode + ";Start date=" + StartTime.ToString("yyyy-MM-dd");
FuncStartTime = DateTime.Now;
try
{
if (DataFeedAvailable(dcb, Username, Password))
{
DataManagerBase dmb = Utils.GetDataManager(Config.DefaultDataManager);
int i = 1;
string[] ss = dmb.GetSymbolStrings(null, null, null);
int succ = 0;
int failed = 0;
string LastError = "";
if (ss.Length == 0)
{
Msg = "No symbols in current database";
}
foreach (string s in ss)
{
try
{
CommonDataProvider cdp;
bool b = true;
if (DownloadMode == DownloadMode.DownloadIfNoData)
{
cdp = (CommonDataProvider)dmb[s, 1];
b = cdp.Count == 0;
}
if (b)
{
Msg = "Importing historical data " + i + "/" + ss.Length + ";" + s + ";succ=" + succ + ";failed=" + failed + ";LastError=" + LastError;
cdp = dcb.GetHistoricalData(s, StartTime, EndTime);
if (cdp.Count != 0)
{
dmb.SaveData(s, cdp, false);
}
}
succ++;
i++;
}
catch (Exception e)
{
LastError = e.Message;
failed++;
}
}
}
}
finally
{
FuncMsg = null;
}
}
}
public DownloadManager(DownloadMode dwnmode)
{
InitializeComponent();
this.Dwnmode = dwnmode;
}
internal static async Task DownloadFileAsync(Stream stream, FileInfo outputFile, DateTimeOffset created, DownloadMode mode, CancellationToken token)
{
if (outputFile.Exists && IsValidNupkg(outputFile))
{
if (mode == DownloadMode.FailIfExists)
{
throw new InvalidOperationException($"File already exists: {outputFile.FullName}");
}
else if (mode == DownloadMode.SkipIfExists)
{
// noop
return;
}
else if (mode == DownloadMode.OverwriteIfNewer)
{
if (created <= outputFile.LastWriteTimeUtc)
{
// noop
return;
}
}
}
var tmp = new FileInfo(Path.Combine(outputFile.Directory.FullName, Guid.NewGuid().ToString()));
try
{
outputFile.Directory.Create();
using (var outputStream = new FileStream(tmp.FullName, FileMode.CreateNew))
{
await stream.CopyToAsync(outputStream, 8192, token);
}
FileUtility.Delete(outputFile.FullName);
FileUtility.Move(tmp.FullName, outputFile.FullName);
File.SetCreationTimeUtc(outputFile.FullName, created.UtcDateTime);
File.SetLastWriteTimeUtc(outputFile.FullName, created.UtcDateTime);
}
finally
{
// Clean up if needed
FileUtility.Delete(tmp.FullName);
}
}
public AssetBundleInfo(string name, long compressedSize, string crc, HashSet <string> parentFileNameSet, HashSet <string> assetPathSet, UnloadMode unloadMode, DownloadMode downloadMode, BundleType bundleType, bool remain, string remoteDir)
{
this.FileName = name;
this.FileCompressedSize = compressedSize;
this.FileCrc = crc;
this.ParentFileNameSet = parentFileNameSet;
this.AssetPathSet = assetPathSet;
this.FileUnloadMode = unloadMode;
this.FileDownloadMode = downloadMode;
this.FileBundleType = bundleType;
this.RemainInInstallPackage = remain;
this.RemoteDir = remoteDir;
this._assetBundle = null;
this._completeness = false;
}
public (List <Dictionary <string, string> > games, List <string> headers) GetData(string gameCode, DownloadMode mode)
{
var url = $"http://us.patch.battle.net:1119/{gameCode}/{mode.ToString()}?nocache={DateTime.Now.Millisecond}";
List <string> fileData = bnetWebClient.DownloadString(url).Trim().Lines().ToList();
List <Dictionary <string, string> > CreatedFileData = new List <Dictionary <string, string> >();
string[] headers = fileData[0].Split('|');
fileData.RemoveAt(0);
for (int i = 0; i < headers.Length; i++)
{
headers[i] = headers[i].Split('!')[0].SplitCamelCase();
}
for (int i = 0; i < fileData.Count; i++)
{
Dictionary <string, string> item = new Dictionary <string, string>();
string[] line = fileData[i].Split('|');
for (int h = 0; h < headers.Length; h++)
{
item.Add(headers[h], line[h]);
}
CreatedFileData.Add(item);
}
return(games : CreatedFileData, headers : headers.ToList());
}
internal void Start(bool resume)
{
ClientEngine.MainLoop.QueueWait(delegate
{
CheckRegisteredAndDisposed();
this.Engine.Start();
// If the torrent was "paused", then just update the state to Downloading and forcefully
// make sure the peers begin sending/receiving again
if (this.State == TorrentState.Paused)
{
Mode = new DownloadMode(this);
return;
}
if (!HasMetadata)
{
if (TrackerManager.CurrentTracker != null)
{
this.TrackerManager.Announce(TorrentEvent.Started);
}
Mode = new MetadataMode(this, _torrentSaveFolder);
#if !DISABLE_DHT
StartDht();
#endif
return;
}
VerifyHashState();
// If the torrent has not been hashed, we start the hashing process then we wait for it to finish
// before attempting to start again
if (!HashChecked)
{
if (State != TorrentState.Hashing)
{
HashCheck(true);
}
return;
}
if (State == TorrentState.Seeding || State == TorrentState.Downloading)
{
return;
}
if (TrackerManager.CurrentTracker != null && !resume)
{
if (this.TrackerManager.CurrentTracker.CanScrape)
{
this.TrackerManager.Scrape();
}
this.TrackerManager.Announce(TorrentEvent.Started); // Tell server we're starting
}
if (this.Complete && this.Settings.InitialSeedingEnabled && ClientEngine.SupportsInitialSeed)
{
Mode = new InitialSeedingMode(this);
}
else
{
Mode = new DownloadMode(this);
}
Engine.Broadcast(this);
#if !DISABLE_DHT
StartDht();
#endif
this.StartTime = DateTime.Now;
this.PieceManager.Reset();
ClientEngine.MainLoop.QueueTimeout(TimeSpan.FromSeconds(2), delegate
{
if (State != TorrentState.Downloading && State != TorrentState.Seeding)
{
return(false);
}
PieceManager.Picker.CancelTimedOutRequests();
return(true);
});
});
}
public static Task <IReadOnlyList <FileInfo> > DownloadNupkgsAsync(string outputDirectory, DownloadMode mode, int maxConcurrentDownloads, CancellationToken token, IEnumerable <CatalogEntry> entries)
{
var entriesArray = entries.ToArray();
if (entries.Distinct().Count() != entriesArray.Length)
{
throw new InvalidOperationException("Duplicate entries detected. Entries must be unique by id/version.");
}
return(RunAsync <FileInfo>(
apply: e => e.DownloadNupkgAsync(outputDirectory, mode, token),
maxThreads: maxConcurrentDownloads,
token: token,
entries: entriesArray));
}
public Patcher(string PatchList, string FilesPath, DownloadMode Mode = DownloadMode.HTTP, AuthMode Auth = AuthMode.NOAUTH)
{
this.PatchListPath = new Uri(PatchList);
this.FilesPath = FilesPath;
Authentication = Auth;
if (Mode != DownloadMode.HTTP)
{
throw new NotImplementedException("FTP Downloading is not Supportet, yet.");
}
}
private void InitCreateResponse(string url)
{
_response = WebFactory.GetHttpResponse(url);
_connectionEstablished = IsResponseValid(_response);
_rangeFrom = Options.RangeFrom;
_downloadMode = Options.DownloadMode;
}
/// <summary>
/// Download the nupkg to a folder.
/// </summary>
public Task <FileInfo> DownloadNupkgAsync(string outputDirectory, DownloadMode mode, CancellationToken token)
{
return(DownloadNupkgAsync(outputDirectory, mode, DateTimeOffset.UtcNow, token));
}
public DownloadEvent(DownloadMode mode, string url, int num)
: base(mode.ToString() + ": " + num.ToString() + " - " + url, null, (int)mode, num) {
}
