private async Task<string> GetUsers( LoadOptions options, int[] users ) {
if ( options.Execute ) {
var req = new Request<User[]> {
MethodName = "execute.mfetch25a",
Token = _api.CurrentToken,
Parameters = new Dictionary<string, string>()
};
req.Parameters.Add( "fields", string.Join( ",", MiscTools.GetUserFields( options.Fields )) );
var uids = users.Select( ( value, index ) => new { value, index } ).GroupBy( a => a.index / 1000 ).Select( a => a.Select( b => b.value ).ToArray() ).ToArray();
for ( int i = 0; i < uids.Length; i++ ) req.Parameters.Add( "u" + i, uids[ i ].ToNCStringA() );
return await _api.Executor.ExecRawAsync( req ).ConfigureAwait( false );
}
return await _api.Users.Get(options.Fields, NameCase.Nom, users ).ConfigureAwait( false );
}
public FrmMain() {
InitializeComponent();
lstFields.DataSource = ((UserFields[])Enum.GetValues(typeof(UserFields))).Where(a => a != UserFields.Anything && a != UserFields.Everything && a != UserFields.None).OrderBy( a=>a.ToString() ).ToArray();
CheckForIllegalCrossThreadCalls = false;
//start, end, (int) nud_threads.Value, txt_outpath.Text, GetFields(), volume, chkCompressOutput.Checked, UpdateProfilesCount, UpdateTraffic, () => _cancel );
options = new LoadOptions() { ShowCount = UpdateProfilesCount, ShowTraffic = UpdateTraffic, CancellationToken = () => _cancel };
this.nud_threads.DataBindings.Add( nameof( nud_threads.Value ), options, nameof( options.Threads ), false, DataSourceUpdateMode.OnPropertyChanged );
this.nud_start.DataBindings.Add(nameof(nud_start.Value), options, nameof(options.Start), false, DataSourceUpdateMode.OnPropertyChanged);
this.nud_end.DataBindings.Add(nameof(nud_end.Value), options, nameof(options.End), false, DataSourceUpdateMode.OnPropertyChanged);
this.nudDelay.DataBindings.Add(nameof(nudDelay.Value), options, nameof(options.Delay), false, DataSourceUpdateMode.OnPropertyChanged);
this.nud_volume.DataBindings.Add(nameof(nud_volume.Value), options, nameof(options.VolumeSize), false, DataSourceUpdateMode.OnPropertyChanged);
this.txt_outpath.DataBindings.Add( nameof( txt_outpath.Text ), options, nameof( options.Path ), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkGZip.DataBindings.Add( nameof( chkGZip.Checked ), options, nameof( options.GZip ), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkExecute.DataBindings.Add(nameof(chkExecute.Checked), options, nameof(options.Execute), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkDelay.DataBindings.Add(nameof(chkDelay.Checked), options, nameof(options.DelayEnabled), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkSubdirs.DataBindings.Add(nameof(chkSubdirs.Checked), options, nameof(options.Subdirs), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkUseCounter.DataBindings.Add(nameof(chkUseCounter.Checked), options, nameof(options.UseCounter), false, DataSourceUpdateMode.OnPropertyChanged);
this.txtSourcefile.DataBindings.Add(nameof(txtSourcefile.Text), options, nameof(options.SourceFile), false, DataSourceUpdateMode.OnPropertyChanged);
}
public async Task LoadUsers(
LoadOptions options ) {
long trafficUsed = 0, usersLoaded = 0;
var p = options.UseCounter ? (IIdProvdier) new RangeIdProvider( options.Start, options.End ) : new FileIdProvdier( options.SourceFile );
int activeThreads = 0;
using ( var semaphore = new SemaphoreSlim( options.Threads ) ) {
Func<int[], Task> getChunk = async i => {
try {
int[] users = i;
if ( options.CancellationToken?.Invoke() ?? false ) return;
var outfile = GetChunkPath(options, users );
if ( !File.Exists( outfile ) ) {
var resp = await GetUsers( options, users ).ConfigureAwait(false);
if ( resp == null ) return;
await SaveFile( outfile, resp, options.GZip ).ConfigureAwait(false);
usersLoaded += users.Length;
trafficUsed += _textEncoding.GetByteCount( resp );
options.ShowTraffic?.Invoke( trafficUsed );
options.ShowCount?.Invoke( usersLoaded );
}
}
catch {}
finally {
--activeThreads;
semaphore.Release();
}
};
int[] current;
while ( (current=await p.Get( options.VolumeSize ).ConfigureAwait( false )).Any() ) {
if (options.CancellationToken?.Invoke() ?? false ) break;
await semaphore.WaitAsync().ConfigureAwait(false);
Console.WriteLine( "Threads: {0}", ++activeThreads );
var tsk = getChunk( current );
if ( options.DelayEnabled)
await Task.Delay( options.Delay ).ConfigureAwait(false);
}
for ( int i = 0; i < options.Threads; i++ ) await semaphore.WaitAsync().ConfigureAwait( false );
}
}
private async Task <string> GetUsers(LoadOptions options, int[] users)
{
if (options.Execute)
{
var req = new Request <User[]> {
MethodName = "execute.mfetch25a",
Token = _api.CurrentToken,
Parameters = new Dictionary <string, string>()
};
req.Parameters.Add("fields", string.Join(",", MiscTools.GetUserFields(options.Fields)));
var uids = users.Select((value, index) => new { value, index }).GroupBy(a => a.index / 1000).Select(a => a.Select(b => b.value).ToArray()).ToArray();
for (int i = 0; i < uids.Length; i++)
{
req.Parameters.Add("u" + i, uids[i].ToNCStringA());
}
return(await _api.Executor.ExecRawAsync(req).ConfigureAwait(false));
}
return(await _api.Users.Get(options.Fields, NameCase.Nom, users).ConfigureAwait(false));
}
public FrmMain()
{
InitializeComponent();
lstFields.DataSource = ((UserFields[])Enum.GetValues(typeof(UserFields))).Where(a => a != UserFields.Anything && a != UserFields.Everything && a != UserFields.None).OrderBy(a => a.ToString()).ToArray();
CheckForIllegalCrossThreadCalls = false;
//start, end, (int) nud_threads.Value, txt_outpath.Text, GetFields(), volume, chkCompressOutput.Checked, UpdateProfilesCount, UpdateTraffic, () => _cancel );
options = new LoadOptions()
{
ShowCount = UpdateProfilesCount, ShowTraffic = UpdateTraffic, CancellationToken = () => _cancel
};
this.nud_threads.DataBindings.Add(nameof(nud_threads.Value), options, nameof(options.Threads), false, DataSourceUpdateMode.OnPropertyChanged);
this.nud_start.DataBindings.Add(nameof(nud_start.Value), options, nameof(options.Start), false, DataSourceUpdateMode.OnPropertyChanged);
this.nud_end.DataBindings.Add(nameof(nud_end.Value), options, nameof(options.End), false, DataSourceUpdateMode.OnPropertyChanged);
this.nudDelay.DataBindings.Add(nameof(nudDelay.Value), options, nameof(options.Delay), false, DataSourceUpdateMode.OnPropertyChanged);
this.nud_volume.DataBindings.Add(nameof(nud_volume.Value), options, nameof(options.VolumeSize), false, DataSourceUpdateMode.OnPropertyChanged);
this.txt_outpath.DataBindings.Add(nameof(txt_outpath.Text), options, nameof(options.Path), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkGZip.DataBindings.Add(nameof(chkGZip.Checked), options, nameof(options.GZip), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkExecute.DataBindings.Add(nameof(chkExecute.Checked), options, nameof(options.Execute), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkDelay.DataBindings.Add(nameof(chkDelay.Checked), options, nameof(options.DelayEnabled), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkSubdirs.DataBindings.Add(nameof(chkSubdirs.Checked), options, nameof(options.Subdirs), false, DataSourceUpdateMode.OnPropertyChanged);
this.chkUseCounter.DataBindings.Add(nameof(chkUseCounter.Checked), options, nameof(options.UseCounter), false, DataSourceUpdateMode.OnPropertyChanged);
this.txtSourcefile.DataBindings.Add(nameof(txtSourcefile.Text), options, nameof(options.SourceFile), false, DataSourceUpdateMode.OnPropertyChanged);
}
private static string GetChunkPath( LoadOptions options, int[] users ) {
var filename = $"{users.First()}_{users.Length}.json{( options.GZip ? ".gz" : "" )}";
var dir = options.Path;
if (options.Subdirs) {
dir = Path.Combine(dir, ( users.First() / 1000000 ).ToString() );
if ( !Directory.Exists(dir) ) Directory.CreateDirectory(dir);
}
return Path.Combine(dir, filename );
}
public async Task LoadUsers(
LoadOptions options)
{
long trafficUsed = 0, usersLoaded = 0;
var p = options.UseCounter ? (IIdProvdier) new RangeIdProvider(options.Start, options.End) : new FileIdProvdier(options.SourceFile);
int activeThreads = 0;
using (var semaphore = new SemaphoreSlim(options.Threads)) {
Func <int[], Task> getChunk = async i => {
try {
int[] users = i;
if (options.CancellationToken?.Invoke() ?? false)
{
return;
}
var outfile = GetChunkPath(options, users);
if (!File.Exists(outfile))
{
var resp = await GetUsers(options, users).ConfigureAwait(false);
if (resp == null)
{
return;
}
await SaveFile(outfile, resp, options.GZip).ConfigureAwait(false);
usersLoaded += users.Length;
trafficUsed += _textEncoding.GetByteCount(resp);
options.ShowTraffic?.Invoke(trafficUsed);
options.ShowCount?.Invoke(usersLoaded);
}
}
catch {}
finally {
--activeThreads;
semaphore.Release();
}
};
int[] current;
while ((current = await p.Get(options.VolumeSize).ConfigureAwait(false)).Any())
{
if (options.CancellationToken?.Invoke() ?? false)
{
break;
}
await semaphore.WaitAsync().ConfigureAwait(false);
Console.WriteLine("Threads: {0}", ++activeThreads);
var tsk = getChunk(current);
if (options.DelayEnabled)
{
await Task.Delay(options.Delay).ConfigureAwait(false);
}
}
for (int i = 0; i < options.Threads; i++)
{
await semaphore.WaitAsync().ConfigureAwait(false);
}
}
}