private async Task<string> GetUsers( LoadOptions options, int[] users ) { if ( options.Execute ) { var req = new Request<User[]> { MethodName = "execute.mfetch25a", Token = _api.CurrentToken, Parameters = new Dictionary<string, string>() }; req.Parameters.Add( "fields", string.Join( ",", MiscTools.GetUserFields( options.Fields )) ); var uids = users.Select( ( value, index ) => new { value, index } ).GroupBy( a => a.index / 1000 ).Select( a => a.Select( b => b.value ).ToArray() ).ToArray(); for ( int i = 0; i < uids.Length; i++ ) req.Parameters.Add( "u" + i, uids[ i ].ToNCStringA() ); return await _api.Executor.ExecRawAsync( req ).ConfigureAwait( false ); } return await _api.Users.Get(options.Fields, NameCase.Nom, users ).ConfigureAwait( false ); }
public FrmMain() { InitializeComponent(); lstFields.DataSource = ((UserFields[])Enum.GetValues(typeof(UserFields))).Where(a => a != UserFields.Anything && a != UserFields.Everything && a != UserFields.None).OrderBy( a=>a.ToString() ).ToArray(); CheckForIllegalCrossThreadCalls = false; //start, end, (int) nud_threads.Value, txt_outpath.Text, GetFields(), volume, chkCompressOutput.Checked, UpdateProfilesCount, UpdateTraffic, () => _cancel ); options = new LoadOptions() { ShowCount = UpdateProfilesCount, ShowTraffic = UpdateTraffic, CancellationToken = () => _cancel }; this.nud_threads.DataBindings.Add( nameof( nud_threads.Value ), options, nameof( options.Threads ), false, DataSourceUpdateMode.OnPropertyChanged ); this.nud_start.DataBindings.Add(nameof(nud_start.Value), options, nameof(options.Start), false, DataSourceUpdateMode.OnPropertyChanged); this.nud_end.DataBindings.Add(nameof(nud_end.Value), options, nameof(options.End), false, DataSourceUpdateMode.OnPropertyChanged); this.nudDelay.DataBindings.Add(nameof(nudDelay.Value), options, nameof(options.Delay), false, DataSourceUpdateMode.OnPropertyChanged); this.nud_volume.DataBindings.Add(nameof(nud_volume.Value), options, nameof(options.VolumeSize), false, DataSourceUpdateMode.OnPropertyChanged); this.txt_outpath.DataBindings.Add( nameof( txt_outpath.Text ), options, nameof( options.Path ), false, DataSourceUpdateMode.OnPropertyChanged); this.chkGZip.DataBindings.Add( nameof( chkGZip.Checked ), options, nameof( options.GZip ), false, DataSourceUpdateMode.OnPropertyChanged); this.chkExecute.DataBindings.Add(nameof(chkExecute.Checked), options, nameof(options.Execute), false, DataSourceUpdateMode.OnPropertyChanged); this.chkDelay.DataBindings.Add(nameof(chkDelay.Checked), options, nameof(options.DelayEnabled), false, DataSourceUpdateMode.OnPropertyChanged); this.chkSubdirs.DataBindings.Add(nameof(chkSubdirs.Checked), options, nameof(options.Subdirs), false, DataSourceUpdateMode.OnPropertyChanged); this.chkUseCounter.DataBindings.Add(nameof(chkUseCounter.Checked), options, nameof(options.UseCounter), false, DataSourceUpdateMode.OnPropertyChanged); this.txtSourcefile.DataBindings.Add(nameof(txtSourcefile.Text), options, nameof(options.SourceFile), false, DataSourceUpdateMode.OnPropertyChanged); }
public async Task LoadUsers( LoadOptions options ) { long trafficUsed = 0, usersLoaded = 0; var p = options.UseCounter ? (IIdProvdier) new RangeIdProvider( options.Start, options.End ) : new FileIdProvdier( options.SourceFile ); int activeThreads = 0; using ( var semaphore = new SemaphoreSlim( options.Threads ) ) { Func<int[], Task> getChunk = async i => { try { int[] users = i; if ( options.CancellationToken?.Invoke() ?? false ) return; var outfile = GetChunkPath(options, users ); if ( !File.Exists( outfile ) ) { var resp = await GetUsers( options, users ).ConfigureAwait(false); if ( resp == null ) return; await SaveFile( outfile, resp, options.GZip ).ConfigureAwait(false); usersLoaded += users.Length; trafficUsed += _textEncoding.GetByteCount( resp ); options.ShowTraffic?.Invoke( trafficUsed ); options.ShowCount?.Invoke( usersLoaded ); } } catch {} finally { --activeThreads; semaphore.Release(); } }; int[] current; while ( (current=await p.Get( options.VolumeSize ).ConfigureAwait( false )).Any() ) { if (options.CancellationToken?.Invoke() ?? false ) break; await semaphore.WaitAsync().ConfigureAwait(false); Console.WriteLine( "Threads: {0}", ++activeThreads ); var tsk = getChunk( current ); if ( options.DelayEnabled) await Task.Delay( options.Delay ).ConfigureAwait(false); } for ( int i = 0; i < options.Threads; i++ ) await semaphore.WaitAsync().ConfigureAwait( false ); } }
private async Task <string> GetUsers(LoadOptions options, int[] users) { if (options.Execute) { var req = new Request <User[]> { MethodName = "execute.mfetch25a", Token = _api.CurrentToken, Parameters = new Dictionary <string, string>() }; req.Parameters.Add("fields", string.Join(",", MiscTools.GetUserFields(options.Fields))); var uids = users.Select((value, index) => new { value, index }).GroupBy(a => a.index / 1000).Select(a => a.Select(b => b.value).ToArray()).ToArray(); for (int i = 0; i < uids.Length; i++) { req.Parameters.Add("u" + i, uids[i].ToNCStringA()); } return(await _api.Executor.ExecRawAsync(req).ConfigureAwait(false)); } return(await _api.Users.Get(options.Fields, NameCase.Nom, users).ConfigureAwait(false)); }
public FrmMain() { InitializeComponent(); lstFields.DataSource = ((UserFields[])Enum.GetValues(typeof(UserFields))).Where(a => a != UserFields.Anything && a != UserFields.Everything && a != UserFields.None).OrderBy(a => a.ToString()).ToArray(); CheckForIllegalCrossThreadCalls = false; //start, end, (int) nud_threads.Value, txt_outpath.Text, GetFields(), volume, chkCompressOutput.Checked, UpdateProfilesCount, UpdateTraffic, () => _cancel ); options = new LoadOptions() { ShowCount = UpdateProfilesCount, ShowTraffic = UpdateTraffic, CancellationToken = () => _cancel }; this.nud_threads.DataBindings.Add(nameof(nud_threads.Value), options, nameof(options.Threads), false, DataSourceUpdateMode.OnPropertyChanged); this.nud_start.DataBindings.Add(nameof(nud_start.Value), options, nameof(options.Start), false, DataSourceUpdateMode.OnPropertyChanged); this.nud_end.DataBindings.Add(nameof(nud_end.Value), options, nameof(options.End), false, DataSourceUpdateMode.OnPropertyChanged); this.nudDelay.DataBindings.Add(nameof(nudDelay.Value), options, nameof(options.Delay), false, DataSourceUpdateMode.OnPropertyChanged); this.nud_volume.DataBindings.Add(nameof(nud_volume.Value), options, nameof(options.VolumeSize), false, DataSourceUpdateMode.OnPropertyChanged); this.txt_outpath.DataBindings.Add(nameof(txt_outpath.Text), options, nameof(options.Path), false, DataSourceUpdateMode.OnPropertyChanged); this.chkGZip.DataBindings.Add(nameof(chkGZip.Checked), options, nameof(options.GZip), false, DataSourceUpdateMode.OnPropertyChanged); this.chkExecute.DataBindings.Add(nameof(chkExecute.Checked), options, nameof(options.Execute), false, DataSourceUpdateMode.OnPropertyChanged); this.chkDelay.DataBindings.Add(nameof(chkDelay.Checked), options, nameof(options.DelayEnabled), false, DataSourceUpdateMode.OnPropertyChanged); this.chkSubdirs.DataBindings.Add(nameof(chkSubdirs.Checked), options, nameof(options.Subdirs), false, DataSourceUpdateMode.OnPropertyChanged); this.chkUseCounter.DataBindings.Add(nameof(chkUseCounter.Checked), options, nameof(options.UseCounter), false, DataSourceUpdateMode.OnPropertyChanged); this.txtSourcefile.DataBindings.Add(nameof(txtSourcefile.Text), options, nameof(options.SourceFile), false, DataSourceUpdateMode.OnPropertyChanged); }
private static string GetChunkPath( LoadOptions options, int[] users ) { var filename = $"{users.First()}_{users.Length}.json{( options.GZip ? ".gz" : "" )}"; var dir = options.Path; if (options.Subdirs) { dir = Path.Combine(dir, ( users.First() / 1000000 ).ToString() ); if ( !Directory.Exists(dir) ) Directory.CreateDirectory(dir); } return Path.Combine(dir, filename ); }
public async Task LoadUsers( LoadOptions options) { long trafficUsed = 0, usersLoaded = 0; var p = options.UseCounter ? (IIdProvdier) new RangeIdProvider(options.Start, options.End) : new FileIdProvdier(options.SourceFile); int activeThreads = 0; using (var semaphore = new SemaphoreSlim(options.Threads)) { Func <int[], Task> getChunk = async i => { try { int[] users = i; if (options.CancellationToken?.Invoke() ?? false) { return; } var outfile = GetChunkPath(options, users); if (!File.Exists(outfile)) { var resp = await GetUsers(options, users).ConfigureAwait(false); if (resp == null) { return; } await SaveFile(outfile, resp, options.GZip).ConfigureAwait(false); usersLoaded += users.Length; trafficUsed += _textEncoding.GetByteCount(resp); options.ShowTraffic?.Invoke(trafficUsed); options.ShowCount?.Invoke(usersLoaded); } } catch {} finally { --activeThreads; semaphore.Release(); } }; int[] current; while ((current = await p.Get(options.VolumeSize).ConfigureAwait(false)).Any()) { if (options.CancellationToken?.Invoke() ?? false) { break; } await semaphore.WaitAsync().ConfigureAwait(false); Console.WriteLine("Threads: {0}", ++activeThreads); var tsk = getChunk(current); if (options.DelayEnabled) { await Task.Delay(options.Delay).ConfigureAwait(false); } } for (int i = 0; i < options.Threads; i++) { await semaphore.WaitAsync().ConfigureAwait(false); } } }