public override Task Init(IWorkerGrain self, PredicateBase predicate, IPrincipalGrain principalGrain) { base.Init(self, predicate, principalGrain); ulong filesize = ((ScanPredicate)predicate).FileSize; tableId = ((ScanPredicate)predicate).TableID; separator = ((ScanPredicate)predicate).Separator; string extensionKey = ""; Guid key = this.GetPrimaryKey(out extensionKey); ulong i = UInt64.Parse(extensionKey); ulong num_grains = (ulong)((ScanPredicate)predicate).NumberOfGrains; ulong partition = filesize / num_grains; ulong start_byte = i * partition; ulong end_byte = num_grains - 1 == i?filesize:(i + 1) * partition; reader = new ScanStreamReader(((ScanPredicate)predicate).File); if (!reader.GetFile(start_byte)) { return(Task.FromException(new Exception("unable to get file"))); } start = start_byte; end = end_byte; if (start != 0) { start += reader.TrySkipFirst(); } Console.WriteLine("Init: start byte: " + start.ToString() + " end byte: " + end.ToString()); return(Task.CompletedTask); }
public async Task Initialize() { string currentDir = Environment.CurrentDirectory; Console.WriteLine("Getting: " + currentDir + "/" + file); reader = new ScanStreamReader(currentDir + "/" + file, separator); if (!reader.GetFile(0)) { throw new Exception("unable to get file"); } await reader.FillBuffer(); }
public async Task <TexeraTuple> NextAsync() { if (reader.IsEOF() && fileNames.Count > 0) { reader.Close(); reader = new ScanStreamReader(folder + "/" + fileNames.Dequeue(), separator); if (!reader.GetFile(0)) { throw new Exception("unable to get file"); } await reader.FillBuffer(); } Pair <TexeraTuple, ulong> res = await reader.ReadTuple(); return(res.First); }
public async Task Initialize() { reader = new ScanStreamReader(file, separator, idxes); if (!reader.GetFile(start)) { throw new Exception("unable to get file"); } if (start != 0) { start += await reader.TrySkipFirst(); } else { await reader.FillBuffer(); } }
public async Task Initialize() { if (folder.StartsWith("http://")) { fileNames = new Queue <string>(Common.Utils.ListFileNameFromHDFSDirectory(folder)); } else { fileNames = new Queue <string>(Directory.GetFiles(folder, "*", SearchOption.TopDirectoryOnly)); } reader = new ScanStreamReader(folder + "/" + fileNames.Dequeue(), separator); if (!reader.GetFile(0)) { throw new Exception("unable to get file"); } await reader.FillBuffer(); }
private bool ReadTuple(out TexeraTuple tx) { try { ulong ByteCount; string res = reader.ReadLine(out ByteCount); start += ByteCount; if (reader.IsEOF()) { start = end + 1; tx = null; return(false); } try { tx = new TexeraTuple(tableId, res.Split(separator)); ++tuple_counter; return(true); } catch { tx = null; Console.WriteLine("Failed to parse the tuple"); return(false); } } catch (Exception ex) { Console.WriteLine("EXCEPTION in Reading Tuples from File - " + ex.ToString()); Console.WriteLine("start_offset: " + start.ToString() + " end_offset: " + end.ToString()); if (!reader.GetFile(start)) { throw new Exception("Reading Tuple: Cannot Get File"); } tx = null; return(false); } }