public override void Ingress <T>(DryadLinqContext context,
IEnumerable <T> source,
Uri dataSetUri,
DryadLinqMetaData metaData,
CompressionScheme compressionScheme,
bool isTemp = false)
{
string account, key, container, blob;
AzureUtils.FromAzureUri(dataSetUri, out account, out key, out container, out blob);
if (compressionScheme != CompressionScheme.None)
{
throw new DryadLinqException("Not implemented: writing to Azure temporary storage with compression enabled");
}
AzureDfsClient client = new AzureDfsClient(account, key, container);
DryadLinqFactory <T> factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (Stream stream = client.GetFileStreamWriterAsync(blob).Result)
{
DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream);
DryadLinqRecordWriter <T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
{
writer.WriteRecordSync(rec);
}
writer.Close();
}
}
internal TableEnumerator(DryadLinqContext context, DataProvider dataProvider, Uri dataSetUri)
{
this.m_current = default(T);
this.m_factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
Stream stream = dataProvider.Egress(context, dataSetUri);
DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream);
this.m_reader = this.m_factory.MakeReader(nativeStream);
this.m_reader.StartWorker();
}
public override void Ingress <T>(DryadLinqContext context,
IEnumerable <T> source,
Uri dataSetUri,
DryadLinqMetaData metaData,
CompressionScheme outputScheme,
bool isTemp,
Expression <Action <IEnumerable <T>, Stream> > serializer)
{
DryadLinqFactory <T> factory = (DryadLinqFactory <T>)DryadLinqCodeGen.GetFactory(context, typeof(T));
using (Stream stream = context.GetHdfsClient.GetDfsStreamWriter(dataSetUri))
{
DryadLinqBlockStream nativeStream = new DryadLinqBlockStream(stream);
DryadLinqRecordWriter <T> writer = factory.MakeWriter(nativeStream);
foreach (T rec in source)
{
writer.WriteRecordSync(rec);
}
writer.Close();
}
}
