public override void Output(IRow input, IUnstructuredWriter output)
{
if (this.recordCol == -1)
{
// Map column names to indices once to avoid mapping overhead for each row.
this.recordCol = input.Schema.IndexOf("record");
this.offsetCol = input.Schema.IndexOf("offset");
this.encodingErrorsCol = input.Schema.IndexOf("encodingErrors");
}
var record = input.Get <byte[]>(recordCol);
var encodingErrors = input.Get <bool>(encodingErrorsCol);
if (encodingErrors && this.replaceInvalidBytes)
{
// Decode the record to substitute invalid bytes with the replacement character.
// Encode the sanitized record again before writing it to the output file.
record = this.nonThrowingEncoding.GetBytes(this.nonThrowingEncoding.GetString(record));
}
if (this.offsetFormat != null)
{
output.BaseStream.Write(
this.nonThrowingEncoding.GetBytes(string.Format(this.offsetFormat, input.Get <long>(this.offsetCol))));
var bom = this.nonThrowingEncoding.GetPreamble();
if (record.Take(bom.Length).SequenceEqual(bom))
{
// This should happen once per file. Prefer code simplicity to performance.
record = record.Skip(bom.Length).ToArray();
}
}
output.BaseStream.Write(record);
}
public override void Output(IRow input, IUnstructuredWriter output)
{
ISchema schema = input.Schema;
if (_ds == null)
{
List <SchemaElement> lse = new List <SchemaElement>();
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
lse.Add(new SchemaElement(col.Name, Type.GetType(getColType(col))));
}
_ds = new DataSet(new Schema(lse));
_tempStream = new MemoryStream();
_resultStream = output.BaseStream;
_writer = new ParquetWriter(output.BaseStream, null, writeroptions);
//create DS based on schema
//input.Schema
}
List <object> ls = new List <object>();
for (int i = 0; i < schema.Count; i++)
{
ls.Add(input.Get <dynamic>(input.Schema[i].Name));
}
Row r = new Row(ls);
_ds.Add(r);
}
public override void Output(IRow input, IUnstructuredWriter output)
{
if (_fileWriter == null)
{
_avSchema = Schema.Parse(_avroSchema) as RecordSchema;
var writer = new GenericDatumWriter <GenericRecord>(_avSchema);
_fileWriter = DataFileWriter <GenericRecord> .OpenWriter(writer, output.BaseStream);
}
var record = new GenericRecord(_avSchema);
foreach (var x in input.Schema)
{
var obj = input.Get <object>(x.Name);
if (obj != null)
{
var objType = obj.GetType();
if (objType.IsGenericType && objType.GetGenericTypeDefinition() == typeof(SqlArray <>))
{
obj = ((System.Collections.IEnumerable)obj).Cast <object>().ToArray();
}
}
record.Add(x.Name, obj);
}
_fileWriter.Append(record);
}
// Output
//
// Outputs the names of the rowset columns in a column separated row and optionally adds their types in a second row.
//
public override void Output(IRow row, IUnstructuredWriter output)
{
if (_first_row_written) { return; }
using (StreamWriter streamWriter = new StreamWriter(output.BaseStream, this._encoding))
{
streamWriter.NewLine = this._row_delim;
ISchema schema = row.Schema;
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
if (i > 0)
{
streamWriter.Write(this._col_delim);
}
var val = _quoting ? AddQuotes(col.Name) : col.Name;
streamWriter.Write(val);
}
streamWriter.WriteLine();
if (_with_types)
{
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
if (i > 0)
{
streamWriter.Write(this._col_delim);
}
var val = _quoting ? AddQuotes(col.Type.FullName) : col.Type.FullName;
streamWriter.Write(val);
}
streamWriter.WriteLine();
}
}
_first_row_written = true;
}
public override void Output(IRow input, IUnstructuredWriter output)
{
using (var streamWriter = new StreamWriter(output.BaseStream))
{
var value = input.Get <int>("value");
streamWriter.WriteLine(value);
}
}
public override void Output(IRow input, IUnstructuredWriter output)
{
var obj = input.Get<object>(0);
byte[] imageArray = (byte[])obj;
using (MemoryStream ms = new MemoryStream(imageArray))
{
var image = Image.FromStream(ms);
image.Save(output.BaseStream, ImageFormat.Jpeg);
}
}
public override void Output(IRow input, IUnstructuredWriter output)
{
var obj = input.Get <object>(0);
byte[] imageArray = (byte[])obj;
using (MemoryStream ms = new MemoryStream(imageArray))
{
var image = Image.FromStream(ms);
image.Save(output.BaseStream, ImageFormat.Jpeg);
}
}
public override void Output(IRow row, IUnstructuredWriter output)
{
ISchema schema = row.Schema;
for (int i = 0; i < schema.Count; i++)
{
object obj = row.Get <object>(i);
if (obj is byte[])
{
output.BaseStream.Write((byte[])obj, 0, ((byte[])obj).Length);
}
}
}
public override void Output(IRow input, IUnstructuredWriter output)
{
var file = input.Get <byte[]>(0);
_outputStream = output.BaseStream;
var entry = _archive.CreateEntry(string.Format(_filePattern, _fileCount++),
CompressionLevel.NoCompression);
using (var ms = new MemoryStream(file))
using (var entryStream = entry.Open())
{
ms.CopyTo(entryStream);
}
}
/// <summary/>
public override void Output(IRow row, IUnstructuredWriter output)
{
// First Row
if(this.writer == null)
{
// Json.Net (writer)
this.writer = new JsonTextWriter(new StreamWriter(output.BaseStream));
// Header (array)
this.writer.WriteStartArray();
}
// Row(s)
WriteRow(row, this.writer);
}
/// <summary/>
public override void Output(IRow row, IUnstructuredWriter output)
{
// First Row
if (this.writer == null)
{
// Json.Net (writer)
this.writer = new JsonTextWriter(new StreamWriter(output.BaseStream));
// Header (array)
this.writer.WriteStartArray();
}
// Row(s)
WriteRow(row, this.writer);
}
public override void Output(IRow input, IUnstructuredWriter output)
{
// If this is the first row, then get the stream output set up by the U-SQL runtime
if (this.outputWriter == null)
{
this.outputWriter = output.BaseStream;
}
// Get the schema of the row
var columnSchema = input.Schema;
// Iterate through the columns in the row and convert the data to XML encoded strings
StringBuilder rowData = new StringBuilder($@"<{xmlDocType}>");
foreach (var column in columnSchema)
{
rowData.Append($@"<{column.Name}>");
// This outputter currently only recognizes int, double, string, and DateTime data.
if (column.Type == typeof(int))
{
rowData.Append($@"{input.Get<int>(column.Name)}");
}
if (column.Type == typeof(double))
{
rowData.Append($@"{input.Get<double>(column.Name)}");
}
if (column.Type == typeof(string))
{
rowData.Append($@"{input.Get<string>(column.Name)}");
}
if (column.Type == typeof(DateTime))
{
rowData.Append($@"{input.Get<DateTime>(column.Name)}");
}
rowData.Append($@"</{column.Name}>");
}
rowData.Append($@"</{xmlDocType}>");
rowData.Append(Environment.NewLine);
// Send the XML encoded string to the output stream
string data = rowData.ToString();
this.outputWriter.Write(Encoding.UTF8.GetBytes(data), 0, data.Length);
}
// void Output(IRow row, IUnstructuredWriter output)
//
// Actual implementation of DriverOutputter that overwrites the Output method of IOutputter.
public override void Output(IRow row, IUnstructuredWriter output)
{
using (StreamWriter streamWriter = new StreamWriter(output.BaseStream, this._encoding))
{
streamWriter.NewLine = this._row_delim;
ISchema schema = row.Schema;
for (int i = 0; i < schema.Count; i++)
{
object val = row.Get <object>(i);
if (i > 0)
{
streamWriter.Write(this._col_delim);
}
this.WriteValue(val, streamWriter);
}
streamWriter.WriteLine();
}
}
public override void Output(IRow row, IUnstructuredWriter output)
{
// First Row
if (this.writer == null)
{
//this.writer = new SequentialWriter<object>(AvroContainer.Create<AnonType_WeatherDataSet>(this.avroSchema, output.BaseStream, Codec.Deflate), 24);
this.serializer = AvroSerializer.Create <AnonType_WeatherDataSet>();
}
AnonType_WeatherDataSet data = new AnonType_WeatherDataSet(
row.Get <string>(row.Schema[0].Name),
row.Get <int?>(row.Schema[1].Name),
row.Get <string>(row.Schema[2].Name),
(double?)row.Get <decimal?>(row.Schema[3].Name),
row.Get <string>(row.Schema[4].Name),
row.Get <string>(row.Schema[5].Name),
row.Get <string>(row.Schema[6].Name),
row.Get <int?>(row.Schema[7].Name)
);
serializer.Serialize(output.BaseStream, data);
}
/// <summary>Output is called at least once per instance</summary>
/// <param name="input">A SQLIP row</param>
/// <param name="output">Wrapper for a Stream</param>
public override void Output(IRow input, IUnstructuredWriter output)
{
IColumn badColumn = input.Schema.FirstOrDefault(col => col.Type != typeof(string));
if (badColumn != null)
{
throw new ArgumentException(string.Format("Column '{0}' must be of type 'string', not '{1}'", badColumn.Name, badColumn.Type.Name));
}
using (var writer = XmlWriter.Create(output.BaseStream, this.fragmentSettings))
{
writer.WriteStartElement(this.rowPath);
foreach (IColumn col in input.Schema)
{
var value = input.Get<string>(col.Name);
if (value != null)
{
// Skip null values in order to distinguish them from empty strings
writer.WriteElementString(this.columnPaths[col.Name] ?? col.Name, value);
}
}
}
}
public override void Output(IRow row, IUnstructuredWriter output)
{
// First Row
if (this.writer == null)
{
this.writer = new SequentialWriter <object>(AvroContainer.CreateGenericWriter(this.avroSchema, output.BaseStream, Codec.Deflate), 24);
this.serializer = AvroSerializer.CreateGeneric(this.avroSchema);
}
dynamic data = new AvroRecord(this.serializer.WriterSchema);
data.station = row.Get <string>(row.Schema[0].Name);
data.datekey = row.Get <int?>(row.Schema[1].Name);
data.element = row.Get <string>(row.Schema[2].Name);
data.value = (double?)row.Get <decimal?>(row.Schema[3].Name);
data.mflag = row.Get <string>(row.Schema[4].Name);
data.qflag = row.Get <string>(row.Schema[5].Name);
data.sflag = row.Get <string>(row.Schema[6].Name);
data.timekey = row.Get <int?>(row.Schema[7].Name);
writer.Write(data);
}
// Output
//
// Outputs the names of the rowset columns in a column separated row and optionally adds their types in a second row.
//
public override void Output(IRow row, IUnstructuredWriter output)
{
if (_first_row_written)
{
return;
}
using (StreamWriter streamWriter = new StreamWriter(output.BaseStream, this._encoding))
{
streamWriter.NewLine = this._row_delim;
ISchema schema = row.Schema;
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
if (i > 0)
{
streamWriter.Write(this._col_delim);
}
var val = _quoting ? AddQuotes(col.Name) : col.Name;
streamWriter.Write(val);
}
streamWriter.WriteLine();
if (_with_types)
{
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
if (i > 0)
{
streamWriter.Write(this._col_delim);
}
var val = _quoting ? AddQuotes(col.Type.FullName) : col.Type.FullName;
streamWriter.Write(val);
}
streamWriter.WriteLine();
}
}
_first_row_written = true;
}
/// <summary>Output is called at least once per instance</summary>
/// <param name="input">A SQLIP row</param>
/// <param name="output">Wrapper for a Stream</param>
public override void Output(IRow input, IUnstructuredWriter output)
{
IColumn badColumn = input.Schema.FirstOrDefault(col => col.Type != typeof(string));
if (badColumn != null)
{
throw new ArgumentException(string.Format("Column '{0}' must be of type 'string', not '{1}'", badColumn.Name, badColumn.Type.Name));
}
using (var writer = XmlWriter.Create(output.BaseStream, this.fragmentSettings))
{
writer.WriteStartElement(this.rowPath);
foreach (IColumn col in input.Schema)
{
var value = input.Get <string>(col.Name);
if (value != null)
{
// Skip null values in order to distinguish them from empty strings
writer.WriteElementString(this.columnPaths[col.Name] ?? col.Name, value);
}
}
}
}
public override void Output(IRow row, IUnstructuredWriter output)
{
var streamWriter = new StreamWriter(output.BaseStream);
// Metadata schema initialization to enumerate column names
var schema = row.Schema;
if (this.row_count == 0)
{
streamWriter.Write("|");
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
streamWriter.Write(" ");
streamWriter.Write(col.Name);
streamWriter.Write(" ");
streamWriter.Write("|");
}
streamWriter.Write("\r\n");
streamWriter.Flush();
streamWriter.Write("|");
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
streamWriter.Write(" ");
streamWriter.Write("---");
streamWriter.Write(" ");
streamWriter.Write("|");
}
streamWriter.Write("\r\n");
streamWriter.Flush();
}
// Data row output
streamWriter.Write("|");
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
string val = "";
try
{
var coltype = col.Type;
if (coltype == typeof(string))
{
val = row.Get <string>(col.Name).ToString();
val = val ?? "NULL";
}
else if (coltype == typeof(float))
{
val = row.Get <float>(col.Name).ToString();
}
else if (coltype == typeof(double))
{
val = row.Get <double>(col.Name).ToString();
}
else if (coltype == typeof(long))
{
val = row.Get <long>(col.Name).ToString();
}
else if (coltype == typeof(Guid))
{
val = row.Get <Guid>(col.Name).ToString();
}
else if (coltype == typeof(int?))
{
val = row.Get <int?> (col.Name).ToString();
val = val ?? "NULL";
}
else if (coltype == typeof(long?))
{
val = row.Get <long?>(col.Name).ToString();
val = val ?? "NULL";
}
else if (coltype == typeof(float?))
{
val = row.Get <float?>(col.Name).ToString();
val = val ?? "NULL";
}
else if (coltype == typeof(double?))
{
val = row.Get <double?> (col.Name).ToString();
val = val ?? "NULL";
}
else
{
val = "UNKNOWNTYPE";
}
}
catch (System.NullReferenceException)
{
// Handling NULL values--keeping them empty
val = "NULL";
}
streamWriter.Write(" ");
streamWriter.Write(val);
streamWriter.Write(" ");
streamWriter.Write("|");
}
streamWriter.Write("\n");
streamWriter.Flush();
this.row_count++;
}
public override void Output(IRow input, IUnstructuredWriter output)
{
}
public override void Output(IRow input, IUnstructuredWriter output)
{
_builder.Add(input, output.BaseStream);
}
public override void Output(IRow row, IUnstructuredWriter output)
{
var buffer = Encoding.UTF8.GetBytes(row.Get <string>("col1"));
output.BaseStream.Write(buffer, 0, buffer.Length);
}
public override void Output(IRow row, IUnstructuredWriter output)
{
var streamWriter = new StreamWriter(output.BaseStream);
// Metadata schema initialization to enumerate column names
var schema = row.Schema;
if (this.row_count == 0)
{
streamWriter.Write("|");
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
streamWriter.Write(" ");
streamWriter.Write(col.Name);
streamWriter.Write(" ");
streamWriter.Write("|");
}
streamWriter.Write("\r\n");
streamWriter.Flush();
streamWriter.Write("|");
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
streamWriter.Write(" ");
streamWriter.Write("---");
streamWriter.Write(" ");
streamWriter.Write("|");
}
streamWriter.Write("\r\n");
streamWriter.Flush();
}
// Data row output
streamWriter.Write("|");
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
string val = "";
try
{
// Data type enumeration--required to match the distinct list of types from OUTPUT statement
switch (col.Type.Name.ToString().ToLower())
{
case "string":
val = row.Get <string>(col.Name).ToString();
break;
case "int":
val = row.Get <int>(col.Name).ToString();
break;
case "float":
val = row.Get <float>(col.Name).ToString();
break;
case "double":
val = row.Get <double>(col.Name).ToString();
break;
case "guid":
val = row.Get <Guid>(col.Name).ToString();
break;
default: break;
}
}
catch (System.NullReferenceException)
{
// Handling NULL values--keeping them empty
val = "NULL";
}
streamWriter.Write(" ");
streamWriter.Write(val);
streamWriter.Write(" ");
streamWriter.Write("|");
}
streamWriter.Write("\n");
streamWriter.Flush();
this.row_count++;
}
// void Output(IRow row, IUnstructuredWriter output)
//
// Actual implementation of DriverOutputter that overwrites the Output method of IOutputter.
public override void Output(IRow row, IUnstructuredWriter output)
{
using (StreamWriter streamWriter = new StreamWriter(output.BaseStream, this._encoding))
{
streamWriter.NewLine = this._row_delim;
ISchema schema = row.Schema;
for (int i = 0; i < schema.Count; i++)
{
object val = row.Get<object>(i);
if (i > 0)
{
streamWriter.Write(this._col_delim);
}
this.WriteValue(val, streamWriter);
}
streamWriter.WriteLine();
}
}