public override void Output(IRow input, IUnstructuredWriter output)
{
if (this.recordCol == -1)
{
// Map column names to indices once to avoid mapping overhead for each row.
this.recordCol = input.Schema.IndexOf("record");
this.offsetCol = input.Schema.IndexOf("offset");
this.encodingErrorsCol = input.Schema.IndexOf("encodingErrors");
}
var record = input.Get <byte[]>(recordCol);
var encodingErrors = input.Get <bool>(encodingErrorsCol);
if (encodingErrors && this.replaceInvalidBytes)
{
// Decode the record to substitute invalid bytes with the replacement character.
// Encode the sanitized record again before writing it to the output file.
record = this.nonThrowingEncoding.GetBytes(this.nonThrowingEncoding.GetString(record));
}
if (this.offsetFormat != null)
{
output.BaseStream.Write(
this.nonThrowingEncoding.GetBytes(string.Format(this.offsetFormat, input.Get <long>(this.offsetCol))));
var bom = this.nonThrowingEncoding.GetPreamble();
if (record.Take(bom.Length).SequenceEqual(bom))
{
// This should happen once per file. Prefer code simplicity to performance.
record = record.Skip(bom.Length).ToArray();
}
}
output.BaseStream.Write(record);
}
public override IRow Process(IRow input, IUpdatableRow output)
{
double lat = input.Get <double>(latColumn);
double lon = input.Get <double>(lonColumn);
GeoLocation loc = new GeoLocation {
Longitude = lon, Latitude = lat
};
var country = _service.FindCountry(loc);
var USstates = _service.FindUsaState(loc);
if (country != null && country.Name != null)
{
output.Set <string>("country", country.Name);
}
else
{
output.Set <string>("country", "");
}
if (USstates != null && USstates.Name != null)
{
output.Set <string>("USstates", USstates.Name);
}
else
{
output.Set <string>("USstates", "");
}
return(output.AsReadOnly());
}
public void TestMyProcessor()
{
//Schema: "a:int, b:int"
USqlColumn <int> col1 = new USqlColumn <int>("a");
USqlColumn <int> col2 = new USqlColumn <int>("b");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
//Generate one row with specified column values
object[] values = new object[2] {
2, 3
};
IRow input = new USqlRow(schema, values);
IUpdatableRow output = input.AsUpdatable();
//Create UDO instance
MyProcessor processor = new MyProcessor(floor: 4);
IRow newOutput = processor.Process(input, output);
//Verify results
Assert.IsTrue(newOutput.Schema.Count == 2);
Assert.IsTrue(newOutput.Get <int>(0) == 2);
Assert.IsTrue(newOutput.Get <int>(1) == 4);
}
public override IEnumerable <IRow> Apply(IRow input, IUpdatableRow output)
{
DateTime startTime = input.Get <DateTime>(startColumn);
DateTime endTime = input.Get <DateTime>(endColumn);
var startDate = startTime.Date;
var endDate = endTime.Date;
DateTime startTimeOfDay;
DateTime endTimeOfDay;
var lastSecond = new TimeSpan(23, 59, 59);
for (var dt = startDate; dt <= endDate; dt = dt.AddDays(1))
{
if (dt == startDate)
{
startTimeOfDay = startTime;
}
else
{
startTimeOfDay = dt;
}
if (dt == endDate)
{
endTimeOfDay = endTime;
}
else
{
endTimeOfDay = dt + lastSecond;
}
output.Set <DateTime>("startTimeOfDay", startTimeOfDay);
output.Set <DateTime>("endTimeOfDay", endTimeOfDay);
yield return(output.AsReadOnly());
}
}
public override IRow Process(IRow input, IUpdatableRow output)
{
var tag = input.Get<string>("Tag");
var category = input.Get<string>("Category");
category = "other";
foreach (var cat in categoryMapper)
{
var categoryName = cat.Key;
var listOfPrefixes = cat.Value;
var found = false;
foreach (var pref in listOfPrefixes)
{
if (tag.StartsWith(pref))
{
category = categoryName;
found = true;
break;
}
}
if (found)
{
break;
}
}
output.Set("Category", category);
return output.AsReadOnly();
}
public void TestMyProcessorWithCollection()
{
//Schema: "a:int, b:int"
USqlSchema schema = new USqlSchema(
new USqlColumn <int>("a"),
new USqlColumn <int>("b")
);
IUpdatableRow output = new USqlRow(schema, null).AsUpdatable();
//Generate Rowset with specified values
List <object[]> values = new List <object[]> {
new object[2] {
2, 3
},
new object[2] {
10, 20
}
};
IEnumerable <IRow> rows = UnitTestHelper.CreateRowsFromValues(schema, values);
IRowset rowset = UnitTestHelper.GetRowsetFromCollection(rows, output.AsReadOnly());
//Create UDO instance
MyProcessor processor = new MyProcessor(floor: 1, enforce: true);
foreach (IRow r in rowset.Rows)
{
IRow after = processor.Process(r, output);
//Verify result
Assert.IsTrue(after.Get <int>(0) == 2);
Assert.IsTrue(after.Get <int>(1) == 4);
break;
}
}
public void TestMyProcessor()
{
// Define the schema for processor input rowset
// Schema: "a:int, b:int"
//
USqlColumn <int> col1 = new USqlColumn <int>("col1");
USqlColumn <int> col2 = new USqlColumn <int>("col2");
List <IColumn> columns = new List <IColumn> {
col1, col2
};
USqlSchema schema = new USqlSchema(columns);
// Generate one row with specified column values as input rowset
//
object[] values = new object[2] {
0, 0
};
IRow input = new USqlRow(schema, values);
IUpdatableRow output = input.AsUpdatable();
// Create processor instance for testing and run the processor with fake input
//
MyProcessor processor = new MyProcessor();
IRow newOutput = processor.Process(input, output);
//Verify results for processor output
//
Assert.IsTrue(newOutput.Schema.Count == 2);
Assert.IsTrue(newOutput.Get <int>(0) == 1);
Assert.IsTrue(newOutput.Get <int>(1) == 5);
}
public override IRow Process(IRow input, IUpdatableRow output)
{
output.Set <int>("DepID", input.Get <int>("DepID"));
output.Set <string>("DepName", input.Get <string>("DepName"));
output.Set <string>("HelloWorld", hw);
return(output.AsReadOnly());
}
public override IRow Process(IRow input, IUpdatableRow output)
{
int a = input.Get <int>("col1");
int b = input.Get <int>("col2");
output.Set <int>("col1", a + 1);
output.Set <int>("col2", b + 5);
return(output.AsReadOnly());
}
// IRow Process(IRow input, IUpdatableRow output)
//
// Actual implementatoin of the user-defined processor. Overwrites the Process method of IProcessor.
public override IRow Process(IRow input, IUpdatableRow output)
{
string first_name = input.Get <string>("first_name");
string last_name = input.Get <string>("last_name");
string name = first_name.Substring(0, 1) + "." + last_name;
output.Set <string>("name", name);
output.Set <int>("id", Int32.Parse(input.Get <string>("id")));
output.Set <string>("zipcode", input.Get <string>("zipcode"));
output.Set <string>("country", input.Get <string>("country"));
return(output.AsReadOnly());
}
public override IRow Process(IRow input, IUpdatableRow output)
{
string UserID = input.Get <string>("UserID");
string Name = input.Get <string>("Name");
string Address = input.Get <string>("Address");
string City = input.Get <string>("City");
string State = input.Get <string>("State");
string PostalCode = input.Get <string>("PostalCode");
string Country = input.Get <string>("Country");
string Phone = input.Get <string>("Phone");
if (CountryTranslation.Keys.Contains(Country))
{
Country = CountryTranslation[Country];
}
output.Set <string>(0, UserID);
output.Set <string>(1, Name);
output.Set <string>(2, Address);
output.Set <string>(3, City);
output.Set <string>(4, State);
output.Set <string>(5, PostalCode);
output.Set <string>(6, Country);
output.Set <string>(7, Phone);
return(output.AsReadOnly());
}
public override void Output(IRow input, IUnstructuredWriter output)
{
if (_fileWriter == null)
{
_avSchema = Schema.Parse(_avroSchema) as RecordSchema;
var writer = new GenericDatumWriter <GenericRecord>(_avSchema);
_fileWriter = DataFileWriter <GenericRecord> .OpenWriter(writer, output.BaseStream);
}
var record = new GenericRecord(_avSchema);
foreach (var x in input.Schema)
{
var obj = input.Get <object>(x.Name);
if (obj != null)
{
var objType = obj.GetType();
if (objType.IsGenericType && objType.GetGenericTypeDefinition() == typeof(SqlArray <>))
{
obj = ((System.Collections.IEnumerable)obj).Cast <object>().ToArray();
}
}
record.Add(x.Name, obj);
}
_fileWriter.Append(record);
}
public override void Output(IRow input, IUnstructuredWriter output)
{
ISchema schema = input.Schema;
if (_ds == null)
{
List <SchemaElement> lse = new List <SchemaElement>();
for (int i = 0; i < schema.Count(); i++)
{
var col = schema[i];
lse.Add(new SchemaElement(col.Name, Type.GetType(getColType(col))));
}
_ds = new DataSet(new Schema(lse));
_tempStream = new MemoryStream();
_resultStream = output.BaseStream;
_writer = new ParquetWriter(output.BaseStream, null, writeroptions);
//create DS based on schema
//input.Schema
}
List <object> ls = new List <object>();
for (int i = 0; i < schema.Count; i++)
{
ls.Add(input.Get <dynamic>(input.Schema[i].Name));
}
Row r = new Row(ls);
_ds.Add(r);
}
/// <summary>Apply is called at least once per instance</summary>
/// <param name="input">A SQLIP row</param>
/// <param name="output">A SQLIP updatable row.</param>
/// <returns>IEnumerable of IRow, one IRow per SQLIP row.</returns>
/// <remarks>Because applier constructor arguments cannot depend on
/// column references, the name of the column to parse is given as a string. Then
/// the actual column value is obtained by calling IRow.Get. The rest of the code
/// is the same as XmlDomExtractor.</remarks>
public override IEnumerable <IRow> Apply(IRow input, IUpdatableRow output)
{
// Make sure that all requested columns are of type string
IColumn column = output.Schema.FirstOrDefault(col => col.Type != typeof(string));
if (column != null)
{
throw new ArgumentException(string.Format("Column '{0}' must be of type 'string', not '{1}'", column.Name, column.Type.Name));
}
XmlDocument xmlDocument = new XmlDocument();
xmlDocument.LoadXml(input.Get <string>(this.xmlColumnName));
foreach (XmlNode xmlNode in xmlDocument.DocumentElement.SelectNodes(this.rowPath))
{
// IUpdatableRow implements a builder pattern to save memory allocations,
// so call output.Set in a loop
foreach (IColumn col in output.Schema)
{
var explicitColumnMapping = this.columnPaths.FirstOrDefault(columnPath => columnPath.Value == col.Name);
XmlNode xml = xmlNode.SelectSingleNode(explicitColumnMapping.Key ?? col.Name);
output.Set(explicitColumnMapping.Value ?? col.Name, xml == null ? null : xml.InnerXml);
}
// then call output.AsReadOnly to build an immutable IRow.
yield return(output.AsReadOnly());
}
}
/// <summary/>
private static void WriteRow(IRow row, JsonTextWriter writer)
{
// Row
// => { c1:v1, c2:v2, ...}
// Header
writer.WriteStartObject();
// Fields
var columns = row.Schema;
for (int i = 0; i < columns.Count; i++)
{
// Note: We simply delegate to Json.Net for all data conversions
// For data conversions beyond what Json.Net supports, do an explicit projection:
// ie: SELECT datetime.ToString(...) AS datetime, ...
object value = row.Get <object>(i);
// Note: We don't bloat the JSON with sparse (null) properties
if (value != null)
{
writer.WritePropertyName(columns[i].Name, escape: true);
writer.WriteValue(value);
}
}
// Footer
writer.WriteEndObject();
}
public override IRow Process(IRow input, IUpdatableRow output)
{
var s = input.Get <string>("name");
output.Set <string>("reversed", Reverse(s));
return(output.AsReadOnly());
}
/// <summary>Apply is called at least once per instance</summary>
/// <param name="input">A SQLIP row</param>
/// <param name="output">A SQLIP updatable row.</param>
/// <returns>IEnumerable of IRow, one IRow per SQLIP row.</returns>
/// <remarks>Because applier constructor arguments cannot depend on
/// column references, the name of the column to parse is given as a string. Then
/// the actual column value is obtained by calling IRow.Get. The rest of the code
/// is the same as XmlDomExtractor.</remarks>
public override IEnumerable<IRow> Apply(IRow input, IUpdatableRow output)
{
// Make sure that all requested columns are of type string
IColumn column = output.Schema.FirstOrDefault(col => col.Type != typeof(string));
if (column != null)
{
throw new ArgumentException(string.Format("Column '{0}' must be of type 'string', not '{1}'", column.Name, column.Type.Name));
}
XmlDocument xmlDocument = new XmlDocument();
xmlDocument.LoadXml(input.Get<string>(this.xmlColumnName));
foreach (XmlNode xmlNode in xmlDocument.DocumentElement.SelectNodes(this.rowPath))
{
// IUpdatableRow implements a builder pattern to save memory allocations,
// so call output.Set in a loop
foreach(IColumn col in output.Schema)
{
var explicitColumnMapping = this.columnPaths.FirstOrDefault(columnPath => columnPath.Value == col.Name);
XmlNode xml = xmlNode.SelectSingleNode(explicitColumnMapping.Key ?? col.Name);
output.Set(explicitColumnMapping.Value ?? col.Name, xml == null ? null : xml.InnerXml);
}
// then call output.AsReadOnly to build an immutable IRow.
yield return output.AsReadOnly();
}
}
public override void Output(IRow input, IUnstructuredWriter output)
{
using (var streamWriter = new StreamWriter(output.BaseStream))
{
var value = input.Get <int>("value");
streamWriter.WriteLine(value);
}
}
public override IRow Process(IRow input, IUpdatableRow output)
{
var imgContent = input.Get <byte[]>("content");
var colorList = input.Get <SqlMap <int, string> >("colors");
using (var ms = new MemoryStream(imgContent))
{
var bMap = Image.FromStream(ms) as Bitmap;
var result = ColorExtractor.GetMostUsedColor(bMap, _topColorCount);
if (result.Any())
{
colorList = new SqlMap <int, string>(result);
}
}
output.Set("colors", colorList);
return(output.AsReadOnly());
}
public override void Output(IRow input, IUnstructuredWriter output)
{
// If this is the first row, then get the stream output set up by the U-SQL runtime
if (this.outputWriter == null)
{
this.outputWriter = output.BaseStream;
}
// Get the schema of the row
var columnSchema = input.Schema;
// Iterate through the columns in the row and convert the data to XML encoded strings
StringBuilder rowData = new StringBuilder($@"<{xmlDocType}>");
foreach (var column in columnSchema)
{
rowData.Append($@"<{column.Name}>");
// This outputter currently only recognizes int, double, string, and DateTime data.
if (column.Type == typeof(int))
{
rowData.Append($@"{input.Get<int>(column.Name)}");
}
if (column.Type == typeof(double))
{
rowData.Append($@"{input.Get<double>(column.Name)}");
}
if (column.Type == typeof(string))
{
rowData.Append($@"{input.Get<string>(column.Name)}");
}
if (column.Type == typeof(DateTime))
{
rowData.Append($@"{input.Get<DateTime>(column.Name)}");
}
rowData.Append($@"</{column.Name}>");
}
rowData.Append($@"</{xmlDocType}>");
rowData.Append(Environment.NewLine);
// Send the XML encoded string to the output stream
string data = rowData.ToString();
this.outputWriter.Write(Encoding.UTF8.GetBytes(data), 0, data.Length);
}
public override IRow Process(IRow input, IUpdatableRow output)
{
var a = input.Get <int>("a");
var b = input.Get <int>("b");
if (a + b > floor)
{
output.Set("a", a);
output.Set("b", b + 1);
}
else if (enforce)
{
throw new Exception("Passthrough wasn't enabled: " + (a + b));
}
return(output.AsReadOnly());
}
/// <summary>Apply is called at least once per instance</summary>
/// <param name="input">A SQLIP row</param>
/// <param name="output">A SQLIP updatable row.</param>
/// <returns>IEnumerable of IRow, one IRow per SQLIP row.</returns>
/// <remarks>Because applier constructor arguments cannot depend on
/// column references, the name of the column to parse is given as a string. Then
/// the actual column value is obtained by calling IRow.Get. The rest of the code
/// is the same as XmlDomExtractor.</remarks>
public override IEnumerable <IRow> Apply(IRow input, IUpdatableRow output)
{
// Make sure that all requested columns are of type string
IColumn column = output.Schema.FirstOrDefault(col => col.Type != typeof(string));
if (column != null)
{
throw new ArgumentException(string.Format("Column '{0}' must be of type 'string', not '{1}'", column.Name, column.Type.Name));
}
XmlDocument xmlDocument = new XmlDocument();
xmlDocument.LoadXml(input.Get <string>(this.xmlColumnName));
XmlNamespaceManager nsmanager = new XmlNamespaceManager(xmlDocument.NameTable);
// If namespace declarations have been provided, add them to the namespace manager
if (this.namespaceDecls != null)
{
foreach (var namespaceDecl in this.namespaceDecls)
{
nsmanager.AddNamespace(namespaceDecl.Key, namespaceDecl.Value);
}
}
foreach (XmlNode xmlNode in xmlDocument.DocumentElement.SelectNodes(this.rowPath, nsmanager))
{
// IUpdatableRow implements a builder pattern to save memory allocations,
// so call output.Set in a loop
foreach (IColumn col in output.Schema)
{
switch (col.Name)
{
// populate hiearchy columns with their XmlNode.Name
case "ElementName":
output.Set <string>("ElementName", xmlNode.Name);
break;
case "ChildName":
output.Set <string>("ChildName", xmlNode.FirstChild?.Name);
break;
case "GrandChildName":
output.Set <string>("GrandChildName", xmlNode.FirstChild?.FirstChild?.Name);
break;
// populate mapped columns with their XPath result from XmlNode.InnerXml
default:
var explicitColumnMapping = this.columnPaths.FirstOrDefault(columnPath => columnPath.Value == col.Name);
var xml = xmlNode.SelectSingleNode(explicitColumnMapping.Key ?? col.Name, nsmanager);
output.Set(explicitColumnMapping.Value ?? col.Name, xml?.InnerXml);
break;
}
}
// then call output.AsReadOnly to build an immutable IRow.
yield return(output.AsReadOnly());
}
}
public override IRow Process(IRow input, IUpdatableRow output)
{
string geohash7 = input.Get <string>("geohash7");
var coord = Geohash.Decode(geohash7);
output.Set <string>("geohash7", geohash7);
output.Set <double>("lat", coord[0]);
output.Set <double>("lon", coord[1]);
return(output.AsReadOnly());
}
public override void Output(IRow input, IUnstructuredWriter output)
{
var obj = input.Get<object>(0);
byte[] imageArray = (byte[])obj;
using (MemoryStream ms = new MemoryStream(imageArray))
{
var image = Image.FromStream(ms);
image.Save(output.BaseStream, ImageFormat.Jpeg);
}
}
public override IRow Process(IRow input, IUpdatableRow output)
{
var tags = this.tagger.ProduceTags(input.Get <byte[]>(this.imgColName));
var stringOfTags = string.Join(";", tags.Select(x => string.Format("{0}:{1}", x.Key, x.Value)));
output.Set <int>(this.numColName, tags.Count);
output.Set <string>(this.tagColName, stringOfTags);
return(output.AsReadOnly());
}
public override IRow Process(IRow input, IUpdatableRow output)
{
// Retrieve the data for the row from the input
string ticker = input.Get <string>("Ticker");
int price = input.Get <int>("Price");
DateTime quoteTime = input.Get <DateTime>("QuoteTime");
// Mark suspicious movements with an "X" in the Suspicious flag column
bool isSuspicious = SuspiciousMovement(ticker, price, quoteTime);
string suspiciousFlag = isSuspicious ? "X" : "";
// Create the output row, including the suspicious flag
output.Set <string>("Ticker", ticker);
output.Set <int>("Price", price);
output.Set <DateTime>("QuoteTime", quoteTime);
output.Set <string>("Suspicious", suspiciousFlag);
// The value returned should be a read-only copy of the output row
return(output.AsReadOnly());
}
public override void Output(IRow input, IUnstructuredWriter output)
{
var obj = input.Get <object>(0);
byte[] imageArray = (byte[])obj;
using (MemoryStream ms = new MemoryStream(imageArray))
{
var image = Image.FromStream(ms);
image.Save(output.BaseStream, ImageFormat.Jpeg);
}
}
// Generate a row that contains the opening price and % change in price for a stock item, identified by the ticker in the input row
public override IEnumerable <IRow> Apply(IRow input, IUpdatableRow output)
{
// Retrieve the ticker, price, and quote time from the input row
string ticker = input.Get <string>("Ticker");
int price = input.Get <int>("Price");
DateTime quoteTime = input.Get <DateTime>("QuoteTime");
// Find the opening price for this stock item
int openingPrice = getOpeningPrice(ticker, price, quoteTime);
// Calculate the % change
int priceDiff = price - openingPrice;
double percentChange = ((double)priceDiff / openingPrice) * 100;
// Generate and return the new row
output.Set <string>("Ticker", ticker);
output.Set <int>("OpeningPrice", openingPrice);
output.Set <double>("PercentChange", percentChange);
yield return(output.AsReadOnly());
}
// IRow Process(IRow input, IUpdatableRow output)
//
// Actual implementatoin of the user-defined processor. Overwrites the Process method of IProcessor.
public override IRow Process(IRow input, IUpdatableRow output)
{
string text = input.Get <string>("country");
if (EnglishCountryNames.CountryTranslation.Keys.Contains(text))
{
text = EnglishCountryNames.CountryTranslation[text];
}
output.Set <string>("country", text);
return(output.AsReadOnly());
}
public override IEnumerable <IRow> Apply(IRow input, IUpdatableRow output)
{
DateTime startTime = input.Get <DateTime>(startColumn);
DateTime endTime = input.Get <DateTime>(endColumn);
var startValueCol = (from x in input.Schema where x.Name == startValueColumn select x).First();
if (startValueCol.Type == typeof(bool))
{
var startValue = input.Get <bool>(startValueColumn);
return(locf <bool>(startTime, endTime, startValue, output));
}
else if (startValueCol.Type == typeof(int))
{
var startValue = input.Get <int>(startValueColumn);
return(locf <int>(startTime, endTime, startValue, output));
}
else if (startValueCol.Type == typeof(double))
{
var startValue = input.Get <double>(startValueColumn);
return(locf <double>(startTime, endTime, startValue, output));
}
else if (startValueCol.Type == typeof(string))
{
var startValue = input.Get <string>(startValueColumn);
return(locf <string>(startTime, endTime, startValue, output));
}
else
{
return(null);
}
}
public override IRow Process(IRow input, IUpdatableRow output)
{
string year = input.Get <string>("year");
string category = input.Get <string>("category");
string winner = input.Get <string>("winner");
string entity = input.Get <string>("entity");
string sentence = string.Empty;
var current = string.Empty;
if (Sentence.TryGetValue(entity, out current))
{
sentence = current;
}
output.Set <string>(0, year);
output.Set <string>(1, category);
output.Set <string>(2, winner);
output.Set <string>(3, entity);
output.Set <string>(4, sentence);
return(output.AsReadOnly());
}
public static dynamic ToExpando(this IRow row)
{
var obj = new ExpandoObject();
((IDictionary <string, object>)obj)["searchaction"] = "upload";
row.Schema.ToList().ForEach(x =>
{
((IDictionary <string, object>)obj)[x.Name] = row.Get <object>(x.Name);
});
return(obj);
}
public override IRow Process(IRow input, IUpdatableRow output)
{
var img = input.Get<byte[]>("image_data");
// load image only once into memory per row
using (StreamImage inImage = new StreamImage(img))
{
output.SetColumnIfExists("equipment_make", inImage.getStreamImageProperty(ImageProperties.equipment_make));
output.SetColumnIfExists("equipment_model", inImage.getStreamImageProperty(ImageProperties.equipment_model));
output.SetColumnIfExists("description", inImage.getStreamImageProperty(ImageProperties.description));
output.SetColumnIfExists("copyright", inImage.getStreamImageProperty(ImageProperties.copyright));
output.SetColumnIfExists("thumbnail", inImage.scaleStreamImageTo(150, 150));
}
return output.AsReadOnly();
}
// void Output(IRow row, IUnstructuredWriter output)
//
// Actual implementation of DriverOutputter that overwrites the Output method of IOutputter.
public override void Output(IRow row, IUnstructuredWriter output)
{
using (StreamWriter streamWriter = new StreamWriter(output.BaseStream, this._encoding))
{
streamWriter.NewLine = this._row_delim;
ISchema schema = row.Schema;
for (int i = 0; i < schema.Count; i++)
{
object val = row.Get<object>(i);
if (i > 0)
{
streamWriter.Write(this._col_delim);
}
this.WriteValue(val, streamWriter);
}
streamWriter.WriteLine();
}
}
// IRow Process(IRow input, IUpdatableRow output)
//
// Actual implementatoin of the user-defined processor. Overwrites the Process method of IProcessor.
public override IRow Process(IRow input, IUpdatableRow output)
{
List<string> list = new List<string>();
foreach (var current in input.Schema)
{
if (current.Type.IsGenericType && current.Type.GetGenericTypeDefinition() == typeof(SqlMap) && current.Type.GetGenericArguments()[0] == typeof(string))
{
list.Add(current.Name);
}
}
Dictionary<string, ArrayList> maps_to_be_changed = new Dictionary<string, ArrayList>();
foreach (var current2 in output.Schema)
{
bool flag = list.Contains(current2.Name);
if (-1 < input.Schema.IndexOf(current2.Name) && !flag)
{
output.Set<object>(current2.Name, input.Get<object>(current2.Name));
}
else if (!flag)
{
foreach (string current3 in list)
{
SqlMap<string, string> sqlMap = input.Get<SqlMap<string, string>>(current3);
SqlArray<string> sqlArray = null;
List<string> list2 = null;
if (sqlMap != null)
{
sqlArray = sqlMap.Keys;
if (sqlMap.Values != null)
{
list2 = sqlMap.Values.ToList<string>();
}
}
int num = (sqlArray == null) ? -1 : sqlArray.ToList<string>().IndexOf(current2.Name);
if (num != -1)
{
output.Set<string>(current2.Name, list2[num]);
if (maps_to_be_changed.Keys.Contains(current3))
{
maps_to_be_changed[current3].Add(current2.Name);
}
else
{
maps_to_be_changed.Add(current3, new ArrayList
{
current2.Name
});
}
break;
}
output.Set<object>(current2.Name, current2.Type.IsValueType ? Activator.CreateInstance(current2.Type) : null);
}
}
}
using (IEnumerator<IColumn> enumerator = output.Schema.GetEnumerator())
{
while (enumerator.MoveNext())
{
IColumn out_col = enumerator.Current;
bool flag = list.Contains(out_col.Name);
if (flag)
{
SqlMap<string, string> sqlMap = input.Get<SqlMap<string, string>>(out_col.Name);
if (maps_to_be_changed != null && maps_to_be_changed.Keys.Contains(out_col.Name))
{
sqlMap = new SqlMap<string, string>(
from kvp in sqlMap
where !maps_to_be_changed[out_col.Name].Contains(kvp.Key)
select kvp);
}
output.Set<SqlMap<string, string>>(out_col.Name, sqlMap);
}
}
}
return output.AsReadOnly();
}
/// <summary/>
private static void WriteRow(IRow row, JsonTextWriter writer)
{
// Row
// => { c1:v1, c2:v2, ...}
// Header
writer.WriteStartObject();
// Fields
var columns = row.Schema;
for(int i=0; i<columns.Count; i++)
{
// Note: We simply delegate to Json.Net for all data conversions
// For data conversions beyond what Json.Net supports, do an explicit projection:
// ie: SELECT datetime.ToString(...) AS datetime, ...
object value = row.Get<object>(i);
// Note: We don't bloat the JSON with sparse (null) properties
if(value != null)
{
writer.WritePropertyName(columns[i].Name, escape:true);
writer.WriteValue(value);
}
}
// Footer
writer.WriteEndObject();
}
// IRow Process(IRow input, IUpdatableRow output)
//
// Actual implementatoin of the user-defined processor. Overwrites the Process method of IProcessor.
public override IRow Process(IRow input, IUpdatableRow output)
{
string text = input.Get<string>("country");
if (EnglishCountryNames.CountryTranslation.Keys.Contains(text))
{
text = EnglishCountryNames.CountryTranslation[text];
}
output.Set<string>("country", text);
return output.AsReadOnly();
}
/// <summary>Output is called at least once per instance</summary>
/// <param name="input">A SQLIP row</param>
/// <param name="output">Wrapper for a Stream</param>
public override void Output(IRow input, IUnstructuredWriter output)
{
IColumn badColumn = input.Schema.FirstOrDefault(col => col.Type != typeof(string));
if (badColumn != null)
{
throw new ArgumentException(string.Format("Column '{0}' must be of type 'string', not '{1}'", badColumn.Name, badColumn.Type.Name));
}
using (var writer = XmlWriter.Create(output.BaseStream, this.fragmentSettings))
{
writer.WriteStartElement(this.rowPath);
foreach (IColumn col in input.Schema)
{
var value = input.Get<string>(col.Name);
if (value != null)
{
// Skip null values in order to distinguish them from empty strings
writer.WriteElementString(this.columnPaths[col.Name] ?? col.Name, value);
}
}
}
}
