Beispiel #1
0
        public override void Output(IRow input, IUnstructuredWriter output)
        {
            if (this.recordCol == -1)
            {
                // Map column names to indices once to avoid mapping overhead for each row.
                this.recordCol         = input.Schema.IndexOf("record");
                this.offsetCol         = input.Schema.IndexOf("offset");
                this.encodingErrorsCol = input.Schema.IndexOf("encodingErrors");
            }

            var record         = input.Get <byte[]>(recordCol);
            var encodingErrors = input.Get <bool>(encodingErrorsCol);

            if (encodingErrors && this.replaceInvalidBytes)
            {
                // Decode the record to substitute invalid bytes with the replacement character.
                // Encode the sanitized record again before writing it to the output file.
                record = this.nonThrowingEncoding.GetBytes(this.nonThrowingEncoding.GetString(record));
            }

            if (this.offsetFormat != null)
            {
                output.BaseStream.Write(
                    this.nonThrowingEncoding.GetBytes(string.Format(this.offsetFormat, input.Get <long>(this.offsetCol))));

                var bom = this.nonThrowingEncoding.GetPreamble();
                if (record.Take(bom.Length).SequenceEqual(bom))
                {
                    // This should happen once per file. Prefer code simplicity to performance.
                    record = record.Skip(bom.Length).ToArray();
                }
            }

            output.BaseStream.Write(record);
        }
Beispiel #2
0
        public override void Output(IRow input, IUnstructuredWriter output)
        {
            ISchema schema = input.Schema;

            if (_ds == null)
            {
                List <SchemaElement> lse = new List <SchemaElement>();
                for (int i = 0; i < schema.Count(); i++)
                {
                    var col = schema[i];
                    lse.Add(new SchemaElement(col.Name, Type.GetType(getColType(col))));
                }
                _ds = new DataSet(new Schema(lse));

                _tempStream   = new MemoryStream();
                _resultStream = output.BaseStream;

                _writer = new ParquetWriter(output.BaseStream, null, writeroptions);

                //create DS based on schema
                //input.Schema
            }

            List <object> ls = new List <object>();

            for (int i = 0; i < schema.Count; i++)
            {
                ls.Add(input.Get <dynamic>(input.Schema[i].Name));
            }
            Row r = new Row(ls);

            _ds.Add(r);
        }
Beispiel #3
0
        public override void Output(IRow input, IUnstructuredWriter output)
        {
            if (_fileWriter == null)
            {
                _avSchema = Schema.Parse(_avroSchema) as RecordSchema;
                var writer = new GenericDatumWriter <GenericRecord>(_avSchema);
                _fileWriter = DataFileWriter <GenericRecord> .OpenWriter(writer, output.BaseStream);
            }

            var record = new GenericRecord(_avSchema);

            foreach (var x in input.Schema)
            {
                var obj = input.Get <object>(x.Name);

                if (obj != null)
                {
                    var objType = obj.GetType();
                    if (objType.IsGenericType && objType.GetGenericTypeDefinition() == typeof(SqlArray <>))
                    {
                        obj = ((System.Collections.IEnumerable)obj).Cast <object>().ToArray();
                    }
                }

                record.Add(x.Name, obj);
            }

            _fileWriter.Append(record);
        }
Beispiel #4
0
 // Output
 //
 // Outputs the names of the rowset columns in a column separated row and optionally adds their types in a second row.
 //
 public override void Output(IRow row, IUnstructuredWriter output)
 {
     if (_first_row_written) { return; }
     using (StreamWriter streamWriter = new StreamWriter(output.BaseStream, this._encoding))
     {
         streamWriter.NewLine = this._row_delim;
         ISchema schema = row.Schema;
         for (int i = 0; i < schema.Count(); i++)
         {
             var col = schema[i];
             if (i > 0)
             {
                 streamWriter.Write(this._col_delim);
             }
             var val = _quoting ? AddQuotes(col.Name) : col.Name;
             streamWriter.Write(val);
         }
         streamWriter.WriteLine();
         if (_with_types)
         {
             for (int i = 0; i < schema.Count(); i++)
             {
                 var col = schema[i];
                 if (i > 0)
                 {
                     streamWriter.Write(this._col_delim);
                 }
                 var val = _quoting ? AddQuotes(col.Type.FullName) : col.Type.FullName;
                 streamWriter.Write(val);
             }
             streamWriter.WriteLine();
         }
     }
     _first_row_written = true;
 }
Beispiel #5
0
 public override void Output(IRow input, IUnstructuredWriter output)
 {
     using (var streamWriter = new StreamWriter(output.BaseStream))
     {
         var value = input.Get <int>("value");
         streamWriter.WriteLine(value);
     }
 }
Beispiel #6
0
 public override void Output(IRow input, IUnstructuredWriter output)
 {
     var obj = input.Get<object>(0);
     byte[] imageArray = (byte[])obj;
     using (MemoryStream ms = new MemoryStream(imageArray))
     {
         var image = Image.FromStream(ms);
         image.Save(output.BaseStream, ImageFormat.Jpeg);
     }
 }
Beispiel #7
0
        public override void Output(IRow input, IUnstructuredWriter output)
        {
            var obj = input.Get <object>(0);

            byte[] imageArray = (byte[])obj;
            using (MemoryStream ms = new MemoryStream(imageArray))
            {
                var image = Image.FromStream(ms);
                image.Save(output.BaseStream, ImageFormat.Jpeg);
            }
        }
Beispiel #8
0
        public override void Output(IRow row, IUnstructuredWriter output)
        {
            ISchema schema = row.Schema;

            for (int i = 0; i < schema.Count; i++)
            {
                object obj = row.Get <object>(i);
                if (obj is byte[])
                {
                    output.BaseStream.Write((byte[])obj, 0, ((byte[])obj).Length);
                }
            }
        }
        public override void Output(IRow input, IUnstructuredWriter output)
        {
            var file = input.Get <byte[]>(0);

            _outputStream = output.BaseStream;

            var entry = _archive.CreateEntry(string.Format(_filePattern, _fileCount++),
                                             CompressionLevel.NoCompression);

            using (var ms = new MemoryStream(file))
                using (var entryStream = entry.Open())
                {
                    ms.CopyTo(entryStream);
                }
        }
Beispiel #10
0
        /// <summary/>
        public override void                    Output(IRow row, IUnstructuredWriter output)
        {
            // First Row
            if(this.writer == null)
            {
                // Json.Net (writer)
                this.writer = new JsonTextWriter(new StreamWriter(output.BaseStream));
                
                // Header (array)
                this.writer.WriteStartArray();
            }

            // Row(s)
            WriteRow(row, this.writer);
        }
Beispiel #11
0
        /// <summary/>
        public override void                    Output(IRow row, IUnstructuredWriter output)
        {
            // First Row
            if (this.writer == null)
            {
                // Json.Net (writer)
                this.writer = new JsonTextWriter(new StreamWriter(output.BaseStream));

                // Header (array)
                this.writer.WriteStartArray();
            }

            // Row(s)
            WriteRow(row, this.writer);
        }
Beispiel #12
0
        public override void Output(IRow input, IUnstructuredWriter output)
        {
            // If this is the first row, then get the stream output set up by the U-SQL runtime
            if (this.outputWriter == null)
            {
                this.outputWriter = output.BaseStream;
            }

            // Get the schema of the row
            var columnSchema = input.Schema;

            // Iterate through the columns in the row and convert the data to XML encoded strings
            StringBuilder rowData = new StringBuilder($@"<{xmlDocType}>");

            foreach (var column in columnSchema)
            {
                rowData.Append($@"<{column.Name}>");

                // This outputter currently only recognizes int, double, string, and DateTime data.
                if (column.Type == typeof(int))
                {
                    rowData.Append($@"{input.Get<int>(column.Name)}");
                }
                if (column.Type == typeof(double))
                {
                    rowData.Append($@"{input.Get<double>(column.Name)}");
                }
                if (column.Type == typeof(string))
                {
                    rowData.Append($@"{input.Get<string>(column.Name)}");
                }
                if (column.Type == typeof(DateTime))
                {
                    rowData.Append($@"{input.Get<DateTime>(column.Name)}");
                }

                rowData.Append($@"</{column.Name}>");
            }
            rowData.Append($@"</{xmlDocType}>");
            rowData.Append(Environment.NewLine);

            // Send the XML encoded string to the output stream
            string data = rowData.ToString();

            this.outputWriter.Write(Encoding.UTF8.GetBytes(data), 0, data.Length);
        }
Beispiel #13
0
 // void Output(IRow row, IUnstructuredWriter output)
 //
 // Actual implementation of DriverOutputter that overwrites the Output method of IOutputter.
 public override void Output(IRow row, IUnstructuredWriter output)
 {
     using (StreamWriter streamWriter = new StreamWriter(output.BaseStream, this._encoding))
     {
         streamWriter.NewLine = this._row_delim;
         ISchema schema = row.Schema;
         for (int i = 0; i < schema.Count; i++)
         {
             object val = row.Get <object>(i);
             if (i > 0)
             {
                 streamWriter.Write(this._col_delim);
             }
             this.WriteValue(val, streamWriter);
         }
         streamWriter.WriteLine();
     }
 }
Beispiel #14
0
        public override void Output(IRow row, IUnstructuredWriter output)
        {
            // First Row
            if (this.writer == null)
            {
                //this.writer = new SequentialWriter<object>(AvroContainer.Create<AnonType_WeatherDataSet>(this.avroSchema, output.BaseStream, Codec.Deflate), 24);
                this.serializer = AvroSerializer.Create <AnonType_WeatherDataSet>();
            }

            AnonType_WeatherDataSet data = new AnonType_WeatherDataSet(
                row.Get <string>(row.Schema[0].Name),
                row.Get <int?>(row.Schema[1].Name),
                row.Get <string>(row.Schema[2].Name),
                (double?)row.Get <decimal?>(row.Schema[3].Name),
                row.Get <string>(row.Schema[4].Name),
                row.Get <string>(row.Schema[5].Name),
                row.Get <string>(row.Schema[6].Name),
                row.Get <int?>(row.Schema[7].Name)
                );

            serializer.Serialize(output.BaseStream, data);
        }
Beispiel #15
0
        /// <summary>Output is called at least once per instance</summary>
        /// <param name="input">A SQLIP row</param>
        /// <param name="output">Wrapper for a Stream</param>
        public override void Output(IRow input, IUnstructuredWriter output)
        {
            IColumn badColumn = input.Schema.FirstOrDefault(col => col.Type != typeof(string));
            if (badColumn != null)
            {
                throw new ArgumentException(string.Format("Column '{0}' must be of type 'string', not '{1}'", badColumn.Name, badColumn.Type.Name));
            }

            using (var writer = XmlWriter.Create(output.BaseStream, this.fragmentSettings))
            {
                writer.WriteStartElement(this.rowPath);
                foreach (IColumn col in input.Schema)
                {
                    var value = input.Get<string>(col.Name);
                    if (value != null)
                    {
                        // Skip null values in order to distinguish them from empty strings
                        writer.WriteElementString(this.columnPaths[col.Name] ?? col.Name, value);
                    }
                }
            }
        }
Beispiel #16
0
        public override void Output(IRow row, IUnstructuredWriter output)
        {
            // First Row
            if (this.writer == null)
            {
                this.writer     = new SequentialWriter <object>(AvroContainer.CreateGenericWriter(this.avroSchema, output.BaseStream, Codec.Deflate), 24);
                this.serializer = AvroSerializer.CreateGeneric(this.avroSchema);
            }

            dynamic data = new AvroRecord(this.serializer.WriterSchema);

            data.station = row.Get <string>(row.Schema[0].Name);
            data.datekey = row.Get <int?>(row.Schema[1].Name);
            data.element = row.Get <string>(row.Schema[2].Name);
            data.value   = (double?)row.Get <decimal?>(row.Schema[3].Name);
            data.mflag   = row.Get <string>(row.Schema[4].Name);
            data.qflag   = row.Get <string>(row.Schema[5].Name);
            data.sflag   = row.Get <string>(row.Schema[6].Name);
            data.timekey = row.Get <int?>(row.Schema[7].Name);

            writer.Write(data);
        }
Beispiel #17
0
 // Output
 //
 // Outputs the names of the rowset columns in a column separated row and optionally adds their types in a second row.
 //
 public override void Output(IRow row, IUnstructuredWriter output)
 {
     if (_first_row_written)
     {
         return;
     }
     using (StreamWriter streamWriter = new StreamWriter(output.BaseStream, this._encoding))
     {
         streamWriter.NewLine = this._row_delim;
         ISchema schema = row.Schema;
         for (int i = 0; i < schema.Count(); i++)
         {
             var col = schema[i];
             if (i > 0)
             {
                 streamWriter.Write(this._col_delim);
             }
             var val = _quoting ? AddQuotes(col.Name) : col.Name;
             streamWriter.Write(val);
         }
         streamWriter.WriteLine();
         if (_with_types)
         {
             for (int i = 0; i < schema.Count(); i++)
             {
                 var col = schema[i];
                 if (i > 0)
                 {
                     streamWriter.Write(this._col_delim);
                 }
                 var val = _quoting ? AddQuotes(col.Type.FullName) : col.Type.FullName;
                 streamWriter.Write(val);
             }
             streamWriter.WriteLine();
         }
     }
     _first_row_written = true;
 }
Beispiel #18
0
        /// <summary>Output is called at least once per instance</summary>
        /// <param name="input">A SQLIP row</param>
        /// <param name="output">Wrapper for a Stream</param>
        public override void Output(IRow input, IUnstructuredWriter output)
        {
            IColumn badColumn = input.Schema.FirstOrDefault(col => col.Type != typeof(string));

            if (badColumn != null)
            {
                throw new ArgumentException(string.Format("Column '{0}' must be of type 'string', not '{1}'", badColumn.Name, badColumn.Type.Name));
            }

            using (var writer = XmlWriter.Create(output.BaseStream, this.fragmentSettings))
            {
                writer.WriteStartElement(this.rowPath);
                foreach (IColumn col in input.Schema)
                {
                    var value = input.Get <string>(col.Name);
                    if (value != null)
                    {
                        // Skip null values in order to distinguish them from empty strings
                        writer.WriteElementString(this.columnPaths[col.Name] ?? col.Name, value);
                    }
                }
            }
        }
Beispiel #19
0
        public override void Output(IRow row, IUnstructuredWriter output)
        {
            var streamWriter = new StreamWriter(output.BaseStream);

            // Metadata schema initialization to enumerate column names
            var schema = row.Schema;


            if (this.row_count == 0)
            {
                streamWriter.Write("|");
                for (int i = 0; i < schema.Count(); i++)
                {
                    var col = schema[i];
                    streamWriter.Write(" ");
                    streamWriter.Write(col.Name);
                    streamWriter.Write(" ");
                    streamWriter.Write("|");
                }
                streamWriter.Write("\r\n");
                streamWriter.Flush();

                streamWriter.Write("|");
                for (int i = 0; i < schema.Count(); i++)
                {
                    var col = schema[i];
                    streamWriter.Write(" ");
                    streamWriter.Write("---");
                    streamWriter.Write(" ");
                    streamWriter.Write("|");
                }
                streamWriter.Write("\r\n");
                streamWriter.Flush();
            }

            // Data row output
            streamWriter.Write("|");
            for (int i = 0; i < schema.Count(); i++)
            {
                var    col = schema[i];
                string val = "";

                try
                {
                    var coltype = col.Type;
                    if (coltype == typeof(string))
                    {
                        val = row.Get <string>(col.Name).ToString();
                        val = val ?? "NULL";
                    }
                    else if (coltype == typeof(float))
                    {
                        val = row.Get <float>(col.Name).ToString();
                    }
                    else if (coltype == typeof(double))
                    {
                        val = row.Get <double>(col.Name).ToString();
                    }
                    else if (coltype == typeof(long))
                    {
                        val = row.Get <long>(col.Name).ToString();
                    }
                    else if (coltype == typeof(Guid))
                    {
                        val = row.Get <Guid>(col.Name).ToString();
                    }
                    else if (coltype == typeof(int?))
                    {
                        val = row.Get <int?> (col.Name).ToString();
                        val = val ?? "NULL";
                    }
                    else if (coltype == typeof(long?))
                    {
                        val = row.Get <long?>(col.Name).ToString();
                        val = val ?? "NULL";
                    }
                    else if (coltype == typeof(float?))
                    {
                        val = row.Get <float?>(col.Name).ToString();
                        val = val ?? "NULL";
                    }
                    else if (coltype == typeof(double?))
                    {
                        val = row.Get <double?> (col.Name).ToString();
                        val = val ?? "NULL";
                    }
                    else
                    {
                        val = "UNKNOWNTYPE";
                    }
                }
                catch (System.NullReferenceException)
                {
                    // Handling NULL values--keeping them empty
                    val = "NULL";
                }
                streamWriter.Write(" ");
                streamWriter.Write(val);
                streamWriter.Write(" ");
                streamWriter.Write("|");
            }
            streamWriter.Write("\n");
            streamWriter.Flush();

            this.row_count++;
        }
Beispiel #20
0
 public override void Output(IRow input, IUnstructuredWriter output)
 {
 }
Beispiel #21
0
 public override void Output(IRow input, IUnstructuredWriter output)
 {
     _builder.Add(input, output.BaseStream);
 }
        public override void Output(IRow row, IUnstructuredWriter output)
        {
            var buffer = Encoding.UTF8.GetBytes(row.Get <string>("col1"));

            output.BaseStream.Write(buffer, 0, buffer.Length);
        }
Beispiel #23
0
        public override void Output(IRow row, IUnstructuredWriter output)
        {
            var streamWriter = new StreamWriter(output.BaseStream);

            // Metadata schema initialization to enumerate column names
            var schema = row.Schema;


            if (this.row_count == 0)
            {
                streamWriter.Write("|");
                for (int i = 0; i < schema.Count(); i++)
                {
                    var col = schema[i];
                    streamWriter.Write(" ");
                    streamWriter.Write(col.Name);
                    streamWriter.Write(" ");
                    streamWriter.Write("|");
                }
                streamWriter.Write("\r\n");
                streamWriter.Flush();

                streamWriter.Write("|");
                for (int i = 0; i < schema.Count(); i++)
                {
                    var col = schema[i];
                    streamWriter.Write(" ");
                    streamWriter.Write("---");
                    streamWriter.Write(" ");
                    streamWriter.Write("|");
                }
                streamWriter.Write("\r\n");
                streamWriter.Flush();
            }

            // Data row output
            streamWriter.Write("|");
            for (int i = 0; i < schema.Count(); i++)
            {
                var    col = schema[i];
                string val = "";
                try
                {
                    // Data type enumeration--required to match the distinct list of types from OUTPUT statement
                    switch (col.Type.Name.ToString().ToLower())
                    {
                    case "string":
                        val = row.Get <string>(col.Name).ToString();
                        break;

                    case "int":
                        val = row.Get <int>(col.Name).ToString();
                        break;

                    case "float":
                        val = row.Get <float>(col.Name).ToString();
                        break;

                    case "double":
                        val = row.Get <double>(col.Name).ToString();
                        break;

                    case "guid":
                        val = row.Get <Guid>(col.Name).ToString();
                        break;

                    default: break;
                    }
                }
                catch (System.NullReferenceException)
                {
                    // Handling NULL values--keeping them empty
                    val = "NULL";
                }
                streamWriter.Write(" ");
                streamWriter.Write(val);
                streamWriter.Write(" ");
                streamWriter.Write("|");
            }
            streamWriter.Write("\n");
            streamWriter.Flush();

            this.row_count++;
        }
Beispiel #24
0
 // void Output(IRow row, IUnstructuredWriter output)
 //
 // Actual implementation of DriverOutputter that overwrites the Output method of IOutputter.
 public override void Output(IRow row, IUnstructuredWriter output)
 {
     using (StreamWriter streamWriter = new StreamWriter(output.BaseStream, this._encoding))
     {
         streamWriter.NewLine = this._row_delim;
         ISchema schema = row.Schema;
         for (int i = 0; i < schema.Count; i++)
         {
             object val = row.Get<object>(i);
             if (i > 0)
             {
                 streamWriter.Write(this._col_delim);
             }
             this.WriteValue(val, streamWriter);
         }
         streamWriter.WriteLine();
     }
 }