コード例 #1
0
        private FileObjectCollection PopulateCollection <T>(FileObject fileObject) where T : IRecord
        {
            FileObjectCollection collection = new FileObjectCollection()
            {
                fileObject
            };

            _instance.TotalFilesFormatted++;
            _instance.TotalRecords += fileObject.RecordCount;

            if (Config.IsKustoConfigured())
            {
                // kusto native format is Csv
                // kusto json ingest is 2 to 3 times slower and does *not* use standard json format. uses json document per line no comma
                // using csv and compression for best performance
                collection = SerializeCsv <T>(fileObject);

                if (Config.KustoCompressed)
                {
                    collection.ForEach(x => x.Stream.Compress());
                }
            }
            else if (Config.IsLogAnalyticsConfigured())
            {
                // la is kusto based but only accepts non compressed json format ingest
                collection = SerializeJson <T>(fileObject);
            }

            collection.ForEach(x => SaveToCache(x));
            return(collection);
        }
コード例 #2
0
        public FileObjectCollection SerializeJson <T>(FileObject fileObject) where T : IRecord
        {
            Log.Debug("enter");
            string sourceFile = fileObject.FileUri.ToLower().TrimEnd(Constants.JsonExtension.ToCharArray());

            fileObject.FileUri = $"{sourceFile}{Constants.JsonExtension}";
            FileObjectCollection collection = new FileObjectCollection();
            string relativeUri = fileObject.RelativeUri.TrimEnd(Constants.JsonExtension.ToCharArray()) + Constants.JsonExtension;

            if (fileObject.Length > Constants.MaxJsonTransmitBytes)
            {
                FileObject newFileObject = new FileObject($"{sourceFile}", fileObject.BaseUri)
                {
                    Status = FileStatus.formatting
                };
                _instance.FileObjects.Add(newFileObject);
                int counter = 0;

                foreach (T record in fileObject.Stream.Read <T>())
                {
                    record.RelativeUri = relativeUri;
                    counter++;

                    if (newFileObject.Length < Constants.WarningJsonTransmitBytes)
                    {
                        newFileObject.Stream.Write <T>(new List <T>(1)
                        {
                            record
                        }, true);
                    }
                    else
                    {
                        collection.Add(newFileObject);
                        record.RelativeUri = relativeUri.TrimEnd(Constants.JsonExtension.ToCharArray()) + $".{counter}{Constants.JsonExtension}";
                        newFileObject      = new FileObject(record.RelativeUri, fileObject.BaseUri)
                        {
                            Status = FileStatus.formatting
                        };
                        _instance.FileObjects.Add(newFileObject);
                    }
                }

                _instance.FileObjects.Remove(fileObject);
                newFileObject.FileUri = $"{sourceFile}.{counter}{Constants.JsonExtension}";
                collection.Add(newFileObject);
            }
            else
            {
                collection.Add(fileObject);
            }

            Log.Debug($"json serialized size: {fileObject.Length} file: {fileObject.FileUri}");
            return(collection);
        }
コード例 #3
0
        private FileObjectCollection SerializeJson <T>(FileObject fileObject, IList <T> records)
        {
            Log.Debug("enter");
            FileObjectCollection collection = new FileObjectCollection()
            {
                fileObject
            };
            int counter = 0;

            string sourceFile = fileObject.FileUri.ToLower().Replace(JsonExtension, "");

            fileObject.FileUri = $"{sourceFile}{JsonExtension}";
            List <byte> jsonSerializedBytes = new List <byte>();

            byte[] leftBracket  = Encoding.UTF8.GetBytes("[");
            byte[] rightBracket = Encoding.UTF8.GetBytes("]");
            byte[] comma        = Encoding.UTF8.GetBytes(",");

            jsonSerializedBytes.AddRange(leftBracket);

            foreach (T record in records)
            {
                byte[] recordBytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(record, new JsonSerializerSettings()
                {
                }));

                if (jsonSerializedBytes.Count + recordBytes.Length + rightBracket.Length > MaxJsonTransmitBytes)
                {
                    jsonSerializedBytes.AddRange(rightBracket);
                    fileObject.Stream.Set(jsonSerializedBytes.ToArray());
                    fileObject.Length = fileObject.Stream.Get().Length;
                    jsonSerializedBytes.Clear();

                    fileObject = new FileObject($"{sourceFile}.{counter}{JsonExtension}", fileObject.BaseUri);
                    jsonSerializedBytes.AddRange(leftBracket);

                    Log.Debug($"json serialized size: {jsonSerializedBytes.Count} file: {fileObject.FileUri}");
                    collection.Add(fileObject);
                }

                jsonSerializedBytes.AddRange(recordBytes);
                jsonSerializedBytes.AddRange(comma);
                counter++;
            }

            jsonSerializedBytes.RemoveRange(jsonSerializedBytes.Count - comma.Length, comma.Length);
            jsonSerializedBytes.AddRange(rightBracket);
            fileObject.Stream.Set(jsonSerializedBytes.ToArray());
            fileObject.Length = fileObject.Stream.Get().Length;

            Log.Debug($"json serialized size: {jsonSerializedBytes.Count} file: {fileObject.FileUri}");
            return(collection);
        }
コード例 #4
0
        public FileObjectCollection SerializeCsv <T>(FileObject fileObject) where T : IRecord
        {
            Log.Debug("enter");
            FileObjectCollection collection = new FileObjectCollection()
            {
                fileObject
            };
            int counter = 0;

            string sourceFile = fileObject.FileUri.ToLower().TrimEnd(Constants.CsvExtension.ToCharArray());

            fileObject.FileUri = $"{sourceFile}{Constants.CsvExtension}";
            List <byte> csvSerializedBytes = new List <byte>();
            string      relativeUri        = fileObject.RelativeUri.TrimEnd(Constants.CsvExtension.ToCharArray()) + Constants.CsvExtension;

            foreach (T record in fileObject.Stream.Read <T>())
            {
                record.RelativeUri = relativeUri;
                byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString());

                if (csvSerializedBytes.Count + recordBytes.Length > Constants.MaxCsvTransmitBytes)
                {
                    record.RelativeUri = relativeUri.TrimEnd(Constants.CsvExtension.ToCharArray()) + $".{counter}{Constants.CsvExtension}";
                    recordBytes        = Encoding.UTF8.GetBytes(record.ToString());

                    fileObject.Stream.Set(csvSerializedBytes.ToArray());
                    csvSerializedBytes.Clear();

                    fileObject = new FileObject(record.RelativeUri, fileObject.BaseUri)
                    {
                        Status = FileStatus.formatting
                    };
                    _instance.FileObjects.Add(fileObject);

                    Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
                    collection.Add(fileObject);
                }

                csvSerializedBytes.AddRange(recordBytes);
                counter++;
            }

            fileObject.Stream.Set(csvSerializedBytes.ToArray());
            Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
            return(collection);
        }
コード例 #5
0
        private FileObjectCollection SerializeJson <T>(FileObject fileObject) where T : IRecord
        {
            Log.Debug("enter");
            string sourceFile = fileObject.FileUri.ToLower().TrimEnd(JsonExtension.ToCharArray());

            fileObject.FileUri = $"{sourceFile}{JsonExtension}";
            FileObjectCollection collection = new FileObjectCollection();
            string relativeUri = fileObject.RelativeUri.TrimEnd(JsonExtension.ToCharArray()) + JsonExtension;


            if (fileObject.Length > MaxJsonTransmitBytes)
            {
                FileObject newFileObject = new FileObject($"{sourceFile}", fileObject.BaseUri);
                int        counter       = 0;

                foreach (T record in fileObject.Stream.Read <T>())
                {
                    record.RelativeUri = relativeUri;
                    counter++;

                    if (newFileObject.Length < WarningJsonTransmitBytes)
                    {
                        newFileObject.Stream.Write <T>(new List <T> {
                            record
                        }, true);
                    }
                    else
                    {
                        collection.Add(newFileObject);
                        record.RelativeUri = relativeUri.TrimEnd(JsonExtension.ToCharArray()) + $".{counter}{JsonExtension}";
                        newFileObject      = new FileObject(record.RelativeUri, fileObject.BaseUri);
                    }
                }

                newFileObject.FileUri = $"{sourceFile}.{counter}{JsonExtension}";
                collection.Add(newFileObject);
            }
            else
            {
                collection.Add(fileObject);
            }

            Log.Debug($"json serialized size: {fileObject.Length} file: {fileObject.FileUri}");
            return(collection);
        }
コード例 #6
0
        private FileObjectCollection SerializeCsv <T>(FileObject fileObject, IList <T> records)
        {
            Log.Debug("enter");
            FileObjectCollection collection = new FileObjectCollection()
            {
                fileObject
            };
            int counter = 0;

            string sourceFile = fileObject.FileUri.ToLower().Replace(CsvExtension, "");

            fileObject.FileUri = $"{sourceFile}{CsvExtension}";
            List <byte> csvSerializedBytes = new List <byte>();

            foreach (T record in records)
            {
                byte[] recordBytes = Encoding.UTF8.GetBytes(record.ToString());

                if (csvSerializedBytes.Count + recordBytes.Length > MaxCsvTransmitBytes)
                {
                    fileObject.Stream.Set(csvSerializedBytes.ToArray());
                    fileObject.Length = fileObject.Stream.Get().Length;
                    csvSerializedBytes.Clear();

                    fileObject = new FileObject($"{sourceFile}.{counter}{CsvExtension}", fileObject.BaseUri);

                    Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
                    collection.Add(fileObject);
                }

                csvSerializedBytes.AddRange(recordBytes);
                counter++;
            }

            fileObject.Stream.Set(csvSerializedBytes.ToArray());
            fileObject.Length = fileObject.Stream.Get().Length;

            Log.Debug($"csv serialized size: {csvSerializedBytes.Count} file: {fileObject.FileUri}");
            return(collection);
        }