Пример #1
0
        private static Action <IDataRow> GetWriter(this IFile file, Interfaces.ILog log)
        {
            var stream   = file.GetWriterStreams(log);
            var rowCount = (long)0;

            if (stream != null)
            {
                switch (file.Format)
                {
                case FileFormat.CSV:
                    var writerStream = stream.GetCsvWriter(file, log);
                    return(row =>
                    {
                        if (row == null)
                        {
                            log?.Info(string.Format(
                                          Localization.GetLocalizationString("{0} line(s) written to {1}"),
                                          rowCount, file.Name));
                            stream.Flush();
                            return;
                        }

                        if (row.SourceName == file.Name)
                        {
                            rowCount = writerStream(row);
                        }
                    });
                }
            }
            return(row => { });
        }
Пример #2
0
        public static Func <IDataRow> GetRecordGroupingFunc(this Func <IDataRow> getLineFunc, IFileConfiguration fileConfig,
                                                            Interfaces.ILog logger)
        {
            var getRow = getLineFunc.GroupRowsFunc(fileConfig.Rows[0], logger);

            return(() => getRow());
        }
Пример #3
0
        private static StreamWriter GetWriterStreams(this IFile file, Interfaces.ILog log)
        {
            var streams = new List <Stream>();

            if (file.MultipleMedia?.Count > 0)
            {
                streams = file.MultipleMedia.Where(x => !x.Disabled).Select(x => x.GetWriterStream(file, log)).ToList();
            }
            else if (!file.Media.Disabled)
            {
                streams.Add(file.Media.GetWriterStream(file, log));
            }
            return(new StreamWriter(new WriteStreamProxy(streams), Encoding.UTF8));
        }
Пример #4
0
        private static Stream GetLocalStream(this IFileMedia media, Interfaces.ILog log)
        {
            if (File.Exists(media.Path) && media.Operation != DataOperation.Append)
            {
                try
                {
                    File.Delete(media.Path);
                }
                catch (Exception ex)
                {
                    log.Error(ex.ToString());
                    return(null);
                }
            }

            return(File.Open(media.Path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read));
        }
Пример #5
0
        public static Func <int> BufferedRead(this ISourceFileContext r, Interfaces.ILog logger)
        {
            if (r?.Stream == null)
            {
                throw new ImporterArgumentOutOfRangeException(Localization.GetLocalizationString("Source stream cannot be null"));
            }
            var buff          = new char[60];
            var position      = 0;
            var length        = 0;
            var rd            = r.Stream;
            var readChars     = (long)0;
            var resultMessage = new Action(() =>
            {
                if (readChars > 0)
                {
                    logger?.Debug($"Loaded {readChars} characters");
                }
            });

            return(() =>
            {
                if (position >= length)
                {
                    if (rd == null || rd.EndOfStream)
                    {
                        resultMessage();
                        readChars = 0;
                        return -1;
                    }
                    length = rd.ReadBlock(buff, 0, 5);
                    readChars += length;
                    position = 0;
                    if (length == 0)
                    {
                        return -1;
                    }
                }

                return buff[position++];
            });
        }
Пример #6
0
        private static async Task FinalizeDatasetUpload(this WaveContext context, Interfaces.ILog log)
        {
            var client = new HttpClient();
            var url    = $"{context.EntryPoint}/services/data/v36.0/sobjects/InsightsExternalData/{context.SetId}";

            log.Info(Localization.GetLocalizationString("Finalizing upload job {0}", context.SetId));
            var content = new StringContent("{\"Action\" : \"Process\"}", Encoding.ASCII);

            content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
            client.AddAuthorization(context);
            var response = await client.PatchAsync(url, content);

            if (response.IsSuccessStatusCode)
            {
                log.Info(Localization.GetLocalizationString("Successfully uploaded {0}", context.Alias));
                return;
            }
            var responseText = await response.Content.ReadAsStringAsync();

            throw new ImporterUploadException($"{response.StatusCode} -{responseText}");
        }
Пример #7
0
        private static async Task InitiateDatasetUpload(this WaveContext context, string xmdJson, DataOperation operation, Interfaces.ILog log)
        {
            var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalData";

            log.Info(Localization.GetLocalizationString("Initializing data upload"));
            var client  = new HttpClient();
            var encJson = Convert.ToBase64String(Encoding.UTF8.GetBytes(xmdJson));
            var payload =
                $"{{\"Format\":\"csv\",\"EdgemartAlias\":\"{context.Alias}\",\"Operation\":\"{operation.ToString()}\",\"Action\":\"None\",\"MetadataJson\":\"{encJson}\"}}";
            var content = new StringContent(payload, Encoding.ASCII);

            content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
            client.AddAuthorization(context);
            var response = await client.PostAsync(url, content);

            var responseType = new { id = "" };
            var responseText = await response.Content.ReadAsStringAsync();

            log.Debug($"Received {responseText}");
            if (response.IsSuccessStatusCode)
            {
                context.SetId = JsonConvert.DeserializeAnonymousType(responseText, responseType).id;
                log.Info(Localization.GetLocalizationString("Received job id {0}", context.SetId));
            }
        }
Пример #8
0
 public void AddLogger(Interfaces.ILog logger)
 {
     loggers.Add(logger);
 }
Пример #9
0
        public static Func <ISourceRow> CsvReader(this Func <int> readNext, ISourceFileContext context, Interfaces.ILog logger)
        {
            var fileConfig = context.FileConfiguration;

            if (fileConfig == null)
            {
                var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
                logger?.Fatal(msg);
                throw new ArgumentException(msg);
            }

            var nullValue = string.IsNullOrWhiteSpace(fileConfig.NullValue) ? "" : fileConfig.NullValue;
            var delimiter = string.IsNullOrWhiteSpace(fileConfig.Delimiter) ? ',' : fileConfig.Delimiter[0];
            var qualifier = string.IsNullOrWhiteSpace(fileConfig.Qualifier) ? '\"' : fileConfig.Qualifier[0];

            var locker   = new object();
            var rowCount = 0;

            logger?.Info(string.Format(Localization.GetLocalizationString("Loading data from {0}..."), fileConfig.Name));
            return(() =>
            {
                lock (locker)
                {
                    var columns = new List <ISourceField>();
                    var hadQualifier = false;
                    Action <StringBuilder> enqueue = clmn =>
                    {
                        var value = clmn.ToString();
                        columns.Add((value.Length > 0 && (value != nullValue || hadQualifier)) ? new SourceField(value) : new SourceField(null));
                    };
                    int c;
                    while ((c = readNext()) >= 0 && (c == '\n' || c == '\r'))
                    {
                        ;
                    }
                    if (c < 0)
                    {
                        logger?.Info(Localization.GetLocalizationString("Loaded {0} line(s) from {1}.", rowCount, fileConfig.Name));
                        return null;
                    }
                    var isQualified = false;
                    var column = new StringBuilder();
                    while (c >= 0)
                    {
                        if (c == qualifier && (column.Length == 0 || hadQualifier))
                        {
                            hadQualifier = true;
                            isQualified = !isQualified;
                            if (isQualified && column.Length > 0)
                            {
                                column.Append((char)c);
                            }
                        }
                        else
                        if (c == delimiter && !isQualified)
                        {
                            enqueue(column);
                            column = new StringBuilder();
                        }
                        else
                        if ((c == '\n' || c == '\r') && !isQualified)
                        {
                            enqueue(column);
                            column = null;
                            break;
                        }
                        else
                        {
                            column.Append((char)c);
                        }
                        c = readNext();
                    }
                    if (c == -1 && column != null)
                    {
                        enqueue(column);
                    }

                    rowCount++;
                    return new SourceRow {
                        Context = context, Fields = columns, LineNumber = rowCount
                    };
                }
            });
        }
Пример #10
0
        private static async Task <Func <MemoryStream, bool, int> > GetDataSenderFunc(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
        {
            var chunkNumber  = 0;
            var queue        = new ConcurrentQueue <MemoryStream>();
            var isFinalizing = false;
            var uploader     = await fileMedia.GetDataChunkUploader(fileConfig, log);

            var senderTask = new Task(() =>
            {
                while (true)
                {
                    while (queue.TryDequeue(out var nextChunk))
                    {
                        try
                        {
                            chunkNumber = uploader(nextChunk, isFinalizing && queue.IsEmpty);
                        }
                        catch
                        {
                            log.Fatal(Localization.GetLocalizationString("Upload has terminated"));
                            return;
                        }
                    }
                    if (isFinalizing && queue.IsEmpty)
                    {
                        return;
                    }
                    Thread.Sleep(100);
                }
            });

            senderTask.Start();
            return((stream, finalize) =>
            {
                if (stream != null)
                {
                    queue.Enqueue(stream);
                    while (queue.Count > 3)
                    {
                        Thread.Sleep(50);
                    }
                }
                isFinalizing = finalize;
                if (isFinalizing)
                {
                    log.Info(Localization.GetLocalizationString("Awaiting for upload to finish."));
                    senderTask.Wait();
                }

                return chunkNumber;
            });
        }
Пример #11
0
        private static async Task <Func <MemoryStream, bool, int> > GetDataChunkUploader(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
        {
            var getContext = await fileMedia.GetWaveContextFunc(fileConfig.Name, log);

            var xmdJson = fileConfig.GetMetadataBuilder();
            var chunkNo = 0;

            return((stream, isFinalizing) =>
            {
                var context = getContext();
                if (string.IsNullOrWhiteSpace(context.SetId))
                {
                    context.InitiateDatasetUpload(xmdJson(), fileMedia.Operation, log).Wait();
                    if (string.IsNullOrWhiteSpace(context.SetId))
                    {
                        throw new ImporterException(
                            Localization.GetLocalizationString("Could not get job id from wave, cannot upload chunks"));
                    }
                }
                stream.Flush();
                var encCSVchunk = Convert.ToBase64String(stream.ToArray());
                var payload =
                    $"{{\"InsightsExternalDataId\":\"{context.SetId}\",\"PartNumber\":{++chunkNo},\"DataFile\":\"{encCSVchunk}\"}}";
                var tryCount = 0;
                while (true)
                {
                    try
                    {
                        var client = new HttpClient();
                        var content = new StringContent(payload, Encoding.ASCII);
                        content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
                        var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalDataPart";
                        client.AddAuthorization(context);
                        log.Debug($"Uploading chunk #{chunkNo}");
                        var response = client.PostAsync(url, content).Result;
                        if (response.IsSuccessStatusCode)
                        {
                            log.Debug($"Uploaded chunk #{chunkNo}");
                            if (isFinalizing)
                            {
                                context.FinalizeDatasetUpload(log).Wait();
                            }
                            return chunkNo;
                        }
                    }
                    catch (Exception ex)
                    {
                        log.Error(Localization.GetLocalizationString("Error while uploading chunk #{0} - {1}", chunkNo, ex.Message));
                        log.Debug(ex.ToString());
                    }
                    if (++tryCount > 4)
                    {
                        throw new ImporterUploadException(Localization.GetLocalizationString("Failed to upload dataset."));
                    }
                    log.Debug(Localization.GetLocalizationString("Retrying to upload chunk#{0}", chunkNo));
                }
            });
        }
Пример #12
0
        public static Func <IDataRow> ParseData(this Func <ISourceRow> getLineFunc, IFileConfiguration fileConfig, Interfaces.ILog logger)
        {
            if (fileConfig == null)
            {
                var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
                logger?.Fatal(msg);
                throw new ArgumentException(msg);
            }
            logger?.Info(string.Format(Localization.GetLocalizationString("Parsing data from {0}"), fileConfig.Name));
            var parsers       = fileConfig.GetRowParsers();
            var currentRecord = (long)0;
            var parsedRecords = (long)0;

            return(() =>
            {
                var line = getLineFunc();
                if (line != null)
                {
                    currentRecord++;
                    try
                    {
                        var row = parsers(line, currentRecord, currentRecord);
                        if (row != null)
                        {
                            parsedRecords++;
                            return row;
                        }
                    }
                    catch (Exception ex)
                    {
                        logger?.Error(Localization.GetLocalizationString("Failed to parse line: \"{0}\"", string.Join(",", line.Fields.Select(x => x.Source))));
                        throw ex;
                    }


                    return new DataRow(
                        new Dictionary <string, IValue>
                    {
                        {
                            "raw",
                            new ValueWrapper <string>(string.Join(",", line.Fields.Select(x => x.Source)),
                                                      Localization.GetLocalizationString("Parse error"), true, string.Join(",", line.Fields.Select(x => x.Source)))
                        }
                    },
                        Localization.GetLocalizationString("Could not parse line."), currentRecord, line.LineNumber, line.Context.SourcePath, line.Context.FileConfiguration.Name);
                }

                logger?.Info(string.Format(Localization.GetLocalizationString("Parsed {0}/{1} records from {2}"), parsedRecords, currentRecord, fileConfig.Name));
                return null;
            });
        }
Пример #13
0
        private static Func <ISourceFileContext, Func <IDataRow> > GetStreamReader(this IFileConfiguration fileConfig, Interfaces.ILog logger)
        {
            switch (fileConfig.Format)
            {
            case FileFormat.Fixed:
                return(context => context?.BufferedRead(logger).FixedWidthReader(context, logger).ParseData(context.FileConfiguration, logger));

            case FileFormat.CSV:
                return(context => context?.BufferedRead(logger).CsvReader(context, logger).ParseData(context.FileConfiguration, logger));

            default:
                return(null);
            }
        }
Пример #14
0
        private static StreamReader GetLocalFileStream(this string filePath, Encoding encoding, Interfaces.ILog logger)
        {
            if (string.IsNullOrWhiteSpace(filePath))
            {
                return(null);
            }

            logger.Debug(string.Format(Localization.GetLocalizationString("Opening source file \"{0}\""),
                                       filePath));
            return(new StreamReader(File.Open(filePath, FileMode.Open, FileAccess.Read, FileShare.Read), encoding));
        }
Пример #15
0
        private static Func <ISourceFileContext> GetSourceStream(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog logger)
        {
            logger?.Debug($"Getting source stream(s) for media \"{fileMedia.MediaType.ToString()}\" - \"{fileMedia.Path}\"");
            switch (fileMedia.MediaType)
            {
            case MediaType.Local:
                var directoryName = Path.GetDirectoryName(Path.GetFullPath(fileMedia.Path));
                logger?.Debug($"Searching for local file(s) \"{Path.GetFullPath(fileMedia.Path)}\"");
                var files = Directory.EnumerateFiles(directoryName, Path.GetFileName(fileMedia.Path),
                                                     fileMedia.IncludeSubfolders ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).OrderBy(x => x).ToList();
                logger?.Debug($"Found {files.Count} file(s) matching the pattern.");
                foreach (var file in files)
                {
                    logger?.Debug(file);
                }

                var getFile = files.GetNextFunc();

                return(() =>
                {
                    var file = getFile();
                    return new SourceFileContext
                    {
                        SourcePath = file,
                        Stream = file.GetLocalFileStream(fileMedia.Encoding.GetFilEncoding(), logger),
                        FileMedia = fileMedia,
                        FileConfiguration = fileConfig
                    };
                });
            }
            return(null);
        }
Пример #16
0
 public static Stream GetWaveStreamWriter(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
 {
     return(new WaveStream(fileMedia, fileConfig, log));
 }
Пример #17
0
        private static Stream GetWriterStream(this IFileMedia media, IFileConfiguration fileConfig, Interfaces.ILog log)
        {
            switch (media.MediaType)
            {
            case MediaType.Local:
                return(media.GetLocalStream(log));

            case MediaType.Wave:
                return(media.GetWaveStreamWriter(fileConfig, log));
            }

            return(null);
        }
Пример #18
0
        private static async Task <Func <WaveContext> > GetWaveContextFunc(this IFileMedia fileMedia, string fileName, Interfaces.ILog log)
        {
            var client  = new HttpClient();
            var url     = $"{fileMedia.ConnectionCredentials.EntryPoint}/services/oauth2/token?grant_type=password&client_id={fileMedia.ConnectionCredentials.ClientId}&client_secret={fileMedia.ConnectionCredentials.ClientSecret}&username={fileMedia.ConnectionCredentials.Login}&password={fileMedia.ConnectionCredentials.Password}{fileMedia.ConnectionCredentials.Token}";
            var content = new StringContent(string.Empty);

            content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
            log.Info(string.Format(
                         Localization.GetLocalizationString("Obtaining access token and entry point from \"{0}\" for \"{1}\""),
                         fileMedia.ConnectionCredentials.EntryPoint, fileMedia.ConnectionCredentials.Login));
            var response = await client.PostAsync(url, content);

            WaveContext context = null;

            if (response.IsSuccessStatusCode)
            {
                log.Info(Localization.GetLocalizationString("Retrieved entry point and access token."));
                var responseType = new { access_token = "", instance_url = "" };
                var result       = JsonConvert.DeserializeAnonymousType(await response.Content.ReadAsStringAsync(), responseType);
                context = new WaveContext
                {
                    Token      = result.access_token,
                    EntryPoint = result.instance_url,
                    Alias      = fileName.Replace(" ", "_")
                };
            }
            else
            {
                log.Fatal($"{response.StatusCode.ToString()}-{response.Content.ReadAsStringAsync()}");
                throw new UnauthorizedAccessException(Localization.GetLocalizationString("Could not get access to wave entry point."));
            }
            return(() => context);
        }
Пример #19
0
        private static IList <Action <IDataRow> > ConfigureWriter(this Func <IFile> file, Interfaces.ILog log)
        {
            var   writers = new List <Action <IDataRow> >();
            IFile fileConfig;

            while ((fileConfig = file()) != null)
            {
                if (!fileConfig.Disabled)
                {
                    var writer = fileConfig.GetWriter(log).GroupRecords(fileConfig, log);
                    if (writer != null)
                    {
                        writers.Add(writer);
                    }
                }
            }

            return(writers);
        }
Пример #20
0
        public static Func <IDataRow, long> GetCsvWriter(this StreamWriter stream, IFile fileConfig, Interfaces.ILog log)
        {
            var  nullValue          = string.IsNullOrWhiteSpace(fileConfig.NullValue) ? "" : fileConfig.NullValue;
            var  delimiter          = string.IsNullOrWhiteSpace(fileConfig.Delimiter)?DEFAULT_DELIMITER[0]:fileConfig.Delimiter[0];
            var  filters            = fileConfig.Rows.Select(x => x.PrepareTargetFilter()).ToList();
            var  rowCount           = (long)0;
            var  getQualifiedString = fileConfig.GetQualifiedStrFunc(log);
            bool isHeadersWritten   = false;

            return(row =>
            {
                if (!isHeadersWritten && fileConfig.HasHeaders)
                {
                    for (var r = 0; r < fileConfig.Rows.Count; r++)
                    {
                        for (var c = 0; c < fileConfig.Rows[r].Columns.Count; c++)
                        {
                            if (c > 0)
                            {
                                stream.Write(delimiter);
                            }
                            stream.Write(fileConfig.Rows[r].Columns[c].Alias ?? fileConfig.Rows[r].Columns[c].Name);
                        }
                        stream.WriteLine();
                    }
                    isHeadersWritten = true;
                }
                if (!string.IsNullOrWhiteSpace(row.Error))
                {
                    log.Error(Localization.GetLocalizationString("Error in file {0}, line {1}", row.SourcePath, row.RawLineNumber) + " - " +
                              (row.Error ?? "") + "\n\r\t" +
                              string.Join(",", row.Columns.Values.Select(x => x.Source)));
                    return rowCount;
                }
                for (var r = 0; r < fileConfig.Rows.Count; r++)
                {
                    if (!filters[r](row))
                    {
                        continue;
                    }
                    var columns = fileConfig.Rows[r].Columns;
                    var rowString = new StringBuilder();
                    for (var c = 0; c < columns.Count; c++)
                    {
                        var column = columns[c];
                        var value = row[column.Alias ?? column.Name];
                        if (value == null)
                        {
                            //log.Warning(Localization.GetLocalizationString("Could not find column {0}", columns[c].Name));
                        }

                        var valueString = (value != null && !value.IsNull) ? getQualifiedString(value.ToString(column.Format), column.Type) : nullValue;
                        rowString.Append(valueString);
                        if (columns.Count - 1 > c)
                        {
                            rowString.Append(delimiter);
                        }
                    }
                    stream.WriteLine(rowString.ToString());
                    rowCount++;
                }

                return rowCount;
            });
        }
Пример #21
0
        public static Func <ISourceRow> FixedWidthReader(this Func <int> readNext, ISourceFileContext context, Interfaces.ILog logger)
        {
            if (context?.FileConfiguration == null)
            {
                var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
                logger?.Fatal(msg);
                throw new ArgumentException(msg);
            }
            var locker   = new object();
            var rowCount = 0;

            logger?.Info(string.Format(Localization.GetLocalizationString("Loading data from {0}..."), context.FileConfiguration.Name));
            var lineReaders = context.FileConfiguration.Rows.Select(x => x.GetLineFunc(context.FileConfiguration)).ToList();

            return(() =>
            {
                lock (locker)
                {
                    var result = new List <ISourceField>();
                    foreach (var reader in lineReaders)
                    {
                        result.AddRange(reader(readNext));
                    }
                    return result.Count > 0 ? new SourceRow {
                        Context = context, Fields = result, LineNumber = ++rowCount
                    } : null;
                }
            });
        }