Esempio n. 1
0
        private static async Task <Func <MemoryStream, bool, int> > GetDataChunkUploader(this IFileMedia fileMedia, IFileConfiguration fileConfig, Interfaces.ILog log)
        {
            var getContext = await fileMedia.GetWaveContextFunc(fileConfig.Name, log);

            var xmdJson = fileConfig.GetMetadataBuilder();
            var chunkNo = 0;

            return((stream, isFinalizing) =>
            {
                var context = getContext();
                if (string.IsNullOrWhiteSpace(context.SetId))
                {
                    context.InitiateDatasetUpload(xmdJson(), fileMedia.Operation, log).Wait();
                    if (string.IsNullOrWhiteSpace(context.SetId))
                    {
                        throw new ImporterException(
                            Localization.GetLocalizationString("Could not get job id from wave, cannot upload chunks"));
                    }
                }
                stream.Flush();
                var encCSVchunk = Convert.ToBase64String(stream.ToArray());
                var payload =
                    $"{{\"InsightsExternalDataId\":\"{context.SetId}\",\"PartNumber\":{++chunkNo},\"DataFile\":\"{encCSVchunk}\"}}";
                var tryCount = 0;
                while (true)
                {
                    try
                    {
                        var client = new HttpClient();
                        var content = new StringContent(payload, Encoding.ASCII);
                        content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
                        var url = $"{context.EntryPoint}/services/data/v41.0/sobjects/InsightsExternalDataPart";
                        client.AddAuthorization(context);
                        log.Debug($"Uploading chunk #{chunkNo}");
                        var response = client.PostAsync(url, content).Result;
                        if (response.IsSuccessStatusCode)
                        {
                            log.Debug($"Uploaded chunk #{chunkNo}");
                            if (isFinalizing)
                            {
                                context.FinalizeDatasetUpload(log).Wait();
                            }
                            return chunkNo;
                        }
                    }
                    catch (Exception ex)
                    {
                        log.Error(Localization.GetLocalizationString("Error while uploading chunk #{0} - {1}", chunkNo, ex.Message));
                        log.Debug(ex.ToString());
                    }
                    if (++tryCount > 4)
                    {
                        throw new ImporterUploadException(Localization.GetLocalizationString("Failed to upload dataset."));
                    }
                    log.Debug(Localization.GetLocalizationString("Retrying to upload chunk#{0}", chunkNo));
                }
            });
        }
Esempio n. 2
0
        public static Func <IDataRow> ParseData(this Func <ISourceRow> getLineFunc, IFileConfiguration fileConfig, Interfaces.ILog logger)
        {
            if (fileConfig == null)
            {
                var msg = Localization.GetLocalizationString("Could not get Source Configuration...");
                logger?.Fatal(msg);
                throw new ArgumentException(msg);
            }
            logger?.Info(string.Format(Localization.GetLocalizationString("Parsing data from {0}"), fileConfig.Name));
            var parsers       = fileConfig.GetRowParsers();
            var currentRecord = (long)0;
            var parsedRecords = (long)0;

            return(() =>
            {
                var line = getLineFunc();
                if (line != null)
                {
                    currentRecord++;
                    try
                    {
                        var row = parsers(line, currentRecord, currentRecord);
                        if (row != null)
                        {
                            parsedRecords++;
                            return row;
                        }
                    }
                    catch (Exception ex)
                    {
                        logger?.Error(Localization.GetLocalizationString("Failed to parse line: \"{0}\"", string.Join(",", line.Fields.Select(x => x.Source))));
                        throw ex;
                    }


                    return new DataRow(
                        new Dictionary <string, IValue>
                    {
                        {
                            "raw",
                            new ValueWrapper <string>(string.Join(",", line.Fields.Select(x => x.Source)),
                                                      Localization.GetLocalizationString("Parse error"), true, string.Join(",", line.Fields.Select(x => x.Source)))
                        }
                    },
                        Localization.GetLocalizationString("Could not parse line."), currentRecord, line.LineNumber, line.Context.SourcePath, line.Context.FileConfiguration.Name);
                }

                logger?.Info(string.Format(Localization.GetLocalizationString("Parsed {0}/{1} records from {2}"), parsedRecords, currentRecord, fileConfig.Name));
                return null;
            });
        }
Esempio n. 3
0
        private static Stream GetLocalStream(this IFileMedia media, Interfaces.ILog log)
        {
            if (File.Exists(media.Path) && media.Operation != DataOperation.Append)
            {
                try
                {
                    File.Delete(media.Path);
                }
                catch (Exception ex)
                {
                    log.Error(ex.ToString());
                    return(null);
                }
            }

            return(File.Open(media.Path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.Read));
        }
Esempio n. 4
0
        public static Func <IDataRow, long> GetCsvWriter(this StreamWriter stream, IFile fileConfig, Interfaces.ILog log)
        {
            var  nullValue          = string.IsNullOrWhiteSpace(fileConfig.NullValue) ? "" : fileConfig.NullValue;
            var  delimiter          = string.IsNullOrWhiteSpace(fileConfig.Delimiter)?DEFAULT_DELIMITER[0]:fileConfig.Delimiter[0];
            var  filters            = fileConfig.Rows.Select(x => x.PrepareTargetFilter()).ToList();
            var  rowCount           = (long)0;
            var  getQualifiedString = fileConfig.GetQualifiedStrFunc(log);
            bool isHeadersWritten   = false;

            return(row =>
            {
                if (!isHeadersWritten && fileConfig.HasHeaders)
                {
                    for (var r = 0; r < fileConfig.Rows.Count; r++)
                    {
                        for (var c = 0; c < fileConfig.Rows[r].Columns.Count; c++)
                        {
                            if (c > 0)
                            {
                                stream.Write(delimiter);
                            }
                            stream.Write(fileConfig.Rows[r].Columns[c].Alias ?? fileConfig.Rows[r].Columns[c].Name);
                        }
                        stream.WriteLine();
                    }
                    isHeadersWritten = true;
                }
                if (!string.IsNullOrWhiteSpace(row.Error))
                {
                    log.Error(Localization.GetLocalizationString("Error in file {0}, line {1}", row.SourcePath, row.RawLineNumber) + " - " +
                              (row.Error ?? "") + "\n\r\t" +
                              string.Join(",", row.Columns.Values.Select(x => x.Source)));
                    return rowCount;
                }
                for (var r = 0; r < fileConfig.Rows.Count; r++)
                {
                    if (!filters[r](row))
                    {
                        continue;
                    }
                    var columns = fileConfig.Rows[r].Columns;
                    var rowString = new StringBuilder();
                    for (var c = 0; c < columns.Count; c++)
                    {
                        var column = columns[c];
                        var value = row[column.Alias ?? column.Name];
                        if (value == null)
                        {
                            //log.Warning(Localization.GetLocalizationString("Could not find column {0}", columns[c].Name));
                        }

                        var valueString = (value != null && !value.IsNull) ? getQualifiedString(value.ToString(column.Format), column.Type) : nullValue;
                        rowString.Append(valueString);
                        if (columns.Count - 1 > c)
                        {
                            rowString.Append(delimiter);
                        }
                    }
                    stream.WriteLine(rowString.ToString());
                    rowCount++;
                }

                return rowCount;
            });
        }