예제 #1
0
        protected override void ExecuteImpl()
        {
            if (!File.Exists(FileName))
            {
                Context.Log(LogSeverity.Debug, this, "can't delete file because it doesn't exist '{FileName}'", PathHelpers.GetFriendlyPathName(FileName));
                return;
            }

            Context.Log(LogSeverity.Information, this, "deleting file '{FileName}'", PathHelpers.GetFriendlyPathName(FileName));

            try
            {
                File.Delete(FileName);
                Context.Log(LogSeverity.Debug, this, "successfully deleted file '{FileName}' in {Elapsed}", PathHelpers.GetFriendlyPathName(FileName),
                            InvocationInfo.LastInvocationStarted.Elapsed);
            }
            catch (Exception ex)
            {
                var exception = new ProcessExecutionException(this, "file deletion failed", ex);
                exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "file deletion failed, file name: {0}, message: {1}",
                                                      FileName, ex.Message));
                exception.Data.Add("FileName", FileName);
                throw exception;
            }
        }
예제 #2
0
        protected override void ExecuteImpl()
        {
            using (var clt = new HttpClient())
            {
                var iocUid = 0;
                try
                {
                    using (Context.CancellationTokenSource.Token.Register(clt.CancelPendingRequests))
                    {
                        iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.httpGet, Url, null, null, null, null,
                                                                "downloading file from {Url} to {FileName}",
                                                                Url, PathHelpers.GetFriendlyPathName(FileName));

                        using (var response = clt.GetStreamAsync(Url).Result)
                            using (var fs = new FileStream(FileName, FileMode.Create))
                            {
                                response.CopyTo(fs);
                            }

                        Context.RegisterIoCommandSuccess(this, IoCommandKind.httpGet, iocUid, Convert.ToInt32(new FileInfo(FileName).Length));
                    }
                }
                catch (Exception ex)
                {
                    Context.RegisterIoCommandFailed(this, IoCommandKind.httpGet, iocUid, null, ex);

                    var exception = new HttpException(this, "file download failed", Url, ex);
                    exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "file download failed, url: {0}, file name: {1}, message: {2}",
                                                          Url, FileName, ex.Message));
                    exception.Data.Add("Url", Url);
                    exception.Data.Add("FileName", FileName);
                    throw exception;
                }
            }
        }
예제 #3
0
        protected override void ExecuteImpl()
        {
            using (var clt = new WebClient())
            {
                var iocUid = 0;
                try
                {
                    using (Context.CancellationTokenSource.Token.Register(clt.CancelAsync))
                    {
                        iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.httpGet, Url, null, null, null, null,
                                                                "downloading file from {Url} to {FileName}",
                                                                Url, PathHelpers.GetFriendlyPathName(FileName));

                        clt.DownloadFile(Url, FileName);

                        Context.RegisterIoCommandSuccess(this, IoCommandKind.httpGet, iocUid, Convert.ToInt32(new FileInfo(FileName).Length));
                    }
                }
                catch (Exception ex)
                {
                    Context.RegisterIoCommandFailed(this, IoCommandKind.httpGet, iocUid, null, ex);

                    var exception = new ProcessExecutionException(this, "file download failed", ex);
                    exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "file download failed, url: {0}, file name: {1}, message: {2}",
                                                          Url, FileName, ex.Message));
                    exception.Data.Add("Url", Url);
                    exception.Data.Add("FileName", FileName);
                    throw exception;
                }
            }
        }
예제 #4
0
        protected override IEnumerable <IRow> Produce()
        {
            var iocUid = Context.RegisterIoCommandStart(this, IoCommandKind.fileRead, FileName, null, null, null, null,
                                                        "reading from {FileName}",
                                                        PathHelpers.GetFriendlyPathName(FileName));

            if (!File.Exists(FileName))
            {
                var exception = new ProcessExecutionException(this, "input file doesn't exist");
                exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "input file doesn't exist: {0}",
                                                      FileName));

                exception.Data.Add("FileName", FileName);

                Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, 0, exception);
                throw exception;
            }

            var columnConfig = ColumnConfiguration?.ToDictionary(x => x.SourceColumn.ToUpperInvariant(), StringComparer.OrdinalIgnoreCase);
            var resultCount  = 0;

            Stream       stream;
            StreamReader reader;

            try
            {
                stream = new FileStream(FileName, FileMode.Open, FileAccess.Read, FileShare.Read);
                reader = new StreamReader(stream);
            }
            catch (Exception ex)
            {
                Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, null, ex);

                var exception = new EtlException(this, "error while opening file", ex);
                exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while opening file: {0}, message: {1}", FileName, ex.Message));
                exception.Data.Add("FileName", FileName);
                throw exception;
            }

            var firstRow      = true;
            var initialValues = new List <KeyValuePair <string, object> >();

            var partList = new List <string>(100);
            var builder  = new StringBuilder(2000);

            // capture for performance
            var columnNames                    = ColumnNames;
            var delimiter                      = Delimiter;
            var treatEmptyStringAsNull         = TreatEmptyStringAsNull;
            var removeSurroundingDoubleQuotes  = RemoveSurroundingDoubleQuotes;
            var throwOnMissingDoubleQuoteClose = ThrowOnMissingDoubleQuoteClose;
            var ignoreColumns                  = IgnoreColumns?.ToHashSet();

            try
            {
                while (!Context.CancellationTokenSource.IsCancellationRequested)
                {
                    string line;
                    try
                    {
                        line = reader.ReadLine();
                        if (line == null)
                        {
                            break;
                        }

                        if (string.IsNullOrEmpty(line))
                        {
                            continue;
                        }
                    }
                    catch (Exception ex)
                    {
                        Context.RegisterIoCommandFailed(this, IoCommandKind.fileRead, iocUid, resultCount, ex);
                        var exception = new EtlException(this, "error while reading data from file", ex);
                        exception.Data.Add("FileName", FileName);
                        exception.AddOpsMessage(string.Format(CultureInfo.InvariantCulture, "error while reading data from file: {0}, message: {1}", FileName, ex.Message));
                        throw exception;
                    }

                    if (line.EndsWith(delimiter))
                    {
                        line = line[0..^ 1];