Exemple #1
0
        /// <summary>
        /// This is the command line wrapper for the "BlobCat" utility. You can use it to concatenate a list of block blobs into a single block blob
        /// This scenario is commonly required in the Big Data space where there are typically multiple output files from ETL pipelines or queries
        /// and those files need to be "joined" together to make one large file.
        /// </summary>
        static int Main(string[] args)
        {
            var parseResult = CommandLine.Parser.Default.ParseArguments <ConcatBlobOptions, FilesToBlobOptions>(args)
                              .MapResult(
                (ConcatBlobOptions opts) => {
                return(BlobCatEngine.BlobToBlob(
                           opts.SourceAccountName,
                           opts.SourceContainer,
                           opts.SourceKey,
                           opts.SourceFilePrefix,
                           opts.SortFilenames,
                           opts.SourceFiles.ToList(),
                           opts.DestAccountName,
                           opts.DestKey,
                           opts.DestContainer,
                           opts.DestFilename) ? 0 : 1);
            },
                (FilesToBlobOptions opts) =>
            {
                return(BlobCatEngine.DiskToBlob(
                           opts.SourceFolder,
                           opts.SourceFilePrefix,
                           opts.SortFilenames,
                           opts.SourceFiles.ToList(),
                           opts.DestAccountName,
                           opts.DestKey,
                           opts.DestContainer,
                           opts.DestFilename
                           ) ? 0 : 1);
            },
                errs => 1);

            return(parseResult);
        }
Exemple #2
0
        /// <summary>
        /// This is the command line wrapper for the "BlobCat" utility. You can use it to concatenate a list of block blobs into a single block blob
        /// This scenario is commonly required in the Big Data space where there are typically multiple output files from ETL pipelines or queries
        /// and those files need to be "joined" together to make one large file.
        /// </summary>
        static int Main(string[] args)
        {
            int retVal = 0;

            // create a logger instance
            var logFilePath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location),
                                           "log_{Date}.txt");

            ProgressBar pbar = null;

            using (var loggerFactory = new LoggerFactory().AddFile(logFilePath, args.Contains("--Debug") ? LogLevel.Debug : LogLevel.Information))
            {
                var myLogger = loggerFactory.CreateLogger("BlobCatCmd");

                var parseResult = Parser.Default.ParseArguments <ConcatBlobOptions, FilesToBlobOptions>(args);

                var progress = new Progress <OpProgress>(opProgress =>
                {
                    if (pbar is null)
                    {
                        if (opProgress.TotalTicks > 0)
                        {
                            if (!(Console.IsInputRedirected || Console.IsOutputRedirected))
                            {
                                pbar = new ProgressBar(opProgress.TotalTicks, "Starting operation", new ProgressBarOptions
                                {
                                    ProgressCharacter   = '.',
                                    ProgressBarOnBottom = true,
                                    // EnableTaskBarProgress = true,
                                    DisplayTimeInRealTime = true
                                });
                            }
                        }
                    }

                    if (pbar != null)
                    {
                        pbar.Tick(opProgress.StatusMessage);
                    }
                });

                retVal = parseResult.MapResult(
                    (ConcatBlobOptions opts) =>
                {
                    return(BlobCatEngine.BlobToBlob(
                               opts.SourceAccountName,
                               opts.SourceContainer,
                               opts.SourceKey,
                               opts.SourceSAS,
                               opts.SourceFilePrefix,
                               opts.SourceEndpointSuffix,
                               opts.SortFilenames,
                               opts.SourceFiles.ToList(),
                               opts.DestAccountName,
                               opts.DestKey,
                               opts.DestSAS,
                               opts.DestContainer,
                               opts.DestFilename,
                               opts.DestEndpointSuffix,
                               opts.ColHeader,
                               opts.FileSeparator,
                               opts.CalcMD5ForBlock,
                               opts.Overwrite,
                               opts.ExecutionTimeout,
                               opts.MaxDOP,
                               opts.UseInbuiltRetry,
                               opts.RetryCount,
                               myLogger,
                               progress).GetAwaiter().GetResult() ? 0 : 1);
                },
                    (FilesToBlobOptions opts) =>
                {
                    return(BlobCatEngine.DiskToBlob(
                               opts.SourceFolder,
                               opts.SourceFilePrefix,
                               opts.SortFilenames,
                               opts.SourceFiles.ToList(),
                               opts.DestAccountName,
                               opts.DestKey,
                               opts.DestSAS,
                               opts.DestContainer,
                               opts.DestFilename,
                               opts.DestEndpointSuffix,
                               opts.ColHeader,
                               opts.FileSeparator,
                               opts.CalcMD5ForBlock,
                               opts.Overwrite,
                               opts.ExecutionTimeout,
                               opts.MaxDOP,
                               opts.UseInbuiltRetry,
                               opts.RetryCount,
                               myLogger,
                               progress
                               ).GetAwaiter().GetResult() ? 0 : 1);
                },
                    errs => 1);
            }

            if (pbar != null)
            {
                pbar.Dispose();
            }

            return(retVal);
        }