示例#1
0
        public void Execute()
        {
            var mapfiles = new MapFileCollection(this.Path);

            foreach (((var src, var trg), var i) in mapfiles.GetFiles().Select((x, i) => (x, i)))
            {
                Log.Info($"{(i + 1).ToString().PadRight(5)} {trg}");
            }
        }
示例#2
0
        /// <summary>
        /// Iterates each daily file in the specified <paramref name="dailyFolder"/> and adds a line for each
        /// day to the approriate coarse file
        /// </summary>
        /// <param name="dailyFolder">The folder with daily data</param>
        /// <param name="coarseFolder">The coarse output folder</param>
        /// <param name="mapFileFolder">Location of the map file for the data</param>
        /// <param name="start">The start time, this is resolve by finding the most recent written coarse file</param>
        /// <param name="exclusions">The symbols to be excluded from processing</param>
        /// <param name="ignoreMapless">Ignore the symbols without a map file.</param>
        private static void ProcessDailyFolder(string dailyFolder, string coarseFolder, string mapFileFolder, DateTime start, HashSet<string> exclusions, bool ignoreMapless)
        {
            const decimal scaleFactor = 10000m;

            Log.Trace("Processing: {0}", dailyFolder);

            var stopwatch = Stopwatch.StartNew();

            // load map files into memory
            var mapFileCollection = new MapFileCollection(mapFileFolder);

            var symbols = 0;
            var maplessCount = 0;
            var dates = new HashSet<DateTime>();

            // instead of opening/closing these constantly, open them once and dispose at the end (~3x speed improvement)
            var writers = new Dictionary<string, StreamWriter>();

            // open up each daily file to get the values and append to the daily coarse files
            foreach (var file in Directory.EnumerateFiles(dailyFolder))
            {
                try
                {
                    var symbol = Path.GetFileNameWithoutExtension(file);
                    if (symbol == null)
                    {
                        Log.Trace("Unable to resolve symbol from file: {0}", file);
                        continue;
                    }

                    symbol = symbol.ToUpper();

                    if (exclusions.Contains(symbol))
                    {
                        Log.Trace("Excluded symbol: {0}", symbol);
                        continue;
                    }

                    if (ignoreMapless && !File.Exists(Path.Combine(mapFileFolder, symbol.ToLower() + ".csv")))
                    {
                        maplessCount++;
                        // Too verbose.
                        //Log.Trace("Excluded mapless symbol: " + symbol);
                        continue;
                    }

                    ZipFile zip;
                    using (var reader = Compression.Unzip(file, out zip))
                    {
                        // 30 period EMA constant
                        const decimal k = 2m / (30 + 1);

                        var seeded = false;
                        var runningAverageVolume = 0m;

                        symbols++;
                        string line;
                        while ((line = reader.ReadLine()) != null)
                        {
                            //20150625.csv
                            var csv = line.Split(',');
                            var date = DateTime.ParseExact(csv[0], DateFormat.TwelveCharacter, CultureInfo.InvariantCulture);
                            if (date <= start) continue;

                            var close = decimal.Parse(csv[4])/scaleFactor;
                            var volume = long.Parse(csv[5]);

                            // compute the current volume EMA for dollar volume calculations
                            runningAverageVolume = seeded
                                ? volume*k + runningAverageVolume*(1 - k)
                                : volume;

                            seeded = true;

                            var dollarVolume = close*runningAverageVolume;

                            var coarseFile = Path.Combine(coarseFolder, date.ToString("yyyyMMdd") + ".csv");
                            dates.Add(date);

                            // this is kind of wierd, but is done to keep symbol resolution in the engine simpler
                            // we're going to map this symbol to the correct map file symbol for the date.
                            // for example, in 2013, GOOG was trading, but today it's GOOGL, and so it's
                            // map file is GOOGL.csv, so we'll map the symbol to GOOGL so the engine can
                            // easily resolve it
                            var mappedSymbol = symbol;
                            var mapFile = mapFileCollection.ResolveMapFile(symbol, date);
                            if (mapFile == null && ignoreMapless)
                            {
                                // if we're ignoring mapless files then we should always be able to resolve this
                                Log.Error(string.Format("Unable to resolve map file for {0} as of {1}", symbol, date.ToShortDateString()));
                            }

                            if (mapFile != null)
                            {
                                mappedSymbol = Path.GetFileNameWithoutExtension(mapFile).ToUpper();
                            }

                            // symbol,close,volume,dollar volume
                            var coarseFileLine = mappedSymbol + "," + close + "," + volume + "," + dollarVolume;

                            StreamWriter writer;
                            if (!writers.TryGetValue(coarseFile, out writer))
                            {
                                writer = new StreamWriter(new FileStream(coarseFile, FileMode.Append, FileAccess.Write, FileShare.Write));
                                writers[coarseFile] = writer;
                            }
                            writer.WriteLine(coarseFileLine);
                        }
                    }

                    if (symbols%1000 == 0)
                    {
                        Log.Trace("Completed processing {0} symbols. Current elapsed: {1} seconds", symbols, stopwatch.Elapsed.TotalSeconds.ToString("0.00"));
                    }
                }
                catch (Exception err)
                {
                    // log the error and continue with the process
                    Log.Error(err.ToString());
                }
            }

            Log.Trace("Saving {0} coarse files to disk", dates.Count);

            // dispose all the writers at the end of processing
            foreach (var writer in writers)
            {
                writer.Value.Dispose();
            }

            stopwatch.Stop();

            Log.Trace("Processed {0} symbols into {1} coarse files in {2} seconds", symbols, dates.Count, stopwatch.Elapsed.TotalSeconds.ToString("0.00"));
            Log.Trace("Excluded {0} mapless symbols.", maplessCount);
        }
示例#3
0
        public static void PackPk3(
            string path,
            MapFileCollection files,
            HashSet <Pk3File> stockFiles)
        {
            var folder = FileUtil.NewTempFolder;

            var stock = new HashSet <string>(stockFiles.Select(x => x.Path));

            foreach ((string source, string target) in files.GetFiles())
            {
                // If exact match found in stock files, skip
                if (stock.Contains(target))
                {
                    Log.Debug($"Skipping '{target}', exists in stock pk3s.");
                    continue;
                }

                string sourceFile = null;
                string targetFile = null;

                // Exact match
                if (File.Exists(source))
                {
                    sourceFile = source;
                    targetFile = target;
                }
                else if (Path.GetExtension(source) == "" ||
                         Path.GetExtension(source) == ".tga" ||
                         Path.GetExtension(source) == ".jpg")
                {
                    string ambiguous = null;

                    if (File.Exists(ambiguous = Path.ChangeExtension(source, "tga")))
                    {
                        Log.Debug($"Using found tga version for file {target}");
                        sourceFile = ambiguous;
                        targetFile = Path.ChangeExtension(target, "tga");
                    }
                    else if (stock.Contains(ambiguous))
                    {
                        Log.Debug($"TGA for missing file '{target}' found in stock pk3s");
                        continue;
                    }
                    else if (File.Exists(ambiguous = Path.ChangeExtension(source, "jpg")))
                    {
                        Log.Debug($"Using found jpg version for file {target}");
                        sourceFile = ambiguous;
                        targetFile = Path.ChangeExtension(target, "jpg");
                    }
                    else if (stock.Contains(ambiguous))
                    {
                        Log.Debug($"JPG for missing file '{target}' found in stock pk3s");
                        continue;
                    }
                }

                if (sourceFile == null || targetFile == null)
                {
                    Log.Error($"Cannot find '{target}' in filesystem or stock pk3s.");
                    continue;
                }

                var fullPathToTarget = Path.Combine(folder, targetFile);
                Directory.CreateDirectory(Path.GetDirectoryName(fullPathToTarget));
                File.Copy(sourceFile, fullPathToTarget, false);

                Log.Debug($" -> {targetFile}");
            }

            ZipFile.CreateFromDirectory(folder, path);
        }