public void ViaSocketWriter() { var ranges = new RangeCollection(); var fileStream = new FileStream(@"C:\Users\jgauffin\Downloads\AspNetMVC3ToolsUpdateSetup.exe", FileMode.Open, FileAccess.Read, FileShare.ReadWrite); ranges.Parse("bytes=0-50000", (int)fileStream.Length); var stream = new ByteRangeStream(ranges, fileStream); var job = new StreamSocketWriterJob(stream); var buffer = new byte[65535]; var args = Substitute.For <SocketAsyncEventArgs>(); args.UserToken.Returns(buffer); job.Write(args); while (true) { if (job.WriteCompleted(5000)) { job.Dispose(); break; } job.Write(args); } }
public void ViaSocketWriter() { var ranges = new RangeCollection(); var fileStream = new FileStream(@"C:\Users\jgauffin\Downloads\AspNetMVC3ToolsUpdateSetup.exe", FileMode.Open, FileAccess.Read, FileShare.ReadWrite); ranges.Parse("bytes=0-50000", (int)fileStream.Length); var stream = new ByteRangeStream(ranges, fileStream); var job = new StreamSocketWriterJob(stream); var buffer = new byte[65535]; var args = Substitute.For<SocketAsyncEventArgs>(); args.UserToken.Returns(buffer); job.Write(args); while (true) { if (job.WriteCompleted(5000)) { job.Dispose(); break; } job.Write(args); } }
public void ReadOneRangePartially() { var stream = CreateTextStream(); var ranges = new RangeCollection(); ranges.Parse("bytes=0-99", (int)stream.Length); byte[] target = new byte[100]; var stream2 = new ByteRangeStream(ranges, stream); stream2.Read(target, 0, 50); stream2.Read(target, 50, 50); Assert.Equal(GetString().Substring(0,100), Encoding.ASCII.GetString(target, 0, 100)); }
public void ReadOneRangePartially() { var stream = CreateTextStream(); var ranges = new RangeCollection(); ranges.Parse("bytes=0-99", (int)stream.Length); byte[] target = new byte[100]; var stream2 = new ByteRangeStream(ranges, stream); stream2.Read(target, 0, 50); stream2.Read(target, 50, 50); Assert.Equal(GetString().Substring(0, 100), Encoding.ASCII.GetString(target, 0, 100)); }
/// <summary> /// Refreshes job status /// </summary> public void RefreshJobStatus() { if (_jobListener != null) { LogEntry oldVersion = (LogEntry)this.MemberwiseClone(); JobState = _jobListener.JobState; ISchedulerJobCounters counters = _jobListener.JobCounters; //_job.GetCounters(); string stateStr = string.Format("{0}/{1}/{2}/{3}/{4}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.CanceledTaskCount, counters.FinishedTaskCount); FailedTaskCount = counters.FailedTaskCount; TaskStatus = stateStr; if (FailedTaskCount > 0) { IEnumerable <ISchedulerTask> tasklist = ClusterSubmitter.GetFailedAndCanceledTasks(_jobListener.Scheduler, _jobListener.Job); string failedTaskRangeAsString = tasklist.Select(task => task.TaskId.JobTaskId).StringJoin(","); if ("" != failedTaskRangeAsString) { this.FailedTasks = RangeCollection.Parse(failedTaskRangeAsString).ToString(); } else { FailedTasks = ""; } } else { FailedTasks = ""; } if (_jobListener.JobState == JobState.Finished) { if (WallTime.Ticks == 0) { DateTime startTime = _job.SubmitTime; DateTime endTime = _job.EndTime; WallTime = endTime - startTime; } if (CpuTime.Ticks == 0) { var tasklist = _job.GetTaskList(null, null, true).Cast <ISchedulerTask>(); var totalTicks = tasklist.Select(task => (task.EndTime - task.StartTime).Ticks).Sum(); CpuTime = new TimeSpan(totalTicks); } } } }
private void HandleRangeRequest(IRequest request) { var rangeHeader = request.Headers["Range"]; var response = request.CreateResponse(HttpStatusCode.PartialContent, "Welcome"); response.ContentType = "application/octet-stream"; response.AddHeader("Accept-Ranges", "bytes"); response.AddHeader("Content-Disposition", @"attachment;filename=""ReallyBigFile.Txt"""); //var fileStream = new FileStream(Environment.CurrentDirectory + @"\Ranges\ReallyBigFile.Txt", FileMode.Open, // FileAccess.Read, FileShare.ReadWrite); var fileStream = new FileStream(@"C:\Users\jgauffin\Downloads\AspNetMVC3ToolsUpdateSetup.exe", FileMode.Open, FileAccess.Read, FileShare.ReadWrite); var ranges = new RangeCollection(); ranges.Parse(rangeHeader.Value, (int)fileStream.Length); response.AddHeader("Content-Range", ranges.ToHtmlHeaderValue((int)fileStream.Length)); response.Body = new ByteRangeStream(ranges, fileStream); Send(response); }
/// <returns>True if item was added. False if it already existed in the range.</returns> override public bool TryAdd(Dictionary <string, string> row, string fileName) { Helper.CheckCondition(row.ContainsKey("rowIndex"), string.Format(@"When auditing tabulation a ""rowIndex"" column is required. (File ""{0}"")", fileName)); Helper.CheckCondition(row.ContainsKey("rowCount"), string.Format(@"When auditing tabulation a ""rowCount"" column is required. (File ""{0}"")", fileName)); RangeCollection rowIndexRange = RangeCollection.Parse(row["rowIndex"]); long rowCount = long.Parse(row["rowCount"]); Helper.CheckCondition(rowIndexRange.IsBetween(0, rowCount - 1), string.Format(@"rowIndex must be at least zero and less than rowCount (File ""{0}"")", fileName)); if (RowCountSoFar == long.MinValue) { RowCountSoFar = rowCount; } else { Helper.CheckCondition(RowCountSoFar == rowCount, string.Format("A different row count was at rowIndex {0} in file {1}", rowIndexRange, fileName)); } bool tryAdd = RowIndexRangeCollection.TryAddRangeCollection(rowIndexRange); return(tryAdd); }
private void ParseIntoInternal(string hlaStr) { Match unambMatch = UnambiguousHlaExpr.Match(hlaStr); if (unambMatch.Success) { string locusStr = unambMatch.Groups["locus"].Value; ParseLocusAndCheckAgainstExisting(hlaStr, locusStr); TryAdd(unambMatch.Groups["number"].Value).Enforce("{0} cannot be parsed. Must have 2, 4 or 6 digits.", hlaStr); } else { Match ambMatch = AmbiguousHlaExpr.Match(hlaStr); if (ambMatch.Success) { string locusStr = ambMatch.Groups["locus"].Value; ParseLocusAndCheckAgainstExisting(hlaStr, locusStr); string commonNum = ambMatch.Groups["commonNum"].Value; string rangeStr = ambMatch.Groups["range"].Value; RangeCollection range = RangeCollection.Parse(rangeStr); foreach (long endingNum in range) { string endingNumStr = endingNum >= 100 ? endingNum.ToString("0000") : endingNum.ToString("00"); // if six digits, then this chunk will be >=100. Else, will be the last 2 digits of a 4 digit type. string typeToAdd = commonNum + endingNumStr; TryAdd(typeToAdd).Enforce("Tried to create an HLA type out of {0} but failed. Must be 2, 4 or 6 digits long.", typeToAdd); } } else { throw new ParseException("Can't parse hla string {0}.", hlaStr); } } }
/// <summary> /// Handle the request. /// </summary> /// <param name="context">HTTP context</param> /// <returns><see cref="ModuleResult.Stop"/> will stop all processing except <see cref="IHttpModule.EndRequest"/>.</returns> /// <remarks>Invoked in turn for all modules unless you return <see cref="ModuleResult.Stop"/>.</remarks> public ModuleResult HandleRequest(IHttpContext context) { // only handle GET and HEAD if (!context.Request.Method.Equals("GET", StringComparison.OrdinalIgnoreCase) && !context.Request.Method.Equals("HEAD", StringComparison.OrdinalIgnoreCase)) { return(ModuleResult.Continue); } // serve a directory if (ListFiles) { if (TryGenerateDirectoryPage(context)) { return(ModuleResult.Stop); } } var header = context.Request.Headers["If-Modified-Since"]; var time = header != null ? DateTime.ParseExact(header.Value, "R", CultureInfo.InvariantCulture) : DateTime.MinValue; var fileContext = new FileContext(context.Request, time); _fileService.GetFile(fileContext); if (!fileContext.IsFound) { return(ModuleResult.Continue); } if (!fileContext.IsModified) { context.Response.StatusCode = (int)HttpStatusCode.NotModified; context.Response.StatusDescription = "Was last modified " + fileContext.LastModifiedAtUtc.ToString("R"); return(ModuleResult.Stop); } var mimeType = MimeTypeProvider.Instance.Get(fileContext.Filename); if (mimeType == null) { context.Response.StatusCode = (int)HttpStatusCode.UnsupportedMediaType; context.Response.StatusDescription = string.Format("File type '{0}' is not supported.", Path.GetExtension(fileContext.Filename)); return(ModuleResult.Stop); } context.Response.AddHeader("Last-Modified", fileContext.LastModifiedAtUtc.ToString("R")); context.Response.AddHeader("Accept-Ranges", "bytes"); context.Response.AddHeader("Content-Disposition", "inline;filename=\"" + Path.GetFileName(fileContext.Filename) + "\""); context.Response.ContentType = mimeType; context.Response.ContentLength = (int)fileContext.FileStream.Length; // ranged/partial transfers var rangeHeader = context.Request.Headers["Range"]; if (rangeHeader != null && !string.IsNullOrEmpty(rangeHeader.Value)) { var ranges = new RangeCollection(); ranges.Parse(rangeHeader.Value, (int)fileContext.FileStream.Length); context.Response.AddHeader("Content-Range", ranges.ToHtmlHeaderValue((int)fileContext.FileStream.Length)); context.Response.Body = new ByteRangeStream(ranges, fileContext.FileStream); context.Response.ContentLength = ranges.TotalLength; context.Response.StatusCode = 206; } else { context.Response.Body = fileContext.FileStream; } // do not include a body when the client only want's to get content information. if (context.Request.Method.Equals("HEAD", StringComparison.OrdinalIgnoreCase)) { context.Response.Body.Dispose(); context.Response.Body = null; } return(ModuleResult.Stop); }
/// <summary> /// Handle the request. /// </summary> /// <param name="context">HTTP context</param> /// <returns><see cref="ModuleResult.Stop"/> will stop all processing except <see cref="IHttpModule.EndRequest"/>.</returns> /// <remarks>Invoked in turn for all modules unless you return <see cref="ModuleResult.Stop"/>.</remarks> public ModuleResult HandleRequest(IHttpContext context) { // only handle GET and HEAD if (!context.Request.HttpMethod.Equals("GET", StringComparison.OrdinalIgnoreCase) && !context.Request.HttpMethod.Equals("HEAD", StringComparison.OrdinalIgnoreCase)) return ModuleResult.Continue; // serve a directory if (AllowFileListing) { if (TryGenerateDirectoryPage(context)) return ModuleResult.Stop; } var header = context.Request.Headers["If-Modified-Since"]; var time = header != null ? DateTime.ParseExact(header, "R", CultureInfo.InvariantCulture) : DateTime.MinValue; var fileContext = new FileContext(context.Request, time); _fileService.GetFile(fileContext); if (!fileContext.IsFound) return ModuleResult.Continue; if (!fileContext.IsModified) { context.Response.StatusCode = (int)HttpStatusCode.NotModified; context.Response.ReasonPhrase = "Was last modified " + fileContext.LastModifiedAtUtc.ToString("R"); return ModuleResult.Stop; } if (fileContext.IsGzipSubstitute) { context.Response.AddHeader("Content-Encoding", "gzip"); } var mimeType = MimeTypeProvider.Instance.Get(fileContext.Filename); if (mimeType == null) { context.Response.StatusCode = (int)HttpStatusCode.UnsupportedMediaType; context.Response.ReasonPhrase = string.Format("File type '{0}' is not supported.", Path.GetExtension(fileContext.Filename)); return ModuleResult.Stop; } context.Response.AddHeader("Last-Modified", fileContext.LastModifiedAtUtc.ToString("R")); context.Response.AddHeader("Accept-Ranges", "bytes"); context.Response.AddHeader("Content-Disposition", "inline;filename=\"" + Path.GetFileName(fileContext.Filename) + "\""); context.Response.ContentType = mimeType; context.Response.ContentLength = (int)fileContext.FileStream.Length; // ranged/partial transfers var rangeStr = context.Request.Headers["Range"]; if (!string.IsNullOrEmpty(rangeStr)) { var ranges = new RangeCollection(); ranges.Parse(rangeStr, (int)fileContext.FileStream.Length); context.Response.AddHeader("Content-Range", ranges.ToHtmlHeaderValue((int)fileContext.FileStream.Length)); context.Response.Body = new ByteRangeStream(ranges, fileContext.FileStream); context.Response.ContentLength = ranges.TotalLength; context.Response.StatusCode = 206; } else context.Response.Body = fileContext.FileStream; // do not include a body when the client only want's to get content information. if (context.Request.HttpMethod.Equals("HEAD", StringComparison.OrdinalIgnoreCase) && context.Response.Body != null) { context.Response.Body.Dispose(); context.Response.Body = null; } return ModuleResult.Stop; }
/// <summary> /// Only call this in a lock... /// </summary> private bool AsynchronousAddRangeToStream(RangeCollection itemsToAdd, int itemCount) { string completedRanges = null; RangeCollection completedRangeCollection = null; //StreamReader reader = null; //StreamWriter writer = null; try { //reader = new StreamReader(rangeFileStream); //completedRanges = reader.ReadToEnd(); completedRanges = _sharedRangeFile.ReadAll().Trim(); completedRangeCollection = completedRanges.Length == 0 ? new RangeCollection() : RangeCollection.Parse(completedRanges); Console.WriteLine("Range read from file: {0}", completedRangeCollection); completedRangeCollection.AddRangeCollection(itemsToAdd); //rangeFileStream.SetLength(0); //writer = new StreamWriter(rangeFileStream); //writer.Write(completedRangeCollection); //writer.Flush(); _sharedRangeFile.Clear(); _sharedRangeFile.Write(completedRangeCollection); Console.WriteLine("Range {0} complete to {1} items? {2}", completedRangeCollection, itemCount, completedRangeCollection.IsComplete(itemCount)); return(completedRangeCollection.IsComplete(itemCount)); } catch (Exception) { Console.WriteLine("ERROR: Trouble adding range."); Console.WriteLine("read from file: {0}", completedRanges); Console.WriteLine("new range: {0}", itemsToAdd); Console.WriteLine("combined: {0}", completedRangeCollection); throw; } }
static void Main(string[] args) { try { ArgCollection argCollection = ArgCollection.GetInstance(args); if (argCollection.ExtractOptionalFlag("help")) { Console.WriteLine(""); Console.WriteLine(UsageMessage); Console.WriteLine(HelpMessage); return; } string optimizerName = argCollection.ExtractOptional <string>("optimizer", "BrentThenGrid"); string keepTestName = argCollection.ExtractOptional <string>("keepTest", "AlwaysKeep"); string skipRowIndexFileNameOrNull = argCollection.ExtractOptional <string>("skipRowIndexFile", null); argCollection.CheckNoMoreOptions(); string treeFileName = argCollection.ExtractNext <string>("treeFile"); string predictorFileName = argCollection.ExtractNext <string>("predictorFile"); string targetFileName = argCollection.ExtractNext <string>("targetFile"); string leafDistributionName = argCollection.ExtractNext <string>("leafDistribution"); string nullDataGeneratorName = argCollection.ExtractNext <string>("nullDataGenerator"); string niceName = argCollection.ExtractNext <string>("niceName"); string outputDirectory = argCollection.ExtractNext <string>("outputDirectory"); RangeCollection pieceIndexRangeCollection = argCollection.ExtractNext <RangeCollection>("pieceIndexRange"); int pieceCount = argCollection.ExtractNext <int>("pieceCount"); RangeCollection nullIndexRangeCollection = argCollection.ExtractNext <RangeCollection>("nullIndexRange"); argCollection.CheckThatEmpty(); if (!PhyloDDriver.ValidateDistribution(leafDistributionName)) { Console.WriteLine("{0} is not a recognized distribution name. Please choose a name from the following list:", leafDistributionName); foreach (string name in PhyloDDriver.GetDistributionNames()) { Console.WriteLine("\t{0}", name); } throw new ArgumentException("Invalid distribution name."); } RangeCollection skipRowIndexRangeCollectionOrNull = (null == skipRowIndexFileNameOrNull) || skipRowIndexFileNameOrNull == "null" ? null : RangeCollection.Parse(File.ReadAllText(skipRowIndexFileNameOrNull)); KeepTest <Dictionary <string, string> > keepTest = KeepTest <Dictionary <string, string> > .GetInstance(null, keepTestName); SpecialFunctions.CheckCondition(pieceIndexRangeCollection.IsBetween(0, pieceCount - 1), "pieceIndex must be at least 0 and less than pieceCount"); SpecialFunctions.CheckCondition(nullIndexRangeCollection.IsBetween(-1, int.MaxValue), "nullIndex must be at least -1"); PhyloTree aPhyloTree = PhyloTree.GetInstance(treeFileName, null); ModelScorer modelScorer = ModelScorer.GetInstance(aPhyloTree, leafDistributionName, optimizerName); ModelEvaluator modelEvaluator = ModelEvaluator.GetInstance(leafDistributionName, modelScorer); PhyloDDriver driver = PhyloDDriver.GetInstance(); driver.Run( modelEvaluator, predictorFileName, targetFileName, leafDistributionName, nullDataGeneratorName, keepTest, skipRowIndexRangeCollectionOrNull, niceName, outputDirectory, pieceIndexRangeCollection, pieceCount, nullIndexRangeCollection, optimizerName); //Console.Write("Press enter to exist."); //Console.Read(); } catch (Exception exception) { Console.WriteLine(""); Console.WriteLine(exception.Message); if (exception.InnerException != null) { Console.WriteLine(exception.InnerException.Message); } Console.WriteLine(""); Console.WriteLine(UsageMessage); throw; } }
/// <summary> /// Does the work. /// </summary> public override void DoWork() { // get our input data and null the field to make sure we don't serialize it back InputData inputData = mInputData; mInputData = null; // get the job-specific names of input files FileDefCollection fileDefs = Job.FileDefs; string treeFileName = Utility.GetNamedFileDef(fileDefs, Constants.TreeFileDefName).LocalName; string predictorFileName = Utility.GetNamedFileDef(fileDefs, Constants.PredictorFileDefName).LocalName; string targetFileName = Utility.GetNamedFileDef(fileDefs, Constants.TargetFileDefName).LocalName; string skipRowIndexFileName = Utility.GetNamedFileDef(fileDefs, Constants.SkipRowIndexFileDefName).LocalName; // construct RangeCollections RangeCollection pieceIndexRangeCollection = RangeCollection.Parse(inputData.PieceIndexRange); RangeCollection nullIndexRangeCollection = RangeCollection.Parse(inputData.NullIndexRange); RangeCollection skipRowIndexRangeCollection; FileInfo fileInfo = new FileInfo(skipRowIndexFileName); if (fileInfo.Length > 0) { skipRowIndexRangeCollection = RangeCollection.Parse(File.ReadAllText(skipRowIndexFileName)); } else { skipRowIndexRangeCollection = null; } // do the rest PhyloTree aPhyloTree = PhyloTree.GetInstance(treeFileName, null); ModelScorer modelScorer = ModelScorer.GetInstance(aPhyloTree, inputData.LeafDistributionName, inputData.OptimizerName); ModelEvaluator modelEvaluator = ModelEvaluator.GetInstance(inputData.LeafDistributionName, modelScorer); KeepTest <Dictionary <string, string> > keepTest = KeepTest <Dictionary <string, string> > .GetInstance(null, inputData.KeepTestName); PhyloDDriver driver = PhyloDDriver.GetInstance(); // create a name for the temporary job sandbox. This directory gets created by driver.Run(...) string agentOutputDirectoryName = Path.Combine(Environment.CurrentDirectory, String.Format(CultureInfo.InvariantCulture, "{0}.{1}", Job.JobId, Task.TaskId)); // save the standard out and standard error in memory streams using (MemoryStream streamOut = new MemoryStream(), streamError = new MemoryStream()) { try { // redirect the outputs using ( StreamWriter writerOut = new StreamWriter(streamOut), writerError = new StreamWriter(streamError)) { Console.SetOut(writerOut); Console.SetError(writerError); try { // run the model string outputFileName = driver.Run( modelEvaluator, predictorFileName, targetFileName, inputData.LeafDistributionName, inputData.NullDataGeneratorName, keepTest, skipRowIndexRangeCollection, inputData.NiceName, agentOutputDirectoryName, pieceIndexRangeCollection, inputData.PieceCount, nullIndexRangeCollection, inputData.OptimizerName); // this is the expected output file name -- save this so it can be written on the master side with the same name. mOutputFileName = Path.GetFileName(outputFileName); mLocalOutputFileName = Path.Combine(inputData.LocalOutputDirectoryName, mOutputFileName); // get the output data string fullOutputPath = Path.Combine(agentOutputDirectoryName, mOutputFileName); if (!File.Exists(fullOutputPath)) { TaskResult.FailureReason = TaskFailureReason.MissingOutput; TaskResult.FailureMessage = String.Format(CultureInfo.CurrentCulture, "Cannot find output file '{0}'", targetFileName); TaskResult.Status = TaskAssignmentStatus.Failed; } using (StreamReader outputData = new StreamReader(fullOutputPath)) { mOutputData = outputData.ReadToEnd(); } } finally { // this finally is to make sure we delete the folder // get rid of the sandbox Directory.Delete(agentOutputDirectoryName, true); } } } finally { // this finally is to make sure we get console output Encoding encoding = Encoding.Default; TaskResult.StandardOutput = encoding.GetString(streamOut.GetBuffer()); TaskResult.StandardError = encoding.GetString(streamError.GetBuffer()); } } }
public void RefreshJobStatus() { if (_job != null) { LogEntry oldVersion = (LogEntry)this.MemberwiseClone(); _job.Refresh(); JobState = _job.State; ISchedulerJobCounters counters = _job.GetCounters(); string stateStr = string.Format("{0}/{1}/{2}/{3}", counters.QueuedTaskCount, counters.RunningTaskCount, counters.FailedTaskCount, counters.FinishedTaskCount); FailedTaskCount = counters.FailedTaskCount; TaskStatus = stateStr; if (FailedTaskCount > 0) { IEnumerable <ISchedulerTask> tasklist = GetFailedTasks(_job); string failedTaskRangeAsString = tasklist.Select(task => task.TaskId.JobTaskId).StringJoin(","); if ("" != failedTaskRangeAsString) { this.FailedTasks = RangeCollection.Parse(failedTaskRangeAsString).ToString(); } else { FailedTasks = ""; } } else { FailedTasks = ""; } if (JobState == JobState.Finished) { if (WallTime.Ticks == 0) { DateTime startTime = _job.SubmitTime; DateTime endTime = _job.EndTime; WallTime = endTime - startTime; } if (CpuTime.Ticks == 0) { var tasklist = _job.GetTaskList(null, null, true).Cast <ISchedulerTask>(); var totalTicks = tasklist.Select(task => (task.EndTime - task.StartTime).Ticks).Sum(); CpuTime = new TimeSpan(totalTicks); } } bool taskStateChanged = FailedTasks != oldVersion.FailedTasks || TaskStatus != oldVersion.TaskStatus; bool jobStateChanged = JobState != oldVersion.JobState || (FailedTaskCount == 0) != (oldVersion.FailedTaskCount == 0) || string.IsNullOrEmpty(FailedTasks) != string.IsNullOrEmpty(oldVersion.FailedTasks) || CpuTime != oldVersion.CpuTime || WallTime != oldVersion.WallTime; //if (_taskStateChangedSinceLastEvent != taskStateChanged || _jobStateChangedSinceLastEvent != jobStateChanged) // Console.WriteLine("bad"); if (taskStateChanged) { RaiseTaskStateChangedEvent(); } if (jobStateChanged) { RaiseJobStateChangedEvent(); } } }