public void DecodeSegment(Segment segment) { segment.status = Segment.Status.YDECODING; lock (segments) { segments.Enqueue(segment); } }
public void addSegment(ref Segment segment) { segment.status = Segment.Status.WRITECACHE; lock (newSegments) { newSegments.Add(segment); } }
private string getFilename(ref Segment segment) { // Create output dictionary if necessary Directory.CreateDirectory(Properties.Settings.Default.outputPath + "\\" + segment.parent.parent.name + "\\"); if (segment.yname != null && segment.yname != "") { return Properties.Settings.Default.outputPath + "\\" + segment.parent.parent.name + "\\" + segment.yname; } else { return Properties.Settings.Default.outputPath + "\\" + segment.parent.parent.name + "\\" + segment.parent.filename; } }
private void writeSegment(Segment seg, WriteStream ws) { ws.stream.Write(seg.data, 0, seg.data.Length); seg.parent.byteprogress += seg.data.Length; seg.data = null; seg.status = Segment.Status.COMPLETE; seg.parent.saveprogress = seg.id; ws.lastuse = DateTime.Now.Millisecond; }
// Updates cache private void Update() { while (keepRunning) { List<Segment> toremove = new List<Segment>(); Segment[] segs = new Segment[newSegments.Count]; lock (newSegments) { newSegments.CopyTo(segs); newSegments.Clear(); } for (int i = 0; i < segs.Length; i++) { processSegment(segs[i]); } foreach (Segment seg in segments) { if (seg.id - 1 == seg.parent.saveprogress) { saveSegment(seg); cacheSize -= (ulong)seg.bytes; toremove.Add(seg); } } foreach (Segment seg in toremove) { segments.Remove(seg); } // Empty out streams thats passed the timeout limit.. foreach (WriteStream ws in streams) { if (ws.lastuse + STREAMTIMEOUT * 1000 < DateTime.Now.Millisecond) { ws.stream.Close(); } } Thread.Sleep(1000); // Sleep 1sec before updating again } }
private void saveSegment(Segment seg) { System.Console.WriteLine("Saving seg: " + seg.id); bool saved = false; // Check if stream is in cache List<WriteStream> toremove = new List<WriteStream>(); List<WriteStream>.Enumerator enumer = streams.GetEnumerator(); if(enumer.MoveNext()) { do { WriteStream ws = (WriteStream)enumer.Current; if (ws.filejob.Equals(seg.parent)) { // Copy from a tempsaved segment if (seg.tempsaved) { FileStream stream = new FileStream(Properties.Settings.Default.tempFolder + "\\" + seg.tempname, FileMode.Open, FileAccess.Read); byte[] buffer = new byte[seg.bytes]; stream.Read(buffer, 0, seg.bytes); // May end up blocking if seg.bytes is larger than actual file seg.data = buffer; stream.Close(); stream.Dispose(); } writeSegment(seg, ws); saved = true; // Is filejob complete? if (seg.id == seg.parent.yparts) { // Close stream ws.stream.Flush(); ws.stream.Close(); ws.stream.Dispose(); // Remove from collection toremove.Add(ws); seg.parent.complete = true; } break; } } while (enumer.MoveNext()); } foreach (WriteStream ws in toremove) { streams.Remove(ws); } // Else open a new stream if (!saved) { // Open new stream.. WriteStream ws = new WriteStream(); ws.stream = new FileStream(seg.parent.outputfilename, FileMode.Append, FileAccess.Write); ws.filejob = seg.parent; writeSegment(seg, ws); // Was this the last part? if (seg.id == seg.parent.yparts || seg.parent.yparts == 0) { // Close stream.. ws.stream.Flush(); ws.stream.Close(); ws.stream.Dispose(); } else { streams.Add(ws); } } }
// Processes new segment private void processSegment(Segment segment) { // is the cache waiting for this part? if (segment.id == 1) { Logging.Instance.Log("(Cache) Recieved first segment"); // First part in a filejob.. segment.parent.outputfilename = getFilename(ref segment); saveSegment(segment); } else if (segment.id - 1 == segment.parent.saveprogress) { // Append to file Logging.Instance.Log("(Cache) Recieved awaited segment"); saveSegment(segment); } else if (cacheSize + (ulong)segment.bytes > (ulong)MAXCACHE * 1024 * 1024) { Logging.Instance.Log("Cache too big, saving to disk"); segment.status = Segment.Status.TEMPCACHED; segment.tempname = "temp" + new Random().Next(9999999); FileStream stream = new FileStream(Properties.Settings.Default.tempFolder + "\\" + segment.tempname, FileMode.Create, FileAccess.Write); stream.Write(segment.data, 0, segment.bytes); segment.data = null; stream.Flush(); stream.Close(); stream.Dispose(); segments.Add(segment); segment.tempsaved = true; } else { // memorycache Logging.Instance.Log("(Cache) Saving segment to cache"); segments.Add(segment); cacheSize += (ulong)segment.bytes; } }
public byte[] RecieveSegment(Segment segment) { try { // Request body... Write("BODY "+ segment.addr +'\n'); Assert(Response(), "222"); NetworkStream stream = GetStream(); stream.ReadTimeout = 10000; byte[] buffer = new byte[segment.bytes]; int read = 0; bool done = false; int chunk; while (!done && (chunk = stream.Read(buffer, read, buffer.Length - read)) > 0 && keepAlive) { read += chunk; segment.parent.parent.ProgressKB += (ulong)chunk; segment.progress += chunk; lock (this) { speed += (uint)chunk; } if (read == buffer.Length) { byte[] newBuffer = new byte[buffer.Length * 2]; Array.Copy(buffer, newBuffer, buffer.Length); buffer = newBuffer; } mainGui.UpdateSingleThreadInfo(this); // Looks for .\r\n (End of message) if (read > 2 && buffer[read - 3] == '.' && buffer[read - 2] == '\r' && buffer[read - 1] == '\n') { // Check that it is in the beginning of a new line if (read < 4 || buffer[read - 5] == '\r' && buffer[read - 4] == '\n') { break; } } } if (!keepAlive) { throw new Exception("KeepAlive is false"); ; } byte[] returnarray = new byte[read - 3]; Array.Copy(buffer, returnarray, read - 3); segment.bytes = read; Logging.Instance.Log("(NNTPConnection(" + id + ")) SEGMENT Complete: id=" + segment.id + " bytes=" + read); return returnarray; } catch (IOException ex) { throw; } }
private void ProcessSegment(Segment segment) { int expectedlenght = 0; // TODO: Check accuracy of this value MemoryStream ms = new MemoryStream(segment.data); string header = ReadLine(ms); while(header.Equals("")) header = ReadLine(ms); int ysize = 0; // TODO: Check accuracy of this value if (header.Contains("=ybegin ") && header.Contains("line=") && header.Contains("size=") && header.Contains("name=")) { try { segment.isYenc = true; ysize = int.Parse(getValue(header, "size")); segment.yname = getValue(header, "name"); segment.yname = EnsureGoodName(segment.yname); segment.parent.filename = segment.yname; if (header.Contains("part=")) { segment.YPart = int.Parse(getValue(header, "part")); if (segment.YPart > segment.parent.yparts) segment.parent.yparts = segment.YPart; string partheader = ReadLine(ms); if (partheader.Contains("=ypart ") && partheader.Contains("begin=") && partheader.Contains("end=")) { int partBegin = int.Parse(getValue(partheader, "begin")); int partEnd = int.Parse(getValue(partheader, "end")); expectedlenght = partEnd - partBegin; } } } catch (Exception ex) { } } if (!segment.isYenc) { segment.bytes = segment.data.Length; WriteCache.Instance.addSegment(ref segment); return; } else { // Start decoding data... byte[] buffer = new byte[1]; byte[] output = new byte[segment.data.Length]; bool cr = false, dot = false, finished = false, decode = true, nl = false, bleh2 = false; // flags int bufpos = 0, lastnl = 0; // Decode byte-for-byte while (ms.Read(buffer, 0, 1) > 0 && !finished) { if (cr && buffer[0] == '\n') { // Discard \r\n by going back one byte and discarding current byte bufpos--; lastnl = bufpos; decode = false; } else if (dot && buffer[0] == '.' && bufpos == lastnl+1) { // Discard current (2nd) dot //string s = ""; //for (int i = 10; i > 0; i--) //{ // s += (char)output[bufpos - 1 - i]; //} //System.Console.WriteLine("LastNL: {0}, last10: {1}", lastnl, s); decode = false; } // Reset flags cr = false; dot = false; // Set flags switch ((int)buffer[0]) { case '=': ms.Read(buffer, 0, 1); if (buffer[0] == 'y') { string yend = ReadLine(ms); if (segment.YPart != 0) segment.crc = getValue(yend, "pcrc32"); else segment.crc = getValue(yend, "crc32"); finished = true; break; } else { for (int i = 0; i < special.Length; i++) { if(buffer[0] == special[i]+64) buffer[0] = (byte)(char)special[i]; } } break; case '\r': cr = true; break; case '.': if(decode) dot = true; break; } if (decode && !finished) { // Decode byte buffer[0] -= 42 %256; output[bufpos] = buffer[0]; bufpos++; } decode = true; } byte[] data = new byte[bufpos]; Array.Copy(output, data, bufpos); segment.data = data; segment.bytes = segment.data.Length; segment.ValidateCRC(); WriteCache.Instance.addSegment(ref segment); } }
//// Generates queues for a filecollection object //public static Queue genQueue(ArrayList list) //{ // Queue queue = new Queue(list.Count); // foreach (FileJob job in list) // { // job.queue = new Queue(job.segments.Count); // foreach (Segment seg in job.segments) // { // job.queue.Enqueue(seg); // } // queue.Enqueue(job); // } // return queue; //} // Parses XML structure into file objects private static List<FileJob> ParseXML(XmlDocument document, FileCollection filecollection) { XmlNodeList nodes = document.GetElementsByTagName("file"); List<FileJob> files = new List<FileJob>(); Random random = new Random(); int collectionID = random.Next(); int fileId = 0; // Parsing for each file node foreach (XmlElement fileNode in nodes) { FileJob filejob = new FileJob(ref filecollection); filejob.filename = ""+collectionID; XmlAttributeCollection collection = fileNode.Attributes; // File Attributes foreach (XmlAttribute attr in collection) { switch (attr.Name) { case "poster": filejob.poster = attr.Value; break; case "date": filejob.date = int.Parse(attr.Value); break; case "subject": if (attr.Value.Contains("\"")) { // try to get inner filename... try { int first = attr.Value.IndexOf("\"") + 1; string filename = attr.Value.Substring(first, attr.Value.Length - first); filename = filename.Substring(0, filename.IndexOf("\"")); filejob.filename = filejob.filename + "-"+ filename.Substring(0, filename.LastIndexOf(".")+4); } catch (Exception ex) { filejob.filename = filejob.filename+"-"+fileId; } } else filejob.filename = filejob.filename + "-" + fileId; filejob.subject = attr.Value; break; } } int maxnumber = 0; foreach (XmlElement subNode in fileNode.ChildNodes) { if (subNode.Name == "segments") { foreach (XmlElement segmentNode in subNode) { Segment segment = new Segment(); segment.addr = "<"+segmentNode.InnerXml+">"; segment.addr = segment.addr.Replace("&", "&"); segment.setParent(ref filejob); foreach (XmlAttribute attr in segmentNode.Attributes) { switch (attr.Name) { case "bytes": segment.bytes = int.Parse(attr.Value); filejob.size += (ulong)segment.bytes; break; case "number": segment.id = int.Parse(attr.Value); if (segment.id > maxnumber) maxnumber = segment.id; break; } } // Check and add element to arraylist if (segment.id != 0 && segment.bytes != 0 && segment.addr.Length > 10) filejob.segments.Add(segment); } } else if (subNode.Name == "groups") { filejob.groups.Add(subNode.FirstChild.InnerText); } } filejob.yparts = maxnumber; // Will fail if source isnt yenc // Sort segments filejob.segments.Sort(new SegmentComparer()); files.Add(filejob); fileId++; } return files; }