Esempio n. 1
0
 public void DecodeSegment(Segment segment)
 {
     segment.status = Segment.Status.YDECODING;
     lock (segments)
     {
         segments.Enqueue(segment);
     }
 }
Esempio n. 2
0
 public void addSegment(ref Segment segment)
 {
     segment.status = Segment.Status.WRITECACHE;
     lock (newSegments)
     {
         newSegments.Add(segment);
     }
 }
Esempio n. 3
0
        private string getFilename(ref Segment segment)
        {
            // Create output dictionary if necessary
            Directory.CreateDirectory(Properties.Settings.Default.outputPath + "\\" + segment.parent.parent.name + "\\");

            if (segment.yname != null && segment.yname != "")
            {
                return Properties.Settings.Default.outputPath + "\\" + segment.parent.parent.name + "\\" + segment.yname;
            }
            else
            {
                return Properties.Settings.Default.outputPath + "\\" + segment.parent.parent.name + "\\" + segment.parent.filename;
            }
        }
Esempio n. 4
0
 private void writeSegment(Segment seg, WriteStream ws)
 {
     ws.stream.Write(seg.data, 0, seg.data.Length);
     seg.parent.byteprogress += seg.data.Length;
     seg.data = null;
     seg.status = Segment.Status.COMPLETE;
     seg.parent.saveprogress = seg.id;
     ws.lastuse = DateTime.Now.Millisecond;
 }
Esempio n. 5
0
        // Updates cache
        private void Update()
        {
            while (keepRunning)
            {
                List<Segment> toremove = new List<Segment>();
                Segment[] segs = new Segment[newSegments.Count];
                lock (newSegments)
                {
                    newSegments.CopyTo(segs);
                    newSegments.Clear();
                }

                for (int i = 0; i < segs.Length; i++)
                {
                    processSegment(segs[i]);
                }

                foreach (Segment seg in segments)
                {
                    if (seg.id - 1 == seg.parent.saveprogress)
                    {
                        saveSegment(seg);
                        cacheSize -= (ulong)seg.bytes;
                        toremove.Add(seg);
                    }
                }
                foreach (Segment seg in toremove)
                {
                    segments.Remove(seg);
                }

                // Empty out streams thats passed the timeout limit..
                foreach (WriteStream ws in streams)
                {
                    if (ws.lastuse + STREAMTIMEOUT * 1000 < DateTime.Now.Millisecond)
                    {
                        ws.stream.Close();
                    }
                }
                Thread.Sleep(1000); // Sleep 1sec before updating again
            }
        }
Esempio n. 6
0
        private void saveSegment(Segment seg)
        {
            System.Console.WriteLine("Saving seg: " + seg.id);
            bool saved = false;
            // Check if stream is in cache
            List<WriteStream> toremove = new List<WriteStream>();
            List<WriteStream>.Enumerator enumer = streams.GetEnumerator();
            if(enumer.MoveNext()) {
                do
                {
                    WriteStream ws = (WriteStream)enumer.Current;
                    if (ws.filejob.Equals(seg.parent))
                    {
                        // Copy from a tempsaved segment
                        if (seg.tempsaved)
                        {
                            FileStream stream = new FileStream(Properties.Settings.Default.tempFolder + "\\" + seg.tempname, FileMode.Open, FileAccess.Read);
                            byte[] buffer = new byte[seg.bytes];
                            stream.Read(buffer, 0, seg.bytes); // May end up blocking if seg.bytes is larger than actual file
                            seg.data = buffer;
                            stream.Close();
                            stream.Dispose();
                        }
                        writeSegment(seg, ws);
                        saved = true;

                        // Is filejob complete?
                        if (seg.id == seg.parent.yparts)
                        {
                            // Close stream
                            ws.stream.Flush();
                            ws.stream.Close();
                            ws.stream.Dispose();
                            // Remove from collection
                            toremove.Add(ws);
                            seg.parent.complete = true;
                        }
                        break;
                    }
                }
                while (enumer.MoveNext());
            }
            foreach (WriteStream ws in toremove)
            {
                streams.Remove(ws);
            }

            // Else open a new stream
            if (!saved)
            {
                // Open new stream..
                WriteStream ws = new WriteStream();
                ws.stream = new FileStream(seg.parent.outputfilename, FileMode.Append, FileAccess.Write);
                ws.filejob = seg.parent;
                writeSegment(seg, ws);
                // Was this the last part?
                if (seg.id == seg.parent.yparts || seg.parent.yparts == 0)
                {
                    // Close stream..
                    ws.stream.Flush();
                    ws.stream.Close();
                    ws.stream.Dispose();
                }
                else
                {
                    streams.Add(ws);
                }
            }
        }
Esempio n. 7
0
        // Processes new segment
        private void processSegment(Segment segment)
        {
            // is the cache waiting for this part?

            if (segment.id == 1)
            {

                Logging.Instance.Log("(Cache) Recieved first segment");
                // First part in a filejob..
                segment.parent.outputfilename = getFilename(ref segment);
                saveSegment(segment);

            }
            else if (segment.id - 1 == segment.parent.saveprogress)
            {
                // Append to file
                Logging.Instance.Log("(Cache) Recieved awaited segment");
                saveSegment(segment);
            }
            else if (cacheSize + (ulong)segment.bytes > (ulong)MAXCACHE * 1024 * 1024)
            {
                Logging.Instance.Log("Cache too big, saving to disk");
                segment.status = Segment.Status.TEMPCACHED;
                segment.tempname = "temp" + new Random().Next(9999999);

                FileStream stream = new FileStream(Properties.Settings.Default.tempFolder + "\\" + segment.tempname, FileMode.Create, FileAccess.Write);
                stream.Write(segment.data, 0, segment.bytes);
                segment.data = null;
                stream.Flush();
                stream.Close();
                stream.Dispose();
                segments.Add(segment);
                segment.tempsaved = true;
            }
            else
            {
                // memorycache
                Logging.Instance.Log("(Cache) Saving segment to cache");
                segments.Add(segment);
                cacheSize += (ulong)segment.bytes;
            }
        }
Esempio n. 8
0
        public byte[] RecieveSegment(Segment segment)
        {
            try
            {
                // Request body...

                Write("BODY "+ segment.addr +'\n');
                Assert(Response(), "222");

                NetworkStream stream = GetStream();

                stream.ReadTimeout = 10000;

                byte[] buffer = new byte[segment.bytes];
                int read = 0;
                bool done = false;
                int chunk;

                while (!done && (chunk = stream.Read(buffer, read, buffer.Length - read)) > 0 && keepAlive)
                {
                    read += chunk;
                    segment.parent.parent.ProgressKB += (ulong)chunk;
                    segment.progress += chunk;
                    lock (this)
                    {
                        speed += (uint)chunk;
                    }
                    if (read == buffer.Length)
                    {
                        byte[] newBuffer = new byte[buffer.Length * 2];
                        Array.Copy(buffer, newBuffer, buffer.Length);
                        buffer = newBuffer;
                    }
                    mainGui.UpdateSingleThreadInfo(this);
                    // Looks for .\r\n (End of message)
                    if (read > 2 && buffer[read - 3] == '.' && buffer[read - 2] == '\r' && buffer[read - 1] == '\n')
                    {
                        // Check that it is in the beginning of a new line
                        if (read < 4 || buffer[read - 5] == '\r' && buffer[read - 4] == '\n')
                        {
                            break;
                        }
                    }
                }
                if (!keepAlive)
                {
                    throw new Exception("KeepAlive is false"); ;
                }
                byte[] returnarray = new byte[read - 3];
                Array.Copy(buffer, returnarray, read - 3);
                segment.bytes = read;
                Logging.Instance.Log("(NNTPConnection(" + id + ")) SEGMENT Complete: id=" + segment.id + " bytes=" + read);
                return returnarray;
            }
            catch (IOException ex)
            {
                throw;
            }
        }
Esempio n. 9
0
        private void ProcessSegment(Segment segment)
        {
            int expectedlenght = 0; // TODO: Check accuracy of this value

            MemoryStream ms = new MemoryStream(segment.data);

            string header = ReadLine(ms);
            while(header.Equals(""))
                header = ReadLine(ms);

            int ysize = 0;  // TODO: Check accuracy of this value

            if (header.Contains("=ybegin ") && header.Contains("line=") && header.Contains("size=") && header.Contains("name="))
            {
                try
                {
                    segment.isYenc = true;
                    ysize = int.Parse(getValue(header, "size"));
                    segment.yname = getValue(header, "name");
                    segment.yname = EnsureGoodName(segment.yname);
                    segment.parent.filename = segment.yname;
                    if (header.Contains("part="))
                    {
                        segment.YPart = int.Parse(getValue(header, "part"));
                        if (segment.YPart > segment.parent.yparts)
                            segment.parent.yparts = segment.YPart;

                        string partheader = ReadLine(ms);
                        if (partheader.Contains("=ypart ") && partheader.Contains("begin=") && partheader.Contains("end="))
                        {
                            int partBegin = int.Parse(getValue(partheader, "begin"));
                            int partEnd = int.Parse(getValue(partheader, "end"));
                            expectedlenght = partEnd - partBegin;
                        }
                    }

                }
                catch (Exception ex)
                {
                }
            }

            if (!segment.isYenc)
            {
                segment.bytes = segment.data.Length;
                WriteCache.Instance.addSegment(ref segment);
                return;
            }
            else
            {

                // Start decoding data...
                byte[] buffer = new byte[1];
                byte[] output = new byte[segment.data.Length];
                bool cr = false, dot = false, finished = false, decode = true, nl = false, bleh2 = false; // flags
                int bufpos = 0, lastnl = 0;

                // Decode byte-for-byte
                while (ms.Read(buffer, 0, 1) > 0 && !finished)
                {
                    if (cr && buffer[0] == '\n')
                    {
                        // Discard \r\n by going back one byte and discarding current byte
                        bufpos--;
                        lastnl = bufpos;
                        decode = false;
                    }
                    else if (dot && buffer[0] == '.' && bufpos == lastnl+1)
                    {
                        // Discard current (2nd) dot
                        //string s = "";
                        //for (int i = 10; i > 0; i--)
                        //{
                        //    s += (char)output[bufpos - 1 - i];
                        //}
                        //System.Console.WriteLine("LastNL: {0}, last10: {1}", lastnl, s);
                        decode = false;
                    }
                    // Reset flags
                    cr = false;
                    dot = false;
                    // Set flags
                    switch ((int)buffer[0])
                    {
                        case '=':
                            ms.Read(buffer, 0, 1);
                            if (buffer[0] == 'y')
                            {
                                string yend = ReadLine(ms);
                                if (segment.YPart != 0)
                                    segment.crc = getValue(yend, "pcrc32");
                                else
                                    segment.crc = getValue(yend, "crc32");
                                finished = true;
                                break;
                            } else {
                                for (int i = 0; i < special.Length; i++)
                                {
                                    if(buffer[0] == special[i]+64)
                                        buffer[0] = (byte)(char)special[i];
                                }
                            }
                            break;
                        case '\r':
                            cr = true;
                            break;
                        case '.':
                            if(decode)
                                dot = true;
                            break;
                    }
                    if (decode && !finished)
                    {
                        // Decode byte
                        buffer[0] -= 42 %256;
                        output[bufpos] = buffer[0];
                        bufpos++;
                    }
                    decode = true;
                }
                byte[] data = new byte[bufpos];
                Array.Copy(output, data, bufpos);
                segment.data = data;
                segment.bytes = segment.data.Length;
                segment.ValidateCRC();
                WriteCache.Instance.addSegment(ref segment);
            }
        }
Esempio n. 10
0
        //// Generates queues for a filecollection object
        //public static Queue genQueue(ArrayList list)
        //{
        //    Queue queue = new Queue(list.Count);
        //    foreach (FileJob job in list)
        //    {
        //        job.queue = new Queue(job.segments.Count);
        //        foreach (Segment seg in job.segments)
        //        {
        //            job.queue.Enqueue(seg);
        //        }
        //        queue.Enqueue(job);
        //    }
        //    return queue;
        //}
        // Parses XML structure into file objects
        private static List<FileJob> ParseXML(XmlDocument document, FileCollection filecollection)
        {
            XmlNodeList nodes = document.GetElementsByTagName("file");
            List<FileJob> files = new List<FileJob>();
            Random random = new Random();
            int collectionID = random.Next();
            int fileId = 0;
            // Parsing for each file node
            foreach (XmlElement fileNode in nodes)
            {
                FileJob filejob = new FileJob(ref filecollection);
                filejob.filename = ""+collectionID;
                XmlAttributeCollection collection = fileNode.Attributes;

                // File Attributes
                foreach (XmlAttribute attr in collection)
                {
                    switch (attr.Name)
                    {
                        case "poster":
                            filejob.poster = attr.Value;
                            break;
                        case "date":
                            filejob.date = int.Parse(attr.Value);
                            break;
                        case "subject":
                            if (attr.Value.Contains("\""))
                            {
                                // try to get inner filename...
                                try
                                {
                                    int first = attr.Value.IndexOf("\"") + 1;
                                    string filename = attr.Value.Substring(first, attr.Value.Length - first);
                                    filename = filename.Substring(0, filename.IndexOf("\""));
                                    filejob.filename = filejob.filename + "-"+ filename.Substring(0, filename.LastIndexOf(".")+4);
                                }
                                catch (Exception ex)
                                {
                                    filejob.filename = filejob.filename+"-"+fileId;
                                }

                            }
                            else
                                filejob.filename = filejob.filename + "-" + fileId;

                            filejob.subject = attr.Value;
                            break;
                    }
                }
                int maxnumber = 0;
                foreach (XmlElement subNode in fileNode.ChildNodes)
                {
                    if (subNode.Name == "segments")
                    {
                        foreach (XmlElement segmentNode in subNode)
                        {
                            Segment segment = new Segment();
                            segment.addr = "<"+segmentNode.InnerXml+">";
                            segment.addr = segment.addr.Replace("&amp;", "&");
                            segment.setParent(ref filejob);
                            foreach (XmlAttribute attr in segmentNode.Attributes)
                            {
                                switch (attr.Name)
                                {
                                    case "bytes":
                                        segment.bytes = int.Parse(attr.Value);
                                        filejob.size += (ulong)segment.bytes;
                                        break;
                                    case "number":
                                        segment.id = int.Parse(attr.Value);
                                        if (segment.id > maxnumber)
                                            maxnumber = segment.id;
                                        break;
                                }
                            }

                            // Check and add element to arraylist
                            if (segment.id != 0 && segment.bytes != 0 && segment.addr.Length > 10)
                                filejob.segments.Add(segment);
                        }
                    } else if (subNode.Name == "groups")
                    {
                        filejob.groups.Add(subNode.FirstChild.InnerText);
                    }
                }
                filejob.yparts = maxnumber; // Will fail if source isnt yenc
                // Sort segments
                filejob.segments.Sort(new SegmentComparer());

                files.Add(filejob);
                fileId++;
            }
            return files;
        }