/// <summary> /// 向 client 发送要处理的直播间信息 /// </summary> /// <param name="client"></param> /// <param name="room"></param> private static void SendTask(CrawlerClient client, StreamRoom room) { room.StartTime = DateTime.Now; room.Clients.Add(client.Name); client.CurrentJobs.Add(room); WebsocketServer.SendString(client.WebSocketContext, JsonConvert.SerializeObject(new Command { Type = CommandType.Issue, Room = room })); Console.WriteLine($@"下发 {room.Roomid} 给 {client.Name}"); }
/// <summary> /// 在一定时间后重试 /// </summary> /// <param name="time"></param> /// <param name="room"></param> private static void RetryRoomAfter(TimeSpan time, StreamRoom room) { if (++room.RetryTime >= 3) { return; } room.RetryAfter = DateTime.Now + time; RetryQueue.Add(room); }
private static void Ws_OnMessage(object sender, MessageEventArgs e) { if (!e.IsText) { return; } var command = JsonConvert.DeserializeObject <Command>(e.Data); if (command.Type != CommandType.Issue) { return; } StreamRoom streamRoom = command.Room; Console.WriteLine("New task: " + streamRoom.Roomid); StreamRooms.Add(streamRoom); Task.Run(() => StreamParser.Parse(streamRoom)) .ContinueWith((task) => { StreamRooms.Remove(streamRoom); if (task.IsFaulted) { string error = task.Exception.ToString(); Console.WriteLine("ERROR: " + streamRoom.Roomid + " " + task.Exception.InnerException.Message); WebSocket.Send(JsonConvert.SerializeObject(new Command { Type = CommandType.CompleteFailed, Room = streamRoom, Error = error })); } else if (task.IsCanceled) { Console.WriteLine("ERROR: GetAsync Timed Out"); WebSocket.Send(JsonConvert.SerializeObject(new Command { Type = CommandType.CompleteFailed, Room = streamRoom, Error = "GetAsync Timed Out" })); } else { StreamMetadata data = task.Result; Console.WriteLine("Success: " + streamRoom.Roomid); WebSocket.Send(JsonConvert.SerializeObject(new Command { Type = CommandType.CompleteSuccess, Room = streamRoom, Metadata = data })); } }); }
/// <summary> /// 重新给直播间分配一个新 client /// </summary> /// <param name="room"></param> private static void RetryRoom(StreamRoom room) { if (++room.RetryTime >= 3) { return; } var client = ConnectedClient.FirstOrDefault(x => x.MaxParallelTask > x.CurrentJobs.Count); if (client == null) { RoomQueue.AddFirst(room); } else { ConnectedClient.Remove(client); ConnectedClient.Add(client); SendTask(client, room); } }
/// <summary> /// 把收集到的数据写入数据库 /// </summary> /// <param name="name"></param> /// <param name="room"></param> /// <param name="metadata"></param> private static void WriteResult(string name, StreamRoom room, StreamMetadata metadata) { Exception exception = null; try { var roominfo = JsonConvert.SerializeObject(room); var onmetadata = JsonConvert.SerializeObject(metadata.FlvMetadata); using (var connection = new MySqlConnection(Config.MySql)) { int result = connection.Execute("INSERT INTO data(`roomid`,`clientname`,`roominfo`," + "`flvhost`,`height`,`width`,`fps`,`encoder`,`video_datarate`,`audio_datarate`," + "`profile`,`level`,`size`,`onmetadata`,`avc_dcr`) VALUES " + "(@Roomid,@name,@roominfo,@FlvHost,@Height,@Width,@Fps,@Encoder," + "@VideoDatarate,@AudioDatarate,@Profile,@Level,@TotalSize,@onmetadata,@AVCDecoderConfigurationRecord)", new { room.Roomid, name, roominfo, metadata.FlvHost, metadata.Height, metadata.Width, metadata.Fps, metadata.Encoder, metadata.VideoDatarate, metadata.AudioDatarate, metadata.Profile, metadata.Level, metadata.TotalSize, onmetadata, metadata.AVCDecoderConfigurationRecord }); } } catch (Exception ex) { exception = ex; } TelegramMessage .Append(DateTime.Now.ToString("HH:mm:ss.f")) .Append("\n") .Append(name) .Append(" #success ") .Append(room.Roomid) .Append("\n") .Append(metadata.Width) .Append("x") .Append(metadata.Height) .Append(" ") .Append(metadata.Fps) .Append("\nV: ") .Append(metadata.VideoDatarate) .Append(" A: ") .Append(metadata.AudioDatarate) .Append(" P: ") .Append(metadata.Profile) .Append(" L: ") .Append(metadata.Level) .Append("\nE: ") .Append(metadata.Encoder); if (exception != null) { TelegramMessage .Append("\n写数据库错误"); Console.WriteLine("写数据库错误 " + exception.ToString()); } TelegramMessage.Append("\n\n"); }
public static async Task <StreamMetadata> Parse(StreamRoom room) { StreamMetadata streamMetadata = new StreamMetadata(); IFlvStreamProcessor Processor = new FlvStreamProcessor(null, (byte[] data) => new FlvMetadata(data), () => new FlvTag()) .Initialize(null, null, EnabledFeature.ClipOnly, AutoCuttingMode.Disabled); Stream _stream = null; HttpResponseMessage _response = null; var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(2)); bool success = false; bool avc = false; Processor.TagProcessed += (sender, e) => { IFlvTag t = e.Tag; if (t.TimeStamp > 10 * 1000) { success = true; try { cancellationTokenSource.Cancel(); } catch (Exception) { } return; } if (!avc && t.IsVideoKeyframe && t.Profile != -1) { avc = true; streamMetadata.Profile = t.Profile; streamMetadata.Level = t.Level; streamMetadata.AVCDecoderConfigurationRecord = t.Data; } if (t.TagType == TagType.VIDEO) { streamMetadata.TotalSize += t.TagSize; } }; try { using (var client = new HttpClient()) { client.Timeout = TimeSpan.FromMinutes(2); client.DefaultRequestHeaders.Accept.Clear(); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*")); client.DefaultRequestHeaders.UserAgent.Clear(); client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"); client.DefaultRequestHeaders.Referrer = new Uri("https://live.bilibili.com"); client.DefaultRequestHeaders.Add("Origin", "https://live.bilibili.com"); string flv_path = GetPlayUrl(room.Roomid); streamMetadata.FlvHost = new Uri(flv_path).Host; _response = await client.GetAsync(flv_path, HttpCompletionOption.ResponseHeadersRead, cancellationTokenSource.Token); } if (_response.StatusCode != HttpStatusCode.OK) { throw new Exception("StatusCode: " + _response.StatusCode); } else { _stream = await _response.Content.ReadAsStreamAsync(); await _ReadStreamLoop(); if (!success) { throw new Exception("Timeout"); } { var metadata = Processor.Metadata.Meta.Where(x => x.Key.Replace("\0", "").Length != 0).ToDictionary( x => x.Key.Replace("\0", ""), x => (x.Value is string str) ? str.Replace("\0", "") : x.Value ); streamMetadata.Width = metadata.ContainsKey("width") ? ((metadata["width"] is int w) ? w : int.Parse(metadata["width"].ToString())) : (metadata.ContainsKey("displayWidth") ? ((metadata["displayWidth"] is int dw) ? dw : int.Parse(metadata["displayWidth"].ToString())) : -1); streamMetadata.Height = metadata.ContainsKey("height") ? ((metadata["height"] is int h) ? h : int.Parse(metadata["height"].ToString())) : (metadata.ContainsKey("displayHeight") ? ((metadata["displayHeight"] is int dh) ? dh : int.Parse(metadata["displayHeight"].ToString())) : -1); streamMetadata.Fps = metadata.ContainsKey("framerate") ? ((metadata["framerate"] is int f) ? f : int.Parse(metadata["framerate"].ToString())) : (metadata.ContainsKey("fps") ? ((metadata["fps"] is int df) ? df : int.Parse(metadata["fps"].ToString())) : -1); streamMetadata.VideoDatarate = metadata.ContainsKey("videodatarate") ? ((metadata["videodatarate"] is int vdr) ? vdr : int.Parse(metadata["videodatarate"].ToString())) : -1; streamMetadata.AudioDatarate = metadata.ContainsKey("audiodatarate") ? ((metadata["audiodatarate"] is int adr) ? adr : int.Parse(metadata["audiodatarate"].ToString())) : -1; streamMetadata.Encoder = metadata.ContainsKey("encoder") ? metadata["encoder"].ToString() : ""; streamMetadata.FlvMetadata = metadata; } } } finally { _CleanupFlvRequest(); } return(streamMetadata); async Task _ReadStreamLoop() { try { const int BUF_SIZE = 1024 * 8; byte[] buffer = new byte[BUF_SIZE]; while (!cancellationTokenSource.Token.IsCancellationRequested) { int bytesRead = await _stream.ReadAsync(buffer, 0, BUF_SIZE, cancellationTokenSource.Token); if (bytesRead != 0) { if (bytesRead != BUF_SIZE) { Processor.AddBytes(buffer.Take(bytesRead).ToArray()); } else { Processor.AddBytes(buffer); } } else { break; } } } catch (Exception e) { if (e is ObjectDisposedException && cancellationTokenSource.Token.IsCancellationRequested) { return; } throw; } } void _CleanupFlvRequest() { if (Processor != null) { Processor.FinallizeFile(); Processor.Dispose(); Processor = null; } _stream?.Dispose(); _stream = null; _response?.Dispose(); _response = null; } }