From f08e523f864317f8cd3db75da39a94d63548c988 Mon Sep 17 00:00:00 2001 From: Martyn Date: Sun, 12 Dec 2021 22:23:30 +0100 Subject: [PATCH] Buffered video player --- examples/bufferedvid/main.go | 36 +++ pkg/bufferedvid/bufferedvid.go | 480 +++++++++++++++++++++++++++++++++ 2 files changed, 516 insertions(+) create mode 100644 examples/bufferedvid/main.go create mode 100644 pkg/bufferedvid/bufferedvid.go diff --git a/examples/bufferedvid/main.go b/examples/bufferedvid/main.go new file mode 100644 index 0000000..d73105e --- /dev/null +++ b/examples/bufferedvid/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "fyne.io/fyne/v2" + "fyne.io/fyne/v2/app" + "fyne.io/fyne/v2/container" + "fyne.io/fyne/v2/widget" + "git.martyn.berlin/martyn/fyne-widgets/pkg/bufferedvid" + "git.martyn.berlin/martyn/fyne-widgets/pkg/layouts" +) + +var videoWidget *bufferedvid.BufferedVidPlayback + +func unimplemented() {} + +func main() { + a := app.New() + w := a.NewWindow("Video") + + videoWidget = bufferedvid.NewBufferedVidPlayback() + videoWidget.VideoFilename = "1 Minute Timer-CH50zuS8DD0.mp4" + layout := layouts.NewFloatingControlsLayout() + layout.FloatingControlsLocation = layouts.FloatingControlsCenter + + buttons := container.NewHBox(widget.NewButton(">", func() { videoWidget.Play() }), widget.NewButton("||", func() { + if videoWidget.IsPaused() { + videoWidget.UnPause() + } else { + videoWidget.Pause() + } + })) + + w.SetContent(container.New(&layout, videoWidget, buttons)) + w.Resize(fyne.NewSize(640, 480)) + w.ShowAndRun() +} diff --git a/pkg/bufferedvid/bufferedvid.go b/pkg/bufferedvid/bufferedvid.go new file mode 100644 index 0000000..bc60544 --- /dev/null +++ b/pkg/bufferedvid/bufferedvid.go @@ -0,0 +1,480 @@ +package bufferedvid + +import ( + "bytes" + "encoding/binary" + "fmt" + "image" + "image/color" + "math/rand" + "os" + "time" + + "fyne.io/fyne/v2" + "fyne.io/fyne/v2/canvas" + "fyne.io/fyne/v2/theme" + "fyne.io/fyne/v2/widget" + "github.com/zergon321/reisen" +) + +func staticNoiseImage(w, h int) *image.RGBA { + i := image.NewRGBA(image.Rect(0, 0, w, h)) + for x := 0; x < w; x++ { + for y := 0; y < h; y++ { + lum := uint8(rand.Float32() * 255) + i.Set(x, y, color.RGBA{lum, lum, lum, 255}) + } + } + return i +} + +const ( + width = 1280 + height = 720 + frameBufferSize = 1024 + sampleRate = 44100 + channelCount = 2 + bitDepth = 8 + sampleBufferSize = 32 * channelCount * bitDepth * 1024 +) + +// player holds all the data +// necessary for playing video. +type player struct { + pix *image.NRGBA + ticker <-chan time.Time + errs <-chan error + frameBuffer <-chan *image.RGBA + audioBuffer <-chan [2]float64 + last time.Time + fps int + paused bool + frameCount int64 +} + +// readVideoAndAudio reads video and audio frames +// from the opened media and sends the decoded +// data to che channels to be played. +func (p *player) readVideoAndAudio(media *reisen.Media) (<-chan *image.RGBA, <-chan [2]float64, chan error, error) { + frameBuffer := make(chan *image.RGBA, frameBufferSize) + sampleBuffer := make(chan [2]float64, sampleBufferSize) + errs := make(chan error) + + err := media.OpenDecode() + + if err != nil { + return nil, nil, nil, err + } + + videoStream := media.VideoStreams()[0] + err = videoStream.Open() + p.frameCount = videoStream.FrameCount() + + if err != nil { + return nil, nil, nil, err + } + + audioStream := media.AudioStreams()[0] + err = audioStream.Open() + + if err != nil { + return nil, nil, nil, err + } + + go func() { + for { + packet, gotPacket, err := media.ReadPacket() + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + if !gotPacket { + break + } + + switch packet.Type() { + case reisen.StreamVideo: + s := media.Streams()[packet.StreamIndex()].(*reisen.VideoStream) + videoFrame, gotFrame, err := s.ReadVideoFrame() + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + if !gotFrame { + break + } + + if videoFrame == nil { + continue + } + + frameBuffer <- videoFrame.Image() + + case reisen.StreamAudio: + s := media.Streams()[packet.StreamIndex()].(*reisen.AudioStream) + audioFrame, gotFrame, err := s.ReadAudioFrame() + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + if !gotFrame { + break + } + + if audioFrame == nil { + continue + } + + // Turn the raw byte data into + // audio samples of type [2]float64. + reader := bytes.NewReader(audioFrame.Data()) + + // See the README.md file for + // detailed scheme of the sample structure. + for reader.Len() > 0 { + sample := [2]float64{0, 0} + var result float64 + err = binary.Read(reader, binary.LittleEndian, &result) + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + sample[0] = result + + err = binary.Read(reader, binary.LittleEndian, &result) + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + sample[1] = result + sampleBuffer <- sample + } + } + } + + videoStream.Close() + audioStream.Close() + media.CloseDecode() + close(frameBuffer) + close(sampleBuffer) + close(errs) + }() + + return frameBuffer, sampleBuffer, errs, nil +} + +// Starts reading samples and frames +// of the media file. +func (p *player) open(fname string) error { + // Sprite for drawing video frames. + p.pix = image.NewNRGBA(image.Rect(0, 0, width, height)) + + // Open the media file. + media, err := reisen.NewMedia(fname) + + if err != nil { + fmt.Printf("Error %s\n", err.Error()) + return err + } + + // Get the FPS for playing + // video frames. + videoFPS, _ := media.Streams()[0].FrameRate() + fmt.Printf("Detected FPS of %d\n", videoFPS) + if videoFPS == 0 { + fmt.Println("Setting fps to 60 as no FPS found") + videoFPS = 60 + } + if videoFPS > 150 { + fmt.Printf("FPS of %d seems way too high, assuming it's *1000, so we'll use %d\n", videoFPS, videoFPS/1000) + videoFPS = videoFPS / 1000 + } + p.fps = videoFPS + + if err != nil { + fmt.Printf("Error %s\n", err.Error()) + return err + } + + // SPF for frame ticker. + spf := 1.0 / float64(videoFPS) + frameDuration, err := time. + ParseDuration(fmt.Sprintf("%fs", spf)) + + if err != nil { + fmt.Printf("Error %s\n", err.Error()) + return err + } + + // Start decoding streams. + p.frameBuffer, p.audioBuffer, + p.errs, err = p.readVideoAndAudio(media) + + if err != nil { + return err + } + + p.ticker = time.Tick(frameDuration) + + // Setup metrics. + p.last = time.Now() + return nil +} + +var _ fyne.WidgetRenderer = (*bufferedVidPlaybackRenderer)(nil) + +type VideoControlsVisibleStruct struct { + PlayButton bool + PauseButton bool + Scrubber bool +} + +type BufferedVidPlayback struct { + widget.BaseWidget + UpdateFPS int64 + VideoFilename string + VideoControlsVisible VideoControlsVisibleStruct + Loop bool + + fpsTimer *time.Ticker + videoOpened bool + playerStruct player + currentFrameID int64 + allFrames []image.RGBA + bufferFilling bool + paused bool +} + +func NewBufferedVidPlayback() *BufferedVidPlayback { + w := &BufferedVidPlayback{} + w.ExtendBaseWidget(w) + w.UpdateFPS = 25 + w.videoOpened = false + w.currentFrameID = 0 + w.bufferFilling = false + w.paused = false + w.Loop = false + + return w +} + +func fileIsOpenable(path string) bool { + _, err := os.Open(path) + if err != nil { + return false + } + return true +} + +func (w *BufferedVidPlayback) Play() { + if !w.videoOpened { + if fileIsOpenable(w.VideoFilename) { + w.playerStruct.open(w.VideoFilename) + w.fpsTimer.Stop() + spf := 1.0 / float64(w.playerStruct.fps) + frameDuration, err := time. + ParseDuration(fmt.Sprintf("%fs", spf)) + + if err != nil { + fmt.Printf("Error %s\n", err.Error()) + } + w.fpsTimer.Reset(frameDuration) + go func() { + // empty the audio buffer in case that's what's jamming things up. + for { + _ = <-w.playerStruct.audioBuffer + } + }() + w.FillBuffer() + w.videoOpened = true + } + } + if w.paused { + w.paused = false + } + w.BaseWidget.Refresh() +} + +func (w *BufferedVidPlayback) Pause() { + if !w.videoOpened || !w.bufferFilling { + if !w.paused { + w.paused = true + } + } +} + +func (w *BufferedVidPlayback) UnPause() { + if !w.videoOpened || !w.bufferFilling { + if w.paused { + w.paused = false + } + } +} + +func (w *BufferedVidPlayback) IsPaused() bool { + return w.paused +} + +func (w *BufferedVidPlayback) Resize(s fyne.Size) { + w.BaseWidget.Resize(s) +} + +func (w *BufferedVidPlayback) FillBuffer() { + w.allFrames = make([]image.RGBA, w.playerStruct.frameCount) + w.bufferFilling = true + w.currentFrameID = 0 + for { + frameImage := <-w.playerStruct.frameBuffer + w.allFrames[w.currentFrameID] = *frameImage + w.currentFrameID = w.currentFrameID + 1 + if w.currentFrameID > w.playerStruct.frameCount-2 { + w.currentFrameID = 0 + w.bufferFilling = false + return + } + } +} + +func (w *BufferedVidPlayback) CreateRenderer() fyne.WidgetRenderer { + w.fpsTimer = time.NewTicker(time.Duration(1000/25) * time.Millisecond) + go func(v *BufferedVidPlayback) { + for { + _ = <-w.fpsTimer.C + v.Refresh() + } + }(w) + return newBufferedVidPlaybackRenderer(w) +} + +type bufferedVidPlaybackRenderer struct { + bufferedVidPlayback *BufferedVidPlayback + currentframe *canvas.Raster + progress *widget.Slider + bufferingProgress *widget.ProgressBar + pausebtn, playbtn *widget.Button +} + +func (r *bufferedVidPlaybackRenderer) actualrenderframe(w, h int) image.Image { + var frameImage *image.RGBA + r.bufferingProgress.Hide() + if r.bufferedVidPlayback.bufferFilling { + r.bufferingProgress.Show() + r.bufferingProgress.Min = 0 + r.bufferingProgress.Max = float64(r.bufferedVidPlayback.playerStruct.frameCount) + r.bufferingProgress.Value = float64(r.bufferedVidPlayback.currentFrameID) + frameImage = staticNoiseImage(w, h) + } else if r.bufferedVidPlayback.videoOpened { + frameImage = &r.bufferedVidPlayback.allFrames[r.bufferedVidPlayback.currentFrameID] + if !r.bufferedVidPlayback.paused { + r.bufferedVidPlayback.currentFrameID = r.bufferedVidPlayback.currentFrameID + 1 + if r.bufferedVidPlayback.currentFrameID > r.bufferedVidPlayback.playerStruct.frameCount-5 { //definitely stopping before the end + r.bufferedVidPlayback.currentFrameID = 0 + if !r.bufferedVidPlayback.Loop { + r.bufferedVidPlayback.paused = true + } + } + } + } else { + frameImage = staticNoiseImage(w, h) //no point doing anything other than returning anyway! + } + return frameImage +} + +func (r *bufferedVidPlaybackRenderer) seek(frame float64) { + if r.bufferedVidPlayback.videoOpened && !r.bufferedVidPlayback.bufferFilling { + r.bufferedVidPlayback.currentFrameID = int64(frame) + } +} + +func newBufferedVidPlaybackRenderer(w *BufferedVidPlayback) *bufferedVidPlaybackRenderer { + renderer := &bufferedVidPlaybackRenderer{ + bufferedVidPlayback: w, + } + renderer.currentframe = canvas.NewRaster(renderer.actualrenderframe) + renderer.progress = widget.NewSlider(0, 1) + renderer.progress.OnChanged = renderer.seek + renderer.playbtn = widget.NewButtonWithIcon("", theme.MediaPlayIcon(), func() { w.Play() }) + renderer.pausebtn = widget.NewButtonWithIcon("", theme.MediaPauseIcon(), func() { + if w.paused { + w.UnPause() + } else { + w.Pause() + } + }) + renderer.bufferingProgress = widget.NewProgressBar() + renderer.bufferingProgress.TextFormatter = func() string { + if w.playerStruct.frameCount > 0 { + return fmt.Sprintf("Buffering (%2.2f%%)...", float64(w.currentFrameID)/float64(w.playerStruct.frameCount)*100) + } + return "Buffering..." + } + return renderer +} + +func (r *bufferedVidPlaybackRenderer) Objects() []fyne.CanvasObject { + // The order is critical, rect is drawn first then currentframe + return []fyne.CanvasObject{r.currentframe, r.bufferingProgress, r.playbtn, r.pausebtn, r.progress} +} + +func (r *bufferedVidPlaybackRenderer) Layout(s fyne.Size) { + videoControlsMaxMin := fyne.Max(r.progress.MinSize().Height, r.pausebtn.MinSize().Height) + videoControlsY := s.Height - videoControlsMaxMin + r.currentframe.Resize(fyne.NewSize(s.Width, videoControlsY)) + r.currentframe.Move(fyne.NewPos(0, 0)) + r.bufferingProgress.Resize(fyne.NewSize(s.Width, r.bufferingProgress.MinSize().Height)) + r.bufferingProgress.Move(fyne.NewPos(0, (s.Height/2)-(r.bufferingProgress.Size().Height))) + r.playbtn.Resize(r.playbtn.MinSize()) + r.playbtn.Move(fyne.NewPos(0, videoControlsY)) + if fyne.Max(r.progress.MinSize().Height, r.playbtn.MinSize().Height) == r.progress.MinSize().Height { + r.playbtn.Resize(fyne.NewSize(r.playbtn.Size().Width, r.progress.Size().Height)) + } + r.pausebtn.Resize(r.pausebtn.MinSize()) + r.pausebtn.Move(fyne.NewPos(r.pausebtn.MinSize().Width, videoControlsY)) + if fyne.Max(r.progress.MinSize().Height, r.pausebtn.MinSize().Height) == r.progress.MinSize().Height { + r.pausebtn.Resize(fyne.NewSize(r.pausebtn.Size().Width, r.progress.Size().Height)) + } + r.progress.Resize(fyne.NewSize(s.Width-r.playbtn.MinSize().Width-r.pausebtn.MinSize().Width, r.progress.MinSize().Height)) + r.progress.Move(fyne.NewPos(r.playbtn.MinSize().Width+r.pausebtn.MinSize().Width, videoControlsY)) + if fyne.Max(r.progress.MinSize().Height, r.pausebtn.MinSize().Height) == r.pausebtn.MinSize().Height { + r.progress.Resize(fyne.NewSize(r.progress.Size().Width, r.pausebtn.Size().Height)) + } +} + +func (r *bufferedVidPlaybackRenderer) MinSize() fyne.Size { + return fyne.NewSize(200, 200) +} + +func (r *bufferedVidPlaybackRenderer) Refresh() { + r.currentframe.Refresh() + r.progress.Min = 0 + r.progress.Max = float64(r.bufferedVidPlayback.playerStruct.frameCount) + if r.bufferedVidPlayback.bufferFilling { + r.progress.Value = 0 + } else { + r.progress.Value = float64(r.bufferedVidPlayback.currentFrameID) + } + r.bufferingProgress.Refresh() + r.currentframe.Refresh() + r.progress.Refresh() + r.pausebtn.Refresh() + r.playbtn.Refresh() +} + +func (r *bufferedVidPlaybackRenderer) Destroy() { + r.bufferedVidPlayback.fpsTimer.Stop() +} // Called when the renderer is destroyed