fyne-widgets/pkg/bufferedvid/bufferedvid.go

package bufferedvid

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"image"
	"image/color"
	"math/rand"
	"os"
	"time"

	"fyne.io/fyne/v2"
	"fyne.io/fyne/v2/canvas"
	"fyne.io/fyne/v2/theme"
	"fyne.io/fyne/v2/widget"
	"github.com/zergon321/reisen"
)

func staticNoiseImage(w, h int) *image.RGBA {
	i := image.NewRGBA(image.Rect(0, 0, w, h))
	for x := 0; x < w; x++ {
		for y := 0; y < h; y++ {
			lum := uint8(rand.Float32() * 255)
			i.Set(x, y, color.RGBA{lum, lum, lum, 255})
		}
	}
	return i
}

const (
	width            = 1280
	height           = 720
	frameBufferSize  = 1024
	sampleRate       = 44100
	channelCount     = 2
	bitDepth         = 8
	sampleBufferSize = 32 * channelCount * bitDepth * 1024
)

// player holds all the data
// necessary for playing video.
type player struct {
	pix         *image.NRGBA
	ticker      <-chan time.Time
	errs        <-chan error
	frameBuffer <-chan *image.RGBA
	audioBuffer <-chan [2]float64
	last        time.Time
	fps         int
	paused      bool
	frameCount  int64
}

// readVideoAndAudio reads video and audio frames
// from the opened media and sends the decoded
// data to che channels to be played.
func (p *player) readVideoAndAudio(media *reisen.Media) (<-chan *image.RGBA, <-chan [2]float64, chan error, error) {
	frameBuffer := make(chan *image.RGBA, frameBufferSize)
	sampleBuffer := make(chan [2]float64, sampleBufferSize)
	errs := make(chan error)

	err := media.OpenDecode()

	if err != nil {
		return nil, nil, nil, err
	}

	videoStream := media.VideoStreams()[0]
	err = videoStream.Open()
	p.frameCount = videoStream.FrameCount()

	if err != nil {
		return nil, nil, nil, err
	}

	audioStream := media.AudioStreams()[0]
	err = audioStream.Open()

	if err != nil {
		return nil, nil, nil, err
	}

	go func() {
		for {
			packet, gotPacket, err := media.ReadPacket()

			if err != nil {
				go func(err error) {
					errs <- err
				}(err)
			}

			if !gotPacket {
				break
			}

			switch packet.Type() {
			case reisen.StreamVideo:
				s := media.Streams()[packet.StreamIndex()].(*reisen.VideoStream)
				videoFrame, gotFrame, err := s.ReadVideoFrame()

				if err != nil {
					go func(err error) {
						errs <- err
					}(err)
				}

				if !gotFrame {
					break
				}

				if videoFrame == nil {
					continue
				}

				frameBuffer <- videoFrame.Image()

			case reisen.StreamAudio:
				s := media.Streams()[packet.StreamIndex()].(*reisen.AudioStream)
				audioFrame, gotFrame, err := s.ReadAudioFrame()

				if err != nil {
					go func(err error) {
						errs <- err
					}(err)
				}

				if !gotFrame {
					break
				}

				if audioFrame == nil {
					continue
				}

				// Turn the raw byte data into
				// audio samples of type [2]float64.
				reader := bytes.NewReader(audioFrame.Data())

				// See the README.md file for
				// detailed scheme of the sample structure.
				for reader.Len() > 0 {
					sample := [2]float64{0, 0}
					var result float64
					err = binary.Read(reader, binary.LittleEndian, &result)

					if err != nil {
						go func(err error) {
							errs <- err
						}(err)
					}

					sample[0] = result

					err = binary.Read(reader, binary.LittleEndian, &result)

					if err != nil {
						go func(err error) {
							errs <- err
						}(err)
					}

					sample[1] = result
					sampleBuffer <- sample
				}
			}
		}

		videoStream.Close()
		audioStream.Close()
		media.CloseDecode()
		close(frameBuffer)
		close(sampleBuffer)
		close(errs)
	}()

	return frameBuffer, sampleBuffer, errs, nil
}

// Starts reading samples and frames
// of the media file.
func (p *player) open(fname string) error {
	// Sprite for drawing video frames.
	p.pix = image.NewNRGBA(image.Rect(0, 0, width, height))

	// Open the media file.
	media, err := reisen.NewMedia(fname)

	if err != nil {
		fmt.Printf("Error %s\n", err.Error())
		return err
	}

	// Get the FPS for playing
	// video frames.
	videoFPS, _ := media.Streams()[0].FrameRate()
	fmt.Printf("Detected FPS of %d\n", videoFPS)
	if videoFPS == 0 {
		fmt.Println("Setting fps to 60 as no FPS found")
		videoFPS = 60
	}
	if videoFPS > 150 {
		fmt.Printf("FPS of %d seems way too high, assuming it's *1000, so we'll use %d\n", videoFPS, videoFPS/1000)
		videoFPS = videoFPS / 1000
	}
	p.fps = videoFPS

	if err != nil {
		fmt.Printf("Error %s\n", err.Error())
		return err
	}

	// SPF for frame ticker.
	spf := 1.0 / float64(videoFPS)
	frameDuration, err := time.
		ParseDuration(fmt.Sprintf("%fs", spf))

	if err != nil {
		fmt.Printf("Error %s\n", err.Error())
		return err
	}

	// Start decoding streams.
	p.frameBuffer, p.audioBuffer,
		p.errs, err = p.readVideoAndAudio(media)

	if err != nil {
		return err
	}

	p.ticker = time.Tick(frameDuration)

	// Setup metrics.
	p.last = time.Now()
	return nil
}

var _ fyne.WidgetRenderer = (*bufferedVidPlaybackRenderer)(nil)

type VideoControlsVisibleStruct struct {
	PlayButton  bool
	PauseButton bool
	Scrubber    bool
}

type BufferedVidPlayback struct {
	widget.BaseWidget
	UpdateFPS            int64
	VideoFilename        string
	VideoControlsVisible VideoControlsVisibleStruct
	Loop                 bool

	fpsTimer       *time.Ticker
	videoOpened    bool
	playerStruct   player
	currentFrameID int64
	allFrames      []image.RGBA
	bufferFilling  bool
	paused         bool
	audioProcessor func([2]float64)
}

func NewBufferedVidPlayback() *BufferedVidPlayback {
	w := &BufferedVidPlayback{}
	w.ExtendBaseWidget(w)
	w.UpdateFPS = 25
	w.videoOpened = false
	w.currentFrameID = 0
	w.bufferFilling = false
	w.paused = false
	w.Loop = false

	return w
}

func fileIsOpenable(path string) bool {
	_, err := os.Open(path)
	if err != nil {
		return false
	}
	return true
}

func (w *BufferedVidPlayback) Play() {
	if !w.videoOpened {
		if fileIsOpenable(w.VideoFilename) {
			w.playerStruct.open(w.VideoFilename)
			w.fpsTimer.Stop()
			spf := 1.0 / float64(w.playerStruct.fps)
			frameDuration, err := time.
				ParseDuration(fmt.Sprintf("%fs", spf))

			if err != nil {
				fmt.Printf("Error %s\n", err.Error())
			}
			w.fpsTimer.Reset(frameDuration)
			go func(p player) {
				for {
					w.processAudioFrame(<-w.playerStruct.audioBuffer)
				}
			}(w.playerStruct)
			w.FillBuffer()
			w.videoOpened = true
		}
	}
	if w.paused {
		w.paused = false
	}
	w.BaseWidget.Refresh()
}

func (w *BufferedVidPlayback) Pause() {
	if !w.videoOpened || !w.bufferFilling {
		if !w.paused {
			w.paused = true
		}
	}
}

func (w *BufferedVidPlayback) UnPause() {
	if !w.videoOpened || !w.bufferFilling {
		if w.paused {
			w.paused = false
		}
	}
}

func (w *BufferedVidPlayback) IsPaused() bool {
	return w.paused
}

func (w *BufferedVidPlayback) Resize(s fyne.Size) {
	w.BaseWidget.Resize(s)
}

func (w *BufferedVidPlayback) FillBuffer() {
	w.allFrames = make([]image.RGBA, w.playerStruct.frameCount)
	w.bufferFilling = true
	w.currentFrameID = 0
	for {
		frameImage := <-w.playerStruct.frameBuffer
		w.allFrames[w.currentFrameID] = *frameImage
		w.currentFrameID = w.currentFrameID + 1
		if w.currentFrameID > w.playerStruct.frameCount-2 {
			w.currentFrameID = 0
			w.bufferFilling = false
			return
		}
	}
}

func (w *BufferedVidPlayback) CreateRenderer() fyne.WidgetRenderer {
	w.fpsTimer = time.NewTicker(time.Duration(1000/25) * time.Millisecond)
	go func(v *BufferedVidPlayback) {
		for {
			_ = <-w.fpsTimer.C
			v.Refresh()
		}
	}(w)
	return newBufferedVidPlaybackRenderer(w)
}

func (w *BufferedVidPlayback) SetAudioProcessorFunc(f func(data [2]float64)) {
	w.audioProcessor = f
}

func (w *BufferedVidPlayback) processAudioFrame(data [2]float64) {
	if w.audioProcessor != nil {
		w.audioProcessor(data)
	} else {
		_ = data
	}
}

type bufferedVidPlaybackRenderer struct {
	bufferedVidPlayback *BufferedVidPlayback
	currentframe        *canvas.Raster
	progress            *widget.Slider
	bufferingProgress   *widget.ProgressBar
	pausebtn, playbtn   *widget.Button
}

func (r *bufferedVidPlaybackRenderer) actualrenderframe(w, h int) image.Image {
	var frameImage *image.RGBA
	r.bufferingProgress.Hide()
	if r.bufferedVidPlayback.bufferFilling {
		r.bufferingProgress.Show()
		r.bufferingProgress.Min = 0
		r.bufferingProgress.Max = float64(r.bufferedVidPlayback.playerStruct.frameCount)
		r.bufferingProgress.Value = float64(r.bufferedVidPlayback.currentFrameID)
		frameImage = staticNoiseImage(w, h)
	} else if r.bufferedVidPlayback.videoOpened {
		frameImage = &r.bufferedVidPlayback.allFrames[r.bufferedVidPlayback.currentFrameID]
		if !r.bufferedVidPlayback.paused {
			r.bufferedVidPlayback.currentFrameID = r.bufferedVidPlayback.currentFrameID + 1
			if r.bufferedVidPlayback.currentFrameID > r.bufferedVidPlayback.playerStruct.frameCount-5 { //definitely stopping before the end
				r.bufferedVidPlayback.currentFrameID = 0
				if !r.bufferedVidPlayback.Loop {
					r.bufferedVidPlayback.paused = true
				}
			}
		}
	} else {
		frameImage = staticNoiseImage(w, h) //no point doing anything other than returning anyway!
	}
	return frameImage
}

func (r *bufferedVidPlaybackRenderer) seek(frame float64) {
	if r.bufferedVidPlayback.videoOpened && !r.bufferedVidPlayback.bufferFilling {
		r.bufferedVidPlayback.currentFrameID = int64(frame)
	}
}

func newBufferedVidPlaybackRenderer(w *BufferedVidPlayback) *bufferedVidPlaybackRenderer {
	renderer := &bufferedVidPlaybackRenderer{
		bufferedVidPlayback: w,
	}
	renderer.currentframe = canvas.NewRaster(renderer.actualrenderframe)
	renderer.progress = widget.NewSlider(0, 1)
	renderer.progress.OnChanged = renderer.seek
	renderer.playbtn = widget.NewButtonWithIcon("", theme.MediaPlayIcon(), func() { w.Play() })
	renderer.pausebtn = widget.NewButtonWithIcon("", theme.MediaPauseIcon(), func() {
		if w.paused {
			w.UnPause()
		} else {
			w.Pause()
		}
	})
	renderer.bufferingProgress = widget.NewProgressBar()
	renderer.bufferingProgress.TextFormatter = func() string {
		if w.playerStruct.frameCount > 0 {
			return fmt.Sprintf("Buffering (%2.2f%%)...", float64(w.currentFrameID)/float64(w.playerStruct.frameCount)*100)
		}
		return "Buffering..."
	}
	return renderer
}

func (r *bufferedVidPlaybackRenderer) Objects() []fyne.CanvasObject {
	// The order is critical, rect is drawn first then currentframe
	return []fyne.CanvasObject{r.currentframe, r.bufferingProgress, r.playbtn, r.pausebtn, r.progress}
}

func (r *bufferedVidPlaybackRenderer) Layout(s fyne.Size) {
	videoControlsMaxMin := fyne.Max(r.progress.MinSize().Height, r.pausebtn.MinSize().Height)
	videoControlsY := s.Height - videoControlsMaxMin
	r.currentframe.Resize(fyne.NewSize(s.Width, videoControlsY))
	r.currentframe.Move(fyne.NewPos(0, 0))
	r.bufferingProgress.Resize(fyne.NewSize(s.Width, r.bufferingProgress.MinSize().Height))
	r.bufferingProgress.Move(fyne.NewPos(0, (s.Height/2)-(r.bufferingProgress.Size().Height)))
	r.playbtn.Resize(r.playbtn.MinSize())
	r.playbtn.Move(fyne.NewPos(0, videoControlsY))
	if fyne.Max(r.progress.MinSize().Height, r.playbtn.MinSize().Height) == r.progress.MinSize().Height {
		r.playbtn.Resize(fyne.NewSize(r.playbtn.Size().Width, r.progress.Size().Height))
	}
	r.pausebtn.Resize(r.pausebtn.MinSize())
	r.pausebtn.Move(fyne.NewPos(r.pausebtn.MinSize().Width, videoControlsY))
	if fyne.Max(r.progress.MinSize().Height, r.pausebtn.MinSize().Height) == r.progress.MinSize().Height {
		r.pausebtn.Resize(fyne.NewSize(r.pausebtn.Size().Width, r.progress.Size().Height))
	}
	r.progress.Resize(fyne.NewSize(s.Width-r.playbtn.MinSize().Width-r.pausebtn.MinSize().Width, r.progress.MinSize().Height))
	r.progress.Move(fyne.NewPos(r.playbtn.MinSize().Width+r.pausebtn.MinSize().Width, videoControlsY))
	if fyne.Max(r.progress.MinSize().Height, r.pausebtn.MinSize().Height) == r.pausebtn.MinSize().Height {
		r.progress.Resize(fyne.NewSize(r.progress.Size().Width, r.pausebtn.Size().Height))
	}
}

func (r *bufferedVidPlaybackRenderer) MinSize() fyne.Size {
	return fyne.NewSize(200, 200)
}

func (r *bufferedVidPlaybackRenderer) Refresh() {
	r.currentframe.Refresh()
	r.progress.Min = 0
	r.progress.Max = float64(r.bufferedVidPlayback.playerStruct.frameCount)
	if r.bufferedVidPlayback.bufferFilling {
		r.progress.Value = 0
	} else {
		r.progress.Value = float64(r.bufferedVidPlayback.currentFrameID)
	}
	r.bufferingProgress.Refresh()
	r.currentframe.Refresh()
	r.progress.Refresh()
	r.pausebtn.Refresh()
	r.playbtn.Refresh()
}

func (r *bufferedVidPlaybackRenderer) Destroy() {
	r.bufferedVidPlayback.fpsTimer.Stop()
} // Called when the renderer is destroyed