diff --git a/examples/vidplayback/main.go b/examples/vidplayback/main.go new file mode 100644 index 0000000..4dabb22 --- /dev/null +++ b/examples/vidplayback/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "fyne.io/fyne/v2" + "fyne.io/fyne/v2/app" + "fyne.io/fyne/v2/container" + "fyne.io/fyne/v2/widget" + "git.martyn.berlin/martyn/fyne-widgets/pkg/layouts" + "git.martyn.berlin/martyn/fyne-widgets/pkg/vidplayback" +) + +var videoWidget *vidplayback.VidPlayback + +func unimplemented() {} + +func main() { + a := app.New() + w := a.NewWindow("Video") + + videoWidget = vidplayback.NewVidPlayback() + videoWidget.VideoFilename = "network.mp4" + layout := layouts.NewFloatingControlsLayout() + layout.FloatingControlsLocation = layouts.FloatingControlsCenter + + buttons := container.NewHBox(widget.NewButton(">", func() { videoWidget.Play() }), widget.NewButton("[]", unimplemented)) + + w.SetContent(container.New(&layout, videoWidget, buttons)) + w.Resize(fyne.NewSize(640, 480)) + w.ShowAndRun() +} diff --git a/examples/vidplayback/network.mp4 b/examples/vidplayback/network.mp4 new file mode 100644 index 0000000..539bfc7 Binary files /dev/null and b/examples/vidplayback/network.mp4 differ diff --git a/pkg/vidplayback/vidplayback.go b/pkg/vidplayback/vidplayback.go new file mode 100644 index 0000000..9a62116 --- /dev/null +++ b/pkg/vidplayback/vidplayback.go @@ -0,0 +1,347 @@ +package vidplayback + +import ( + "bytes" + "encoding/binary" + "fmt" + "image" + "image/color" + "image/draw" + "log" + "math/rand" + "time" + + "fyne.io/fyne/v2" + "fyne.io/fyne/v2/canvas" + "fyne.io/fyne/v2/widget" + "github.com/zergon321/reisen" +) + +var _ fyne.WidgetRenderer = (*vidPlaybackRenderer)(nil) + +const ( + width = 1280 + height = 720 + frameBufferSize = 1024 + sampleRate = 44100 + channelCount = 2 + bitDepth = 8 + sampleBufferSize = 32 * channelCount * bitDepth * 1024 +) + +// player holds all the data +// necessary for playing video. +type player struct { + pix *image.NRGBA + ticker <-chan time.Time + errs <-chan error + frameBuffer <-chan *image.RGBA + last time.Time + deltaTime float64 + paused bool + wid *VidPlayback +} + +func (p *player) readVideoAndAudio(media *reisen.Media) (<-chan *image.RGBA, <-chan [2]float64, chan error, error) { + frameBuffer := make(chan *image.RGBA, frameBufferSize) + sampleBuffer := make(chan [2]float64, sampleBufferSize) + errs := make(chan error) + + err := media.OpenDecode() + + if err != nil { + return nil, nil, nil, err + } + + videoStream := media.VideoStreams()[0] + err = videoStream.Open() + + if err != nil { + return nil, nil, nil, err + } + + audioStream := media.AudioStreams()[0] + err = audioStream.Open() + + if err != nil { + return nil, nil, nil, err + } + + go func() { + for { + packet, gotPacket, err := media.ReadPacket() + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + if !gotPacket { + break + } + + switch packet.Type() { + case reisen.StreamVideo: + s := media.Streams()[packet.StreamIndex()].(*reisen.VideoStream) + videoFrame, gotFrame, err := s.ReadVideoFrame() + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + if !gotFrame { + break + } + + if videoFrame == nil { + continue + } + + frameBuffer <- videoFrame.Image() + + case reisen.StreamAudio: + s := media.Streams()[packet.StreamIndex()].(*reisen.AudioStream) + audioFrame, gotFrame, err := s.ReadAudioFrame() + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + if !gotFrame { + break + } + + if audioFrame == nil { + continue + } + + // Turn the raw byte data into + // audio samples of type [2]float64. + reader := bytes.NewReader(audioFrame.Data()) + + // See the README.md file for + // detailed scheme of the sample structure. + for reader.Len() > 0 { + sample := [2]float64{0, 0} + var result float64 + err = binary.Read(reader, binary.LittleEndian, &result) + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + sample[0] = result + + err = binary.Read(reader, binary.LittleEndian, &result) + + if err != nil { + go func(err error) { + errs <- err + }(err) + } + + sample[1] = result + sampleBuffer <- sample + } + } + } + + videoStream.Close() + audioStream.Close() + media.CloseDecode() + close(frameBuffer) + close(sampleBuffer) + close(errs) + }() + + return frameBuffer, sampleBuffer, errs, nil +} + +// Starts reading samples and frames +// of the media file. +func (p *player) open(fname string) error { + fmt.Printf("Opening %s...\n", fname) + // Sprite for drawing video frames. + p.pix = image.NewNRGBA(image.Rect(0, 0, width, height)) + + // Open the media file. + media, err := reisen.NewMedia(fname) + + if err != nil { + fmt.Printf("Open error %s\n", err.Error()) + return err + } + + // Get the FPS for playing + // video frames. + videoFPS, _ := media.Streams()[0].FrameRate() + fmt.Printf("FPS %d\n", videoFPS) + if videoFPS == 0 { + fmt.Printf("Assuming 60fps as FPS was 0.\n") + videoFPS = 60 + } + + // SPF for frame ticker. + spf := 1.0 / float64(videoFPS) + frameDuration, err := time. + ParseDuration(fmt.Sprintf("%fs", spf)) + + if err != nil { + fmt.Printf("Error %s\n", err.Error()) + return err + } + + // Start decoding streams. + var sampleSource <-chan [2]float64 + p.frameBuffer, sampleSource, + p.errs, err = p.readVideoAndAudio(media) + + if err != nil { + fmt.Printf("Error %s\n", err.Error()) + return err + } + + // Start playing audio samples. + //speaker.Play(p.streamSamples(sampleSource)) + // nooope. + _ = sampleSource + + p.ticker = time.Tick(frameDuration) + + // Setup metrics. + p.last = time.Now() + return nil +} + +func (p *player) update(screen *canvas.Image) error { + // Compute dt. + p.deltaTime = time.Since(p.last).Seconds() + p.last = time.Now() + + // Check for incoming errors. + select { + case err, ok := <-p.errs: + if ok { + fmt.Printf("Error %s\n", err.Error()) + return err + } + + default: + } + + // Read video frames and draw them. + select { + case <-p.ticker: + frame, ok := <-p.frameBuffer + + if ok { + p.pix.Pix = frame.Pix + screen.Refresh() + fmt.Print(".") + p.wid.Refresh() + } + + default: + } + + return nil +} + +type VidPlayback struct { + widget.BaseWidget + VideoFilename string + + play player +} + +func NewVidPlayback() *VidPlayback { + v := &VidPlayback{} + v.play.wid = v + return v +} + +func (v *VidPlayback) Resize(s fyne.Size) { + v.BaseWidget.Resize(s) +} + +func (v *VidPlayback) CreateRenderer() fyne.WidgetRenderer { + return newVidPlaybackRenderer(v) +} + +func (v *VidPlayback) Play() { + _ = v.play.open(v.VideoFilename) + v.play.paused = false +} + +type vidPlaybackRenderer struct { + vidPlayback *VidPlayback + currentframe *canvas.Image +} + +func newVidPlaybackRenderer(v *VidPlayback) *vidPlaybackRenderer { + renderer := &vidPlaybackRenderer{ + vidPlayback: v, + } + i := image.NewRGBA(image.Rect(0, 0, 640, 480)) + for x := 0; x < 640; x++ { + for y := 0; y < 480; y++ { + lum := uint8(rand.Float32() * 255) + i.Set(x, y, color.RGBA{lum, lum, lum, 255}) + } + } + renderer.currentframe = canvas.NewImageFromImage(i) + go func() { + for { + if v.play.paused { + time.Sleep(time.Millisecond * 8) + continue + } + + err := v.play.update(renderer.currentframe) + if err != nil { + log.Println("Error playing:", err) + } + } + }() + return renderer +} + +func (r *vidPlaybackRenderer) Objects() []fyne.CanvasObject { + return []fyne.CanvasObject{r.currentframe} +} + +func (r *vidPlaybackRenderer) Layout(s fyne.Size) { + r.currentframe.Resize(s) +} + +func (r *vidPlaybackRenderer) MinSize() fyne.Size { + return fyne.NewSize(200, 200) +} + +func (r *vidPlaybackRenderer) Refresh() { + + b := r.currentframe.Image.Bounds() + fmt.Print("*") + m := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy())) + draw.Draw(m, m.Bounds(), r.currentframe.Image, b.Min, draw.Src) + + for x := 0; x < b.Dx(); x++ { + for y := 0; y < b.Dy(); y++ { + lum := uint8(rand.Float32() * 255) + m.Set(x, y, color.RGBA{lum, lum, lum, 255}) + } + } + fmt.Print("+") + r.currentframe.Image = m + r.currentframe.Refresh() +} + +func (r *vidPlaybackRenderer) Destroy() { +} // Called when the renderer is destroyed