Use faster Y flip of the video encoders (OpenGL/N64)

giongto35 · Sep 29, 2023 · c1c4731 · c1c4731
1 parent a901c84
commit c1c4731
Show file tree

Hide file tree

Showing 8 changed files with 29 additions and 3 deletions.
diff --git a/pkg/worker/caged/libretro/frontend.go b/pkg/worker/caged/libretro/frontend.go
@@ -24,6 +24,7 @@ type Emulator interface {
 	LoadCore(name string)
 	LoadGame(path string) error
 	FPS() int
+	Flipped() bool
 	AudioSampleRate() int
 	IsPortrait() bool
 	// Start is called after LoadGame
@@ -239,6 +240,7 @@ func (f *Frontend) Start() {
 	}
 }
 
+func (f *Frontend) Flipped() bool                 { return f.nano.IsGL() }
 func (f *Frontend) FrameSize() (int, int)         { return f.nano.GeometryBase() }
 func (f *Frontend) FPS() int                      { return f.nano.VideoFramerate() }
 func (f *Frontend) HashPath() string              { return f.storage.GetSavePath() }

diff --git a/pkg/worker/caged/libretro/nanoarch/nanoarch.go b/pkg/worker/caged/libretro/nanoarch/nanoarch.go
@@ -251,7 +251,7 @@ func (n *Nanoarch) LoadGame(path string) error {
 	n.stopped.Store(false)
 
 	if n.Video.gl.enabled {
-		setRotation(image.F180) // flip Y coordinates of OpenGL
+		//setRotation(image.F180) // flip Y coordinates of OpenGL
 		bufS := uint(n.sysAvInfo.geometry.max_width*n.sysAvInfo.geometry.max_height) * n.Video.BPP
 		graphics.SetBuffer(int(bufS))
 		n.log.Info().Msgf("Set buffer: %v", byteCountBinary(int64(bufS)))
@@ -351,6 +351,7 @@ func (n *Nanoarch) Run() {
 	}
 }
 
+func (n *Nanoarch) IsGL() bool      { return n.Video.gl.enabled }
 func (n *Nanoarch) IsStopped() bool { return n.stopped.Load() }
 
 func videoSetPixelFormat(format uint32) (C.bool, error) {

diff --git a/pkg/worker/coordinatorhandlers.go b/pkg/worker/coordinatorhandlers.go
@@ -112,6 +112,9 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke
 			app.Close()
 			return api.EmptyPacket
 		}
+		if app.Flipped() {
+			m.SetVideoFlip(true)
+		}
 
 		// make the room
 		r = room.NewRoom[*room.GameSession](uid, app, w.router.Users(), m)

diff --git a/pkg/worker/encoder/encoder.go b/pkg/worker/encoder/encoder.go
@@ -16,6 +16,7 @@ type (
 		LoadBuf(input []byte)
 		Encode() []byte
 		IntraRefresh()
+		SetFlip(bool)
 		Shutdown() error
 	}
 )
@@ -65,6 +66,8 @@ func (vp *VideoEncoder) Encode(img InFrame) OutFrame {
 	return nil
 }
 
+func (vp *VideoEncoder) SetFlip(b bool) { vp.encoder.SetFlip(b) }
+
 func (vp *VideoEncoder) Stop() {
 	vp.stopped.Store(true)
 	vp.mu.Lock()

diff --git a/pkg/worker/encoder/h264/libx264.go b/pkg/worker/encoder/h264/libx264.go
@@ -44,7 +44,8 @@ type Nal struct {
 const RcCrf = 1
 
 const (
-	CspI420 = 0x0002 // yuv 4:2:0 planar
+	CspI420  = 0x0002 // yuv 4:2:0 planar
+	CspVflip = 0x1000 /* the csp is vertically flipped */
 
 	// CspMask      = 0x00ff /* */
 	// CspNone      = 0x0000 /* Invalid mode     */
@@ -65,7 +66,6 @@ const (
 	//CspBgra      = 0x000f /* packed bgr 32bits */
 	//CspRgb       = 0x0010 /* packed rgb 24bits */
 	//CspMax       = 0x0011 /* end of list */
-	//CspVflip     = 0x1000 /* the csp is vertically flipped */
 	//CspHighDepth = 0x2000 /* the csp has a depth of 16 bits per pixel component */
 )
 

diff --git a/pkg/worker/encoder/h264/x264.go b/pkg/worker/encoder/h264/x264.go
@@ -139,6 +139,14 @@ func (e *H264) IntraRefresh() {
 	// !to implement
 }
 
+func (e *H264) SetFlip(b bool) {
+	if b {
+		e.in.Img.ICsp |= CspVflip
+	} else {
+		e.in.Img.ICsp &= ^CspVflip
+	}
+}
+
 func (e *H264) Shutdown() error {
 	e.y = nil
 	e.u = nil

diff --git a/pkg/worker/encoder/vpx/libvpx.go b/pkg/worker/encoder/vpx/libvpx.go
@@ -84,8 +84,11 @@ type Vpx struct {
 	image      C.vpx_image_t
 	codecCtx   C.vpx_codec_ctx_t
 	kfi        C.int
+	flipped    bool
 }
 
+func (vpx *Vpx) SetFlip(b bool) { vpx.flipped = b }
+
 type Options struct {
 	// Target bandwidth to use for this stream, in kilobits per second.
 	Bitrate uint
@@ -134,6 +137,9 @@ func NewEncoder(w, h int, opts *Options) (*Vpx, error) {
 
 func (vpx *Vpx) LoadBuf(yuv []byte) {
 	C.vpx_img_read(&vpx.image, unsafe.Pointer(&yuv[0]))
+	if vpx.flipped {
+		C.vpx_img_flip(&vpx.image)
+	}
 }
 
 // Encode encodes yuv image with the VPX8 encoder.
@@ -166,5 +172,6 @@ func (vpx *Vpx) Shutdown() error {
 	C.vpx_img_free(&vpx.image)
 	//}
 	C.vpx_codec_destroy(&vpx.codecCtx)
+	vpx.flipped = false
 	return nil
 }
diff --git a/pkg/worker/media/media.go b/pkg/worker/media/media.go
@@ -189,3 +189,5 @@ func (wmp *WebrtcMediaPipe) initVideo(w, h int, conf config.Video) error {
 }
 
 func (wmp *WebrtcMediaPipe) ProcessVideo(v app.Video) []byte { return wmp.enc.Encode(&v.Frame) }
+
+func (wmp *WebrtcMediaPipe) SetVideoFlip(b bool) { wmp.enc.SetFlip(b) }