Add the initial mouse support

pkg/api/user.go

@@ -12,8 +12,9 @@ type (
 		PlayerIndex int    `json:"player_index"`
 	}
 	GameStartUserResponse struct {
-		RoomId string        `json:"roomId"`
-		Av     *AppVideoInfo `json:"av"`
+		RoomId  string        `json:"roomId"`
+		Av      *AppVideoInfo `json:"av"`
+		KbMouse bool          `json:"kb_mouse"`
 	}
 	IceServer struct {
 		Urls       string `json:"urls,omitempty"`

pkg/api/worker.go

@@ -33,8 +33,9 @@ type (
 	}
 	StartGameResponse struct {
 		Room
-		AV     *AppVideoInfo `json:"av"`
-		Record bool
+		AV      *AppVideoInfo `json:"av"`
+		Record  bool          `json:"record"`
+		KbMouse bool          `json:"kb_mouse"`
 	}
 	RecordGameRequest[T Id] struct {
 		StatefulRoom[T]

pkg/config/config.yaml

@@ -186,6 +186,7 @@ emulator:
       #       A list of device IDs to bind to the input ports.
       #       Some cores allow binding multiple devices to a single port (DosBox), but typically,
       #       you should bind just one device to one port.
+      #   - kbMouseSupport (bool) -- (temp) a flag if the core needs the keyboard and mouse on the client
       #   - vfr (bool)
       #    (experimental)
       #       Enable variable frame rate only for cores that can't produce a constant frame rate.

pkg/config/emulator.go

@@ -47,6 +47,7 @@ type LibretroCoreConfig struct {
 	Height          int
 	Hid             map[int][]int
 	IsGlAllowed     bool
+	KbMouseSupport  bool
 	Lib             string
 	Options         map[string]string
 	Roms            []string

pkg/coordinator/userapi.go

@@ -37,6 +37,6 @@ func (u *User) SendWebrtcOffer(sdp string) { u.Notify(api.WebrtcOffer, sdp) }
 func (u *User) SendWebrtcIceCandidate(candidate string) { u.Notify(api.WebrtcIce, candidate) }
 
 // StartGame signals the user that everything is ready to start a game.
-func (u *User) StartGame(av *api.AppVideoInfo) {
-	u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av})
+func (u *User) StartGame(av *api.AppVideoInfo, kbMouse bool) {
+	u.Notify(api.StartGame, api.GameStartUserResponse{RoomId: u.w.RoomId, Av: av, KbMouse: kbMouse})
 }

pkg/coordinator/userhandlers.go

@@ -56,7 +56,7 @@ func (u *User) HandleStartGame(rq api.GameStartUserRequest, launcher games.Launc
 		return
 	}
 	u.log.Info().Str("id", startGameResp.Rid).Msg("Received room response from worker")
-	u.StartGame(startGameResp.AV)
+	u.StartGame(startGameResp.AV, startGameResp.KbMouse)
 
 	// send back recording status
 	if conf.Recording.Enabled && rq.Record {

pkg/network/webrtc/webrtc.go

@@ -17,6 +17,7 @@ type Peer struct {
 	log        *logger.Logger
 	OnMessage  func(data []byte)
 	OnKeyboard func(data []byte)
+	OnMouse    func(data []byte)
 
 	a *webrtc.TrackLocalStaticSample
 	v *webrtc.TrackLocalStaticSample
@@ -109,6 +110,17 @@ func (p *Peer) NewCall(vCodec, aCodec string, onICECandidate func(ice any)) (sdp
 	})
 	p.log.Debug().Msg("Added [keyboard] chan")
 
+	mChan, err := p.addDataChannel("mouse")
+	if err != nil {
+		return "", err
+	}
+	mChan.OnMessage(func(m webrtc.DataChannelMessage) {
+		if p.OnMouse != nil {
+			p.OnMouse(m.Data)
+		}
+	})
+	p.log.Debug().Msg("Added [mouse] chan")
+
 	p.conn.OnICEConnectionStateChange(p.handleICEState(func() { p.log.Info().Msg("Connected") }))
 	// Stream provider supposes to send offer
 	offer, err := p.conn.CreateOffer(nil)

pkg/worker/caged/app/app.go

@@ -15,6 +15,8 @@ type App interface {
 	SetDataCb(func([]byte))
 	InputGamepad(port int, data []byte)
 	InputKeyboard(port int, data []byte)
+	InputMouse(port int, data []byte)
+	KbMouseSupport() bool
 }
 
 type Audio struct {

pkg/worker/caged/libretro/caged.go

@@ -88,6 +88,8 @@ func (c *Caged) ViewportSize() (int, int)            { return c.base.ViewportSiz
 func (c *Caged) Scale() float64                      { return c.Emulator.Scale() }
 func (c *Caged) InputGamepad(port int, data []byte)  { c.base.Input(port, RetroPad, data) }
 func (c *Caged) InputKeyboard(port int, data []byte) { c.base.Input(port, Keyboard, data) }
+func (c *Caged) InputMouse(port int, data []byte)    { c.base.Input(port, Mouse, data) }
+func (c *Caged) KbMouseSupport() bool                { return c.base.KbMouseSupport() }
 func (c *Caged) Start()                              { go c.Emulator.Start() }
 func (c *Caged) SetSaveOnClose(v bool)               { c.base.SaveOnClose = v }
 func (c *Caged) SetSessionId(name string)            { c.base.SetSessionId(name) }

pkg/worker/caged/libretro/frontend.go

@@ -73,6 +73,7 @@ type Device byte
 const (
 	RetroPad = Device(nanoarch.RetroPad)
 	Keyboard = Device(nanoarch.Keyboard)
+	Mouse    = Device(nanoarch.Mouse)
 )
 
 var (
@@ -151,6 +152,7 @@ func (f *Frontend) LoadCore(emu string) {
 		Options:         conf.Options,
 		UsesLibCo:       conf.UsesLibCo,
 		CoreAspectRatio: conf.CoreAspectRatio,
+		KbMouseSupport:  conf.KbMouseSupport,
 	}
 	f.mu.Lock()
 	scale := 1.0
@@ -278,6 +280,7 @@ func (f *Frontend) FrameSize() (int, int)         { return f.nano.BaseWidth(), f
 func (f *Frontend) HasSave() bool                 { return os.Exists(f.HashPath()) }
 func (f *Frontend) HashPath() string              { return f.storage.GetSavePath() }
 func (f *Frontend) IsPortrait() bool              { return f.nano.IsPortrait() }
+func (f *Frontend) KbMouseSupport() bool          { return f.nano.KbMouseSupport() }
 func (f *Frontend) LoadGame(path string) error    { return f.nano.LoadGame(path) }
 func (f *Frontend) PixFormat() uint32             { return f.nano.Video.PixFmt.C }
 func (f *Frontend) RestoreGameState() error       { return f.Load() }
@@ -300,6 +303,8 @@ func (f *Frontend) Input(port int, d Device, data []byte) {
 		f.nano.InputRetropad(port, data)
 	case Keyboard:
 		f.nano.InputKeyboard(port, data)
+	case Mouse:
+		f.nano.InputMouse(port, data)
 	}
 }
 

pkg/worker/caged/libretro/nanoarch/input.go

@@ -14,14 +14,13 @@ const (
 	Pressed
 )
 
-const RetroDeviceTypeShift = 8
-
 // InputState stores full controller state.
 // It consists of:
 //   - uint16 button values
 //   - int16 analog stick values
+type InputState [maxPort]RetroPadState
+
 type (
-	InputState    [maxPort]RetroPadState
 	RetroPadState struct {
 		keys uint32
 		axes [dpadAxes]int32
@@ -31,13 +30,31 @@ type (
 		mod  uint16
 		mu   sync.Mutex
 	}
+	MouseState struct {
+		dx, dy  atomic.Int32
+		buttons atomic.Int32
+	}
 )
 
+type MouseBtnState int32
+
 type Device byte
 
 const (
 	RetroPad Device = iota
 	Keyboard
+	Mouse
+)
+
+const (
+	MouseMove = iota
+	MouseButton
+)
+
+const (
+	MouseLeft MouseBtnState = 1 << iota
+	MouseRight
+	MouseMiddle
 )
 
 const (
@@ -66,24 +83,40 @@ func (s *InputState) IsDpadTouched(port uint, axis uint) (shift C.int16_t) {
 	return C.int16_t(atomic.LoadInt32(&s[port].axes[axis]))
 }
 
-func RetroDeviceSubclass(base, id int) int { return ((id + 1) << RetroDeviceTypeShift) | base }
-
 func NewKeyboardState() KeyboardState {
 	return KeyboardState{
 		keys: make(map[uint]struct{}),
 		mod:  0,
 	}
 }
 
-func (ks *KeyboardState) Set(press bool, key uint, mod uint16) {
+// SetKey sets keyboard state.
+//
+//	0 1 2 3 4 5 6
+//	[ KEY ] P MOD
+//
+//	KEY contains Libretro code of the keyboard key (4 bytes).
+//	P contains 0 or 1 if the key is pressed (1 byte).
+//	MOD contains bitmask for Alt | Ctrl | Meta | Shift keys press state (2 bytes).
+//
+// Returns decoded state from the input bytes.
+func (ks *KeyboardState) SetKey(data []byte) (pressed bool, key uint, mod uint16) {
+	if len(data) != 7 {
+		return
+	}
+
+	pressed = data[4] == 1
+	key = uint(binary.BigEndian.Uint32(data))
+	mod = binary.BigEndian.Uint16(data[5:])
 	ks.mu.Lock()
-	if press {
+	if pressed {
 		ks.keys[key] = struct{}{}
 	} else {
 		delete(ks.keys, key)
 	}
 	ks.mod = mod
 	ks.mu.Unlock()
+	return
 }
 
 func (ks *KeyboardState) Pressed(key uint) C.int16_t {
@@ -96,9 +129,32 @@ func (ks *KeyboardState) Pressed(key uint) C.int16_t {
 	return Released
 }
 
-func decodeKeyboardState(data []byte) (bool, uint, uint16) {
-	press := data[4] == 1
-	key := uint(binary.LittleEndian.Uint32(data))
-	mod := binary.LittleEndian.Uint16(data[5:])
-	return press, key, mod
+// ShiftPos sets mouse relative position state.
+//
+//	0  1 2  3
+//	[dx] [dy]
+//
+//	dx and dy are relative mouse coordinates
+func (ms *MouseState) ShiftPos(data []byte) {
+	if len(data) != 4 {
+		return
+	}
+	dx := int16(data[0])<<8 + int16(data[1])
+	dy := int16(data[2])<<8 + int16(data[3])
+	ms.dx.Add(int32(dx))
+	ms.dy.Add(int32(dy))
+}
+
+func (ms *MouseState) PopX() C.int16_t { return C.int16_t(ms.dx.Swap(0)) }
+func (ms *MouseState) PopY() C.int16_t { return C.int16_t(ms.dy.Swap(0)) }
+
+// SetButtons sets the state MouseBtnState of mouse buttons.
+func (ms *MouseState) SetButtons(data byte) { ms.buttons.Store(int32(data)) }
+
+func (ms *MouseState) Buttons() (l, r, m bool) {
+	mbs := MouseBtnState(ms.buttons.Load())
+	l = mbs&MouseLeft != 0
+	r = mbs&MouseRight != 0
+	m = mbs&MouseMiddle != 0
+	return
 }

pkg/worker/caged/libretro/nanoarch/input_test.go

@@ -1,6 +1,7 @@
 package nanoarch
 
 import (
+	"encoding/binary"
 	"math/rand"
 	"sync"
 	"testing"
@@ -19,3 +20,54 @@ func TestConcurrentInput(t *testing.T) {
 	}
 	wg.Wait()
 }
+
+func TestMousePos(t *testing.T) {
+	data := []byte{0, 0, 0, 0}
+
+	dx := 1111
+	dy := 2222
+
+	binary.BigEndian.PutUint16(data, uint16(dx))
+	binary.BigEndian.PutUint16(data[2:], uint16(dy))
+
+	ms := MouseState{}
+	ms.ShiftPos(data)
+
+	x := int(ms.PopX())
+	y := int(ms.PopY())
+
+	if x != dx || y != dy {
+		t.Errorf("invalid state, %v = %v, %v = %v", dx, x, dy, y)
+	}
+
+	if ms.dx.Load() != 0 || ms.dy.Load() != 0 {
+		t.Errorf("coordinates weren't cleared")
+	}
+}
+
+func TestMouseButtons(t *testing.T) {
+	tests := []struct {
+		name string
+		data byte
+		l    bool
+		r    bool
+		m    bool
+	}{
+		{name: "l+r+m+", data: 1 + 2 + 4, l: true, r: true, m: true},
+		{name: "l-r-m-", data: 0},
+		{name: "l-r+m-", data: 2, r: true},
+		{name: "l+r-m+", data: 1 + 4, l: true, m: true},
+	}
+
+	ms := MouseState{}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ms.SetButtons(test.data)
+			l, r, m := ms.Buttons()
+			if l != test.l || r != test.r || m != test.m {
+				t.Errorf("wrong button state: %v -> %v, %v, %v", test.data, l, r, m)
+			}
+		})
+	}
+}

pkg/worker/caged/libretro/nanoarch/nanoarch.go

@@ -402,7 +402,7 @@ func (n *Nanoarch) InputMouse(_ int, data []byte) {
 	case MouseMove:
 		n.mouse.ShiftPos(state)
 	case MouseButton:
-		n.mouse.SetButtons(state)
+		n.mouse.SetButtons(state[0])
 	}
 }
 

pkg/worker/coordinatorhandlers.go

@@ -126,12 +126,14 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke
 				}
 			}
 
-			data, err := api.Wrap(api.Out{T: uint8(api.AppVideoChange), Payload: api.AppVideoInfo{
-				W: m.VideoW,
-				H: m.VideoH,
-				A: app.AspectRatio(),
-				S: int(app.Scale()),
-			}})
+			data, err := api.Wrap(api.Out{
+				T: uint8(api.AppVideoChange),
+				Payload: api.AppVideoInfo{
+					W: m.VideoW,
+					H: m.VideoH,
+					A: app.AspectRatio(),
+					S: int(app.Scale()),
+				}})
 			if err != nil {
 				c.log.Error().Err(err).Msgf("wrap")
 			}
@@ -173,10 +175,15 @@ func (c *coordinator) HandleGameStart(rq api.StartGameRequest[com.Uid], w *Worke
 	s := room.WithWebRTC(user.Session)
 	s.OnMessage = func(data []byte) { r.App().InputGamepad(user.Index, data) }
 	s.OnKeyboard = func(data []byte) { r.App().InputKeyboard(user.Index, data) }
+	s.OnMouse = func(data []byte) { r.App().InputMouse(user.Index, data) }
 
 	c.RegisterRoom(r.Id())
 
-	response := api.StartGameResponse{Room: api.Room{Rid: r.Id()}, Record: w.conf.Recording.Enabled}
+	response := api.StartGameResponse{
+		Room:    api.Room{Rid: r.Id()},
+		Record:  w.conf.Recording.Enabled,
+		KbMouse: r.App().KbMouseSupport(),
+	}
 	if r.App().AspectEnabled() {
 		ww, hh := r.App().ViewportSize()
 		response.AV = &api.AppVideoInfo{W: ww, H: hh, A: r.App().AspectRatio(), S: int(r.App().Scale())}