From 5679e3c0d75ff83c4e9214d251c4259d759ae260 Mon Sep 17 00:00:00 2001 From: Lukas Herman Date: Thu, 26 Dec 2019 19:08:03 -0800 Subject: [PATCH] Add SelectSettings algorithm and Codec to video constraints --- const.go | 9 +++++ examples/simple/main.go | 9 ++++- mediadevices.go | 71 +++++++++++++++++++++++++++++++++------ mediastreamconstraints.go | 31 ++++++++++++++++- pkg/driver/manager.go | 14 ++++++-- track.go | 7 ++-- 6 files changed, 121 insertions(+), 20 deletions(-) create mode 100644 const.go diff --git a/const.go b/const.go new file mode 100644 index 0000000..1ca0116 --- /dev/null +++ b/const.go @@ -0,0 +1,9 @@ +package mediadevices + +import "github.com/pion/webrtc/v2" + +type Codec string + +const ( + CodecH264 = webrtc.H264 +) diff --git a/examples/simple/main.go b/examples/simple/main.go index 1c53e49..a88a99a 100644 --- a/examples/simple/main.go +++ b/examples/simple/main.go @@ -31,7 +31,14 @@ func main() { mediaDevices := mediadevices.NewMediaDevices(peerConnection) - s, err := mediaDevices.GetUserMedia(mediadevices.MediaStreamConstraints{}) + s, err := mediaDevices.GetUserMedia(mediadevices.MediaStreamConstraints{ + Video: mediadevices.VideoTrackConstraints{ + Enabled: true, + Width: 800, // This is just an ideal value + Height: 480, // This is just an ideal value + Codec: mediadevices.CodecH264, // This is default, you may omit this + }, + }) if err != nil { panic(err) } diff --git a/mediadevices.go b/mediadevices.go index 425df15..d86d7cd 100644 --- a/mediadevices.go +++ b/mediadevices.go @@ -1,6 +1,9 @@ package mediadevices import ( + "fmt" + "math" + "github.com/pion/mediadevices/pkg/driver" "github.com/pion/webrtc/v2" ) @@ -20,23 +23,69 @@ type mediaDevices struct { func (m *mediaDevices) GetUserMedia(constraints MediaStreamConstraints) (MediaStream, error) { // TODO: It should return media stream based on constraints - d := driver.Manager.Query()[0] - err := d.Open() - if err != nil { - return nil, err - } - vd := d.(driver.VideoDriver) - spec := vd.Specs()[0] + trackers := make([]tracker, 0) + if constraints.Video.Enabled { + tracker, err := m.videoSelect(constraints.Video) + if err != nil { + return nil, err + } - tracker, err := newVideoTrack(m.pc, vd, spec, webrtc.H264) - if err != nil { - return nil, err + trackers = append(trackers, tracker) } - s, err := NewMediaStream(tracker) + s, err := NewMediaStream(trackers...) if err != nil { return nil, err } return s, nil } + +func (m *mediaDevices) videoSelect(constraints VideoTrackConstraints) (tracker, error) { + videoFilterFn := driver.FilterKind(driver.Video) + drivers := driver.Manager.Query(videoFilterFn) + + var bestDriver driver.VideoDriver + var bestSpec driver.VideoSpec + minFitnessDist := math.Inf(1) + + for _, d := range drivers { + wasClosed := d.Status() == driver.StateClosed + + if wasClosed { + err := d.Open() + if err != nil { + // Skip this driver if we failed to open because we can't get the specs + continue + } + } + + vd := d.(driver.VideoDriver) + for _, spec := range vd.Specs() { + fitnessDist := constraints.fitnessDistance(spec) + + if fitnessDist < minFitnessDist { + minFitnessDist = fitnessDist + bestDriver = vd + bestSpec = spec + } + } + + if wasClosed { + // Since it was closed, we should close it to avoid a leak + d.Close() + } + } + + if bestDriver == nil { + return nil, fmt.Errorf("failed to find the best setting") + } + + if bestDriver.Status() == driver.StateClosed { + err := bestDriver.Open() + if err != nil { + return nil, fmt.Errorf("failed in opening the best video driver") + } + } + return newVideoTrack(m.pc, bestDriver, bestSpec, constraints.Codec) +} diff --git a/mediastreamconstraints.go b/mediastreamconstraints.go index 2ffb5ed..5511be1 100644 --- a/mediastreamconstraints.go +++ b/mediastreamconstraints.go @@ -1,8 +1,37 @@ package mediadevices +import "github.com/pion/mediadevices/pkg/driver" + +import "math" + type MediaStreamConstraints struct { Audio MediaTrackConstraints - Video MediaTrackConstraints + Video VideoTrackConstraints } type MediaTrackConstraints bool + +type VideoTrackConstraints struct { + Enabled bool + Width, Height int + Codec Codec +} + +// fitnessDistance is an implementation for https://w3c.github.io/mediacapture-main/#dfn-fitness-distance +func (c *VideoTrackConstraints) fitnessDistance(s driver.VideoSpec) float64 { + var dist float64 + + if s.Width != c.Width { + actualWidth := float64(s.Width) + idealWidth := float64(c.Width) + dist += math.Abs(actualWidth-idealWidth) / math.Max(math.Abs(actualWidth), math.Abs(idealWidth)) + } + + if s.Height != c.Height { + actualHeight := float64(s.Height) + idealHeight := float64(c.Height) + dist += math.Abs(actualHeight-idealHeight) / math.Max(math.Abs(actualHeight), math.Abs(idealHeight)) + } + + return dist +} diff --git a/pkg/driver/manager.go b/pkg/driver/manager.go index 3eefec9..22baac0 100644 --- a/pkg/driver/manager.go +++ b/pkg/driver/manager.go @@ -2,6 +2,14 @@ package driver import "fmt" +type FilterFn func(Driver) bool + +func FilterKind(k Kind) FilterFn { + return func(d Driver) bool { + return d.Info().Kind == k + } +} + type manager struct { drivers map[string]Driver } @@ -21,10 +29,12 @@ func (m *manager) register(a Adapter) error { return nil } -func (m *manager) Query() []Driver { +func (m *manager) Query(f FilterFn) []Driver { results := make([]Driver, 0) for _, d := range m.drivers { - results = append(results, d) + if ok := f(d); ok { + results = append(results, d) + } } return results diff --git a/track.go b/track.go index 55b4471..e1889f8 100644 --- a/track.go +++ b/track.go @@ -1,7 +1,6 @@ package mediadevices import ( - "fmt" "math/rand" "github.com/pion/codec" @@ -25,7 +24,7 @@ type videoTrack struct { encoder codec.Encoder } -func newVideoTrack(pc *webrtc.PeerConnection, d driver.VideoDriver, spec driver.VideoSpec, codecName string) (*videoTrack, error) { +func newVideoTrack(pc *webrtc.PeerConnection, d driver.VideoDriver, spec driver.VideoSpec, codecName Codec) (*videoTrack, error) { var err error decoder, err := frame.NewDecoder(spec.FrameFormat) if err != nil { @@ -35,7 +34,7 @@ func newVideoTrack(pc *webrtc.PeerConnection, d driver.VideoDriver, spec driver. var payloadType uint8 var encoder codec.Encoder switch codecName { - case webrtc.H264: + default: payloadType = webrtc.DefaultPayloadTypeH264 encoder, err = h264.NewEncoder(h264.Options{ Width: spec.Width, @@ -43,8 +42,6 @@ func newVideoTrack(pc *webrtc.PeerConnection, d driver.VideoDriver, spec driver. Bitrate: 1000000, MaxFrameRate: 30, }) - default: - err = fmt.Errorf("%s is currently not supported", codecName) } if err != nil {