diff --git a/.gitmodules b/.gitmodules index 2dc7f24..12c1c69 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "pion/webrtc"] path = pion/webrtc url = https://github.com/pion/webrtc.git +[submodule "pion/rtp"] + path = pion/rtp + url = https://github.com/pion/rtp.git diff --git a/TODO b/TODO index d93b094..32807db 100644 --- a/TODO +++ b/TODO @@ -3,7 +3,7 @@ - [x] camera switcher: discovered that it is broken https://github.com/aiortc/aiortc/issues/304 meanwhile Pion has been supporting it for 3 years https://github.com/pion/webrtc/pull/1527 beginning of port from python/aiortc to go/pion ... maybe other issues around performance will be resolved too by doing so. - [x] write a go version of the compressed_vipc decoder demo -- [ ] write a Pion-compatible vision IPC track +- [ ] write a Pion-compatible vision IPC track by looking at https://github.com/pion/webrtc/blob/master/examples/rtp-to-webrtc/main.go and https://github.com/asticode/go-astiav/blob/master/examples/transcoding/main.go - battery view - lowcam view - multicam diff --git a/go.mod b/go.mod index 5e4707a..3023b6a 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,10 @@ module github.com/kfatehi/webrtc-body go 1.19 -require github.com/pebbe/zmq4 v1.2.9 +require ( + github.com/pebbe/zmq4 v1.2.9 + github.com/pion/webrtc/v3 v3.1.55 +) require ( github.com/google/uuid v1.3.0 // indirect @@ -22,8 +25,6 @@ require ( github.com/pion/transport/v2 v2.0.1 // indirect github.com/pion/turn/v2 v2.1.0 // indirect github.com/pion/udp v0.1.4 // indirect - github.com/pion/webrtc/v3 v3.1.55 // indirect - github.com/stretchr/testify v1.8.1 // indirect golang.org/x/crypto v0.6.0 // indirect golang.org/x/net v0.6.0 // indirect golang.org/x/sys v0.5.0 // indirect diff --git a/go.work b/go.work index f95c59c..76ce52b 100644 --- a/go.work +++ b/go.work @@ -7,4 +7,5 @@ use ( ./go-capnproto2 ./cereal/gen/go ./pion/webrtc + ./pion/rtp ) diff --git a/go.work.sum b/go.work.sum index 87a6f14..6431499 100644 --- a/go.work.sum +++ b/go.work.sum @@ -5,6 +5,7 @@ github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/lxn/win v0.0.0-20210218163916-a377121e959e/go.mod h1:KxxjdtRkfNoYDCUP5ryK7XJJNTnpC8atvtmTheChOtk= github.com/otiai10/gosseract v2.2.1+incompatible/go.mod h1:XrzWItCzCpFRZ35n3YtVTgq5bLAhFIkascoRo8G32QE= +github.com/pion/webrtc/v3 v3.1.55/go.mod h1:M1gU5mnvvo4e1nnLvF23esYz0nZAFOtbU/wq44MSfbc= github.com/robotn/gohook v0.31.3 h1:kGX8iukJ9ensVRwRKnTtdojAMQOpa6KFnXDi4OA4RaI= github.com/robotn/gohook v0.31.3/go.mod h1:wyGik0yb4iwCfJjDprtNkTyxkgQWuKoVPQ3hkz6+6js= github.com/robotn/xgb v0.0.0-20190912153532-2cb92d044934 h1:2lhSR8N3T6I30q096DT7/5AKEIcf1vvnnWAmS0wfnNY= @@ -29,4 +30,3 @@ golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeap golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= diff --git a/main.go b/main.go index 188bb53..e847674 100644 --- a/main.go +++ b/main.go @@ -1,35 +1,99 @@ package main import ( + "errors" "fmt" + "io" "log" "secureput" + "strings" + "github.com/asticode/go-astiav" "github.com/pion/webrtc/v3" ) -var vipctrack *VisionIpcTrack +var visionTrack *VisionIpcTrack -func StopTrack() { - if vipctrack != nil { - vipctrack.Stop() - vipctrack = nil - } -} - -func ReplaceTrack(prefix string, pc *webrtc.PeerConnection) { - StopTrack() +func ReplaceTrack(prefix string, peerConnection *webrtc.PeerConnection) { var err error - vipctrack, err = NewVisionIpcTrack(prefix + "EncodeData") + if visionTrack != nil { + visionTrack.Stop() + } + visionTrack, err = NewVisionIpcTrack(prefix + "EncodeData") if err != nil { log.Fatal(fmt.Errorf("main: creating track failed: %w", err)) } - go vipctrack.Start() - - webrtc.newtrack - for frame := range vipctrack.Frame { - // Do something with decoded frame + // Create a video track + videoTrack, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video", "pion") + if err != nil { + panic(err) } + rtpSender, err := peerConnection.AddTrack(videoTrack) + if err != nil { + panic(err) + } + // Later on, we will use rtpSender.ReplaceTrack() for graceful track replacement + + // Read incoming RTCP packets + // Before these packets are returned they are processed by interceptors. For things + // like NACK this needs to be called. + go func() { + rtcpBuf := make([]byte, 1500) + for { + if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil { + return + } + } + }() + + // Set the handler for ICE connection state + // This will notify you when the peer has connected/disconnected + peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) { + fmt.Printf("Connection State has changed %s \n", connectionState.String()) + if connectionState.String() == "disconnected" { + visionTrack.Stop() + } else if connectionState == webrtc.ICEConnectionStateFailed { + visionTrack.Stop() + if closeErr := peerConnection.Close(); closeErr != nil { + log.Println(fmt.Errorf("main: peer connection closed due to error: %w", err)) + } + } else if connectionState.String() == "connected" { + astiav.SetLogLevel(astiav.LogLevelDebug) + astiav.SetLogCallback(func(l astiav.LogLevel, fmt, msg, parent string) { + log.Printf("ffmpeg log: %s (level: %d)\n", strings.TrimSpace(msg), l) + }) + go func() { + go visionTrack.Start() + defer visionTrack.Stop() + for visionTrack != nil { + for frame := range visionTrack.Frame { + // Do something with decoded frame + fmt.Println(frame.Roll) + + // so right now frame is a raw decoded AVFrame + // https://github.com/FFmpeg/FFmpeg/blob/n5.0/libavutil/frame.h#L317 + // avframe := frame.Frame + + // we need to: + // 1. transcode to h264 with adaptive bitrate using astiav + + // 2. create an RTP packet with h264 data inside using pion's rtp packetizer + var rtpPacket []byte + // 3. write the RTP packet to the videoTrack + + if _, err = videoTrack.Write(rtpPacket); err != nil { + if errors.Is(err, io.ErrClosedPipe) { + // The peerConnection has been closed. + return + } + + log.Println(fmt.Errorf("rtp write error: %w", err)) + } + } + } + }() + } + }) } func main() { @@ -45,11 +109,7 @@ func main() { signal.OnPeerConnectionCreated = func(pc *webrtc.PeerConnection) { ReplaceTrack("road", pc) } - signal.OnICEConnectionStateChange = func(connectionState webrtc.ICEConnectionState) { - if connectionState.String() == "disconnected" { - StopTrack() - } - } for { + select {} } } diff --git a/pion/rtp b/pion/rtp new file mode 160000 index 0000000..0e2d3fe --- /dev/null +++ b/pion/rtp @@ -0,0 +1 @@ +Subproject commit 0e2d3fee736375987792d023c7851e526651b9fb diff --git a/signaling b/signaling index ce4623b..eb06131 160000 --- a/signaling +++ b/signaling @@ -1 +1 @@ -Subproject commit ce4623b9c957e87b9009512933e1507f32d83eae +Subproject commit eb061314f5d3308062e20ea33b6f6941a11ff836