add: basic webrtc is working

2023-09-28 20:35:50 -07:00
parent 7fbd4fff69
commit 19bb6c49b4
22 changed files with 2615 additions and 330 deletions
@@ -8,6 +8,9 @@ import (
 	"connectrpc.com/grpcreflect"
 	servicepb "github.com/chathaway-codes/home-sensors/v2/gen/genconnect"
 	"github.com/chathaway-codes/home-sensors/v2/pkg/signaler"
+	"github.com/rs/cors"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
 )

 func main() {
@@ -26,7 +29,40 @@ func main() {
 	fmt.Printf("Got path %s\n", path)
 	mux.Handle(servicepb.NewSignalerServiceHandler(signaler.New()))

-	if err := http.ListenAndServe("127.0.0.1:8080", mux); err != nil {
+	corsHandler := cors.New(cors.Options{
+		AllowedMethods: []string{
+			http.MethodGet,
+			http.MethodPost,
+		},
+		AllowedOrigins: []string{"example.com"},
+		AllowedHeaders: []string{
+			"Accept-Encoding",
+			"Authorization",
+			"Content-Encoding",
+			"Content-Type",
+			"Connect-Protocol-Version",
+			"Connect-Timeout-Ms",
+			"Connect-Accept-Encoding",  // Unused in web browsers, but added for future-proofing
+			"Connect-Content-Encoding", // Unused in web browsers, but added for future-proofing
+			"Grpc-Timeout",             // Used for gRPC-web
+			"X-Grpc-Web",               // Used for gRPC-web
+			"X-User-Agent",             // Used for gRPC-web
+		},
+		ExposedHeaders: []string{
+			"Content-Encoding",         // Unused in web browsers, but added for future-proofing
+			"Connect-Content-Encoding", // Unused in web browsers, but added for future-proofing
+			"Grpc-Status",              // Required for gRPC-web
+			"Grpc-Message",             // Required for gRPC-web
+		},
+	})
+	handler := corsHandler.Handler(mux)
+
+	server := &http.Server{
+		Addr:    "0.0.0.0:8080",
+		Handler: h2c.NewHandler(handler, &http2.Server{}),
+	}
+
+	if err := server.ListenAndServe(); err != nil {
 		log.Fatalf("Failed to listen for HTTP traffic: %v", err)
 	}
 }
@@ -5,39 +5,61 @@ package main

 import (
 	"context"
+	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"os"
+	"sync"
 	"time"

 	"connectrpc.com/connect"
 	pb "github.com/chathaway-codes/home-sensors/v2/gen"
 	servicepb "github.com/chathaway-codes/home-sensors/v2/gen/genconnect"
+	"github.com/google/uuid"
 	"github.com/pion/webrtc/v3"
 	"github.com/pion/webrtc/v3/pkg/media"
-	"github.com/pion/webrtc/v3/pkg/media/h264reader"
+	"github.com/pion/webrtc/v3/pkg/media/ivfreader"
+	"google.golang.org/protobuf/encoding/prototext"
 	"google.golang.org/protobuf/proto"
 )

-const (
-	videoFileName     = "/home/charles/Downloads/simpsons_movie_1080p_hddvd_trailer/The Simpsons Movie - 1080p Trailer.mp4"
-	oggPageDuration   = time.Millisecond * 20
-	h264FrameDuration = time.Millisecond * 33
+var (
+	videoFileName = flag.String("in", "/home/charles/Downloads/simpsons_movie_1080p_hddvd_trailer/output.ivf", "Where to load data from; if set to -, stdin will be used")
 )

 func withAuth[T any](token string, v *T) *connect.Request[T] {
 	req := connect.NewRequest[T](v)
-	req.Header().Add("authorization", "Bearer "+token)
+	req.Header().Add("Authorization", "Bearer "+token)
 	return req
 }

 func main() { //nolint
+	flag.Parse()
 	ctx := context.Background()
-	httpClient := &http.Client{}
-	client := servicepb.NewSignalerServiceClient(httpClient, "http://localhost:8080/")
+
+	/*httpClient := &http.Client{
+		Transport: &http2.Transport{
+			AllowHTTP: true,
+			DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) {
+				// If you're also using this client for non-h2c traffic, you may want
+				// to delegate to tls.Dial if the network isn't TCP or the addr isn't
+				// in an allowlist.
+				log.Printf("Connecting to %s : %s", network, addr)
+				return net.Dial(network, addr)
+			},
+			// Don't forget timeouts!
+		},
+	}*/
+	vid, err := newVideo(ctx)
+	if err != nil {
+		log.Fatalf("Failed to start video: %v", err)
+	}
+	client := servicepb.NewSignalerServiceClient(http.DefaultClient, "http://192.168.0.65:8080/")
 	authToken, err := client.CreateAuthToken(ctx, connect.NewRequest(&pb.CreateAuthTokenRequest{
+		Home: "home1234",
 		Type: &pb.CreateAuthTokenRequest_Camera_{
 			Camera: &pb.CreateAuthTokenRequest_Camera{
 				Id: "movie",
@@ -48,20 +70,113 @@ func main() { //nolint
 		log.Fatalf("Failed to get auth token: %v", err)
 	}
 	token := authToken.Msg.GetToken()
-
-	// Assert that we have an audio or video file
-	_, err = os.Stat(videoFileName)
-	haveVideoFile := !os.IsNotExist(err)
-
-	iceConnectedCtx, iceConnectedCtxCancel := context.WithCancel(context.Background())
+	log.Printf("Got token %s", prototext.Format(authToken.Msg))

 	// Create a new RTCPeerConnection
+	log.Printf("Waiting for connections")

-	// Wait for a session request
-	session, err := client.PopSession(ctx, withAuth(token, &pb.PopSessionRequest{}))
-	if err != nil {
-		log.Fatalf("error creating session: %v", err)
+	for {
+		// Wait for a session request
+		session, err := client.PopSession(ctx, withAuth(token, &pb.PopSessionRequest{}))
+		if err != nil {
+			log.Fatalf("error creating session: %v", err)
+		}
+		go handleSession(ctx, client, token, session, vid)
 	}
+}
+
+type video struct {
+	mu        sync.Mutex
+	listeners map[string]chan<- []byte
+	codec     string
+}
+
+func newVideo(ctx context.Context) (*video, error) {
+	var err error
+	// Assert that we have an audio or video file
+	videoFileName := *videoFileName
+	var videoIn io.Reader
+	if videoFileName == "-" {
+		videoIn = os.Stdin
+	} else {
+		videoIn, err = os.Open(videoFileName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to open %q: %v", videoFileName, err)
+		}
+
+	}
+
+	ivf, header, err := ivfreader.NewWith(videoIn)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read video: %v", err)
+	}
+
+	// Determine video codec
+	var trackCodec string
+	switch header.FourCC {
+	case "AV01":
+		trackCodec = webrtc.MimeTypeAV1
+	case "VP90":
+		trackCodec = webrtc.MimeTypeVP9
+	case "VP80":
+		trackCodec = webrtc.MimeTypeVP8
+	default:
+		return nil, fmt.Errorf("unable to handle FourCC %s", header.FourCC)
+	}
+
+	vid := &video{
+		listeners: make(map[string]chan<- []byte),
+		codec:     trackCodec,
+	}
+	go func() {
+		// Send our video file frame at a time. Pace our sending so we send it at the same speed it should be played back as.
+		// This isn't required since the video is timestamped, but we will such much higher loss if we send all at once.
+		//
+		// It is important to use a time.Ticker instead of time.Sleep because
+		// * avoids accumulating skew, just calling time.Sleep didn't compensate for the time spent parsing the data
+		// * works around latency issues with Sleep (see https://github.com/golang/go/issues/44343)
+		ticker := time.NewTicker(time.Millisecond * time.Duration((float32(header.TimebaseNumerator)/float32(header.TimebaseDenominator))*1000))
+		for ; true; <-ticker.C {
+			frame, _, ivfErr := ivf.ParseNextFrame()
+			if errors.Is(ivfErr, io.EOF) {
+				fmt.Printf("All video frames parsed and sent")
+			}
+
+			if ivfErr != nil {
+				panic(ivfErr)
+			}
+
+			vid.mu.Lock()
+			for _, lis := range vid.listeners {
+				lis <- frame
+			}
+			vid.mu.Unlock()
+		}
+	}()
+
+	return vid, nil
+}
+
+func (v *video) Join() (<-chan []byte, string, func()) {
+	v.mu.Lock()
+	defer v.mu.Unlock()
+
+	myID := uuid.New().String()
+	ch := make(chan []byte)
+	v.listeners[myID] = ch
+
+	return ch, v.codec, func() {
+		v.mu.Lock()
+		defer v.mu.Unlock()
+
+		delete(v.listeners, myID)
+	}
+}
+
+func handleSession(ctx context.Context, client servicepb.SignalerServiceClient, token string, session *connect.Response[pb.Session], vid *video) {
+	var err error
+	log.Printf("New session")
+
 	peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{
 		ICEServers: []webrtc.ICEServer{
 			{
@@ -69,78 +184,60 @@ func main() { //nolint
 			},
 		},
 	})
-	if err != nil {
-		panic(err)
-	}
+
+	// We use the cancel func to signal that the stream is ready
+	iceConnectedCtx, iceConnectedCtxCancel := context.WithCancel(context.Background())
 	defer func() {
-		if cErr := peerConnection.Close(); cErr != nil {
-			fmt.Printf("cannot close peerConnection: %v\n", cErr)
+		if err := peerConnection.Close(); err != nil {
+			fmt.Printf("cannot close peerConnection: %v\n", err)
 		}
 	}()

-	if haveVideoFile {
-		// Create a video track
-		videoTrack, videoTrackErr := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video", "pion")
-		if videoTrackErr != nil {
-			panic(videoTrackErr)
-		}
-
-		rtpSender, videoTrackErr := peerConnection.AddTrack(videoTrack)
-		if videoTrackErr != nil {
-			panic(videoTrackErr)
-		}
-
-		// Read incoming RTCP packets
-		// Before these packets are returned they are processed by interceptors. For things
-		// like NACK this needs to be called.
-		go func() {
-			rtcpBuf := make([]byte, 1500)
-			for {
-				if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
-					return
-				}
-			}
-		}()
-
-		go func() {
-			// Open a H264 file and start reading using our IVFReader
-			file, h264Err := os.Open(videoFileName)
-			if h264Err != nil {
-				panic(h264Err)
-			}
-
-			h264, h264Err := h264reader.NewReader(file)
-			if h264Err != nil {
-				panic(h264Err)
-			}
-
-			// Wait for connection established
-			<-iceConnectedCtx.Done()
-
-			// Send our video file frame at a time. Pace our sending so we send it at the same speed it should be played back as.
-			// This isn't required since the video is timestamped, but we will such much higher loss if we send all at once.
-			//
-			// It is important to use a time.Ticker instead of time.Sleep because
-			// * avoids accumulating skew, just calling time.Sleep didn't compensate for the time spent parsing the data
-			// * works around latency issues with Sleep (see https://github.com/golang/go/issues/44343)
-			ticker := time.NewTicker(h264FrameDuration)
-			for ; true; <-ticker.C {
-				nal, h264Err := h264.NextNAL()
-				if h264Err == io.EOF {
-					fmt.Printf("All video frames parsed and sent")
-					os.Exit(0)
-				}
-				if h264Err != nil {
-					panic(h264Err)
-				}
-
-				if h264Err = videoTrack.WriteSample(media.Sample{Data: nal.Data, Duration: h264FrameDuration}); h264Err != nil {
-					panic(h264Err)
-				}
-			}
-		}()
+	// connect to the video stream; the cleanup is done in the goroutine which
+	// consumes the framess
+	ch, trackCodec, cleanUp := vid.Join()
+	// Create a video track
+	videoTrack, videoTrackErr := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: trackCodec}, "video", "pion")
+	if videoTrackErr != nil {
+		log.Printf("Failed to create video track: %v", err)
 	}

+	rtpSender, err := peerConnection.AddTrack(videoTrack)
+	if err != nil {
+		log.Printf("Failed to add track to connection: %v", err)
+	}
+
+	// Read incoming RTCP packets
+	// Before these packets are returned they are processed by interceptors. For things
+	// like NACK this needs to be called.
+	go func() {
+		rtcpBuf := make([]byte, 1500)
+		for {
+			if _, _, err := rtpSender.Read(rtcpBuf); err != nil {
+				return
+			}
+		}
+	}()
+
+	go func() {
+		defer cleanUp()
+		readyToSend := false
+		for frame := range ch {
+			select {
+			case <-iceConnectedCtx.Done():
+				readyToSend = true
+			default:
+				// do nothing
+			}
+			if !readyToSend {
+				continue
+			}
+			if err := videoTrack.WriteSample(media.Sample{Data: frame, Duration: time.Second}); err != nil {
+				panic(err)
+			}
+		}
+	}()
+
 	// Set the handler for ICE connection state
 	// This will notify you when the peer has connected/disconnected
 	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
@@ -160,15 +257,27 @@ func main() { //nolint
 			// Use webrtc.PeerConnectionStateDisconnected if you are interested in detecting faster timeout.
 			// Note that the PeerConnection may come back from PeerConnectionStateDisconnected.
 			fmt.Println("Peer Connection has gone to failed exiting")
-			os.Exit(0)
+			return
 		}
 	})

 	peerConnection.OnICECandidate(func(i *webrtc.ICECandidate) {
 		if i == nil {
+			if _, err := client.CreateIceMessage(ctx, withAuth(token, &pb.CreateIceMessageRequest{
+				SessionIdentifier: session.Msg.GetId(),
+				IceMessage: &pb.IceMessage{
+					Type: &pb.IceMessage_NoMoreCandidates{},
+				},
+			})); err != nil {
+				log.Fatalf("Error sending done w/ candidates: %v", err)
+			}
 			return
 		}
 		c := i.ToJSON()
+		var usernameFragment *string
+		if c.UsernameFragment != nil {
+			usernameFragment = proto.String(*c.UsernameFragment)
+		}
 		client.CreateIceMessage(ctx, withAuth(token, &pb.CreateIceMessageRequest{
 			SessionIdentifier: session.Msg.GetId(),
 			IceMessage: &pb.IceMessage{
@@ -177,74 +286,85 @@ func main() { //nolint
 						Candidate:        c.Candidate,
 						SdpMid:           c.SDPMid,
 						SdpLineIndex:     proto.Int32(int32(*c.SDPMLineIndex)),
-						UsernameFragment: proto.String(*c.UsernameFragment),
+						UsernameFragment: usernameFragment,
 					},
 				},
 			},
 		}))
 	})
+	log.Printf("Spawning helper")
+
+	// helper which sends answers, waits for

 	// Add ICE candidates from remote
-	go func() {
-		for {
-			msg, err := client.PopIceMessage(ctx, withAuth(token, &pb.PopIceMessageRequest{
-				SessionIdentifier: session.Msg.GetId(),
-			}))
-			if err != nil {
-				log.Fatalf("failed to pop ice message: %v", err)
+	for {
+		msg, err := client.PopIceMessage(ctx, withAuth(token, &pb.PopIceMessageRequest{
+			SessionIdentifier: session.Msg.GetId(),
+		}))
+		if err != nil {
+			log.Printf("failed to pop ice message: %v", err)
+		}
+		//log.Printf("Got ice message: %v", prototext.Format(msg.Msg))
+		switch msg.Msg.Type.(type) {
+		case *pb.IceMessage_Candidate:
+			candidate := msg.Msg.GetCandidate()
+			var sdpMLine *uint16
+			if candidate.SdpLineIndex != nil {
+				t := uint16(candidate.GetSdpLineIndex())
+				sdpMLine = &t
+			}
+			if err := peerConnection.AddICECandidate(webrtc.ICECandidateInit{
+				Candidate:     candidate.GetCandidate(),
+				SDPMid:        candidate.SdpMid,
+				SDPMLineIndex: sdpMLine,
+			}); err != nil {
+				log.Fatalf("Failed to add ice candidate: %v", err)
 			}
-			switch msg.Msg.Type.(type) {
-			case *pb.IceMessage_Candidate:
-				candidate := msg.Msg.GetCandidate()
-				var sdpMLine *uint16
-				if candidate.SdpLineIndex != nil {
-					t := uint16(candidate.GetSdpLineIndex())
-					sdpMLine = &t
-				}
-				if err := peerConnection.AddICECandidate(webrtc.ICECandidateInit{
-					Candidate:     candidate.GetCandidate(),
-					SDPMid:        candidate.SdpMid,
-					SDPMLineIndex: sdpMLine,
-				}); err != nil {
-					log.Fatalf("Failed to add ice candidate: %v", err)
-				}
-			case *pb.IceMessage_Session:
-				session := msg.Msg.GetSession()

-				offer := webrtc.SessionDescription{
-					Type: webrtc.SDPType(session.SdpType),
-					SDP:  session.Sdp,
-				}
-				if err := peerConnection.SetLocalDescription(offer); err != nil {
-					log.Fatalf("Failed to set location description: %v", err)
-				}
+			// Send back an answer
+			answer, err := peerConnection.CreateAnswer(nil)
+			if err != nil {
+				log.Printf("Candidate failed")
+				continue
+			}

-				// Send back an answer
-				answer, err := peerConnection.CreateAnswer(nil)
-				if err != nil {
-					log.Fatalf("Failed to create an answer: %v", err)
-				}
-				if err := peerConnection.SetRemoteDescription(answer); err != nil {
-					log.Fatalf("Failed to set remote description: %v", err)
-				}
+			if err := peerConnection.SetLocalDescription(answer); err != nil {
+				log.Printf("Failed to set local description: %v", err)
+			}

-				_, err = client.CreateIceMessage(ctx, withAuth(token, &pb.CreateIceMessageRequest{
-					IceMessage: &pb.IceMessage{
-						Type: &pb.IceMessage_Session{
-							Session: &pb.IceSessionDescription{
-								SdpType: int64(answer.Type),
-								Sdp:     answer.SDP,
-							},
+			_, err = client.CreateIceMessage(ctx, withAuth(token, &pb.CreateIceMessageRequest{
+				SessionIdentifier: session.Msg.GetId(),
+				IceMessage: &pb.IceMessage{
+					Type: &pb.IceMessage_Session{
+						Session: &pb.IceSessionDescription{
+							SdpType: int64(answer.Type),
+							Sdp:     answer.SDP,
 						},
 					},
-				}))
-				if err != nil {
-					log.Fatalf("Failed to send answer: %v", err)
-				}
+				},
+			}))
+			if err != nil {
+				log.Printf("Failed to send answer: %v", err)
 			}
-		}
-	}()
+		case *pb.IceMessage_Session:
+			iceSession := msg.Msg.GetSession()

-	// Block forever
-	select {}
+			switch iceSession.SdpType {
+			case int64(webrtc.SDPTypeOffer):
+				offer := webrtc.SessionDescription{
+					Type: webrtc.SDPType(iceSession.SdpType),
+					SDP:  iceSession.Sdp,
+				}
+
+				if err := peerConnection.SetRemoteDescription(offer); err != nil {
+					log.Fatalf("Failed to set remote description: %v", err)
+				}
+			default:
+				log.Printf("unexpected sdp type: %v", webrtc.SDPType(iceSession.SdpType).String())
+			}
+			log.Printf("Accepted promise!")
+		case *pb.IceMessage_NoMoreCandidates:
+			// do nothing
+		}
+	}
 }