From 02e6e320283a6a427e91285c985e2ec1281529dc Mon Sep 17 00:00:00 2001 From: David Halls Date: Sat, 21 Aug 2021 12:08:22 +0100 Subject: [PATCH] Start restructure to allow video to be restarted --- site/example.html | 4 +- site/example.js | 334 +++++++++++++++++++++++----------------------- site/gl-canvas.js | 2 + site/hls.js | 7 + 4 files changed, 181 insertions(+), 166 deletions(-) diff --git a/site/example.html b/site/example.html index fe04978..25c0a8c 100644 --- a/site/example.html +++ b/site/example.html @@ -43,11 +43,11 @@
- +
- +
diff --git a/site/example.js b/site/example.js index 7dd670c..3f5190b 100644 --- a/site/example.js +++ b/site/example.js @@ -10,6 +10,8 @@ import { max_video_encoder_config, } from './resolution.js'; +const target_frame_rate = 30; + const ingestion_url_el = document.getElementById('ingestion-url'); ingestion_url_el.value = localStorage.getItem('streamana-example-ingestion-url'); @@ -144,7 +146,15 @@ async function start() { error_alert_el_parent.removeChild(error_alert_el); } - let camera_stream, gl_canvas, canvas_stream, lock_portrait = false, done = false; + // get aspect ratio of encoder + console.log(`encoder resolution: ${video_encoder_config.width}x${video_encoder_config.height}`); + const ar_encoder = video_encoder_config.ratio; + const ar_encoder_inv = 1/ar_encoder; + + const zoom_video = zoom_video_el.checked; + const lock_portrait = screen.orientation.type.startsWith('portrait') && lock_portrait_el.checked; + let video_el, gl_canvas, canvas_stream, camera_stream, done = false; + function cleanup(err) { if (err) { console.error(err); @@ -193,11 +203,87 @@ async function start() { camera_swap_el.classList.add('d-none'); } + function update() { + // update the canvas + if (gl_canvas.onLoop()) { + // get aspect ratio of video + const ar_video = video_el.videoWidth / video_el.videoHeight; + + // Note: we need to use canvas_el_parent.parentNode.offsetWidth + // to take into account margins + let width, height; + const ar_parent = canvas_el_parent.parentNode.offsetWidth / + canvas_el_parent.offsetHeight; + if (lock_portrait) { + if (zoom_video) { + if (ar_video < ar_encoder_inv) { + if (ar_parent >= ar_video) { + height = canvas_el_parent.offsetHeight * ar_encoder_inv; + width = canvas_el_parent.offsetHeight; + } else { + height = canvas_el_parent.parentNode.offsetWidth / (video_encoder_config.width * ar_video / video_encoder_config.height); + width = canvas_el_parent.parentNode.offsetWidth / ar_video; + } + } else if (ar_parent >= ar_video) { + height = canvas_el_parent.offsetHeight * ar_video; + width = canvas_el_parent.offsetHeight / (video_encoder_config.height / ar_video / video_encoder_config.width); + } else { + height = canvas_el_parent.parentNode.offsetWidth; + width = canvas_el_parent.parentNode.offsetWidth / ar_encoder_inv; + } + } else if (ar_parent >= ar_encoder_inv) { + height = canvas_el_parent.offsetHeight * ar_encoder_inv; + width = canvas_el_parent.offsetHeight; + } else { + height = canvas_el_parent.parentNode.offsetWidth; + width = canvas_el_parent.parentNode.offsetWidth / ar_encoder_inv; + } + } else if (zoom_video) { + if (ar_video < ar_encoder) { + if (ar_parent >= ar_video) { + width = canvas_el_parent.offsetHeight * ar_encoder; + height = canvas_el_parent.offsetHeight; + } else { + width = canvas_el_parent.parentNode.offsetWidth / (video_encoder_config.height * ar_video / video_encoder_config.width); + height = canvas_el_parent.parentNode.offsetWidth / ar_video; + } + } else if (ar_parent >= ar_video) { + width = canvas_el_parent.offsetHeight * ar_video; + height = canvas_el_parent.offsetHeight / (video_encoder_config.width / ar_video / video_encoder_config.height); + } else { + width = canvas_el_parent.parentNode.offsetWidth; + height = canvas_el_parent.parentNode.offsetWidth / ar_encoder; + } + } else if (ar_parent >= ar_encoder) { + width = canvas_el_parent.offsetHeight * ar_encoder; + height = canvas_el_parent.offsetHeight; + } else { + width = canvas_el_parent.parentNode.offsetWidth; + height = canvas_el_parent.parentNode.offsetWidth / ar_encoder; + } + canvas_el.style.width = `${width}px`; + canvas_el.style.height = `${height}px`; + // TODO: + // select which camera to use (front/rear)? + // have an icon which click to swap between user and environment + // we'll need to close the camera stream and start it again + // but leave the canvas going + // check behaviour when rotate phone + // chrome bug when rotate (sometimes half page doesn't render) + // chrome inspect not working + // allow select audio and video devices + // will need an audio element for "indirection" + // performance on mobile + // a40 no buffers currently available in the reader queue + // windows, android, iOS, find a mac to test + } + } + try { // create video element which will be used for grabbing the frames to // write to a canvas so we can apply webgl shaders // also used to get the native video dimensions - const video_el = document.createElement('video'); + video_el = document.createElement('video'); video_el.muted = true; video_el.playsInline = true; @@ -207,13 +293,93 @@ async function start() { video_el.src = 'empty.mp4'; await video_el.play(); + canvas_el.addEventListener('webglcontextlost', cleanup); + + // set canvas dimensions to same as encoder so its gets all the output + canvas_el.width = video_encoder_config.width; + canvas_el.height = video_encoder_config.height; + + // use glsl-canvas to make managing webgl stuff easier + gl_canvas = new GlCanvas(canvas_el, { + // as an example, greyscale the stream + fragmentString: shader + }); + + gl_canvas.on('error', cleanup); + + // tell canvas to use frames from video + gl_canvas.setTexture('u_texture', video_el); + + // check whether we're locking portrait mode or zooming (display without bars) + if (lock_portrait) { + // rotate the canvas + canvas_el.classList.add('rotate'); + canvas_el.classList.remove('mw-100', 'mh-100'); + canvas_el_parent.classList.remove('mx-auto'); + + // lock to portrait mode + try { + await screen.orientation.lock('portrait'); + } catch (ex) { + if (ex.name === 'SecurityError') { + if (!document.fullscreenElement) { + await document.documentElement.requestFullscreen(); + } + await screen.orientation.lock('portrait'); + } else if (ex.name !== 'NotSupportedError') { + throw ex; + } + } + } else if (zoom_video) { + // we're going to remove the bars for local display only + canvas_el.classList.add('zoom'); + canvas_el.classList.remove('mw-100', 'mh-100'); + canvas_el_parent.classList.remove('mx-auto'); + } + + // if we're locked to portrait mode, tell the shader to rotate the video + gl_canvas.setUniform('u_rotate', lock_portrait); + + // capture video from the canvas + // Note: Safari on iOS doesn't get any data, might be related to + // https://bugs.webkit.org/show_bug.cgi?id=181663 + //const frame_rate = video_settings.frameRate; + canvas_stream = canvas_el.captureStream(target_frame_rate); + + // HLS from the canvas stream to the ingestion URL + hls = new HLS(canvas_stream, ingestion_url, ffmpeg_lib_url, target_frame_rate, lock_portrait); + hls.addEventListener('run', () => console.log('HLS running')); + hls.addEventListener('exit', ev => { + const msg = `HLS exited with status ${ev.detail.code}`; + if (ev.detail.code === 0) { + console.log(msg); + cleanup(); + } else { + cleanup(msg); + } + }); + hls.addEventListener('error', cleanup); + hls.addEventListener('start', function () { + if (done) { + this.end(true); + } + waiting_el.classList.add('d-none'); + if (!lock_portrait) { + camera_swap_el.classList.remove('d-none'); + } + canvas_el.classList.remove('invisible'); + go_live_el.disabled = false; + update(); + }); + hls.addEventListener('update', update); + // capture video from webcam const camera_video_constraints = { width: video_encoder_config.width, height: video_encoder_config.height, frameRate: { - ideal: 30, - max: 30 + ideal: target_frame_rate, + max: target_frame_rate }, facingMode: facing_mode }; @@ -236,181 +402,21 @@ async function start() { facing_mode = video_settings.facingMode || 'user'; localStorage.setItem('streamana-facing-mode', facing_mode); - canvas_el.addEventListener('webglcontextlost', cleanup); - - // use glsl-canvas to make managing webgl stuff easier - // because it's not visible, client dimensions are zero so we - // need to substitute actual dimensions instead - gl_canvas = new GlCanvas(canvas_el, { - // as an example, greyscale the stream - fragmentString: shader - }); - - gl_canvas.on('error', cleanup); - - // tell canvas to use frames from video - gl_canvas.setTexture('u_texture', video_el); - // wait for video to load (must come after gl_canvas.setTexture() since it // registers a loadeddata handler which then registers a play handler) video_el.addEventListener('loadeddata', async function () { try { console.log(`video resolution: ${this.videoWidth}x${this.videoHeight}`); - console.log(`encoder resolution: ${video_encoder_config.width}x${video_encoder_config.height}`); - - // set aspect ratios of video and encoder - const ar_encoder = video_encoder_config.ratio; - const ar_encoder_inv = 1/ar_encoder; - - // set canvas dimensions to same as encoder so its gets all the output - canvas_el.width = video_encoder_config.width; - canvas_el.height = video_encoder_config.height; - - // check whether we're locking portrait mode or zooming (display without bars) - let zoom_video = zoom_video_el.checked; - if ((this.videoWidth < this.videoHeight) && lock_portrait_el.checked) { - lock_portrait = true; - // rotate the canvas - canvas_el.classList.add('rotate'); - canvas_el.classList.remove('mw-100', 'mh-100'); - canvas_el_parent.classList.remove('mx-auto'); - - // lock to portrait mode - try { - await screen.orientation.lock('portrait'); - } catch (ex) { - if (ex.name === 'SecurityError') { - if (!document.fullscreenElement) { - await document.documentElement.requestFullscreen(); - } - await screen.orientation.lock('portrait'); - } else if (ex.name !== 'NotSupportedError') { - throw ex; - } - } - } else if (zoom_video) { - // we're going to remove the bars for local display only - canvas_el.classList.add('zoom'); - canvas_el.classList.remove('mw-100', 'mh-100'); - canvas_el_parent.classList.remove('mx-auto'); - } - - // if we're locked to portrait mode, tell the shader to rotate the video - gl_canvas.setUniform('u_rotate', lock_portrait); // start the camera video this.play(); - // capture video from the canvas - // Note: Safari on iOS doesn't get any data, might be related to - // https://bugs.webkit.org/show_bug.cgi?id=181663 - const frame_rate = video_settings.frameRate; - canvas_stream = canvas_el.captureStream(frame_rate); - // add audio if present const audio_tracks = camera_stream.getAudioTracks(); if (audio_tracks.length > 0) { canvas_stream.addTrack(audio_tracks[0]); } - const update = () => { - // update the canvas - if (gl_canvas.onLoop()) { - const ar_video = this.videoWidth / this.videoHeight; - - // Note: we need to use canvas_el_parent.parentNode.offsetWidth - // to take into account margins - let width, height; - const ar_parent = canvas_el_parent.parentNode.offsetWidth / - canvas_el_parent.offsetHeight; - if (lock_portrait) { - if (zoom_video) { - if (ar_video < ar_encoder_inv) { - if (ar_parent >= ar_video) { - height = canvas_el_parent.offsetHeight * ar_encoder_inv; - width = canvas_el_parent.offsetHeight; - } else { - height = canvas_el_parent.parentNode.offsetWidth / (video_encoder_config.width * ar_video / video_encoder_config.height); - width = canvas_el_parent.parentNode.offsetWidth / ar_video; - } - } else if (ar_parent >= ar_video) { - height = canvas_el_parent.offsetHeight * ar_video; - width = canvas_el_parent.offsetHeight / (video_encoder_config.height / ar_video / video_encoder_config.width); - } else { - height = canvas_el_parent.parentNode.offsetWidth; - width = canvas_el_parent.parentNode.offsetWidth / ar_encoder_inv; - } - } else if (ar_parent >= ar_encoder_inv) { - height = canvas_el_parent.offsetHeight * ar_encoder_inv; - width = canvas_el_parent.offsetHeight; - } else { - height = canvas_el_parent.parentNode.offsetWidth; - width = canvas_el_parent.parentNode.offsetWidth / ar_encoder_inv; - } - } else if (zoom_video) { - if (ar_video < ar_encoder) { - if (ar_parent >= ar_video) { - width = canvas_el_parent.offsetHeight * ar_encoder; - height = canvas_el_parent.offsetHeight; - } else { - width = canvas_el_parent.parentNode.offsetWidth / (video_encoder_config.height * ar_video / video_encoder_config.width); - height = canvas_el_parent.parentNode.offsetWidth / ar_video; - } - } else if (ar_parent >= ar_video) { - width = canvas_el_parent.offsetHeight * ar_video; - height = canvas_el_parent.offsetHeight / (video_encoder_config.width / ar_video / video_encoder_config.height); - } else { - width = canvas_el_parent.parentNode.offsetWidth; - height = canvas_el_parent.parentNode.offsetWidth / ar_encoder; - } - } else if (ar_parent >= ar_encoder) { - width = canvas_el_parent.offsetHeight * ar_encoder; - height = canvas_el_parent.offsetHeight; - } else { - width = canvas_el_parent.parentNode.offsetWidth; - height = canvas_el_parent.parentNode.offsetWidth / ar_encoder; - } - canvas_el.style.width = `${width}px`; - canvas_el.style.height = `${height}px`; - // TODO: - // select which camera to use (front/rear)? - // have an icon which click to swap between user and environment - // we'll need to close the camera stream and start it again - // but leave the canvas going - // check behaviour when rotate phone - // chrome bug when rotate (sometimes half page doesn't render) - // chrome inspect not working - // allow select audio and video devices - // performance on mobile - // a40 no buffers currently available in the reader queue - // windows, android, iOS, find a mac to test - } - }; - - // start HLS from the canvas stream to the ingestion URL - hls = new HLS(canvas_stream, ingestion_url, ffmpeg_lib_url, frame_rate, lock_portrait); - hls.addEventListener('run', () => console.log('HLS running')); - hls.addEventListener('exit', ev => { - const msg = `HLS exited with status ${ev.detail.code}`; - if (ev.detail.code === 0) { - console.log(msg); - cleanup(); - } else { - cleanup(msg); - } - }); - hls.addEventListener('error', cleanup); - hls.addEventListener('start', function () { - if (done) { - this.end(true); - } - waiting_el.classList.add('d-none'); - camera_swap_el.classList.remove('d-none'); - canvas_el.classList.remove('invisible'); - go_live_el.disabled = false; - update(); - }); - hls.addEventListener('update', update); await hls.start(); } catch (ex) { cleanup(ex); diff --git a/site/gl-canvas.js b/site/gl-canvas.js index e95edb6..556379a 100644 --- a/site/gl-canvas.js +++ b/site/gl-canvas.js @@ -6,6 +6,8 @@ const { Canvas } = await importUMD('./glsl-canvas.min.js'); export class GlCanvas extends Canvas { constructor(canvas_el, options) { super(new Proxy(canvas_el, { + // because it's not visible, client dimensions are zero so we + // need to substitute actual dimensions instead get: (target, name, receiver) => { if (name === 'getBoundingClientRect') { return () => new DOMRect(0, 0, target.width, target.height); diff --git a/site/hls.js b/site/hls.js index 523f7ce..265a578 100644 --- a/site/hls.js +++ b/site/hls.js @@ -21,9 +21,14 @@ export class HLS extends EventTarget { } this.update_event = new CustomEvent('update'); this.sending = false; + this.started = false; } async start() { + if (this.started) { + return; + } + // if audio isn't present, add a silent track if (this.stream.getAudioTracks().length === 0) { console.warn("No audio present, adding silence"); @@ -68,6 +73,8 @@ export class HLS extends EventTarget { console.log("Using MediaRecorder MP4"); } } + + this.started = true; } async media_recorder(mimeType) {