javascript - 将图像和视频传递到 WebWorker

问题描述

我正在<video>使用OpenCV.js. 该处理有一个特别慢的操作，可能需要 100 毫秒到 400 毫秒。即使我在延迟/setTimeout循环中执行此操作并且也尝试过承诺，但这会导致视频中出现大量阻塞和滞后/断断续续。

因此，我想将此操作移至WebWorker. 我面临的问题是没有明显的方法可以将视频传输到WebWorker. 我读到这可以通过画布完成，但这似乎很复杂/效率低下。有没有更简单的方法来做到这一点？

我正在使用这样的实现，OpenCV.js但我的计算成本更高，并且使用 webRTC。

本质上，我对 a 的问题有WebWorker两个：

OpenCV.js 中的图像是从标签中读取<img>的<canvas>。我怎样才能将这些传递给一个WebWorker？
OpenCV.js 中的视频直接从<video>标签中读取。我当前的设置非常方便，但我怎样才能将它传递给WebWorker.
如何从内部加载 WASM (opencv.js) 文件WebWorker

我的代码

<!DOCTYPE html>
<html lang="en">
<script src="./opencv.js" type="text/javascript"></script>

<head>
    <meta charset="utf-8" />
</head>

<body>
    <video id="video"></video>
    <script type="text/javascript">
        var video; // INPUT WEBRTC as <video> tag
        var drawingCanvas;
        var img; // INPUT AS <img> tag
        var cameras = ["user", "environment"]; // USUAL CAMERA TYPES

        // WAITS FOR WASM TO LOAD
        cv['onRuntimeInitialized'] = () => {
            // LOADING INDICATOR STOP

            var options = []; // CAMERAS AVAILABLE
            navigator.mediaDevices.enumerateDevices().then((devices) => {
                let index = 0;
                devices.find((device) => {
                    if (device.kind === 'videoinput') {
                        if (device.deviceId == '') {
                            options.push({
                                audio: false,
                                video: {
                                    facingMode: {
                                        exact: cameras[index]
                                    }
                                }
                            });
                            index++;
                        } else {
                            options.push({
                                audio: false,
                                video: {
                                    deviceId: {
                                        exact: device.deviceId
                                    }
                                }
                            });
                        }
                    }
                });

                if (options.length == 0) {
                    console.log("NO DEVICES FOUND");
                } else {

                    navigator.mediaDevices.getUserMedia(options[options.length - 1]).then(stream => {

                        video = document.getElementById('video');
                        video.setAttribute('playsinline', 'playsinline');
                        video.setAttribute('id', 'v');
                        video.setAttribute('position', 'absolute');
                        video.setAttribute('top', '0');
                        video.setAttribute('left', '0');

                        //document.body.appendChild(video);
                        try {
                            video.srcObject = stream;
                            video.style.display = 'block';
                            video.play();
                        } catch (error) {
                            video.src = URL.createObjectURL(stream);
                            video.style.display = 'block';
                            video.play();
                        }

                        let videoSettings = stream.getVideoTracks()[0].getSettings()

                        video.width = window.innerWidth; //videoSettings.width/2;
                        video.height = window.innerWidth * (videoSettings.height / videoSettings.width); //

                        img = document.createElement('img');
                        img.setAttribute("id", "img1");
                        img.setAttribute("src", "volviclogo.png");
                        document.body.appendChild(img);

                        brisk = new cv.BRISK(30, 3, 1); //new cv.BRISK(30, 1, 3);

                        isDetecting = true;

                        setTimeout(mainFunction, 1000);

                    })
                }
            }).catch(err => {
                console.log(err);
            });

        }

        // VIDEO MAX FRAME RATE
        var FPS = 24;

        // OPENCV VARS CACHED
        var brisk;
        var img1;
        var kp1;
        var des1;
        var img1Raw;

        // ADDITIONAL LOADINGS FIRST PASS
        var isFirst = true;

        // TRACKING OF VIDEO FEED
        var cap;
        var frame;

        function mainFunction() {

            if (isFirst) {
                cap = new cv.VideoCapture(video);
                frame = new cv.Mat(video.height, video.width, cv.CV_8UC4);
                cap.read(frame);
            }

            subFunction();

            function subFunction() {
                try {

                    let begin = Date.now();

                    cap.read(frame);
                    img2 = new cv.Mat();
                    const mask = new cv.Mat();
                    kp2 = new cv.KeyPointVector();
                    des2 = new cv.Mat();
                    cv.cvtColor(frame, img2, cv.COLOR_RGBA2GRAY, 0);
                    brisk.detectAndCompute(img2, mask, kp2, des2, false); // SUPER SLOW

                    img2.delete();

                    if (isFirst) {
                        img1Raw = cv.imread("img1");
                        img1 = new cv.Mat();
                        cv.cvtColor(img1Raw, img1, cv.COLOR_RGBA2GRAY, 0);
                        kp1 = new cv.KeyPointVector();
                        des1 = new cv.Mat();
                        brisk.detectAndCompute(img1, mask, kp1, des1, false); // SUPER SLOW BUT ONLY ONCE
                        isFirst = false;
                    }

                    kp2.delete();
                    des2.delete();

                    let delay = 1000 / FPS - (Date.now() - begin);
                    setTimeout(subFunction, delay);

                } catch (err) {
                    console.log("ERROR");
                    console.log(err);
                }
            }
        }
    </script>

</body>

</html>

标签： javascripthtmlopencvweb-worker

javascript - 将图像和视频传递到 WebWorker

问题描述

解决方案

推荐阅读