playcanvas · willeastcott · Feb 10, 2025 · Feb 10, 2025 · Feb 10, 2025 · Feb 10, 2025
diff --git a/examples/ar-avatar.html b/examples/ar-avatar.html
@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no, viewport-fit=cover">
+        <title>PlayCanvas Web Components - AR Avatar</title>
+        <script type="importmap">
+            {
+                "imports": {
+                    "playcanvas": "../node_modules/playcanvas/build/playcanvas.mjs",
+                    "@mediapipe/tasks-vision": "../node_modules/@mediapipe/tasks-vision/vision_bundle.mjs"
+                }
+            }
+        </script>
+
+        <script type="module" src="../dist/pwc.mjs"></script>
+        <link rel="stylesheet" href="css/example.css">
+    </head>
+    <body>
+        <pc-app>
+            <pc-asset src="assets/scripts/camera-feed.mjs"></pc-asset>
+            <pc-asset src="assets/scripts/face-landmarks.mjs"></pc-asset>
+            <pc-asset src="assets/scripts/morph-update.mjs"></pc-asset>
+            <pc-asset src="assets/models/raccoon-head.glb" id="raccoon"></pc-asset>
+            <!-- Scene -->
+            <pc-scene>
+                <!-- Camera -->
+                <pc-entity name="camera" position="0 0 1">
+                    <pc-camera clear-color="0 0 0 0" fov="80"></pc-camera>
+                    <pc-scripts>
+                        <pc-script name="cameraFeed"></pc-script>
+                        <pc-script name="faceLandmarks"></pc-script>
+                    </pc-scripts>
+                </pc-entity>
+                <!-- Light -->
+                <pc-entity name="light" rotation="45 0 0">
+                    <pc-light></pc-light>
+                </pc-entity>
+                <!-- Raccoon Avatar -->
+                <pc-entity name="raccoon" scale="-1 1 1">
+                    <pc-model asset="raccoon"></pc-model>
+                    <pc-scripts>
+                        <pc-script name="morphUpdate"></pc-script>
+                    </pc-scripts>
+                </pc-entity>
+            </pc-scene>
+        </pc-app>
+        <script type="module" src="js/example.mjs"></script>
+    </body>
+</html>
diff --git a/examples/assets/models/raccoon-head.glb b/examples/assets/models/raccoon-head.glb
diff --git a/examples/assets/scripts/camera-feed.mjs b/examples/assets/scripts/camera-feed.mjs
@@ -0,0 +1,85 @@
+import { Script } from 'playcanvas';
+
+/**
+ * A script that displays a live camera feed from the device's camera.
+ *
+ * This script creates a video element and requests access to the device's camera.
+ * It then streams the camera's video to the video element and plays it.
+ */
+export class CameraFeed extends Script {
+    /**
+     * Whether to flip the video stream horizontally to behave like a mirror.
+     *
+     * @type {boolean}
+     * @attribute
+     */
+    mirror = true;
+
+    /**
+     * @type {HTMLVideoElement|null}
+     */
+    video = null;
+
+    createVideoElement() {
+        const video = document.createElement('video');
+
+        // Enable inline playback, autoplay, and mute (important for mobile)
+        video.setAttribute('playsinline', '');
+        video.autoplay = true;
+        video.muted = true;
+
+        // Style the video element to fill the viewport and cover it like CSS background-size: cover
+        video.style.position = 'absolute';
+        video.style.top = '0';
+        video.style.left = '0';
+        video.style.width = '100%';
+        video.style.height = '100%';
+        video.style.objectFit = 'cover';
+
+        // Set a negative z-index so the video appears behind the canvas
+        video.style.zIndex = '-1';
+
+        // Mirror the video stream, if chosen.
+        if (this.mirror) {
+            video.style.transform = 'scaleX(-1)';
+        }
+
+        return video;
+    }
+
+    initialize() {
+        this.video = this.createVideoElement();
+
+        // Insert the video element into the DOM.
+        document.body.appendChild(this.video);
+
+        this.on('destroy', () => {
+            if (this.video && this.video.srcObject) {
+                // Stop the video stream
+                this.video.srcObject.getTracks().forEach(track => track.stop());
+            }
+            // Remove the video element from the DOM
+            document.body.removeChild(this.video);
+            this.video = null;
+        });
+
+        // Request access to the webcam
+        if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
+            navigator.mediaDevices.getUserMedia({ video: true, audio: false })
+            .then((stream) => {
+                // Stream the webcam to the video element and play it
+                this.video.srcObject = stream;
+                this.video.play();
+            })
+            .catch((error) => {
+                console.error('Error accessing the webcam:', error);
+            });
+        } else {
+            console.error('getUserMedia is not supported in this browser.');
+        }
+    }
+
+    update(dt) {
+        // Optional per-frame logic.
+    }
+}
diff --git a/examples/assets/scripts/face-landmarks.mjs b/examples/assets/scripts/face-landmarks.mjs
@@ -0,0 +1,36 @@
+import { FaceLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
+import { Script } from 'playcanvas';
+
+export class FaceLandmarks extends Script {
+    /** @type {FaceLandmarker} */
+    faceLandmarker;
+
+    async initialize() {
+        const wasmFileset = await FilesetResolver.forVisionTasks(
+            '../node_modules/@mediapipe/tasks-vision/wasm'
+        );
+        this.faceLandmarker = await FaceLandmarker.createFromOptions(wasmFileset, {
+            baseOptions: {
+                modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
+                delegate: 'GPU'
+            },
+            outputFaceBlendshapes: true,
+            outputFacialTransformationMatrixes: true,
+            runningMode: 'VIDEO',
+            numFaces: 1
+        });
+    }
+
+    update(dt) {
+        if (this.faceLandmarker) {
+            const video = document.querySelector('video');
+            const detections = this.faceLandmarker.detectForVideo(video, Date.now());
+            if (detections && detections.faceBlendshapes) {
+                if (detections.faceBlendshapes.length > 0) {
+                    const { categories } = detections.faceBlendshapes[0];
+                    this.app.fire('face:blendshapes', categories);
+                }
+            }
+        }
+    }
+}
diff --git a/examples/assets/scripts/morph-update.mjs b/examples/assets/scripts/morph-update.mjs
@@ -0,0 +1,18 @@
+import { Script } from 'playcanvas';
+
+export class MorphUpdate extends Script {
+    initialize() {
+        this.app.on('face:blendshapes', (categories) => {
+            const renders = this.entity.findComponents('render');
+            for (const render of renders) {
+                for (const meshInstance of render.meshInstances) {
+                    if (meshInstance.morphInstance) {
+                        for (const category of categories) {
+                            meshInstance.morphInstance.setWeight(category.categoryName, category.score);
+                        }
+                    }
+                }
+            }
+        });
+    }
+}
diff --git a/examples/js/example-list.mjs b/examples/js/example-list.mjs
@@ -1,6 +1,7 @@
 export const examples = [
     { name: 'Animation', path: 'animation.html' },
     { name: 'Annotations', path: 'annotations.html' },
+    { name: 'AR Avatar', path: 'ar-avatar.html' },
     { name: 'Basic Shapes', path: 'basic-shapes.html' },
     { name: 'Car Configurator', path: 'car-configurator.html' },
     { name: 'FPS Controller', path: 'fps-controller.html' },

diff --git a/lib/meshopt_decoder.module.d.ts b/lib/meshopt_decoder.module.d.ts
@@ -0,0 +1,15 @@
+// This file is part of meshoptimizer library and is distributed under the terms of MIT License.
+// Copyright (C) 2016-2022, by Arseny Kapoulkine ([email protected])
+export const MeshoptDecoder: {
+    supported: boolean;
+    ready: Promise<void>;
+
+    decodeVertexBuffer: (target: Uint8Array, count: number, size: number, source: Uint8Array, filter?: string) => void;
+    decodeIndexBuffer: (target: Uint8Array, count: number, size: number, source: Uint8Array) => void;
+    decodeIndexSequence: (target: Uint8Array, count: number, size: number, source: Uint8Array) => void;
+
+    decodeGltfBuffer: (target: Uint8Array, count: number, size: number, source: Uint8Array, mode: string, filter?: string) => void;
+
+    useWorkers: (count: number) => void;
+    decodeGltfBufferAsync: (count: number, size: number, source: Uint8Array, mode: string, filter?: string) => Promise<Uint8Array>;
+};
diff --git a/lib/meshopt_decoder.module.js b/lib/meshopt_decoder.module.js
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -37,6 +37,7 @@
     "watch": "rollup -c -w"
   },
   "devDependencies": {
+    "@mediapipe/tasks-vision": "^0.10.21",
     "@playcanvas/eslint-config": "2.0.9",
     "@rollup/plugin-commonjs": "28.0.2",
     "@rollup/plugin-node-resolve": "16.0.0",

diff --git a/src/asset.ts b/src/asset.ts
@@ -1,5 +1,7 @@
 import { Asset } from 'playcanvas';
 
+import { MeshoptDecoder } from '../lib/meshopt_decoder.module.js';
+
 const extToType = new Map([
     ['bin', 'binary'],
     ['css', 'css'],
@@ -20,6 +22,45 @@ const extToType = new Map([
     ['webp', 'texture']
 ]);
 
+
+// provide buffer view callback so we can handle models compressed with MeshOptimizer
+// https://github.com/zeux/meshoptimizer
+const processBufferView = (
+    gltfBuffer: any,
+    buffers: Array<any>,
+    continuation: (err: string, result: any) => void
+) => {
+    if (gltfBuffer.extensions && gltfBuffer.extensions.EXT_meshopt_compression) {
+        const extensionDef = gltfBuffer.extensions.EXT_meshopt_compression;
+
+        Promise.all([MeshoptDecoder.ready, buffers[extensionDef.buffer]]).then((promiseResult) => {
+            const buffer = promiseResult[1];
+
+            const byteOffset = extensionDef.byteOffset || 0;
+            const byteLength = extensionDef.byteLength || 0;
+
+            const count = extensionDef.count;
+            const stride = extensionDef.byteStride;
+
+            const result = new Uint8Array(count * stride);
+            const source = new Uint8Array(buffer.buffer, buffer.byteOffset + byteOffset, byteLength);
+
+            MeshoptDecoder.decodeGltfBuffer(
+                result,
+                count,
+                stride,
+                source,
+                extensionDef.mode,
+                extensionDef.filter
+            );
+
+            continuation(null, result);
+        });
+    } else {
+        continuation(null, null);
+    }
+};
+
 /**
  * The AssetElement interface provides properties and methods for manipulating
  * {@link https://developer.playcanvas.com/user-manual/engine/web-components/tags/pc-asset/ | `<pc-asset>`} elements.
@@ -54,10 +95,21 @@ class AssetElement extends HTMLElement {
             return;
         }
 
-        this.asset = new Asset(id, type, { url: src });
+        if (type === 'container') {
+            this.asset = new Asset(id, type, { url: src }, undefined, {
+                // @ts-ignore TODO no definition in pc
+                bufferView: {
+                    processAsync: processBufferView.bind(this)
+                }
+            });
+        } else {
+            this.asset = new Asset(id, type, { url: src });
+        }
+
         this.asset.preload = !this._lazy;
     }
 
+
     destroyAsset() {
         if (this.asset) {
             this.asset.unload();