From 7f07fb0d6ab5f089d112789ca181ac6a7148575b Mon Sep 17 00:00:00 2001
From: Will Eastcott <will@playcanvas.com>
Date: Mon, 10 Feb 2025 19:12:30 +0000
Subject: [PATCH 1/2] Enable head tracking for AR avatar example

---
 examples/ar-avatar.html                    |  4 ++--
 examples/assets/scripts/camera-feed.mjs    |  2 +-
 examples/assets/scripts/face-landmarks.mjs | 16 ++++++++++++++--
 examples/assets/scripts/morph-update.mjs   |  1 +
 4 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/examples/ar-avatar.html b/examples/ar-avatar.html
index f189371..2d46798 100644
--- a/examples/ar-avatar.html
+++ b/examples/ar-avatar.html
@@ -25,7 +25,7 @@
             <!-- Scene -->
             <pc-scene>
                 <!-- Camera -->
-                <pc-entity name="camera" position="0 0 1">
+                <pc-entity name="camera">
                     <pc-camera clear-color="0 0 0 0" fov="80"></pc-camera>
                     <pc-scripts>
                         <pc-script name="cameraFeed"></pc-script>
@@ -37,7 +37,7 @@
                     <pc-light></pc-light>
                 </pc-entity>
                 <!-- Raccoon Avatar -->
-                <pc-entity name="raccoon" scale="-1 1 1">
+                <pc-entity name="raccoon" scale="100 100 100">
                     <pc-model asset="raccoon"></pc-model>
                     <pc-scripts>
                         <pc-script name="morphUpdate"></pc-script>
diff --git a/examples/assets/scripts/camera-feed.mjs b/examples/assets/scripts/camera-feed.mjs
index 279717c..b953ca6 100644
--- a/examples/assets/scripts/camera-feed.mjs
+++ b/examples/assets/scripts/camera-feed.mjs
@@ -13,7 +13,7 @@ export class CameraFeed extends Script {
      * @type {boolean}
      * @attribute
      */
-    mirror = true;
+    mirror = false;
 
     /**
      * @type {HTMLVideoElement|null}
diff --git a/examples/assets/scripts/face-landmarks.mjs b/examples/assets/scripts/face-landmarks.mjs
index 24a1e4c..92ae536 100644
--- a/examples/assets/scripts/face-landmarks.mjs
+++ b/examples/assets/scripts/face-landmarks.mjs
@@ -1,5 +1,5 @@
 import { FaceLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
-import { Script } from 'playcanvas';
+import { Mat4, Script } from 'playcanvas';
 
 export class FaceLandmarks extends Script {
     /** @type {FaceLandmarker} */
@@ -27,12 +27,24 @@ export class FaceLandmarks extends Script {
             if (video && video.readyState >= HTMLMediaElement.HAVE_ENOUGH_DATA) {
                 const detections = this.faceLandmarker.detectForVideo(video, Date.now());
                 if (detections && detections.faceBlendshapes) {
+                    if (detections.facialTransformationMatrixes.length > 0) {
+                        const { data } = detections.facialTransformationMatrixes[0];
+                        const matrix = new Mat4();
+                        matrix.set(data).invert();
+                        const position = matrix.getTranslation();
+                        const rotation = matrix.getEulerAngles();
+                        this.entity.setPosition(position);
+                        this.entity.setEulerAngles(rotation);
+                    }
+
                     if (detections.faceBlendshapes.length > 0) {
                         const { categories } = detections.faceBlendshapes[0];
-                        this.app.fire('face:blendshapes', categories);
+                        const transform = detections.facialTransformationMatrixes[0];
+                        this.app.fire('face:blendshapes', categories, transform);
                     }
                 }
             }
         }
     }
+
 }
diff --git a/examples/assets/scripts/morph-update.mjs b/examples/assets/scripts/morph-update.mjs
index 38d2b5e..3c59422 100644
--- a/examples/assets/scripts/morph-update.mjs
+++ b/examples/assets/scripts/morph-update.mjs
@@ -4,6 +4,7 @@ export class MorphUpdate extends Script {
     initialize() {
         this.app.on('face:blendshapes', (categories) => {
             const renders = this.entity.findComponents('render');
+
             for (const render of renders) {
                 for (const meshInstance of render.meshInstances) {
                     if (meshInstance.morphInstance) {

From 169915c2307c2f840a28b650277f18c279ce54c0 Mon Sep 17 00:00:00 2001
From: Will Eastcott <will@playcanvas.com>
Date: Mon, 10 Feb 2025 19:18:09 +0000
Subject: [PATCH 2/2] Tweaks

---
 examples/assets/scripts/face-landmarks.mjs | 4 +---
 examples/assets/scripts/morph-update.mjs   | 1 -
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/examples/assets/scripts/face-landmarks.mjs b/examples/assets/scripts/face-landmarks.mjs
index 92ae536..9473fe8 100644
--- a/examples/assets/scripts/face-landmarks.mjs
+++ b/examples/assets/scripts/face-landmarks.mjs
@@ -39,12 +39,10 @@ export class FaceLandmarks extends Script {
 
                     if (detections.faceBlendshapes.length > 0) {
                         const { categories } = detections.faceBlendshapes[0];
-                        const transform = detections.facialTransformationMatrixes[0];
-                        this.app.fire('face:blendshapes', categories, transform);
+                        this.app.fire('face:blendshapes', categories);
                     }
                 }
             }
         }
     }
-
 }
diff --git a/examples/assets/scripts/morph-update.mjs b/examples/assets/scripts/morph-update.mjs
index 3c59422..38d2b5e 100644
--- a/examples/assets/scripts/morph-update.mjs
+++ b/examples/assets/scripts/morph-update.mjs
@@ -4,7 +4,6 @@ export class MorphUpdate extends Script {
     initialize() {
         this.app.on('face:blendshapes', (categories) => {
             const renders = this.entity.findComponents('render');
-
             for (const render of renders) {
                 for (const meshInstance of render.meshInstances) {
                     if (meshInstance.morphInstance) {