refactored yellow orb and audio analyser

src/components/KeyPressHandler.tsx

@@ -96,20 +96,20 @@ const KeyPressHandler = () => {
       mainSubscene !== "level_selection" &&
       scene === "main"
     ) {
-      if (now > lainIdleCounter.current + 10000) {
-        lainManager({ event: getRandomIdleLainAnim() });
-        // after one idle animation plays, the second comes sooner than it would after a regular keypress
-        lainIdleCounter.current = now - 2500;
-      }
-      if (now > idleSceneCounter.current + 15000) {
-        idleManager(getRandomIdleMedia());
-        sceneManager({ event: "play_idle_media" });
-        // put it on lock until the next action, since while the idle media plays, the
-        // Date.now() value keeps increasing, which can result in another idle media playing right after one finishes
-        // one way to work around this would be to modify the value depending on the last played idle media's duration
-        // but i'm way too lazy for that
-        idleSceneCounter.current = -1;
-      }
+      // if (now > lainIdleCounter.current + 10000) {
+      //   lainManager({ event: getRandomIdleLainAnim() });
+      //   // after one idle animation plays, the second comes sooner than it would after a regular keypress
+      //   lainIdleCounter.current = now - 2500;
+      // }
+      // if (now > idleSceneCounter.current + 15000) {
+      //   idleManager(getRandomIdleMedia());
+      //   sceneManager({ event: "play_idle_media" });
+      //   // put it on lock until the next action, since while the idle media plays, the
+      //   // Date.now() value keeps increasing, which can result in another idle media playing right after one finishes
+      //   // one way to work around this would be to modify the value depending on the last played idle media's duration
+      //   // but i'm way too lazy for that
+      //   idleSceneCounter.current = -1;
+      // }
     }
   });
 

src/components/LainSpeak.tsx

@@ -39,12 +39,12 @@ const LainSpeak = (props: LainTaKProps) => {
     />
   );
 
-  const analyser = useStore((state) => state.audioAnalyser);
-
   const mouthRef = useRef<THREE.SpriteMaterial>();
+  const audioAnalyser = useStore((state) => state.audioAnalyser);
+
   useFrame(() => {
-    if (analyser) {
-      const freq = parseInt(String(analyser.getAverageFrequency()));
+    if (audioAnalyser) {
+      const freq = parseInt(String(audioAnalyser.getAverageFrequency()));
 
       if (mouthRef.current) {
         if (freq >= 50) {

src/components/MainScene/YellowOrb.tsx

@@ -1,118 +1,111 @@
-import React, { memo, useRef, useState } from "react";
+import React, { memo, useMemo, useRef } from "react";
 import { useFrame, useLoader } from "react-three-fiber";
 import * as THREE from "three";
 import orbSprite from "../../static/sprite/orb.png";
-import { useStore } from "../../store";
-
-// initialize outside the component otherwise it gets overwritten when it rerenders
-let orbIdx = 0;
-let orbDirectionChangeCount = 0;
-let renderOrder = -1;
 
 type YellowOrbProps = {
   visible: boolean;
-}
+};
 
-const YellowOrb = (props: YellowOrbProps) => {
+const YellowOrb = memo((props: YellowOrbProps) => {
   const orbRef = useRef<THREE.Object3D>();
-  const [orbDirection, setOrbDirection] = useState("left");
-  const [currentCurve, setCurrentCurve] = useState("first");
+  const idxRef = useRef(0);
+  const directionChangeCountRef = useRef(0);
+  // left or right (0/1)
+  const directionRef = useRef(0);
+  // first curve and second curve (0/1)
+  const curveIdxRef = useRef(0);
 
   const orbSpriteTexture = useLoader(THREE.TextureLoader, orbSprite);
 
   // first one goes from up to down left to right
-  const firstCurve = new THREE.QuadraticBezierCurve3(
-    new THREE.Vector3(1.2, 0, 0),
-    new THREE.Vector3(0.5, -0.8, 0),
-    new THREE.Vector3(-1.2, 1, 0)
-  );
-
   // second one goes from down to up left to right
-  const secondCurve = new THREE.QuadraticBezierCurve3(
-    new THREE.Vector3(-1.2, -0.8, 0),
-    new THREE.Vector3(-0.5, -0.1, 0),
-    new THREE.Vector3(1.2, 0.8, 0)
+  const curves = useMemo(
+    () => [
+      new THREE.QuadraticBezierCurve3(
+        new THREE.Vector3(1.2, 0, 0),
+        new THREE.Vector3(0.5, -0.8, 0),
+        new THREE.Vector3(-1.2, 1, 0)
+      ),
+      new THREE.QuadraticBezierCurve3(
+        new THREE.Vector3(-1.2, -0.8, 0),
+        new THREE.Vector3(-0.5, -0.1, 0),
+        new THREE.Vector3(1.2, 0.8, 0)
+      ),
+    ],
+    []
   );
 
   useFrame(() => {
     if (props.visible) {
-      let orbPosFirst = firstCurve.getPoint(orbIdx / 250);
-      let orbPosSecond = secondCurve.getPoint(orbIdx / 250);
+      const orbPosFirst = curves[0].getPoint(idxRef.current / 250);
+      const orbPosSecond = curves[1].getPoint(idxRef.current / 250);
 
       if (orbPosFirst.x < -1.4) {
-        switch (currentCurve) {
-          case "first":
-            setOrbDirection("right");
-            renderOrder = 0;
-            break;
-          case "second":
-            setOrbDirection("left");
-            break;
+        if (curveIdxRef.current === 0) {
+          directionRef.current = 1;
+          if (orbRef.current) orbRef.current.renderOrder = 0;
+        } else {
+          directionRef.current = 0;
         }
-        orbDirectionChangeCount++;
+        if (directionChangeCountRef.current) directionChangeCountRef.current++;
       }
 
       if (orbPosFirst.x > 1.4) {
-        switch (currentCurve) {
-          case "first":
-            setOrbDirection("left");
-            break;
-          case "second":
-            setOrbDirection("right");
-            renderOrder = -1;
-            break;
+        if (curveIdxRef.current === 0) {
+          directionRef.current = 0;
+        } else {
+          directionRef.current = 1;
+
+          if (orbRef.current) orbRef.current.renderOrder = -1;
         }
-        orbDirectionChangeCount++;
+        if (directionChangeCountRef.current) directionChangeCountRef.current++;
       }
 
-      if (orbDirection === "left") {
-        switch (currentCurve) {
-          case "first":
-            orbIdx++;
-            break;
-          case "second":
-            orbIdx--;
-            break;
+      if (directionRef.current === 0) {
+        if (curveIdxRef.current === 0) {
+          idxRef.current++;
+        } else {
+          idxRef.current--;
         }
       } else {
-        switch (currentCurve) {
-          case "first":
-            orbIdx--;
-            break;
-          case "second":
-            orbIdx++;
-            break;
+        if (curveIdxRef.current === 0) {
+          idxRef.current--;
+        } else {
+          idxRef.current++;
         }
       }
 
-      if (orbDirectionChangeCount % 6 === 0 && orbDirectionChangeCount !== 0) {
-        orbDirectionChangeCount = 0;
-        switch (currentCurve) {
-          case "first":
-            orbIdx = 250;
-            setCurrentCurve("second");
-            break;
-          case "second":
-            orbIdx = 0;
-            setCurrentCurve("first");
-            break;
+      if (
+        directionChangeCountRef.current % 6 === 0 &&
+        directionChangeCountRef.current !== 0
+      ) {
+        directionChangeCountRef.current = 0;
+        if (curveIdxRef.current === 0) {
+          idxRef.current = 250;
+          curveIdxRef.current = 1;
+        } else {
+          idxRef.current = 0;
+          curveIdxRef.current = 0;
         }
-        setOrbDirection("left");
+        directionRef.current = 0;
       }
 
-      if (currentCurve === "first") {
-        orbRef.current!.position.x = orbPosFirst.x;
-        orbRef.current!.position.y = orbPosFirst.y;
-      } else {
-        orbRef.current!.position.x = orbPosSecond.x;
-        orbRef.current!.position.y = orbPosSecond.y;
+      if (orbRef.current) {
+        if (curveIdxRef.current === 0) {
+          orbRef.current.position.x = orbPosFirst.x;
+          orbRef.current.position.y = orbPosFirst.y;
+        } else {
+          orbRef.current.position.x = orbPosSecond.x;
+          orbRef.current.position.y = orbPosSecond.y;
+        }
       }
     }
   });
 
   return (
     <group position={[0, -0.1, 1]}>
-      <sprite scale={[0.5, 0.5, 0.5]} ref={orbRef} renderOrder={renderOrder}>
+      <sprite scale={[0.5, 0.5, 0.5]} ref={orbRef}>
         <spriteMaterial
           attach="material"
           map={orbSpriteTexture}
@@ -122,6 +115,6 @@ const YellowOrb = (props: YellowOrbProps) => {
       </sprite>
     </group>
   );
-};
+});
 
 export default YellowOrb;

src/components/MediaPlayer.tsx

@@ -37,8 +37,6 @@ const MediaPlayer = () => {
 
       if (percentageElapsed % 5 === 0 && percentageElapsed !== 0) {
         setPercentageElapsed(percentageElapsed);
-        if (percentageElapsed === 100 && videoRef.current)
-          videoRef.current.currentTime = 0;
       }
     }
   }, [setPercentageElapsed, videoRef]);

src/components/MediaScene/AudioVisualizer/AudioVisualizer.tsx

@@ -5,8 +5,7 @@ import AudioVisualizerColumn from "./AudioVisualizerColumn";
 import { useStore } from "../../../store";
 
 const AudioVisualizer = memo(() => {
-  const analyser = useStore((state) => state.audioAnalyser);
-
+  const audioAnalyser = useStore(state=> state.audioAnalyser)
   const columnRefs = useMemo(
     () =>
       Array.from({ length: 15 }, () => [
@@ -19,8 +18,8 @@ const AudioVisualizer = memo(() => {
   );
 
   useFrame(() => {
-    if (analyser) {
-      const frequencyData = analyser.getFrequencyData();
+    if (audioAnalyser) {
+      const frequencyData = audioAnalyser.getFrequencyData();
 
       columnRefs.forEach((refArray, idx) => {
         const ref1 = refArray[0];

src/core/setters/media/mediaManager.ts

@@ -10,18 +10,13 @@ const mediaManager = (eventState: any) => {
 
   const updateRightSide = useStore.getState().updateRightSide;
 
-  const setAudioAnalyser = useStore.getState().setAudioAnalyser;
+  const setPercentageElapsed = useStore.getState().setPercentageElapsed;
 
   const playMedia = () => {
     const mediaElement = document.getElementById("media") as HTMLMediaElement;
 
     if (mediaElement && mediaElement.paused) {
-      const listener = new THREE.AudioListener();
-      const audio = new THREE.Audio(listener);
-
-      audio.setMediaElementSource(mediaElement);
-
-      setAudioAnalyser(new THREE.AudioAnalyser(audio, 2048));
+      setPercentageElapsed(0);
 
       mediaElement.play();
     }

src/scenes/EndScene.tsx

@@ -5,11 +5,7 @@ import { useStore } from "../store";
 import EndSelectionScreen from "../components/EndScene/EndSelectionScreen";
 
 const EndScene = () => {
-  const setAudioAnalyser = useStore((state) => state.setAudioAnalyser);
-
-  const mediaPlayedCount = useStore(
-    (state) => state.endMediaPlayedCount
-  );
+  const mediaPlayedCount = useStore((state) => state.endMediaPlayedCount);
 
   const mainCylinderRef = useRef<THREE.Object3D>();
 
@@ -37,13 +33,6 @@ const EndScene = () => {
           "media"
         ) as HTMLMediaElement;
 
-        const listener = new THREE.AudioListener();
-        const audio = new THREE.Audio(listener);
-
-        audio.setMediaElementSource(mediaElement);
-
-        setAudioAnalyser(new THREE.AudioAnalyser(audio, 2048));
-
         if (mediaElement) {
           mediaElement.play();
           setIsIntro(false);
@@ -55,7 +44,7 @@ const EndScene = () => {
         setSceneOutro(true);
       }, 4000);
     }
-  }, [mediaPlayedCount, setAudioAnalyser]);
+  }, [mediaPlayedCount]);
 
   // return mediaPlayedCount > 0 ? (
   //   <>

src/scenes/MediaScene.tsx

@@ -1,5 +1,5 @@
 import React, { useEffect } from "react";
-import { useStore } from "../store";
+import { createAudioAnalyser, useStore } from "../store";
 import LeftSide from "../components/MediaScene/Selectables/LeftSide";
 import RightSide from "../components/MediaScene/Selectables/RightSide";
 import AudioVisualizer from "../components/MediaScene/AudioVisualizer/AudioVisualizer";
@@ -10,6 +10,8 @@ import GreenTextRenderer from "../components/TextRenderer/GreenTextRenderer";
 import MediaYellowTextAnimator from "../components/TextRenderer/MediaYellowTextAnimator";
 
 const MediaScene = () => {
+  const setAudioAnalyser = useStore((state) => state.setAudioAnalyser);
+
   const activeNodeMedia = useStore((state) => state.activeNode.media_file);
 
   useEffect(() => {
@@ -29,6 +31,8 @@ const MediaScene = () => {
     const trackElement = document.getElementById("track") as HTMLTrackElement;
 
     if (mediaElement) {
+      setAudioAnalyser(createAudioAnalyser());
+
       mediaElement.currentTime = 0;
       import("../static/webvtt/" + nodeName + ".vtt")
         .then((vtt) => {
@@ -49,7 +53,7 @@ const MediaScene = () => {
         });
       }
     }
-  }, [nodeMedia, nodeName]);
+  }, [nodeMedia, nodeName, setAudioAnalyser]);
 
   return (
     <perspectiveCamera position-z={3}>

src/scenes/TaKScene.tsx

@@ -1,18 +1,18 @@
 import React, { useEffect, useState } from "react";
 import LainSpeak from "../components/LainSpeak";
-import * as THREE from "three";
-import { useStore } from "../store";
+import { createAudioAnalyser, useStore } from "../store";
 
 const TaKScene = () => {
-  const setAudioAnalyser = useStore((state) => state.setAudioAnalyser);
   const setScene = useStore((state) => state.setScene);
+  const setAudioAnalyser = useStore((state) => state.setAudioAnalyser);
+
+  const nodeMedia = useStore((state) => state.activeNode.media_file);
+  const nodeName = useStore((state) => state.activeNode.node_name);
 
   const [isIntro, setIsIntro] = useState(true);
   const [isOutro, setIsOutro] = useState(false);
 
-  const percentageElapsed = useStore(
-    (state) => state.mediaPercentageElapsed
-  );
+  const percentageElapsed = useStore((state) => state.mediaPercentageElapsed);
 
   useEffect(() => {
     if (percentageElapsed === 100) {
@@ -26,21 +26,35 @@ const TaKScene = () => {
   useEffect(() => {
     setTimeout(() => {
       const mediaElement = document.getElementById("media") as HTMLMediaElement;
-
-      const listener = new THREE.AudioListener();
-      const audio = new THREE.Audio(listener);
-
-      audio.setMediaElementSource(mediaElement);
-
-      setAudioAnalyser(new THREE.AudioAnalyser(audio, 2048));
+      const trackElement = document.getElementById("track") as HTMLTrackElement;
 
       if (mediaElement) {
+        setAudioAnalyser(createAudioAnalyser());
         mediaElement.currentTime = 0;
-        mediaElement.play();
+        import("../static/webvtt/" + nodeName + ".vtt")
+          .then((vtt) => {
+            if (vtt) trackElement.src = vtt.default;
+          })
+          // some entries have no spoken words, so the file doesnt exist. we catch that here.
+          .catch((e) => console.log(e));
+
+        if (nodeMedia.includes("XA")) {
+          import("../static/audio/" + nodeMedia + ".ogg").then((media) => {
+            mediaElement.src = media.default;
+            mediaElement.load();
+            mediaElement.play();
+          });
+        } else {
+          import("../static/movie/" + nodeMedia + "[0].webm").then((media) => {
+            mediaElement.src = media.default;
+            mediaElement.load();
+            mediaElement.play();
+          });
+        }
         setIsIntro(false);
       }
     }, 3800);
-  }, [setAudioAnalyser]);
+  }, [nodeMedia, nodeName]);
 
   return <LainSpeak intro={isIntro} outro={isOutro} />;
 };

src/store.ts

@@ -6,6 +6,7 @@ import { NodeDataType } from "./components/MainScene/Site";
 import { getNodeById } from "./utils/node-utils";
 import site_a from "./resources/site_a.json";
 
+
 type State = {
   currentScene: string;
 
@@ -48,7 +49,7 @@ type State = {
   permissionDenied: boolean;
 
   // media/media scene
-  audioAnalyser: undefined | THREE.AudioAnalyser;
+  audioAnalyser: any;
   mediaPercentageElapsed: number;
   mediaComponentMatrix: [["play", "exit"], ["fstWord", "sndWord", "thirdWord"]];
   mediaComponentMatrixIndices: {
@@ -172,7 +173,7 @@ export const useStore = create(
       showingAbout: false,
       permissionDenied: false,
 
-      // media / media scene
+      // media scene
       audioAnalyser: undefined,
       mediaPercentageElapsed: 0,
       mediaComponentMatrix: [
@@ -316,7 +317,8 @@ export const useStore = create(
       setPermissionDenied: (to: boolean) =>
         set(() => ({ permissionDenied: to })),
 
-      // media/media scene setters
+      // media scene setters
+      setAudioAnalyser: (to: any) => set(() => ({ audioAnalyser: to })),
       toggleMediaSide: () =>
         set((state) => ({
           mediaComponentMatrixIndices: {
@@ -343,8 +345,6 @@ export const useStore = create(
         })),
       setPercentageElapsed: (to: number) =>
         set(() => ({ mediaPercentageElapsed: to })),
-      setAudioAnalyser: (to: THREE.AudioAnalyser) =>
-        set(() => ({ audioAnalyser: to })),
       setWordSelected: (to: boolean) => set(() => ({ wordSelected: to })),
 
       // idle media setters
@@ -537,3 +537,13 @@ export const playAudio = (audio: HTMLAudioElement) => {
   audio.loop = false;
   audio.play();
 };
+
+export const createAudioAnalyser = () => {
+    const mediaElement = document.getElementById("media") as HTMLMediaElement;
+    const listener = new THREE.AudioListener();
+    const audio = new THREE.Audio(listener);
+
+    audio.setMediaElementSource(mediaElement);
+
+    return new THREE.AudioAnalyser(audio, 2048);
+};

src/utils/node-utils.ts

@@ -67,8 +67,7 @@ export const isNodeVisible = (
     ? (node.unlocked_by === "" ||
         gameProgress[node.unlocked_by as keyof typeof gameProgress]
           .is_viewed) &&
-        gameProgress[node.node_name as keyof typeof gameProgress].is_visible &&
-        node.required_final_video_viewcount < 1
+        gameProgress[node.node_name as keyof typeof gameProgress].is_visible
     : false;
 };