audio visualizer works properly, needs optimization

src/components/MediaScene/AudioVisualizer/AudioVisualizer.tsx

@@ -1,9 +1,12 @@
 import React, { useMemo, useRef, useState } from "react";
 import * as THREE from "three";
 import { useFrame } from "react-three-fiber";
+import AudioVisualizerColumn from "./AudioVisualizerColumn";
+import { useAudioVisualizerStore } from "../../../store";
 
 const AudioVisualizer = () => {
-  const analysedData = useMemo(() => {
+  const setFrequency = useAudioVisualizerStore((state) => state.setFrequency);
+  const analyser = useMemo(() => {
     const listener = new THREE.AudioListener();
 
     const audio = new THREE.Audio(listener);
@@ -11,19 +14,31 @@ const AudioVisualizer = () => {
       document.getElementById("media") as HTMLMediaElement
     );
 
-    const analyser = new THREE.AudioAnalyser(audio, 32);
-
-    return analyser.getFrequencyData()[0];
+    return new THREE.AudioAnalyser(audio, 2048);
   }, []);
 
-  const t = useRef<any>();
-  useFrame(() => {});
+  useFrame(() => {
+    if (!(document.getElementById("media") as HTMLMediaElement).paused)
+      setFrequency(analyser.getFrequencyData());
+  });
+
   return (
     <>
-      <mesh scale={[analysedData, 0.5, 0.5]} position={[0, 0, 0]} ref={t}>
-        <meshBasicMaterial attach="material" />
-        <planeBufferGeometry attach="geometry" />
-      </mesh>
+      <AudioVisualizerColumn position={[0, 0, 0]} idx={0} />
+      <AudioVisualizerColumn position={[0, -0.5, 0]} idx={1} />
+      <AudioVisualizerColumn position={[0, -1, 0]} idx={2} />
+      <AudioVisualizerColumn position={[0, -1.5, 0]} idx={3} />
+      <AudioVisualizerColumn position={[0, -2, 0]} idx={4} />
+      <AudioVisualizerColumn position={[0, -2.5, 0]} idx={5} />
+      <AudioVisualizerColumn position={[0, -3, 0]} idx={6} />
+      <AudioVisualizerColumn position={[0, -3.5, 0]} idx={7} />
+      <AudioVisualizerColumn position={[0, -4, 0]} idx={8} />
+      <AudioVisualizerColumn position={[0, -4.5, 0]} idx={9} />
+      <AudioVisualizerColumn position={[0, -5, 0]} idx={10} />
+      <AudioVisualizerColumn position={[0, -5.5, 0]} idx={11} />
+      <AudioVisualizerColumn position={[0, -6, 0]} idx={12} />
+      <AudioVisualizerColumn position={[0, -6.5, 0]} idx={13} />
+      <AudioVisualizerColumn position={[0, -7, 0]} idx={14} />
     </>
   );
 };

src/components/MediaScene/AudioVisualizer/AudioVisualizerColumn.tsx

@@ -0,0 +1,112 @@
+import React, { useEffect, useMemo, useRef, useState } from "react";
+import audioVisualizerOrangeOrb from "../../../static/sprite/audio_visual_orb_orange.png";
+import audioVisualizerYellowOrb from "../../../static/sprite/audio_visual_orb_yellow.png";
+import { useLoader } from "react-three-fiber";
+import * as THREE from "three";
+import { useAudioVisualizerStore, useBlueOrbStore } from "../../../store";
+import { useSpring, a } from "@react-spring/three";
+import level_y_values from "../../../resources/level_y_values.json";
+
+type AudioVisualizerColumnProps = {
+  position: number[];
+  idx: number;
+};
+
+const AudioVisualizerColumn = (props: AudioVisualizerColumnProps) => {
+  const orangeAudioVisualizerOrb = useLoader(
+    THREE.TextureLoader,
+    audioVisualizerOrangeOrb
+  );
+  const yellowAudioVisualizerOrb = useLoader(
+    THREE.TextureLoader,
+    audioVisualizerYellowOrb
+  );
+
+  const [
+    { fstOrbVisible, sndOrbVisible, thirdOrbVisible, fourthOrbVisible },
+    set,
+  ] = useSpring(() => ({
+    fstOrbVisible: false,
+    sndOrbVisible: false,
+    thirdOrbVisible: false,
+    fourthOrbVisible: false,
+    config: { duration: 800 },
+  }));
+
+  useEffect(() => {
+    useAudioVisualizerStore.subscribe(set, (state) => ({
+      fstOrbVisible: state.frequency[props.idx * 32] >= 64,
+      sndOrbVisible: state.frequency[props.idx * 32] >= 128,
+      thirdOrbVisible: state.frequency[props.idx * 32] >= 192,
+      fourthOrbVisible: state.frequency[props.idx * 32] >= 255,
+    }));
+  }, [props.position, set]);
+
+  return (
+    <group position={props.position as [number, number, number]}>
+      <a.mesh
+        scale={[0.4, 0.4, 0.4]}
+        position={[-5.2, 3.8, 0]}
+        visible={fstOrbVisible}
+        renderOrder={10}
+      >
+        <meshBasicMaterial
+          attach="material"
+          map={orangeAudioVisualizerOrb}
+          transparent={true}
+          opacity={0.5}
+          depthTest={false}
+        />
+        <planeBufferGeometry attach="geometry" />
+      </a.mesh>
+      <a.mesh
+        scale={[0.4, 0.4, 0.4]}
+        position={[-4.65, 3.8, 0]}
+        visible={sndOrbVisible}
+        renderOrder={5}
+      >
+        <meshBasicMaterial
+          attach="material"
+          map={yellowAudioVisualizerOrb}
+          opacity={0.5}
+          transparent={true}
+          depthTest={false}
+        />
+        <planeBufferGeometry attach="geometry" />
+      </a.mesh>
+
+      <a.mesh
+        scale={[0.4, 0.4, 0.4]}
+        position={[-4.1, 3.8, 0]}
+        visible={thirdOrbVisible}
+        renderOrder={5}
+      >
+        <meshBasicMaterial
+          attach="material"
+          map={yellowAudioVisualizerOrb}
+          opacity={0.5}
+          transparent={true}
+          depthTest={false}
+        />
+        <planeBufferGeometry attach="geometry" />
+      </a.mesh>
+      <a.mesh
+        scale={[0.4, 0.4, 0.4]}
+        position={[-3.55, 3.8, 0]}
+        visible={fourthOrbVisible}
+        renderOrder={10}
+      >
+        <meshBasicMaterial
+          attach="material"
+          map={yellowAudioVisualizerOrb}
+          opacity={0.5}
+          transparent={true}
+          depthTest={false}
+        />
+        <planeBufferGeometry attach="geometry" />
+      </a.mesh>
+    </group>
+  );
+};
+
+export default AudioVisualizerColumn;

src/components/MediaScene/MediaPlayer.tsx

@@ -1,6 +1,5 @@
-import React, { useCallback, useRef, useState } from "react";
-import ReactPlayer from "react-player";
-import test from "../../static/movie/test.webm";
+import React, { useCallback, useRef } from "react";
+import test from "../../static/movie/LAIN01.XA[18].ogg";
 import { useMediaStore } from "../../store";
 
 const MediaPlayer = () => {

src/react-app-env.d.ts

@@ -4,8 +4,13 @@ declare module "*.webm" {
   export default src;
 }
 
-declare module "threejs-slice-geometry" {
-  const src: any;
+declare module "*.mp3" {
+  const src: string;
+  export default src;
+}
+
+declare module "*.ogg" {
+  const src: string;
   export default src;
 }
 

src/store.ts

@@ -124,6 +124,11 @@ type TextRendererState = {
   toggleGreenText: () => void;
 };
 
+type AudioVisualizerState = {
+  frequency: Uint8Array;
+  setFrequency: (to: Uint8Array) => void;
+};
+
 export const useTextRendererStore = create<TextRendererState>((set) => ({
   // yellow text
   yellowText: "Play",
@@ -253,7 +258,8 @@ export const useMediaStore = create<MediaState>((set) => ({
     set(() => ({ lastActiveLeftSideElement: to })),
   setLastActiveRightSideElement: (to) =>
     set(() => ({ lastActiveRightSideElement: to })),
-  setMediaPercentageElapsed: (to) => set(() => ({ mediaPercentageElapsed: to })),
+  setMediaPercentageElapsed: (to) =>
+    set(() => ({ mediaPercentageElapsed: to })),
 }));
 
 export const useMediaWordStore = create<MediaWordState>((set) => ({
@@ -324,3 +330,8 @@ export const useMediaWordStore = create<MediaWordState>((set) => ({
           : state.wordPositionDataStructIdx + val,
     })),
 }));
+
+export const useAudioVisualizerStore = create<AudioVisualizerState>((set) => ({
+  frequency: new Uint8Array(),
+  setFrequency: (to) => set(() => ({ frequency: to })),
+}));