Re-implement music visualizer

nix/package.nix

@@ -47,7 +47,13 @@ in {
   };
 
   shell = pkgs.npmlock2nix.shell {
-    inherit buildInputs prePatch node_modules_attrs;
+    inherit prePatch node_modules_attrs;
+
+    buildInputs =
+      buildInputs
+      ++ (with pkgs; [
+        clang-tools
+      ]);
 
     src = nix-filter {
       root = self;

package-lock.json

@@ -7812,6 +7812,11 @@
       "integrity": "sha512-Gvzk7OZpiqKSkxsQvO/mbTN1poglhmAV7gR/DdIrRrSMXraRQQlfikRJOr3Nb9GTMPC5kof948Zy6jJZIFtDvQ==",
       "dev": true
     },
+    "react-use-error-boundary": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/react-use-error-boundary/-/react-use-error-boundary-3.0.0.tgz",
+      "integrity": "sha512-5urkfyU3ZzInEMSHe1gxtDzlQAHs0krTt0V6h8H2L5nXhDKq3OYXnCs9lGHDkEkYvLmsphw8ap5g8uYfvrkJng=="
+    },
     "read-chunk": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-1.0.1.tgz",

package.yaml

@@ -22,6 +22,7 @@ dependencies:
   # React-redux stuff
   react: ^18.2.0
   react-dom: ^18.2.0
+  react-use-error-boundary: ^3.0.0 # TODO(tlater): Remove when react implement their own
   redux: ^4.2.0
   '@reduxjs/toolkit': ^1.8.3
   react-redux: ^8.0.2

src/lib/scss/_custom-bulma.scss

@@ -40,5 +40,7 @@ $content-heading-color: $text;
 $hr-background-color: $grey-light;
 $hr-height: 1px;
 
+$pre-background: $grey-darker;
+
 @import "~/node_modules/bulma";
 @import "./_navbar";

src/music/features/musicplayer/MusicPlayer.tsx

@@ -1,13 +1,17 @@
 import React from "react";
 
 import Controls from "../controls/Controls";
+import Visualizer from "../visualizer/Visualizer";
 import { musicPlayer } from "./musicPlayerSlice";
 
 function MusicPlayer() {
     return (
         <div className="is-flex-grow-1 is-flex is-flex-direction-column">
             <div className="is-flex-grow-1 is-overflow-hidden">
-                <div></div>
+                <Visualizer
+                    audioContext={musicPlayer.audioContext}
+                    audioNode={musicPlayer.audioNode}
+                />
             </div>
             <div className="is-flex-grow-0">
                 <Controls />

src/music/features/visualizer/Renderer.ts

@@ -0,0 +1,241 @@
+import { Shader } from "./Shader";
+import { mat4 } from "gl-matrix";
+
+import { Cube } from "./cube";
+import vertexSource from "./vertices.glsl";
+import fragmentSource from "./fragments.glsl";
+
+const ROTATION_SPEED = 0.0005;
+
+class RendererError extends Error {}
+
+class Renderer {
+    private canvas: HTMLCanvasElement;
+
+    private analyser: AnalyserNode;
+    private analyserData: Uint8Array;
+
+    private time: number;
+    private buffers: {
+        indices?: WebGLBuffer;
+        positions?: WebGLBuffer;
+        normals?: WebGLBuffer;
+        fft?: WebGLBuffer;
+    };
+
+    constructor(
+        context: AudioContext,
+        node: AudioNode,
+        canvas: HTMLCanvasElement
+    ) {
+        const analyser = context.createAnalyser();
+        analyser.fftSize = 2048;
+        analyser.smoothingTimeConstant = 0.8;
+        node.connect(analyser);
+
+        this.canvas = canvas;
+        this.analyser = analyser;
+        this.analyserData = new Uint8Array(analyser.frequencyBinCount);
+
+        this.time = 0;
+        this.buffers = {};
+    }
+
+    resize() {
+        this.canvas.width = this.canvas.parentElement!.clientWidth;
+        this.canvas.height = this.canvas.parentElement!.clientHeight;
+    }
+
+    initializeScene() {
+        this.resize();
+
+        const gl = this.canvas.getContext("webgl2");
+        if (gl === null) {
+            throw new RendererError("WebGL (2) is unsupported on this browser");
+        }
+
+        const shader = Shader.builder(gl)
+            .addShader(vertexSource, gl.VERTEX_SHADER)
+            .addShader(fragmentSource, gl.FRAGMENT_SHADER)
+            .addAttribute("aVertexPosition")
+            .addAttribute("aVertexNormal")
+            .addAttribute("aHeight")
+            .addUniforms("uProjectionMatrix")
+            .addUniforms("uModelViewMatrix")
+            .addUniforms("uNormalMatrix")
+            .build();
+
+        this.initGL(gl, shader);
+        this.initBuffers(gl);
+        this.drawScene(gl, shader);
+    }
+
+    initBuffers(gl: WebGLRenderingContext) {
+        // Scale down the unit cube before we use it
+        Cube.vertices = Cube.vertices.map(
+            (num: number) => num / this.analyser.frequencyBinCount
+        );
+
+        // Position buffer
+        const positionBuffer = gl.createBuffer();
+
+        if (positionBuffer === null) {
+            throw new Error("could not initialize position buffer");
+        }
+
+        gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
+        gl.bufferData(gl.ARRAY_BUFFER, Cube.vertices, gl.STATIC_DRAW);
+        this.buffers.positions = positionBuffer;
+
+        // Index buffer
+        const indexBuffer = gl.createBuffer();
+
+        if (indexBuffer === null) {
+            throw new Error("could not initialize index buffer");
+        }
+
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, indexBuffer);
+        gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, Cube.indices, gl.STATIC_DRAW);
+        this.buffers.indices = indexBuffer;
+
+        // Surface normal buffer
+        const normalBuffer = gl.createBuffer();
+
+        if (normalBuffer === null) {
+            throw new Error("could not initialize normal buffer");
+        }
+
+        gl.bindBuffer(gl.ARRAY_BUFFER, normalBuffer);
+        gl.bufferData(gl.ARRAY_BUFFER, Cube.normals, gl.STATIC_DRAW);
+        this.buffers.normals = normalBuffer;
+
+        // fft data buffer
+        const fftBuffer = gl.createBuffer();
+
+        if (fftBuffer === null) {
+            throw new Error("could not initialize fft buffer");
+        }
+
+        // No need to initialize this buffer here since we will be
+        // updating it as soon as we start rendering anyway.
+        this.buffers.fft = fftBuffer;
+    }
+
+    initGL(gl: WebGLRenderingContext, shader: Shader) {
+        gl.useProgram(shader.program);
+        gl.clearColor(0.0, 0.0, 0.0, 1.0);
+        gl.clearDepth(1.0);
+        gl.enable(gl.DEPTH_TEST);
+        gl.depthFunc(gl.LESS);
+    }
+
+    updateMatrices(gl: WebGLRenderingContext, shader: Shader) {
+        const projectionMatrix = mat4.create();
+        mat4.perspective(
+            projectionMatrix,
+            (45 * Math.PI) / 180,
+            gl.canvas.clientWidth / gl.canvas.clientHeight,
+            0.1,
+            100.0
+        );
+        gl.uniformMatrix4fv(
+            shader.getUniform("uProjectionMatrix"),
+            false,
+            projectionMatrix
+        );
+
+        const modelViewMatrix = mat4.create();
+        mat4.translate(modelViewMatrix, modelViewMatrix, [0.0, 0.0, -1.5]);
+        mat4.rotateX(modelViewMatrix, modelViewMatrix, Math.PI / 6);
+        mat4.rotateY(
+            modelViewMatrix,
+            modelViewMatrix,
+            this.time * ROTATION_SPEED
+        );
+        mat4.translate(modelViewMatrix, modelViewMatrix, [-1.0, 0.0, 0.0]);
+        gl.uniformMatrix4fv(
+            shader.getUniform("uModelViewMatrix"),
+            false,
+            modelViewMatrix
+        );
+
+        const normalMatrix = mat4.create();
+        mat4.invert(normalMatrix, modelViewMatrix);
+        mat4.transpose(normalMatrix, normalMatrix);
+        gl.uniformMatrix4fv(
+            shader.getUniform("uNormalMatrix"),
+            false,
+            normalMatrix
+        );
+    }
+
+    drawScene(gl: WebGL2RenderingContext, shader: Shader) {
+        if (
+            this.buffers.indices === undefined ||
+            this.buffers.positions === undefined ||
+            this.buffers.normals === undefined ||
+            this.buffers.fft === undefined
+        ) {
+            throw new Error("failed to create buffers before rendering");
+        }
+
+        this.updateMatrices(gl, shader);
+
+        // Update cube buffers
+        gl.bindBuffer(gl.ARRAY_BUFFER, this.buffers.positions);
+        gl.vertexAttribPointer(
+            shader.getAttribute("aVertexPosition"),
+            3,
+            gl.FLOAT,
+            false,
+            0,
+            0
+        );
+        gl.enableVertexAttribArray(shader.getAttribute("aVertexPosition"));
+
+        gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.buffers.indices);
+
+        gl.bindBuffer(gl.ARRAY_BUFFER, this.buffers.normals);
+        gl.vertexAttribPointer(
+            shader.getAttribute("aVertexNormal"),
+            3,
+            gl.FLOAT,
+            false,
+            0,
+            0
+        );
+        gl.enableVertexAttribArray(shader.getAttribute("aVertexNormal"));
+
+        // Update fft
+        this.analyser.getByteFrequencyData(this.analyserData);
+
+        gl.bindBuffer(gl.ARRAY_BUFFER, this.buffers.fft);
+        gl.bufferData(gl.ARRAY_BUFFER, this.analyserData, gl.STREAM_DRAW);
+        gl.vertexAttribPointer(
+            shader.getAttribute("aHeight"),
+            1,
+            gl.UNSIGNED_BYTE,
+            false,
+            0,
+            0
+        );
+        gl.vertexAttribDivisor(shader.getAttribute("aHeight"), 1);
+        gl.enableVertexAttribArray(shader.getAttribute("aHeight"));
+
+        gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
+        gl.drawElementsInstanced(
+            gl.TRIANGLES,
+            36,
+            gl.UNSIGNED_SHORT,
+            0,
+            this.analyser.frequencyBinCount
+        );
+
+        requestAnimationFrame((time) => {
+            this.time = time;
+            this.drawScene(gl, shader);
+        });
+    }
+}
+
+export { Renderer, RendererError };

src/music/features/visualizer/Shader.ts

@@ -0,0 +1,183 @@
+type ShaderType =
+    | WebGLRenderingContext["VERTEX_SHADER"]
+    | WebGLRenderingContext["FRAGMENT_SHADER"];
+
+interface ShaderSource {
+    source: string;
+    kind: ShaderType;
+}
+
+type ShaderAttributes = Map<string, number>;
+type ShaderUniforms = Map<string, WebGLUniformLocation>;
+
+class ShaderError extends Error {}
+
+class Shader {
+    private program_: WebGLProgram;
+    private attributes_: ShaderAttributes;
+    private uniforms_: ShaderUniforms;
+
+    constructor(
+        program: WebGLProgram,
+        attributes: ShaderAttributes,
+        uniforms: ShaderUniforms
+    ) {
+        this.program_ = program;
+        this.attributes_ = attributes;
+        this.uniforms_ = uniforms;
+    }
+
+    static builder(gl: WebGLRenderingContext): ShaderBuilder {
+        return new ShaderBuilder(gl);
+    }
+
+    get program(): WebGLProgram {
+        return this.program_;
+    }
+
+    public getAttribute(name: string): number {
+        const attribute = this.attributes_.get(name);
+
+        if (attribute === undefined) {
+            throw new ShaderError(`undefined shader attribute: ${name}`);
+        }
+
+        return attribute;
+    }
+
+    public getUniform(name: string): WebGLUniformLocation {
+        const uniform = this.uniforms_.get(name);
+
+        if (uniform === undefined) {
+            throw new ShaderError(`undefined shader uniform: ${name}`);
+        }
+
+        return uniform;
+    }
+
+    get uniforms(): ShaderUniforms {
+        return this.uniforms_;
+    }
+}
+
+class ShaderBuilder {
+    private gl: WebGLRenderingContext;
+    private sources: Array<ShaderSource>;
+    private attributes: Array<string>;
+    private uniforms: Array<string>;
+
+    public constructor(gl: WebGLRenderingContext) {
+        this.gl = gl;
+        this.sources = new Array<ShaderSource>();
+        this.attributes = new Array<string>();
+        this.uniforms = new Array<string>();
+    }
+
+    public addShader(source: string, kind: ShaderType): ShaderBuilder {
+        this.sources.push({ source, kind });
+        return this;
+    }
+
+    public addAttribute(name: string): ShaderBuilder {
+        this.attributes.push(name);
+        return this;
+    }
+
+    public addUniforms(name: string): ShaderBuilder {
+        this.uniforms.push(name);
+        return this;
+    }
+
+    public build(): Shader {
+        // Load, compile and link shader sources
+        const shaders = this.sources.map(({ source, kind }) => {
+            return this.loadShader(source, kind);
+        });
+
+        const shaderProgram = this.gl.createProgram();
+        if (shaderProgram === null) {
+            throw new ShaderError("failed to create shader program");
+        }
+
+        for (const shader of shaders) {
+            this.gl.attachShader(shaderProgram, shader);
+        }
+
+        this.gl.linkProgram(shaderProgram);
+        if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) {
+            let message = "failed to link shader program";
+            const log = this.gl.getProgramInfoLog(shaderProgram);
+            if (log !== null) {
+                message = `failed to link shader program: ${log}`;
+            }
+
+            throw new ShaderError(message);
+        }
+
+        // Find attribute and uniform locations
+        const attributes = this.attributes.reduce((acc, attribute) => {
+            const attributeLocation = this.gl.getAttribLocation(
+                shaderProgram,
+                attribute
+            );
+
+            if (attributeLocation === -1) {
+                throw new ShaderError(
+                    `shader attribute '${attribute}' could not be found`
+                );
+            }
+
+            return new Map<string, number>([
+                ...acc,
+                [attribute, attributeLocation],
+            ]);
+        }, new Map<string, number>());
+
+        const uniforms = this.uniforms.reduce((acc, uniform) => {
+            const uniformLocation = this.gl.getUniformLocation(
+                shaderProgram,
+                uniform
+            );
+
+            if (uniformLocation === null) {
+                throw new ShaderError(
+                    `shader uniform '${uniform}' could not be found`
+                );
+            }
+
+            return new Map<string, WebGLUniformLocation>([
+                ...acc,
+                [uniform, uniformLocation],
+            ]);
+        }, new Map<string, WebGLUniformLocation>());
+
+        // Build actual shader object
+        return new Shader(shaderProgram, attributes, uniforms);
+    }
+
+    private loadShader(source: string, kind: ShaderType): WebGLShader {
+        const shader = this.gl.createShader(kind);
+        if (shader === null) {
+            throw new ShaderError(`failed to initialize shader "${source}"`);
+        }
+
+        this.gl.shaderSource(shader, source);
+        this.gl.compileShader(shader);
+
+        if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
+            let message = `failed to compile shader "${source}"`;
+            const log = this.gl.getShaderInfoLog(shader);
+            if (log !== null) {
+                message = `failed to compile shader "${source}": ${log}`;
+            }
+
+            this.gl.deleteShader(shader);
+
+            throw new ShaderError(message);
+        }
+
+        return shader;
+    }
+}
+
+export { Shader, ShaderError };

src/music/features/visualizer/Visualizer.tsx

@@ -0,0 +1,87 @@
+import React, { useCallback, useState } from "react";
+import { Renderer, RendererError } from "./Renderer";
+import { ShaderError } from "./Shader";
+
+function Visualizer({
+    audioContext,
+    audioNode,
+}: {
+    audioContext: AudioContext;
+    audioNode: AudioNode;
+}) {
+    const [renderError, setRenderError] = useState<JSX.Element | null>(null);
+
+    const canvas = useCallback(
+        (canvas: HTMLCanvasElement | null) => {
+            // If we're rendering an error message, we won't be
+            // setting a canvas.
+            //
+            // Also, nonintuitively, renderError will be null here on
+            // subsequent iterations, so we can't rely on it to
+            // identify errors.
+            if (canvas === null) {
+                return;
+            }
+
+            const renderer = new Renderer(audioContext, audioNode, canvas);
+            try {
+                renderer.initializeScene();
+            } catch (error) {
+                // Log so we don't lose the stack trace
+                console.log(error);
+
+                if (error instanceof ShaderError) {
+                    setRenderError(
+                        <span>
+                            Failed to compile shader; This is a bug, feel free
+                            to contact me with this error message:
+                            <pre>
+                                <code className="has-text-danger">
+                                    {error.message}
+                                </code>
+                            </pre>
+                        </span>
+                    );
+                } else if (error instanceof RendererError) {
+                    setRenderError(
+                        <span>
+                            This browser does not support WebGL 2, sadly. This
+                            demo uses WebGL and specifically instanced drawing,
+                            so unfortunately this means it can't run on your
+                            browser/device.
+                        </span>
+                    );
+                } else if (error instanceof Error) {
+                    setRenderError(
+                        <span>
+                            Something went very wrong; apologies, either your
+                            browser is not behaving or there's a serious bug.
+                            You can contact me with this error message:
+                            <pre>
+                                <code className="has-text-danger">
+                                    {error.message}
+                                </code>
+                            </pre>
+                        </span>
+                    );
+                } else {
+                    setRenderError(
+                        <span>
+                            Something went very wrong; apologies, either your
+                            browser is not behaving or there's a serious bug.
+                        </span>
+                    );
+                }
+            }
+        },
+        [audioContext, audioNode]
+    );
+
+    if (renderError === null) {
+        return <canvas ref={canvas} style={{ display: "block" }}></canvas>;
+    } else {
+        return renderError;
+    }
+}
+
+export default Visualizer;

src/music/features/visualizer/cube.ts

@@ -0,0 +1,84 @@
+/**  * A hand-written 3d model of a cube.
+ *
+ * If this ever needs to be more than this, consider moving it to a
+ * proper .obj model.
+ */
+const Cube = {
+    // prettier-ignore
+    vertices: new Float32Array([
+       -1.0, -1.0,  1.0,
+        1.0, -1.0,  1.0,
+        1.0,  1.0,  1.0,
+       -1.0,  1.0,  1.0,
+
+       -1.0, -1.0, -1.0,
+       -1.0,  1.0, -1.0,
+        1.0,  1.0, -1.0,
+        1.0, -1.0, -1.0,
+
+       -1.0,  1.0, -1.0,
+       -1.0,  1.0,  1.0,
+        1.0,  1.0,  1.0,
+        1.0,  1.0, -1.0,
+
+       -1.0, -1.0, -1.0,
+        1.0, -1.0, -1.0,
+        1.0, -1.0,  1.0,
+       -1.0, -1.0,  1.0,
+
+        1.0, -1.0, -1.0,
+        1.0,  1.0, -1.0,
+        1.0,  1.0,  1.0,
+        1.0, -1.0,  1.0,
+
+       -1.0, -1.0, -1.0,
+       -1.0, -1.0,  1.0,
+       -1.0,  1.0,  1.0,
+       -1.0,  1.0, -1.0,
+    ]),
+
+    // prettier-ignore
+    indices: new Uint16Array([
+        0,  1,  2,   0,  2,  3,
+        4,  5,  6,   4,  6,  7,
+        8,  9, 10,   8, 10, 11,
+        12, 13, 14,  12, 14, 15,
+        16, 17, 18,  16, 18, 19,
+        20, 21, 22,  20, 22, 23,
+    ]),
+
+    // prettier-ignore
+    normals: new Float32Array([
+        0.0,  0.0,  1.0,
+        0.0,  0.0,  1.0,
+        0.0,  0.0,  1.0,
+        0.0,  0.0,  1.0,
+
+        0.0,  0.0, -1.0,
+        0.0,  0.0, -1.0,
+        0.0,  0.0, -1.0,
+        0.0,  0.0, -1.0,
+
+        0.0,  1.0,  0.0,
+        0.0,  1.0,  0.0,
+        0.0,  1.0,  0.0,
+        0.0,  1.0,  0.0,
+
+        0.0, -1.0,  0.0,
+        0.0, -1.0,  0.0,
+        0.0, -1.0,  0.0,
+        0.0, -1.0,  0.0,
+
+        1.0,  0.0,  0.0,
+        1.0,  0.0,  0.0,
+        1.0,  0.0,  0.0,
+        1.0,  0.0,  0.0,
+
+        -1.0,  0.0,  0.0,
+        -1.0,  0.0,  0.0,
+        -1.0,  0.0,  0.0,
+        -1.0,  0.0,  0.0
+    ]),
+};
+
+export { Cube };

src/music/features/visualizer/fragments.glsl

@@ -0,0 +1,8 @@
+#version 300 es
+
+precision highp float;
+
+flat in vec4 vColor;
+out vec4 color;
+
+void main() { color = vColor; }

src/music/features/visualizer/fragments.glsl.d.ts

@@ -0,0 +1,2 @@
+declare const fragments: string;
+export default fragments;

src/music/features/visualizer/vertices.glsl

@@ -0,0 +1,40 @@
+#version 300 es
+
+#define BASE_COLOR vec4(1.0, 1.0, 1.0, 1.0)
+#define AMBIENT_LIGHT vec3(0.3, 0.3, 0.3)
+#define LIGHT_DIRECTION normalize(vec3(0.85, 0.8, 0.75))
+#define LIGHT_COLOR vec3(1.0, 1.0, 1.0)
+
+precision highp float;
+
+layout(location = 0) in vec4 aVertexPosition;
+layout(location = 1) in vec3 aVertexNormal;
+layout(location = 2) in float aHeight;
+flat out vec4 vColor;
+
+uniform mat4 uModelViewMatrix;
+uniform mat4 uProjectionMatrix;
+uniform mat4 uNormalMatrix;
+
+void main() {
+  float instanceX =
+      aVertexPosition.x + float(gl_InstanceID) * 2.0 * abs(aVertexPosition.x);
+  float vertexY =
+      aVertexPosition.y > 0.0 ? aVertexPosition.y * aHeight : aVertexPosition.y;
+
+  gl_Position = uProjectionMatrix * uModelViewMatrix *
+                vec4(instanceX, vertexY, aVertexPosition.zw);
+
+  vec4 transformedNormal = uNormalMatrix * vec4(aVertexNormal, 1.0);
+  float directionalLight =
+      max(dot(transformedNormal.xyz, LIGHT_DIRECTION), 0.0);
+
+  if (aHeight == 0.0) {
+    vColor = vec4(0.0, 0.0, 0.0, 0.0);
+  } else {
+    vec4 transformedNormal = uNormalMatrix * vec4(aVertexNormal, 1.0);
+    float directionalLight =
+        max(dot(transformedNormal.xyz, LIGHT_DIRECTION), 0.0);
+    vColor = vec4(AMBIENT_LIGHT + (directionalLight * LIGHT_COLOR), 1.0);
+  }
+}

src/music/features/visualizer/vertices.glsl.d.ts

@@ -0,0 +1,2 @@
+declare const vertices: string;
+export default vertices;

tsconfig.json

@@ -5,7 +5,8 @@
     "esModuleInterop": true,
     "jsx": "react",
     "isolatedModules": true,
-    "target": "es5",
+    "target": "es2015",
+    "moduleResolution": "node",
     "plugins": [
       {
         "name": "typescript-eslint-language-service"