From ff39947b97e708e8cb3519377fdb9f780ffc0524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= Date: Sat, 6 Aug 2022 18:04:38 +0100 Subject: [PATCH] Re-implement music visualizer --- nix/package.nix | 8 +- package-lock.json | 5 + package.yaml | 1 + src/lib/scss/_custom-bulma.scss | 2 + .../features/musicplayer/MusicPlayer.tsx | 6 +- src/music/features/visualizer/Renderer.ts | 241 ++++++++++++++++++ src/music/features/visualizer/Shader.ts | 183 +++++++++++++ src/music/features/visualizer/Visualizer.tsx | 87 +++++++ src/music/features/visualizer/cube.ts | 84 ++++++ src/music/features/visualizer/fragments.glsl | 8 + .../features/visualizer/fragments.glsl.d.ts | 2 + src/music/features/visualizer/vertices.glsl | 40 +++ .../features/visualizer/vertices.glsl.d.ts | 2 + tsconfig.json | 3 +- 14 files changed, 669 insertions(+), 3 deletions(-) create mode 100644 src/music/features/visualizer/Renderer.ts create mode 100644 src/music/features/visualizer/Shader.ts create mode 100644 src/music/features/visualizer/Visualizer.tsx create mode 100644 src/music/features/visualizer/cube.ts create mode 100644 src/music/features/visualizer/fragments.glsl create mode 100644 src/music/features/visualizer/fragments.glsl.d.ts create mode 100644 src/music/features/visualizer/vertices.glsl create mode 100644 src/music/features/visualizer/vertices.glsl.d.ts diff --git a/nix/package.nix b/nix/package.nix index 7363031..a932f07 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -47,7 +47,13 @@ in { }; shell = pkgs.npmlock2nix.shell { - inherit buildInputs prePatch node_modules_attrs; + inherit prePatch node_modules_attrs; + + buildInputs = + buildInputs + ++ (with pkgs; [ + clang-tools + ]); src = nix-filter { root = self; diff --git a/package-lock.json b/package-lock.json index 63b65aa..3542461 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7812,6 +7812,11 @@ "integrity": "sha512-Gvzk7OZpiqKSkxsQvO/mbTN1poglhmAV7gR/DdIrRrSMXraRQQlfikRJOr3Nb9GTMPC5kof948Zy6jJZIFtDvQ==", "dev": true }, + "react-use-error-boundary": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/react-use-error-boundary/-/react-use-error-boundary-3.0.0.tgz", + "integrity": "sha512-5urkfyU3ZzInEMSHe1gxtDzlQAHs0krTt0V6h8H2L5nXhDKq3OYXnCs9lGHDkEkYvLmsphw8ap5g8uYfvrkJng==" + }, "read-chunk": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-1.0.1.tgz", diff --git a/package.yaml b/package.yaml index db811fd..43a09dc 100644 --- a/package.yaml +++ b/package.yaml @@ -22,6 +22,7 @@ dependencies: # React-redux stuff react: ^18.2.0 react-dom: ^18.2.0 + react-use-error-boundary: ^3.0.0 # TODO(tlater): Remove when react implement their own redux: ^4.2.0 '@reduxjs/toolkit': ^1.8.3 react-redux: ^8.0.2 diff --git a/src/lib/scss/_custom-bulma.scss b/src/lib/scss/_custom-bulma.scss index d5ed286..769137b 100644 --- a/src/lib/scss/_custom-bulma.scss +++ b/src/lib/scss/_custom-bulma.scss @@ -40,5 +40,7 @@ $content-heading-color: $text; $hr-background-color: $grey-light; $hr-height: 1px; +$pre-background: $grey-darker; + @import "~/node_modules/bulma"; @import "./_navbar"; diff --git a/src/music/features/musicplayer/MusicPlayer.tsx b/src/music/features/musicplayer/MusicPlayer.tsx index e9f6e32..abb3cfb 100644 --- a/src/music/features/musicplayer/MusicPlayer.tsx +++ b/src/music/features/musicplayer/MusicPlayer.tsx @@ -1,13 +1,17 @@ import React from "react"; import Controls from "../controls/Controls"; +import Visualizer from "../visualizer/Visualizer"; import { musicPlayer } from "./musicPlayerSlice"; function MusicPlayer() { return (
-
+
diff --git a/src/music/features/visualizer/Renderer.ts b/src/music/features/visualizer/Renderer.ts new file mode 100644 index 0000000..bfc688d --- /dev/null +++ b/src/music/features/visualizer/Renderer.ts @@ -0,0 +1,241 @@ +import { Shader } from "./Shader"; +import { mat4 } from "gl-matrix"; + +import { Cube } from "./cube"; +import vertexSource from "./vertices.glsl"; +import fragmentSource from "./fragments.glsl"; + +const ROTATION_SPEED = 0.0005; + +class RendererError extends Error {} + +class Renderer { + private canvas: HTMLCanvasElement; + + private analyser: AnalyserNode; + private analyserData: Uint8Array; + + private time: number; + private buffers: { + indices?: WebGLBuffer; + positions?: WebGLBuffer; + normals?: WebGLBuffer; + fft?: WebGLBuffer; + }; + + constructor( + context: AudioContext, + node: AudioNode, + canvas: HTMLCanvasElement + ) { + const analyser = context.createAnalyser(); + analyser.fftSize = 2048; + analyser.smoothingTimeConstant = 0.8; + node.connect(analyser); + + this.canvas = canvas; + this.analyser = analyser; + this.analyserData = new Uint8Array(analyser.frequencyBinCount); + + this.time = 0; + this.buffers = {}; + } + + resize() { + this.canvas.width = this.canvas.parentElement!.clientWidth; + this.canvas.height = this.canvas.parentElement!.clientHeight; + } + + initializeScene() { + this.resize(); + + const gl = this.canvas.getContext("webgl2"); + if (gl === null) { + throw new RendererError("WebGL (2) is unsupported on this browser"); + } + + const shader = Shader.builder(gl) + .addShader(vertexSource, gl.VERTEX_SHADER) + .addShader(fragmentSource, gl.FRAGMENT_SHADER) + .addAttribute("aVertexPosition") + .addAttribute("aVertexNormal") + .addAttribute("aHeight") + .addUniforms("uProjectionMatrix") + .addUniforms("uModelViewMatrix") + .addUniforms("uNormalMatrix") + .build(); + + this.initGL(gl, shader); + this.initBuffers(gl); + this.drawScene(gl, shader); + } + + initBuffers(gl: WebGLRenderingContext) { + // Scale down the unit cube before we use it + Cube.vertices = Cube.vertices.map( + (num: number) => num / this.analyser.frequencyBinCount + ); + + // Position buffer + const positionBuffer = gl.createBuffer(); + + if (positionBuffer === null) { + throw new Error("could not initialize position buffer"); + } + + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + gl.bufferData(gl.ARRAY_BUFFER, Cube.vertices, gl.STATIC_DRAW); + this.buffers.positions = positionBuffer; + + // Index buffer + const indexBuffer = gl.createBuffer(); + + if (indexBuffer === null) { + throw new Error("could not initialize index buffer"); + } + + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, indexBuffer); + gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, Cube.indices, gl.STATIC_DRAW); + this.buffers.indices = indexBuffer; + + // Surface normal buffer + const normalBuffer = gl.createBuffer(); + + if (normalBuffer === null) { + throw new Error("could not initialize normal buffer"); + } + + gl.bindBuffer(gl.ARRAY_BUFFER, normalBuffer); + gl.bufferData(gl.ARRAY_BUFFER, Cube.normals, gl.STATIC_DRAW); + this.buffers.normals = normalBuffer; + + // fft data buffer + const fftBuffer = gl.createBuffer(); + + if (fftBuffer === null) { + throw new Error("could not initialize fft buffer"); + } + + // No need to initialize this buffer here since we will be + // updating it as soon as we start rendering anyway. + this.buffers.fft = fftBuffer; + } + + initGL(gl: WebGLRenderingContext, shader: Shader) { + gl.useProgram(shader.program); + gl.clearColor(0.0, 0.0, 0.0, 1.0); + gl.clearDepth(1.0); + gl.enable(gl.DEPTH_TEST); + gl.depthFunc(gl.LESS); + } + + updateMatrices(gl: WebGLRenderingContext, shader: Shader) { + const projectionMatrix = mat4.create(); + mat4.perspective( + projectionMatrix, + (45 * Math.PI) / 180, + gl.canvas.clientWidth / gl.canvas.clientHeight, + 0.1, + 100.0 + ); + gl.uniformMatrix4fv( + shader.getUniform("uProjectionMatrix"), + false, + projectionMatrix + ); + + const modelViewMatrix = mat4.create(); + mat4.translate(modelViewMatrix, modelViewMatrix, [0.0, 0.0, -1.5]); + mat4.rotateX(modelViewMatrix, modelViewMatrix, Math.PI / 6); + mat4.rotateY( + modelViewMatrix, + modelViewMatrix, + this.time * ROTATION_SPEED + ); + mat4.translate(modelViewMatrix, modelViewMatrix, [-1.0, 0.0, 0.0]); + gl.uniformMatrix4fv( + shader.getUniform("uModelViewMatrix"), + false, + modelViewMatrix + ); + + const normalMatrix = mat4.create(); + mat4.invert(normalMatrix, modelViewMatrix); + mat4.transpose(normalMatrix, normalMatrix); + gl.uniformMatrix4fv( + shader.getUniform("uNormalMatrix"), + false, + normalMatrix + ); + } + + drawScene(gl: WebGL2RenderingContext, shader: Shader) { + if ( + this.buffers.indices === undefined || + this.buffers.positions === undefined || + this.buffers.normals === undefined || + this.buffers.fft === undefined + ) { + throw new Error("failed to create buffers before rendering"); + } + + this.updateMatrices(gl, shader); + + // Update cube buffers + gl.bindBuffer(gl.ARRAY_BUFFER, this.buffers.positions); + gl.vertexAttribPointer( + shader.getAttribute("aVertexPosition"), + 3, + gl.FLOAT, + false, + 0, + 0 + ); + gl.enableVertexAttribArray(shader.getAttribute("aVertexPosition")); + + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.buffers.indices); + + gl.bindBuffer(gl.ARRAY_BUFFER, this.buffers.normals); + gl.vertexAttribPointer( + shader.getAttribute("aVertexNormal"), + 3, + gl.FLOAT, + false, + 0, + 0 + ); + gl.enableVertexAttribArray(shader.getAttribute("aVertexNormal")); + + // Update fft + this.analyser.getByteFrequencyData(this.analyserData); + + gl.bindBuffer(gl.ARRAY_BUFFER, this.buffers.fft); + gl.bufferData(gl.ARRAY_BUFFER, this.analyserData, gl.STREAM_DRAW); + gl.vertexAttribPointer( + shader.getAttribute("aHeight"), + 1, + gl.UNSIGNED_BYTE, + false, + 0, + 0 + ); + gl.vertexAttribDivisor(shader.getAttribute("aHeight"), 1); + gl.enableVertexAttribArray(shader.getAttribute("aHeight")); + + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.drawElementsInstanced( + gl.TRIANGLES, + 36, + gl.UNSIGNED_SHORT, + 0, + this.analyser.frequencyBinCount + ); + + requestAnimationFrame((time) => { + this.time = time; + this.drawScene(gl, shader); + }); + } +} + +export { Renderer, RendererError }; diff --git a/src/music/features/visualizer/Shader.ts b/src/music/features/visualizer/Shader.ts new file mode 100644 index 0000000..1aaad68 --- /dev/null +++ b/src/music/features/visualizer/Shader.ts @@ -0,0 +1,183 @@ +type ShaderType = + | WebGLRenderingContext["VERTEX_SHADER"] + | WebGLRenderingContext["FRAGMENT_SHADER"]; + +interface ShaderSource { + source: string; + kind: ShaderType; +} + +type ShaderAttributes = Map; +type ShaderUniforms = Map; + +class ShaderError extends Error {} + +class Shader { + private program_: WebGLProgram; + private attributes_: ShaderAttributes; + private uniforms_: ShaderUniforms; + + constructor( + program: WebGLProgram, + attributes: ShaderAttributes, + uniforms: ShaderUniforms + ) { + this.program_ = program; + this.attributes_ = attributes; + this.uniforms_ = uniforms; + } + + static builder(gl: WebGLRenderingContext): ShaderBuilder { + return new ShaderBuilder(gl); + } + + get program(): WebGLProgram { + return this.program_; + } + + public getAttribute(name: string): number { + const attribute = this.attributes_.get(name); + + if (attribute === undefined) { + throw new ShaderError(`undefined shader attribute: ${name}`); + } + + return attribute; + } + + public getUniform(name: string): WebGLUniformLocation { + const uniform = this.uniforms_.get(name); + + if (uniform === undefined) { + throw new ShaderError(`undefined shader uniform: ${name}`); + } + + return uniform; + } + + get uniforms(): ShaderUniforms { + return this.uniforms_; + } +} + +class ShaderBuilder { + private gl: WebGLRenderingContext; + private sources: Array; + private attributes: Array; + private uniforms: Array; + + public constructor(gl: WebGLRenderingContext) { + this.gl = gl; + this.sources = new Array(); + this.attributes = new Array(); + this.uniforms = new Array(); + } + + public addShader(source: string, kind: ShaderType): ShaderBuilder { + this.sources.push({ source, kind }); + return this; + } + + public addAttribute(name: string): ShaderBuilder { + this.attributes.push(name); + return this; + } + + public addUniforms(name: string): ShaderBuilder { + this.uniforms.push(name); + return this; + } + + public build(): Shader { + // Load, compile and link shader sources + const shaders = this.sources.map(({ source, kind }) => { + return this.loadShader(source, kind); + }); + + const shaderProgram = this.gl.createProgram(); + if (shaderProgram === null) { + throw new ShaderError("failed to create shader program"); + } + + for (const shader of shaders) { + this.gl.attachShader(shaderProgram, shader); + } + + this.gl.linkProgram(shaderProgram); + if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) { + let message = "failed to link shader program"; + const log = this.gl.getProgramInfoLog(shaderProgram); + if (log !== null) { + message = `failed to link shader program: ${log}`; + } + + throw new ShaderError(message); + } + + // Find attribute and uniform locations + const attributes = this.attributes.reduce((acc, attribute) => { + const attributeLocation = this.gl.getAttribLocation( + shaderProgram, + attribute + ); + + if (attributeLocation === -1) { + throw new ShaderError( + `shader attribute '${attribute}' could not be found` + ); + } + + return new Map([ + ...acc, + [attribute, attributeLocation], + ]); + }, new Map()); + + const uniforms = this.uniforms.reduce((acc, uniform) => { + const uniformLocation = this.gl.getUniformLocation( + shaderProgram, + uniform + ); + + if (uniformLocation === null) { + throw new ShaderError( + `shader uniform '${uniform}' could not be found` + ); + } + + return new Map([ + ...acc, + [uniform, uniformLocation], + ]); + }, new Map()); + + // Build actual shader object + return new Shader(shaderProgram, attributes, uniforms); + } + + private loadShader(source: string, kind: ShaderType): WebGLShader { + const shader = this.gl.createShader(kind); + if (shader === null) { + throw new ShaderError(`failed to initialize shader "${source}"`); + } + + this.gl.shaderSource(shader, source); + this.gl.compileShader(shader); + + if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) { + let message = `failed to compile shader "${source}"`; + const log = this.gl.getShaderInfoLog(shader); + if (log !== null) { + message = `failed to compile shader "${source}": ${log}`; + } + + this.gl.deleteShader(shader); + + throw new ShaderError(message); + } + + return shader; + } +} + +export { Shader, ShaderError }; diff --git a/src/music/features/visualizer/Visualizer.tsx b/src/music/features/visualizer/Visualizer.tsx new file mode 100644 index 0000000..a940a96 --- /dev/null +++ b/src/music/features/visualizer/Visualizer.tsx @@ -0,0 +1,87 @@ +import React, { useCallback, useState } from "react"; +import { Renderer, RendererError } from "./Renderer"; +import { ShaderError } from "./Shader"; + +function Visualizer({ + audioContext, + audioNode, +}: { + audioContext: AudioContext; + audioNode: AudioNode; +}) { + const [renderError, setRenderError] = useState(null); + + const canvas = useCallback( + (canvas: HTMLCanvasElement | null) => { + // If we're rendering an error message, we won't be + // setting a canvas. + // + // Also, nonintuitively, renderError will be null here on + // subsequent iterations, so we can't rely on it to + // identify errors. + if (canvas === null) { + return; + } + + const renderer = new Renderer(audioContext, audioNode, canvas); + try { + renderer.initializeScene(); + } catch (error) { + // Log so we don't lose the stack trace + console.log(error); + + if (error instanceof ShaderError) { + setRenderError( + + Failed to compile shader; This is a bug, feel free + to contact me with this error message: + 
+                                
+                                    {error.message}
+                                
+
+ + ); + } else if (error instanceof RendererError) { + setRenderError( + + This browser does not support WebGL 2, sadly. This + demo uses WebGL and specifically instanced drawing, + so unfortunately this means it can't run on your + browser/device. + + ); + } else if (error instanceof Error) { + setRenderError( + + Something went very wrong; apologies, either your + browser is not behaving or there's a serious bug. + You can contact me with this error message: + 
+                                
+                                    {error.message}
+                                
+
+ + ); + } else { + setRenderError( + + Something went very wrong; apologies, either your + browser is not behaving or there's a serious bug. + + ); + } + } + }, + [audioContext, audioNode] + ); + + if (renderError === null) { + return ; + } else { + return renderError; + } +} + +export default Visualizer; diff --git a/src/music/features/visualizer/cube.ts b/src/music/features/visualizer/cube.ts new file mode 100644 index 0000000..ea044de --- /dev/null +++ b/src/music/features/visualizer/cube.ts @@ -0,0 +1,84 @@ +/** * A hand-written 3d model of a cube. + * + * If this ever needs to be more than this, consider moving it to a + * proper .obj model. + */ +const Cube = { + // prettier-ignore + vertices: new Float32Array([ + -1.0, -1.0, 1.0, + 1.0, -1.0, 1.0, + 1.0, 1.0, 1.0, + -1.0, 1.0, 1.0, + + -1.0, -1.0, -1.0, + -1.0, 1.0, -1.0, + 1.0, 1.0, -1.0, + 1.0, -1.0, -1.0, + + -1.0, 1.0, -1.0, + -1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, + 1.0, 1.0, -1.0, + + -1.0, -1.0, -1.0, + 1.0, -1.0, -1.0, + 1.0, -1.0, 1.0, + -1.0, -1.0, 1.0, + + 1.0, -1.0, -1.0, + 1.0, 1.0, -1.0, + 1.0, 1.0, 1.0, + 1.0, -1.0, 1.0, + + -1.0, -1.0, -1.0, + -1.0, -1.0, 1.0, + -1.0, 1.0, 1.0, + -1.0, 1.0, -1.0, + ]), + + // prettier-ignore + indices: new Uint16Array([ + 0, 1, 2, 0, 2, 3, + 4, 5, 6, 4, 6, 7, + 8, 9, 10, 8, 10, 11, + 12, 13, 14, 12, 14, 15, + 16, 17, 18, 16, 18, 19, + 20, 21, 22, 20, 22, 23, + ]), + + // prettier-ignore + normals: new Float32Array([ + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0, + 0.0, 0.0, 1.0, + + 0.0, 0.0, -1.0, + 0.0, 0.0, -1.0, + 0.0, 0.0, -1.0, + 0.0, 0.0, -1.0, + + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + 0.0, 1.0, 0.0, + + 0.0, -1.0, 0.0, + 0.0, -1.0, 0.0, + 0.0, -1.0, 0.0, + 0.0, -1.0, 0.0, + + 1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + 1.0, 0.0, 0.0, + + -1.0, 0.0, 0.0, + -1.0, 0.0, 0.0, + -1.0, 0.0, 0.0, + -1.0, 0.0, 0.0 + ]), +}; + +export { Cube }; diff --git a/src/music/features/visualizer/fragments.glsl b/src/music/features/visualizer/fragments.glsl new file mode 100644 index 0000000..b0d6784 --- /dev/null +++ b/src/music/features/visualizer/fragments.glsl @@ -0,0 +1,8 @@ +#version 300 es + +precision highp float; + +flat in vec4 vColor; +out vec4 color; + +void main() { color = vColor; } diff --git a/src/music/features/visualizer/fragments.glsl.d.ts b/src/music/features/visualizer/fragments.glsl.d.ts new file mode 100644 index 0000000..5f4fa30 --- /dev/null +++ b/src/music/features/visualizer/fragments.glsl.d.ts @@ -0,0 +1,2 @@ +declare const fragments: string; +export default fragments; diff --git a/src/music/features/visualizer/vertices.glsl b/src/music/features/visualizer/vertices.glsl new file mode 100644 index 0000000..8a66554 --- /dev/null +++ b/src/music/features/visualizer/vertices.glsl @@ -0,0 +1,40 @@ +#version 300 es + +#define BASE_COLOR vec4(1.0, 1.0, 1.0, 1.0) +#define AMBIENT_LIGHT vec3(0.3, 0.3, 0.3) +#define LIGHT_DIRECTION normalize(vec3(0.85, 0.8, 0.75)) +#define LIGHT_COLOR vec3(1.0, 1.0, 1.0) + +precision highp float; + +layout(location = 0) in vec4 aVertexPosition; +layout(location = 1) in vec3 aVertexNormal; +layout(location = 2) in float aHeight; +flat out vec4 vColor; + +uniform mat4 uModelViewMatrix; +uniform mat4 uProjectionMatrix; +uniform mat4 uNormalMatrix; + +void main() { + float instanceX = + aVertexPosition.x + float(gl_InstanceID) * 2.0 * abs(aVertexPosition.x); + float vertexY = + aVertexPosition.y > 0.0 ? aVertexPosition.y * aHeight : aVertexPosition.y; + + gl_Position = uProjectionMatrix * uModelViewMatrix * + vec4(instanceX, vertexY, aVertexPosition.zw); + + vec4 transformedNormal = uNormalMatrix * vec4(aVertexNormal, 1.0); + float directionalLight = + max(dot(transformedNormal.xyz, LIGHT_DIRECTION), 0.0); + + if (aHeight == 0.0) { + vColor = vec4(0.0, 0.0, 0.0, 0.0); + } else { + vec4 transformedNormal = uNormalMatrix * vec4(aVertexNormal, 1.0); + float directionalLight = + max(dot(transformedNormal.xyz, LIGHT_DIRECTION), 0.0); + vColor = vec4(AMBIENT_LIGHT + (directionalLight * LIGHT_COLOR), 1.0); + } +} diff --git a/src/music/features/visualizer/vertices.glsl.d.ts b/src/music/features/visualizer/vertices.glsl.d.ts new file mode 100644 index 0000000..dae6095 --- /dev/null +++ b/src/music/features/visualizer/vertices.glsl.d.ts @@ -0,0 +1,2 @@ +declare const vertices: string; +export default vertices; diff --git a/tsconfig.json b/tsconfig.json index 1a93d76..8e0a4bb 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,7 +5,8 @@ "esModuleInterop": true, "jsx": "react", "isolatedModules": true, - "target": "es5", + "target": "es2015", + "moduleResolution": "node", "plugins": [ { "name": "typescript-eslint-language-service"