implement image to audio conversion

Author: p2r3

package.json

@@ -23,6 +23,7 @@
     "meyda": "^5.6.3",
     "mime": "^4.1.0",
     "pdftoimg-js": "^0.2.5",
-    "vite-plugin-static-copy": "^3.1.6"
+    "vite-plugin-static-copy": "^3.1.6",
+    "wavefile": "^11.0.0"
   }
 }

src/handlers/meyda.ts

@@ -1,6 +1,7 @@
 import type { FileData, FileFormat, FormatHandler } from "../FormatHandler.ts";
 
 import Meyda from "meyda";
+import { WaveFile } from "wavefile";
 
 class meydaHandler implements FormatHandler {
 
@@ -11,27 +12,27 @@ class meydaHandler implements FormatHandler {
       format: "png",
       extension: "png",
       mime: "image/png",
-      from: false,
+      from: true,
       to: true,
-      internal: "png"
+      internal: "image"
     },
     {
       name: "Joint Photographic Experts Group JFIF",
       format: "jpeg",
       extension: "jpg",
       mime: "image/jpeg",
-      from: false,
+      from: true,
       to: true,
-      internal: "jpeg"
+      internal: "image"
     },
     {
       name: "WebP",
       format: "webp",
       extension: "webp",
       mime: "image/webp",
-      from: false,
+      from: true,
       to: true,
-      internal: "webp"
+      internal: "image"
     }
   ];
   public ready: boolean = false;
@@ -43,14 +44,14 @@ class meydaHandler implements FormatHandler {
   async init () {
 
     const dummy = document.createElement("audio");
-    if (dummy.canPlayType("audio/wav")) this.supportedFormats.push({
+    this.supportedFormats.push({
       name: "Waveform Audio File Format",
       format: "wav",
       extension: "wav",
       mime: "audio/wav",
-      from: true,
-      to: false,
-      internal: "wav"
+      from: dummy.canPlayType("audio/wav") !== "",
+      to: true,
+      internal: "audio"
     });
     if (dummy.canPlayType("audio/mpeg")) this.supportedFormats.push({
       name: "MP3 Audio",
@@ -59,7 +60,7 @@ class meydaHandler implements FormatHandler {
       mime: "audio/mpeg",
       from: true,
       to: false,
-      internal: "mp3"
+      internal: "audio"
     });
     if (dummy.canPlayType("audio/ogg")) this.supportedFormats.push({
       name: "Ogg Audio",
@@ -68,7 +69,7 @@ class meydaHandler implements FormatHandler {
       mime: "audio/ogg",
       from: true,
       to: false,
-      internal: "ogg"
+      internal: "audio"
     });
     if (dummy.canPlayType("audio/flac")) this.supportedFormats.push({
       name: "Free Lossless Audio Codec",
@@ -77,11 +78,13 @@ class meydaHandler implements FormatHandler {
       mime: "audio/flac",
       from: true,
       to: false,
-      internal: "flac"
+      internal: "audio"
     });
     dummy.remove();
 
-    this.#audioContext = new AudioContext();
+    this.#audioContext = new AudioContext({
+      sampleRate: 44100
+    });
 
     this.#canvas = document.createElement("canvas");
     const ctx = this.#canvas.getContext("2d");
@@ -94,7 +97,7 @@ class meydaHandler implements FormatHandler {
 
   async doConvert (
     inputFiles: FileData[],
-    _inputFormat: FileFormat,
+    inputFormat: FileFormat,
     outputFormat: FileFormat
   ): Promise<FileData[]> {
     if (
@@ -107,48 +110,151 @@ class meydaHandler implements FormatHandler {
     }
     const outputFiles: FileData[] = [];
 
-    for (const inputFile of inputFiles) {
-
-      const inputBytes = new Uint8Array(inputFile.bytes);
-      const audioData = await this.#audioContext.decodeAudioData(inputBytes.buffer);
-
-      Meyda.bufferSize = 2048;
-      Meyda.sampleRate = audioData.sampleRate;
-      const samples = audioData.getChannelData(0);
-      const imageWidth = Math.floor(samples.length / Meyda.bufferSize);
-      const imageHeight = Meyda.bufferSize / 2;
-
-      this.#canvas.width = imageWidth;
-      this.#canvas.height = imageHeight;
-
-      for (let i = 0; i < imageWidth; i ++) {
-        const frame = samples.slice(i * Meyda.bufferSize, (i + 1) * Meyda.bufferSize);
-        const filtered = Meyda.windowing(frame, "hanning");
-        const spectrum = Meyda.extract("amplitudeSpectrum", filtered);
-        if (!(spectrum instanceof Float32Array)) throw "Failed to extract audio features!";
-        const pixels = new Uint8ClampedArray(spectrum.length * 4);
-        for (let i = 0; i < spectrum.length; i ++) {
-          const int = Math.floor(spectrum[i] * 16777215);
-          pixels[i * 4] = int & 0xFF;
-          pixels[i * 4 + 1] = (int >> 8) & 0xFF;
-          pixels[i * 4 + 2] = (int >> 16) & 0xFF;
-          pixels[i * 4 + 3] = 0xFF;
+    const inputIsImage = (inputFormat.internal === "image");
+    const outputIsImage = (outputFormat.internal === "image");
+
+    const bufferSize = 2048;
+
+    if (inputIsImage === outputIsImage) {
+      throw "Invalid input/output format.";
+    }
+
+    if (inputIsImage) {
+      for (const inputFile of inputFiles) {
+
+        this.#ctx.clearRect(0, 0, this.#canvas.width, this.#canvas.width);
+
+        const blob = new Blob([inputFile.bytes as BlobPart], { type: inputFormat.mime });
+        const url = URL.createObjectURL(blob);
+
+        const image = new Image();
+        await new Promise((resolve, reject) => {
+          image.addEventListener("load", resolve);
+          image.addEventListener("error", reject);
+          image.src = url;
+        });
+
+        const imageWidth = image.naturalWidth;
+        const imageHeight = image.naturalHeight;
+
+        this.#canvas.width = imageWidth;
+        this.#canvas.height = imageHeight;
+        this.#ctx.drawImage(image, 0, 0);
+
+        const imageData = this.#ctx.getImageData(0, 0, imageWidth, imageHeight);
+        const pixelBuffer = imageData.data as Uint8ClampedArray;
+
+        const sampleRate = this.#audioContext.sampleRate;
+
+        const audioData = new Float32Array(imageWidth * bufferSize);
+
+        // Precompute sine and cosine waves for each frequency
+        const sineWaves = new Float32Array(imageHeight * bufferSize);
+        const cosineWaves = new Float32Array(imageHeight * bufferSize);
+        for (let y = 0; y < imageHeight; y ++) {
+          const frequency = (y / imageHeight) * (sampleRate / 2);
+          for (let s = 0; s < bufferSize; s ++) {
+            const timeInSeconds = s / sampleRate;
+            const angle = 2 * Math.PI * frequency * timeInSeconds;
+            sineWaves[y * bufferSize + s] = Math.sin(angle);
+            cosineWaves[y * bufferSize + s] = Math.cos(angle);
+          }
         }
-        const imageData = new ImageData(pixels as ImageDataArray, 1, imageHeight);
-        this.#ctx.putImageData(imageData, i, 0);
+
+        for (let x = 0; x < imageWidth; x ++) {
+          for (let y = 0; y < imageHeight; y ++) {
+            const pixelIndex = (x + y * imageWidth) * 4;
+
+            // Extract amplitude from R and G channels
+            const magInt = pixelBuffer[pixelIndex] + (pixelBuffer[pixelIndex + 1] << 8);
+            const amplitude = magInt / 65535;
+            // Extract phase from B channel
+            const phase = (pixelBuffer[pixelIndex + 2] / 255) * (2 * Math.PI) - Math.PI;
+
+            for (let s = 0; s < bufferSize; s ++) {
+              const timeIndex = x * bufferSize + s;
+              audioData[timeIndex] += amplitude * (
+                cosineWaves[y * bufferSize + s] * Math.cos(phase)
+                - sineWaves[y * bufferSize + s] * Math.sin(phase)
+              );
+            }
+          }
+        }
+
+        // Normalize output
+        let max = 0;
+        for (let i = 0; i < imageWidth * bufferSize; i ++) {
+          const magnitude = Math.abs(audioData[i]);
+          if (magnitude > max) max = magnitude;
+        }
+        for (let i = 0; i < imageWidth * bufferSize; i ++) {
+          audioData[i] /= max;
+        }
+
+        const wav = new WaveFile();
+        wav.fromScratch(1, sampleRate, "32f", audioData);
+
+        const bytes = wav.toBuffer();
+        const name = inputFile.name.split(".")[0] + "." + outputFormat.extension;
+        outputFiles.push({ bytes, name });
+
       }
+    } else {
+      for (const inputFile of inputFiles) {
 
-      const bytes: Uint8Array = await new Promise((resolve, reject) => {
-        this.#canvas!.toBlob((blob) => {
-          if (!blob) return reject("Canvas output failed.");
-          blob.arrayBuffer().then(buf => resolve(new Uint8Array(buf)));
-        }, outputFormat.mime);
-      });
-      const name = inputFile.name.split(".")[0] + "." + outputFormat.extension;
-      outputFiles.push({ bytes, name });
+        const inputBytes = new Uint8Array(inputFile.bytes);
+        const audioData = await this.#audioContext.decodeAudioData(inputBytes.buffer);
 
+        Meyda.bufferSize = bufferSize;
+        Meyda.sampleRate = audioData.sampleRate;
+        const samples = audioData.getChannelData(0);
+        const imageWidth = Math.floor(samples.length / Meyda.bufferSize);
+        const imageHeight = Meyda.bufferSize / 2;
+
+        this.#canvas.width = imageWidth;
+        this.#canvas.height = imageHeight;
+
+        for (let i = 0; i < imageWidth; i ++) {
+
+          const frame = samples.slice(i * Meyda.bufferSize, (i + 1) * Meyda.bufferSize);
+          const spectrum = Meyda.extract("complexSpectrum", frame);
+          if (!spectrum || !("real" in spectrum) || !("imag" in spectrum)) {
+            throw "Failed to extract audio features!";
+          }
+          const real = spectrum.real as Float32Array;
+          const imaginary = spectrum.imag as Float32Array;
+
+          const pixels = new Uint8ClampedArray(imageHeight * 4);
+          for (let j = 0; j < imageHeight; j ++) {
+            const magnitude = Math.sqrt(real[j] * real[j] + imaginary[j] * imaginary[j]);
+            const phase = Math.atan2(imaginary[j], real[j]);
+            // Encode magnitude in R, G channels
+            const magInt = Math.floor(Math.min(magnitude * 65535, 65535));
+            pixels[j * 4] = magInt & 0xFF;
+            pixels[j * 4 + 1] = (magInt >> 8) & 0xFF;
+            // Encode phase in B channel
+            const phaseNormalized = Math.floor(((phase + Math.PI) / (2 * Math.PI)) * 255);
+            pixels[j * 4 + 2] = phaseNormalized;
+            pixels[j * 4 + 3] = 0xFF;
+          }
+          const imageData = new ImageData(pixels as ImageDataArray, 1, imageHeight);
+          this.#ctx.putImageData(imageData, i, 0);
+
+        }
+
+        const bytes: Uint8Array = await new Promise((resolve, reject) => {
+          this.#canvas!.toBlob((blob) => {
+            if (!blob) return reject("Canvas output failed.");
+            blob.arrayBuffer().then(buf => resolve(new Uint8Array(buf)));
+          }, outputFormat.mime);
+        });
+        const name = inputFile.name.split(".")[0] + "." + outputFormat.extension;
+        outputFiles.push({ bytes, name });
+
+      }
     }
 
+
     return outputFiles;
   }