From 1d336575690dfe2721e8fbe2646b54eae0e0ec5b Mon Sep 17 00:00:00 2001
From: N-Pex <npexdev@gmail.com>
Date: Fri, 25 Oct 2024 11:56:27 +0200
Subject: [PATCH 1/3] Generate base64 waveform in audio message body

---
 src/components/messages/AudioPlayer.vue       |  7 ++
 src/components/messages/AudioWaveformView.vue | 79 +++++++++++++++++++
 src/plugins/utils.js                          | 39 +++++++++
 3 files changed, 125 insertions(+)
 create mode 100644 src/components/messages/AudioWaveformView.vue

diff --git a/src/components/messages/AudioPlayer.vue b/src/components/messages/AudioPlayer.vue
index cf7472c..f0194b3 100644
--- a/src/components/messages/AudioPlayer.vue
+++ b/src/components/messages/AudioPlayer.vue
@@ -6,15 +6,22 @@
     <div class="play-time">
       {{ currentTime }} / {{ totalTime }}
     </div>
+    <div style="position:relative;flex: 1 1 100%">
     <v-slider @change="seeked" :disabled="!info.url" color="currentColor" track-color="#cccccc" class="play-progress" :value="info.playPercent" min="0"
       max="100" />
+      <div style="position:absolute;left:18px;right:18px;top:0;bottom:0;height:100%">
+      <AudioWaveformView :event="event" style="width:100%;height:100%" />
+      </div>
+    </div>
   </div>
 </template>
 
 <script>
 import util from "../../plugins/utils";
+import AudioWaveformView from "./AudioWaveformView.vue";
 
 export default {
+  components: { AudioWaveformView },
   props: {
     event: {
       type: Object,
diff --git a/src/components/messages/AudioWaveformView.vue b/src/components/messages/AudioWaveformView.vue
new file mode 100644
index 0000000..5f4dc8c
--- /dev/null
+++ b/src/components/messages/AudioWaveformView.vue
@@ -0,0 +1,79 @@
+<template>
+    <canvas ref="waveform">
+    </canvas>
+</template>
+
+<script>
+//  import util from "../../plugins/utils";
+
+export default {
+    props: {
+        event: {
+            type: Object,
+            default: function () {
+                return null;
+            },
+        },
+    },
+    data() {
+        return {
+        };
+    },
+    mounted() {
+        this.drawWaveform(this.event);
+    },
+    methods: {
+        drawWaveform(event) {
+            const canvas = this.$refs.waveform;
+            if (canvas) {
+                const dpr = window.devicePixelRatio || 1;
+                canvas.width = canvas.offsetWidth * dpr;
+                canvas.height = (canvas.offsetHeight) * dpr;
+
+                const ctx = canvas.getContext("2d");
+
+                ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+                const base64content = event ? event.getContent().formatted_body : undefined;
+                if (base64content) {
+                    const data = atob(base64content).split('').map(function (c) { return c.charCodeAt(0); });
+
+                    const drawLineSegment = (ctx, x, y0, data) => {
+                        ctx.lineWidth = 1; // how thick the line is
+                        ctx.strokeStyle = "#000"; // what color our line is
+                        ctx.beginPath();
+                        ctx.moveTo(x, y0 - data);
+                        ctx.lineTo(x, y0 + data);
+                        ctx.stroke();
+                    };
+
+                    console.log("W", canvas.width, canvas.height, canvas.offsetWidth, canvas.offsetHeight);
+
+                    const width = canvas.width;
+                    const height = canvas.height;
+                    const samples = data.length;
+                    for (let i = 0; i < width; i++) {
+                        if (i % 4 == 2 || i % 4 == 3) continue;
+                        const iSample = Math.floor(width == 0 ? 0 :(1000 * i) / width);
+                        const sample = iSample < samples ? data[iSample] : 0;
+                        drawLineSegment(ctx, i, height/2, (sample / 255) * height / 2);
+                    }
+                }
+            }
+
+        }
+    },
+    watch: {
+        event: {
+            immediate: false,
+            handler(event) {
+                this.drawWaveform(event);
+            }
+        }
+    }
+};
+</script>
+
+<style lang="scss">
+@import "@/assets/css/chat.scss";
+</style>
\ No newline at end of file
diff --git a/src/plugins/utils.js b/src/plugins/utils.js
index 56368b3..6af17d5 100644
--- a/src/plugins/utils.js
+++ b/src/plugins/utils.js
@@ -424,6 +424,9 @@ class Util {
                     promise
                         .then((response) => {
                             messageContent.url = response.content_uri;
+                            return (msgtype == 'm.audio' ? this.generateWaveform(fileContents, messageContent) : true);
+                        })
+                        .then(() => {
                             return this.sendMessage(matrixClient, roomId, "m.room.message", messageContent)
                         })
                         .then(result => {
@@ -478,6 +481,9 @@ class Util {
                             return reject(response.error);
                         }
                         encryptedFile.url = response.content_uri;
+                        return (msgtype == 'm.audio' ? this.generateWaveform(fileContents, messageContent) : true);
+                    })
+                    .then(() => {
                         return this.sendMessage(matrixClient, roomId, "m.room.message", messageContent)
                     })
                     .then(result => {
@@ -495,6 +501,39 @@ class Util {
         return uploadPromise;
     }
 
+    generateWaveform(data, messageContent) {
+        const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
+        if (audioCtx) {
+            return audioCtx.decodeAudioData(data)
+                .then((audioBuffer) => {
+                    const rawData = audioBuffer.getChannelData(0); // We only need to work with one channel of data
+                    const samples = 1000; // Number of samples we want to have in our final data set
+                    const blockSize = Math.floor(rawData.length / samples); // the number of samples in each subdivision
+                    let filteredData = [];
+                    for (let i = 0; i < samples; i++) {
+                      let blockStart = blockSize * i; // the location of the first sample in the block
+                      let sum = 0;
+                      for (let j = 0; j < blockSize; j++) {
+                        sum = sum + Math.abs(rawData[blockStart + j]) // find the sum of all the samples in the block
+                      }
+                      filteredData.push(sum / blockSize); // divide the sum by the block size to get the average
+                    }
+
+                    // Normalize
+                    const multiplier = Math.pow(Math.max(...filteredData), -1);
+                    filteredData = filteredData.map(n => n * multiplier);
+
+                    // Integerize
+                    filteredData = filteredData.map(n => parseInt((n * 255).toFixed()));
+                    
+                    const base64 = Buffer.from(filteredData).toString('base64'); //.replace(/=/g, '')
+                    console.log("BASE64", base64);
+                    messageContent.format = "org.matrix.custom.html";
+                    messageContent.formatted_body = base64;
+                })
+        }
+    }
+
     /**
      * Return what "mode" to use for the given room.
      *

From 1de3cc148e68d313ea52c4132351f3b7eac5780f Mon Sep 17 00:00:00 2001
From: N-Pex <npexdev@gmail.com>
Date: Fri, 25 Oct 2024 12:46:41 +0200
Subject: [PATCH 2/3] Switch to instead use SVG format for content body

---
 src/components/messages/AudioWaveformView.vue | 76 +++++--------------
 src/plugins/utils.js                          | 15 +++-
 2 files changed, 33 insertions(+), 58 deletions(-)

diff --git a/src/components/messages/AudioWaveformView.vue b/src/components/messages/AudioWaveformView.vue
index 5f4dc8c..905fbbc 100644
--- a/src/components/messages/AudioWaveformView.vue
+++ b/src/components/messages/AudioWaveformView.vue
@@ -1,10 +1,9 @@
 <template>
-    <canvas ref="waveform">
-    </canvas>
+    <div v-html="content" class="waveform">
+    </div>
 </template>
 
 <script>
-//  import util from "../../plugins/utils";
 
 export default {
     props: {
@@ -15,65 +14,32 @@ export default {
             },
         },
     },
-    data() {
-        return {
-        };
-    },
-    mounted() {
-        this.drawWaveform(this.event);
-    },
-    methods: {
-        drawWaveform(event) {
-            const canvas = this.$refs.waveform;
-            if (canvas) {
-                const dpr = window.devicePixelRatio || 1;
-                canvas.width = canvas.offsetWidth * dpr;
-                canvas.height = (canvas.offsetHeight) * dpr;
-
-                const ctx = canvas.getContext("2d");
-
-                ctx.clearRect(0, 0, canvas.width, canvas.height);
-
-                const base64content = event ? event.getContent().formatted_body : undefined;
-                if (base64content) {
-                    const data = atob(base64content).split('').map(function (c) { return c.charCodeAt(0); });
-
-                    const drawLineSegment = (ctx, x, y0, data) => {
-                        ctx.lineWidth = 1; // how thick the line is
-                        ctx.strokeStyle = "#000"; // what color our line is
-                        ctx.beginPath();
-                        ctx.moveTo(x, y0 - data);
-                        ctx.lineTo(x, y0 + data);
-                        ctx.stroke();
-                    };
-
-                    console.log("W", canvas.width, canvas.height, canvas.offsetWidth, canvas.offsetHeight);
-
-                    const width = canvas.width;
-                    const height = canvas.height;
-                    const samples = data.length;
-                    for (let i = 0; i < width; i++) {
-                        if (i % 4 == 2 || i % 4 == 3) continue;
-                        const iSample = Math.floor(width == 0 ? 0 :(1000 * i) / width);
-                        const sample = iSample < samples ? data[iSample] : 0;
-                        drawLineSegment(ctx, i, height/2, (sample / 255) * height / 2);
+    computed: {
+        content() {
+            return this.$sanitize(
+                this.event ? this.event.getContent().formatted_body : "",
+                {
+                    allowedTags: ["svg", "path"],
+                    allowedAttributes: {
+                        'svg': ["viewbox", "fill", "preserveaspectratio", "xmlns"],
+                        'path': ["d", "style"]
                     }
                 }
-            }
-
+            );
         }
     },
-    watch: {
-        event: {
-            immediate: false,
-            handler(event) {
-                this.drawWaveform(event);
-            }
-        }
-    }
 };
 </script>
 
 <style lang="scss">
 @import "@/assets/css/chat.scss";
+
+.waveform svg {
+    width: 100%;
+    height: 100%;
+
+    path {
+        stroke: black !important;
+    }
+}
 </style>
\ No newline at end of file
diff --git a/src/plugins/utils.js b/src/plugins/utils.js
index 6af17d5..86346d3 100644
--- a/src/plugins/utils.js
+++ b/src/plugins/utils.js
@@ -526,10 +526,19 @@ class Util {
                     // Integerize
                     filteredData = filteredData.map(n => parseInt((n * 255).toFixed()));
                     
-                    const base64 = Buffer.from(filteredData).toString('base64'); //.replace(/=/g, '')
-                    console.log("BASE64", base64);
+
+                    // Generate SVG of waveform
+                    let svg = `<svg viewBox="0 0 ${samples} 255" fill="none" preserveAspectRatio="none" xmlns="http://www.w3.org/2000/svg">`;
+                    svg += `<path d="`;
+                    filteredData.forEach((d, i) => {
+                        const delta = d / 2; 
+                        svg += `M${i} ${128 - delta}V${128 + delta}`;
+                    });
+                    svg += `" style="fill:none;stroke:green;stroke-width:1" />`;
+                    svg += "</svg>";
+
                     messageContent.format = "org.matrix.custom.html";
-                    messageContent.formatted_body = base64;
+                    messageContent.formatted_body = svg;
                 })
         }
     }

From f1bde3a7a0fdc4050315a3dd0225a15d5cd7cce0 Mon Sep 17 00:00:00 2001
From: N-Pex <npexdev@gmail.com>
Date: Fri, 25 Oct 2024 12:50:14 +0200
Subject: [PATCH 3/3] Check audio context support

---
 src/plugins/utils.js | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/plugins/utils.js b/src/plugins/utils.js
index 86346d3..bdc8646 100644
--- a/src/plugins/utils.js
+++ b/src/plugins/utils.js
@@ -502,13 +502,16 @@ class Util {
     }
 
     generateWaveform(data, messageContent) {
+        if (!(window.AudioContext || window.webkitAudioContext)) {
+            return; // No support
+        }
         const audioCtx = new (window.AudioContext || window.webkitAudioContext)();
         if (audioCtx) {
             return audioCtx.decodeAudioData(data)
                 .then((audioBuffer) => {
-                    const rawData = audioBuffer.getChannelData(0); // We only need to work with one channel of data
-                    const samples = 1000; // Number of samples we want to have in our final data set
-                    const blockSize = Math.floor(rawData.length / samples); // the number of samples in each subdivision
+                    const rawData = audioBuffer.getChannelData(0); // TODO - currently using only 1 channel
+                    const samples = 1000; // Number of samples
+                    const blockSize = Math.floor(rawData.length / samples);
                     let filteredData = [];
                     for (let i = 0; i < samples; i++) {
                       let blockStart = blockSize * i; // the location of the first sample in the block