From 035069f505c9c4e38bdc36d4c8b4802f8d1e406b Mon Sep 17 00:00:00 2001
From: N-Pex <npexdev@gmail.com>
Date: Thu, 10 Jul 2025 09:46:07 +0200
Subject: [PATCH] Support video thumbnails

---
 src/components/file_mode/ThumbnailView.vue | 25 +++++-
 src/models/attachment.ts                   |  9 +++
 src/models/attachmentManager.ts            | 89 +++++++++++++++++++---
 src/plugins/utils.js                       | 19 ++++-
 4 files changed, 124 insertions(+), 18 deletions(-)

diff --git a/src/components/file_mode/ThumbnailView.vue b/src/components/file_mode/ThumbnailView.vue
index a3f7f58..0e58dc1 100644
--- a/src/components/file_mode/ThumbnailView.vue
+++ b/src/components/file_mode/ThumbnailView.vue
@@ -1,7 +1,7 @@
 <template>
   <div ref="thumbnailRef" style="width: 100%; height: 100%">
-    <v-responsive v-if="isVideo && source" :class="{ 'thumbnail-item': true, preview: previewOnly }">
-      <video :src="source" :controls="!previewOnly" class="w-100 h-100">
+    <v-responsive v-if="isVideo && (source || poster)" :class="{ 'thumbnail-item': true, preview: previewOnly }">
+      <video :src="source" :poster="poster" :controls="!previewOnly" class="w-100 h-100">
         {{ $t("fallbacks.video_file") }}
       </video>
     </v-responsive>
@@ -50,6 +50,26 @@ const emits = defineEmits<ThumbnailEmits>();
 
 let { isVideo, isImage, fileTypeIcon, fileTypeIconClass, fileName, fileSize } = useThumbnail(file?.file ?? item?.event);
 const fileURL: Ref<string | undefined> = ref(undefined);
+const poster: Ref<string | undefined> = ref(undefined);
+
+const updatePoster = () => {
+  if (props.item && isVideo.value) {
+    if (props.item.thumbnail) {
+      poster.value = props.item.thumbnail;
+    } else {
+      props.item
+        .loadThumbnail()
+        .then((url) => {
+          poster.value = url.data;
+        })
+        .catch(() => {
+          poster.value = undefined;
+        });
+    }
+  }
+};
+
+updatePoster();
 
 watch(props, (props: ThumbnailProps) => {
   const updates = useThumbnail(props.file?.file ?? props.item?.event);
@@ -59,6 +79,7 @@ watch(props, (props: ThumbnailProps) => {
   fileTypeIconClass = updates.fileTypeIconClass;
   fileName = updates.fileName;
   fileSize = updates.fileSize;
+  updatePoster();
 });
 
 const source = computed(() => {
diff --git a/src/models/attachment.ts b/src/models/attachment.ts
index 0653edc..0b8765f 100644
--- a/src/models/attachment.ts
+++ b/src/models/attachment.ts
@@ -41,6 +41,14 @@ export type AttachmentSendInfo = {
   randomTranslationY: number; // For UI effects
 };
 
+export type AttachmentThumbnail = {
+  data: Uint8Array;
+  mimetype: string;
+  size: number;
+  w: number;
+  h: number;
+}
+
 export type Attachment = {
   status: "loading" | "loaded";
   file: File;
@@ -51,6 +59,7 @@ export type Attachment = {
   src?: string;
   proof?: Proof;
   sendInfo?: AttachmentSendInfo;
+  thumbnail?: AttachmentThumbnail;
 };
 
 export type AttachmentBatch = {
diff --git a/src/models/attachmentManager.ts b/src/models/attachmentManager.ts
index e155084..c726756 100644
--- a/src/models/attachmentManager.ts
+++ b/src/models/attachmentManager.ts
@@ -1,13 +1,24 @@
 import { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk";
-import { EventAttachment, EventAttachmentLoadSrcOptions, EventAttachmentUrlData, EventAttachmentUrlPromise, EventAttachmentUrlType, KeanuEvent, KeanuEventExtension } from "./eventAttachment";
+import {
+  EventAttachment,
+  EventAttachmentUrlData,
+  EventAttachmentUrlType,
+  KeanuEvent,
+  KeanuEventExtension,
+} from "./eventAttachment";
 import axios, { AxiosRequestConfig, AxiosResponse } from "axios";
 import { Counter, ModeOfOperation } from "aes-js";
-import { Attachment, AttachmentBatch, AttachmentSendInfo } from "./attachment";
+import {
+  Attachment,
+  AttachmentBatch,
+  AttachmentSendInfo,
+  AttachmentThumbnail,
+} from "./attachment";
 import proofmode from "../plugins/proofmode";
 import imageSize from "image-size";
 import imageResize from "image-resize";
-import { computed, isRef, Reactive, reactive, ref, Ref } from "vue";
-import utils from "@/plugins/utils";
+import { computed, Reactive, reactive, ref, Ref } from "vue";
+import utils, { THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT } from "@/plugins/utils";
 
 export class AttachmentManager {
   matrixClient: MatrixClient;
@@ -107,6 +118,60 @@ export class AttachmentManager {
         };
         reader.readAsDataURL(file);
       });
+    } else if (file.type.startsWith("video/")) {
+      let url = URL.createObjectURL(file);
+      const thumb: AttachmentThumbnail | undefined = await new Promise((resolve) => {
+        if (url) {
+          try {
+            let canvas: HTMLCanvasElement = document.createElement("canvas");
+
+            canvas.width = 320;
+            canvas.height = 240;
+            let video: HTMLVideoElement = document.createElement("video");
+            video.addEventListener("loadedmetadata", function () {
+              if (video.videoWidth > THUMBNAIL_MAX_WIDTH || video.videoHeight > THUMBNAIL_MAX_HEIGHT) {
+                var aspect = video.videoWidth / video.videoHeight;
+                canvas.width = parseInt(
+                  (video.videoWidth > video.videoHeight ? THUMBNAIL_MAX_WIDTH : THUMBNAIL_MAX_HEIGHT * aspect).toFixed()
+                );
+                canvas.height = parseInt(
+                  (video.videoWidth > video.videoHeight ? THUMBNAIL_MAX_WIDTH / aspect : THUMBNAIL_MAX_HEIGHT).toFixed()
+                );
+              } else {
+                canvas.width = video.videoWidth;
+                canvas.height = video.videoHeight;
+              }
+            });
+            video.addEventListener("loadeddata", (e) => {
+              const ctx = canvas.getContext("2d");
+              if (ctx) {
+                ctx.drawImage(video, 0, 0, video.videoWidth, video.videoHeight, 0, 0, canvas.width, canvas.height);
+                // ctx.fillStyle = "white";
+                // ctx.fillText("Thumbnail", 10, 10);
+              }
+              canvas.toBlob((b) => {
+                b?.arrayBuffer().then((data) => {
+                  resolve({
+                    data: new Uint8Array(data),
+                    mimetype: b.type,
+                    size: b.size,
+                    w: canvas.width,
+                    h: canvas.height,
+                  });
+                });
+              }, "image/png");
+            });
+            video.src = url;
+            video.currentTime = 0.1;
+          } catch (error) {
+            console.error("Failed to get video thumbnail: " + error);
+            resolve(undefined);
+          }
+        } else {
+          resolve(undefined);
+        }
+      });
+      attachment.thumbnail = thumb;
     }
 
     attachment.status = "loaded";
@@ -126,7 +191,6 @@ export class AttachmentManager {
     return 0;
   }
 
-
   public getEventAttachment(event: KeanuEvent): Reactive<EventAttachment> {
     let entry = this.cache.get(event.getId());
     if (entry !== undefined) {
@@ -149,7 +213,7 @@ export class AttachmentManager {
     });
     attachment.loadSrc = () => {
       if (attachment.src) {
-        return Promise.resolve({data: attachment.src, type: "src"});
+        return Promise.resolve({ data: attachment.src, type: "src" });
       } else if (attachment.srcPromise) {
         return attachment.srcPromise;
       }
@@ -159,11 +223,11 @@ export class AttachmentManager {
         attachment.src = (res as EventAttachmentUrlData).data;
         return res;
       }) as Promise<EventAttachmentUrlData>;
-      return attachment.srcPromise as Promise<{data:string,type:EventAttachmentUrlType}>;
+      return attachment.srcPromise as Promise<{ data: string; type: EventAttachmentUrlType }>;
     };
     attachment.loadThumbnail = () => {
       if (attachment.thumbnail) {
-        return Promise.resolve({data: attachment.thumbnail, type: "thumbnail"});
+        return Promise.resolve({ data: attachment.thumbnail, type: "thumbnail" });
       } else if (attachment.thumbnailPromise) {
         return attachment.thumbnailPromise;
       }
@@ -183,7 +247,7 @@ export class AttachmentManager {
       const promise = this._loadEventAttachmentOrThumbnail(event, false, true, (percent) => {
         attachment.srcProgress = percent;
       }).then((res) => {
-        return {data: res.data as Blob};
+        return { data: res.data as Blob };
       });
       return promise;
     };
@@ -209,7 +273,7 @@ export class AttachmentManager {
     thumbnail: boolean,
     asBlob: boolean,
     progress?: (percent: number) => void
-  ): Promise<EventAttachmentUrlData | {data: Blob, type: EventAttachmentUrlType}> {
+  ): Promise<EventAttachmentUrlData | { data: Blob; type: EventAttachmentUrlType }> {
     await this.matrixClient.decryptEventIfNeeded(event);
 
     let urltype: EventAttachmentUrlType = thumbnail ? "thumbnail" : "src";
@@ -301,7 +365,7 @@ export class AttachmentManager {
     const response = await axios.get(url, options);
     const bytes = decrypt ? await this.decryptData(file, response) : { buffer: response.data };
     const blob = new Blob([bytes.buffer], { type: mime });
-    return asBlob ? {data:  blob, type: urltype} : {data: URL.createObjectURL(blob), type: urltype};
+    return asBlob ? { data: blob, type: urltype } : { data: URL.createObjectURL(blob), type: urltype };
   }
 
   private b64toBuffer(val: any) {
@@ -485,7 +549,8 @@ export const createUploadBatch = (
                   }
                 },
                 eventId,
-                attachment.dimensions
+                attachment.dimensions,
+                attachment.thumbnail
               )
               .then((mediaEventId: string) => {
                 // Look at last item rotation, flipping the sign on this, so looks more like a true stack
diff --git a/src/plugins/utils.js b/src/plugins/utils.js
index 34b0ca0..a2fb9f3 100644
--- a/src/plugins/utils.js
+++ b/src/plugins/utils.js
@@ -25,8 +25,8 @@ export const ROOM_TYPE_CHANNEL = "im.keanu.room_type_channel";
 
 export const STATE_EVENT_ROOM_TYPE = "im.keanu.room_type";
 
-const THUMBNAIL_MAX_WIDTH = 160;
-const THUMBNAIL_MAX_HEIGHT = 160;
+export const THUMBNAIL_MAX_WIDTH = 160;
+export const THUMBNAIL_MAX_HEIGHT = 160;
 
 // Install extended localized format
 dayjs.extend(localizedFormat);
@@ -391,7 +391,7 @@ class Util {
     return [encryptedBytes, encryptedFile];
   }
 
-  sendFile(matrixClient, roomId, file, onUploadProgress, threadRoot, dimensions) {
+  sendFile(matrixClient, roomId, file, onUploadProgress, threadRoot, dimensions, thumbnail) {
     const uploadPromise = new UploadPromise(undefined);
     uploadPromise.wrappedPromise = new Promise((resolve, reject) => {
       var reader = new FileReader();
@@ -419,11 +419,22 @@ class Util {
 
           var description = file.name;
           var msgtype = "m.file";
+          
+          if (thumbnail) {
+            thumbnailData = thumbnail.data;
+            thumbnailInfo = {
+              mimetype: thumbnail.mimetype,
+              size: thumbnail.size,
+              w: thumbnail.w,
+              h: thumbnail.h,
+            };
+          } 
+          
           if (file.type.startsWith("image/")) {
             msgtype = "m.image";
 
             // Generate thumbnail?
-            if (dimensions) {
+            if (dimensions && !thumbnail) {
               const w = dimensions.width;
               const h = dimensions.height;
               if (w > THUMBNAIL_MAX_WIDTH || h > THUMBNAIL_MAX_HEIGHT) {