Support video thumbnails

src/models/attachment.ts

@@ -41,6 +41,14 @@ export type AttachmentSendInfo = {
   randomTranslationY: number; // For UI effects
 };
 
+export type AttachmentThumbnail = {
+  data: Uint8Array;
+  mimetype: string;
+  size: number;
+  w: number;
+  h: number;
+}
+
 export type Attachment = {
   status: "loading" | "loaded";
   file: File;
@@ -51,6 +59,7 @@ export type Attachment = {
   src?: string;
   proof?: Proof;
   sendInfo?: AttachmentSendInfo;
+  thumbnail?: AttachmentThumbnail;
 };
 
 export type AttachmentBatch = {

src/models/attachmentManager.ts

@@ -1,13 +1,24 @@
 import { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk";
-import { EventAttachment, EventAttachmentLoadSrcOptions, EventAttachmentUrlData, EventAttachmentUrlPromise, EventAttachmentUrlType, KeanuEvent, KeanuEventExtension } from "./eventAttachment";
+import {
+  EventAttachment,
+  EventAttachmentUrlData,
+  EventAttachmentUrlType,
+  KeanuEvent,
+  KeanuEventExtension,
+} from "./eventAttachment";
 import axios, { AxiosRequestConfig, AxiosResponse } from "axios";
 import { Counter, ModeOfOperation } from "aes-js";
-import { Attachment, AttachmentBatch, AttachmentSendInfo } from "./attachment";
+import {
+  Attachment,
+  AttachmentBatch,
+  AttachmentSendInfo,
+  AttachmentThumbnail,
+} from "./attachment";
 import proofmode from "../plugins/proofmode";
 import imageSize from "image-size";
 import imageResize from "image-resize";
-import { computed, isRef, Reactive, reactive, ref, Ref } from "vue";
-import utils from "@/plugins/utils";
+import { computed, Reactive, reactive, ref, Ref } from "vue";
+import utils, { THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT } from "@/plugins/utils";
 
 export class AttachmentManager {
   matrixClient: MatrixClient;
@@ -107,6 +118,60 @@ export class AttachmentManager {
         };
         reader.readAsDataURL(file);
       });
+    } else if (file.type.startsWith("video/")) {
+      let url = URL.createObjectURL(file);
+      const thumb: AttachmentThumbnail | undefined = await new Promise((resolve) => {
+        if (url) {
+          try {
+            let canvas: HTMLCanvasElement = document.createElement("canvas");
+
+            canvas.width = 320;
+            canvas.height = 240;
+            let video: HTMLVideoElement = document.createElement("video");
+            video.addEventListener("loadedmetadata", function () {
+              if (video.videoWidth > THUMBNAIL_MAX_WIDTH || video.videoHeight > THUMBNAIL_MAX_HEIGHT) {
+                var aspect = video.videoWidth / video.videoHeight;
+                canvas.width = parseInt(
+                  (video.videoWidth > video.videoHeight ? THUMBNAIL_MAX_WIDTH : THUMBNAIL_MAX_HEIGHT * aspect).toFixed()
+                );
+                canvas.height = parseInt(
+                  (video.videoWidth > video.videoHeight ? THUMBNAIL_MAX_WIDTH / aspect : THUMBNAIL_MAX_HEIGHT).toFixed()
+                );
+              } else {
+                canvas.width = video.videoWidth;
+                canvas.height = video.videoHeight;
+              }
+            });
+            video.addEventListener("loadeddata", (e) => {
+              const ctx = canvas.getContext("2d");
+              if (ctx) {
+                ctx.drawImage(video, 0, 0, video.videoWidth, video.videoHeight, 0, 0, canvas.width, canvas.height);
+                // ctx.fillStyle = "white";
+                // ctx.fillText("Thumbnail", 10, 10);
+              }
+              canvas.toBlob((b) => {
+                b?.arrayBuffer().then((data) => {
+                  resolve({
+                    data: new Uint8Array(data),
+                    mimetype: b.type,
+                    size: b.size,
+                    w: canvas.width,
+                    h: canvas.height,
+                  });
+                });
+              }, "image/png");
+            });
+            video.src = url;
+            video.currentTime = 0.1;
+          } catch (error) {
+            console.error("Failed to get video thumbnail: " + error);
+            resolve(undefined);
+          }
+        } else {
+          resolve(undefined);
+        }
+      });
+      attachment.thumbnail = thumb;
     }
 
     attachment.status = "loaded";
@@ -126,7 +191,6 @@ export class AttachmentManager {
     return 0;
   }
 
-
   public getEventAttachment(event: KeanuEvent): Reactive<EventAttachment> {
     let entry = this.cache.get(event.getId());
     if (entry !== undefined) {
@@ -149,7 +213,7 @@ export class AttachmentManager {
     });
     attachment.loadSrc = () => {
       if (attachment.src) {
-        return Promise.resolve({data: attachment.src, type: "src"});
+        return Promise.resolve({ data: attachment.src, type: "src" });
       } else if (attachment.srcPromise) {
         return attachment.srcPromise;
       }
@@ -159,11 +223,11 @@ export class AttachmentManager {
         attachment.src = (res as EventAttachmentUrlData).data;
         return res;
       }) as Promise<EventAttachmentUrlData>;
-      return attachment.srcPromise as Promise<{data:string,type:EventAttachmentUrlType}>;
+      return attachment.srcPromise as Promise<{ data: string; type: EventAttachmentUrlType }>;
     };
     attachment.loadThumbnail = () => {
       if (attachment.thumbnail) {
-        return Promise.resolve({data: attachment.thumbnail, type: "thumbnail"});
+        return Promise.resolve({ data: attachment.thumbnail, type: "thumbnail" });
       } else if (attachment.thumbnailPromise) {
         return attachment.thumbnailPromise;
       }
@@ -183,7 +247,7 @@ export class AttachmentManager {
       const promise = this._loadEventAttachmentOrThumbnail(event, false, true, (percent) => {
         attachment.srcProgress = percent;
       }).then((res) => {
-        return {data: res.data as Blob};
+        return { data: res.data as Blob };
       });
       return promise;
     };
@@ -209,7 +273,7 @@ export class AttachmentManager {
     thumbnail: boolean,
     asBlob: boolean,
     progress?: (percent: number) => void
-  ): Promise<EventAttachmentUrlData | {data: Blob, type: EventAttachmentUrlType}> {
+  ): Promise<EventAttachmentUrlData | { data: Blob; type: EventAttachmentUrlType }> {
     await this.matrixClient.decryptEventIfNeeded(event);
 
     let urltype: EventAttachmentUrlType = thumbnail ? "thumbnail" : "src";
@@ -301,7 +365,7 @@ export class AttachmentManager {
     const response = await axios.get(url, options);
     const bytes = decrypt ? await this.decryptData(file, response) : { buffer: response.data };
     const blob = new Blob([bytes.buffer], { type: mime });
-    return asBlob ? {data:  blob, type: urltype} : {data: URL.createObjectURL(blob), type: urltype};
+    return asBlob ? { data: blob, type: urltype } : { data: URL.createObjectURL(blob), type: urltype };
   }
 
   private b64toBuffer(val: any) {
@@ -485,7 +549,8 @@ export const createUploadBatch = (
                   }
                 },
                 eventId,
-                attachment.dimensions
+                attachment.dimensions,
+                attachment.thumbnail
               )
               .then((mediaEventId: string) => {
                 // Look at last item rotation, flipping the sign on this, so looks more like a true stack