diff --git a/src/components/chatMixin.js b/src/components/chatMixin.js index 40c1d5d..23c1941 100644 --- a/src/components/chatMixin.js +++ b/src/components/chatMixin.js @@ -52,6 +52,7 @@ import ReadMarker from "./messages/ReadMarker.vue"; import RoomTombstone from "./messages/composition/RoomTombstone.vue"; import roomDisplayOptionsMixin from "./roomDisplayOptionsMixin"; import roomTypeMixin from "./roomTypeMixin"; +import MessageThreadSending from "./messages/composition/MessageThreadSending.vue"; export const ROOM_READ_MARKER_EVENT_PLACEHOLDER = { getId: () => "ROOM_READ_MARKER", getTs: () => Date.now() }; @@ -225,6 +226,9 @@ export default { } return null; } + if (!isForExport && event.uploadBatch) { + return MessageThreadSending; + } if (event.isMxThread) { // Outgoing thread return isForExport ? MessageThreadExport : MessageThread; diff --git a/src/components/file_mode/SendAttachmentsLayout.vue b/src/components/file_mode/SendAttachmentsLayout.vue index 1edb68c..668545c 100644 --- a/src/components/file_mode/SendAttachmentsLayout.vue +++ b/src/components/file_mode/SendAttachmentsLayout.vue @@ -245,8 +245,8 @@ export default defineComponent({ batch: { type: Object, - default: function () { - return reactive(createUploadBatch(null, null, 0)); + default: () => { + return createUploadBatch(null, null); }, }, }, @@ -294,7 +294,7 @@ export default defineComponent({ this.currentItemIndex = newValue.length - 1; } }, - deep: true, + deep: 1, }, }, methods: { @@ -323,6 +323,7 @@ export default defineComponent({ }, sendAll() { this.status = this.mainStatuses.SENDING; + this.$emit("close"); this.batch .send(this.messageInput && this.messageInput.length > 0 ? this.messageInput : this.defaultRootMessageText) .then(() => { diff --git a/src/components/file_mode/ThumbnailView.vue b/src/components/file_mode/ThumbnailView.vue index ac00161..f6e83f2 100644 --- a/src/components/file_mode/ThumbnailView.vue +++ b/src/components/file_mode/ThumbnailView.vue @@ -23,7 +23,7 @@ + + + + diff --git a/src/models/attachment.ts b/src/models/attachment.ts index 2c03f08..e80b2c4 100644 --- a/src/models/attachment.ts +++ b/src/models/attachment.ts @@ -1,5 +1,5 @@ import { ComputedRef, Ref } from "vue"; -import { Proof } from "./proof"; +import { Proof, ProofHintFlags } from "./proof"; export class UploadPromise { wrappedPromise: Promise; @@ -39,6 +39,7 @@ export type AttachmentSendInfo = { randomRotation: number; // For UI effects randomTranslationX: number; // For UI effects randomTranslationY: number; // For UI effects + proofHintFlags?: ProofHintFlags; }; export type AttachmentThumbnail = { @@ -58,15 +59,15 @@ export type Attachment = { useScaled: boolean; src?: string; proof?: Proof; - sendInfo?: AttachmentSendInfo; thumbnail?: AttachmentThumbnail; + sendInfo: AttachmentSendInfo; }; export type AttachmentBatch = { - sendingStatus: Ref<"initial" | "sending" | "sent" | "canceled" | "failed">; + sendingStatus: Ref; sendingRootEventId: Ref; - sendingPromise: Ref | undefined>; - attachments: Ref; + progressPercent: Ref; + attachments: Attachment[]; attachmentsSentCount: ComputedRef; attachmentsSending: ComputedRef; attachmentsSent: ComputedRef; @@ -78,4 +79,3 @@ export type AttachmentBatch = { cancel: () => void; cancelSendAttachment: (attachment: Attachment) => void; }; - diff --git a/src/models/attachmentManager.ts b/src/models/attachmentManager.ts index 852fb11..fdcf1e0 100644 --- a/src/models/attachmentManager.ts +++ b/src/models/attachmentManager.ts @@ -1,4 +1,4 @@ -import { MatrixClient, MatrixEvent, Room } from "matrix-js-sdk"; +import { MatrixClient, MatrixEvent, Room, RoomEvent } from "matrix-js-sdk"; import { EventAttachment, EventAttachmentUrlData, @@ -8,11 +8,18 @@ import { } from "./eventAttachment"; import axios, { AxiosRequestConfig, AxiosResponse } from "axios"; import { Counter, ModeOfOperation } from "aes-js"; -import { Attachment, AttachmentBatch, AttachmentSendInfo, AttachmentThumbnail } from "./attachment"; +import { + Attachment, + AttachmentBatch, + AttachmentSendInfo, + AttachmentSendStatus, + AttachmentThumbnail, +} from "./attachment"; import proofmode from "../plugins/proofmode"; import imageResize from "image-resize"; -import { computed, Reactive, reactive, ref, Ref, shallowReactive } from "vue"; +import { computed, ref, Ref, shallowReactive, unref } from "vue"; import utils, { THUMBNAIL_MAX_WIDTH, THUMBNAIL_MAX_HEIGHT } from "@/plugins/utils"; +import { extractProofHintFlags } from "./proof"; export class AttachmentManager { matrixClient: MatrixClient; @@ -20,7 +27,8 @@ export class AttachmentManager { maxSizeUploads: number; maxSizeAutoDownloads: number; - cache: Map>; + cache: Map; + cacheUploads: Map; constructor(matrixClient: MatrixClient, useAuthedMedia: boolean, maxSizeAutoDownloads: number) { this.matrixClient = matrixClient; @@ -29,6 +37,7 @@ export class AttachmentManager { this.maxSizeAutoDownloads = maxSizeAutoDownloads; this.cache = new Map(); + this.cacheUploads = new Map(); // Get max upload size this.matrixClient @@ -40,7 +49,7 @@ export class AttachmentManager { } public createUpload(room: Room) { - return createUploadBatch(this.matrixClient, room, this.maxSizeUploads); + return createUploadBatch(this, room); } public createAttachment(file: File): Attachment { @@ -48,6 +57,16 @@ export class AttachmentManager { status: "loading", file: file, useScaled: false, + sendInfo: { + status: "initial", + statusDate: 0, + mediaEventId: undefined, + progress: 0, + promise: undefined, + randomRotation: 0, + randomTranslationX: 0, + randomTranslationY: 0, + }, }; const ra = shallowReactive(a); this.prepareUpload(ra); @@ -182,15 +201,15 @@ export class AttachmentManager { return 0; } - public getEventAttachment(event: KeanuEvent): Reactive { - let entry = this.cache.get(event.getId()); + public getEventAttachment(event: KeanuEvent): EventAttachment { + let entry = this.cache.get(event.getId() ?? "invalid"); if (entry !== undefined) { return entry; } const fileSize = this.getSrcFileSize(event); - const attachment: Reactive = shallowReactive({ + const attachment: EventAttachment = { event: event, name: this.getFileName(event), srcSize: fileSize, @@ -201,7 +220,7 @@ export class AttachmentManager { loadThumbnail: () => Promise.reject("Not implemented"), loadBlob: () => Promise.reject("Not implemented"), release: () => Promise.reject("Not implemented"), - }); + }; attachment.loadSrc = () => { if (attachment.src) { return Promise.resolve({ data: attachment.src, type: "src" }); @@ -246,7 +265,7 @@ export class AttachmentManager { // TODO - figure out logic if (entry) { // TODO - abortable promises - this.cache.delete(event.getId()); + this.cache.delete(event.getId() ?? "invalid"); if (attachment.src) { URL.revokeObjectURL(attachment.src); } @@ -255,7 +274,9 @@ export class AttachmentManager { } } }; - this.cache.set(event.getId(), attachment!); + if (event.getId()) { + this.cache.set(event.getId()!, attachment!); + } return attachment; } @@ -391,16 +412,35 @@ export class AttachmentManager { } } -export const createUploadBatch = ( - matrixClient: MatrixClient | null, - room: Room | null, - maxSizeUploads: number -): AttachmentBatch => { - const sendingStatus: Ref<"initial" | "sending" | "sent" | "canceled" | "failed"> = ref("initial"); +export const createUploadBatch = (manager: AttachmentManager | null, room: Room | null): AttachmentBatch => { + const matrixClient = manager?.matrixClient; + const maxSizeUploads = manager?.maxSizeUploads ?? 0; + + const txnId = utils.randomPass(); + console.error("Created txnId", txnId); + + const sendingStatus: Ref = ref("initial"); const sendingRootEventId: Ref = ref(undefined); const sendingPromise: Ref | undefined> = ref(undefined); const attachments: Ref = ref([]); + const totalOriginalFileSize: Ref = ref(0); + const progressPercent: Ref = ref(0); + + const updateProgress = () => { + // Use relative sizes of the original files to determine how many percent + // the individual files contribute to the total progress. + const progress = attachments.value.reduce((cb, item) => { + const info = item.sendInfo; + const thisFileCurrent = item.file.size; + const max = totalOriginalFileSize.value > 0 ? thisFileCurrent / totalOriginalFileSize.value : 0; + const q = (info.progress * max) / 100; + return cb + q; + }, 0); + const percent = parseInt(Math.floor(progress * 100).toFixed()); + progressPercent.value = percent; + }; + const attachmentsSentCount = computed(() => { return attachments.value.reduce((a, elem) => (elem.sendInfo?.status == "sent" ? a + 1 : a), 0); }); @@ -486,9 +526,10 @@ export const createUploadBatch = ( } }; - const send = (message: string): Promise => { + const send = async (message: string): Promise => { if (!matrixClient || !room) return Promise.reject("Not configured"); sendingStatus.value = "sending"; + attachments.value.forEach((attachment) => { let sendInfo: AttachmentSendInfo = { status: "initial", @@ -499,25 +540,54 @@ export const createUploadBatch = ( randomTranslationX: 0, randomTranslationY: 0, promise: undefined, + proofHintFlags: extractProofHintFlags(attachment.proof), }; attachment.sendInfo = shallowReactive(sendInfo); + attachment.proof = undefined; }); - const sendingPromise = utils - .sendTextMessage(matrixClient, room.roomId, message) + totalOriginalFileSize.value = attachments.value.reduce((cb, item) => { + const thisFileCurrent = item.scaledFile && item.useScaled ? item.scaledFile.size : item.file.size; + return cb + thisFileCurrent; + }, 0); + + const onLocalEchoUpdated = (event: KeanuEvent) => { + console.error("Local echo updated", event.getTxnId(), event.uploadBatch); + if (!event.uploadBatch && event.getTxnId() === txnId) { + event.uploadBatch = { + sendingStatus, + sendingRootEventId, + progressPercent, + attachments: unref(attachments), + attachmentsSentCount, + attachmentsSending, + attachmentsSent, + addAttachments, + removeAttachment, + isTooLarge, + canSend, + send, + cancel, + cancelSendAttachment, + }; + } + }; + room.on(RoomEvent.LocalEchoUpdated, onLocalEchoUpdated); + + const promise = utils + .sendTextMessage(matrixClient, room.roomId, message, undefined, undefined, txnId) .then((eventId: string) => { sendingRootEventId.value = eventId; - // Use the eventId as a thread root for all the media let promiseChain = Promise.resolve(); const getItemPromise = (index: number) => { if (index < attachments.value.length) { const attachment = attachments.value[index]; - const item = attachment.sendInfo!; - if (item.status !== "initial") { + const item = attachment; + if (item.sendInfo.status !== "initial") { return getItemPromise(++index); } - item.status = "sending"; + item.sendInfo.status = "sending"; let file = (() => { if (attachment.scaledFile && attachment.useScaled) { @@ -536,10 +606,11 @@ export const createUploadBatch = ( file, ({ loaded, total }: { loaded: number; total: number }) => { if (loaded == total) { - item.progress = 100; + item.sendInfo.progress = 100; } else if (total > 0) { - item.progress = (100 * loaded) / total; + item.sendInfo.progress = (100 * loaded) / total; } + updateProgress(); }, eventId, attachment.dimensions, @@ -561,23 +632,23 @@ export const createUploadBatch = ( signY = -1; } } - item.randomRotation = signR * (2 + Math.random() * 10); - item.randomTranslationX = signX * Math.random() * 20; - item.randomTranslationY = signY * Math.random() * 20; - item.mediaEventId = mediaEventId; - item.status = "sent"; - item.statusDate = Date.now(); + item.sendInfo.randomRotation = signR * (2 + Math.random() * 10); + item.sendInfo.randomTranslationX = signX * Math.random() * 20; + item.sendInfo.randomTranslationY = signY * Math.random() * 20; + item.sendInfo.mediaEventId = mediaEventId; + item.sendInfo.status = "sent"; + item.sendInfo.statusDate = Date.now(); }) .catch((ignorederr: any) => { - if (item.promise?.aborted) { - item.status = "canceled"; + if (item.sendInfo.promise?.aborted) { + item.sendInfo.status = "canceled"; } else { console.error("ERROR", ignorederr); - item.status = "failed"; + item.sendInfo.status = "failed"; } return Promise.resolve(); }); - item.promise = itemPromise; + item.sendInfo.promise = itemPromise; return itemPromise.then(() => getItemPromise(++index)); } else return Promise.resolve(); }; @@ -589,16 +660,20 @@ export const createUploadBatch = ( sendingRootEventId.value = undefined; }) .catch((err: any) => { - console.error("ERROR", err); + console.error("Upload error", err); + }) + .finally(() => { + room.off(RoomEvent.LocalEchoUpdated, onLocalEchoUpdated); }); - return sendingPromise; + sendingPromise.value = promise; + return promise; }; return { sendingStatus, sendingRootEventId, - sendingPromise, - attachments, + progressPercent, + attachments: unref(attachments), attachmentsSentCount, attachmentsSending, attachmentsSent, diff --git a/src/models/eventAttachment.ts b/src/models/eventAttachment.ts index 0635501..d7ae64f 100644 --- a/src/models/eventAttachment.ts +++ b/src/models/eventAttachment.ts @@ -1,4 +1,5 @@ import { MatrixEvent, Room } from "matrix-js-sdk"; +import { AttachmentBatch } from "./attachment"; export type KeanuEventExtension = { isMxThread?: boolean; @@ -6,6 +7,7 @@ export type KeanuEventExtension = { isPinned?: boolean; parentThread?: MatrixEvent & KeanuEventExtension; replyEvent?: MatrixEvent & KeanuEventExtension; + uploadBatch?: AttachmentBatch; } export type KeanuEvent = MatrixEvent & KeanuEventExtension; diff --git a/src/models/proof.ts b/src/models/proof.ts index 5d43922..deee838 100644 --- a/src/models/proof.ts +++ b/src/models/proof.ts @@ -53,6 +53,72 @@ export type Proof = { data?: any; name?: string; json?: string; - integrity?: { pgp?: any; c2pa?: any; exif?: {[key: string]: string | Object}; opentimestamps?: any }; + integrity?: { pgp?: any; c2pa?: C2PAData; exif?: {[key: string]: string | Object}; opentimestamps?: any }; ai?: { inferenceResult?: AIInferenceResult}; } + +export type ProofHintFlags = { + aiGenerated?: boolean; + aiEdited?: boolean; + screenshot?: boolean; + camera?: boolean; +} + +export const extractProofHintFlags = (proof?: Proof): (ProofHintFlags | undefined) => { + if (!proof) return undefined; + + let screenshot = false; + let camera = false; + let aiGenerated = false; + let aiEdited = false; + let valid = false; + + try { + let results = proof.integrity?.c2pa?.manifest_info.validation_results?.activeManifest; + if (results) { + valid = results.failure.length == 0 && results.success.length > 0; + } + + const manifests = Object.values(proof.integrity?.c2pa?.manifest_info.manifests ?? {}); + for (const manifest of manifests) { + for (const assertion of manifest.assertions) { + if (assertion.label === "c2pa.actions") { + const actions = (assertion.data as C2PAActionsAssertion)?.actions ?? []; + const a = actions.find((a) => a.action === "c2pa.created"); + if (a) { + // creator.value = a.softwareAgent; + // dateCreated.value = dayjs(Date.parse(manifest.signature_info.time)); + if (a.digitalSourceType === C2PASourceTypeScreenCapture) { + screenshot = true; + } + if ( + a.digitalSourceType === C2PASourceTypeDigitalCapture || + a.digitalSourceType === C2PASourceTypeComputationalCapture || + a.digitalSourceType === C2PASourceTypeCompositeCapture + ) { + camera = true; + } + if ( + a.digitalSourceType === C2PASourceTypeTrainedAlgorithmicMedia || + a.digitalSourceType === C2PASourceTypeCompositeWithTrainedAlgorithmicMedia + ) { + aiGenerated = true; + } + return; + } + } + } + } + if (valid) { + const flags: ProofHintFlags = { + aiGenerated, + aiEdited, + screenshot, + camera + }; + return flags; + } + } catch (error) { + } + return undefined; +}; \ No newline at end of file diff --git a/src/plugins/touch.js b/src/plugins/touch.js new file mode 100644 index 0000000..7608c90 --- /dev/null +++ b/src/plugins/touch.js @@ -0,0 +1,13 @@ +import Hammer from "hammerjs"; + +export function singleOrDoubleTapRecognizer(element) { + // reference: https://codepen.io/jtangelder/pen/xxYyJQ + const hm = new Hammer.Manager(element); + + hm.add(new Hammer.Tap({ event: "doubletap", taps: 2 })); + hm.add(new Hammer.Tap({ event: "singletap" })); + + hm.get("doubletap").recognizeWith("singletap"); + hm.get("singletap").requireFailure("doubletap"); + return hm; +} diff --git a/src/plugins/utils.js b/src/plugins/utils.js index e76ada4..05aa3f2 100644 --- a/src/plugins/utils.js +++ b/src/plugins/utils.js @@ -4,7 +4,6 @@ import imageResize from "image-resize"; import { AutoDiscovery, Method } from "matrix-js-sdk"; import User from "../models/user"; import prettyBytes from "pretty-bytes"; -import Hammer from "hammerjs"; import { Thread } from "matrix-js-sdk/lib/models/thread"; import { imageSize } from "image-size"; import dayjs from "dayjs"; @@ -144,13 +143,29 @@ class Util { var file = null; let decrypt = true; if (!!content.info && !!content.info.thumbnail_url) { - url = matrixClient.mxcUrlToHttp(content.info.thumbnail_url, undefined, undefined, undefined, undefined, undefined, useAuthedMedia); + url = matrixClient.mxcUrlToHttp( + content.info.thumbnail_url, + undefined, + undefined, + undefined, + undefined, + undefined, + useAuthedMedia + ); decrypt = false; if (content.info.thumbnail_info) { mime = content.info.thumbnail_info.mimetype; } } else if (content.url != null) { - url = matrixClient.mxcUrlToHttp(content.url, undefined, undefined, undefined, undefined, undefined, useAuthedMedia); + url = matrixClient.mxcUrlToHttp( + content.url, + undefined, + undefined, + undefined, + undefined, + undefined, + useAuthedMedia + ); decrypt = false; if (content.info) { mime = content.info.mimetype; @@ -175,13 +190,17 @@ class Util { throw new Error("No url found!"); } - const response = await axios - .get(url, useAuthedMedia ? { - responseType: "arraybuffer", - headers: { - Authorization: `Bearer ${matrixClient.getAccessToken()}`, - }, - } : { responseType: "arraybuffer" }); + const response = await axios.get( + url, + useAuthedMedia + ? { + responseType: "arraybuffer", + headers: { + Authorization: `Bearer ${matrixClient.getAccessToken()}`, + }, + } + : { responseType: "arraybuffer" } + ); const bytes = decrypt ? await this.decryptIfNeeded(file, response) : { buffer: response.data }; return URL.createObjectURL(new Blob([bytes.buffer], { type: mime })); } @@ -217,7 +236,7 @@ class Util { }); } - sendTextMessage(matrixClient, roomId, text, editedEvent, replyToEvent) { + sendTextMessage(matrixClient, roomId, text, editedEvent, replyToEvent, txnId) { var content = ContentHelpers.makeTextMessage(text); if (editedEvent) { content["m.relates_to"] = { @@ -248,7 +267,7 @@ class Util { .join("\n"); content.body = prefix + "\n\n" + content.body; } - return this.sendMessage(matrixClient, roomId, "m.room.message", content); + return this.sendMessage(matrixClient, roomId, "m.room.message", content, txnId); } sendQuickReaction(matrixClient, roomId, emoji, event, extraData = {}) { @@ -306,10 +325,10 @@ class Util { return this.sendMessage(matrixClient, roomId, "org.matrix.msc3381.poll.response", content); } - sendMessage(matrixClient, roomId, eventType, content) { + sendMessage(matrixClient, roomId, eventType, content, txnId) { return new Promise((resolve, reject) => { matrixClient - .sendEvent(roomId, eventType, content, undefined, undefined) + .sendEvent(roomId, eventType, content, txnId, undefined) .then((result) => { console.log("Message sent: ", result); resolve(result.event_id); @@ -419,7 +438,7 @@ class Util { var description = file.name; var msgtype = "m.file"; - + if (thumbnail) { thumbnailData = thumbnail.data; thumbnailInfo = { @@ -428,8 +447,8 @@ class Util { w: thumbnail.w, h: thumbnail.h, }; - } - + } + if (file.type.startsWith("image/")) { msgtype = "m.image"; @@ -446,7 +465,9 @@ class Util { width: newWidth, height: newHeight, outputType: "blob", - }).catch(() => {return Promise.resolve(undefined)}); + }).catch(() => { + return Promise.resolve(undefined); + }); if (scaled && file.size > scaled.size) { thumbnailData = new Uint8Array(await scaled.arrayBuffer()); thumbnailInfo = { @@ -483,12 +504,19 @@ class Util { messageContent.filename = file.name; } + let totalBytes = 0; + let thumbBytes = 0; + const useEncryption = matrixClient.isRoomEncrypted(roomId); const dataUploadOpts = { type: useEncryption ? "application/octet-stream" : file.type, name: description, - progressHandler: onUploadProgress, + progressHandler: ({ loaded, total }) => { + if (onUploadProgress) { + onUploadProgress({ loaded: loaded + thumbBytes, total: totalBytes }); + } + }, onlyContentUri: false, }; @@ -498,6 +526,8 @@ class Util { data = encryptedBytes; } + totalBytes = data.length; + if (thumbnailData) { messageContent.thumbnail_info = thumbnailInfo; if (useEncryption) { @@ -505,10 +535,16 @@ class Util { messageContent.info.thumbnail_file = encryptedFile; thumbnailData = encryptedBytes; } + thumbBytes = thumbnailData.length; + totalBytes += thumbnailData.length; const thumbnailUploadOpts = { type: useEncryption ? "application/octet-stream" : file.type, name: "thumb:" + description, - progressHandler: onUploadProgress, + progressHandler: ({ loaded, total }) => { + if (onUploadProgress) { + onUploadProgress({ loaded: loaded, total: totalBytes }); + } + }, onlyContentUri: false, }; const thumbUploadPromise = matrixClient.uploadContent(thumbnailData, thumbnailUploadOpts); @@ -1162,18 +1198,6 @@ class Util { return mobileTabletPattern.test(userAgent); } - singleOrDoubleTabRecognizer(element) { - // reference: https://codepen.io/jtangelder/pen/xxYyJQ - const hm = new Hammer.Manager(element); - - hm.add(new Hammer.Tap({ event: "doubletap", taps: 2 })); - hm.add(new Hammer.Tap({ event: "singletap" })); - - hm.get("doubletap").recognizeWith("singletap"); - hm.get("singletap").requireFailure("doubletap"); - return hm; - } - /** * Possibly convert numerals to local representation (currently only for "bo" locale) * @param str String in which to convert numerals [0-9]