import utils from "@/plugins/utils"; import dayjs from "dayjs"; export type AIInferenceResult = { aiGenerated: boolean; aiProbability: number; humanProbability: number; }; export const C2PASourceTypeScreenCapture = "http://cv.iptc.org/newscodes/digitalsourcetype/screenCapture"; export const C2PASourceTypeDigitalCapture = "http://cv.iptc.org/newscodes/digitalsourcetype/digitalCapture"; export const C2PASourceTypeComputationalCapture = "http://cv.iptc.org/newscodes/digitalsourcetype/computationalCapture"; export const C2PASourceTypeCompositeCapture = "http://cv.iptc.org/newscodes/digitalsourcetype/compositeCapture"; export const C2PASourceTypeTrainedAlgorithmicMedia = "http://cv.iptc.org/newscodes/digitalsourcetype/trainedAlgorithmicMedia"; export const C2PASourceTypeCompositeWithTrainedAlgorithmicMedia = "http://cv.iptc.org/newscodes/digitalsourcetype/compositeWithTrainedAlgorithmicMedia"; export type C2PAActionsAssertion = { actions: { action: string; softwareAgent?: string; digitalSourceType?: string; }[]; }; export type C2PAAssertion = { label: string; data: C2PAActionsAssertion | undefined; }; export type C2PAManifest = { assertions: C2PAAssertion[]; signature_info: { time: string; }; }; export type C2PAValidationResults = { activeManifest?: { failure: any[]; success: any[]; informational: any[]; }; }; export type C2PAManifestInfo = { active_manifest: string; manifests: { [key: string]: C2PAManifest }; validation_results?: C2PAValidationResults; }; export type C2PAData = { manifest_info: C2PAManifestInfo; }; export type Proof = { data?: any; name?: string; json?: string; integrity?: { pgp?: any; c2pa?: C2PAData; exif?: { [key: string]: string | Object }; opentimestamps?: any }; ai?: { inferenceResult?: AIInferenceResult }; }; export type MediaMetadataGenerator = "unknown" | "camera" | "screenshot" | "ai"; export type MediaMetadataPropertySource = "c2pa" | "exif" | "metadata"; export type MediaMetadataEdit = { editor: string; date?: Date; }; export type MediaMetadataLocation = { latitude: string; longitude: string; source: MediaMetadataPropertySource; }; export type MediaInterventionFlags = { creationDate?: Date; generator?: MediaMetadataGenerator; modified?: boolean; containsC2PA?: boolean; containsEXIF?: boolean; }; export type MediaMetadata = { device?: string; creationDate?: Date; creationDateSource?: MediaMetadataPropertySource; generator?: MediaMetadataGenerator; generatorSource?: MediaMetadataPropertySource; edits?: MediaMetadataEdit[]; containsC2PA?: boolean; containsEXIF?: boolean; location?: MediaMetadataLocation; }; type FlagMatchRule = { field: string; match: string[]; description: string; }; type FlagMatchRulePathSegment = { object: any; path: string; }; type FlagMatchRuleValue = { path: FlagMatchRulePathSegment[]; value: string | object; }; type FlagMatchInfo = { field: string; value: string; re: string; }; type FlagValue = { path: string; transform?: (value: any, match: FlagMatchRuleValue) => any; matches?: string[]; description?: string; // Not currently used }; const pathsC2PASource = (): FlagValue[] => { return [ { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:Make", transform: getExifMakeModelPrefixed, }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:Model", transform: getExifMakeModelPrefixed, }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/softwareAgent/name", }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/softwareAgent", }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/../../../claim_generator", }, ]; }; const pathsExifSource = (): FlagValue[] => { return [ { path: "integrity/exif/Make", transform: getExifMakeModelNoPrefix }, { path: "integrity/exif/Model", transform: getExifMakeModelNoPrefix }, ]; }; const pathsC2PACreationDate = (): FlagValue[] => { return [ { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/when", }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/../../metadata/dateTime", }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/../../../signature_info/time", }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:DateTimeOriginal", transform: (value, match) => { // Check if we have a timestamp offset let offset = extractFlagValues("../exif:OffsetTimeOriginal", match.path)?.at(0)?.value; return dayjs(Date.parse(value + (offset ? offset : ""))).format("lll"); }, }, ]; }; const pathsC2PACamera = (): FlagValue[] => { return [ { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:SceneType", matches: ["^1$", "^directly photographed image$"] }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:LensMake" }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:LensModel" }, ]; }; const pathsExifCamera = (): FlagValue[] => { return [ { path: "integrity/exif/SceneType", matches: ["^1$", "^directly photographed image$"] }, { path: "integrity/exif/LensMake" }, { path: "integrity/exif/LensModel" }, ]; }; const pathsExifScreenshot = (): FlagValue[] => { return [ { path: "integrity/exif/UserComment", transform: exifStringTransform, matches: ["^Screenshot$"], }, ]; }; const pathsMetaScreenshot = (): FlagValue[] => { return [ { path: "name", matches: ["screenshot"], } ]; }; const pathsC2PALocation = (): FlagValue[] => { return [ { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/GPSLatitude" }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/GPSLongitude" }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/GPSLatitudeRef" }, { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/GPSLongitudeRef" }, ]; }; const pathsExifLocation = (): FlagValue[] => { return [ { path: "integrity/exif/GPSLatitude" }, { path: "integrity/exif/GPSLongitude" }, { path: "integrity/exif/GPSLatitudeRef" }, { path: "integrity/exif/GPSLongitudeRef" }, ]; }; const pathsMetaAI = (): FlagValue[] => { const knownAIServices = [ "ChatGPT", "OpenAI-API", "Adobe Firefly", "RunwayML", "Runway AI", "Google AI", "Stable Diffusion" ]; return [ { path: "name", matches: ["^DALL_E_", "^Gen-3"], description: "File name" }, { path: "integrity/c2pa/manifest_info/manifests[]/claim_generator", matches: knownAIServices, description: "C2PA claim generator", }, { path: "iptc/Credit", matches: knownAIServices, description: "IPTC Credit" }, { path: "iptc/Provider", matches: knownAIServices, description: "IPTC Provider" }, { path: "iptc/ImageSupplier[]", matches: knownAIServices, description: "IPTC ImageSupplier" }, { path: "iptc/ImageCreator[]", matches: knownAIServices, description: "IPTC ImageCreator" }, ]; }; const getExifValue = (val: string): string => { return val.replace(/^"(.+(?="$))"$/, "$1"); }; const toDegrees = (dms: string | undefined, direction: string) => { if (!dms || dms.length == 0) return undefined; var parts = dms.split(/deg|min|sec/); var d = parts[0]; var m = parts[1]; var s = parts[2]; var deg = (Number(d) + Number(m) / 60 + Number(s) / 3600).toFixed(6); if (direction == "S" || direction == "W") { deg = "-" + deg; } return deg; }; const pathsExifCreationDate = (): FlagValue[] => { return [ { path: "integrity/exif/DateTimeOriginal", transform: (value, match) => { let dateTimeOriginal = getExifValue(value); // Check if we have a timestamp offset let offset = extractFlagValues("../OffsetTimeOriginal", match.path)?.at(0)?.value as string; return dayjs(Date.parse(dateTimeOriginal + (offset ? getExifValue(offset) : ""))).format("lll"); }, }, ]; }; const getExifMakeModelNoPrefix = (ignoredvalue: any, match: FlagMatchRuleValue): string => { return getExifMakeModel(match, ""); }; const getExifMakeModelPrefixed = (ignoredvalue: any, match: FlagMatchRuleValue): string => { return getExifMakeModel(match, "exif:"); }; const exifStringTransform = (value: any, match: FlagMatchRuleValue): string => { try { const s = value as string; if (s) { if (s.toLowerCase().startsWith("0x")) { if (s.toLowerCase().startsWith("0x554e49434f444500")) { // Unicode const buffer = Buffer.from(s.substring(18), "hex"); return buffer.toString("utf-8"); // } else if (s.toLowerCase().startsWith("0x4a49530000000000")) { // // JIS // const buffer = Buffer.from(s.substring(18), 'hex'); // return buffer.toString("ascii"); } else if ( s.toLowerCase().startsWith("0x4153434949000000") || s.toLowerCase().startsWith("0x0000000000000000") ) { // Ascii const buffer = Buffer.from(s.substring(18), "hex"); console.log(buffer.toString("ascii")); return buffer.toString("ascii"); } } } } catch (e) {} return value; }; const getExifMakeModel = (match: FlagMatchRuleValue, prefix: string): string => { // Make and model let makeAndModel = ""; const make = extractFlagValues(`../${prefix}Make`, match.path)?.at(0)?.value as string; const model = extractFlagValues(`../${prefix}Model`, match.path)?.at(0)?.value as string; if (make) { makeAndModel += getExifValue(make); } if (model) { if (makeAndModel.length > 0) { makeAndModel += ", "; } makeAndModel += getExifValue(model); } return makeAndModel; }; const extractFlagValues = (flagPath: string, path: FlagMatchRulePathSegment[]): FlagMatchRuleValue[] => { const getValues = (keys: string[], path: FlagMatchRulePathSegment[]): FlagMatchRuleValue[] | undefined => { if (keys.length == 0 || path.length == 0) return undefined; const o = path[0].object; let key = keys[0]; if (key === "..") { return getValues(keys.slice(1), path.slice(1)); } const lastBracket = key.lastIndexOf("["); const hasBracket = key.endsWith("]") && lastBracket > 0; let optionalConstraint: String | undefined = undefined; if (hasBracket) { optionalConstraint = key.substring(lastBracket + 1, key.length - 1); key = key.substring(0, lastBracket); } let nextObject = o[key]; let omatches: any[] = nextObject ? Array.isArray(nextObject) ? nextObject : hasBracket ? Object.values(nextObject) : [nextObject] : []; // Any constraints controlling what array object(s) to consider? if (optionalConstraint) { if (optionalConstraint.startsWith("!!")) { // Ignore this path in the tree if ANY of the values contain the constraint match. const [prop, val] = optionalConstraint.substring(2).split("="); let valarray = val.split("|"); if (omatches.some((item) => valarray.includes(item[prop]))) { omatches = []; } } else if (optionalConstraint.startsWith("!")) { const [prop, val] = optionalConstraint.substring(1).split("="); let valarray = val.split("|"); omatches = omatches.filter((m) => { return valarray.includes(m[prop]); }); } else { const [prop, val] = optionalConstraint.split("="); let valarray = val.split("|"); omatches = omatches.filter((m) => { return valarray.includes(m[prop]); }); } } if (omatches.length > 0) { if (keys.length == 1) { return omatches.map((oin, i) => { return { value: oin, path: [{ object: oin, path: key + (omatches.length > 1 ? `[${i}]` : "") }, ...path] }; }); } else if (omatches.length == 1) { return getValues(keys.slice(1), [{ object: omatches[0], path: key }, ...path]); } else { return omatches.reduce((res: FlagMatchRuleValue[] | undefined, oin: any, i: number) => { let matches = getValues(keys.slice(1), [{ object: oin, path: key + "[" + i + "]" }, ...path]); if (matches) { const r2 = res || []; r2.push(...matches); return r2; } return res; }, undefined); } } else { return undefined; } }; let result: FlagMatchRuleValue[] = []; try { let keys = flagPath.split("/"); result = getValues(keys, path) ?? []; } catch (e) { console.error("Invalid RE", e); } return result; }; const getFirstWithData = (flagValues: FlagValue[], path: FlagMatchRulePathSegment[]): string | undefined => { for (let idx = 0; idx < flagValues.length; idx++) { const result = extractFlagValues(flagValues[idx].path, path); if (result.length > 0) { try { let first = result[0]; let val: string | undefined = first.value as string; let transform = flagValues[idx].transform; if (val && transform) { val = transform(val, first); } if (val && flagValues[idx].matches) { if (!flagValues[idx].matches!.some((m) => { const re = new RegExp(m, "gi"); return re.test(val!); })) { val = undefined; } } if (val) { return val; } } catch (error) {} } } return undefined; }; const getFirstWithDataAsDate = (flagValues: FlagValue[], path: FlagMatchRulePathSegment[]): Date | undefined => { let val = getFirstWithData(flagValues, path); if (val) { try { let date = new Date(Date.parse(val)); if (isNaN(date.valueOf())) { // Try EXIF format date = utils.parseExifDate(val); } return date; } catch (error) {} } return undefined; }; const getMultiple = (flagValues: FlagValue[], path: FlagMatchRulePathSegment[]): (string | undefined)[] => { let results: (string | undefined)[] = new Array(flagValues.length); for (let idx = 0; idx < flagValues.length; idx++) { const result = extractFlagValues(flagValues[idx].path, path); if (result.length > 0) { results[idx] = result[0].value as string; } else { results[idx] = undefined; } } return results; }; export const mediaMetadataToMediaInterventionFlags = (mediaMetadata: MediaMetadata): MediaInterventionFlags => { return { creationDate: mediaMetadata.creationDate, generator: mediaMetadata.generator, modified: mediaMetadata.edits && mediaMetadata.edits.length > 0, containsC2PA: mediaMetadata.containsC2PA, containsEXIF: mediaMetadata.containsEXIF, }; }; export const extractMediaMetadata = (proof?: Proof): MediaMetadata | undefined => { if (!proof) return undefined; let edits: MediaMetadataEdit[] | undefined = undefined; let valid = false; try { let results = proof.integrity?.c2pa?.manifest_info.validation_results?.activeManifest; if (results) { valid = results.failure.length == 0 && results.success.length > 0; } const rootMatchPath = [{ object: proof, path: "" }]; let source: string | undefined = undefined; let dateCreated: Date | undefined = undefined; let dateCreatedSource: MediaMetadataPropertySource | undefined = undefined; source = getFirstWithData(pathsC2PASource(), rootMatchPath); if (!source) { source = getFirstWithData(pathsExifSource(), rootMatchPath); } dateCreated = getFirstWithDataAsDate(pathsC2PACreationDate(), rootMatchPath); if (dateCreated) { dateCreatedSource = "c2pa"; } else { dateCreated = getFirstWithDataAsDate(pathsExifCreationDate(), rootMatchPath); if (dateCreated) { dateCreatedSource = "exif"; } } let generator: MediaMetadataGenerator = "unknown"; let generatorSource: MediaMetadataPropertySource | undefined = undefined; let digitalSourceType = extractFlagValues( "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/digitalSourceType", rootMatchPath )?.at(0)?.value as string; if ([C2PASourceTypeScreenCapture].includes(digitalSourceType)) { generator = "screenshot"; generatorSource = "c2pa"; } else if ( [C2PASourceTypeDigitalCapture, C2PASourceTypeComputationalCapture, C2PASourceTypeCompositeCapture].includes( digitalSourceType ) ) { generator = "camera"; generatorSource = "c2pa"; } else if ( [C2PASourceTypeTrainedAlgorithmicMedia, C2PASourceTypeCompositeWithTrainedAlgorithmicMedia].includes( digitalSourceType ) ) { generator = "ai"; generatorSource = "c2pa"; } else if (getFirstWithData(pathsC2PACamera(), rootMatchPath)) { generator = "camera"; generatorSource = "c2pa"; } else if (getFirstWithData(pathsExifCamera(), rootMatchPath)) { generator = "camera"; generatorSource = "exif"; } else if (getFirstWithData(pathsExifScreenshot(), rootMatchPath)) { generator = "screenshot"; generatorSource = "exif"; } else if (getFirstWithData(pathsMetaScreenshot(), rootMatchPath)) { generator = "screenshot"; generatorSource = "metadata"; } else if (getFirstWithData(pathsMetaAI(), rootMatchPath)) { generator = "ai"; generatorSource = "metadata"; } console.error("PROOF", proof); const c2paEdits = extractFlagValues( "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[!!action=c2pa.created]", rootMatchPath ); if (c2paEdits.length > 0) { edits = c2paEdits.map((edit) => { return { editor: getFirstWithData( [{ path: "softwareAgent/name" }, { path: "softwareAgent" }, { path: "../../../claim_generator" }], edit.path ) ?? "", date: getFirstWithDataAsDate( [{ path: "when" }, { path: "../../metadata/dateTime" }, { path: "../../../signature_info/time" }], edit.path ), }; }); } // Location let location: MediaMetadataLocation | undefined = undefined; let locationSource: MediaMetadataPropertySource = "c2pa" let [lat, lon, latref, lonref] = getMultiple(pathsC2PALocation(), rootMatchPath); if (!lat || !lon) { [lat, lon, latref, lonref] = getMultiple(pathsExifLocation(), rootMatchPath); locationSource = "exif" } if (lat && lon) { try { const latitude = toDegrees(getExifValue(lat), getExifValue(latref ?? "")); const longitude = toDegrees(getExifValue(lon), getExifValue(lonref ?? "")); if (latitude && longitude && latitude.length > 0 && longitude.length > 0) { location = { latitude, longitude, source: "exif", }; } } catch (error) {} } // Do we have any data? Else, return "undefined", we don't just want to send an object with all defaults. if (source === undefined && dateCreated === undefined && generator === "unknown" && (!edits || edits.length == 0)) { return undefined; } const flags: MediaMetadata = { device: source, creationDate: dateCreated, creationDateSource: dateCreatedSource, generator: generator, generatorSource: generatorSource, edits: edits, containsC2PA: proof.integrity?.c2pa !== undefined, containsEXIF: proof.integrity?.exif !== undefined, location: location, }; return flags; } catch (error) {} return undefined; };