keanu-weblite/src/models/proof.ts

import utils from "@/plugins/utils";
import dayjs from "dayjs";

export type AIInferenceResult = {
  aiGenerated: boolean;
  aiProbability: number;
  humanProbability: number;
};

export const C2PASourceTypeScreenCapture = "http://cv.iptc.org/newscodes/digitalsourcetype/screenCapture";
export const C2PASourceTypeDigitalCapture = "http://cv.iptc.org/newscodes/digitalsourcetype/digitalCapture";
export const C2PASourceTypeComputationalCapture = "http://cv.iptc.org/newscodes/digitalsourcetype/computationalCapture";
export const C2PASourceTypeCompositeCapture = "http://cv.iptc.org/newscodes/digitalsourcetype/compositeCapture";
export const C2PASourceTypeTrainedAlgorithmicMedia =
  "http://cv.iptc.org/newscodes/digitalsourcetype/trainedAlgorithmicMedia";
export const C2PASourceTypeCompositeWithTrainedAlgorithmicMedia =
  "http://cv.iptc.org/newscodes/digitalsourcetype/compositeWithTrainedAlgorithmicMedia";

export type C2PAActionsAssertion = {
  actions: {
    action: string;
    softwareAgent?: string;
    digitalSourceType?: string;
  }[];
};

export type C2PAAssertion = {
  label: string;
  data: C2PAActionsAssertion | undefined;
};

export type C2PAManifest = {
  assertions: C2PAAssertion[];
  signature_info: {
    time: string;
  };
};

export type C2PAValidationResults = {
  activeManifest?: {
    failure: any[];
    success: any[];
    informational: any[];
  };
};

export type C2PAManifestInfo = {
  active_manifest: string;
  manifests: { [key: string]: C2PAManifest };
  validation_results?: C2PAValidationResults;
};

export type C2PAData = {
  manifest_info: C2PAManifestInfo;
};

export type Proof = {
  data?: any;
  name?: string;
  json?: string;
  integrity?: { pgp?: any; c2pa?: C2PAData; exif?: { [key: string]: string | Object }; opentimestamps?: any };
  ai?: { inferenceResult?: AIInferenceResult };
};

export type MediaMetadataGenerator = "unknown" | "camera" | "screenshot" | "ai";
export type MediaMetadataPropertySource = "c2pa" | "exif" | "metadata";

export type MediaMetadataEdit = {
  editor: string;
  date?: Date;
};

export type MediaMetadataLocation = {
  latitude: string;
  longitude: string;
  source: MediaMetadataPropertySource;
};

export type MediaInterventionFlags = {
  creationDate?: Date;
  generator?: MediaMetadataGenerator;
  modified?: boolean;
  containsC2PA?: boolean;
  containsEXIF?: boolean;
};

export type MediaMetadata = {
  device?: string;
  creationDate?: Date;
  creationDateSource?: MediaMetadataPropertySource;
  generator?: MediaMetadataGenerator;
  generatorSource?: MediaMetadataPropertySource;
  edits?: MediaMetadataEdit[];
  containsC2PA?: boolean;
  containsEXIF?: boolean;
  location?: MediaMetadataLocation;
};

type FlagMatchRule = {
  field: string;
  match: string[];
  description: string;
};

type FlagMatchRulePathSegment = {
  object: any;
  path: string;
};

type FlagMatchRuleValue = {
  path: FlagMatchRulePathSegment[];
  value: string | object;
};

type FlagMatchInfo = {
  field: string;
  value: string;
  re: string;
};

type FlagValue = {
  path: string;
  transform?: (value: any, match: FlagMatchRuleValue) => any;
  matches?: string[];
  description?: string; // Not currently used
};

const pathsC2PASource = (): FlagValue[] => {
  return [
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:Make",
      transform: getMakeModel,
    },
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:Model",
      transform: getMakeModel,
    },
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/softwareAgent/name",
    },
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/softwareAgent",
    },
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/tiff:Make",
      transform: getMakeModel,
    },
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/tiff:Model",
      transform: getMakeModel,
    },

    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/../../../claim_generator",
    },
  ];
};

const pathsExifSource = (): FlagValue[] => {
  return [
    { path: "integrity/exif/Make", transform: getMakeModel },
    { path: "integrity/exif/Model", transform: getMakeModel },
  ];
};

const pathsC2PACreationDate = (): FlagValue[] => {
  return [
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/when",
    },
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/../../metadata/dateTime",
    },
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/../../../signature_info/time",
    },
    {
      path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:DateTimeOriginal",
      transform: (value, match) => {
        // Check if we have a timestamp offset
        let offset = extractFlagValues("../exif:OffsetTimeOriginal", match.path)?.at(0)?.value;
        return dayjs(Date.parse(value + (offset ? offset : ""))).format("lll");
      },
    },
  ];
};

const pathsC2PACamera = (): FlagValue[] => {
  return [
    { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:SceneType", matches: ["^1$", "^directly photographed image$"] },
    { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:LensMake" },
    { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exif:LensModel" },
    { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exifEX:LensMake" },
    { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/exifEX:LensModel" },
  ];
};

const pathsExifCamera = (): FlagValue[] => {
  return [
    { path: "integrity/exif/SceneType", matches: ["^1$", "^directly photographed image$"] },
    { path: "integrity/exif/LensMake" },
    { path: "integrity/exif/LensModel" },
  ];
};

const pathsExifScreenshot = (): FlagValue[] => {
  return [
    {
      path: "integrity/exif/UserComment",
      transform: exifStringTransform,
      matches: ["^Screenshot$"],
    },
  ];
};

const pathsMetaScreenshot = (): FlagValue[] => {
return [  
    {
      path: "name",
      matches: ["screenshot"],
    }
  ];
};

const pathsC2PALocation = (): FlagValue[] => {
  return [
    { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/GPSLatitude" },
    { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/GPSLongitude" },
    { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/GPSLatitudeRef" },
    { path: "integrity/c2pa/manifest_info/manifests[]/assertions[label=stds.exif]/data/GPSLongitudeRef" },
  ];
};

const pathsExifLocation = (): FlagValue[] => {
  return [
    { path: "integrity/exif/GPSLatitude" },
    { path: "integrity/exif/GPSLongitude" },
    { path: "integrity/exif/GPSLatitudeRef" },
    { path: "integrity/exif/GPSLongitudeRef" },
  ];
};

const pathsMetaAI = (): FlagValue[] => {
  const knownAIServices = [
    "ChatGPT",
    "OpenAI-API",
    "Adobe Firefly",
    "RunwayML",
    "Runway AI",
    "Google AI",
    "Stable Diffusion"
  ];
  return [
    { path: "name", matches: ["^DALL_E_", "^Gen-3"], description: "File name" },
    {
      path: "integrity/c2pa/manifest_info/manifests[]/claim_generator",
      matches: knownAIServices,
      description: "C2PA claim generator",
    },
    { path: "iptc/Credit", matches: knownAIServices, description: "IPTC Credit" },
    { path: "iptc/Provider", matches: knownAIServices, description: "IPTC Provider" },
    { path: "iptc/ImageSupplier[]", matches: knownAIServices, description: "IPTC ImageSupplier" },
    { path: "iptc/ImageCreator[]", matches: knownAIServices, description: "IPTC ImageCreator" },
  ];
};

const getExifValue = (val: string): string => {
  return val.replace(/^"(.+(?="$))"$/, "$1");
};

const toDegrees = (dms: string | undefined, direction: string) => {
  if (!dms || dms.length == 0) return undefined;
  var parts = dms.split(/deg|min|sec/);
  var d = parts[0];
  var m = parts[1];
  var s = parts[2];
  var deg = (Number(d) + Number(m) / 60 + Number(s) / 3600).toFixed(6);
  if (direction == "S" || direction == "W") {
    deg = "-" + deg;
  }
  return deg;
};

const pathsExifCreationDate = (): FlagValue[] => {
  return [
    {
      path: "integrity/exif/DateTimeOriginal",
      transform: (value, match) => {
        let dateTimeOriginal = getExifValue(value);
        // Check if we have a timestamp offset
        let offset = extractFlagValues("../OffsetTimeOriginal", match.path)?.at(0)?.value as string;
        return dayjs(Date.parse(dateTimeOriginal + (offset ? getExifValue(offset) : ""))).format("lll");
      },
    },
  ];
};

const getMakeModel = (ignoredval: string, match: FlagMatchRuleValue): string => {
  let prefix: string = "";

  const pathLeaf = match.path.at(0)?.path;
  if (pathLeaf && pathLeaf.indexOf(":") >= 0) {
    prefix = pathLeaf.substring(0, pathLeaf.indexOf(":") + 1);
  }

    // Make and model
  let makeAndModel = "";
  const make = extractFlagValues(`../${prefix}Make`, match.path)?.at(0)?.value as string;
  const model = extractFlagValues(`../${prefix}Model`, match.path)?.at(0)?.value as string;
  if (make) {
    makeAndModel += getExifValue(make);
  }
  if (model) {
    if (makeAndModel.length > 0) {
      makeAndModel += ", ";
    }
    makeAndModel += getExifValue(model);
  }
  return makeAndModel;
};

const exifStringTransform = (value: any, match: FlagMatchRuleValue): string => {
  try {
    const s = value as string;
    if (s) {
      if (s.toLowerCase().startsWith("0x")) {
        if (s.toLowerCase().startsWith("0x554e49434f444500")) {
          // Unicode
          const buffer = Buffer.from(s.substring(18), "hex");
          return buffer.toString("utf-8");
          // } else if (s.toLowerCase().startsWith("0x4a49530000000000")) {
          //   // JIS
          //   const buffer = Buffer.from(s.substring(18), 'hex');
          //   return buffer.toString("ascii");
        } else if (
          s.toLowerCase().startsWith("0x4153434949000000") ||
          s.toLowerCase().startsWith("0x0000000000000000")
        ) {
          // Ascii
          const buffer = Buffer.from(s.substring(18), "hex");
          console.log(buffer.toString("ascii"));
          return buffer.toString("ascii");
        }
      }
    }
  } catch (e) {}
  return value;
};

const extractFlagValues = (flagPath: string, path: FlagMatchRulePathSegment[]): FlagMatchRuleValue[] => {
  const getValues = (keys: string[], path: FlagMatchRulePathSegment[]): FlagMatchRuleValue[] | undefined => {
    if (keys.length == 0 || path.length == 0) return undefined;

    const o = path[0].object;
    let key = keys[0];

    if (key === "..") {
      return getValues(keys.slice(1), path.slice(1));
    }

    const lastBracket = key.lastIndexOf("[");
    const hasBracket = key.endsWith("]") && lastBracket > 0;

    let optionalConstraint: String | undefined = undefined;

    if (hasBracket) {
      optionalConstraint = key.substring(lastBracket + 1, key.length - 1);
      key = key.substring(0, lastBracket);
    }

    let nextObject = o[key];
    let omatches: any[] = nextObject
      ? Array.isArray(nextObject)
        ? nextObject
        : hasBracket
        ? Object.values(nextObject)
        : [nextObject]
      : [];

    // Any constraints controlling what array object(s) to consider?
    if (optionalConstraint) {
      if (optionalConstraint.startsWith("!!")) {
        // Ignore this path in the tree if ANY of the values contain the constraint match.
        const [prop, val] = optionalConstraint.substring(2).split("=");
        let valarray = val.split("|");
        if (omatches.some((item) => valarray.includes(item[prop]))) {
          omatches = [];
        }
      } else if (optionalConstraint.startsWith("!")) {
        const [prop, val] = optionalConstraint.substring(1).split("=");
        let valarray = val.split("|");
        omatches = omatches.filter((m) => {
          return valarray.includes(m[prop]);
        });
      } else {
        const [prop, val] = optionalConstraint.split("=");
        let valarray = val.split("|");
        omatches = omatches.filter((m) => {
          return valarray.includes(m[prop]);
        });
      }
    }

    if (omatches.length > 0) {
      if (keys.length == 1) {
        return omatches.map((oin, i) => {
          return { value: oin, path: [{ object: oin, path: key + (omatches.length > 1 ? `[${i}]` : "") }, ...path] };
        });
      } else if (omatches.length == 1) {
        return getValues(keys.slice(1), [{ object: omatches[0], path: key }, ...path]);
      } else {
        return omatches.reduce((res: FlagMatchRuleValue[] | undefined, oin: any, i: number) => {
          let matches = getValues(keys.slice(1), [{ object: oin, path: key + "[" + i + "]" }, ...path]);
          if (matches) {
            const r2 = res || [];
            r2.push(...matches);
            return r2;
          }
          return res;
        }, undefined);
      }
    } else {
      return undefined;
    }
  };

  let result: FlagMatchRuleValue[] = [];
  try {
    let keys = flagPath.split("/");
    result = getValues(keys, path) ?? [];
  } catch (e) {
    console.error("Invalid RE", e);
  }
  return result;
};

const getFirstWithData = (flagValues: FlagValue[], path: FlagMatchRulePathSegment[]): string | undefined => {
  for (let idx = 0; idx < flagValues.length; idx++) {
    const result = extractFlagValues(flagValues[idx].path, path);
    if (result.length > 0) {
      try {
        let first = result[0];
        let val: string | undefined = first.value as string;
        let transform = flagValues[idx].transform;
        if (val && transform) {
          val = transform(val, first);
        }
        if (val && flagValues[idx].matches) {
          if (!flagValues[idx].matches!.some((m) => {
            const re = new RegExp(m, "gi");
            return re.test(val!);
          })) {
            val = undefined;
          }
        }
        if (val) {
          return val;
        }
      } catch (error) {}
    }
  }
  return undefined;
};

const getFirstWithDataAsDate = (flagValues: FlagValue[], path: FlagMatchRulePathSegment[]): Date | undefined => {
  let val = getFirstWithData(flagValues, path);
  if (val) {
    try {
      let date = new Date(Date.parse(val));
      if (isNaN(date.valueOf())) {
        // Try EXIF format
        date = utils.parseExifDate(val);
      }
      return date;
    } catch (error) {}
  }
  return undefined;
};

const getMultiple = (flagValues: FlagValue[], path: FlagMatchRulePathSegment[]): (string | undefined)[] => {
  let results: (string | undefined)[] = new Array(flagValues.length);
  for (let idx = 0; idx < flagValues.length; idx++) {
    const result = extractFlagValues(flagValues[idx].path, path);
    if (result.length > 0) {
      results[idx] = result[0].value as string;
    } else {
      results[idx] = undefined;
    }
  }
  return results;
};

export const mediaMetadataToMediaInterventionFlags = (mediaMetadata: MediaMetadata): MediaInterventionFlags => {
  return {
    creationDate: mediaMetadata.creationDate,
    generator: mediaMetadata.generator,
    modified: mediaMetadata.edits && mediaMetadata.edits.length > 0,
    containsC2PA: mediaMetadata.containsC2PA,
    containsEXIF: mediaMetadata.containsEXIF,
  };
};

export const extractMediaMetadata = (proof?: Proof): MediaMetadata | undefined => {
  if (!proof) return undefined;

  let edits: MediaMetadataEdit[] | undefined = undefined;
  let valid = false;

  try {
    let results = proof.integrity?.c2pa?.manifest_info.validation_results?.activeManifest;
    if (results) {
      valid = results.failure.length == 0 && results.success.length > 0;
    }

    const rootMatchPath = [{ object: proof, path: "" }];

    let source: string | undefined = undefined;
    let dateCreated: Date | undefined = undefined;
    let dateCreatedSource: MediaMetadataPropertySource | undefined = undefined;

    source = getFirstWithData(pathsC2PASource(), rootMatchPath);
    if (!source) {
      source = getFirstWithData(pathsExifSource(), rootMatchPath);
    }

    dateCreated = getFirstWithDataAsDate(pathsC2PACreationDate(), rootMatchPath);
    if (dateCreated) {
      dateCreatedSource = "c2pa";
    } else {
      dateCreated = getFirstWithDataAsDate(pathsExifCreationDate(), rootMatchPath);
      if (dateCreated) {
        dateCreatedSource = "exif";
      }
    }

    let generator: MediaMetadataGenerator = "unknown";
    let generatorSource: MediaMetadataPropertySource | undefined = undefined;

    let digitalSourceType = extractFlagValues(
      "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[action=c2pa.created]/digitalSourceType",
      rootMatchPath
    )?.at(0)?.value as string;
    if ([C2PASourceTypeScreenCapture].includes(digitalSourceType)) {
      generator = "screenshot";
      generatorSource = "c2pa";
    } else if (
      [C2PASourceTypeDigitalCapture, C2PASourceTypeComputationalCapture, C2PASourceTypeCompositeCapture].includes(
        digitalSourceType
      )
    ) {
      generator = "camera";
      generatorSource = "c2pa";
    } else if (
      [C2PASourceTypeTrainedAlgorithmicMedia, C2PASourceTypeCompositeWithTrainedAlgorithmicMedia].includes(
        digitalSourceType
      )
    ) {
      generator = "ai";
      generatorSource = "c2pa";
    } else if (getFirstWithData(pathsC2PACamera(), rootMatchPath)) {
      generator = "camera";
      generatorSource = "c2pa";
    } else if (getFirstWithData(pathsExifCamera(), rootMatchPath)) {
      generator = "camera";
      generatorSource = "exif";
    } else if (getFirstWithData(pathsExifScreenshot(), rootMatchPath)) {
      generator = "screenshot";
      generatorSource = "exif";
    } else if (getFirstWithData(pathsMetaScreenshot(), rootMatchPath)) {
      generator = "screenshot";
      generatorSource = "metadata";
    } else if (getFirstWithData(pathsMetaAI(), rootMatchPath)) {
      generator = "ai";
      generatorSource = "metadata";
    }

    console.error("PROOF", proof);

    const c2paEdits = extractFlagValues(
      "integrity/c2pa/manifest_info/manifests[]/assertions[label=c2pa.actions|c2pa.actions.v2]/data/actions[!!action=c2pa.created]",
      rootMatchPath
    );
    if (c2paEdits.length > 0) {
      edits = c2paEdits.map((edit) => {
        return {
          editor:
            getFirstWithData(
              [{ path: "softwareAgent/name" }, { path: "softwareAgent" }, { path: "../../../claim_generator" }],
              edit.path
            ) ?? "",
          date: getFirstWithDataAsDate(
            [{ path: "when" }, { path: "../../metadata/dateTime" }, { path: "../../../signature_info/time" }],
            edit.path
          ),
        };
      });
    }

    // Location
    let location: MediaMetadataLocation | undefined = undefined;
    let locationSource: MediaMetadataPropertySource = "c2pa"
    let [lat, lon, latref, lonref] = getMultiple(pathsC2PALocation(), rootMatchPath);
    if (!lat || !lon) {
      [lat, lon, latref, lonref] = getMultiple(pathsExifLocation(), rootMatchPath);
      locationSource = "exif"
    }
    if (lat && lon) {
      try {
        const latitude = toDegrees(getExifValue(lat), getExifValue(latref ?? ""));
        const longitude = toDegrees(getExifValue(lon), getExifValue(lonref ?? ""));
        if (latitude && longitude && latitude.length > 0 && longitude.length > 0) {
          location = {
            latitude,
            longitude,
            source: "exif",
          };
        }
      } catch (error) {}
    }

    // Do we have any data? Else, return "undefined", we don't just want to send an object with all defaults.
    if (source === undefined && dateCreated === undefined && generator === "unknown" && (!edits || edits.length == 0)) {
      return undefined;
    }

    const flags: MediaMetadata = {
      device: source,
      creationDate: dateCreated,
      creationDateSource: dateCreatedSource,
      generator: generator,
      generatorSource: generatorSource,
      edits: edits,
      containsC2PA: proof.integrity?.c2pa !== undefined,
      containsEXIF: proof.integrity?.exif !== undefined,
      location: location,
    };
    return flags;
  } catch (error) {}
  return undefined;
};