MediaWiki:Gadget-LabelScanIndexer.js: Unterschied zwischen den Versionen

Version vom 22. November 2025, 19:25 Uhr

/* Gadget: LabelScanIndexer (Auto-Save, lokal, ESM via externem Shim)
 * Erzeugt Embeddings lokal (CLIP) und speichert in MediaWiki:Gadget-LabelScan-index.json
 * Läuft nur auf "Hilfe:LabelScan-Indexer"
 */

/* global mw */
(function () {
  'use strict';

  // ---------- Seitenerkennung ----------
  var NS = mw.config.get('wgNamespaceNumber'); // 12 = Hilfe/Help
  var TITLE = mw.config.get('wgTitle');        // Titel ohne Namespace
  var ON_PAGE = (NS === 12 && TITLE === 'LabelScan-Indexer');
  if (!ON_PAGE) { return; }

  var INDEX_TITLE = 'MediaWiki:Gadget-LabelScan-index.json';

  // ---------- Lokale Pfade ----------
  var TRANSFORMERS_SHIM = '/vendor/transformers/esm-shim.js'; // neu: externes Modul
  var WASM_DIR          = '/vendor/transformers/';            // enthält ort-wasm*.wasm
  var MODEL_ID          = 'Xenova/clip-vit-base-patch32';
  var LOCAL_MODEL_PATH  = '/models';

  // Files für Sanity-Check
  var CHECK_URLS = [
    LOCAL_MODEL_PATH + '/Xenova/clip-vit-base-patch32/preprocessor_config.json',
    LOCAL_MODEL_PATH + '/Xenova/clip-vit-base-patch32/onnx/vision_model_quantized.onnx'
  ];

  // ---------- Helpers ----------
  function $(id) { return document.getElementById(id); }
  function status(t) { var el = $('idx-status'); if (el) el.textContent = t || ''; }
  function log(){ try{ console.log.apply(console, ['[LabelScanIndexer]'].concat([].slice.call(arguments))); }catch(_){} }
  function warn(){ try{ console.warn.apply(console, ['[LabelScanIndexer]'].concat([].slice.call(arguments))); }catch(_){} }
  function err(){ try{ console.error.apply(console, ['[LabelScanIndexer]'].concat([].slice.call(arguments))); }catch(_){} }

  function hasInterfaceRight() {
    var groups = mw.config.get('wgUserGroups') || [];
    for (var i = 0; i < groups.length; i++) {
      if (groups[i] === 'interface-admin' || groups[i] === 'sysop') return true;
    }
    return false;
  }

  function float32ToBase64(vec) {
    var bytes = new Uint8Array(vec.buffer);
    var bin = '';
    var chunk = 0x8000;
    for (var i = 0; i < bytes.length; i += chunk) {
      bin += String.fromCharCode.apply(null, bytes.subarray(i, i + chunk));
    }
    return btoa(bin);
  }

  function fileToCanvasExif(file) {
    return new Promise(function (resolve, reject) {
      if ('createImageBitmap' in window) {
        createImageBitmap(file, { imageOrientation: 'from-image' }).then(function (bmp) {
          if ('OffscreenCanvas' in window) {
            var c1 = new OffscreenCanvas(bmp.width, bmp.height);
            c1.getContext('2d').drawImage(bmp, 0, 0);
            resolve(c1);
          } else {
            var c2 = document.createElement('canvas');
            c2.width = bmp.width; c2.height = bmp.height;
            c2.getContext('2d').drawImage(bmp, 0, 0);
            resolve(c2);
          }
        })["catch"](reject);
      } else {
        var url = URL.createObjectURL(file);
        var im = new Image();
        im.onload = function () {
          var c3 = document.createElement('canvas');
          c3.width = im.width; c3.height = im.height;
          c3.getContext('2d').drawImage(im, 0, 0);
          URL.revokeObjectURL(url);
          resolve(c3);
        };
        im.onerror = function (e) { URL.revokeObjectURL(url); reject(e); };
        im.src = url;
      }
    });
  }

  function canvasToBlobPromise(canvas) {
    if (canvas.convertToBlob) {
      return canvas.convertToBlob({ type: 'image/jpeg', quality: 0.95 });
    }
    return new Promise(function (resolve) {
      canvas.toBlob(function (b) { resolve(b); }, 'image/jpeg', 0.95);
    });
  }

  function timeoutPromise(p, ms, label) {
    return new Promise(function (resolve, reject) {
      var to = setTimeout(function(){ reject(new Error('Timeout: ' + (label||'operation') + ' nach ' + ms + ' ms')); }, ms);
      p.then(function(x){ clearTimeout(to); resolve(x); }, function(e){ clearTimeout(to); reject(e); });
    });
  }

  function headOk(url) {
    return fetch(url, { method: 'GET', cache: 'no-store' }).then(function(res){
      if (!res.ok) throw new Error('HTTP '+res.status+' bei '+url);
      return true;
    });
  }

  function preflightCheck() {
    log('Preflight-Check…');
    return Promise.all(CHECK_URLS.map(function(u){
      return timeoutPromise(headOk(u), 8000, 'check '+u).then(function(){ log('OK', u); return true; }, function(e){ throw new Error('Fehler beim Laden: '+u+'\n→ '+e.message); });
    }));
  }

  // ---------- ESM laden über externes Modul (kein inline) ----------
  function loadModuleFile(url) {
    return new Promise(function (resolve, reject) {
      var s = document.createElement('script');
      s.type = 'module';
      s.src = url;
      s.onload = function () { resolve(); };
      s.onerror = function () { reject(new Error('Module load failed: ' + url)); };
      document.head.appendChild(s);
    });
  }

  var _libPromise = null;
  function ensureLib() {
    if (_libPromise) return _libPromise;
    _libPromise = preflightCheck().then(function(){
      log('lade Transformers (ESM via Shim)…', TRANSFORMERS_SHIM);
      return loadModuleFile(TRANSFORMERS_SHIM).then(function () {
        var t0 = Date.now();
        return new Promise(function (resolve, reject) {
          (function spin() {
            if (window.transformers && typeof window.transformers === 'object') {
              // Env konfigurieren
              var env = window.transformers.env;
              env.allowLocalModels = true;
              env.allowRemoteModels = false;
              env.localModelPath   = LOCAL_MODEL_PATH;

              env.backends = env.backends || {};
              env.backends.onnx = env.backends.onnx || {};
              // Forciere WASM (WebGPU kann je nach Browser/CSP zicken)
              env.backends.onnx.preferredBackend = 'wasm';
              env.backends.onnx.wasm = env.backends.onnx.wasm || {};
              env.backends.onnx.wasm.wasmPaths = WASM_DIR;

              log('Transformers bereit.');
              resolve(window.transformers);
            } else if (Date.now() - t0 > 10000) {
              reject(new Error('Transformers-ESM nicht verfügbar (Timeout).'));
            } else {
              setTimeout(spin, 50);
            }
          })();
        });
      });
    });
    return _libPromise;
  }

  var _modelPromise = null;
  function ensureModel() {
    if (_modelPromise) return _modelPromise;
    _modelPromise = ensureLib().then(function (tf) {
      status('Modell laden …');
      log('lade Processor & Model…', MODEL_ID);

      var p = Promise.all([
        tf.AutoProcessor.from_pretrained(MODEL_ID),
        tf.CLIPVisionModelWithProjection.from_pretrained(MODEL_ID, { quantized: true })
      ]).then(function (arr) {
        var pack = { mod: tf, processor: arr[0], model: arr[1] };
        try {
          var backend = (pack.model && pack.model.session && pack.model.session.executionProvider) || 'unknown';
          log('Modell geladen | Backend:', backend);
        } catch (e) { log('Modell geladen'); }
        return pack;
      });

      return timeoutPromise(p, 25000, 'Model from_pretrained');
    });

    return _modelPromise;
  }

  function buildEmbeddingFromFile(file) {
    return ensureModel().then(function (pack) {
      status('Bild vorbereiten …');
      return timeoutPromise(fileToCanvasExif(file), 8000, 'Canvas aus Bild').then(function (canvas) {
        return timeoutPromise(canvasToBlobPromise(canvas), 8000, 'Canvas→Blob').then(function (blob) {
          status('Bild analysieren …');
          return timeoutPromise(pack.mod.RawImage.fromBlob(blob), 8000, 'RawImage').then(function (raw) {
            return timeoutPromise(pack.processor(raw, { return_tensors: 'pt' }), 12000, 'Processor').then(function (inputs) {
              return timeoutPromise(pack.model.forward({ pixel_values: inputs.pixel_values }), 20000, 'Model forward').then(function (out) {
                var vec = (out && out.image_embeds && out.image_embeds.data) || (out && out.image_embeds);
                if (!(vec instanceof Float32Array)) throw new Error('Embedding-Format unerwartet');

                // Normieren
                var i, n = 0;
                for (i = 0; i < vec.length; i++) n += vec[i] * vec[i];
                var norm = Math.sqrt(n) || 1;
                var v = new Float32Array(vec.length);
                for (i = 0; i < vec.length; i++) v[i] = vec[i] / norm;
                return v;
              });
            });
          });
        });
      });
    });
  }

  // ---------- Index laden/speichern ----------
  function fetchIndexJSON() {
    var url = mw.util.getUrl(INDEX_TITLE, { action: 'raw', ctype: 'application/json' });
    return fetch(url, { cache: 'no-store' }).then(function (res) {
      if (!res.ok) throw new Error('Index nicht ladbar: ' + res.status);
      return res.text();
    }).then(function (txt) {
      try { return JSON.parse(txt || '[]') || []; }
      catch (e) { return []; }
    });
  }

  function saveIndexJSON(newArray, summary) {
    return mw.loader.using(['mediawiki.api']).then(function () {
      var api = new mw.Api();
      var text = JSON.stringify(newArray, null, 2) + '\n';
      return api.postWithToken('csrf', {
        action: 'edit',
        title: INDEX_TITLE,
        text: text,
        summary: summary || 'LabelScan: +1 embedding (Auto-Indexer)',
        nocreate: 0,
        bot: 1
      });
    });
  }

  // ---------- Neu: Duplikat-Erkennung über EMBED ----------
  function findEntryByEmbed(indexArray, embedB64) {
    if (!indexArray || !indexArray.length || !embedB64) return null;
    for (var i = 0; i < indexArray.length; i++) {
      var it = indexArray[i];
      if (!it || typeof it.embed !== 'string') continue;
      if (it.embed === embedB64) {
        return it; // Duplikat gefunden
      }
    }
    return null;
  }

  // ---------- Click-Handler ----------
  var runBtn = document.getElementById('idx-run');
  if (!runBtn) {
    warn('Button #idx-run nicht gefunden – ist das HTML auf der Seite eingebunden?');
  } else {
    runBtn.addEventListener('click', function () {
      if (!hasInterfaceRight()) {
        alert('⚠️ Du brauchst Admin/Interface-Rechte (editinterface).');
        return;
      }

      var titleEl = $('idx-title');
      var thumbEl = $('idx-thumb');
      var fileEl  = $('idx-file');

      var title = titleEl ? String(titleEl.value || '').trim() : '';
      var thumb = thumbEl ? String(thumbEl.value || '').trim() : '';
      var file  = (fileEl && fileEl.files && fileEl.files[0]) ? fileEl.files[0] : null;

      if (!title) { alert('Titel fehlt.'); return; }
      if (!file)  { alert('Bitte eine Bilddatei wählen.'); return; }

      runBtn.disabled = true;
      status('Embedding berechnen …');
      log('Start embedding…', title, file && file.name);

      buildEmbeddingFromFile(file).then(function (vec) {
        var b64 = float32ToBase64(vec);
        var outBox = $('idx-out');
        if (outBox) outBox.value = JSON.stringify({ title: title, thumb: thumb, embed: b64 }, null, 2);

        status('Index laden …');
        return fetchIndexJSON().then(function (arr) {
          // NEU: Duplikat-Check über EMBED
          var existing = findEntryByEmbed(arr, b64);
          if (existing) {
            log('Duplikat-Embedding erkannt, nichts gespeichert.', existing);
            status('Embedding bereits im Index – nichts gespeichert.');
            alert(
              'Dieses Bild (Embedding) ist bereits im LabelScan-Index hinterlegt.\n' +
              'Vorhandener Eintrag: "' + (existing.title || 'unbekannt') + '".\n\n' +
              'Es wurde nichts geändert.'
            );
            // Signal nach außen: Speichern übersprungen
            return 'SKIP_DUPLICATE';
          }

          // Kein Duplikat → anhängen & speichern
          arr.push({ title: title, thumb: thumb, embed: b64 });
          status('Speichern …');
          return saveIndexJSON(arr, 'LabelScan: +1 embedding für "' + title + '"');
        });
      }).then(function (result) {
        if (result === 'SKIP_DUPLICATE') {
          log('Speichern übersprungen (Duplikat-Embedding).');
          // Status ist oben bereits gesetzt
        } else {
          status('Gespeichert ✅');
          log('Done.');
        }
      })["catch"](function (e) {
        err(e);
        status('Fehler ❌ ' + (e && e.message ? e.message : e));
        alert(
          'Fehler beim Erzeugen/Speichern:\n\n' +
          (e && e.message ? e.message : e) +
          '\n\nPrüfe bitte in der Konsole die [LabelScanIndexer]-Logs.'
        );
      }).then(function () {
        runBtn.disabled = false;
      });
    });
  }

  log('bereit');
})();