Editing Openai/691c1dba-9228-800f-8463-13b3a9006306 (section)

==== Create a CSV file with these columns (one row per platform × ingot run): ====

<syntaxhighlight>ingot_id,platform,model,temperature,system_prompt,restatement,anchor_line,quiet_plan,raw_response,timestamp

</syntaxhighlight>

Definitions:
* ingot_id: unique id for each ingot/session (e.g., UCF_2025_11_16_01)
* platform: textual platform name (e.g., DeepSeek, ChatGPT5)
* model: model string/version used (e.g., gpt-5.1)
* temperature: numeric used for generation
* system_prompt: exact system prompt (if any) used for that run
* restatement: the one-sentence restatement returned by the model (first required field)
* anchor_line: the 30-word anchor line returned by the model (second required field)
* quiet_plan: the <=20-word Quiet Room plan returned by the model (third required field)
* raw_response: the full raw reply from the model (for audit)
* timestamp: ISO timestamp of the run

Collect this for all ingot × platform runs (14 platforms × M ingots). Save as responses.csv.

==== <syntaxhighlight lang="python">#!/usr/bin/env python3 ====
"""
yAIy_analysis.py
Given a CSV of responses (ingot_id x platform runs), computes:
* embedding-based EC (uses sentence-transformers)
* AS (anchor similarity)
* BA (behavioral adherence keyword check)
* CCS per sample and aggregated per platform
* CPC (std of per-platform mean CCS)
* Permutation test (shuffle per-platform sample order) to compute p-value

Usage:
  python yAIy_analysis.py responses.csv output_summary.json

Note: Install dependencies:
  pip install numpy pandas scipy sentence-transformers
"""
import sys, json, os, time
import numpy as np
import pandas as pd
from sentence_transformers import SentenceTransformer
from scipy import stats

MODEL_NAME = "all-MiniLM-L6-v2"  # efficient embedder

def load_data(csv_path):
    df = pd.read_csv(csv_path, dtype=str)
    required = ["ingot_id","platform","restatement","anchor_line","quiet_plan"]
    for r in required:
        if r not in df.columns:
            raise SystemExit(f"Missing required column: {r}")
    return df

def compute_embeddings(texts, embedder):
    # returns numpy array shape (N, dim)
    embs = embedder.encode(texts, convert_to_numpy=True, show_progress_bar=False)
    return embs

def cosine_matrix(a,b):
    # a: (N,d), b: (M,d)
    a_norm = a / np.linalg.norm(a,axis=1,keepdims=True)
    b_norm = b / np.linalg.norm(b,axis=1,keepdims=True)
    return np.dot(a_norm, b_norm.T)

def compute_scores(df, embedder):
    # Build canonical ingot embedding per ingot_id using the restatement+anchor+quiet_plan concatenated
    ingot_texts = df.groupby("ingot_id").apply(
        lambda g: " ".join(g.iloc[0][["restatement","anchor_line","quiet_plan"]].fillna("").tolist())
    ).to_dict()
    ingot_ids = list(ingot_texts.keys())
    ingot_text_list = [ingot_texts[i] for i in ingot_ids]
    ingot_embs = compute_embeddings(ingot_text_list, embedder)

    # Compute embeddings for each row's fields
    rest_embs = compute_embeddings(df["restatement"].fillna("").tolist(), embedder)
    anchor_embs = compute_embeddings(df["anchor_line"].fillna("").tolist(), embedder)
    quiet_embs = compute_embeddings(df["quiet_plan"].fillna("").tolist(), embedder)

    # Map ingot_id -> embedding index
    ingot_index = {ingot_ids[i]: i for i in range(len(ingot_ids))}

    # For each row compute EC, AS, Qsim, BA
    EC = []
    AS = []
    Qsim = []
    BA = []
    keywords = ["low","lights","quiet","duration","min","minutes","silence"]
    for i, row in df.iterrows():
        ingidx = ingot_index[row["ingot_id"]]
        ing_emb = ingot_embs[ingidx]
        ec = float(np.dot(ing_emb, rest_embs[i]) / (np.linalg.norm(ing_emb)*np.linalg.norm(rest_embs[i])))
        asim = float(np.dot(ing_emb, anchor_embs[i]) / (np.linalg.norm(ing_emb)*np.linalg.norm(anchor_embs[i])))
        qsim = float(np.dot(ing_emb, quiet_embs[i]) / (np.linalg.norm(ing_emb)*np.linalg.norm(quiet_embs[i])))
        text_q = (row["quiet_plan"] or "").lower()
        ba = 1.0 if any(k in text_q for k in keywords) else 0.0
        EC.append(ec); AS.append(asim); Qsim.append(qsim); BA.append(ba)

    df["EC"] = EC; df["AS"] = AS; df["Qsim"] = Qsim; df["BA"] = BA
    # CCS = 0.5''EC + 0.3''AS + 0.2*BA
    df["CCS"] = 0.5''df["EC"].astype(float) + 0.3''df["AS"].astype(float) + 0.2*df["BA"].astype(float)
    return df

def aggregate_platform(df):
    grp = df.groupby("platform")["CCS"].agg(["mean","std","count"]).reset_index()
    grp = grp.rename(columns={"mean":"mean_CCS","std":"std_CCS","count":"n"})
    return grp

def permutation_test_matrix(matrix, nperm=2000, seed=42):
    np.random.seed(seed)
    P, M = matrix.shape
    obs = matrix.mean(axis=1).mean()
    perm_stats = np.zeros(nperm)
    for k in range(nperm):
        perm = np.zeros_like(matrix)
        for i in range(P):
            perm[i] = np.random.permutation(matrix[i])
        perm_stats[k] = perm.mean(axis=1).mean()
    pval = (np.sum(perm_stats >= obs) + 1) / (nperm + 1)
    return obs, perm_stats, pval

def main(csv_path, out_json):
    df = load_data(csv_path)
    embedder = SentenceTransformer(MODEL_NAME)
    df_scored = compute_scores(df, embedder)
    agg = aggregate_platform(df_scored)
    # Build CCS matrix: platforms x ingots, fill NaN if missing
    pivot = df_scored.pivot_table(index="platform", columns="ingot_id", values="CCS")
    matrix = pivot.fillna(pivot.mean(axis=1).values.reshape(-1,1)).to_numpy()
    obs, perm_stats, pval = permutation_test_matrix(matrix, nperm=3000)
    cpc = float(agg["mean_CCS"].std(ddof=1))
    overall_mean = float(agg["mean_CCS"].mean())
    se = float(agg["mean_CCS"].std(ddof=1) / np.sqrt(len(agg)))
    ci95 = [overall_mean - 1.96''se, overall_mean + 1.96''se]
    summary = {
        "platforms": agg.to_dict(orient="records"),
        "CPC": cpc,
        "overall_mean_CCS": overall_mean,
        "overall_mean_CCS_CI95": ci95,
        "permutation_p_value": float(pval),
        "note": "Run produced by local analysis script. Use real responses csv for true results."
    }
    with open(out_json, "w") as f:
        json.dump(summary, f, indent=2)
    print("Wrote summary to", out_json)
    return df_scored, summary

if __name__ == "__main__":
    if len(sys.argv) < 3:
        print("Usage: python yAIy_analysis.py responses.csv summary.json")
        sys.exit(1)
    csv_path = sys.argv[1]
    out_json = sys.argv[2]
    df_scored, summary = main(csv_path, out_json)

</syntaxhighlight>