/* agent_learn.c — the SLOW LOOP: the agent LEARNS its world model from experience, not handed it.
*
* belief_field_agent.c was handed the operators. Here the agent BABBLES (random actions), records
* (action, observation) experience, and learns the controlled HMM (emission E, action-conditioned
* transitions T[a]) by controlled Baum-Welch (EM), then controls using only the learned model. Result:
* the learned model recovers the shift dynamics (T[a=1] from state 0 peaks at ~0.9 on 'to=1') and controls
* AS WELL AS the agent handed the true model, far better than acting on raw observation. The world model is
* now ACQUIRED, not given -- the last autonomy piece. Tested on two emission structures (symmetric and
* asymmetric per-state) to show learning is robust; both close the loop.
*
* HONEST PATH (kept as a record). Two wrong turns preceded this: (1) a first version wired mom.c's spectral
* observable-operator read directly in -- it failed (worse than random) because the uncontrolled OOM does
* not transfer trivially to a controlled process (controlled spectral learning / PSR is genuinely subtle;
* that remains the right tool for FRACTAL-hard structure, per mom.c, but a proper task, not a quick wire-in).
* (2) EM then learned NOTHING (control = chance, T uniform), which I misdiagnosed as a symmetry "valley";
* it was actually an off-by-one ACTION-INDEXING bug -- the transition (t-1)->t is governed by Ab[t], not
* Ab[t-1]. Fixed, EM recovers the operators cleanly in BOTH environments. The lesson: characterize the bug,
* do not narrate a valley that is not there.
*
* build: cc -O2 -Wall -std=c11 -o agent_learn agent_learn.c -lm
*/
#include <math.h>
#include <stdio.h>
#define K 3
#define TN 0.15
#define GOAL 0
#define TGT 0
#define NB 100000
#define ITERS 60
#define CONTROL 300000
static unsigned long long RNG;
static unsigned long long xs(void){ RNG^=RNG<<13; RNG^=RNG>>7; RNG^=RNG<<17; return RNG; }
static double urand(void){ return (double)(xs()>>11)*(1.0/9007199254740992.0); }
static int rk(void){ return (int)(xs()%K); }
static double Emat[K][K]; /* current true emission (set per environment) */
static void set_symmetric(void){ for(int s=0;s<K;s++) for(int x=0;x<K;x++) Emat[s][x]=(s==x)?0.60:0.20; }
static void set_asymmetric(void){ double A[K][K]={{0.70,0.20,0.10},{0.15,0.70,0.15},{0.10,0.25,0.65}}; for(int s=0;s<K;s++)for(int x=0;x<K;x++) Emat[s][x]=A[s][x]; }
static double Ptrans(int j,int i,int a){ return (1.0-TN)*((j==(i+a)%K)?1.0:0.0) + TN/K; }
static double Pemit (int x,int s){ return Emat[s][x]; }
static int step_env(int s,int a){ return (urand()<1.0-TN)? (s+a)%K : rk(); }
static int emit_env(int s){ double u=urand(),a=0; for(int x=0;x<K;x++){ a+=Emat[s][x]; if(u<a) return x; } return K-1; }
static double run_true(void){
RNG=0x9e3779b97f4a7c15ULL; int s=rk(); long hit=0; double b[K]={1.0/K,1.0/K,1.0/K};
for(long t=0;t<CONTROL;t++){
int best=0; double bv=-1; for(int a=0;a<K;a++){ double m=0; for(int i=0;i<K;i++) m+=b[i]*Ptrans(GOAL,i,a); if(m>bv){bv=m;best=a;} }
s=step_env(s,best); int x=emit_env(s); if(s==GOAL) hit++;
double bp[K]={0,0,0}; for(int j=0;j<K;j++){ double v=0; for(int i=0;i<K;i++) v+=b[i]*Ptrans(j,i,best); bp[j]=v*Pemit(x,j); }
double z=0; for(int j=0;j<K;j++) z+=bp[j]; if(z<1e-300)z=1e-300; for(int j=0;j<K;j++) b[j]=bp[j]/z;
}
return (double)hit/CONTROL;
}
static double run_nomodel(void){
RNG=0x9e3779b97f4a7c15ULL; int s=rk(),lastx=0; long hit=0;
for(long t=0;t<CONTROL;t++){ int a=((GOAL-lastx)%K+K)%K; s=step_env(s,a); int x=emit_env(s); if(s==GOAL)hit++; lastx=x; }
return (double)hit/CONTROL;
}
static double E[K][K], Tr[K][K][K], pi[K];
static int Xb[NB], Ab[NB];
static double al[NB][K], be[NB][K], cs[NB];
static void em(void){
RNG=0xd1b54a32d192ed03ULL; int s=rk();
for(long t=0;t<NB;t++){ int a=rk(); s=step_env(s,a); Ab[t]=a; Xb[t]=emit_env(s); }
for(int i=0;i<K;i++){ pi[i]=1.0/K; double z=0; for(int x=0;x<K;x++){ E[i][x]=(i==x)?0.5:0.25; z+=E[i][x]; } for(int x=0;x<K;x++) E[i][x]/=z;
for(int a=0;a<K;a++) for(int j=0;j<K;j++) Tr[a][i][j]=1.0/K; }
for(int it=0; it<ITERS; it++){
double s0=0; for(int i=0;i<K;i++){ al[0][i]=pi[i]*E[i][Xb[0]]; s0+=al[0][i]; }
if(s0<1e-300)s0=1e-300; cs[0]=s0; for(int i=0;i<K;i++) al[0][i]/=s0;
for(long t=1;t<NB;t++){ double st=0; int a=Ab[t]; /* transition (t-1)->t is governed by Ab[t] */
for(int j=0;j<K;j++){ double v=0; for(int i=0;i<K;i++) v+=al[t-1][i]*Tr[a][i][j]; al[t][j]=v*E[j][Xb[t]]; st+=al[t][j]; }
if(st<1e-300)st=1e-300; cs[t]=st; for(int j=0;j<K;j++) al[t][j]/=st; }
for(int i=0;i<K;i++) be[NB-1][i]=1.0;
for(long t=NB-2;t>=0;t--){ int a=Ab[t+1]; /* transition t->(t+1) is governed by Ab[t+1] */
for(int i=0;i<K;i++){ double v=0; for(int j=0;j<K;j++) v+=Tr[a][i][j]*E[j][Xb[t+1]]*be[t+1][j]; be[t][i]=v/cs[t+1]; } }
double Enum[K][K]={{0}},Eden[K]={0},Tnum[K][K][K]={{{0}}},Tden[K][K]={{0}},pin[K]={0};
for(long t=0;t<NB;t++){ for(int i=0;i<K;i++){ double g=al[t][i]*be[t][i]; Enum[i][Xb[t]]+=g; Eden[i]+=g; if(t==0) pin[i]=g; } }
for(long t=0;t<NB-1;t++){ int a=Ab[t+1]; /* transition t->(t+1) is governed by Ab[t+1] */
for(int i=0;i<K;i++){ double gi=0; for(int j=0;j<K;j++){ double xi=al[t][i]*Tr[a][i][j]*E[j][Xb[t+1]]*be[t+1][j]/cs[t+1]; Tnum[a][i][j]+=xi; gi+=xi; } Tden[a][i]+=gi; } }
for(int i=0;i<K;i++){ double z=Eden[i]<1e-12?1e-12:Eden[i]; for(int x=0;x<K;x++) E[i][x]=Enum[i][x]/z; }
for(int a=0;a<K;a++) for(int i=0;i<K;i++){ double z=Tden[a][i]<1e-12?1e-12:Tden[a][i]; for(int j=0;j<K;j++) Tr[a][i][j]=Tnum[a][i][j]/z; }
double zp=0; for(int i=0;i<K;i++) zp+=pin[i]; if(zp<1e-12)zp=1e-12; for(int i=0;i<K;i++) pi[i]=pin[i]/zp;
}
}
static double run_learned(int gstar){
RNG=0x9e3779b97f4a7c15ULL; int s=rk(); long hit=0; double b[K]; for(int i=0;i<K;i++) b[i]=pi[i];
for(long t=0;t<CONTROL;t++){
int best=0; double bv=-1; for(int a=0;a<K;a++){ double m=0; for(int i=0;i<K;i++) m+=b[i]*Tr[a][i][gstar]; if(m>bv){bv=m;best=a;} }
s=step_env(s,best); int x=emit_env(s); if(s==GOAL) hit++;
double bp[K]={0,0,0}; for(int j=0;j<K;j++){ double v=0; for(int i=0;i<K;i++) v+=b[i]*Tr[best][i][j]; bp[j]=v*E[j][x]; }
double z=0; for(int j=0;j<K;j++) z+=bp[j]; if(z<1e-300)z=1e-300; for(int j=0;j<K;j++) b[j]=bp[j]/z;
}
return (double)hit/CONTROL;
}
static void run_mode(const char*name){
em();
int gstar=0; for(int i=1;i<K;i++) if(E[i][TGT]>E[gstar][TGT]) gstar=i;
double occ_learn=run_learned(gstar), occ_true=run_true(), occ_nom=run_nomodel();
/* is the learned transition informative? peak-off-diagonal test on T[a=1] */
double diag1=Tr[1][0][1]; /* should be high (~shift) if learned */
printf(" --- %s ---\n",name);
printf(" TRUE model %.3f | LEARNED %.3f | NO-MODEL %.3f | chance %.3f\n", occ_true,occ_learn,occ_nom,1.0/K);
printf(" learned T[a=1] from-state-0 -> [%.2f %.2f %.2f] (peak at 'to=1' if the shift was learned)\n",Tr[1][0][0],Tr[1][0][1],Tr[1][0][2]);
if(occ_learn>occ_nom+0.03 && occ_learn>occ_true-0.06)
printf(" => slow loop CLOSES: learned model controls ~as well as the true model (shift T[1][0]->to=1 = %.2f).\n\n",diag1);
else
printf(" => learned model did not reach handed-model control here -- read the numbers before claiming success.\n\n");
}
int main(void){
printf("=== Agent LEARNS its world model from experience (slow loop closed) ===\n");
printf(" Babble %d, controlled EM %d iters, control %d. Goal=state %d via observation TGT=%d.\n\n",NB,ITERS,CONTROL,GOAL,TGT);
set_symmetric(); run_mode("SYMMETRIC per-state emissions");
set_asymmetric(); run_mode("ASYMMETRIC per-state emissions");
printf(" READ: in BOTH environments the agent learned E and T[a] from raw babbling and controls ~as well as\n");
printf(" the agent handed the true model -- the world model is ACQUIRED, not given. Combined with the seed\n");
printf(" (belief_field_agent.c), the loop is now autonomous: babble -> LEARN the model (slow) -> infer by\n");
printf(" belief-mating (fast) -> decide toward the setpoint -> act. mom.c stays the directed-read tool for\n");
printf(" fractal-hard structure where local EM would stall (Part A's mess3), which this benign env is not.\n");
return 0;
}