// proto-F-ranked-gold.cu
// ---------------------------------------------------------------------------
// (1) RANKED-MI value upgrade (Linas's measure, verified from
// learn-lang-diary-part-four.lyx:249):
// rankedMI(a,x) = log2( count(a,x) / sqrt(marg[a]*marg[x]) )
// = plain MI with HALF the marginal penalty -> tempers the rare-word bias.
// Shows COUNT vs MI vs RANKED-MI side by side; verifies GPU==CPU.
//
// (2) GOLD CROSS-CHECK: emits gold.scm — the SAME co-occurrence edges as
// (Evaluation (Predicate "cooc") (List (Concept a)(Concept x))) plus a
// MeetLink query per anchor-pair — so the REAL opencog pattern-matcher
// (cog-execute!) answers the identical query. We compare its result to our
// intersection (which == our GPU result, proven in Proto-D/E). Writes the
// expected sets to expected.txt for an automatic diff.
//
// Build: nvcc -O2 -arch=sm_75 -o proto-F proto-F-ranked-gold.cu
// Run: ./proto-F <book.txt>... ; then: guile gold.scm | sort | diff - expected.txt
// ---------------------------------------------------------------------------
#include <cstdio>
#include <cstdint>
#include <cctype>
#include <cmath>
#include <vector>
#include <string>
#include <unordered_map>
#include <algorithm>
#include <random>
#include <cuda_runtime.h>
#define CUDA_OK(c) do{cudaError_t e=(c); if(e!=cudaSuccess){\
printf("CUDA ERR %s:%d %s\n",__FILE__,__LINE__,cudaGetErrorString(e));return -1;}}while(0)
struct Corpus{ int W=0; std::vector<std::string> id2word;
std::vector<uint32_t> row_off,col_idx; std::vector<float> cnt;
std::vector<double> marg; double N=0; };
static bool load_corpus(const std::vector<std::string>& files,int WIN,Corpus& C){
std::unordered_map<std::string,int> vocab; std::vector<std::vector<int>> sents;
for(auto& f:files){ FILE* fp=fopen(f.c_str(),"rb"); if(!fp){printf("open fail %s\n",f.c_str());return false;}
std::string tok; std::vector<int> snt; int ch;
auto fw=[&](){ if(tok.empty())return; auto it=vocab.find(tok); int id;
if(it==vocab.end()){id=C.id2word.size();vocab[tok]=id;C.id2word.push_back(tok);}else id=it->second;
snt.push_back(id); tok.clear(); };
auto fs=[&](){ if(snt.size()>1)sents.push_back(snt); snt.clear(); };
while((ch=fgetc(fp))!=EOF){ if(isalpha(ch)||ch=='\'')tok.push_back((char)tolower(ch));
else{fw(); if(ch=='.'||ch=='!'||ch=='?')fs();} } fw();fs();fclose(fp); }
C.W=C.id2word.size();
std::vector<std::unordered_map<uint32_t,float>> nb(C.W);
for(auto&s:sents)for(int i=0;i<(int)s.size();i++)
for(int j=std::max(0,i-WIN);j<=std::min((int)s.size()-1,i+WIN);j++) if(i!=j) nb[s[i]][(uint32_t)s[j]]+=1.f;
C.row_off.assign(C.W+1,0); C.marg.assign(C.W,0.0); long E=0; for(int w=0;w<C.W;w++)E+=nb[w].size();
C.col_idx.reserve(E);C.cnt.reserve(E);
for(int w=0;w<C.W;w++){ std::vector<std::pair<uint32_t,float>> v(nb[w].begin(),nb[w].end());
std::sort(v.begin(),v.end()); C.row_off[w+1]=C.row_off[w]+v.size();
for(auto&p:v){C.col_idx.push_back(p.first);C.cnt.push_back(p.second);C.marg[w]+=p.second;C.N+=p.second;} }
printf("Corpus: %d sentences, %d words, %ld edges, N=%.0f\n",(int)sents.size(),C.W,E,C.N);
return true;
}
__device__ __forceinline__ int bs(const uint32_t* a,int n,uint32_t k){int lo=0,hi=n-1;
while(lo<=hi){int m=(lo+hi)>>1;uint32_t v=a[m];if(v==k)return m;if(v<k)lo=m+1;else hi=m-1;}return -1;}
// per match: id, count-sum, MI-sum, rankedMI-sum
__global__ void qscore(const uint32_t* ro,const uint32_t* ci,const float* cn,const double* mg,double N,
const uint32_t* anc,const uint32_t* qo,int nq,
uint32_t* oid,float* ocnt,float* omi,float* ormi,int st,int* oc){
int q=blockIdx.x; if(q>=nq)return; int ka=qo[q+1]-qo[q]; const uint32_t* A=anc+qo[q]; if(!ka)return;
int pv=0; uint32_t pl=ro[A[0]+1]-ro[A[0]]; for(int i=1;i<ka;i++){uint32_t l=ro[A[i]+1]-ro[A[i]];if(l<pl){pl=l;pv=i;}}
uint32_t p=A[pv]; const uint32_t* P=ci+ro[p]; const float* PC=cn+ro[p];
for(int t=threadIdx.x;t<(int)pl;t+=blockDim.x){ uint32_t x=P[t]; bool all=true;
float sc=PC[t]; double mi=log2(N*(double)PC[t]/(mg[p]*mg[x]));
double rmi=log2((double)PC[t]/sqrt(mg[p]*mg[x]));
for(int i=0;i<ka&&all;i++){ if(i==pv)continue; uint32_t a=A[i]; const uint32_t* AA=ci+ro[a]; int la=ro[a+1]-ro[a];
int q2=bs(AA,la,x); if(q2<0)all=false; else{float c=(cn+ro[a])[q2]; sc+=c;
mi+=log2(N*(double)c/(mg[a]*mg[x])); rmi+=log2((double)c/sqrt(mg[a]*mg[x]));} }
if(all){int idx=atomicAdd(&oc[q],1); if(idx<st){oid[q*st+idx]=x;ocnt[q*st+idx]=sc;omi[q*st+idx]=(float)mi;ormi[q*st+idx]=(float)rmi;}}
}
}
int main(int argc,char**argv){
if(argc<2){printf("usage: %s <book.txt>...\n",argv[0]);return 1;}
std::vector<std::string> files; for(int i=1;i<argc;i++)files.push_back(argv[i]);
printf("=== Prototype F: ranked-MI + gold cross-check ===\n\n");
Corpus C; if(!load_corpus(files,2,C))return 1; long E=C.col_idx.size();
auto wid=[&](const char*w)->int{for(int i=0;i<C.W;i++)if(C.id2word[i]==w)return i;return -1;};
uint32_t *d_ro,*d_ci; float* d_cn; double* d_mg;
CUDA_OK(cudaMalloc(&d_ro,C.row_off.size()*4));CUDA_OK(cudaMalloc(&d_ci,E*4));
CUDA_OK(cudaMalloc(&d_cn,E*4));CUDA_OK(cudaMalloc(&d_mg,C.W*8));
CUDA_OK(cudaMemcpy(d_ro,C.row_off.data(),C.row_off.size()*4,cudaMemcpyHostToDevice));
CUDA_OK(cudaMemcpy(d_ci,C.col_idx.data(),E*4,cudaMemcpyHostToDevice));
CUDA_OK(cudaMemcpy(d_cn,C.cnt.data(),E*4,cudaMemcpyHostToDevice));
CUDA_OK(cudaMemcpy(d_mg,C.marg.data(),C.W*8,cudaMemcpyHostToDevice));
struct R{uint32_t id;float c,mi,rmi;};
auto run=[&](std::vector<std::vector<uint32_t>>&Q,int st,std::vector<std::vector<R>>&out)->int{
int nq=Q.size(); std::vector<uint32_t> anc,qo(nq+1,0);
for(int i=0;i<nq;i++){qo[i+1]=qo[i]+Q[i].size();for(auto a:Q[i])anc.push_back(a);}
uint32_t *da,*dq,*di; float *dc,*dm,*dr; int* dct;
CUDA_OK(cudaMalloc(&da,anc.size()*4));CUDA_OK(cudaMalloc(&dq,(nq+1)*4));
CUDA_OK(cudaMalloc(&di,(size_t)nq*st*4));CUDA_OK(cudaMalloc(&dc,(size_t)nq*st*4));
CUDA_OK(cudaMalloc(&dm,(size_t)nq*st*4));CUDA_OK(cudaMalloc(&dr,(size_t)nq*st*4));CUDA_OK(cudaMalloc(&dct,nq*4));
CUDA_OK(cudaMemcpy(da,anc.data(),anc.size()*4,cudaMemcpyHostToDevice));
CUDA_OK(cudaMemcpy(dq,qo.data(),(nq+1)*4,cudaMemcpyHostToDevice));CUDA_OK(cudaMemset(dct,0,nq*4));
qscore<<<nq,128>>>(d_ro,d_ci,d_cn,d_mg,C.N,da,dq,nq,di,dc,dm,dr,st,dct); CUDA_OK(cudaDeviceSynchronize());
std::vector<int> ct(nq);std::vector<uint32_t> oi((size_t)nq*st);std::vector<float> oc((size_t)nq*st),om((size_t)nq*st),orr((size_t)nq*st);
CUDA_OK(cudaMemcpy(ct.data(),dct,nq*4,cudaMemcpyDeviceToHost));
CUDA_OK(cudaMemcpy(oi.data(),di,(size_t)nq*st*4,cudaMemcpyDeviceToHost));
CUDA_OK(cudaMemcpy(oc.data(),dc,(size_t)nq*st*4,cudaMemcpyDeviceToHost));
CUDA_OK(cudaMemcpy(om.data(),dm,(size_t)nq*st*4,cudaMemcpyDeviceToHost));
CUDA_OK(cudaMemcpy(orr.data(),dr,(size_t)nq*st*4,cudaMemcpyDeviceToHost));
out.assign(nq,{}); for(int q=0;q<nq;q++){int c=std::min(ct[q],st);
for(int i=0;i<c;i++)out[q].push_back({oi[(size_t)q*st+i],oc[(size_t)q*st+i],om[(size_t)q*st+i],orr[(size_t)q*st+i]});}
cudaFree(da);cudaFree(dq);cudaFree(di);cudaFree(dc);cudaFree(dm);cudaFree(dr);cudaFree(dct);return 0;};
// ---- (1) COUNT vs MI vs RANKED-MI ----------------------------------
struct QE{const char*name;std::vector<const char*>a;};
std::vector<QE> ex={{"mr+mrs",{"mr","mrs"}},{"said+have+not",{"said","have","not"}},{"could+would",{"could","would"}}};
printf("\n--- COUNT vs MI vs RANKED-MI (top-8) ---\n");
for(auto&e:ex){ std::vector<uint32_t> anc; bool ok=true; for(auto w:e.a){int id=wid(w);if(id<0){ok=false;break;}anc.push_back(id);}
if(!ok){printf(" [%s] skipped\n",e.name);continue;}
std::vector<std::vector<uint32_t>> Q={anc}; std::vector<std::vector<R>> r; if(run(Q,8192,r))return 1; auto v=r[0];
auto pr=[&](const char* lbl,auto cmp){ auto s=v; std::sort(s.begin(),s.end(),cmp);
printf(" %-10s",lbl); for(int i=0;i<(int)std::min((size_t)8,s.size());i++)printf("%s ",C.id2word[s[i].id].c_str()); printf("\n"); };
printf(" [%s] %zu matches\n",e.name,v.size());
pr("count:",[](const R&a,const R&b){return a.c>b.c;});
pr("MI:",[](const R&a,const R&b){return a.mi>b.mi;});
pr("rankedMI:",[](const R&a,const R&b){return a.rmi>b.rmi;});
}
// ---- verify GPU == CPU for MI and ranked-MI ------------------------
std::mt19937 rng(99); std::uniform_int_distribution<int> wd(0,C.W-1);
const int NQ=2000; std::vector<std::vector<uint32_t>> Q(NQ);
for(int i=0;i<NQ;i++){int k=2+rng()%3;for(int j=0;j<k;j++)Q[i].push_back(wd(rng));}
std::vector<std::vector<R>> rg; if(run(Q,8192,rg))return 1;
long mis=0; double dmi=0,drmi=0;
for(int i=0;i<NQ;i++){auto&anc=Q[i]; int pv=0;uint32_t pl=C.row_off[anc[0]+1]-C.row_off[anc[0]];
for(size_t k=1;k<anc.size();k++){uint32_t l=C.row_off[anc[k]+1]-C.row_off[anc[k]];if(l<pl){pl=l;pv=k;}}
std::unordered_map<uint32_t,std::pair<float,float>> cpu; uint32_t p=anc[pv];
for(uint32_t t=C.row_off[p];t<C.row_off[p+1];t++){uint32_t x=C.col_idx[t];bool all=true;
double mi=log2(C.N*(double)C.cnt[t]/(C.marg[p]*C.marg[x])),rmi=log2((double)C.cnt[t]/sqrt(C.marg[p]*C.marg[x]));
for(size_t k=0;k<anc.size()&&all;k++){if((int)k==pv)continue;uint32_t a=anc[k];
auto b=C.col_idx.begin()+C.row_off[a],en=C.col_idx.begin()+C.row_off[a+1];auto it=std::lower_bound(b,en,x);
if(it==en||*it!=x)all=false;else{float c=C.cnt[C.row_off[a]+(it-b)];mi+=log2(C.N*(double)c/(C.marg[a]*C.marg[x]));rmi+=log2((double)c/sqrt(C.marg[a]*C.marg[x]));}}
if(all)cpu[x]={(float)mi,(float)rmi};}
if(rg[i].size()!=cpu.size()){mis++;continue;}
for(auto&t:rg[i]){auto it=cpu.find(t.id);if(it==cpu.end()){mis++;break;}
dmi=std::max(dmi,(double)std::fabs(it->second.first-t.mi)); drmi=std::max(drmi,(double)std::fabs(it->second.second-t.rmi));}}
printf("\n--- verify: %d queries set-mismatch=%ld maxdiff MI=%.2e rankedMI=%.2e -> %s ---\n",
NQ,mis,dmi,drmi,(mis==0&&dmi<1e-2&&drmi<1e-2)?"PASS GPU==CPU":"FAIL");
// ---- (2) GOLD CROSS-CHECK: emit gold.scm + expected.txt ------------
std::vector<std::pair<const char*,const char*>> gold={{"mr","mrs"},{"could","would"}};
FILE* scm=fopen("gold.scm","w"); FILE* exp=fopen("expected.txt","w");
fprintf(scm,"; auto-generated; real opencog pattern-matcher cross-check\n(use-modules (opencog) (opencog exec))\n");
int gi=0;
for(auto&g:gold){ int a=wid(g.first),b=wid(g.second); if(a<0||b<0)continue; gi++;
// emit only edges incident to a and b
auto emit=[&](int w){ for(uint32_t t=C.row_off[w];t<C.row_off[w+1];t++)
fprintf(scm,"(Evaluation (Predicate \"cooc\") (List (Concept \"%s\") (Concept \"%s\")))\n",
C.id2word[w].c_str(),C.id2word[C.col_idx[t]].c_str()); };
emit(a); emit(b);
// Meet query: $X with cooc(a,$X) AND cooc(b,$X)
fprintf(scm,
"(define r%d (cog-execute! (Meet (TypedVariable (Variable \"$X\") (Type 'ConceptNode))\n"
" (And (Evaluation (Predicate \"cooc\") (List (Concept \"%s\") (Variable \"$X\")))\n"
" (Evaluation (Predicate \"cooc\") (List (Concept \"%s\") (Variable \"$X\")))))))\n"
"(for-each (lambda (x) (format #t \"Q%d ~a\\n\" (cog-name x)))\n"
" (sort (cog-outgoing-set r%d) (lambda (p q) (string<? (cog-name p)(cog-name q)))))\n",
gi,g.first,g.second,gi,gi);
// expected = CPU intersection (== our GPU per Proto-D/E)
std::vector<std::string> inter;
{auto ab=C.col_idx.begin()+C.row_off[a],ae=C.col_idx.begin()+C.row_off[a+1];
auto bb=C.col_idx.begin()+C.row_off[b],be=C.col_idx.begin()+C.row_off[b+1];
std::vector<uint32_t> ix; std::set_intersection(ab,ae,bb,be,std::back_inserter(ix));
for(auto x:ix) inter.push_back(C.id2word[x]); std::sort(inter.begin(),inter.end()); }
for(auto&s:inter) fprintf(exp,"Q%d %s\n",gi,s.c_str());
printf(" gold pair [%s,%s]: expected %zu common words (CPU=GPU)\n",g.first,g.second,inter.size());
}
fclose(scm); fclose(exp);
printf("\nWrote gold.scm + expected.txt. Cross-check:\n guile gold.scm 2>/dev/null | sort | diff - <(sort expected.txt) && echo MATCH\n");
cudaFree(d_ro);cudaFree(d_ci);cudaFree(d_cn);cudaFree(d_mg); return 0;
}