#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
// Read repo_map.json
const repoMapData = JSON.parse(fs.readFileSync('repo_map.json', 'utf8'));
// Support both old array format and new object format with _meta
const repos = repoMapData.repos || repoMapData;
// Initialize data structures for semantic maps
const dependencyGraph = {};
const techStack = {};
const maturityIndex = {};
// Progress tracking
let processedCount = 0;
const totalRepos = repos.length;
console.log(`Starting Phase 1A: Processing ${totalRepos} repositories...`);
// Process each repository
for (const repo of repos) {
try {
processedCount++;
console.log(`\n[${processedCount}/${totalRepos}] Processing: ${repo.name}`);
// Create directory for this repo
const repoDir = path.join('skills', repo.name);
if (!fs.existsSync(repoDir)) {
fs.mkdirSync(repoDir, { recursive: true });
}
// Initialize structures
dependencyGraph[repo.name] = {
dependencies: [],
devDependencies: [],
reverseDependencies: []
};
techStack[repo.name] = {
language: repo.language,
runtime: null,
framework: null,
dependencies: []
};
maturityIndex[repo.name] = {
lastUpdated: null,
stars: 0,
issuesCount: 0,
isArchived: false
};
// Fetch package.json if it exists
let packageJson = null;
try {
const packageContent = execSync(
`gh api repos/holepunchto/${repo.name}/contents/package.json --jq '.content' | base64 -d`,
{ encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }
);
packageJson = JSON.parse(packageContent);
// Extract dependencies
if (packageJson.dependencies) {
dependencyGraph[repo.name].dependencies = Object.keys(packageJson.dependencies);
techStack[repo.name].dependencies = Object.keys(packageJson.dependencies);
}
if (packageJson.devDependencies) {
dependencyGraph[repo.name].devDependencies = Object.keys(packageJson.devDependencies);
}
} catch (e) {
console.log(` ⚠ No package.json found`);
}
// Fetch README
let readme = '';
try {
readme = execSync(
`gh api repos/holepunchto/${repo.name}/readme --jq '.content' | base64 -d`,
{ encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }
);
} catch (e) {
console.log(` ⚠ No README found`);
}
// Fetch repo metadata
let repoMetadata = null;
try {
const metadataStr = execSync(
`gh api repos/holepunchto/${repo.name} --jq '{stars: .stargazers_count, issues: .open_issues_count, updated: .updated_at, archived: .archived}'`,
{ encoding: 'utf8', stdio: ['pipe', 'pipe', 'ignore'] }
);
repoMetadata = JSON.parse(metadataStr);
maturityIndex[repo.name] = {
lastUpdated: repoMetadata.updated,
stars: repoMetadata.stars,
issuesCount: repoMetadata.issues,
isArchived: repoMetadata.archived
};
} catch (e) {
console.log(` ⚠ Could not fetch repo metadata`);
}
// Detect runtime from dependencies (improved logic based on pear-docs analysis)
let detectedRuntime = null;
// Strategy 1: Check repo name patterns
if (repo.name.startsWith('bare-')) {
detectedRuntime = 'bare';
} else if (repo.name.startsWith('pear-')) {
detectedRuntime = 'pear';
}
// Strategy 2: If not detected by name, check dependencies
if (!detectedRuntime && packageJson) {
const allDeps = [
...Object.keys(packageJson.dependencies || {}),
...Object.keys(packageJson.devDependencies || {})
];
// Count bare-* and pear-* dependencies
const bareDeps = allDeps.filter(d => d.startsWith('bare-')).length;
const pearDeps = allDeps.filter(d => d.startsWith('pear-')).length;
// If 2+ bare dependencies, likely bare runtime
if (bareDeps >= 2) {
detectedRuntime = 'bare';
}
// If 2+ pear dependencies, likely pear runtime
else if (pearDeps >= 2) {
detectedRuntime = 'pear';
}
// Check for 'universal' indicator
else if (repo.name.includes('universal')) {
detectedRuntime = 'universal';
}
// Check package.json engines field
else if (packageJson.engines && packageJson.engines.node) {
detectedRuntime = 'node';
}
}
techStack[repo.name].runtime = detectedRuntime;
// Create skills.md — but NEVER overwrite a Phase-1B-enriched file. This
// Phase-1A generator only writes the placeholder API section; an enriched
// skills.md (no placeholder marker) holds real API docs we must preserve.
const skillsPath = path.join(repoDir, 'skills.md');
const enriched = fs.existsSync(skillsPath) &&
!fs.readFileSync(skillsPath, 'utf8').includes('To be enriched in Phase 1B');
if (enriched) {
console.log(` ↩ Preserved enriched skills.md`);
} else {
const skillsContent = generateSkillsMarkdown(repo, packageJson, readme, repoMetadata);
fs.writeFileSync(skillsPath, skillsContent);
console.log(` ✓ Created skills.md`);
}
// Rate limiting: sleep briefly every 10 repos
if (processedCount % 10 === 0) {
console.log(` 💤 Brief pause to respect rate limits...`);
execSync('sleep 1');
}
} catch (error) {
console.error(` ✗ Error processing ${repo.name}:`, error.message);
}
}
// Calculate reverse dependencies (fixed to use exact matching only)
console.log('\n📊 Calculating reverse dependencies...');
for (const repoName in dependencyGraph) {
const deps = dependencyGraph[repoName].dependencies;
for (const dep of deps) {
// Check if this dep is in our ecosystem - EXACT MATCH ONLY
const depInEcosystem = repos.find(r => r.name === dep);
if (depInEcosystem && dependencyGraph[depInEcosystem.name]) {
dependencyGraph[depInEcosystem.name].reverseDependencies.push(repoName);
}
}
}
// Deduplicate reverse dependencies
console.log('🔧 Deduplicating reverse dependencies...');
for (const repoName in dependencyGraph) {
const reverseDeps = dependencyGraph[repoName].reverseDependencies;
dependencyGraph[repoName].reverseDependencies = [...new Set(reverseDeps)];
}
// Save semantic maps
console.log('\n💾 Saving semantic maps...');
fs.writeFileSync('dependency-graph.json', JSON.stringify(dependencyGraph, null, 2));
fs.writeFileSync('tech-stack.json', JSON.stringify(techStack, null, 2));
fs.writeFileSync('maturity-index.json', JSON.stringify(maturityIndex, null, 2));
console.log(`\n✅ Phase 1A Complete!`);
console.log(` Processed: ${processedCount}/${totalRepos} repositories`);
console.log(` Created: ${processedCount} skills.md files`);
console.log(` Generated: dependency-graph.json, tech-stack.json, maturity-index.json`);
function generateSkillsMarkdown(repo, packageJson, readme, metadata) {
let content = `# ${repo.name}\n\n`;
content += `**Repository**: ${repo.url}\n`;
content += `**Language**: ${repo.language}\n`;
if (metadata) {
content += `**Stars**: ${metadata.stars}\n`;
content += `**Last Updated**: ${metadata.updated}\n`;
content += `**Status**: ${metadata.archived ? '⚠️ Archived' : '✓ Active'}\n`;
}
content += `\n## Description\n\n`;
content += repo.description || 'No description available';
if (packageJson) {
content += `\n\n## Installation\n\n`;
content += `\`\`\`bash\n`;
content += `npm install ${packageJson.name || repo.name}\n`;
content += `\`\`\`\n`;
if (packageJson.dependencies && Object.keys(packageJson.dependencies).length > 0) {
content += `\n## Dependencies\n\n`;
Object.entries(packageJson.dependencies).forEach(([name, version]) => {
content += `- ${name}: ${version}\n`;
});
}
if (packageJson.main || packageJson.exports) {
content += `\n## Entry Point\n\n`;
content += `- Main: ${packageJson.main || 'N/A'}\n`;
if (packageJson.exports) {
content += `- Exports: ${JSON.stringify(packageJson.exports, null, 2)}\n`;
}
}
}
if (readme && readme.length > 0) {
content += `\n## README Summary\n\n`;
// Extract first few paragraphs from README
const readmeLines = readme.split('\n').slice(0, 20);
content += readmeLines.join('\n');
content += `\n\n[See full README](${repo.url})\n`;
}
content += `\n## API Documentation\n\n`;
content += `_To be enriched in Phase 1B with deep code analysis_\n`;
content += `\n## Usage Examples\n\n`;
content += `_To be added in Phase 1B_\n`;
content += `\n## Semantic Type\n\n`;
content += `_To be classified in Phase 2_\n`;
content += `\n## Related Packages\n\n`;
content += `_To be analyzed in Phase 2_\n`;
return content;
}