#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
// Configuration
const TIER = process.argv[2] || '1';
const tiers = JSON.parse(fs.readFileSync('tiers.json', 'utf8'));
const tierKey = `tier${TIER}`;
const tierData = tiers[tierKey];
if (!tierData) {
console.error(`❌ Invalid tier: ${TIER}`);
console.error(`Available tiers: ${Object.keys(tiers).join(', ')}`);
process.exit(1);
}
console.log(`\n🚀 Phase 1B: ${tierData.name}`);
console.log(`📦 Processing ${tierData.repos.length} repositories\n`);
// Create repos directory
const reposDir = path.join('repos', tierKey);
if (!fs.existsSync(reposDir)) {
fs.mkdirSync(reposDir, { recursive: true });
}
let successCount = 0;
let skipCount = 0;
// Process each repo
for (const repoName of tierData.repos) {
console.log(`\n[${ tierData.repos.indexOf(repoName) + 1}/${tierData.repos.length}] Processing: ${repoName}`);
const repoPath = path.join(reposDir, repoName);
try {
// Clone if not exists
if (!fs.existsSync(repoPath)) {
console.log(` 📥 Cloning...`);
execSync(
`git clone --depth 1 https://github.com/holepunchto/${repoName}.git ${repoPath}`,
{ stdio: ['pipe', 'pipe', 'pipe'] }
);
console.log(` ✓ Cloned`);
} else {
console.log(` ⏭️ Already cloned, skipping...`);
skipCount++;
}
// Analyze repo
console.log(` 🔍 Analyzing...`);
const analysis = analyzeRepo(repoName, repoPath);
// Save analysis
const skillsDir = path.join('skills', repoName);
const apiExtractPath = path.join(skillsDir, 'api-extract.json');
fs.writeFileSync(apiExtractPath, JSON.stringify(analysis, null, 2));
console.log(` 💾 Saved API extract`);
// Update skills.md
console.log(` 📝 Updating skills.md...`);
updateSkillsMarkdown(repoName, analysis);
console.log(` ✓ Updated skills.md`);
successCount++;
} catch (error) {
console.error(` ✗ Error: ${error.message}`);
}
}
console.log(`\n✅ Phase 1B Tier ${TIER} Complete!`);
console.log(` Successful: ${successCount}/${tierData.repos.length}`);
console.log(` Skipped (already cloned): ${skipCount}`);
// ============================================================================
// Analysis Functions
// ============================================================================
function analyzeRepo(repoName, repoPath) {
const analysis = {
repo: repoName,
timestamp: new Date().toISOString(),
entryPoint: null,
exports: {},
api: {},
readme: {},
patterns: []
};
// 1. Analyze package.json
const packageJsonPath = path.join(repoPath, 'package.json');
if (fs.existsSync(packageJsonPath)) {
const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
analysis.entryPoint = pkg.main || pkg.exports || 'index.js';
analysis.exports.packageJson = {
main: pkg.main,
exports: pkg.exports,
type: pkg.type
};
}
// 2. Analyze README
const readmePath = path.join(repoPath, 'README.md');
if (fs.existsSync(readmePath)) {
const readme = fs.readFileSync(readmePath, 'utf8');
analysis.readme = parseReadme(readme);
}
// 3. Find and parse entry point
const entryFiles = findEntryPoint(repoPath, analysis.entryPoint);
if (entryFiles.length > 0) {
analysis.api = parseEntryPoint(repoPath, entryFiles[0]);
}
// 4. Detect patterns
analysis.patterns = detectPatterns(repoPath, analysis.readme.content);
return analysis;
}
function findEntryPoint(repoPath, hint) {
const candidates = [
hint,
'index.js',
'lib/index.js',
'src/index.js',
'index.mjs',
'lib/index.mjs'
].filter(Boolean);
const found = [];
for (const candidate of candidates) {
const fullPath = path.join(repoPath, candidate);
if (fs.existsSync(fullPath)) {
found.push(candidate);
}
}
return found;
}
function parseReadme(content) {
const sections = {};
let currentSection = 'intro';
let currentContent = [];
const lines = content.split('\n');
for (const line of lines) {
// Check for section headers
if (line.startsWith('## ')) {
// Save previous section
if (currentContent.length > 0) {
sections[currentSection] = currentContent.join('\n');
}
// Start new section
currentSection = line.substring(3).toLowerCase().trim();
currentContent = [];
} else {
currentContent.push(line);
}
}
// Save last section
if (currentContent.length > 0) {
sections[currentSection] = currentContent.join('\n');
}
// Extract usage examples (code blocks)
const examples = [];
const codeBlockRegex = /```(?:javascript|js)?\n([\s\S]*?)```/g;
let match;
while ((match = codeBlockRegex.exec(content)) !== null) {
examples.push(match[1].trim());
}
return {
content: content.substring(0, 1000), // First 1000 chars
sections: Object.keys(sections),
hasApi: 'api' in sections || 'usage' in sections,
hasExamples: examples.length > 0,
exampleCount: examples.length,
examples: examples.slice(0, 3) // First 3 examples
};
}
function parseEntryPoint(repoPath, entryFile) {
try {
const fullPath = path.join(repoPath, entryFile);
const content = fs.readFileSync(fullPath, 'utf8');
const api = {
file: entryFile,
exports: [],
classes: [],
functions: []
};
// Simple regex-based parsing (not perfect but good enough)
// Look for module.exports or export
const exportMatches = content.match(/module\.exports\s*=\s*(\w+)/g);
if (exportMatches) {
exportMatches.forEach(m => {
const name = m.match(/=\s*(\w+)/)[1];
api.exports.push({ type: 'default', name });
});
}
// Look for class definitions
const classMatches = content.match(/class\s+(\w+)/g);
if (classMatches) {
classMatches.forEach(m => {
const name = m.match(/class\s+(\w+)/)[1];
api.classes.push({ name });
});
}
// Look for function exports
const funcMatches = content.match(/(?:export\s+)?function\s+(\w+)/g);
if (funcMatches) {
funcMatches.forEach(m => {
const name = m.match(/function\s+(\w+)/)[1];
api.functions.push({ name });
});
}
return api;
} catch (error) {
return { error: error.message };
}
}
function detectPatterns(repoPath, readmeContent) {
const patterns = [];
// Read package.json dependencies
const packageJsonPath = path.join(repoPath, 'package.json');
let dependencies = [];
if (fs.existsSync(packageJsonPath)) {
const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
dependencies = Object.keys(pkg.dependencies || {});
}
// P2P patterns
if (dependencies.includes('hypercore') || readmeContent.includes('hypercore')) {
patterns.push('append-only-log');
}
if (dependencies.includes('hyperswarm') || readmeContent.includes('hyperswarm')) {
patterns.push('peer-discovery');
}
if (dependencies.includes('hyperdht') || readmeContent.includes('hyperdht')) {
patterns.push('dht-discovery');
}
if (dependencies.includes('protomux') || readmeContent.includes('protomux')) {
patterns.push('stream-multiplexing');
}
if (readmeContent.includes('replicate') || readmeContent.includes('replication')) {
patterns.push('replication');
}
if (dependencies.includes('compact-encoding') || dependencies.includes('sub-encoder')) {
patterns.push('binary-encoding');
}
if (dependencies.includes('sodium-native') || dependencies.includes('sodium-universal')) {
patterns.push('cryptography');
}
if (dependencies.includes('streamx') || dependencies.includes('bare-stream')) {
patterns.push('streaming');
}
return patterns;
}
function updateSkillsMarkdown(repoName, analysis) {
const skillsPath = path.join('skills', repoName, 'skills.md');
if (!fs.existsSync(skillsPath)) {
console.log(` ⚠️ skills.md not found, skipping update`);
return;
}
let content = fs.readFileSync(skillsPath, 'utf8');
// Replace the "API Documentation" section
const apiSection = generateApiSection(analysis);
content = content.replace(
/## API Documentation\n\n_To be enriched in Phase 1B with deep code analysis_/,
`## API Documentation\n\n${apiSection}`
);
// Replace the "Usage Examples" section
if (analysis.readme.hasExamples) {
const examplesSection = generateExamplesSection(analysis);
content = content.replace(
/## Usage Examples\n\n_To be added in Phase 1B_/,
`## Usage Examples\n\n${examplesSection}`
);
}
// Add P2P Patterns section before "Semantic Type"
if (analysis.patterns.length > 0) {
const patternsSection = `## P2P Patterns\n\n${analysis.patterns.map(p => `- ${p}`).join('\n')}\n\n`;
content = content.replace(
/## Semantic Type/,
`${patternsSection}## Semantic Type`
);
}
fs.writeFileSync(skillsPath, content);
}
function generateApiSection(analysis) {
let section = '';
if (analysis.readme.hasApi) {
section += `_Extracted from README and code analysis_\n\n`;
} else {
section += `_Extracted from code analysis_\n\n`;
}
if (analysis.api.classes && analysis.api.classes.length > 0) {
section += `### Classes\n\n`;
analysis.api.classes.forEach(cls => {
section += `- \`${cls.name}\`\n`;
});
section += `\n`;
}
if (analysis.api.functions && analysis.api.functions.length > 0) {
section += `### Functions\n\n`;
analysis.api.functions.forEach(fn => {
section += `- \`${fn.name}()\`\n`;
});
section += `\n`;
}
if (analysis.api.exports && analysis.api.exports.length > 0) {
section += `### Exports\n\n`;
analysis.api.exports.forEach(exp => {
section += `- ${exp.type}: \`${exp.name}\`\n`;
});
section += `\n`;
}
section += `\n_See [api-extract.json](./api-extract.json) for detailed API information._\n`;
return section;
}
function generateExamplesSection(analysis) {
let section = '';
if (analysis.readme.examples && analysis.readme.examples.length > 0) {
section += `_Extracted from README_\n\n`;
analysis.readme.examples.forEach((example, i) => {
section += `### Example ${i + 1}\n\n`;
section += `\`\`\`javascript\n${example}\n\`\`\`\n\n`;
});
}
return section;
}