#!/usr/bin/env node
/**
* Phase 2: Semantic Mapping Builder
*
* Generates semantic categorization files for all 529 holepunchto repos:
* - semantic-categories.json - Categorize repos by official pear-docs taxonomy
* - use-case-map.json - Map repos to common P2P use cases
* - pattern-map.json - Map P2P patterns to repos
* - platform-support.json - Platform compatibility
* - stability-index.json - Maturity and stability classification
*/
const fs = require('fs');
const path = require('path');
console.log('🚀 Phase 2: Building Semantic Maps\n');
// Load Phase 1 data
console.log('📥 Loading Phase 1 data...');
const repoMapData = JSON.parse(fs.readFileSync('repo_map.json', 'utf8'));
// Support both old array format and new object format with _meta
const repoMapArray = repoMapData.repos || repoMapData;
const dependencyGraph = JSON.parse(fs.readFileSync('dependency-graph.json', 'utf8'));
const techStack = JSON.parse(fs.readFileSync('tech-stack.json', 'utf8'));
const maturityIndex = JSON.parse(fs.readFileSync('maturity-index.json', 'utf8'));
const tiers = JSON.parse(fs.readFileSync('tiers.json', 'utf8'));
// Convert repo_map array to object keyed by repo name
const repoMap = {};
for (const repo of repoMapArray) {
repoMap[repo.name] = repo;
}
// Get list of all repos
const allRepos = Object.keys(repoMap);
console.log(`✓ Loaded data for ${allRepos.length} repos\n`);
// Load API extract data for deeply analyzed repos
const deeplyAnalyzedRepos = [
...tiers.tier1.repos,
...tiers.tier2.repos,
...tiers.tier3.repos,
...tiers.tier4.repos
];
const apiData = {};
for (const repo of deeplyAnalyzedRepos) {
const apiFile = `skills/${repo}/api-extract.json`;
if (fs.existsSync(apiFile)) {
try {
apiData[repo] = JSON.parse(fs.readFileSync(apiFile, 'utf8'));
} catch (e) {
// Skip if file exists but is malformed
}
}
}
console.log(`✓ Loaded API data for ${Object.keys(apiData).length} deeply analyzed repos\n`);
// Official taxonomy from pear-docs
const officialBuildingBlocks = [
'hypercore', 'hyperbee', 'hyperdrive', 'autobase', 'hyperdht', 'hyperswarm'
];
const officialHelpers = [
'corestore', 'localdrive', 'mirror-drive', 'hyperswarm-secret-stream',
'compact-encoding', 'protomux'
];
// ============================================================================
// 1. SEMANTIC CATEGORIES
// ============================================================================
console.log('📋 Building semantic-categories.json...');
const semanticCategories = {};
for (const repo of allRepos) {
let category = null;
let subcategory = null;
let confidence = 'low';
let reasoning = '';
let alternateCategories = [];
// Check official lists first (highest confidence)
if (officialBuildingBlocks.includes(repo)) {
category = 'Building Blocks';
subcategory = 'Core Primitives';
confidence = 'high';
reasoning = 'Listed in official pear-docs building blocks';
} else if (officialHelpers.includes(repo)) {
category = 'Helpers';
subcategory = 'Support Libraries';
confidence = 'high';
reasoning = 'Listed in official pear-docs helpers';
}
// Naming patterns
else if (repo.startsWith('bare-')) {
category = 'Bare Modules';
subcategory = 'Standard Library';
confidence = 'high';
reasoning = 'Naming pattern: bare-* indicates Bare runtime standard library';
} else if (repo.startsWith('pear-')) {
category = 'Pear Modules';
subcategory = 'Application Runtime';
confidence = 'high';
reasoning = 'Naming pattern: pear-* indicates Pear application module';
} else if (repo.startsWith('hyper')) {
category = 'Building Blocks';
subcategory = 'Extended Primitives';
confidence = 'medium';
reasoning = 'Naming pattern: hyper* typically indicates P2P primitive';
alternateCategories = ['Helpers'];
} else if (repo.startsWith('lib')) {
category = 'Infrastructure';
subcategory = 'Native Libraries';
confidence = 'medium';
reasoning = 'Naming pattern: lib* indicates native/C library binding';
} else if (repo.startsWith('cmake-')) {
category = 'Tools';
subcategory = 'Build Systems';
confidence = 'high';
reasoning = 'Naming pattern: cmake-* indicates build system tool';
} else if (repo.endsWith('-example') || repo.includes('example')) {
category = 'Examples';
subcategory = 'Demo Applications';
confidence = 'high';
reasoning = 'Naming pattern: *-example indicates example/demo application';
}
// Check description keywords
else if (repoMap[repo].description) {
const desc = repoMap[repo].description.toLowerCase();
if (desc.includes('tool') || desc.includes('cli')) {
category = 'Tools';
subcategory = 'CLI Applications';
confidence = 'medium';
reasoning = 'Description indicates tool or CLI application';
} else if (desc.includes('test') || desc.includes('testing')) {
category = 'Tools';
subcategory = 'Testing';
confidence = 'medium';
reasoning = 'Description indicates testing framework or tool';
} else if (desc.includes('helper') || desc.includes('utility')) {
category = 'Helpers';
subcategory = 'Utilities';
confidence = 'medium';
reasoning = 'Description indicates helper or utility library';
} else {
category = 'Infrastructure';
subcategory = 'Miscellaneous';
confidence = 'low';
reasoning = 'Unable to determine from name or description';
}
} else {
category = 'Infrastructure';
subcategory = 'Miscellaneous';
confidence = 'low';
reasoning = 'No description available, default classification';
}
semanticCategories[repo] = {
category,
subcategory,
confidence,
reasoning,
alternateCategories
};
}
fs.writeFileSync('semantic-categories.json', JSON.stringify(semanticCategories, null, 2));
console.log(`✓ Created semantic-categories.json (${allRepos.length} repos categorized)\n`);
// Generate statistics
const categoryStats = {};
for (const repo of allRepos) {
const cat = semanticCategories[repo].category;
categoryStats[cat] = (categoryStats[cat] || 0) + 1;
}
console.log(' Category distribution:');
for (const [cat, count] of Object.entries(categoryStats).sort((a, b) => b[1] - a[1])) {
console.log(` - ${cat}: ${count} repos`);
}
const confidenceStats = {};
for (const repo of allRepos) {
const conf = semanticCategories[repo].confidence;
confidenceStats[conf] = (confidenceStats[conf] || 0) + 1;
}
console.log('\n Confidence distribution:');
for (const [conf, count] of Object.entries(confidenceStats).sort((a, b) => b[1] - a[1])) {
console.log(` - ${conf}: ${count} repos (${Math.round(count / allRepos.length * 100)}%)`);
}
console.log('');
// ============================================================================
// 2. USE CASE MAP
// ============================================================================
console.log('📋 Building use-case-map.json...');
// Define common P2P use cases
const useCaseDefinitions = {
'distributed-database': {
description: 'Building a distributed key-value database with P2P replication',
requiredRepos: ['hypercore', 'hyperbee'],
optionalRepos: ['autobase', 'corestore', 'hypercore-crypto'],
patterns: ['append-only-log', 'b-tree-index', 'replication'],
examples: []
},
'p2p-file-sharing': {
description: 'Peer-to-peer file sharing and synchronization',
requiredRepos: ['hypercore', 'hyperdrive', 'hyperswarm'],
optionalRepos: ['mirror-drive', 'localdrive', 'corestore'],
patterns: ['file-system', 'replication', 'peer-discovery'],
examples: []
},
'real-time-messaging': {
description: 'Real-time P2P messaging and chat applications',
requiredRepos: ['hyperswarm', 'hyperswarm-secret-stream'],
optionalRepos: ['protomux', 'compact-encoding', 'noise-handshake'],
patterns: ['peer-discovery', 'streaming', 'cryptography'],
examples: []
},
'distributed-log': {
description: 'Distributed append-only log for event sourcing',
requiredRepos: ['hypercore'],
optionalRepos: ['corestore', 'hypercore-crypto', 'compact-encoding'],
patterns: ['append-only-log', 'replication', 'cryptography'],
examples: []
},
'p2p-networking': {
description: 'Low-level P2P networking and peer discovery',
requiredRepos: ['hyperdht', 'hyperswarm'],
optionalRepos: ['dht-rpc', 'hyperswarm-dht-relay', 'noise-handshake'],
patterns: ['dht-discovery', 'peer-discovery', 'cryptography'],
examples: []
},
'multi-writer-collab': {
description: 'Multi-writer collaboration with conflict resolution',
requiredRepos: ['autobase', 'hypercore'],
optionalRepos: ['corestore', 'hyperbee'],
patterns: ['append-only-log', 'multi-writer', 'replication'],
examples: []
},
'encrypted-storage': {
description: 'Encrypted P2P storage with access control',
requiredRepos: ['hypercore', 'hyperdrive'],
optionalRepos: ['hypercore-crypto', 'sodium-native'],
patterns: ['cryptography', 'append-only-log', 'file-system'],
examples: []
},
'p2p-streaming': {
description: 'Peer-to-peer video/audio streaming',
requiredRepos: ['hyperswarm', 'hyperbeam'],
optionalRepos: ['protomux', 'compact-encoding'],
patterns: ['streaming', 'peer-discovery', 'binary-encoding'],
examples: []
},
'dht-application': {
description: 'DHT-based distributed hash table application',
requiredRepos: ['hyperdht'],
optionalRepos: ['dht-rpc', 'hyperswarm'],
patterns: ['dht-discovery', 'peer-discovery'],
examples: []
},
'bare-runtime-app': {
description: 'Building applications with Bare JavaScript runtime',
requiredRepos: ['bare'],
optionalRepos: ['bare-fs', 'bare-net', 'bare-crypto'],
patterns: [],
examples: []
},
'pear-application': {
description: 'Building P2P desktop applications with Pear runtime',
requiredRepos: ['pear'],
optionalRepos: ['pear-desktop', 'hyperswarm', 'hyperdrive'],
patterns: ['peer-discovery', 'file-system'],
examples: []
}
};
// Build reverse index: repo → use cases
const repoUseCases = {};
for (const [useCase, def] of Object.entries(useCaseDefinitions)) {
for (const repo of [...def.requiredRepos, ...def.optionalRepos]) {
if (!repoUseCases[repo]) repoUseCases[repo] = [];
repoUseCases[repo].push(useCase);
}
}
const useCaseMap = {
useCases: useCaseDefinitions,
repoUseCases
};
fs.writeFileSync('use-case-map.json', JSON.stringify(useCaseMap, null, 2));
console.log(`✓ Created use-case-map.json (${Object.keys(useCaseDefinitions).length} use cases defined)\n`);
// ============================================================================
// 3. PATTERN MAP
// ============================================================================
console.log('📋 Building pattern-map.json...');
// Define P2P patterns
const patternDefinitions = {
'append-only-log': {
description: 'Immutable, append-only data structure for distributed systems',
implementedBy: ['hypercore'],
usedBy: ['hyperbee', 'hyperdrive', 'autobase', 'hyperblobs'],
whenToUse: 'Need tamper-proof, verifiable history of events or data',
alternatives: []
},
'replication': {
description: 'Synchronize data between peers in a P2P network',
implementedBy: ['hypercore', 'hyperdrive'],
usedBy: ['mirror-drive', 'autobase', 'hyperbee'],
whenToUse: 'Need to sync data across multiple peers',
alternatives: []
},
'streaming': {
description: 'Stream data between peers in real-time',
implementedBy: ['hypercore', 'protomux'],
usedBy: ['hyperbeam', 'pear-radio', 'hyperswarm'],
whenToUse: 'Need real-time data transfer or video/audio streaming',
alternatives: []
},
'cryptography': {
description: 'Cryptographic operations for P2P security',
implementedBy: ['sodium-native', 'hypercore-crypto'],
usedBy: ['hypercore', 'hyperdrive', 'noise-handshake'],
whenToUse: 'Need encryption, signing, or verification',
alternatives: []
},
'peer-discovery': {
description: 'Discover and connect to peers in P2P network',
implementedBy: ['hyperswarm', 'hyperdht'],
usedBy: ['hyperbeam', 'hyperssh', 'hyperswarm-seeders'],
whenToUse: 'Need to find peers without central server',
alternatives: []
},
'binary-encoding': {
description: 'Efficient binary encoding for network protocols',
implementedBy: ['compact-encoding'],
usedBy: ['hypercore', 'protomux', 'hyperbee'],
whenToUse: 'Need efficient wire protocol encoding',
alternatives: []
},
'stream-multiplexing': {
description: 'Multiplex multiple streams over single connection',
implementedBy: ['protomux'],
usedBy: ['hypercore', 'hyperswarm'],
whenToUse: 'Need multiple logical streams over one connection',
alternatives: []
},
'dht-discovery': {
description: 'Distributed hash table for decentralized peer discovery',
implementedBy: ['hyperdht'],
usedBy: ['hyperswarm', 'dht-rpc'],
whenToUse: 'Need fully decentralized peer discovery',
alternatives: []
},
'b-tree-index': {
description: 'B-tree indexing for efficient lookups in append-only log',
implementedBy: ['hyperbee'],
usedBy: [],
whenToUse: 'Need key-value lookups on append-only log',
alternatives: []
},
'file-system': {
description: 'P2P filesystem with replication',
implementedBy: ['hyperdrive', 'localdrive'],
usedBy: ['mirror-drive'],
whenToUse: 'Need distributed filesystem',
alternatives: []
}
};
// Build reverse index from API data
const repoPatterns = {};
for (const repo of allRepos) {
repoPatterns[repo] = {
implements: [],
uses: []
};
// Check if repo implements patterns (from API data)
if (apiData[repo] && apiData[repo].patterns) {
repoPatterns[repo].implements = apiData[repo].patterns;
}
// Check pattern definitions for usage
for (const [pattern, def] of Object.entries(patternDefinitions)) {
if (def.usedBy.includes(repo)) {
repoPatterns[repo].uses.push(pattern);
}
}
}
const patternMap = {
patterns: patternDefinitions,
repoPatterns
};
fs.writeFileSync('pattern-map.json', JSON.stringify(patternMap, null, 2));
console.log(`✓ Created pattern-map.json (${Object.keys(patternDefinitions).length} patterns defined)\n`);
// ============================================================================
// 4. PLATFORM SUPPORT
// ============================================================================
console.log('📋 Building platform-support.json...');
const platformSupport = {};
for (const repo of allRepos) {
const runtime = (techStack[repo] && techStack[repo].runtime) || null;
platformSupport[repo] = {
platforms: {
bare: runtime === 'bare' || repo.startsWith('bare-'),
node: runtime === 'node' || runtime === 'universal',
pear: runtime === 'pear' || repo.startsWith('pear-'),
browser: runtime === 'universal' && !repo.includes('native')
},
runtime: runtime || 'unknown',
nativeModules: repo.includes('native') || repo.startsWith('lib'),
confidence: runtime ? 'high' : 'low'
};
}
fs.writeFileSync('platform-support.json', JSON.stringify(platformSupport, null, 2));
console.log(`✓ Created platform-support.json (${allRepos.length} repos)\n`);
// ============================================================================
// 5. STABILITY INDEX
// ============================================================================
console.log('📋 Building stability-index.json...');
const stabilityIndex = {};
for (const repo of allRepos) {
const maturity = maturityIndex[repo] || {};
const stars = maturity.stars || 0;
const archived = maturity.archived || false;
const lastUpdated = maturity.lastUpdated || '';
// Calculate age in days
let daysSinceUpdate = 9999;
if (lastUpdated) {
const updateDate = new Date(lastUpdated);
const now = new Date();
daysSinceUpdate = Math.floor((now - updateDate) / (1000 * 60 * 60 * 24));
}
// Determine stability
let stability = 'experimental';
let maturityLevel = 'early';
let developmentStatus = 'inactive';
let recommendedFor = 'learning';
if (archived) {
stability = 'archived';
maturityLevel = 'archived';
developmentStatus = 'archived';
recommendedFor = 'reference-only';
} else if (officialBuildingBlocks.includes(repo) || officialHelpers.includes(repo)) {
stability = 'stable';
maturityLevel = 'mature';
developmentStatus = 'active';
recommendedFor = 'production';
} else if (stars >= 100) {
stability = 'stable';
maturityLevel = 'established';
developmentStatus = daysSinceUpdate < 180 ? 'active' : 'maintenance';
recommendedFor = 'production';
} else if (stars >= 20) {
stability = 'stable';
maturityLevel = 'growing';
developmentStatus = daysSinceUpdate < 365 ? 'active' : 'inactive';
recommendedFor = daysSinceUpdate < 365 ? 'production' : 'evaluation';
} else if (stars >= 5) {
stability = 'experimental';
maturityLevel = 'early';
developmentStatus = daysSinceUpdate < 365 ? 'active' : 'inactive';
recommendedFor = 'evaluation';
} else {
stability = 'experimental';
maturityLevel = 'prototype';
developmentStatus = 'inactive';
recommendedFor = 'learning';
}
const reasoning = archived
? 'Repository is archived'
: officialBuildingBlocks.includes(repo) || officialHelpers.includes(repo)
? 'Official building block or helper, actively maintained'
: stars >= 100
? 'High star count, established project'
: stars >= 20
? 'Moderate star count, growing project'
: 'Low activity or new project';
stabilityIndex[repo] = {
stability,
maturity: maturityLevel,
stars,
lastUpdated,
daysSinceUpdate: daysSinceUpdate === 9999 ? null : daysSinceUpdate,
archived,
developmentStatus,
recommendedFor,
reasoning
};
}
fs.writeFileSync('stability-index.json', JSON.stringify(stabilityIndex, null, 2));
console.log(`✓ Created stability-index.json (${allRepos.length} repos)\n`);
// Generate stability statistics
const stabilityStats = {};
for (const repo of allRepos) {
const stability = stabilityIndex[repo].stability;
stabilityStats[stability] = (stabilityStats[stability] || 0) + 1;
}
console.log(' Stability distribution:');
for (const [stab, count] of Object.entries(stabilityStats).sort((a, b) => b[1] - a[1])) {
console.log(` - ${stab}: ${count} repos`);
}
console.log('');
// ============================================================================
// SUMMARY
// ============================================================================
console.log('✅ Phase 2 Complete!\n');
console.log('Generated files:');
console.log(' ✓ semantic-categories.json');
console.log(' ✓ use-case-map.json');
console.log(' ✓ pattern-map.json');
console.log(' ✓ platform-support.json');
console.log(' ✓ stability-index.json');
console.log('');
console.log('Summary:');
console.log(` - ${allRepos.length} repos categorized`);
console.log(` - ${Object.keys(useCaseDefinitions).length} use cases defined`);
console.log(` - ${Object.keys(patternDefinitions).length} patterns mapped`);
console.log(` - ${Object.keys(apiData).length} repos with deep pattern analysis`);
console.log('');
console.log('Ready for Phase 3: Sub-Agent Development');