| … | … | @@ -210,6 +210,15 @@ export default { |
| 210 | 210 | // 9125f3c missed: that commit set 'anonymous' in relay-node/index.js and |
| 211 | 211 | // bare-relay.js, but the value here is merged in AFTER those defaults, so it |
| 212 | 212 | // won. A default install was silently back on 'authenticated-user'. |
| 213 | // Who owns this relay. The TRUST axis: quorum diversity, and any claim that | |
| 214 | // a quorum spans independent parties, is counted from this. Boxes under one | |
| 215 | // owner must share one value even when they are in different datacenters — | |
| 216 | // use failureDomain to keep them scheduled apart. | |
| 217 | operator: null, | |
| 218 | // Where this relay fails together. The SCHEDULING axis: distinct values make | |
| 219 | // the replica scheduler spread across hosts/racks/datacenters. Safe to make | |
| 220 | // per-box; it asserts nothing about independence. | |
| 221 | failureDomain: null, | |
| 213 | 222 | serviceDefaultPeerRole: 'anonymous', |
| 214 | 223 | serviceAdminAllowlist: [], |
| 215 | 224 | enableServices: false, |
| … | … | @@ -318,6 +318,12 @@ export function buildCapabilityDoc (opts = {}) { |
| 318 | 318 | // floor was being met by an accounting artefact. |
| 319 | 319 | const operator = (typeof config.operator === 'string' && config.operator.trim()) || null |
| 320 | 320 | |
| 321 | // failureDomain — the SCHEDULING axis, kept distinct from `operator`. | |
| 322 | // "If this host dies, what dies with it?" rather than "who could collude?". | |
| 323 | // Two boxes one owner runs in different datacenters should spread replicas | |
| 324 | // (distinct failure domains) while counting as ONE operator. | |
| 325 | const failureDomain = (typeof config.failureDomain === 'string' && config.failureDomain.trim()) || null | |
| 326 | ||
| 321 | 327 | // onionGatewayUrl — when the Tor transport is running with a hidden service, |
| 322 | 328 | // the .onion forwards to the relay's HTTP API/gateway port, so the read plane |
| 323 | 329 | // (/catalog.json, /v1/hyper/:key, /.well-known/hiverelay.json) is reachable |
| … | … | @@ -399,6 +405,7 @@ export function buildCapabilityDoc (opts = {}) { |
| 399 | 405 | // signable shape byte-identical (the checked-in capability-doc conformance |
| 400 | 406 | // vector pins it) and changes the doc only where it carries information. |
| 401 | 407 | ...(operator ? { operator } : {}), |
| 408 | ...(failureDomain ? { failureDomain } : {}), | |
| 402 | 409 | // indexRoom — z32 link to this relay's schema-sheets index room, if a |
| 403 | 410 | // sidecar has published one. Additive: clients that don't understand it |
| 404 | 411 | // ignore it and fall back to catalogBeeKey / /catalog.json. schemaVersion |
| … | … | @@ -110,11 +110,43 @@ export function selectQuorum (candidates, opts = {}) { |
| 110 | 110 | * with highest-scoring candidates — we want SOMETHING served over |
| 111 | 111 | * nothing when the pool is small or concentrated. |
| 112 | 112 | */ |
| 113 | /** | |
| 114 | * The two axes, kept apart on purpose. | |
| 115 | * | |
| 116 | * `failureDomain` answers "if this host/rack/datacenter dies, what else dies | |
| 117 | * with it?" — the SCHEDULING question. `operator` answers "who could collude | |
| 118 | * or be compelled?" — the TRUST question, and the one a G3 claim rests on. | |
| 119 | * | |
| 120 | * They were one field, and the consequence was not theoretical: the fleet | |
| 121 | * deploy script assigned `hive-foundation-utah`, `hive-foundation-utah-us`, | |
| 122 | * `hive-foundation-singapore`, `hive-foundation-singapore-2` … distinct IDs | |
| 123 | * for one owner, with a comment saying they exist so the scheduler treats the | |
| 124 | * boxes as separate nodes. That is the right answer for spreading and the | |
| 125 | * wrong one for independence: it made a fourteen-box, single-owner fleet | |
| 126 | * report up to fourteen operators and satisfy any minOperators floor. | |
| 127 | * | |
| 128 | * Splitting them lets spreading keep working while the independence count | |
| 129 | * becomes honest — which means some quorums will now correctly fail their | |
| 130 | * minOperators floor. That is the intended outcome, not a regression. | |
| 131 | */ | |
| 132 | function failureDomainOf (r) { | |
| 133 | return r.failureDomain || r.operator || r.pubkey | |
| 134 | } | |
| 135 | ||
| 136 | function operatorOf (r) { | |
| 137 | // No `|| pubkey` fallback. Defaulting an undeclared operator to the relay's | |
| 138 | // own key is precisely what manufactured diversity out of nothing; an | |
| 139 | // undeclared operator is UNKNOWN, and unknowns are not independent from | |
| 140 | // each other. | |
| 141 | return r.operator || '__undeclared__' | |
| 142 | } | |
| 143 | ||
| 113 | 144 | function selectDiverse (pool, size, minRegions, minOperators) { |
| 114 | 145 | const ranked = [...pool].sort(byScoreDesc) |
| 115 | 146 | const selected = [] |
| 116 | 147 | const taken = new Set() |
| 117 | 148 | const seenRegions = new Set() |
| 149 | const seenDomains = new Set() | |
| 118 | 150 | const seenOperators = new Set() |
| 119 | 151 | |
| 120 | 152 | while (selected.length < size) { |
| … | … | @@ -123,8 +155,10 @@ function selectDiverse (pool, size, minRegions, minOperators) { |
| 123 | 155 | for (const r of ranked) { |
| 124 | 156 | if (taken.has(r.pubkey)) continue |
| 125 | 157 | const region = r.region || '__unknown__' |
| 126 | const op = r.operator || r.pubkey | |
| 127 | const gain = (seenRegions.has(region) ? 0 : 1) + (seenOperators.has(op) ? 0 : 1) | |
| 158 | // SPREADING axis. Distinct hosts under one owner are still worth | |
| 159 | // spreading across — losing a box should not lose every replica — so | |
| 160 | // this deliberately does NOT use `operator`. | |
| 161 | const gain = (seenRegions.has(region) ? 0 : 1) + (seenDomains.has(failureDomainOf(r)) ? 0 : 1) | |
| 128 | 162 | if (!best || gain > bestGain || (gain === bestGain && byScoreDesc(r, best) < 0)) { |
| 129 | 163 | best = r |
| 130 | 164 | bestGain = gain |
| … | … | @@ -134,7 +168,9 @@ function selectDiverse (pool, size, minRegions, minOperators) { |
| 134 | 168 | selected.push(best) |
| 135 | 169 | taken.add(best.pubkey) |
| 136 | 170 | seenRegions.add(best.region || '__unknown__') |
| 137 | seenOperators.add(best.operator || best.pubkey) | |
| 171 | seenDomains.add(failureDomainOf(best)) | |
| 172 | // INDEPENDENCE axis, counted separately and never widened by spreading. | |
| 173 | seenOperators.add(operatorOf(best)) | |
| 138 | 174 | } |
| 139 | 175 | |
| 140 | 176 | // If we couldn't reach minRegions, the caller should know — emit a |
| … | … | @@ -153,7 +189,12 @@ function selectDiverse (pool, size, minRegions, minOperators) { |
| 153 | 189 | observedRegions: seenRegions.size, |
| 154 | 190 | requiredRegions: minRegions, |
| 155 | 191 | observedOperators: seenOperators.size, |
| 156 | requiredOperators: minOperators | |
| 192 | requiredOperators: minOperators, | |
| 193 | observedFailureDomains: seenDomains.size, | |
| 194 | // Distinguishes "one owner, well spread" from "genuinely independent". | |
| 195 | // Both can serve the same replicas; only the second survives collusion | |
| 196 | // or a single legal demand. | |
| 197 | operatorsUndeclared: seenOperators.has('__undeclared__') | |
| 157 | 198 | } |
| 158 | 199 | } |
| 159 | 200 | |
| … | … | @@ -228,11 +269,13 @@ export function describeQuorum (selected) { |
| 228 | 269 | return { size: 0, regions: [], operators: [], warning: null } |
| 229 | 270 | } |
| 230 | 271 | const regions = [...new Set(selected.map(r => r.region || '__unknown__'))] |
| 231 | const operators = [...new Set(selected.map(r => r.operator || r.pubkey))] | |
| 272 | const operators = [...new Set(selected.map(operatorOf))] | |
| 273 | const failureDomains = [...new Set(selected.map(failureDomainOf))] | |
| 232 | 274 | return { |
| 233 | 275 | size: selected.length, |
| 234 | 276 | regions, |
| 235 | 277 | operators, |
| 278 | failureDomains, | |
| 236 | 279 | warning: selected.diversityWarning || null |
| 237 | 280 | } |
| 238 | 281 | } |
| … | … | @@ -25,14 +25,34 @@ SINGAPORE_API_KEY="${SINGAPORE_API_KEY:-}" |
| 25 | 25 | SINGAPORE2_API_KEY="${SINGAPORE2_API_KEY:-}" |
| 26 | 26 | BERN_API_KEY="${BERN_API_KEY:-}" |
| 27 | 27 | |
| 28 | # Per-relay operator identifiers (for AutoHeal v2 sybil resistance). | |
| 29 | # Singapore-1 and Singapore-2 share a region but get distinct operator IDs | |
| 30 | # so the diversity-enforced replica scheduler treats them as separate nodes. | |
| 31 | UTAH_OPERATOR="${UTAH_OPERATOR:-hive-foundation-utah}" | |
| 32 | UTAH_US_OPERATOR="${UTAH_US_OPERATOR:-hive-foundation-utah-us}" | |
| 33 | SINGAPORE_OPERATOR="${SINGAPORE_OPERATOR:-hive-foundation-singapore}" | |
| 34 | SINGAPORE2_OPERATOR="${SINGAPORE2_OPERATOR:-hive-foundation-singapore-2}" | |
| 35 | BERN_OPERATOR="${BERN_OPERATOR:-hive-foundation-bern}" | |
| 28 | # Operator identity — WHO OWNS THE RELAY. This is the trust axis: quorum | |
| 29 | # independence, and any claim that a quorum spans parties who cannot collude, | |
| 30 | # is counted from this value. Every box below is run by one operator, so they | |
| 31 | # all declare the SAME id. A fourteen-box single-owner fleet is one operator, | |
| 32 | # and saying so is what makes a future independence claim believable. | |
| 33 | # | |
| 34 | # This previously assigned a distinct id per box (hive-foundation-utah, | |
| 35 | # -utah-us, -singapore, -singapore-2 ...) with a comment explaining that they | |
| 36 | # existed so the replica scheduler would treat the boxes as separate nodes. | |
| 37 | # That is the right goal and was the wrong field: it made one owner report as | |
| 38 | # up to fourteen operators and satisfy any minOperators floor automatically. | |
| 39 | # Spreading now lives in FAILURE_DOMAIN below. | |
| 40 | HIVE_OPERATOR="${HIVE_OPERATOR:-hive-foundation}" | |
| 41 | UTAH_OPERATOR="${UTAH_OPERATOR:-$HIVE_OPERATOR}" | |
| 42 | UTAH_US_OPERATOR="${UTAH_US_OPERATOR:-$HIVE_OPERATOR}" | |
| 43 | SINGAPORE_OPERATOR="${SINGAPORE_OPERATOR:-$HIVE_OPERATOR}" | |
| 44 | SINGAPORE2_OPERATOR="${SINGAPORE2_OPERATOR:-$HIVE_OPERATOR}" | |
| 45 | BERN_OPERATOR="${BERN_OPERATOR:-$HIVE_OPERATOR}" | |
| 46 | ||
| 47 | # Failure domain — WHAT DIES TOGETHER. This is the scheduling axis, and it is | |
| 48 | # per-box on purpose: losing one host should not lose every replica. Distinct | |
| 49 | # values here assert nothing about independence, so they are free to be as | |
| 50 | # granular as the topology warrants. | |
| 51 | UTAH_FAILURE_DOMAIN="${UTAH_FAILURE_DOMAIN:-cloudzy-utah}" | |
| 52 | UTAH_US_FAILURE_DOMAIN="${UTAH_US_FAILURE_DOMAIN:-cloudzy-utah-us}" | |
| 53 | SINGAPORE_FAILURE_DOMAIN="${SINGAPORE_FAILURE_DOMAIN:-cloudzy-singapore-1}" | |
| 54 | SINGAPORE2_FAILURE_DOMAIN="${SINGAPORE2_FAILURE_DOMAIN:-cloudzy-singapore-2}" | |
| 55 | BERN_FAILURE_DOMAIN="${BERN_FAILURE_DOMAIN:-cloudzy-bern}" | |
| 36 | 56 | |
| 37 | 57 | deploy_server() { |
| 38 | 58 | local IP=$1 |
| … | … | @@ -98,6 +98,20 @@ test('publishes operator identity so quorum diversity can be counted honestly', |
| 98 | 98 | 'whitespace is not an operator identity') |
| 99 | 99 | }) |
| 100 | 100 | |
| 101 | test('publishes failureDomain separately from operator', async (t) => { | |
| 102 | // Two axes, deliberately distinct. `operator` is who could collude; | |
| 103 | // `failureDomain` is what dies together. Boxes one owner runs in different | |
| 104 | // datacenters share an operator and must NOT share a failure domain. | |
| 105 | const doc = buildCapabilityDoc({ | |
| 106 | relay: { config: { operator: 'hive-foundation', failureDomain: 'cloudzy-singapore-2', regions: ['APAC'] } } | |
| 107 | }) | |
| 108 | t.is(doc.operator, 'hive-foundation') | |
| 109 | t.is(doc.failureDomain, 'cloudzy-singapore-2') | |
| 110 | ||
| 111 | t.absent('failureDomain' in buildCapabilityDoc({ relay: { config: {} } }), | |
| 112 | 'omitted when undeclared, so the default signable shape is unchanged') | |
| 113 | }) | |
| 114 | ||
| 101 | 115 | test('advertises hiverelay-forward only when the opt-in forward relay is enabled', async (t) => { |
| 102 | 116 | const base = { config: { discovery: { dht: true } } } |
| 103 | 117 |
| … | … | @@ -233,3 +233,59 @@ test('relays missing region are bucketed as __unknown__', async (t) => { |
| 233 | 233 | // Should still select both (different operators count as diversity) |
| 234 | 234 | t.is(selected.length, 2) |
| 235 | 235 | }) |
| 236 | ||
| 237 | // ── operator vs failure domain ─────────────────────────────────────────── | |
| 238 | // These two were one field. The fleet deploy script gave each box a distinct | |
| 239 | // "operator" id (hive-foundation-utah, -utah-us, -singapore, -singapore-2 …) | |
| 240 | // with a comment stating they existed so the replica scheduler would treat the | |
| 241 | // boxes as separate nodes. Correct goal, wrong field: it made a single-owner | |
| 242 | // fleet report as many operators and clear any minOperators floor for free. | |
| 243 | ||
| 244 | test('one owner across many hosts counts as ONE operator', async (t) => { | |
| 245 | // The real fleet shape: one owner, boxes in different datacenters. | |
| 246 | const fleet = [ | |
| 247 | { pubkey: 'a1', region: 'NA', operator: 'hive-foundation', failureDomain: 'cloudzy-utah', score: 0.9 }, | |
| 248 | { pubkey: 'a2', region: 'NA', operator: 'hive-foundation', failureDomain: 'cloudzy-utah-us', score: 0.85 }, | |
| 249 | { pubkey: 'a3', region: 'APAC', operator: 'hive-foundation', failureDomain: 'cloudzy-singapore-1', score: 0.8 }, | |
| 250 | { pubkey: 'a4', region: 'APAC', operator: 'hive-foundation', failureDomain: 'cloudzy-singapore-2', score: 0.75 }, | |
| 251 | { pubkey: 'a5', region: 'ME', operator: 'hive-foundation', failureDomain: 'cloudzy-dubai', score: 0.7 } | |
| 252 | ] | |
| 253 | const selected = selectQuorum(fleet, { strategy: 'diverse', size: 5, minOperators: 3 }) | |
| 254 | const desc = describeQuorum(selected) | |
| 255 | ||
| 256 | t.is(desc.operators.length, 1, 'one owner is one operator, however many boxes') | |
| 257 | t.ok(desc.failureDomains.length >= 4, 'but replicas still spread across hosts') | |
| 258 | t.ok(selected.diversityWarning, 'and the independence floor honestly fails') | |
| 259 | t.is(selected.diversityWarning.reason, 'insufficient-operator-diversity') | |
| 260 | t.is(selected.diversityWarning.observedOperators, 1) | |
| 261 | }) | |
| 262 | ||
| 263 | test('spreading still works when only failure domains differ', async (t) => { | |
| 264 | // Two boxes, same owner, same region, different hosts. Losing one host must | |
| 265 | // not lose both replicas, so the scheduler must still treat them as distinct. | |
| 266 | const pair = [ | |
| 267 | { pubkey: 'b1', region: 'NA', operator: 'hive-foundation', failureDomain: 'host-1', score: 0.9 }, | |
| 268 | { pubkey: 'b2', region: 'NA', operator: 'hive-foundation', failureDomain: 'host-1', score: 0.8 }, | |
| 269 | { pubkey: 'b3', region: 'NA', operator: 'hive-foundation', failureDomain: 'host-2', score: 0.7 } | |
| 270 | ] | |
| 271 | const selected = selectQuorum(pair, { strategy: 'diverse', size: 2, minRegions: 1, minOperators: 1 }) | |
| 272 | const domains = new Set(selected.map(s => s.failureDomain)) | |
| 273 | t.is(domains.size, 2, 'picks across two hosts rather than doubling up on one') | |
| 274 | }) | |
| 275 | ||
| 276 | test('an undeclared operator is unknown, not independent', async (t) => { | |
| 277 | // The old fallback was `operator || pubkey`, which minted a fresh operator | |
| 278 | // identity from every relay key — manufacturing independence out of silence. | |
| 279 | const anon = [ | |
| 280 | { pubkey: 'c1', region: 'NA', score: 0.9 }, | |
| 281 | { pubkey: 'c2', region: 'EU', score: 0.8 }, | |
| 282 | { pubkey: 'c3', region: 'APAC', score: 0.7 } | |
| 283 | ] | |
| 284 | const selected = selectQuorum(anon, { strategy: 'diverse', size: 3, minRegions: 3, minOperators: 3 }) | |
| 285 | const desc = describeQuorum(selected) | |
| 286 | ||
| 287 | t.is(desc.operators.length, 1, 'three undeclared relays are not three operators') | |
| 288 | t.ok(selected.diversityWarning, 'undeclared operators cannot satisfy an independence floor') | |
| 289 | t.is(selected.diversityWarning.operatorsUndeclared, true) | |
| 290 | t.is(desc.failureDomains.length, 3, 'they are still spread, falling back to pubkey') | |
| 291 | }) |