diff --git a/.gitignore b/.gitignore index 5a541bd..c764717 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ dist/ node_modules/ +coverage/ *.swp *.swo diff --git a/README.md b/README.md index 3aa2260..343e9f0 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,110 @@ If we express views and filter rules as JSON-Logic, we can easily include them in records. +# Development / Demo + +## Setup + +Install [`nvm`](https://nvm.sh) + +Clone repo +```bash +git clone https://gitea.dgov.io/ladd/rhizome +``` + +Use `nvm` to install and activate the target nodejs version +```bash +nvm install +``` + +Install nodejs packages +```bash +npm install +``` + +## Build + +Compile Typescript +```bash +npm run build +``` + +During development, it's useful to run the compiler in watch mode: +```bash +npm run build:watch +``` + +## Run tests + +```bash +npm run test +``` + +## Run test coverage report + +```bash +npm run test:coverage +``` + +## Run multiple live nodes locally as separate processes +To demonstrate the example application, you can open multiple terminals, and in each terminal execute something like the following: + +```bash +export DEBUG="*,-express" +export RHIZOME_REQUEST_BIND_PORT=4000 +export RHIZOME_PUBLISH_BIND_PORT=4001 +export RHIZOME_SEED_PEERS='localhost:4002, localhost:4004' +export RHIZOME_HTTP_API_PORT=3000 +export RHIZOME_PEER_ID=peer1 +npm run example-app +``` + +```bash +export DEBUG="*,-express" +export RHIZOME_REQUEST_BIND_PORT=4002 +export RHIZOME_PUBLISH_BIND_PORT=4003 +export RHIZOME_SEED_PEERS='localhost:4000, localhost:4004' +export RHIZOME_HTTP_API_PORT=3001 +export RHIZOME_PEER_ID=peer2 +npm run example-app +``` + +```bash +export DEBUG="*,-express" +export RHIZOME_REQUEST_BIND_PORT=4004 +export RHIZOME_PUBLISH_BIND_PORT=4005 +export RHIZOME_SEED_PEERS='localhost:4000, localhost:4002' +export RHIZOME_HTTP_API_PORT=3002 +export RHIZOME_PEER_ID=peer3 +npm run example-app +``` + +In a separate terminal, you can use `curl` to interact with an instance. + +`jq` is helpful for formatting the json responses. + +Query the number of peers seen by a given node (including itself) +```bash +curl -s http://localhost:3000/peers/count | jq +``` + +Query the list of peers seen by a given node (including itself) +```bash +curl -s http://localhost:3000/peers | jq +``` + +Query the number of deltas ingested by this node +```bash +curl -s http://localhost:3000/deltas/count | jq +``` + +Query the list of deltas ingested by this node +```bash +curl -s http://localhost:3000/deltas | jq +``` + +# More About Concepts + ## Clocks? Do we want to involve a time synchronization protocol? e.g. ntpd @@ -78,94 +182,3 @@ Considerations imposed by Tinc would include * IP addressing * public key management -# Development / Demo - -## Setup - -Install [`nvm`](https://nvm.sh) - -Clone repo -```bash -git clone https://gitea.dgov.io/ladd/rhizome -``` - -Use `nvm` to install and activate the target nodejs version -```bash -nvm install -``` - -Install nodejs packages -```bash -npm install -``` - -## Build - -Compile Typescript -```bash -npm run build -``` - -During development, it's useful to run the compiler in watch mode: -```bash -npm run build:watch -``` - -## Run - -To demonstrate the example application, you can open multiple terminals, and in each terminal execute something like the following: - -```bash -export DEBUG="*,-express" -export RHIZOME_REQUEST_BIND_PORT=4000 -export RHIZOME_PUBLISH_BIND_PORT=4001 -export RHIZOME_SEED_PEERS='localhost:4002, localhost:4004' -export RHIZOME_HTTP_API_PORT=3000 -export RHIZOME_PEER_ID=peer1 -npm run example-app -``` - -```bash -export DEBUG="*,-express" -export RHIZOME_REQUEST_BIND_PORT=4002 -export RHIZOME_PUBLISH_BIND_PORT=4003 -export RHIZOME_SEED_PEERS='localhost:4000, localhost:4004' -export RHIZOME_HTTP_API_PORT=3001 -export RHIZOME_PEER_ID=peer2 -npm run example-app -``` - -```bash -export DEBUG="*,-express" -export RHIZOME_REQUEST_BIND_PORT=4004 -export RHIZOME_PUBLISH_BIND_PORT=4005 -export RHIZOME_SEED_PEERS='localhost:4000, localhost:4002' -export RHIZOME_HTTP_API_PORT=3002 -export RHIZOME_PEER_ID=peer3 -npm run example-app -``` - -In a separate terminal, you can use `curl` to interact with an instance. - -`jq` is helpful for formatting the json responses. - -Query the number of peers seen by a given node (including itself) -```bash -curl -s http://localhost:3000/peers/count | jq -``` - -Query the list of peers seen by a given node (including itself) -```bash -curl -s http://localhost:3000/peers | jq -``` - -Query the number of deltas ingested by this node -```bash -curl -s http://localhost:3000/deltas/count | jq -``` - -Query the list of deltas ingested by this node -```bash -curl -s http://localhost:3000/deltas | jq -``` - diff --git a/__tests__/lossless.ts b/__tests__/lossless.ts index 2863cb9..f0ae55f 100644 --- a/__tests__/lossless.ts +++ b/__tests__/lossless.ts @@ -135,7 +135,22 @@ describe('Lossless', () => { return creator === 'A' && host === 'H'; }; - expect(lossless.view(filter)).toEqual({ + expect(lossless.view(undefined, filter)).toEqual({ + ace: { + referencedAs: ["1"], + properties: { + value: [{ + creator: 'A', + host: 'H', + pointers: [ + {"1": "ace"}, + ] + }] + } + } + }); + + expect(lossless.view(["ace"], filter)).toEqual({ ace: { referencedAs: ["1"], properties: { diff --git a/__tests__/lossy.ts b/__tests__/lossy.ts index e0c0b38..c910fde 100644 --- a/__tests__/lossy.ts +++ b/__tests__/lossy.ts @@ -1,7 +1,6 @@ -import Debug from "debug"; import {Lossless, LosslessViewMany} from "../src/lossless"; import {Lossy, firstValueFromLosslessViewOne, valueFromCollapsedDelta} from "../src/lossy"; -const debug = Debug('test:lossy'); +import {PointerTarget} from "../src/types"; describe('Lossy', () => { describe('se a provided function to resolve entity views', () => { @@ -36,9 +35,9 @@ describe('Lossy', () => { it('example summary', () => { type Role = { - actor: string, - film: string, - role: string + actor: PointerTarget, + film: PointerTarget, + role: PointerTarget }; type Summary = { @@ -47,14 +46,12 @@ describe('Lossy', () => { const resolver = (losslessView: LosslessViewMany): Summary => { const roles: Role[] = []; - debug('resolving roles'); for (const [id, ent] of Object.entries(losslessView)) { if (ent.referencedAs.includes("role")) { const {delta, value: actor} = firstValueFromLosslessViewOne(ent, "actor") ?? {}; if (!delta) continue; // TODO: panic if (!actor) continue; // TODO: panic const film = valueFromCollapsedDelta(delta, "film"); - debug(`role ${id}`, {actor, film}); if (!film) continue; // TODO: panic roles.push({ role: id, diff --git a/__tests__/run/001-single-node.ts b/__tests__/run/001-single-node.ts index 3310f9b..8a2a728 100644 --- a/__tests__/run/001-single-node.ts +++ b/__tests__/run/001-single-node.ts @@ -17,9 +17,8 @@ describe('Run', () => { await app.stop(); }); - it('can put a new user', async () => { - const {httpAddr, httpPort} = app.config; - const res = await fetch(`http://${httpAddr}:${httpPort}/users`, { + it('can put a new user and fetch it', async () => { + const res = await fetch(`${app.apiUrl}/user`, { method: 'PUT', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ @@ -38,5 +37,18 @@ describe('Run', () => { age: 263 } }); + + await new Promise((resolve) => setTimeout(resolve, 100)); + + const res2 = await fetch(`${app.apiUrl}/user/peon-1`); + const data2 = await res2.json(); + expect(data2).toMatchObject({ + id: "peon-1", + properties: { + name: "Peon", + age: 263 + } + }); + }); }); diff --git a/__tests__/run/002-two-nodes.ts b/__tests__/run/002-two-nodes.ts index 5b367d0..c22b75b 100644 --- a/__tests__/run/002-two-nodes.ts +++ b/__tests__/run/002-two-nodes.ts @@ -28,7 +28,7 @@ describe('Run', () => { debug('apps[0].apiUrl', apps[0].apiUrl); debug('apps[1].apiUrl', apps[1].apiUrl); - const res = await fetch(`${apps[0].apiUrl}/users`, { + const res = await fetch(`${apps[0].apiUrl}/user`, { method: 'PUT', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ @@ -50,7 +50,7 @@ describe('Run', () => { await new Promise((resolve) => setTimeout(resolve, 100)); - const res2 = await fetch(`${apps[1].apiUrl}/users/peon-1`); + const res2 = await fetch(`${apps[1].apiUrl}/user/peon-1`); const data2 = await res2.json(); debug('data2', data2); expect(data2).toMatchObject({ diff --git a/markdown/coverage_report.md b/markdown/coverage_report.md new file mode 100644 index 0000000..b383344 --- /dev/null +++ b/markdown/coverage_report.md @@ -0,0 +1,38 @@ + +> rhizome-node@1.0.0 test +> jest --coverage + +PASS __tests__/lossy.ts +PASS __tests__/lossless.ts +PASS __tests__/peer-address.ts +PASS __tests__/run/002-two-nodes.ts +PASS __tests__/run/001-single-node.ts +----------------------|---------|----------|---------|---------|---------------------------------------------------- +File | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s +----------------------|---------|----------|---------|---------|---------------------------------------------------- +All files | 86.86 | 62.41 | 82.7 | 87.38 | + src | 87.94 | 68.06 | 82.45 | 88.09 | + collection.ts | 88.31 | 71.42 | 66.66 | 90.54 | 62-65,114-122,172 + config.ts | 94.44 | 89.65 | 50 | 94.44 | 22 + deltas.ts | 64.44 | 50 | 76.92 | 64.44 | 27-30,42-46,55-56,64-73 + entity.ts | 100 | 100 | 100 | 100 | + http-api.ts | 59.7 | 13.04 | 38.88 | 59.7 | 32,37,44-60,66,79-80,85-92,100,121,129-130,145-151 + lossless.ts | 98.27 | 91.66 | 100 | 100 | 96 + lossy.ts | 100 | 85.71 | 100 | 100 | 38 + node.ts | 100 | 100 | 100 | 100 | + peers.ts | 96.82 | 100 | 100 | 96.61 | 125-126 + pub-sub.ts | 100 | 100 | 100 | 100 | + request-reply.ts | 95.65 | 0 | 100 | 95.34 | 46,59 + typed-collection.ts | 100 | 100 | 100 | 100 | + types.ts | 100 | 100 | 100 | 100 | + src/util | 74.54 | 31.81 | 82.35 | 78 | + md-files.ts | 74.54 | 31.81 | 82.35 | 78 | 52-56,90-94,108-115 + util | 100 | 100 | 100 | 100 | + app.ts | 100 | 100 | 100 | 100 | +----------------------|---------|----------|---------|---------|---------------------------------------------------- + +Test Suites: 5 passed, 5 total +Tests: 7 passed, 7 total +Snapshots: 0 total +Time: 3.709 s, estimated 5 s +Ran all test suites. diff --git a/package.json b/package.json index 278fbd8..e3e4df4 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "build": "tsc", "build:watch": "tsc --watch", "lint": "eslint", - "test": "jest" + "test": "jest", + "test:coverage": "./scripts/coverage.sh" }, "jest": { "testEnvironment": "node", diff --git a/scripts/coverage.sh b/scripts/coverage.sh new file mode 100755 index 0000000..952d3a3 --- /dev/null +++ b/scripts/coverage.sh @@ -0,0 +1,16 @@ +#!/bin/env bash + +force=false + +while [[ -n "$1" ]]; do + case "$1" in + -f | --force) + force=true + ;; + esac + shift +done + +dest="./markdown/coverage_report.md" + +npm run test -- --coverage 2>&1 | tee "$dest" diff --git a/src/collection.ts b/src/collection.ts index a99e7db..9e15652 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -3,44 +3,45 @@ // It should enable operations like removing a property removes the value from the entities in the collection // It could then be further extended with e.g. table semantics like filter, sort, join +import Debug from 'debug'; import {randomUUID} from "node:crypto"; import EventEmitter from "node:events"; +import {Entity} from "./entity"; +import {Lossless, LosslessViewMany} from "./lossless"; +import {firstValueFromLosslessViewOne, Lossy, LossyViewMany, LossyViewOne} from "./lossy"; import {RhizomeNode} from "./node"; -import {Entity, EntityProperties, EntityPropertiesDeltaBuilder} from "./object-layer"; import {Delta} from "./types"; - -// type Property = { -// name: string, -// type: number | string; -// } - -// class EntityType { -// name: string; -// properties?: Property[]; -// constructor(name: string) { -// this.name = name; -// } -// } +const debug = Debug('collection'); export class Collection { rhizomeNode?: RhizomeNode; name: string; entities = new Map(); eventStream = new EventEmitter(); + lossless = new Lossless(); // TODO: Really just need one global Lossless instance constructor(name: string) { this.name = name; } + // Instead of trying to update our final view of the entity with every incoming delta, + // let's try this: + // - keep a lossless view (of everything) + // - build a lossy view when needed + // This approach is simplistic, but can then be optimized and enhanced. + rhizomeConnect(rhizomeNode: RhizomeNode) { this.rhizomeNode = rhizomeNode; rhizomeNode.deltaStream.subscribeDeltas((delta: Delta) => { // TODO: Make sure this is the kind of delta we're looking for - this.applyDelta(delta); + debug(`collection ${this.name} received delta ${JSON.stringify(delta)}`); + this.lossless.ingestDelta(delta); }); rhizomeNode.httpApi.serveCollection(this); + + debug(`connected ${this.name} to rhizome`); } // Applies the javascript rules for updating object values, @@ -55,7 +56,6 @@ export class Collection { entity.id = entityId; eventType = 'create'; } - const deltaBulider = new EntityPropertiesDeltaBuilder(this.rhizomeNode!, entityId); if (!properties) { // Let's interpret this as entity deletion @@ -74,69 +74,50 @@ export class Collection { } if (local && changed) { // If this is a change, let's generate a delta - deltaBulider.add(key, value); + if (!this.rhizomeNode) throw new Error(`${this.name} collection not connected to rhizome`); + const delta: Delta = { + creator: this.rhizomeNode.config.creator, + host: this.rhizomeNode.config.peerId, + pointers: [{ + localContext: this.name, + target: entityId, + targetContext: key + }, { + localContext: key, + target: value + }] + }; + deltas?.push(delta); + // We append to the array the caller may provide // We can update this count as we receive network confirmation for deltas entity.ahead += 1; } anyChanged = anyChanged || changed; }); - // We've noted that we may be ahead of the server, let's update our - // local image of this entity. - //* In principle, this system can recreate past or alternative states. - //* At worst, by replaying all the deltas up to a particular point. - //* Some sort of checkpointing strategy would probably be helpful. - //* Furthermore, if we can implement reversible transformations, - //* it would then be efficient to calculate the state of the system with - //* specific deltas removed. We could use it to extract a measurement - //* of the effects of some deltas' inclusion or exclusion, the - //* evaluation of which may lend evidence to some possible arguments. this.entities.set(entityId, entity); + if (anyChanged) { - deltas?.push(deltaBulider.delta); eventType = eventType || 'update'; } } if (eventType) { + // TODO: Reconcile this with lossy view approach this.eventStream.emit(eventType, entity); } return entity; } - // We can update our local image of the entity, but we should annotate it - // to indicate that we have not yet received any confirmation of this delta - // having been propagated. - // Later when we receive deltas regarding this entity we can detect when - // we have received back an image that matches our target. - - // So we need a function to generate one or more deltas for each call to put/ - // maybe we stage them and wait for a call to commit() that initiates the - // assembly and transmission of one or more deltas - - applyDelta(delta: Delta) { - // TODO: handle delta representing entity deletion - const idPtr = delta.pointers.find(({localContext}) => localContext === 'id'); - if (!idPtr) { - console.error('encountered delta with no entity id', delta); - return; - } - const properties: EntityProperties = {}; - delta.pointers.filter(({localContext}) => localContext !== 'id') - .forEach(({localContext: key, target: value}) => { - properties[key] = value; - }, {}); - const entityId = idPtr.target as string; - // TODO: Handle the scenario where this update has been superceded by a newer one locally - this.updateEntity(entityId, properties); - } onCreate(cb: (entity: Entity) => void) { + // TODO: Reconcile this with lossy view approach this.eventStream.on('create', (entity: Entity) => { cb(entity); }); } onUpdate(cb: (entity: Entity) => void) { + // TODO: Reconcile this with lossy view approach this.eventStream.on('update', (entity: Entity) => { cb(entity); }); @@ -145,25 +126,46 @@ export class Collection { put(entityId: string | undefined, properties: object): Entity { const deltas: Delta[] = []; const entity = this.updateEntity(entityId, properties, true, deltas); + + debug(`put ${entityId} generated deltas:`, JSON.stringify(deltas)); + + // updateEntity may have generated some deltas for us to store and publish deltas.forEach(async (delta: Delta) => { + + // record this delta just as if we had received it from a peer delta.receivedFrom = this.rhizomeNode!.myRequestAddr; this.rhizomeNode!.deltaStream.deltasAccepted.push(delta); + + // publish the delta to our subscribed peers await this.rhizomeNode!.deltaStream.publishDelta(delta); + debug(`published delta ${JSON.stringify(delta)}`); + + // ingest the delta as though we had received it from a peer + this.lossless.ingestDelta(delta); }); return entity; } - del(entityId: string) { - const deltas: Delta[] = []; - this.updateEntity(entityId, undefined, true, deltas); - deltas.forEach(async (delta: Delta) => { - this.rhizomeNode!.deltaStream.deltasAccepted.push(delta); - await this.rhizomeNode!.deltaStream.publishDelta(delta); - }); - } - - get(id: string): Entity | undefined { - return this.entities.get(id); + get(id: string): LossyViewOne | undefined { + // Now with lossy view approach, instead of just returning what we already have, + // let's compute our view now. + // return this.entities.get(id); + const lossy = new Lossy(this.lossless); + const resolver = (losslessView: LosslessViewMany) => { + const lossyView: LossyViewMany = {}; + debug('lossless view', JSON.stringify(losslessView)); + for (const [id, ent] of Object.entries(losslessView)) { + lossyView[id] = {id, properties: {}}; + for (const key of Object.keys(ent.properties)) { + const {value} = firstValueFromLosslessViewOne(ent, key) || {}; + debug(`[ ${key} ] = ${value}`); + lossyView[id].properties[key] = value; + } + } + return lossyView; + }; + const res = lossy.resolve(resolver, [id]) as LossyViewMany;; + return res[id]; } getIds(): string[] { diff --git a/src/context.ts b/src/context.ts index e69de29..f8d5957 100644 --- a/src/context.ts +++ b/src/context.ts @@ -0,0 +1,24 @@ +// A delta represents an assertion from a given (perspective/POV/context). +// So we want it to be fluent to express these in the local context, +// and propagated as deltas in a configurable manner; i.e. configurable batches or immediate + +// import {Delta} from './types'; + +export class Entity { +} + +export class Context { +} + +export class Assertion { +} + +export class View { +} + +export class User { + +} + +export function example() { +} diff --git a/src/entity.ts b/src/entity.ts new file mode 100644 index 0000000..2dbe872 --- /dev/null +++ b/src/entity.ts @@ -0,0 +1,25 @@ +// The goal here is to provide a translation for +// entities and their properties +// to and from (sequences of) deltas. + +// How can our caller define the entities and their properties? +// - As typescript types? +// - As typescript interfaces? +// - As typescript classes? + +import {PropertyTypes} from "./types"; + +export type EntityProperties = { + [key: string]: PropertyTypes; +}; + +export class Entity { + id: string; + properties: EntityProperties = {}; + ahead = 0; + + constructor(id: string) { + this.id = id; + } +} + diff --git a/src/example-app.ts b/src/example-app.ts index 057a473..962c604 100644 --- a/src/example-app.ts +++ b/src/example-app.ts @@ -1,6 +1,6 @@ import Debug from 'debug'; import {RhizomeNode} from "./node"; -import {Entity} from "./object-layer"; +import {Entity} from "./entity"; import {TypedCollection} from "./typed-collection"; const debug = Debug('example-app'); diff --git a/src/http-api.ts b/src/http-api.ts index 29bfb98..1cf61f8 100644 --- a/src/http-api.ts +++ b/src/http-api.ts @@ -24,42 +24,7 @@ export class HttpApi { } start() { - // Scan and watch for markdown files - this.mdFiles.readDir(); - this.mdFiles.readReadme(); - this.mdFiles.watchDir(); - this.mdFiles.watchReadme(); - - // Serve README - this.router.get('/html/README', (_req: express.Request, res: express.Response) => { - const html = this.mdFiles.getReadmeHTML(); - res.setHeader('content-type', 'text/html').send(html); - }); - - // Serve markdown files as html - this.router.get('/html/:name', (req: express.Request, res: express.Response) => { - let html = this.mdFiles.getHtml(req.params.name); - if (!html) { - res.status(404); - html = htmlDocFromMarkdown('# 404\n\n## [Index](/html)'); - } - res.setHeader('content-type', 'text/html'); - res.send(html); - }); - - // Serve index - { - let md = `# Files\n\n`; - md += `[README](/html/README)\n\n`; - for (const name of this.mdFiles.list()) { - md += `- [${name}](./${name})\n`; - } - const html = htmlDocFromMarkdown(md); - - this.router.get('/html', (_req: express.Request, res: express.Response) => { - res.setHeader('content-type', 'text/html').send(html); - }); - } + // --------------- deltas ---------------- // Serve list of all deltas accepted // TODO: This won't scale well @@ -72,6 +37,8 @@ export class HttpApi { res.json(this.rhizomeNode.deltaStream.deltasAccepted.length); }); + // --------------- peers ---------------- + // Get the list of peers seen by this node (including itself) this.router.get("/peers", (_req: express.Request, res: express.Response) => { res.json(this.rhizomeNode.peers.peers.map(({reqAddr, publishAddr, isSelf, isSeedPeer}) => { @@ -99,6 +66,43 @@ export class HttpApi { res.json(this.rhizomeNode.peers.peers.length); }); + // ----------------- html --------------------- + + // Scan and watch for markdown files + this.mdFiles.readDir(); + this.mdFiles.readReadme(); + this.mdFiles.watchDir(); + this.mdFiles.watchReadme(); + + // Serve README + this.router.get('/html/README', (_req: express.Request, res: express.Response) => { + const html = this.mdFiles.getReadmeHTML(); + res.setHeader('content-type', 'text/html').send(html); + }); + + // Serve markdown files as html + this.router.get('/html/:name', (req: express.Request, res: express.Response) => { + const {name} = req.params; + let html = this.mdFiles.getHtml(name); + if (!html) { + res.status(404); + html = htmlDocFromMarkdown(`# 404 Not Found: ${name}\n\n ## [Index](/html)`); + } + res.setHeader('content-type', 'text/html'); + res.send(html); + }); + + // Serve index + { + const html = this.mdFiles.generateIndex(); + + this.router.get('/html', (_req: express.Request, res: express.Response) => { + res.setHeader('content-type', 'text/html').send(html); + }); + } + + // ------------------- server --------------------- + const {httpAddr, httpPort} = this.rhizomeNode.config; this.server = this.app.listen({ port: httpPort, diff --git a/src/lossless.ts b/src/lossless.ts index e9d0686..2c42578 100644 --- a/src/lossless.ts +++ b/src/lossless.ts @@ -1,39 +1,32 @@ // Deltas target entities. // We can maintain a record of all the targeted entities, and the deltas that targeted them -import {Delta, DeltaFilter, PropertyTypes} from "./types"; - -type DomainEntityID = string; -type PropertyID = string; +import Debug from 'debug'; +import {Delta, DeltaFilter, DomainEntityID, Properties, PropertyID, PropertyTypes} from "./types"; +const debug = Debug('lossless'); export type CollapsedPointer = {[key: string]: PropertyTypes}; + export type CollapsedDelta = Omit & { pointers: CollapsedPointer[]; }; + export type LosslessViewOne = { referencedAs: string[]; properties: { [key: PropertyID]: CollapsedDelta[] } }; + export type LosslessViewMany = { [key: DomainEntityID]: LosslessViewOne; }; class DomainEntityMap extends Map {}; -class DomainEntityProperty { - id: PropertyID; - deltas = new Set(); - - constructor(id: PropertyID) { - this.id = id; - } -} - class DomainEntity { id: DomainEntityID; - properties = new Map(); + properties = new Map>(); constructor(id: DomainEntityID) { this.id = id; @@ -44,15 +37,29 @@ class DomainEntity { .filter(({target}) => target === this.id) .map(({targetContext}) => targetContext) .filter((targetContext) => typeof targetContext === 'string'); + for (const targetContext of targetContexts) { - let property = this.properties.get(targetContext); - if (!property) { - property = new DomainEntityProperty(targetContext); - this.properties.set(targetContext, property); + let propertyDeltas = this.properties.get(targetContext); + if (!propertyDeltas) { + propertyDeltas = new Set(); + this.properties.set(targetContext, propertyDeltas); } - property.deltas.add(delta); + + debug(`adding delta for entity ${this.id}`); + propertyDeltas.add(delta); } } + + toJSON() { + const properties: {[key: PropertyID]: number} = {}; + for (const [key, deltas] of this.properties.entries()) { + properties[key] = deltas.size; + } + return { + id: this.id, + properties + }; + } } export class Lossless { @@ -63,47 +70,70 @@ export class Lossless { .filter(({targetContext}) => !!targetContext) .map(({target}) => target) .filter((target) => typeof target === 'string') + for (const target of targets) { let ent = this.domainEntities.get(target); + if (!ent) { ent = new DomainEntity(target); this.domainEntities.set(target, ent); } + + debug('before add, domain entity:', JSON.stringify(ent)); + ent.addDelta(delta); + + debug('after add, domain entity:', JSON.stringify(ent)); } } //TODO: json logic -- view(deltaFilter?: FilterExpr) { - view(deltaFilter?: DeltaFilter): LosslessViewMany { + view(entityIds?: DomainEntityID[], deltaFilter?: DeltaFilter): LosslessViewMany { const view: LosslessViewMany = {}; - for (const ent of this.domainEntities.values()) { + entityIds = entityIds ?? Array.from(this.domainEntities.keys()); + for (const id of entityIds) { + const ent = this.domainEntities.get(id); + if (!ent) continue; + + debug(`domain entity ${id}`, JSON.stringify(ent)); + const referencedAs = new Set(); - view[ent.id] = { - referencedAs: [], - properties: {} - }; - for (const prop of ent.properties.values()) { - view[ent.id].properties[prop.id] = view[ent.id].properties[prop.id] || []; - for (const delta of prop.deltas) { + const properties: { + [key: PropertyID]: CollapsedDelta[] + } = {}; + + for (const [key, deltas] of ent.properties.entries()) { + properties[key] = properties[key] || []; + + for (const delta of deltas) { + if (deltaFilter) { const include = deltaFilter(delta); if (!include) continue; } + const pointers: CollapsedPointer[] = []; + for (const {localContext, target} of delta.pointers) { pointers.push({[localContext]: target}); if (target === ent.id) { referencedAs.add(localContext); } } + const collapsedDelta: CollapsedDelta = { ...delta, pointers }; - view[ent.id].referencedAs = Array.from(referencedAs.values()); - view[ent.id].properties[prop.id].push(collapsedDelta); + + properties[key].push(collapsedDelta); } } + + view[ent.id] = { + referencedAs: Array.from(referencedAs.values()), + properties + }; } return view; } diff --git a/src/lossy.ts b/src/lossy.ts index 8780c7e..6bb4895 100644 --- a/src/lossy.ts +++ b/src/lossy.ts @@ -5,24 +5,36 @@ // We can achieve this via functional expression, encoded as JSON-Logic. // Fields in the output can be described as transformations +import Debug from 'debug'; import {CollapsedDelta, Lossless, LosslessViewMany, LosslessViewOne} from "./lossless"; -import {DeltaFilter} from "./types"; +import {DeltaFilter, DomainEntityID, Properties} from "./types"; +const debug = Debug('lossy'); -type Resolver = (losslessView: LosslessViewMany) => unknown; +export type LossyViewOne = { + id: DomainEntityID; + properties: Properties; +}; + +export type LossyViewMany = { + [key: DomainEntityID]: LossyViewOne; +}; + +type Resolver = (losslessView: LosslessViewMany) => LossyViewMany | unknown; // Extract a particular value from a delta's pointers -export function valueFromCollapsedDelta(delta: CollapsedDelta, key: string): string | undefined { - const pointers = delta.pointers; - for (const pointer of pointers || []) { - const [[k, value]] = Object.entries(pointer); - if (k === key && typeof value === "string") { - return value; +export function valueFromCollapsedDelta(delta: CollapsedDelta, key: string): string | number | undefined { + for (const pointer of delta.pointers) { + for (const [k, value] of Object.entries(pointer)) { + if (k === key && (typeof value === "string" || typeof value === "number")) { + return value; + } } } } // Example function for resolving a value for an entity by taking the first value we find -export function firstValueFromLosslessViewOne(ent: LosslessViewOne, key: string): {delta: CollapsedDelta, value: string} | undefined { +export function firstValueFromLosslessViewOne(ent: LosslessViewOne, key: string): {delta: CollapsedDelta, value: string | number} | undefined { + debug(`trying to get value for ${key} from ${JSON.stringify(ent.properties[key])}`); for (const delta of ent.properties[key] || []) { const value = valueFromCollapsedDelta(delta, key); if (value) return {delta, value}; @@ -36,8 +48,8 @@ export class Lossy { this.lossless = lossless; } - resolve(fn: Resolver, deltaFilter?: DeltaFilter) { - return fn(this.lossless.view(deltaFilter)); + resolve(fn: Resolver, entityIds?: DomainEntityID[], deltaFilter?: DeltaFilter) { + return fn(this.lossless.view(entityIds, deltaFilter)); } } diff --git a/src/node.ts b/src/node.ts index 32a0a6e..c5be620 100644 --- a/src/node.ts +++ b/src/node.ts @@ -67,15 +67,28 @@ export class RhizomeNode { } async start() { + // Start ZeroMQ publish and reply sockets this.pubSub.start(); this.requestReply.start(); + + // Start HTTP server if (this.config.httpEnable) { this.httpApi.start(); } + + // Wait a short time for sockets to initialize await new Promise((resolve) => setTimeout(resolve, 500)); + + // Subscribe to seed peers this.peers.subscribeToSeeds(); + + // Wait a short time for sockets to initialize await new Promise((resolve) => setTimeout(resolve, 500)); + + // Ask all peers for all deltas this.peers.askAllPeersForDeltas(); + + // Wait to receive all deltas await new Promise((resolve) => setTimeout(resolve, 1000)); } diff --git a/src/object-layer.ts b/src/object-layer.ts deleted file mode 100644 index bab164d..0000000 --- a/src/object-layer.ts +++ /dev/null @@ -1,48 +0,0 @@ -// The goal here is to provide a translation for -// entities and their properties -// to and from (sequences of) deltas. - -// How can our caller define the entities and their properties? -// - As typescript types? -// - As typescript interfaces? -// - As typescript classes? - -import {RhizomeNode} from "./node"; -import {Delta, PropertyTypes} from "./types"; - -export type EntityProperties = { - [key: string]: PropertyTypes; -}; - -export class Entity { - id: string; - properties: EntityProperties = {}; - ahead = 0; - - constructor(id: string) { - this.id = id; - } -} - -// TODO: Use leveldb for storing view snapshots - -export class EntityPropertiesDeltaBuilder { - delta: Delta; - - constructor(rhizomeNode: RhizomeNode, entityId: string) { - this.delta = { - creator: rhizomeNode.config.creator, - host: rhizomeNode.config.peerId, - pointers: [{ - localContext: 'id', - target: entityId, - targetContext: 'properties' - }] - }; - } - - add(localContext: string, target: PropertyTypes) { - this.delta.pointers.push({localContext, target}); - } -} - diff --git a/src/typed-collection.ts b/src/typed-collection.ts index 28c188c..24f5f90 100644 --- a/src/typed-collection.ts +++ b/src/typed-collection.ts @@ -1,12 +1,13 @@ import { Collection } from './collection'; -import {Entity, EntityProperties} from './object-layer'; +import {Entity, EntityProperties} from './entity'; +import {LossyViewOne} from './lossy'; export class TypedCollection extends Collection { put(id: string | undefined, properties: T): Entity { return super.put(id, properties); } - get(id: string): Entity | undefined { + get(id: string): LossyViewOne | undefined { return super.get(id); } } diff --git a/src/types.ts b/src/types.ts index 816347e..5745496 100644 --- a/src/types.ts +++ b/src/types.ts @@ -41,7 +41,10 @@ export type DeltaFilter = (delta: Delta) => boolean; export type PropertyTypes = string | number | undefined; -export type Properties = {[key: string]: PropertyTypes}; +export type DomainEntityID = string; +export type PropertyID = string; + +export type Properties = {[key: PropertyID]: PropertyTypes}; export class PeerAddress { addr: string; diff --git a/src/util/md-files.ts b/src/util/md-files.ts index b6d2b82..ce96150 100644 --- a/src/util/md-files.ts +++ b/src/util/md-files.ts @@ -11,7 +11,10 @@ const docConverter = new Converter({ tasklists: true }); -export const htmlDocFromMarkdown = (md: string): string => docConverter.makeHtml(md); +export type Markdown = string; +export type Html = string; + +export const htmlDocFromMarkdown = (md: Markdown): Html => docConverter.makeHtml(md); type mdFileInfo = { name: string, @@ -27,7 +30,15 @@ export class MDFiles { readFile(name: string) { const md = readFileSync(join('./markdown', `${name}.md`)).toString(); - const html = htmlDocFromMarkdown(md); + let m = ""; + + // Add title and render the markdown + m += `# File: [${name}](/html/${name})\n\n---\n\n${md}`; + + // Add footer with the nav menu + m += `\n\n---\n\n${this.generateIndex()}`; + + const html = htmlDocFromMarkdown(m); this.files.set(name, {name, md, html}); } @@ -49,6 +60,15 @@ export class MDFiles { return Array.from(this.files.keys()); } + generateIndex(): Markdown { + let md = `# [Index](/html)\n\n`; + md += `[README](/html/README)\n\n`; + for (const name of this.list()) { + md += `- [${name}](/html/${name})\n`; + } + return htmlDocFromMarkdown(md); + } + readDir() { // Read list of markdown files from directory and // render each markdown file as html diff --git a/util/app.ts b/util/app.ts index fcbd69e..81895e6 100644 --- a/util/app.ts +++ b/util/app.ts @@ -25,7 +25,7 @@ export class App extends RhizomeNode { ...config, }); - const users = new TypedCollection("users"); + const users = new TypedCollection("user"); users.rhizomeConnect(this); const {httpAddr, httpPort} = this.config;