ladd
/
rhizome
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactored to use lossless/lossy view mechanisms for resolving get requests

This commit is contained in:
Ladd Hoffman 2024-12-26 15:59:03 -06:00
parent 13be73f821
commit 3f0b5bec4e
22 changed files with 481 additions and 302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
dist/
node_modules/
coverage/
*.swp
*.swo

195
README.md
View File

@ -16,6 +16,110 @@
If we express views and filter rules as JSON-Logic, we can easily include them in records.
# Development / Demo
## Setup
Install [`nvm`](https://nvm.sh)
Clone repo
```bash
git clone https://gitea.dgov.io/ladd/rhizome
```
Use `nvm` to install and activate the target nodejs version
```bash
nvm install
```
Install nodejs packages
```bash
npm install
```
## Build
Compile Typescript
```bash
npm run build
```
During development, it's useful to run the compiler in watch mode:
```bash
npm run build:watch
```
## Run tests
```bash
npm run test
```
## Run test coverage report
```bash
npm run test:coverage
```
## Run multiple live nodes locally as separate processes
To demonstrate the example application, you can open multiple terminals, and in each terminal execute something like the following:
```bash
export DEBUG="*,-express"
export RHIZOME_REQUEST_BIND_PORT=4000
export RHIZOME_PUBLISH_BIND_PORT=4001
export RHIZOME_SEED_PEERS='localhost:4002, localhost:4004'
export RHIZOME_HTTP_API_PORT=3000
export RHIZOME_PEER_ID=peer1
npm run example-app
```
```bash
export DEBUG="*,-express"
export RHIZOME_REQUEST_BIND_PORT=4002
export RHIZOME_PUBLISH_BIND_PORT=4003
export RHIZOME_SEED_PEERS='localhost:4000, localhost:4004'
export RHIZOME_HTTP_API_PORT=3001
export RHIZOME_PEER_ID=peer2
npm run example-app
```
```bash
export DEBUG="*,-express"
export RHIZOME_REQUEST_BIND_PORT=4004
export RHIZOME_PUBLISH_BIND_PORT=4005
export RHIZOME_SEED_PEERS='localhost:4000, localhost:4002'
export RHIZOME_HTTP_API_PORT=3002
export RHIZOME_PEER_ID=peer3
npm run example-app
```
In a separate terminal, you can use `curl` to interact with an instance.
`jq` is helpful for formatting the json responses.
Query the number of peers seen by a given node (including itself)
```bash
curl -s http://localhost:3000/peers/count | jq
```
Query the list of peers seen by a given node (including itself)
```bash
curl -s http://localhost:3000/peers | jq
```
Query the number of deltas ingested by this node
```bash
curl -s http://localhost:3000/deltas/count | jq
```
Query the list of deltas ingested by this node
```bash
curl -s http://localhost:3000/deltas | jq
```
# More About Concepts
## Clocks?
Do we want to involve a time synchronization protocol? e.g. ntpd
@ -78,94 +182,3 @@ Considerations imposed by Tinc would include
* IP addressing
* public key management
# Development / Demo
## Setup
Install [`nvm`](https://nvm.sh)
Clone repo
```bash
git clone https://gitea.dgov.io/ladd/rhizome
```
Use `nvm` to install and activate the target nodejs version
```bash
nvm install
```
Install nodejs packages
```bash
npm install
```
## Build
Compile Typescript
```bash
npm run build
```
During development, it's useful to run the compiler in watch mode:
```bash
npm run build:watch
```
## Run
To demonstrate the example application, you can open multiple terminals, and in each terminal execute something like the following:
```bash
export DEBUG="*,-express"
export RHIZOME_REQUEST_BIND_PORT=4000
export RHIZOME_PUBLISH_BIND_PORT=4001
export RHIZOME_SEED_PEERS='localhost:4002, localhost:4004'
export RHIZOME_HTTP_API_PORT=3000
export RHIZOME_PEER_ID=peer1
npm run example-app
```
```bash
export DEBUG="*,-express"
export RHIZOME_REQUEST_BIND_PORT=4002
export RHIZOME_PUBLISH_BIND_PORT=4003
export RHIZOME_SEED_PEERS='localhost:4000, localhost:4004'
export RHIZOME_HTTP_API_PORT=3001
export RHIZOME_PEER_ID=peer2
npm run example-app
```
```bash
export DEBUG="*,-express"
export RHIZOME_REQUEST_BIND_PORT=4004
export RHIZOME_PUBLISH_BIND_PORT=4005
export RHIZOME_SEED_PEERS='localhost:4000, localhost:4002'
export RHIZOME_HTTP_API_PORT=3002
export RHIZOME_PEER_ID=peer3
npm run example-app
```
In a separate terminal, you can use `curl` to interact with an instance.
`jq` is helpful for formatting the json responses.
Query the number of peers seen by a given node (including itself)
```bash
curl -s http://localhost:3000/peers/count | jq
```
Query the list of peers seen by a given node (including itself)
```bash
curl -s http://localhost:3000/peers | jq
```
Query the number of deltas ingested by this node
```bash
curl -s http://localhost:3000/deltas/count | jq
```
Query the list of deltas ingested by this node
```bash
curl -s http://localhost:3000/deltas | jq
```

17
__tests__/lossless.ts
View File

@ -135,7 +135,22 @@ describe('Lossless', () => {
return creator === 'A' && host === 'H';
};
expect(lossless.view(filter)).toEqual({
expect(lossless.view(undefined, filter)).toEqual({
ace: {
referencedAs: ["1"],
properties: {
value: [{
creator: 'A',
host: 'H',
pointers: [
{"1": "ace"},
]
}]
}
}
});
expect(lossless.view(["ace"], filter)).toEqual({
ace: {
referencedAs: ["1"],
properties: {

11
__tests__/lossy.ts
View File

@ -1,7 +1,6 @@
import Debug from "debug";
import {Lossless, LosslessViewMany} from "../src/lossless";
import {Lossy, firstValueFromLosslessViewOne, valueFromCollapsedDelta} from "../src/lossy";
const debug = Debug('test:lossy');
import {PointerTarget} from "../src/types";
describe('Lossy', () => {
describe('se a provided function to resolve entity views', () => {
@ -36,9 +35,9 @@ describe('Lossy', () => {
it('example summary', () => {
type Role = {
actor: string,
film: string,
role: string
actor: PointerTarget,
film: PointerTarget,
role: PointerTarget
};
type Summary = {
@ -47,14 +46,12 @@ describe('Lossy', () => {
const resolver = (losslessView: LosslessViewMany): Summary => {
const roles: Role[] = [];
debug('resolving roles');
for (const [id, ent] of Object.entries(losslessView)) {
if (ent.referencedAs.includes("role")) {
const {delta, value: actor} = firstValueFromLosslessViewOne(ent, "actor") ?? {};
if (!delta) continue; // TODO: panic
if (!actor) continue; // TODO: panic
const film = valueFromCollapsedDelta(delta, "film");
debug(`role ${id}`, {actor, film});
if (!film) continue; // TODO: panic
roles.push({
role: id,

18
__tests__/run/001-single-node.ts
View File

@ -17,9 +17,8 @@ describe('Run', () => {
await app.stop();
});
it('can put a new user', async () => {
const {httpAddr, httpPort} = app.config;
const res = await fetch(`http://${httpAddr}:${httpPort}/users`, {
it('can put a new user and fetch it', async () => {
const res = await fetch(`${app.apiUrl}/user`, {
method: 'PUT',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
@ -38,5 +37,18 @@ describe('Run', () => {
age: 263
}
});
await new Promise((resolve) => setTimeout(resolve, 100));
const res2 = await fetch(`${app.apiUrl}/user/peon-1`);
const data2 = await res2.json();
expect(data2).toMatchObject({
id: "peon-1",
properties: {
name: "Peon",
age: 263
}
});
});
});

4
__tests__/run/002-two-nodes.ts
View File

@ -28,7 +28,7 @@ describe('Run', () => {
debug('apps[0].apiUrl', apps[0].apiUrl);
debug('apps[1].apiUrl', apps[1].apiUrl);
const res = await fetch(`${apps[0].apiUrl}/users`, {
const res = await fetch(`${apps[0].apiUrl}/user`, {
method: 'PUT',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
@ -50,7 +50,7 @@ describe('Run', () => {
await new Promise((resolve) => setTimeout(resolve, 100));
const res2 = await fetch(`${apps[1].apiUrl}/users/peon-1`);
const res2 = await fetch(`${apps[1].apiUrl}/user/peon-1`);
const data2 = await res2.json();
debug('data2', data2);
expect(data2).toMatchObject({

38
markdown/coverage_report.md Normal file
View File

@ -0,0 +1,38 @@
> rhizome-node@1.0.0 test
> jest --coverage
PASS __tests__/lossy.ts
PASS __tests__/lossless.ts
PASS __tests__/peer-address.ts
PASS __tests__/run/002-two-nodes.ts
PASS __tests__/run/001-single-node.ts
----------------------|---------|----------|---------|---------|----------------------------------------------------
File | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s
----------------------|---------|----------|---------|---------|----------------------------------------------------
All files | 86.86 | 62.41 | 82.7 | 87.38 |
src | 87.94 | 68.06 | 82.45 | 88.09 |
collection.ts | 88.31 | 71.42 | 66.66 | 90.54 | 62-65,114-122,172
config.ts | 94.44 | 89.65 | 50 | 94.44 | 22
deltas.ts | 64.44 | 50 | 76.92 | 64.44 | 27-30,42-46,55-56,64-73
entity.ts | 100 | 100 | 100 | 100 |
http-api.ts | 59.7 | 13.04 | 38.88 | 59.7 | 32,37,44-60,66,79-80,85-92,100,121,129-130,145-151
lossless.ts | 98.27 | 91.66 | 100 | 100 | 96
lossy.ts | 100 | 85.71 | 100 | 100 | 38
node.ts | 100 | 100 | 100 | 100 |
peers.ts | 96.82 | 100 | 100 | 96.61 | 125-126
pub-sub.ts | 100 | 100 | 100 | 100 |
request-reply.ts | 95.65 | 0 | 100 | 95.34 | 46,59
typed-collection.ts | 100 | 100 | 100 | 100 |
types.ts | 100 | 100 | 100 | 100 |
src/util | 74.54 | 31.81 | 82.35 | 78 |
md-files.ts | 74.54 | 31.81 | 82.35 | 78 | 52-56,90-94,108-115
util | 100 | 100 | 100 | 100 |
app.ts | 100 | 100 | 100 | 100 |
----------------------|---------|----------|---------|---------|----------------------------------------------------
Test Suites: 5 passed, 5 total
Tests: 7 passed, 7 total
Snapshots: 0 total
Time: 3.709 s, estimated 5 s
Ran all test suites.

3
package.json
View File

@ -7,7 +7,8 @@
"build": "tsc",
"build:watch": "tsc --watch",
"lint": "eslint",
"test": "jest"
"test": "jest",
"test:coverage": "./scripts/coverage.sh"
},
"jest": {
"testEnvironment": "node",

16
scripts/coverage.sh Executable file
View File

@ -0,0 +1,16 @@
#!/bin/env bash
force=false
while [[ -n "$1" ]]; do
case "$1" in
-f | --force)
force=true
;;
esac
shift
done
dest="./markdown/coverage_report.md"
npm run test -- --coverage 2>&1 | tee "$dest"

132
src/collection.ts
View File

@ -3,44 +3,45 @@
// It should enable operations like removing a property removes the value from the entities in the collection
// It could then be further extended with e.g. table semantics like filter, sort, join
import Debug from 'debug';
import {randomUUID} from "node:crypto";
import EventEmitter from "node:events";
import {Entity} from "./entity";
import {Lossless, LosslessViewMany} from "./lossless";
import {firstValueFromLosslessViewOne, Lossy, LossyViewMany, LossyViewOne} from "./lossy";
import {RhizomeNode} from "./node";
import {Entity, EntityProperties, EntityPropertiesDeltaBuilder} from "./object-layer";
import {Delta} from "./types";
// type Property = {
// name: string,
// type: number | string;
// }
// class EntityType {
// name: string;
// properties?: Property[];
// constructor(name: string) {
// this.name = name;
// }
// }
const debug = Debug('collection');
export class Collection {
rhizomeNode?: RhizomeNode;
name: string;
entities = new Map<string, Entity>();
eventStream = new EventEmitter();
lossless = new Lossless(); // TODO: Really just need one global Lossless instance
constructor(name: string) {
this.name = name;
}
// Instead of trying to update our final view of the entity with every incoming delta,
// let's try this:
// - keep a lossless view (of everything)
// - build a lossy view when needed
// This approach is simplistic, but can then be optimized and enhanced.
rhizomeConnect(rhizomeNode: RhizomeNode) {
this.rhizomeNode = rhizomeNode;
rhizomeNode.deltaStream.subscribeDeltas((delta: Delta) => {
// TODO: Make sure this is the kind of delta we're looking for
this.applyDelta(delta);
debug(`collection ${this.name} received delta ${JSON.stringify(delta)}`);
this.lossless.ingestDelta(delta);
});
rhizomeNode.httpApi.serveCollection(this);
debug(`connected ${this.name} to rhizome`);
}
// Applies the javascript rules for updating object values,
@ -55,7 +56,6 @@ export class Collection {
entity.id = entityId;
eventType = 'create';
}
const deltaBulider = new EntityPropertiesDeltaBuilder(this.rhizomeNode!, entityId);
if (!properties) {
// Let's interpret this as entity deletion
@ -74,69 +74,50 @@ export class Collection {
}
if (local && changed) {
// If this is a change, let's generate a delta
deltaBulider.add(key, value);
if (!this.rhizomeNode) throw new Error(`${this.name} collection not connected to rhizome`);
const delta: Delta = {
creator: this.rhizomeNode.config.creator,
host: this.rhizomeNode.config.peerId,
pointers: [{
localContext: this.name,
target: entityId,
targetContext: key
}, {
localContext: key,
target: value
}]
};
deltas?.push(delta);
// We append to the array the caller may provide
// We can update this count as we receive network confirmation for deltas
entity.ahead += 1;
}
anyChanged = anyChanged || changed;
});
// We've noted that we may be ahead of the server, let's update our
// local image of this entity.
//* In principle, this system can recreate past or alternative states.
//* At worst, by replaying all the deltas up to a particular point.
//* Some sort of checkpointing strategy would probably be helpful.
//* Furthermore, if we can implement reversible transformations,
//* it would then be efficient to calculate the state of the system with
//* specific deltas removed. We could use it to extract a measurement
//* of the effects of some deltas' inclusion or exclusion, the
//* evaluation of which may lend evidence to some possible arguments.
this.entities.set(entityId, entity);
if (anyChanged) {
deltas?.push(deltaBulider.delta);
eventType = eventType || 'update';
}
}
if (eventType) {
// TODO: Reconcile this with lossy view approach
this.eventStream.emit(eventType, entity);
}
return entity;
}
// We can update our local image of the entity, but we should annotate it
// to indicate that we have not yet received any confirmation of this delta
// having been propagated.
// Later when we receive deltas regarding this entity we can detect when
// we have received back an image that matches our target.
// So we need a function to generate one or more deltas for each call to put/
// maybe we stage them and wait for a call to commit() that initiates the
// assembly and transmission of one or more deltas
applyDelta(delta: Delta) {
// TODO: handle delta representing entity deletion
const idPtr = delta.pointers.find(({localContext}) => localContext === 'id');
if (!idPtr) {
console.error('encountered delta with no entity id', delta);
return;
}
const properties: EntityProperties = {};
delta.pointers.filter(({localContext}) => localContext !== 'id')
.forEach(({localContext: key, target: value}) => {
properties[key] = value;
}, {});
const entityId = idPtr.target as string;
// TODO: Handle the scenario where this update has been superceded by a newer one locally
this.updateEntity(entityId, properties);
}
onCreate(cb: (entity: Entity) => void) {
// TODO: Reconcile this with lossy view approach
this.eventStream.on('create', (entity: Entity) => {
cb(entity);
});
}
onUpdate(cb: (entity: Entity) => void) {
// TODO: Reconcile this with lossy view approach
this.eventStream.on('update', (entity: Entity) => {
cb(entity);
});
@ -145,25 +126,46 @@ export class Collection {
put(entityId: string | undefined, properties: object): Entity {
const deltas: Delta[] = [];
const entity = this.updateEntity(entityId, properties, true, deltas);
debug(`put ${entityId} generated deltas:`, JSON.stringify(deltas));
// updateEntity may have generated some deltas for us to store and publish
deltas.forEach(async (delta: Delta) => {
// record this delta just as if we had received it from a peer
delta.receivedFrom = this.rhizomeNode!.myRequestAddr;
this.rhizomeNode!.deltaStream.deltasAccepted.push(delta);
// publish the delta to our subscribed peers
await this.rhizomeNode!.deltaStream.publishDelta(delta);
debug(`published delta ${JSON.stringify(delta)}`);
// ingest the delta as though we had received it from a peer
this.lossless.ingestDelta(delta);
});
return entity;
}
del(entityId: string) {
const deltas: Delta[] = [];
this.updateEntity(entityId, undefined, true, deltas);
deltas.forEach(async (delta: Delta) => {
this.rhizomeNode!.deltaStream.deltasAccepted.push(delta);
await this.rhizomeNode!.deltaStream.publishDelta(delta);
});
}
get(id: string): Entity | undefined {
return this.entities.get(id);
get(id: string): LossyViewOne | undefined {
// Now with lossy view approach, instead of just returning what we already have,
// let's compute our view now.
// return this.entities.get(id);
const lossy = new Lossy(this.lossless);
const resolver = (losslessView: LosslessViewMany) => {
const lossyView: LossyViewMany = {};
debug('lossless view', JSON.stringify(losslessView));
for (const [id, ent] of Object.entries(losslessView)) {
lossyView[id] = {id, properties: {}};
for (const key of Object.keys(ent.properties)) {
const {value} = firstValueFromLosslessViewOne(ent, key) || {};
debug(`[ ${key} ] = ${value}`);
lossyView[id].properties[key] = value;
}
}
return lossyView;
};
const res = lossy.resolve(resolver, [id]) as LossyViewMany;;
return res[id];
}
getIds(): string[] {

24
src/context.ts
View File

@ -0,0 +1,24 @@
// A delta represents an assertion from a given (perspective/POV/context).
// So we want it to be fluent to express these in the local context,
// and propagated as deltas in a configurable manner; i.e. configurable batches or immediate
// import {Delta} from './types';
export class Entity {
}
export class Context {
}
export class Assertion {
}
export class View {
}
export class User {
}
export function example() {
}

25
src/entity.ts Normal file
View File

@ -0,0 +1,25 @@
// The goal here is to provide a translation for
// entities and their properties
// to and from (sequences of) deltas.
// How can our caller define the entities and their properties?
// - As typescript types?
// - As typescript interfaces?
// - As typescript classes?
import {PropertyTypes} from "./types";
export type EntityProperties = {
[key: string]: PropertyTypes;
};
export class Entity {
id: string;
properties: EntityProperties = {};
ahead = 0;
constructor(id: string) {
this.id = id;
}
}

2
src/example-app.ts
View File

@ -1,6 +1,6 @@
import Debug from 'debug';
import {RhizomeNode} from "./node";
import {Entity} from "./object-layer";
import {Entity} from "./entity";
import {TypedCollection} from "./typed-collection";
const debug = Debug('example-app');

76
src/http-api.ts
View File

@ -24,42 +24,7 @@ export class HttpApi {
}
start() {
// Scan and watch for markdown files
this.mdFiles.readDir();
this.mdFiles.readReadme();
this.mdFiles.watchDir();
this.mdFiles.watchReadme();
// Serve README
this.router.get('/html/README', (_req: express.Request, res: express.Response) => {
const html = this.mdFiles.getReadmeHTML();
res.setHeader('content-type', 'text/html').send(html);
});
// Serve markdown files as html
this.router.get('/html/:name', (req: express.Request, res: express.Response) => {
let html = this.mdFiles.getHtml(req.params.name);
if (!html) {
res.status(404);
html = htmlDocFromMarkdown('# 404\n\n## [Index](/html)');
}
res.setHeader('content-type', 'text/html');
res.send(html);
});
// Serve index
{
let md = `# Files\n\n`;
md += `[README](/html/README)\n\n`;
for (const name of this.mdFiles.list()) {
md += `- [${name}](./${name})\n`;
}
const html = htmlDocFromMarkdown(md);
this.router.get('/html', (_req: express.Request, res: express.Response) => {
res.setHeader('content-type', 'text/html').send(html);
});
}
// --------------- deltas ----------------
// Serve list of all deltas accepted
// TODO: This won't scale well
@ -72,6 +37,8 @@ export class HttpApi {
res.json(this.rhizomeNode.deltaStream.deltasAccepted.length);
});
// --------------- peers ----------------
// Get the list of peers seen by this node (including itself)
this.router.get("/peers", (_req: express.Request, res: express.Response) => {
res.json(this.rhizomeNode.peers.peers.map(({reqAddr, publishAddr, isSelf, isSeedPeer}) => {
@ -99,6 +66,43 @@ export class HttpApi {
res.json(this.rhizomeNode.peers.peers.length);
});
// ----------------- html ---------------------
// Scan and watch for markdown files
this.mdFiles.readDir();
this.mdFiles.readReadme();
this.mdFiles.watchDir();
this.mdFiles.watchReadme();
// Serve README
this.router.get('/html/README', (_req: express.Request, res: express.Response) => {
const html = this.mdFiles.getReadmeHTML();
res.setHeader('content-type', 'text/html').send(html);
});
// Serve markdown files as html
this.router.get('/html/:name', (req: express.Request, res: express.Response) => {
const {name} = req.params;
let html = this.mdFiles.getHtml(name);
if (!html) {
res.status(404);
html = htmlDocFromMarkdown(`# 404 Not Found: ${name}\n\n ## [Index](/html)`);
}
res.setHeader('content-type', 'text/html');
res.send(html);
});
// Serve index
{
const html = this.mdFiles.generateIndex();
this.router.get('/html', (_req: express.Request, res: express.Response) => {
res.setHeader('content-type', 'text/html').send(html);
});
}
// ------------------- server ---------------------
const {httpAddr, httpPort} = this.rhizomeNode.config;
this.server = this.app.listen({
port: httpPort,

90
src/lossless.ts
View File

@ -1,39 +1,32 @@
// Deltas target entities.
// We can maintain a record of all the targeted entities, and the deltas that targeted them
import {Delta, DeltaFilter, PropertyTypes} from "./types";
type DomainEntityID = string;
type PropertyID = string;
import Debug from 'debug';
import {Delta, DeltaFilter, DomainEntityID, Properties, PropertyID, PropertyTypes} from "./types";
const debug = Debug('lossless');
export type CollapsedPointer = {[key: string]: PropertyTypes};
export type CollapsedDelta = Omit<Delta, 'pointers'> & {
pointers: CollapsedPointer[];
};
export type LosslessViewOne = {
referencedAs: string[];
properties: {
[key: PropertyID]: CollapsedDelta[]
}
};
export type LosslessViewMany = {
[key: DomainEntityID]: LosslessViewOne;
};
class DomainEntityMap extends Map<DomainEntityID, DomainEntity> {};
class DomainEntityProperty {
id: PropertyID;
deltas = new Set<Delta>();
constructor(id: PropertyID) {
this.id = id;
}
}
class DomainEntity {
id: DomainEntityID;
properties = new Map<PropertyID, DomainEntityProperty>();
properties = new Map<PropertyID, Set<Delta>>();
constructor(id: DomainEntityID) {
this.id = id;
@ -44,15 +37,29 @@ class DomainEntity {
.filter(({target}) => target === this.id)
.map(({targetContext}) => targetContext)
.filter((targetContext) => typeof targetContext === 'string');
for (const targetContext of targetContexts) {
let property = this.properties.get(targetContext);
if (!property) {
property = new DomainEntityProperty(targetContext);
this.properties.set(targetContext, property);
let propertyDeltas = this.properties.get(targetContext);
if (!propertyDeltas) {
propertyDeltas = new Set<Delta>();
this.properties.set(targetContext, propertyDeltas);
}
property.deltas.add(delta);
debug(`adding delta for entity ${this.id}`);
propertyDeltas.add(delta);
}
}
toJSON() {
const properties: {[key: PropertyID]: number} = {};
for (const [key, deltas] of this.properties.entries()) {
properties[key] = deltas.size;
}
return {
id: this.id,
properties
};
}
}
export class Lossless {
@ -63,47 +70,70 @@ export class Lossless {
.filter(({targetContext}) => !!targetContext)
.map(({target}) => target)
.filter((target) => typeof target === 'string')
for (const target of targets) {
let ent = this.domainEntities.get(target);
if (!ent) {
ent = new DomainEntity(target);
this.domainEntities.set(target, ent);
}
debug('before add, domain entity:', JSON.stringify(ent));
ent.addDelta(delta);
debug('after add, domain entity:', JSON.stringify(ent));
}
}
//TODO: json logic -- view(deltaFilter?: FilterExpr) {
view(deltaFilter?: DeltaFilter): LosslessViewMany {
view(entityIds?: DomainEntityID[], deltaFilter?: DeltaFilter): LosslessViewMany {
const view: LosslessViewMany = {};
for (const ent of this.domainEntities.values()) {
entityIds = entityIds ?? Array.from(this.domainEntities.keys());
for (const id of entityIds) {
const ent = this.domainEntities.get(id);
if (!ent) continue;
debug(`domain entity ${id}`, JSON.stringify(ent));
const referencedAs = new Set<string>();
view[ent.id] = {
referencedAs: [],
properties: {}
};
for (const prop of ent.properties.values()) {
view[ent.id].properties[prop.id] = view[ent.id].properties[prop.id] || [];
for (const delta of prop.deltas) {
const properties: {
[key: PropertyID]: CollapsedDelta[]
} = {};
for (const [key, deltas] of ent.properties.entries()) {
properties[key] = properties[key] || [];
for (const delta of deltas) {
if (deltaFilter) {
const include = deltaFilter(delta);
if (!include) continue;
}
const pointers: CollapsedPointer[] = [];
for (const {localContext, target} of delta.pointers) {
pointers.push({[localContext]: target});
if (target === ent.id) {
referencedAs.add(localContext);
}
}
const collapsedDelta: CollapsedDelta = {
...delta,
pointers
};
view[ent.id].referencedAs = Array.from(referencedAs.values());
view[ent.id].properties[prop.id].push(collapsedDelta);
properties[key].push(collapsedDelta);
}
}
view[ent.id] = {
referencedAs: Array.from(referencedAs.values()),
properties
};
}
return view;
}

34
src/lossy.ts
View File

@ -5,24 +5,36 @@
// We can achieve this via functional expression, encoded as JSON-Logic.
// Fields in the output can be described as transformations
import Debug from 'debug';
import {CollapsedDelta, Lossless, LosslessViewMany, LosslessViewOne} from "./lossless";
import {DeltaFilter} from "./types";
import {DeltaFilter, DomainEntityID, Properties} from "./types";
const debug = Debug('lossy');
type Resolver = (losslessView: LosslessViewMany) => unknown;
export type LossyViewOne = {
id: DomainEntityID;
properties: Properties;
};
export type LossyViewMany = {
[key: DomainEntityID]: LossyViewOne;
};
type Resolver = (losslessView: LosslessViewMany) => LossyViewMany | unknown;
// Extract a particular value from a delta's pointers
export function valueFromCollapsedDelta(delta: CollapsedDelta, key: string): string | undefined {
const pointers = delta.pointers;
for (const pointer of pointers || []) {
const [[k, value]] = Object.entries(pointer);
if (k === key && typeof value === "string") {
return value;
export function valueFromCollapsedDelta(delta: CollapsedDelta, key: string): string | number | undefined {
for (const pointer of delta.pointers) {
for (const [k, value] of Object.entries(pointer)) {
if (k === key && (typeof value === "string" || typeof value === "number")) {
return value;
}
}
}
}
// Example function for resolving a value for an entity by taking the first value we find
export function firstValueFromLosslessViewOne(ent: LosslessViewOne, key: string): {delta: CollapsedDelta, value: string} | undefined {
export function firstValueFromLosslessViewOne(ent: LosslessViewOne, key: string): {delta: CollapsedDelta, value: string | number} | undefined {
debug(`trying to get value for ${key} from ${JSON.stringify(ent.properties[key])}`);
for (const delta of ent.properties[key] || []) {
const value = valueFromCollapsedDelta(delta, key);
if (value) return {delta, value};
@ -36,8 +48,8 @@ export class Lossy {
this.lossless = lossless;
}
resolve(fn: Resolver, deltaFilter?: DeltaFilter) {
return fn(this.lossless.view(deltaFilter));
resolve(fn: Resolver, entityIds?: DomainEntityID[], deltaFilter?: DeltaFilter) {
return fn(this.lossless.view(entityIds, deltaFilter));
}
}

13
src/node.ts
View File

@ -67,15 +67,28 @@ export class RhizomeNode {
}
async start() {
// Start ZeroMQ publish and reply sockets
this.pubSub.start();
this.requestReply.start();
// Start HTTP server
if (this.config.httpEnable) {
this.httpApi.start();
}
// Wait a short time for sockets to initialize
await new Promise((resolve) => setTimeout(resolve, 500));
// Subscribe to seed peers
this.peers.subscribeToSeeds();
// Wait a short time for sockets to initialize
await new Promise((resolve) => setTimeout(resolve, 500));
// Ask all peers for all deltas
this.peers.askAllPeersForDeltas();
// Wait to receive all deltas
await new Promise((resolve) => setTimeout(resolve, 1000));
}

48
src/object-layer.ts
View File

@ -1,48 +0,0 @@
// The goal here is to provide a translation for
// entities and their properties
// to and from (sequences of) deltas.
// How can our caller define the entities and their properties?
// - As typescript types?
// - As typescript interfaces?
// - As typescript classes?
import {RhizomeNode} from "./node";
import {Delta, PropertyTypes} from "./types";
export type EntityProperties = {
[key: string]: PropertyTypes;
};
export class Entity {
id: string;
properties: EntityProperties = {};
ahead = 0;
constructor(id: string) {
this.id = id;
}
}
// TODO: Use leveldb for storing view snapshots
export class EntityPropertiesDeltaBuilder {
delta: Delta;
constructor(rhizomeNode: RhizomeNode, entityId: string) {
this.delta = {
creator: rhizomeNode.config.creator,
host: rhizomeNode.config.peerId,
pointers: [{
localContext: 'id',
target: entityId,
targetContext: 'properties'
}]
};
}
add(localContext: string, target: PropertyTypes) {
this.delta.pointers.push({localContext, target});
}
}

5
src/typed-collection.ts
View File

@ -1,12 +1,13 @@
import { Collection } from './collection';
import {Entity, EntityProperties} from './object-layer';
import {Entity, EntityProperties} from './entity';
import {LossyViewOne} from './lossy';
export class TypedCollection<T extends EntityProperties> extends Collection {
put(id: string | undefined, properties: T): Entity {
return super.put(id, properties);
}
get(id: string): Entity | undefined {
get(id: string): LossyViewOne | undefined {
return super.get(id);
}
}

5
src/types.ts
View File

@ -41,7 +41,10 @@ export type DeltaFilter = (delta: Delta) => boolean;
export type PropertyTypes = string | number | undefined;
export type Properties = {[key: string]: PropertyTypes};
export type DomainEntityID = string;
export type PropertyID = string;
export type Properties = {[key: PropertyID]: PropertyTypes};
export class PeerAddress {
addr: string;

24
src/util/md-files.ts
View File

@ -11,7 +11,10 @@ const docConverter = new Converter({
tasklists: true
});
export const htmlDocFromMarkdown = (md: string): string => docConverter.makeHtml(md);
export type Markdown = string;
export type Html = string;
export const htmlDocFromMarkdown = (md: Markdown): Html => docConverter.makeHtml(md);
type mdFileInfo = {
name: string,
@ -27,7 +30,15 @@ export class MDFiles {
readFile(name: string) {
const md = readFileSync(join('./markdown', `${name}.md`)).toString();
const html = htmlDocFromMarkdown(md);
let m = "";
// Add title and render the markdown
m += `# File: [${name}](/html/${name})\n\n---\n\n${md}`;
// Add footer with the nav menu
m += `\n\n---\n\n${this.generateIndex()}`;
const html = htmlDocFromMarkdown(m);
this.files.set(name, {name, md, html});
}
@ -49,6 +60,15 @@ export class MDFiles {
return Array.from(this.files.keys());
}
generateIndex(): Markdown {
let md = `# [Index](/html)\n\n`;
md += `[README](/html/README)\n\n`;
for (const name of this.list()) {
md += `- [${name}](/html/${name})\n`;
}
return htmlDocFromMarkdown(md);
}
readDir() {
// Read list of markdown files from directory and
// render each markdown file as html

2
util/app.ts
View File

@ -25,7 +25,7 @@ export class App extends RhizomeNode {
...config,
});
const users = new TypedCollection<User>("users");
const users = new TypedCollection<User>("user");
users.rhizomeConnect(this);
const {httpAddr, httpPort} = this.config;