diff --git a/__tests__/run/001-single-node.ts b/__tests__/run/001-single-node.ts index 03dcc48..2398f25 100644 --- a/__tests__/run/001-single-node.ts +++ b/__tests__/run/001-single-node.ts @@ -1,4 +1,4 @@ -import {App} from "../../util/app"; +import {App} from "../util/app"; describe('Run', () => { let app: App; @@ -23,16 +23,18 @@ describe('Run', () => { method: 'PUT', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ - name: "Peon", id: "peon-1", - age: 263 + properties: { + name: "Peon", + age: 263 + } }) }); const data = await res.json(); expect(data).toMatchObject({ + id: "peon-1", properties: { name: "Peon", - id: "peon-1", age: 263 } }); diff --git a/__tests__/run/002-two-nodes.ts b/__tests__/run/002-two-nodes.ts index 05848d8..832c72c 100644 --- a/__tests__/run/002-two-nodes.ts +++ b/__tests__/run/002-two-nodes.ts @@ -1,4 +1,6 @@ -import {App} from '../../util/app'; +import Debug from 'debug'; +import {App} from '../util/app'; +const debug = Debug('test:two'); describe('Run', () => { const apps: App[] = []; @@ -6,10 +8,14 @@ describe('Run', () => { beforeAll(async () => { apps[0] = new App({ httpEnable: true, + peerId: 'app0', }); apps[1] = new App({ httpEnable: true, + peerId: 'app1', }); + apps[0].config.seedPeers.push(apps[1].myRequestAddr); + apps[1].config.seedPeers.push(apps[0].myRequestAddr); await Promise.all(apps.map((app) => app.start())); }); @@ -19,40 +25,38 @@ describe('Run', () => { }); it('can create a record on node 0 and read it on node 1', async () => { + debug('apps[0].apiUrl', apps[0].apiUrl); + debug('apps[1].apiUrl', apps[1].apiUrl); + const res = await fetch(`${apps[0].apiUrl}/users`, { method: 'PUT', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ - name: "Peon", id: "peon-1", - age: 263 + properties: { + name: "Peon", + age: 263 + } }) }); const data = await res.json(); expect(data).toMatchObject({ + id: "peon-1", properties: { name: "Peon", - id: "peon-1", age: 263 } }); await new Promise((resolve) => setTimeout(resolve, 100)); - const res2 = await fetch(`${apps[0].apiUrl}/users`, { - method: 'PUT', - headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({ - name: "Peon", - id: "peon-1", - age: 263 - }) - }); + const res2 = await fetch(`${apps[1].apiUrl}/users/peon-1`); const data2 = await res2.json(); + debug('data2', data2); expect(data2).toMatchObject({ + id: "peon-1", properties: { name: "Peon", - id: "peon-1", age: 263 } }); diff --git a/util/app.ts b/__tests__/util/app.ts similarity index 74% rename from util/app.ts rename to __tests__/util/app.ts index 51c583c..2084916 100644 --- a/util/app.ts +++ b/__tests__/util/app.ts @@ -1,5 +1,5 @@ -import {RhizomeNode, RhizomeNodeConfig} from "../src/node"; -import {TypedCollection} from "../src/typed-collection"; +import {RhizomeNode, RhizomeNodeConfig} from "../../src/node"; +import {TypedCollection} from "../../src/typed-collection"; type User = { id?: string; @@ -28,7 +28,8 @@ export class App extends RhizomeNode { const users = new TypedCollection("users"); users.rhizomeConnect(this); - this.apiUrl = `http://${this.config.httpAddr}:${this.config.httpPort}`; + const {httpAddr, httpPort} = this.config; + this.apiUrl = `http://${httpAddr}:${httpPort}`; } } diff --git a/package.json b/package.json index a5b6d8e..278fbd8 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,8 @@ "level": "^9.0.0", "object-hash": "^3.0.0", "showdown": "^2.1.0", - "zeromq": "^6.1.2" + "zeromq": "^6.1.2", + "util": "./util/" }, "devDependencies": { "@eslint/js": "^9.17.0", diff --git a/src/collection.ts b/src/collection.ts index 8cb3ca5..a99e7db 100644 --- a/src/collection.ts +++ b/src/collection.ts @@ -66,6 +66,7 @@ export class Collection { } else { let anyChanged = false; Object.entries(properties).forEach(([key, value]) => { + if (key === 'id') return; let changed = false; if (entity.properties && entity.properties[key] !== value) { entity.properties[key] = value; diff --git a/src/http-api.ts b/src/http-api.ts index 76417df..29bfb98 100644 --- a/src/http-api.ts +++ b/src/http-api.ts @@ -1,124 +1,26 @@ import Debug from "debug"; -import express from "express"; -import {FSWatcher} from "fs"; -import {readdirSync, readFileSync, watch} from "fs"; +import express, {Express, Router} from "express"; import {Server} from "http"; -import path, {join} from "path"; -import {Converter} from "showdown"; import {Collection} from "./collection"; import {RhizomeNode} from "./node"; import {Delta} from "./types"; +import {htmlDocFromMarkdown, MDFiles} from "./util/md-files"; const debug = Debug('http-api'); -const docConverter = new Converter({ - completeHTMLDocument: true, - // simpleLineBreaks: true, - tables: true, - tasklists: true -}); - -const htmlDocFromMarkdown = (md: string): string => docConverter.makeHtml(md); - -type mdFileInfo = { - name: string, - md: string, - html: string -}; - -class MDFiles { - files = new Map(); - readme?: mdFileInfo; - dirWatcher?: FSWatcher; - readmeWatcher?: FSWatcher; - - readFile(name: string) { - const md = readFileSync(join('./markdown', `${name}.md`)).toString(); - const html = htmlDocFromMarkdown(md); - this.files.set(name, {name, md, html}); - } - - readReadme() { - const md = readFileSync('./README.md').toString(); - const html = htmlDocFromMarkdown(md); - this.readme = {name: 'README', md, html}; - } - - getReadmeHTML() { - return this.readme?.html; - } - - getHtml(name: string): string | undefined { - return this.files.get(name)?.html; - } - - list(): string[] { - return Array.from(this.files.keys()); - } - - readDir() { - // Read list of markdown files from directory and - // render each markdown file as html - readdirSync('./markdown/') - .filter((f) => f.endsWith('.md')) - .map((name) => path.parse(name).name) - .forEach((name) => this.readFile(name)); - } - - watchDir() { - this.dirWatcher = watch('./markdown', null, (eventType, filename) => { - if (!filename) return; - if (!filename.endsWith(".md")) return; - - const name = path.parse(filename).name; - - switch (eventType) { - case 'rename': { - debug(`file ${name} renamed`); - // Remove it from memory and re-scan everything - this.files.delete(name); - this.readDir(); - break; - } - case 'change': { - debug(`file ${name} changed`); - // Re-read this file - this.readFile(name) - break; - } - } - }); - } - - watchReadme() { - this.readmeWatcher = watch('./README.md', null, (eventType, filename) => { - if (!filename) return; - - switch (eventType) { - case 'change': { - debug(`README file changed`); - // Re-read this file - this.readReadme() - break; - } - } - }); - } - - close() { - this.dirWatcher?.close(); - this.readmeWatcher?.close(); - } -} - export class HttpApi { rhizomeNode: RhizomeNode; - app = express(); + app: Express; + router: Router; mdFiles = new MDFiles(); server?: Server; constructor(rhizomeNode: RhizomeNode) { this.rhizomeNode = rhizomeNode; + this.app = express(); + this.router = Router(); + this.app.use(express.json()); + this.app.use(this.router); } start() { @@ -129,13 +31,13 @@ export class HttpApi { this.mdFiles.watchReadme(); // Serve README - this.app.get('/html/README', (_req: express.Request, res: express.Response) => { + this.router.get('/html/README', (_req: express.Request, res: express.Response) => { const html = this.mdFiles.getReadmeHTML(); res.setHeader('content-type', 'text/html').send(html); }); // Serve markdown files as html - this.app.get('/html/:name', (req: express.Request, res: express.Response) => { + this.router.get('/html/:name', (req: express.Request, res: express.Response) => { let html = this.mdFiles.getHtml(req.params.name); if (!html) { res.status(404); @@ -154,24 +56,24 @@ export class HttpApi { } const html = htmlDocFromMarkdown(md); - this.app.get('/html', (_req: express.Request, res: express.Response) => { + this.router.get('/html', (_req: express.Request, res: express.Response) => { res.setHeader('content-type', 'text/html').send(html); }); } // Serve list of all deltas accepted // TODO: This won't scale well - this.app.get("/deltas", (_req: express.Request, res: express.Response) => { + this.router.get("/deltas", (_req: express.Request, res: express.Response) => { res.json(this.rhizomeNode.deltaStream.deltasAccepted); }); // Get the number of deltas ingested by this node - this.app.get("/deltas/count", (_req: express.Request, res: express.Response) => { + this.router.get("/deltas/count", (_req: express.Request, res: express.Response) => { res.json(this.rhizomeNode.deltaStream.deltasAccepted.length); }); // Get the list of peers seen by this node (including itself) - this.app.get("/peers", (_req: express.Request, res: express.Response) => { + this.router.get("/peers", (_req: express.Request, res: express.Response) => { res.json(this.rhizomeNode.peers.peers.map(({reqAddr, publishAddr, isSelf, isSeedPeer}) => { const deltasAcceptedCount = this.rhizomeNode.deltaStream.deltasAccepted .filter((delta: Delta) => { @@ -193,12 +95,16 @@ export class HttpApi { }); // Get the number of peers seen by this node (including itself) - this.app.get("/peers/count", (_req: express.Request, res: express.Response) => { + this.router.get("/peers/count", (_req: express.Request, res: express.Response) => { res.json(this.rhizomeNode.peers.peers.length); }); const {httpAddr, httpPort} = this.rhizomeNode.config; - this.server = this.app.listen(httpPort, httpAddr, () => { + this.server = this.app.listen({ + port: httpPort, + host: httpAddr, + exclusive: true + }, () => { debug(`HTTP API bound to ${httpAddr}:${httpPort}`); }); } @@ -207,27 +113,31 @@ export class HttpApi { const {name} = collection; // Get the ID of all domain entities - this.app.get(`/${name}/ids`, (_req: express.Request, res: express.Response) => { + this.router.get(`/${name}/ids`, (_req: express.Request, res: express.Response) => { res.json({ids: collection.getIds()}); }); // Get a single domain entity by ID - this.app.get(`/${name}/:id`, (req: express.Request, res: express.Response) => { + this.router.get(`/${name}/:id`, (req: express.Request, res: express.Response) => { const {params: {id}} = req; const ent = collection.get(id); + if (!ent) { + res.status(404).send({error: "Not Found"}); + return; + } res.json(ent); }); // Add a new domain entity // TODO: schema validation - this.app.put(`/${name}`, (req: express.Request, res: express.Response) => { - const {body: properties} = req; - const ent = collection.put(properties.id, properties); + this.router.put(`/${name}`, (req: express.Request, res: express.Response) => { + const {body: {id, properties}} = req; + const ent = collection.put(id, properties); res.json(ent); }); // Update a domain entity - this.app.put(`/${name}/:id`, (req: express.Request, res: express.Response) => { + this.router.put(`/${name}/:id`, (req: express.Request, res: express.Response) => { const {body: properties, params: {id}} = req; if (properties.id && properties.id !== id) { res.status(400).json({error: "ID Mismatch", param: id, property: properties.id}); diff --git a/src/peers.ts b/src/peers.ts index 82f0331..0429431 100644 --- a/src/peers.ts +++ b/src/peers.ts @@ -7,7 +7,7 @@ import {PeerRequest, RequestSocket, ResponseSocket} from "./request-reply"; import {Delta, PeerAddress} from "./types"; const debug = Debug('peers'); -export enum PeerMethods { +export enum RequestMethods { GetPublishAddress, AskForDeltas } @@ -28,7 +28,7 @@ class Peer { this.isSeedPeer = !!SEED_PEERS.find((seedPeer) => reqAddr.isEqual(seedPeer)); } - async request(method: PeerMethods): Promise { + async request(method: RequestMethods): Promise { if (!this.reqSock) { this.reqSock = new RequestSocket(this.reqAddr); } @@ -38,7 +38,7 @@ class Peer { async subscribeDeltas() { if (!this.publishAddr) { debug(`requesting publish addr from peer ${this.reqAddr.toAddrString()}`); - const res = await this.request(PeerMethods.GetPublishAddress); + const res = await this.request(RequestMethods.GetPublishAddress); this.publishAddr = PeerAddress.fromString(res.toString()); debug(`received publish addr ${this.publishAddr.toAddrString()} from peer ${this.reqAddr.toAddrString()}`); } @@ -63,7 +63,7 @@ class Peer { // Third pass should find a way to reduce the number of deltas transmitted. // TODO: requestTimeout - const res = await this.request(PeerMethods.AskForDeltas); + const res = await this.request(RequestMethods.AskForDeltas); const deltas = JSON.parse(res.toString()); return deltas; } @@ -82,12 +82,12 @@ export class Peers { this.rhizomeNode.requestReply.registerRequestHandler(async (req: PeerRequest, res: ResponseSocket) => { debug('inspecting peer request'); switch (req.method) { - case PeerMethods.GetPublishAddress: { + case RequestMethods.GetPublishAddress: { debug('it\'s a request for our publish address'); await res.send(this.rhizomeNode.myPublishAddr.toAddrString()); break; } - case PeerMethods.AskForDeltas: { + case RequestMethods.AskForDeltas: { debug('it\'s a request for deltas'); // TODO: stream these rather than // trying to write them all in one message diff --git a/src/request-reply.ts b/src/request-reply.ts index 029e912..6eb1ff6 100644 --- a/src/request-reply.ts +++ b/src/request-reply.ts @@ -1,13 +1,13 @@ import {Request, Reply, Message} from 'zeromq'; import {EventEmitter} from 'node:events'; -import {PeerMethods} from './peers'; +import {RequestMethods} from './peers'; import Debug from 'debug'; import {RhizomeNode} from './node'; import {PeerAddress} from './types'; const debug = Debug('request-reply'); export type PeerRequest = { - method: PeerMethods; + method: RequestMethods; }; export type RequestHandler = (req: PeerRequest, res: ResponseSocket) => void; @@ -22,7 +22,7 @@ export class RequestSocket { debug(`Request socket connecting to ${addrStr}`); } - async request(method: PeerMethods): Promise { + async request(method: RequestMethods): Promise { const req: PeerRequest = { method }; diff --git a/src/util/md-files.ts b/src/util/md-files.ts new file mode 100644 index 0000000..b6d2b82 --- /dev/null +++ b/src/util/md-files.ts @@ -0,0 +1,105 @@ +import Debug from "debug"; +import {FSWatcher, readdirSync, readFileSync, watch} from "fs"; +import path, {join} from "path"; +import {Converter} from "showdown"; +const debug = Debug('md-files'); + +const docConverter = new Converter({ + completeHTMLDocument: true, + // simpleLineBreaks: true, + tables: true, + tasklists: true +}); + +export const htmlDocFromMarkdown = (md: string): string => docConverter.makeHtml(md); + +type mdFileInfo = { + name: string, + md: string, + html: string +}; + +export class MDFiles { + files = new Map(); + readme?: mdFileInfo; + dirWatcher?: FSWatcher; + readmeWatcher?: FSWatcher; + + readFile(name: string) { + const md = readFileSync(join('./markdown', `${name}.md`)).toString(); + const html = htmlDocFromMarkdown(md); + this.files.set(name, {name, md, html}); + } + + readReadme() { + const md = readFileSync('./README.md').toString(); + const html = htmlDocFromMarkdown(md); + this.readme = {name: 'README', md, html}; + } + + getReadmeHTML() { + return this.readme?.html; + } + + getHtml(name: string): string | undefined { + return this.files.get(name)?.html; + } + + list(): string[] { + return Array.from(this.files.keys()); + } + + readDir() { + // Read list of markdown files from directory and + // render each markdown file as html + readdirSync('./markdown/') + .filter((f) => f.endsWith('.md')) + .map((name) => path.parse(name).name) + .forEach((name) => this.readFile(name)); + } + + watchDir() { + this.dirWatcher = watch('./markdown', null, (eventType, filename) => { + if (!filename) return; + if (!filename.endsWith(".md")) return; + + const name = path.parse(filename).name; + + switch (eventType) { + case 'rename': { + debug(`file ${name} renamed`); + // Remove it from memory and re-scan everything + this.files.delete(name); + this.readDir(); + break; + } + case 'change': { + debug(`file ${name} changed`); + // Re-read this file + this.readFile(name) + break; + } + } + }); + } + + watchReadme() { + this.readmeWatcher = watch('./README.md', null, (eventType, filename) => { + if (!filename) return; + + switch (eventType) { + case 'change': { + debug(`README file changed`); + // Re-read this file + this.readReadme() + break; + } + } + }); + } + + close() { + this.dirWatcher?.close(); + this.readmeWatcher?.close(); + } +} diff --git a/tsconfig.json b/tsconfig.json index a8c6919..4b3def9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,9 +6,6 @@ "moduleResolution": "Node", "sourceMap": true, "baseUrl": ".", - "paths": { - "@/*": ["src/*"] - }, "outDir": "dist", "importsNotUsedAsValues": "remove", "strict": true,