diff --git a/package-lock.json b/package-lock.json index fda7378..b975d24 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "husky": "^8.0.0", "node-fetch": "^3.3.1", "semver": "^7.5.3", + "tmp-promise": "^3.0.3", "tsup": "^7.2.0", "typescript": "^5.0.0", "vitest": "^0.34.0" @@ -7731,6 +7732,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "dependencies": { + "rimraf": "^3.0.0" + }, + "engines": { + "node": ">=8.17.0" + } + }, + "node_modules/tmp-promise": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", + "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", + "dev": true, + "dependencies": { + "tmp": "^0.2.0" + } + }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -14735,6 +14757,24 @@ "integrity": "sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ==", "dev": true }, + "tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dev": true, + "requires": { + "rimraf": "^3.0.0" + } + }, + "tmp-promise": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", + "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", + "dev": true, + "requires": { + "tmp": "^0.2.0" + } + }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", diff --git a/package.json b/package.json index 6fcf5d9..e407673 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "husky": "^8.0.0", "node-fetch": "^3.3.1", "semver": "^7.5.3", + "tmp-promise": "^3.0.3", "tsup": "^7.2.0", "typescript": "^5.0.0", "vitest": "^0.34.0" diff --git a/src/main.ts b/src/main.ts index 6cfac5a..522f2b4 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1 +1,2 @@ export { getDeployStore, getStore } from './store_factory.ts' +export { BlobsServer } from './server.ts' diff --git a/src/server.test.ts b/src/server.test.ts new file mode 100644 index 0000000..62e59c0 --- /dev/null +++ b/src/server.test.ts @@ -0,0 +1,114 @@ +import { promises as fs } from 'node:fs' +import { env, version as nodeVersion } from 'node:process' + +import semver from 'semver' +import tmp from 'tmp-promise' +import { describe, test, expect, beforeAll, afterEach } from 'vitest' + +import { getStore } from './main.js' +import { BlobsServer } from './server.js' + +beforeAll(async () => { + if (semver.lt(nodeVersion, '18.0.0')) { + const nodeFetch = await import('node-fetch') + + // @ts-expect-error Expected type mismatch between native implementation and node-fetch + globalThis.fetch = nodeFetch.default + // @ts-expect-error Expected type mismatch between native implementation and node-fetch + globalThis.Request = nodeFetch.Request + // @ts-expect-error Expected type mismatch between native implementation and node-fetch + globalThis.Response = nodeFetch.Response + // @ts-expect-error Expected type mismatch between native implementation and node-fetch + globalThis.Headers = nodeFetch.Headers + } +}) + +afterEach(() => { + delete env.NETLIFY_BLOBS_CONTEXT +}) + +const siteID = '9a003659-aaaa-0000-aaaa-63d3720d8621' +const key = '54321' +const token = 'my-very-secret-token' + +describe('Local server', () => { + test('Reads and writes from the file system', async () => { + const directory = await tmp.dir() + const server = new BlobsServer({ + directory: directory.path, + token, + }) + const { port } = await server.start() + const blobs = getStore({ + edgeURL: `http://localhost:${port}`, + name: 'mystore', + token, + siteID, + }) + + await blobs.set(key, 'value 1') + expect(await blobs.get(key)).toBe('value 1') + + await blobs.set(key, 'value 2') + expect(await blobs.get(key)).toBe('value 2') + + await blobs.delete(key) + expect(await blobs.get(key)).toBe(null) + + await server.stop() + await fs.rm(directory.path, { force: true, recursive: true }) + }) + + test('Separates keys from different stores', async () => { + const directory = await tmp.dir() + const server = new BlobsServer({ + directory: directory.path, + token, + }) + const { port } = await server.start() + + const store1 = getStore({ + edgeURL: `http://localhost:${port}`, + name: 'mystore1', + token, + siteID, + }) + const store2 = getStore({ + edgeURL: `http://localhost:${port}`, + name: 'mystore2', + token, + siteID, + }) + + await store1.set(key, 'value 1 for store 1') + await store2.set(key, 'value 1 for store 2') + + expect(await store1.get(key)).toBe('value 1 for store 1') + expect(await store2.get(key)).toBe('value 1 for store 2') + + await server.stop() + await fs.rm(directory.path, { force: true, recursive: true }) + }) + + test('If a token is set, rejects any requests with an invalid `authorization` header', async () => { + const directory = await tmp.dir() + const server = new BlobsServer({ + directory: directory.path, + token, + }) + const { port } = await server.start() + const blobs = getStore({ + edgeURL: `http://localhost:${port}`, + name: 'mystore', + token: 'another token', + siteID, + }) + + await expect(async () => await blobs.get(key)).rejects.toThrowError( + 'get operation has failed: store returned a 403 response', + ) + + await server.stop() + await fs.rm(directory.path, { force: true, recursive: true }) + }) +}) diff --git a/src/server.ts b/src/server.ts new file mode 100644 index 0000000..9dc5d94 --- /dev/null +++ b/src/server.ts @@ -0,0 +1,232 @@ +import { createReadStream, createWriteStream, promises as fs } from 'node:fs' +import http from 'node:http' +import { tmpdir } from 'node:os' +import { basename, dirname, join, resolve } from 'node:path' + +import { isNodeError, Logger } from './util.ts' + +interface BlobsServerOptions { + /** + * Whether debug-level information should be logged, such as internal errors + * or information about incoming requests. + */ + debug?: boolean + + /** + * Base directory to read and write files from. + */ + directory: string + + /** + * Function to log messages. Defaults to `console.log`. + */ + logger?: Logger + + /** + * Port to run the server on. Defaults to a random port. + */ + port?: number + + /** + * Static authentication token that should be present in all requests. If not + * supplied, no authentication check is performed. + */ + token?: string +} + +export class BlobsServer { + private debug: boolean + private directory: string + private logger: Logger + private port: number + private server?: http.Server + private token?: string + + constructor({ debug, directory, logger, port, token }: BlobsServerOptions) { + this.debug = debug === true + this.directory = directory + this.logger = logger ?? console.log + this.port = port || 0 + this.token = token + } + + logDebug(...message: unknown[]) { + if (!this.debug) { + return + } + + this.logger('[Netlify Blobs server]', ...message) + } + + async delete(req: http.IncomingMessage, res: http.ServerResponse) { + const { dataPath } = this.getFilePathFromURL(req.url) + + if (!dataPath) { + return this.sendResponse(req, res, 400) + } + + try { + await fs.rm(dataPath, { recursive: true }) + } catch (error: unknown) { + if (isNodeError(error) && error.code === 'ENOENT') { + return this.sendResponse(req, res, 404) + } + + return this.sendResponse(req, res, 500) + } + + return this.sendResponse(req, res, 200) + } + + get(req: http.IncomingMessage, res: http.ServerResponse) { + const { dataPath } = this.getFilePathFromURL(req.url) + + if (!dataPath) { + return this.sendResponse(req, res, 400) + } + + const stream = createReadStream(dataPath) + + stream.on('error', (error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') { + return this.sendResponse(req, res, 404) + } + + return this.sendResponse(req, res, 500) + }) + stream.on('finish', () => this.sendResponse(req, res, 200)) + stream.pipe(res) + } + + async put(req: http.IncomingMessage, res: http.ServerResponse) { + const { dataPath } = this.getFilePathFromURL(req.url) + + if (!dataPath) { + return this.sendResponse(req, res, 400) + } + + try { + // We can't have multiple requests writing to the same file, which could + // lead to corrupted data. Ideally we'd have a mechanism where the last + // request wins, but that requires a more advanced state manager. For + // now, we address this by writing data to a temporary file and then + // moving it to the right path after the write has succeeded. + const tempDirectory = await fs.mkdtemp(join(tmpdir(), 'netlify-blobs')) + const tempPath = join(tempDirectory, basename(dataPath)) + + await new Promise((resolve, reject) => { + req.pipe(createWriteStream(tempPath)) + req.on('end', resolve) + req.on('error', reject) + }) + + await fs.mkdir(dirname(dataPath), { recursive: true }) + await fs.rename(tempPath, dataPath) + await fs.rm(tempDirectory, { force: true, recursive: true }) + } catch (error) { + this.logDebug('Error when writing data:', error) + + return this.sendResponse(req, res, 500) + } + + return this.sendResponse(req, res, 200) + } + + /** + * Returns the path to the local file associated with a given combination of + * site ID, store name, and object, which are extracted from a URL path. + */ + getFilePathFromURL(urlPath?: string) { + if (!urlPath) { + return {} + } + + const [, siteID, storeName, key] = urlPath.split('/') + + if (!siteID || !storeName || !key) { + return {} + } + + const dataPath = resolve(this.directory, 'entries', siteID, storeName, key) + + return { dataPath } + } + + handleRequest(req: http.IncomingMessage, res: http.ServerResponse) { + if (!this.validateAccess(req)) { + return this.sendResponse(req, res, 403) + } + + switch (req.method) { + case 'DELETE': + return this.delete(req, res) + + case 'GET': + return this.get(req, res) + + case 'PUT': + return this.put(req, res) + + default: + return this.sendResponse(req, res, 405) + } + } + + sendResponse(req: http.IncomingMessage, res: http.ServerResponse, status: number) { + this.logDebug(`${req.method} ${req.url}: ${status}`) + + res.writeHead(status) + res.end() + } + + async start(): Promise<{ address: string; family: string; port: number }> { + await fs.mkdir(this.directory, { recursive: true }) + + const server = http.createServer((req, res) => this.handleRequest(req, res)) + + this.server = server + + return new Promise((resolve, reject) => { + server.listen(this.port, () => { + const address = server.address() + + if (!address || typeof address === 'string') { + return reject(new Error('Server cannot be started on a pipe or Unix socket')) + } + + resolve(address) + }) + }) + } + + async stop() { + if (!this.server) { + return + } + + await new Promise((resolve, reject) => { + this.server?.close((error?: NodeJS.ErrnoException) => { + if (error) { + return reject(error) + } + + resolve(null) + }) + }) + } + + validateAccess(req: http.IncomingMessage) { + if (!this.token) { + return true + } + + const { authorization = '' } = req.headers + const parts = authorization.split(' ') + + if (parts.length !== 2 || parts[0].toLowerCase() !== 'bearer') { + return false + } + + return parts[1] === this.token + } +} diff --git a/src/util.ts b/src/util.ts new file mode 100644 index 0000000..6fc2585 --- /dev/null +++ b/src/util.ts @@ -0,0 +1,3 @@ +export const isNodeError = (error: unknown): error is NodeJS.ErrnoException => error instanceof Error + +export type Logger = (...message: unknown[]) => void