From b0b81d3d6b6d1c947e5c30d41f4554d7c767c943 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Fri, 21 Jul 2023 00:22:19 +1200 Subject: [PATCH 1/4] feat: upgrade to latest chunkd --- packages/builder/package.json | 2 +- .../builder/src/__benchmark__/tar.validate.ts | 4 +- packages/builder/src/__test__/binary.test.ts | 8 +-- packages/builder/src/__test__/tar.test.ts | 4 +- packages/builder/src/binary.index.builder.ts | 4 +- packages/cli/package.json | 4 +- packages/cli/src/commands/create.ts | 4 +- packages/cli/src/commands/serve.ts | 4 +- packages/cli/src/commands/tar.ts | 15 +++-- packages/cli/src/commands/validate.ts | 4 +- packages/cli/src/commands/version.ts | 4 +- packages/cli/src/url.ts | 12 ++++ packages/cli/src/util/file.tree.ts | 9 ++- packages/cli/tsconfig.json | 2 +- packages/core/package.json | 5 +- packages/core/src/binary.index.ts | 50 ++++++++-------- .../src/binary/__test__/binary.header.test.ts | 5 +- .../core/src/binary/__test__/binary.test.ts | 10 ++-- packages/core/src/cotar.ts | 16 ++--- yarn.lock | 59 +++++++++---------- 20 files changed, 117 insertions(+), 108 deletions(-) create mode 100644 packages/cli/src/url.ts diff --git a/packages/builder/package.json b/packages/builder/package.json index 62b55f80..80007477 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -33,7 +33,7 @@ "*.png" ], "devDependencies": { - "@chunkd/source-file": "^10.0.3", + "@chunkd/source-file": "^11.0.0", "@types/node": "^18.11.9" } } diff --git a/packages/builder/src/__benchmark__/tar.validate.ts b/packages/builder/src/__benchmark__/tar.validate.ts index 884af6fe..91f3fa99 100644 --- a/packages/builder/src/__benchmark__/tar.validate.ts +++ b/packages/builder/src/__benchmark__/tar.validate.ts @@ -1,4 +1,4 @@ -import { SourceMemory } from '@chunkd/core'; +import { SourceMemory } from '@chunkd/source-memory'; import { promises as fs } from 'fs'; import { CotarIndexBuilder } from '../binary.index.builder.js'; import { CotarIndex } from '@cotar/core'; @@ -9,7 +9,7 @@ async function main(): Promise { const fd = await fs.open('test.tar', 'r'); const res = await CotarIndexBuilder.create(fd); - const source = new SourceMemory('Memory', res.buffer); + const source = new SourceMemory(new URL('memory://memory'), res.buffer); const cotarIndex = new CotarIndex(source, { version: 2, count: res.count, magic: 'COT' }); for (let i = 0; i < 5; i++) { diff --git a/packages/builder/src/__test__/binary.test.ts b/packages/builder/src/__test__/binary.test.ts index 1fe270e8..01d06b46 100644 --- a/packages/builder/src/__test__/binary.test.ts +++ b/packages/builder/src/__test__/binary.test.ts @@ -1,4 +1,4 @@ -import { SourceMemory } from '@chunkd/core'; +import { SourceMemory } from '@chunkd/source-memory'; import { SourceFile } from '@chunkd/source-file'; import { Cotar, CotarIndex, IndexHeaderSize, IndexV2RecordSize } from '@cotar/core'; import fnv1a from '@sindresorhus/fnv1a'; @@ -47,8 +47,8 @@ describe('CotarBinary.fake', () => { assert.equal(tarIndexV2.toString('base64'), ExpectedRecordV2); const cotar = new Cotar( - new SourceMemory('Tar', Buffer.from('0123456789')), - await CotarIndex.create(new SourceMemory('index', tarIndexV2)), + new SourceMemory(new URL('memory://tar'), Buffer.from('0123456789')), + await CotarIndex.create(new SourceMemory(new URL('memory://index'), tarIndexV2)), ); assert.deepEqual(await cotar.index.find('tiles/0/0/0.pbf.gz'), { offset: 0, size: 1 }); @@ -64,7 +64,7 @@ describe('CotarBinary.fake', () => { it('should load v2 from a combined tar & header', async () => { const tar = Buffer.concat([Buffer.from('0123456789'), tarIndexV2]); - const cotar = await Cotar.fromTar(new SourceMemory('Combined', tar)); + const cotar = await Cotar.fromTar(new SourceMemory(new URL('memory://Combined'), tar)); assert.equal(cotar.index.sourceOffset, 10); assert.deepEqual(cotar.index.metadata, { magic: 'COT', version: 2, count: 4 }); diff --git a/packages/builder/src/__test__/tar.test.ts b/packages/builder/src/__test__/tar.test.ts index ef29c60a..8644408a 100644 --- a/packages/builder/src/__test__/tar.test.ts +++ b/packages/builder/src/__test__/tar.test.ts @@ -1,6 +1,6 @@ /** Start Header */ // ^-- this line is used for testing -import { SourceMemory } from '@chunkd/core'; +import { SourceMemory } from '@chunkd/source-memory'; import { SourceFile } from '@chunkd/source-file'; import * as cp from 'child_process'; import { promises as fs } from 'fs'; @@ -49,7 +49,7 @@ describe('TarReader', () => { const res = await CotarIndexBuilder.create(source); - const index = await CotarIndex.create(new SourceMemory('index', res.buffer)); + const index = await CotarIndex.create(new SourceMemory(new URL('memory://index'), res.buffer)); assert.equal(res.count >= 3, true); const tarTest = await index.find('tar.test.js'); diff --git a/packages/builder/src/binary.index.builder.ts b/packages/builder/src/binary.index.builder.ts index 8e1eb405..4d8c00a7 100644 --- a/packages/builder/src/binary.index.builder.ts +++ b/packages/builder/src/binary.index.builder.ts @@ -1,4 +1,4 @@ -import { SourceMemory } from '@chunkd/core'; +import { SourceMemory } from '@chunkd/source-memory'; import { LogType } from './log.js'; import { AsyncFileDescriptor, AsyncFileRead, AsyncReader, TarIndexResult } from './tar.index.js'; import { TarReader } from './tar.js'; @@ -160,7 +160,7 @@ export const CotarIndexBuilder = { /** Validate that the index matches the input file */ async validate(getBytes: AsyncReader, index: Buffer, logger?: LogType): Promise { - const binIndex = await CotarIndex.create(new SourceMemory('cotar', index)); + const binIndex = await CotarIndex.create(new SourceMemory(new URL('memory://cotar'), index)); return TarReader.validate(getBytes, binIndex, logger); }, }; diff --git a/packages/cli/package.json b/packages/cli/package.json index 99b0ff12..83e659b4 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -21,8 +21,8 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, "dependencies": { - "@chunkd/fs": "^10.0.6", - "@chunkd/source-file": "^10.0.3", + "@chunkd/fs": "^11.0.0", + "@chunkd/source-file": "^11.0.0", "@cotar/core": "^5.5.0", "@cotar/builder": "^5.5.0", "@cotar/tar": "^5.4.1", diff --git a/packages/cli/src/commands/create.ts b/packages/cli/src/commands/create.ts index daa04cd6..12461065 100644 --- a/packages/cli/src/commands/create.ts +++ b/packages/cli/src/commands/create.ts @@ -1,4 +1,4 @@ -import { SourceMemory } from '@chunkd/core'; +import { SourceMemory } from '@chunkd/source-memory'; import { CotarIndexBuilder, CotarIndexOptions, TarReader } from '@cotar/builder'; import { CotarIndex } from '@cotar/core'; import { command, number, option, positional } from 'cmd-ts'; @@ -65,7 +65,7 @@ async function toTarIndex( const { buffer, count } = await CotarIndexBuilder.create(fd, opts, logger); logger.info({ count, size: buffer.length, duration: Date.now() - startTime }, 'Cotar.Index:Created'); - const index = await CotarIndex.create(new SourceMemory('index', buffer)); + const index = await CotarIndex.create(new SourceMemory(new URL('memory://index'), buffer)); await TarReader.validate(fd, index, logger); await fd.close(); return buffer; diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index a04faa4d..deb4fc7f 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -13,7 +13,7 @@ function rewriteFile(fileName: string, buf: ArrayBuffer, args: { baseUrl: string const text = Buffer.from(buf).toString(); return Buffer.from(text.replace(/template="\//g, `template="${args.baseUrl}/v1/file/`)); } - return buf; + return Buffer.from(buf); } export const commandServe = command({ @@ -76,7 +76,7 @@ export const commandServe = command({ } res.writeHead(200); - res.write(args.disableRewrite ? file : rewriteFile(fileName, file, args)); + res.write(args.disableRewrite ? Buffer.from(file) : rewriteFile(fileName, file, args)); res.end(); logger.info({ action: 'file:get', fileName, status: 200, duration: toDuration(startTime) }, req.url); } diff --git a/packages/cli/src/commands/tar.ts b/packages/cli/src/commands/tar.ts index f929acab..2b712ef3 100644 --- a/packages/cli/src/commands/tar.ts +++ b/packages/cli/src/commands/tar.ts @@ -1,10 +1,11 @@ -import { fsa } from '@chunkd/fs'; +import { fsa, toArray } from '@chunkd/fs'; import { TarBuilder } from '@cotar/tar'; import { command, positional, restPositionals } from 'cmd-ts'; import { existsSync } from 'fs'; import { performance } from 'perf_hooks'; import { force, toDuration, verbose } from '../common.js'; import { logger } from '../log.js'; +import { Url } from '../url.js'; export const commandTar = command({ name: 'tar', @@ -13,7 +14,7 @@ export const commandTar = command({ force, verbose, output: positional({ displayName: 'Output', description: 'Output tar file location' }), - input: restPositionals({ displayName: 'Input', description: 'Input locations' }), + input: restPositionals({ displayName: 'Input', description: 'Input locations', type: Url }), }, async handler(args) { if (args.verbose) logger.level = 'debug'; @@ -31,9 +32,13 @@ export const commandTar = command({ const tarBuilder = new TarBuilder(args.output); for (const input of args.input.sort()) { - const files = await fsa.toArray(fsa.list(input)); - files.sort((a, b) => a.localeCompare(b)); - for (const file of files) await tarBuilder.write(file, await fsa.read(file)); + const files = await toArray(fsa.list(input)); + console.log( + input, + files.map((f) => f.href), + ); + files.sort((a, b) => a.href.localeCompare(b.href)); + for (const file of files) await tarBuilder.write(file.href.slice(input.href.length - 1), await fsa.read(file)); } await tarBuilder.close(); diff --git a/packages/cli/src/commands/validate.ts b/packages/cli/src/commands/validate.ts index 97cbe289..33dbfdf6 100644 --- a/packages/cli/src/commands/validate.ts +++ b/packages/cli/src/commands/validate.ts @@ -41,8 +41,8 @@ async function countHashTable(index: CotarIndex): Promise { let filled = 0; for (let i = 0; i < slotCount; i++) { const offset = index.sourceOffset + i * CotarIndex.Header.Record + CotarIndex.Header.Size; - await index.source.loadBytes(offset, CotarIndex.Header.Record); - if (index.source.getUint32(offset) > 0 || index.source.getUint32(offset + 4) > 0) filled++; + const buffer = Buffer.from(await index.source.fetch(offset, CotarIndex.Header.Record)); + if (buffer.readUInt32LE(0) > 0 || buffer.readUInt32LE(4) > 0) filled++; } return filled; diff --git a/packages/cli/src/commands/version.ts b/packages/cli/src/commands/version.ts index d4779de0..705fdabb 100644 --- a/packages/cli/src/commands/version.ts +++ b/packages/cli/src/commands/version.ts @@ -1,7 +1,5 @@ import { fsa } from '@chunkd/fs'; import { command } from 'cmd-ts'; -import path from 'path'; -import { fileURLToPath } from 'url'; export const commandVersion = command({ name: 'version', @@ -9,7 +7,7 @@ export const commandVersion = command({ args: {}, async handler() { const packageJson = await fsa.readJson>( - fileURLToPath(path.join(import.meta.url, '..', '..', '..', '..', 'package.json')), + new URL('../../../../package.json', import.meta.url), ); console.log(packageJson['name'], packageJson['version']); diff --git a/packages/cli/src/url.ts b/packages/cli/src/url.ts new file mode 100644 index 00000000..319ff0f6 --- /dev/null +++ b/packages/cli/src/url.ts @@ -0,0 +1,12 @@ +import { Type } from 'cmd-ts'; +import { pathToFileURL } from 'node:url'; + +export const Url: Type = { + async from(str) { + try { + return new URL(str); + } catch (e) { + return pathToFileURL(str); + } + }, +}; diff --git a/packages/cli/src/util/file.tree.ts b/packages/cli/src/util/file.tree.ts index 71179061..2fa87bf3 100644 --- a/packages/cli/src/util/file.tree.ts +++ b/packages/cli/src/util/file.tree.ts @@ -1,4 +1,4 @@ -import { ChunkSource } from '@chunkd/core'; +import { Source } from '@chunkd/source'; import { TarReader } from '@cotar/builder'; import path from 'path'; @@ -12,14 +12,13 @@ function toFolderName(f: string): string { /** Create a tree from a tar file */ export class FileTree { nodes: Map> = new Map(); - source: ChunkSource; - constructor(source: ChunkSource) { + source: Source; + constructor(source: Source) { this.source = source; } getBytes = async (offset: number, count: number): Promise => { - await this.source.loadBytes(offset, count); - const bytes = await this.source.bytes(offset, count); + const bytes = await this.source.fetch(offset, count); return Buffer.from(bytes); }; diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index f7ad86d3..b3db2b4b 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "lib": ["es2018"], + "lib": ["es2022", "DOM"], "outDir": "build" }, "include": ["src"], diff --git a/packages/core/package.json b/packages/core/package.json index 65467ee9..4d19dc04 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -19,7 +19,7 @@ }, "license": "MIT", "dependencies": { - "@chunkd/core": "^10.2.0", + "@chunkd/source": "^11.0.0", "@sindresorhus/fnv1a": "^3.0.0" }, "publishConfig": { @@ -31,7 +31,8 @@ "*.png" ], "devDependencies": { - "@chunkd/source-file": "^10.0.3", + "@chunkd/source-file": "^11.0.0", + "@chunkd/source-memory": "^11.0.0", "@types/node": "^18.11.9" } } diff --git a/packages/core/src/binary.index.ts b/packages/core/src/binary.index.ts index 2f97fc36..54e0f5c4 100644 --- a/packages/core/src/binary.index.ts +++ b/packages/core/src/binary.index.ts @@ -1,4 +1,4 @@ -import { ChunkSource } from '@chunkd/core'; +import { Source } from '@chunkd/source'; import fnv1a from '@sindresorhus/fnv1a'; import { CotarIndexRecord } from './cotar.js'; import { IndexHeaderSize, IndexMagic, IndexV2RecordSize } from './format.js'; @@ -6,22 +6,20 @@ import { IndexHeaderSize, IndexMagic, IndexV2RecordSize } from './format.js'; const Big32 = BigInt(32); const BigUint32Max = BigInt(2 ** 32 - 1); -export function readMetadata(bytes: Uint8Array): CotarMetadata { +export function readMetadata(bytes: ArrayBuffer): CotarMetadata { + const buf = new DataView(bytes); // Read the first three bytes as magic 'COT' string - const magic = String.fromCharCode(bytes[0]) + String.fromCharCode(bytes[1]) + String.fromCharCode(bytes[2]); + const magic = + String.fromCharCode(buf.getUint8(0)) + String.fromCharCode(buf.getUint8(1)) + String.fromCharCode(buf.getUint8(2)); // Version number is a uint8 - const version = bytes[3]; + const version = buf.getUint8(3); + const count = buf.getUint32(4, true); - // Record count from uint32 - const byteA = bytes[4]; - const byteB = bytes[5] << 8; - const byteC = bytes[6] << 16; - const byteD = bytes[7] * 0x1000000; return { magic, version, - count: (byteA | byteB | byteC) + byteD, + count, }; } @@ -42,14 +40,14 @@ export class CotarIndex { Record: IndexV2RecordSize, Magic: IndexMagic, }; - source: ChunkSource; + source: Source; sourceOffset: number; /** Should the metadata be read from the header or the footer */ isHeader = true; metadata: CotarMetadata; - constructor(source: ChunkSource, metadata: CotarMetadata, sourceOffset = 0) { + constructor(source: Source, metadata: CotarMetadata, sourceOffset = 0) { this.source = source; this.sourceOffset = sourceOffset; this.metadata = metadata; @@ -59,31 +57,30 @@ export class CotarIndex { return fnv1a(path, { size: 64 }); } - static async loadMetadata(source: ChunkSource, sourceOffset: number, isHeader: boolean): Promise { + static async loadMetadata(source: Source, sourceOffset: number, isHeader: boolean): Promise { if (isHeader) { - await source.loadBytes(sourceOffset, source.chunkSize); - const bytes = source.bytes(sourceOffset, IndexHeaderSize); + const bytes = await source.fetch(sourceOffset, IndexHeaderSize); return readMetadata(bytes); } // TODO ideally this would a file inside the tar // however different tar programs seem to have differing suffixes some have "2x512" bytes of 0 others pad them out further - const bytes = await source.fetchBytes(-IndexHeaderSize); - return readMetadata(new Uint8Array(bytes)); + const bytes = await source.fetch(-IndexHeaderSize); + return readMetadata(bytes); } - static async getMetadata(source: ChunkSource, sourceOffset: number, isHeader: boolean): Promise { + static async getMetadata(source: Source, sourceOffset: number, isHeader: boolean): Promise { const metadata = await this.loadMetadata(source, sourceOffset, isHeader); if (metadata.magic !== IndexMagic) { - throw new Error(`Invalid source: ${source.uri} invalid magic found: ${metadata.magic}`); + throw new Error(`Invalid source: ${source.url} invalid magic found: ${metadata.magic}`); } if (metadata.version !== 2) { - throw new Error(`Invalid source: ${source.uri} invalid version found: ${metadata.version}`); + throw new Error(`Invalid source: ${source.url} invalid version found: ${metadata.version}`); } return metadata; } - static async create(source: ChunkSource, sourceOffset = 0, isHeader = true): Promise { + static async create(source: Source, sourceOffset = 0, isHeader = true): Promise { const metadata = await this.getMetadata(source, sourceOffset, isHeader); return new CotarIndex(source, metadata, sourceOffset); } @@ -108,16 +105,17 @@ export class CotarIndex { let index = startIndex; while (true) { const offset = this.sourceOffset + index * IndexV2RecordSize + IndexHeaderSize; - await this.source.loadBytes(offset, IndexV2RecordSize); + const bytes = await this.source.fetch(offset, IndexV2RecordSize); + const view = new Uint32Array(bytes); - startHashLow = this.source.getUint32(offset); - startHashHigh = this.source.getUint32(offset + 4); + startHashLow = view[0]; + startHashHigh = view[1]; // Found the file if (startHashHigh === hashHigh && startHashLow === hashLow) { // Tar offsets are block aligned to 512byte blocks - const fileOffset = this.source.getUint32(offset + 8) * 512; - const fileSize = this.source.getUint32(offset + 12); + const fileOffset = view[2] * 512; + const fileSize = view[3]; return { offset: fileOffset, size: fileSize }; } // Found a gap in the hash table (file doesn't exist) diff --git a/packages/core/src/binary/__test__/binary.header.test.ts b/packages/core/src/binary/__test__/binary.header.test.ts index 93ae338b..2167ee4a 100644 --- a/packages/core/src/binary/__test__/binary.header.test.ts +++ b/packages/core/src/binary/__test__/binary.header.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; import { readMetadata } from '../../binary.index.js'; +import { SourceMemory } from '@chunkd/source-memory'; const Example = { v1: { @@ -32,12 +33,12 @@ describe('CotarBinaryHeaderFooter', () => { }); it('should parse v1 header', () => { - const header = readMetadata(Example.v1.buf); + const header = readMetadata(SourceMemory.toArrayBuffer(Example.v1.buf)); assert.deepEqual(header, Example.v1.header); }); it('should parse v2 header', () => { - const header = readMetadata(Example.v2.buf); + const header = readMetadata(SourceMemory.toArrayBuffer(Example.v2.buf)); assert.deepEqual(header, Example.v2.header); }); }); diff --git a/packages/core/src/binary/__test__/binary.test.ts b/packages/core/src/binary/__test__/binary.test.ts index ed8fca52..fffe1625 100644 --- a/packages/core/src/binary/__test__/binary.test.ts +++ b/packages/core/src/binary/__test__/binary.test.ts @@ -1,4 +1,4 @@ -import { SourceMemory } from '@chunkd/core'; +import { SourceMemory } from '@chunkd/source-memory'; import fnv1a from '@sindresorhus/fnv1a'; import assert from 'node:assert'; import { describe, it } from 'node:test'; @@ -54,8 +54,8 @@ describe('CotarBinary.fake', () => { assert.equal(tarIndexV2.toString('base64'), ExpectedRecordV2); const cotar = new Cotar( - new SourceMemory('Tar', Buffer.from('0123456789')), - await CotarIndex.create(new SourceMemory('index', tarIndexV2)), + new SourceMemory(new URL('memory://tar'), Buffer.from('0123456789')), + await CotarIndex.create(new SourceMemory(new URL('memory://index'), tarIndexV2)), ); assert.deepEqual(await cotar.index.find('tiles/0/0/0.pbf.gz'), { offset: 0, size: 1 }); @@ -70,8 +70,8 @@ describe('CotarBinary.fake', () => { it('should load v2 from a combined tar & header', async () => { const tar = Buffer.concat([Buffer.from('0123456789'), tarIndexV2]); - - const cotar = await Cotar.fromTar(new SourceMemory('Combined', tar)); + const source = new SourceMemory(new URL('memory://combined'), tar); + const cotar = await Cotar.fromTar(source); // assert.equal(cotar.index.sourceOffset, 10); assert.deepEqual(cotar.index.metadata, { magic: 'COT', version: 2, count: 4 }); diff --git a/packages/core/src/cotar.ts b/packages/core/src/cotar.ts index 2193e463..c7d3285d 100644 --- a/packages/core/src/cotar.ts +++ b/packages/core/src/cotar.ts @@ -1,4 +1,4 @@ -import { ChunkSource } from '@chunkd/core'; +import { Source } from '@chunkd/source'; import { CotarIndex } from './binary.index.js'; import { IndexV2RecordSize, IndexSize } from './format.js'; @@ -8,25 +8,26 @@ export interface CotarIndexRecord { } export class Cotar { - source: ChunkSource; + source: Source; index: CotarIndex; /** * @param source Chunked source of the tar files * @param index */ - constructor(source: ChunkSource, index: CotarIndex) { + constructor(source: Source, index: CotarIndex) { this.source = source; this.index = index; } - static async fromTar(source: ChunkSource): Promise { + static async fromTar(source: Source): Promise { // Load the last file in the tar archive const metadata = await CotarIndex.getMetadata(source, 0, false); - const size = await source.size; + const stat = await source.head(); + if (stat == null || stat.size == null) throw new Error(`Failed to fetch metadata for ${source.url}`); const metadataSize = IndexV2RecordSize; - const startOffset = size - (metadata.count * metadataSize + IndexSize); + const startOffset = stat.size - (metadata.count * metadataSize + IndexSize); const index = new CotarIndex(source, metadata, startOffset); return new Cotar(source, index); @@ -41,7 +42,6 @@ export class Cotar { const index = await this.index.find(fileName); if (index == null) return null; - await this.source.loadBytes(index.offset, index.size); - return this.source.bytes(index.offset, index.size); + return await this.source.fetch(index.offset, index.size); } } diff --git a/yarn.lock b/yarn.lock index e9658a1b..cfc1fb77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -28,45 +28,40 @@ resolved "https://registry.yarnpkg.com/@chunkd/core/-/core-10.2.0.tgz#2c58bee623d78ee838568752dbfc20d5e7729a7b" integrity sha512-11c7jTgKdvVqmdfX4y6zhh/MXYZbtWu+ZnLHQjuqeGEEbyxDhmsUXVX+xjEYJ+bE8Y3+fNBys+pHJscnJPQHWw== -"@chunkd/fs@^10.0.6": - version "10.0.6" - resolved "https://registry.yarnpkg.com/@chunkd/fs/-/fs-10.0.6.tgz#0da10db5f2ff39dd0ae4ce10aa8dfa995c57d448" - integrity sha512-khbB1S8vOcpYR/yggqIPmhSXnMJpJpgvGBhD+QkSdRMVIiFYO4iT6+RqMSj9MQqo+LyZDLKSP/0GBuU4rsVpSw== - dependencies: - "@chunkd/core" "^10.2.0" - "@chunkd/source-file" "^10.0.3" - "@chunkd/source-http" "^10.1.1" - optionalDependencies: - "@chunkd/source-aws" "^10.2.0" - "@chunkd/source-google-cloud" "^10.0.3" - -"@chunkd/source-aws@^10.2.0": - version "10.2.0" - resolved "https://registry.yarnpkg.com/@chunkd/source-aws/-/source-aws-10.2.0.tgz#a96b1740d420904efce11a3ba774009f172ee3ab" - integrity sha512-O/qqNYmp73V28rnL+d1xx+vQqVQdall2VGKFU/QwoMuCHS8IGA+01LHo0cxerTZnBIVaelBATuiglY2q7pJ7HA== +"@chunkd/fs@^11.0.0": + version "11.0.0" + resolved "https://registry.yarnpkg.com/@chunkd/fs/-/fs-11.0.0.tgz#f37d1b7274e169135303cebf81a06f9ebb28ac0f" + integrity sha512-yIx20xJd3NFheOqDSnx/nLFSHGvsSoTaVRASsnCoMofZ6fSxCBoOAiltjzVKHmdUscsu8vIdlD5p6C7H/tmAyw== dependencies: - "@chunkd/core" "^10.2.0" + "@chunkd/source" "^11.0.0" + "@chunkd/source-file" "^11.0.0" + "@chunkd/source-http" "^11.0.0" -"@chunkd/source-file@^10.0.3": - version "10.0.3" - resolved "https://registry.yarnpkg.com/@chunkd/source-file/-/source-file-10.0.3.tgz#de8dadab79e265395acdbcc15632cdec459d1258" - integrity sha512-sU6lEEExfQD7kSGkwdQ2sSt4UjudZTDzhCxy5QUQKR+A3uyaAcezFVMQzc9LH3bLtGRE7tKQifnsUmi3WWpthw== +"@chunkd/source-file@^11.0.0": + version "11.0.0" + resolved "https://registry.yarnpkg.com/@chunkd/source-file/-/source-file-11.0.0.tgz#e2ce35320c82d93460642e5c96c2d6863b7176cf" + integrity sha512-i1yzhLsOXRJRVpuISpdgfKwuHSlKV67dbSm8hO8xQ4kOUJujT1D2vmiZo570QoeA4E1K9q/XdubGRU38HaVcKg== dependencies: - "@chunkd/core" "^10.2.0" + "@chunkd/source" "^11.0.0" -"@chunkd/source-google-cloud@^10.0.3": - version "10.0.3" - resolved "https://registry.yarnpkg.com/@chunkd/source-google-cloud/-/source-google-cloud-10.0.3.tgz#924027c549ebb13f101d19fe8ab8d3e56eba446d" - integrity sha512-pLgvYxjrNux8jAtxU8OuHqod0opnXAq7vjjYnLWwGTm6BdhgyVGdd+Nn0PZHnUTKZgpLZjkdZKWGJNRIAAQDqw== +"@chunkd/source-http@^11.0.0": + version "11.0.0" + resolved "https://registry.yarnpkg.com/@chunkd/source-http/-/source-http-11.0.0.tgz#7db6dac12a15c7e7d3140b9ad792d3c8c764b70d" + integrity sha512-peBgZoCKHxnOh0DYgnh6HWadiNxExQtWv9IjGyAWhuvdr/YKHix7a4vTP+hx82f0Fr4Znajv5Sc9C5N+EJxavw== dependencies: - "@chunkd/core" "^10.2.0" + "@chunkd/source" "^11.0.0" -"@chunkd/source-http@^10.1.1": - version "10.1.1" - resolved "https://registry.yarnpkg.com/@chunkd/source-http/-/source-http-10.1.1.tgz#c09bb47ea46d4bd646cf1a73a7ad6b1eb7dbde23" - integrity sha512-lhkN0bZP5Q1psaLOrUVr8OuBNLrvI5oPDEf1uEYW3/hfJpWrCZU0WCz9xbuS8Fk5RR7qd7y274aJf3h6pSoJwQ== +"@chunkd/source-memory@^11.0.0": + version "11.0.0" + resolved "https://registry.yarnpkg.com/@chunkd/source-memory/-/source-memory-11.0.0.tgz#4d45e1b5ea2feb2e95e90911fe0ceb69b21841da" + integrity sha512-RdmTgpMUJN9nJSuCBX7KhaS0pJEfiMmk/EtRGvse5lEnJMKmMnt5bHbTsvHIE5ONkT+kchwm53KdDKorAMRadA== dependencies: - "@chunkd/core" "^10.2.0" + "@chunkd/source" "^11.0.0" + +"@chunkd/source@^11.0.0": + version "11.0.0" + resolved "https://registry.yarnpkg.com/@chunkd/source/-/source-11.0.0.tgz#8407b1356df5bc2e0b07e0f7be10b60acda3c96f" + integrity sha512-cDdANEMB6J4JEbN5+qDoiiy0CxGtiS3Td+X+A/7R8TW0b2XoKHoqX1/LwTrcdtVRNTOU5QHd83mhau/SbpJb2Q== "@eslint-community/eslint-utils@^4.2.0": version "4.4.0" From 44d2ab48975150b9a11f8b53df3717a8696d07a5 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Wed, 26 Jul 2023 12:56:36 +1200 Subject: [PATCH 2/4] build: upgrade to latest chunkd --- packages/builder/package.json | 1 - packages/cli/package.json | 2 +- packages/core/package.json | 2 +- yarn.lock | 31 +++++++++++++------------------ 4 files changed, 15 insertions(+), 21 deletions(-) diff --git a/packages/builder/package.json b/packages/builder/package.json index 80007477..a9e299f2 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -20,7 +20,6 @@ "license": "MIT", "dependencies": { "@cotar/core": "^5.5.0", - "@chunkd/core": "^10.2.0", "@sindresorhus/fnv1a": "^3.0.0", "binparse": "^2.0.1" }, diff --git a/packages/cli/package.json b/packages/cli/package.json index 83e659b4..7f66b4d9 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -21,7 +21,7 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, "dependencies": { - "@chunkd/fs": "^11.0.0", + "@chunkd/fs": "^11.0.1", "@chunkd/source-file": "^11.0.0", "@cotar/core": "^5.5.0", "@cotar/builder": "^5.5.0", diff --git a/packages/core/package.json b/packages/core/package.json index 4d19dc04..3e521e86 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -32,7 +32,7 @@ ], "devDependencies": { "@chunkd/source-file": "^11.0.0", - "@chunkd/source-memory": "^11.0.0", + "@chunkd/source-memory": "^11.0.1", "@types/node": "^18.11.9" } } diff --git a/yarn.lock b/yarn.lock index cfc1fb77..d222266e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23,19 +23,14 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@chunkd/core@^10.2.0": - version "10.2.0" - resolved "https://registry.yarnpkg.com/@chunkd/core/-/core-10.2.0.tgz#2c58bee623d78ee838568752dbfc20d5e7729a7b" - integrity sha512-11c7jTgKdvVqmdfX4y6zhh/MXYZbtWu+ZnLHQjuqeGEEbyxDhmsUXVX+xjEYJ+bE8Y3+fNBys+pHJscnJPQHWw== - -"@chunkd/fs@^11.0.0": - version "11.0.0" - resolved "https://registry.yarnpkg.com/@chunkd/fs/-/fs-11.0.0.tgz#f37d1b7274e169135303cebf81a06f9ebb28ac0f" - integrity sha512-yIx20xJd3NFheOqDSnx/nLFSHGvsSoTaVRASsnCoMofZ6fSxCBoOAiltjzVKHmdUscsu8vIdlD5p6C7H/tmAyw== +"@chunkd/fs@^11.0.1": + version "11.0.1" + resolved "https://registry.yarnpkg.com/@chunkd/fs/-/fs-11.0.1.tgz#76186f0639b2b0c2b1ac8be756bb68b115f84db4" + integrity sha512-hACdgc63MVW99/PzHj06UpzCytd49VdXmnmFze96ZoO2QLAIKppFGFWANI2MwFN99N+3lc0zCH4UZD/KS02LIg== dependencies: "@chunkd/source" "^11.0.0" "@chunkd/source-file" "^11.0.0" - "@chunkd/source-http" "^11.0.0" + "@chunkd/source-http" "^11.0.1" "@chunkd/source-file@^11.0.0": version "11.0.0" @@ -44,17 +39,17 @@ dependencies: "@chunkd/source" "^11.0.0" -"@chunkd/source-http@^11.0.0": - version "11.0.0" - resolved "https://registry.yarnpkg.com/@chunkd/source-http/-/source-http-11.0.0.tgz#7db6dac12a15c7e7d3140b9ad792d3c8c764b70d" - integrity sha512-peBgZoCKHxnOh0DYgnh6HWadiNxExQtWv9IjGyAWhuvdr/YKHix7a4vTP+hx82f0Fr4Znajv5Sc9C5N+EJxavw== +"@chunkd/source-http@^11.0.1": + version "11.0.1" + resolved "https://registry.yarnpkg.com/@chunkd/source-http/-/source-http-11.0.1.tgz#30285e0e1558c2f5eaa9f097c5587297b88f8d4b" + integrity sha512-DUDOKmnCubN+mAMxWIqUxdnyVg8FNM4O6+CnlblyT8KOln85A52/hxpFvp41zzkzsTGJTyyMd3pYvpQglthRYg== dependencies: "@chunkd/source" "^11.0.0" -"@chunkd/source-memory@^11.0.0": - version "11.0.0" - resolved "https://registry.yarnpkg.com/@chunkd/source-memory/-/source-memory-11.0.0.tgz#4d45e1b5ea2feb2e95e90911fe0ceb69b21841da" - integrity sha512-RdmTgpMUJN9nJSuCBX7KhaS0pJEfiMmk/EtRGvse5lEnJMKmMnt5bHbTsvHIE5ONkT+kchwm53KdDKorAMRadA== +"@chunkd/source-memory@^11.0.1": + version "11.0.1" + resolved "https://registry.yarnpkg.com/@chunkd/source-memory/-/source-memory-11.0.1.tgz#f9de81d881c97475ea9afa8690de94d93daa17fa" + integrity sha512-GbLG3OkTmzk2XKmjZM7tRh2K6OhPC0rzQUJaJdzdXaqZ/0SmN4itER8NTFw0ZxAQYQjilk/3ucKof6Q+gpAtkw== dependencies: "@chunkd/source" "^11.0.0" From 89ce3156fb0eaa53a881e3d9155b99e6e9ca19a7 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Mon, 7 Aug 2023 13:09:15 +1200 Subject: [PATCH 3/4] fix(cli): make tar paths relative to current working directory --- packages/cli/package.json | 2 +- packages/cli/src/commands/tar.ts | 32 ++++++++++++++++++++++++-------- yarn.lock | 9 +++++---- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index 7f66b4d9..74fab730 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -21,7 +21,7 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, "dependencies": { - "@chunkd/fs": "^11.0.1", + "@chunkd/fs": "^11.0.2", "@chunkd/source-file": "^11.0.0", "@cotar/core": "^5.5.0", "@cotar/builder": "^5.5.0", diff --git a/packages/cli/src/commands/tar.ts b/packages/cli/src/commands/tar.ts index 2b712ef3..15a240c3 100644 --- a/packages/cli/src/commands/tar.ts +++ b/packages/cli/src/commands/tar.ts @@ -6,6 +6,15 @@ import { performance } from 'perf_hooks'; import { force, toDuration, verbose } from '../common.js'; import { logger } from '../log.js'; import { Url } from '../url.js'; +import { cwd } from 'process'; +import { pathToFileURL } from 'node:url'; + +function toRelative(url: URL, baseUrl: URL): string { + if (!url.href.startsWith(baseUrl.href)) throw new Error('File is not relative: ' + url + ' vs ' + baseUrl); + + if (baseUrl.href.endsWith('/')) return url.href.slice(baseUrl.href.length); + return url.href.slice(baseUrl.href.length + 1); +} export const commandTar = command({ name: 'tar', @@ -28,17 +37,24 @@ export const commandTar = command({ return; } + const workingDir = pathToFileURL(cwd()); + const startTime = performance.now(); const tarBuilder = new TarBuilder(args.output); - for (const input of args.input.sort()) { - const files = await toArray(fsa.list(input)); - console.log( - input, - files.map((f) => f.href), - ); - files.sort((a, b) => a.href.localeCompare(b.href)); - for (const file of files) await tarBuilder.write(file.href.slice(input.href.length - 1), await fsa.read(file)); + for (const input of args.input) { + const stat = await fsa.head(input); + if (stat != null && !stat.isDirectory) { + // Found a file add the file directly + await tarBuilder.write(toRelative(input, workingDir), await fsa.read(input)); + } else { + const files = await toArray(fsa.list(input)); + + files.sort((a, b) => a.href.localeCompare(b.href)); + for (const file of files) { + await tarBuilder.write(toRelative(file, workingDir), await fsa.read(file)); + } + } } await tarBuilder.close(); diff --git a/yarn.lock b/yarn.lock index d222266e..8da970d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23,14 +23,15 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@chunkd/fs@^11.0.1": - version "11.0.1" - resolved "https://registry.yarnpkg.com/@chunkd/fs/-/fs-11.0.1.tgz#76186f0639b2b0c2b1ac8be756bb68b115f84db4" - integrity sha512-hACdgc63MVW99/PzHj06UpzCytd49VdXmnmFze96ZoO2QLAIKppFGFWANI2MwFN99N+3lc0zCH4UZD/KS02LIg== +"@chunkd/fs@^11.0.2": + version "11.0.2" + resolved "https://registry.yarnpkg.com/@chunkd/fs/-/fs-11.0.2.tgz#fd55ec83c83dc22344db6a2957c12b65883f21c8" + integrity sha512-cLby9YyG2k8Qu9xTEdXREwA+bVxChSNhXEHGzkFh0rev1H5C/dQB5Z6dn/AtLMr2H0D5pgqqlCveH0KruS4pyQ== dependencies: "@chunkd/source" "^11.0.0" "@chunkd/source-file" "^11.0.0" "@chunkd/source-http" "^11.0.1" + "@chunkd/source-memory" "^11.0.1" "@chunkd/source-file@^11.0.0": version "11.0.0" From a26bda52fb66e438c0c717fa5ef38abddf039f00 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Fri, 11 Aug 2023 09:39:57 +1200 Subject: [PATCH 4/4] refactor: remove new URL --- packages/builder/src/__benchmark__/tar.validate.ts | 2 +- packages/builder/src/__test__/binary.test.ts | 6 +++--- packages/builder/src/__test__/tar.test.ts | 2 +- packages/builder/src/binary.index.builder.ts | 2 +- packages/cli/src/commands/create.ts | 2 +- packages/core/src/binary/__test__/binary.test.ts | 6 +++--- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/builder/src/__benchmark__/tar.validate.ts b/packages/builder/src/__benchmark__/tar.validate.ts index 91f3fa99..1ed30f79 100644 --- a/packages/builder/src/__benchmark__/tar.validate.ts +++ b/packages/builder/src/__benchmark__/tar.validate.ts @@ -9,7 +9,7 @@ async function main(): Promise { const fd = await fs.open('test.tar', 'r'); const res = await CotarIndexBuilder.create(fd); - const source = new SourceMemory(new URL('memory://memory'), res.buffer); + const source = new SourceMemory('memory://memory', res.buffer); const cotarIndex = new CotarIndex(source, { version: 2, count: res.count, magic: 'COT' }); for (let i = 0; i < 5; i++) { diff --git a/packages/builder/src/__test__/binary.test.ts b/packages/builder/src/__test__/binary.test.ts index 01d06b46..39e89731 100644 --- a/packages/builder/src/__test__/binary.test.ts +++ b/packages/builder/src/__test__/binary.test.ts @@ -47,8 +47,8 @@ describe('CotarBinary.fake', () => { assert.equal(tarIndexV2.toString('base64'), ExpectedRecordV2); const cotar = new Cotar( - new SourceMemory(new URL('memory://tar'), Buffer.from('0123456789')), - await CotarIndex.create(new SourceMemory(new URL('memory://index'), tarIndexV2)), + new SourceMemory('memory://tar', Buffer.from('0123456789')), + await CotarIndex.create(new SourceMemory('memory://index', tarIndexV2)), ); assert.deepEqual(await cotar.index.find('tiles/0/0/0.pbf.gz'), { offset: 0, size: 1 }); @@ -64,7 +64,7 @@ describe('CotarBinary.fake', () => { it('should load v2 from a combined tar & header', async () => { const tar = Buffer.concat([Buffer.from('0123456789'), tarIndexV2]); - const cotar = await Cotar.fromTar(new SourceMemory(new URL('memory://Combined'), tar)); + const cotar = await Cotar.fromTar(new SourceMemory('memory://Combined', tar)); assert.equal(cotar.index.sourceOffset, 10); assert.deepEqual(cotar.index.metadata, { magic: 'COT', version: 2, count: 4 }); diff --git a/packages/builder/src/__test__/tar.test.ts b/packages/builder/src/__test__/tar.test.ts index 8644408a..607fa5f6 100644 --- a/packages/builder/src/__test__/tar.test.ts +++ b/packages/builder/src/__test__/tar.test.ts @@ -49,7 +49,7 @@ describe('TarReader', () => { const res = await CotarIndexBuilder.create(source); - const index = await CotarIndex.create(new SourceMemory(new URL('memory://index'), res.buffer)); + const index = await CotarIndex.create(new SourceMemory('memory://index', res.buffer)); assert.equal(res.count >= 3, true); const tarTest = await index.find('tar.test.js'); diff --git a/packages/builder/src/binary.index.builder.ts b/packages/builder/src/binary.index.builder.ts index 4d8c00a7..eb322486 100644 --- a/packages/builder/src/binary.index.builder.ts +++ b/packages/builder/src/binary.index.builder.ts @@ -160,7 +160,7 @@ export const CotarIndexBuilder = { /** Validate that the index matches the input file */ async validate(getBytes: AsyncReader, index: Buffer, logger?: LogType): Promise { - const binIndex = await CotarIndex.create(new SourceMemory(new URL('memory://cotar'), index)); + const binIndex = await CotarIndex.create(new SourceMemory('memory://cotar', index)); return TarReader.validate(getBytes, binIndex, logger); }, }; diff --git a/packages/cli/src/commands/create.ts b/packages/cli/src/commands/create.ts index 12461065..6a4269df 100644 --- a/packages/cli/src/commands/create.ts +++ b/packages/cli/src/commands/create.ts @@ -65,7 +65,7 @@ async function toTarIndex( const { buffer, count } = await CotarIndexBuilder.create(fd, opts, logger); logger.info({ count, size: buffer.length, duration: Date.now() - startTime }, 'Cotar.Index:Created'); - const index = await CotarIndex.create(new SourceMemory(new URL('memory://index'), buffer)); + const index = await CotarIndex.create(new SourceMemory('memory://index', buffer)); await TarReader.validate(fd, index, logger); await fd.close(); return buffer; diff --git a/packages/core/src/binary/__test__/binary.test.ts b/packages/core/src/binary/__test__/binary.test.ts index fffe1625..7ae9e67c 100644 --- a/packages/core/src/binary/__test__/binary.test.ts +++ b/packages/core/src/binary/__test__/binary.test.ts @@ -54,8 +54,8 @@ describe('CotarBinary.fake', () => { assert.equal(tarIndexV2.toString('base64'), ExpectedRecordV2); const cotar = new Cotar( - new SourceMemory(new URL('memory://tar'), Buffer.from('0123456789')), - await CotarIndex.create(new SourceMemory(new URL('memory://index'), tarIndexV2)), + new SourceMemory('memory://tar', Buffer.from('0123456789')), + await CotarIndex.create(new SourceMemory('memory://index', tarIndexV2)), ); assert.deepEqual(await cotar.index.find('tiles/0/0/0.pbf.gz'), { offset: 0, size: 1 }); @@ -70,7 +70,7 @@ describe('CotarBinary.fake', () => { it('should load v2 from a combined tar & header', async () => { const tar = Buffer.concat([Buffer.from('0123456789'), tarIndexV2]); - const source = new SourceMemory(new URL('memory://combined'), tar); + const source = new SourceMemory('memory://combined', tar); const cotar = await Cotar.fromTar(source); // assert.equal(cotar.index.sourceOffset, 10); assert.deepEqual(cotar.index.metadata, { magic: 'COT', version: 2, count: 4 });