build: upgrade to latest chunkd

packages/builder/package.json

@@ -20,7 +20,6 @@
   "license": "MIT",
   "dependencies": {
     "@cotar/core": "^5.5.0",
-    "@chunkd/core": "^10.2.0",
     "@sindresorhus/fnv1a": "^3.0.0",
     "binparse": "^2.0.1"
   },
@@ -33,7 +32,7 @@
     "*.png"
   ],
   "devDependencies": {
-    "@chunkd/source-file": "^10.0.3",
+    "@chunkd/source-file": "^11.0.0",
     "@types/node": "^18.11.9"
   }
 }

packages/builder/src/__benchmark__/tar.validate.ts

@@ -1,4 +1,4 @@
-import { SourceMemory } from '@chunkd/core';
+import { SourceMemory } from '@chunkd/source-memory';
 import { promises as fs } from 'fs';
 import { CotarIndexBuilder } from '../binary.index.builder.js';
 import { CotarIndex } from '@cotar/core';
@@ -9,7 +9,7 @@ async function main(): Promise<void> {
   const fd = await fs.open('test.tar', 'r');
   const res = await CotarIndexBuilder.create(fd);
 
-  const source = new SourceMemory('Memory', res.buffer);
+  const source = new SourceMemory('memory://memory', res.buffer);
   const cotarIndex = new CotarIndex(source, { version: 2, count: res.count, magic: 'COT' });
 
   for (let i = 0; i < 5; i++) {

packages/builder/src/__test__/binary.test.ts

@@ -1,4 +1,4 @@
-import { SourceMemory } from '@chunkd/core';
+import { SourceMemory } from '@chunkd/source-memory';
 import { SourceFile } from '@chunkd/source-file';
 import { Cotar, CotarIndex, IndexHeaderSize, IndexV2RecordSize } from '@cotar/core';
 import fnv1a from '@sindresorhus/fnv1a';
@@ -47,8 +47,8 @@ describe('CotarBinary.fake', () => {
     assert.equal(tarIndexV2.toString('base64'), ExpectedRecordV2);
 
     const cotar = new Cotar(
-      new SourceMemory('Tar', Buffer.from('0123456789')),
-      await CotarIndex.create(new SourceMemory('index', tarIndexV2)),
+      new SourceMemory('memory://tar', Buffer.from('0123456789')),
+      await CotarIndex.create(new SourceMemory('memory://index', tarIndexV2)),
     );
 
     assert.deepEqual(await cotar.index.find('tiles/0/0/0.pbf.gz'), { offset: 0, size: 1 });
@@ -64,7 +64,7 @@ describe('CotarBinary.fake', () => {
   it('should load v2 from a combined tar & header', async () => {
     const tar = Buffer.concat([Buffer.from('0123456789'), tarIndexV2]);
 
-    const cotar = await Cotar.fromTar(new SourceMemory('Combined', tar));
+    const cotar = await Cotar.fromTar(new SourceMemory('memory://Combined', tar));
     assert.equal(cotar.index.sourceOffset, 10);
     assert.deepEqual(cotar.index.metadata, { magic: 'COT', version: 2, count: 4 });
 

packages/builder/src/__test__/tar.test.ts

@@ -1,6 +1,6 @@
 /** Start Header */
 // ^-- this line is used for testing
-import { SourceMemory } from '@chunkd/core';
+import { SourceMemory } from '@chunkd/source-memory';
 import { SourceFile } from '@chunkd/source-file';
 import * as cp from 'child_process';
 import { promises as fs } from 'fs';
@@ -49,7 +49,7 @@ describe('TarReader', () => {
 
     const res = await CotarIndexBuilder.create(source);
 
-    const index = await CotarIndex.create(new SourceMemory('index', res.buffer));
+    const index = await CotarIndex.create(new SourceMemory('memory://index', res.buffer));
     assert.equal(res.count >= 3, true);
 
     const tarTest = await index.find('tar.test.js');

packages/builder/src/binary.index.builder.ts

@@ -1,4 +1,4 @@
-import { SourceMemory } from '@chunkd/core';
+import { SourceMemory } from '@chunkd/source-memory';
 import { LogType } from './log.js';
 import { AsyncFileDescriptor, AsyncFileRead, AsyncReader, TarIndexResult } from './tar.index.js';
 import { TarReader } from './tar.js';
@@ -160,7 +160,7 @@ export const CotarIndexBuilder = {
 
   /** Validate that the index matches the input file */
   async validate(getBytes: AsyncReader, index: Buffer, logger?: LogType): Promise<number> {
-    const binIndex = await CotarIndex.create(new SourceMemory('cotar', index));
+    const binIndex = await CotarIndex.create(new SourceMemory('memory://cotar', index));
     return TarReader.validate(getBytes, binIndex, logger);
   },
 };

packages/cli/package.json

@@ -21,8 +21,8 @@
     "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
   },
   "dependencies": {
-    "@chunkd/fs": "^10.0.6",
-    "@chunkd/source-file": "^10.0.3",
+    "@chunkd/fs": "^11.0.2",
+    "@chunkd/source-file": "^11.0.0",
     "@cotar/core": "^5.5.0",
     "@cotar/builder": "^5.5.0",
     "@cotar/tar": "^5.4.1",

packages/cli/src/commands/create.ts

@@ -1,4 +1,4 @@
-import { SourceMemory } from '@chunkd/core';
+import { SourceMemory } from '@chunkd/source-memory';
 import { CotarIndexBuilder, CotarIndexOptions, TarReader } from '@cotar/builder';
 import { CotarIndex } from '@cotar/core';
 import { command, number, option, positional } from 'cmd-ts';
@@ -65,7 +65,7 @@ async function toTarIndex(
   const { buffer, count } = await CotarIndexBuilder.create(fd, opts, logger);
 
   logger.info({ count, size: buffer.length, duration: Date.now() - startTime }, 'Cotar.Index:Created');
-  const index = await CotarIndex.create(new SourceMemory('index', buffer));
+  const index = await CotarIndex.create(new SourceMemory('memory://index', buffer));
   await TarReader.validate(fd, index, logger);
   await fd.close();
   return buffer;

packages/cli/src/commands/serve.ts

@@ -13,7 +13,7 @@ function rewriteFile(fileName: string, buf: ArrayBuffer, args: { baseUrl: string
     const text = Buffer.from(buf).toString();
     return Buffer.from(text.replace(/template="\//g, `template="${args.baseUrl}/v1/file/`));
   }
-  return buf;
+  return Buffer.from(buf);
 }
 
 export const commandServe = command({
@@ -76,7 +76,7 @@ export const commandServe = command({
       }
 
       res.writeHead(200);
-      res.write(args.disableRewrite ? file : rewriteFile(fileName, file, args));
+      res.write(args.disableRewrite ? Buffer.from(file) : rewriteFile(fileName, file, args));
       res.end();
       logger.info({ action: 'file:get', fileName, status: 200, duration: toDuration(startTime) }, req.url);
     }

packages/cli/src/commands/tar.ts

@@ -1,10 +1,20 @@
-import { fsa } from '@chunkd/fs';
+import { fsa, toArray } from '@chunkd/fs';
 import { TarBuilder } from '@cotar/tar';
 import { command, positional, restPositionals } from 'cmd-ts';
 import { existsSync } from 'fs';
 import { performance } from 'perf_hooks';
 import { force, toDuration, verbose } from '../common.js';
 import { logger } from '../log.js';
+import { Url } from '../url.js';
+import { cwd } from 'process';
+import { pathToFileURL } from 'node:url';
+
+function toRelative(url: URL, baseUrl: URL): string {
+  if (!url.href.startsWith(baseUrl.href)) throw new Error('File is not relative: ' + url + ' vs ' + baseUrl);
+
+  if (baseUrl.href.endsWith('/')) return url.href.slice(baseUrl.href.length);
+  return url.href.slice(baseUrl.href.length + 1);
+}
 
 export const commandTar = command({
   name: 'tar',
@@ -13,7 +23,7 @@ export const commandTar = command({
     force,
     verbose,
     output: positional({ displayName: 'Output', description: 'Output tar file location' }),
-    input: restPositionals({ displayName: 'Input', description: 'Input locations' }),
+    input: restPositionals({ displayName: 'Input', description: 'Input locations', type: Url }),
   },
   async handler(args) {
     if (args.verbose) logger.level = 'debug';
@@ -27,13 +37,24 @@ export const commandTar = command({
       return;
     }
 
+    const workingDir = pathToFileURL(cwd());
+
     const startTime = performance.now();
     const tarBuilder = new TarBuilder(args.output);
 
-    for (const input of args.input.sort()) {
-      const files = await fsa.toArray(fsa.list(input));
-      files.sort((a, b) => a.localeCompare(b));
-      for (const file of files) await tarBuilder.write(file, await fsa.read(file));
+    for (const input of args.input) {
+      const stat = await fsa.head(input);
+      if (stat != null && !stat.isDirectory) {
+        // Found a file add the file directly
+        await tarBuilder.write(toRelative(input, workingDir), await fsa.read(input));
+      } else {
+        const files = await toArray(fsa.list(input));
+
+        files.sort((a, b) => a.href.localeCompare(b.href));
+        for (const file of files) {
+          await tarBuilder.write(toRelative(file, workingDir), await fsa.read(file));
+        }
+      }
     }
 
     await tarBuilder.close();

packages/cli/src/commands/validate.ts

@@ -41,8 +41,8 @@ async function countHashTable(index: CotarIndex): Promise<number> {
   let filled = 0;
   for (let i = 0; i < slotCount; i++) {
     const offset = index.sourceOffset + i * CotarIndex.Header.Record + CotarIndex.Header.Size;
-    await index.source.loadBytes(offset, CotarIndex.Header.Record);
-    if (index.source.getUint32(offset) > 0 || index.source.getUint32(offset + 4) > 0) filled++;
+    const buffer = Buffer.from(await index.source.fetch(offset, CotarIndex.Header.Record));
+    if (buffer.readUInt32LE(0) > 0 || buffer.readUInt32LE(4) > 0) filled++;
   }
 
   return filled;

packages/cli/src/commands/version.ts

@@ -1,15 +1,13 @@
 import { fsa } from '@chunkd/fs';
 import { command } from 'cmd-ts';
-import path from 'path';
-import { fileURLToPath } from 'url';
 
 export const commandVersion = command({
   name: 'version',
   description: 'Dump CLI version information',
   args: {},
   async handler() {
     const packageJson = await fsa.readJson<Record<string, unknown>>(
-      fileURLToPath(path.join(import.meta.url, '..', '..', '..', '..', 'package.json')),
+      new URL('../../../../package.json', import.meta.url),
     );
 
     console.log(packageJson['name'], packageJson['version']);

packages/cli/src/url.ts

@@ -0,0 +1,12 @@
+import { Type } from 'cmd-ts';
+import { pathToFileURL } from 'node:url';
+
+export const Url: Type<string, URL> = {
+  async from(str) {
+    try {
+      return new URL(str);
+    } catch (e) {
+      return pathToFileURL(str);
+    }
+  },
+};

packages/cli/src/util/file.tree.ts

@@ -1,4 +1,4 @@
-import { ChunkSource } from '@chunkd/core';
+import { Source } from '@chunkd/source';
 import { TarReader } from '@cotar/builder';
 import path from 'path';
 
@@ -12,14 +12,13 @@ function toFolderName(f: string): string {
 /** Create a tree from a tar file */
 export class FileTree {
   nodes: Map<string, Set<string>> = new Map();
-  source: ChunkSource;
-  constructor(source: ChunkSource) {
+  source: Source;
+  constructor(source: Source) {
     this.source = source;
   }
 
   getBytes = async (offset: number, count: number): Promise<Buffer> => {
-    await this.source.loadBytes(offset, count);
-    const bytes = await this.source.bytes(offset, count);
+    const bytes = await this.source.fetch(offset, count);
     return Buffer.from(bytes);
   };
 

packages/cli/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "lib": ["es2018"],
+    "lib": ["es2022", "DOM"],
     "outDir": "build"
   },
   "include": ["src"],

packages/core/package.json

@@ -19,7 +19,7 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@chunkd/core": "^10.2.0",
+    "@chunkd/source": "^11.0.0",
     "@sindresorhus/fnv1a": "^3.0.0"
   },
   "publishConfig": {
@@ -31,7 +31,8 @@
     "*.png"
   ],
   "devDependencies": {
-    "@chunkd/source-file": "^10.0.3",
+    "@chunkd/source-file": "^11.0.0",
+    "@chunkd/source-memory": "^11.0.1",
     "@types/node": "^18.11.9"
   }
 }