From e7c0a09f4d1b2062f730c635f278246d979e9a40 Mon Sep 17 00:00:00 2001 From: "Mark S. Miller" Date: Mon, 3 Apr 2023 20:42:45 -0700 Subject: [PATCH] fix: CopyBytes new binary Passable type --- packages/marshal/src/deeplyFulfilled.js | 3 + packages/marshal/src/encodePassable.js | 17 ++++ packages/marshal/src/encodeToCapData.js | 4 + packages/marshal/src/encodeToSmallcaps.js | 4 + packages/marshal/src/rankOrder.js | 22 +++++ packages/pass-style/src/copyBytes.js | 84 +++++++++++++++++++ packages/pass-style/src/passStyleOf.js | 3 + packages/pass-style/src/typeGuards.js | 30 +++++++ packages/pass-style/src/types.js | 22 +++-- packages/patterns/src/keys/checkKey.js | 3 + packages/patterns/src/keys/compareKeys.js | 20 +++++ .../patterns/src/patterns/internal-types.js | 1 + .../patterns/src/patterns/patternMatchers.js | 41 +++++++++ packages/patterns/src/types.js | 9 +- 14 files changed, 256 insertions(+), 7 deletions(-) create mode 100644 packages/pass-style/src/copyBytes.js diff --git a/packages/marshal/src/deeplyFulfilled.js b/packages/marshal/src/deeplyFulfilled.js index ebedc93a6d..cddd8eb971 100644 --- a/packages/marshal/src/deeplyFulfilled.js +++ b/packages/marshal/src/deeplyFulfilled.js @@ -61,6 +61,9 @@ export const deeplyFulfilled = async val => { const valPs = val.map(p => deeplyFulfilled(p)); return E.when(Promise.all(valPs), vals => harden(vals)); } + case 'copyBytes': { + return val; + } case 'tagged': { const tag = getTag(val); return E.when(deeplyFulfilled(val.payload), payload => diff --git a/packages/marshal/src/encodePassable.js b/packages/marshal/src/encodePassable.js index 387a88b402..0de77a27bf 100644 --- a/packages/marshal/src/encodePassable.js +++ b/packages/marshal/src/encodePassable.js @@ -16,6 +16,7 @@ import { * @template {Passable} [T=Passable] * @typedef {import('@endo/pass-style').CopyRecord} CopyRecord */ +/** @typedef {import('@endo/pass-style').CopyBytes} CopyBytes */ /** @typedef {import('./types.js').RankCover} RankCover */ const { quote: q, Fail } = assert; @@ -270,6 +271,18 @@ const decodeArray = (encoded, decodePassable) => { return harden(elements); }; +/** + * @param {CopyBytes} copyBytes + * @param {(copyBytes: CopyBytes) => string} _encodePassable + * @returns {string} + */ +const encodeCopyBytes = (copyBytes, _encodePassable) => { + // TODO implement + throw Fail`encodePassable(copyData) not yet implemented: ${copyBytes}`; + // eslint-disable-next-line no-unreachable + return ''; // Just for the type +}; + const encodeRecord = (record, encodePassable) => { const names = recordNames(record); const values = recordValues(record, names); @@ -381,6 +394,9 @@ export const makeEncodePassable = (encodeOptions = {}) => { case 'copyArray': { return encodeArray(passable, encodePassable); } + case 'copyBytes': { + return encodeCopyBytes(passable, encodePassable); + } case 'copyRecord': { return encodeRecord(passable, encodePassable); } @@ -500,6 +516,7 @@ export const passStylePrefixes = { tagged: ':', promise: '?', copyArray: '[', + copyBytes: '', // TODO pick a prefix boolean: 'b', number: 'f', bigint: 'np', diff --git a/packages/marshal/src/encodeToCapData.js b/packages/marshal/src/encodeToCapData.js index ca89725978..e3126bab76 100644 --- a/packages/marshal/src/encodeToCapData.js +++ b/packages/marshal/src/encodeToCapData.js @@ -196,6 +196,10 @@ export const makeEncodeToCapData = (encodeOptions = {}) => { case 'copyArray': { return passable.map(encodeToCapDataRecur); } + case 'copyBytes': { + // TODO implement + throw Fail`marsal of copyBytes not yet implemented: ${passable}`; + } case 'tagged': { return { [QCLASS]: 'tagged', diff --git a/packages/marshal/src/encodeToSmallcaps.js b/packages/marshal/src/encodeToSmallcaps.js index f96c51b841..c925c51d3d 100644 --- a/packages/marshal/src/encodeToSmallcaps.js +++ b/packages/marshal/src/encodeToSmallcaps.js @@ -230,6 +230,10 @@ export const makeEncodeToSmallcaps = (encodeOptions = {}) => { case 'copyArray': { return passable.map(encodeToSmallcapsRecur); } + case 'copyBytes': { + // TODO implement + throw Fail`marsal of copyBytes not yet implemented: ${passable}`; + } case 'tagged': { return { '#tag': encodeToSmallcapsRecur(getTag(passable)), diff --git a/packages/marshal/src/rankOrder.js b/packages/marshal/src/rankOrder.js index f8d8d9d95c..3891c5f779 100644 --- a/packages/marshal/src/rankOrder.js +++ b/packages/marshal/src/rankOrder.js @@ -57,6 +57,8 @@ export const trivialComparator = (left, right) => const passStyleRanks = /** @type {PassStyleRanksRecord} */ ( fromEntries( entries(passStylePrefixes) + // TODO Until copyBytes prefix is chosen + .filter(([_style, prefixes]) => prefixes.length >= 1) // Sort entries by ascending prefix. .sort(([_leftStyle, leftPrefixes], [_rightStyle, rightPrefixes]) => { return trivialComparator(leftPrefixes, rightPrefixes); @@ -211,6 +213,26 @@ export const makeComparatorKit = (compareRemotables = (_x, _y) => 0) => { // If array X is a prefix of array Y, then X has an earlier rank than Y. return comparator(left.length, right.length); } + case 'copyBytes': { + const leftArray = new Uint8Array(left.slice()); + const rightArray = new Uint8Array(right.slice()); + const byteLen = Math.min(left.byteLength, right.byteLength); + for (let i = 0; i < byteLen; i += 1) { + const leftByte = leftArray[i]; + const rightByte = rightArray[i]; + if (leftByte < rightByte) { + return -1; + } + if (leftByte > rightByte) { + return 1; + } + } + // If all corresponding bytes are the same, + // then according to their lengths. + // Thus, if the data of CopyBytes X is a prefix of + // the data of CopyBytes Y, then X is smaller than Y. + return comparator(left.byteLength, right.byteLength); + } case 'tagged': { // Lexicographic by `[Symbol.toStringTag]` then `.payload`. const labelComp = comparator(getTag(left), getTag(right)); diff --git a/packages/pass-style/src/copyBytes.js b/packages/pass-style/src/copyBytes.js new file mode 100644 index 0000000000..3c30d2ad95 --- /dev/null +++ b/packages/pass-style/src/copyBytes.js @@ -0,0 +1,84 @@ +/// + +import { assertChecker } from './passStyle-helpers.js'; + +/** @typedef {import('./types.js').CopyBytes} CopyBytes */ + +const { Fail } = assert; +const { setPrototypeOf } = Object; +const { apply } = Reflect; + +/** + * @type {WeakSet} + */ +const genuineCopyBytes = new WeakSet(); + +const slice = ArrayBuffer.prototype.slice; +const sliceOf = (buffer, start, end) => apply(slice, buffer, [start, end]); + +/** + * A CopyBytes is much like an ArrayBuffer, but immutable. + * It cannot be used as an ArrayBuffer argument when a genuine ArrayBuffer is + * needed. But a `copyBytes.slice()` is a genuine ArrayBuffer, initially with + * a copy of the copyByte's data. + * + * On platforms that support freezing ArrayBuffer, like perhaps a future XS, + * (TODO) the intention is that `copyBytes` could hold on to a single frozen + * one and return it for every call to `arrayBuffer.slice`, rather than making + * a fresh copy each time. + * + * @param {ArrayBuffer} arrayBuffer + * @returns {CopyBytes} + */ +export const makeCopyBytes = arrayBuffer => { + try { + // Both validates and gets an exclusive copy. + // This `arrayBuffer` must not escape, to emulate immutability. + arrayBuffer = sliceOf(arrayBuffer); + } catch { + Fail`Expected genuine ArrayBuffer" ${arrayBuffer}`; + } + /** @type {CopyBytes} */ + const copyBytes = { + // Can't say it this way because it confuses TypeScript + // __proto__: ArrayBuffer.prototype, + byteLength: arrayBuffer.byteLength, + slice(start, end) { + return sliceOf(arrayBuffer, start, end); + }, + [Symbol.toStringTag]: 'CopyBytes', + }; + setPrototypeOf(copyBytes, ArrayBuffer.prototype); + harden(copyBytes); + genuineCopyBytes.add(copyBytes); + return copyBytes; +}; +harden(makeCopyBytes); + +/** + * TODO: This technique for recognizing genuine CopyBytes is incompatible + * with our normal assumption of uncontrolled multiple instantiation of + * a single module. However, our only alternative to this technique is + * unprivileged re-validation of open data, which is incompat with our + * need to encapsulate `arrayBuffer`, the genuinely mutable ArrayBuffer. + * + * @param {unknown} candidate + * @param {import('./types.js').Checker} [check] + * @returns {boolean} + */ +const canBeValid = (candidate, check = undefined) => + // @ts-expect-error `has` argument can actually be anything. + genuineCopyBytes.has(candidate); + +/** + * @type {import('./internal-types.js').PassStyleHelper} + */ +export const CopyBytesHelper = harden({ + styleName: 'copyBytes', + + canBeValid, + + assertValid: (candidate, _passStyleOfRecur) => { + canBeValid(candidate, assertChecker); + }, +}); diff --git a/packages/pass-style/src/passStyleOf.js b/packages/pass-style/src/passStyleOf.js index 0a49fe71a1..9b63198010 100644 --- a/packages/pass-style/src/passStyleOf.js +++ b/packages/pass-style/src/passStyleOf.js @@ -6,6 +6,7 @@ import { isPromise } from '@endo/promise-kit'; import { isObject, isTypedArray, PASS_STYLE } from './passStyle-helpers.js'; import { CopyArrayHelper } from './copyArray.js'; +import { CopyBytesHelper } from './copyBytes.js'; import { CopyRecordHelper } from './copyRecord.js'; import { TaggedHelper } from './tagged.js'; import { ErrorHelper } from './error.js'; @@ -36,6 +37,7 @@ const makeHelperTable = passStyleHelpers => { const HelperTable = { __proto__: null, copyArray: undefined, + copyBytes: undefined, copyRecord: undefined, tagged: undefined, error: undefined, @@ -211,6 +213,7 @@ export const passStyleOf = (globalThis && globalThis[PassStyleOfEndowmentSymbol]) || makePassStyleOf([ CopyArrayHelper, + CopyBytesHelper, CopyRecordHelper, TaggedHelper, ErrorHelper, diff --git a/packages/pass-style/src/typeGuards.js b/packages/pass-style/src/typeGuards.js index 2d2f13e267..19ea226ed1 100644 --- a/packages/pass-style/src/typeGuards.js +++ b/packages/pass-style/src/typeGuards.js @@ -5,6 +5,7 @@ import { passStyleOf } from './passStyleOf.js'; * @template {Passable} [T=Passable] * @typedef {import('./types.js').CopyArray} CopyArray */ +/** @typedef {import('./types.js').CopyBytes} CopyBytes */ /** * @template {Passable} [T=Passable] * @typedef {import('./types.js').CopyRecord} CopyRecord @@ -23,6 +24,16 @@ const { Fail, quote: q } = assert; const isCopyArray = arr => passStyleOf(arr) === 'copyArray'; harden(isCopyArray); +/** + * Check whether the argument is a pass-by-copy binary data, AKA a "copyBytes" + * in @endo/marshal terms + * + * @param {Passable} arr + * @returns {arr is CopyBytes} + */ +const isCopyBytes = arr => passStyleOf(arr) === 'copyBytes'; +harden(isCopyBytes); + /** * Check whether the argument is a pass-by-copy record, AKA a * "copyRecord" in @endo/marshal terms @@ -59,6 +70,23 @@ const assertCopyArray = (array, optNameOfArray = 'Alleged array') => { }; harden(assertCopyArray); +/** + * @callback AssertCopyBytes + * @param {Passable} array + * @param {string=} optNameOfArray + * @returns {asserts array is CopyBytes} + */ + +/** @type {AssertCopyBytes} */ +const assertCopyBytes = (array, optNameOfArray = 'Alleged copyBytes') => { + const passStyle = passStyleOf(array); + passStyle === 'copyBytes' || + Fail`${q( + optNameOfArray, + )} ${array} must be a pass-by-copy binary data, not ${q(passStyle)}`; +}; +harden(assertCopyBytes); + /** * @callback AssertRecord * @param {Passable} record @@ -99,8 +127,10 @@ harden(assertRemotable); export { assertRecord, assertCopyArray, + assertCopyBytes, assertRemotable, isRemotable, isRecord, isCopyArray, + isCopyBytes, }; diff --git a/packages/pass-style/src/types.js b/packages/pass-style/src/types.js index 7d187e1096..0c7e26d9a5 100644 --- a/packages/pass-style/src/types.js +++ b/packages/pass-style/src/types.js @@ -8,7 +8,7 @@ export {}; /** * @typedef { PrimitiveStyle | - * 'copyRecord' | 'copyArray' | 'tagged' | + * 'copyRecord' | 'copyArray' | 'copyBytes' | 'tagged' | * 'remotable' | * 'error' | 'promise' * } PassStyle @@ -29,6 +29,7 @@ export {}; * | 'string' | 'symbol'). * * Containers aggregate other Passables into * * sequences as CopyArrays (PassStyle 'copyArray'), or + * * sequences of 8-bit bytes (PassStyle 'copyBytes'), or * * string-keyed dictionaries as CopyRecords (PassStyle 'copyRecord'), or * * higher-order types as CopyTaggeds (PassStyle 'tagged'). * * PassableCaps (PassStyle 'remotable' | 'promise') expose local values to @@ -53,10 +54,9 @@ export {}; * * A Passable is PureData when its entire data structure is free of PassableCaps * (remotables and promises) and error objects. - * PureData is an arbitrary composition of primitive values into CopyArray - * and/or - * CopyRecord and/or CopyTagged containers (or a single primitive value with no - * container), and is fully pass-by-copy. + * PureData is an arbitrary composition of primitive values into CopyArray, + * CopyBytes, CopyRecord, and/or CopyTagged containers + * (or a single primitive value with no container), and is fully pass-by-copy. * * This restriction assures absence of side effects and interleaving risks *given* * that none of the containers can be a Proxy instance. @@ -93,6 +93,18 @@ export {}; * A Passable sequence of Passable values. */ +/** + * @typedef {{ + * [Symbol.toStringTag]: string, + * byteLength: number, + * slice: (start?: number, end?: number) => ArrayBuffer, + * }} CopyBytes + * It has the same structural type. But because it is not a builtin ArrayBuffer, + * it does not have the same nominal type; meaning, it cannot be used as an + * argument where an ArrayBuffer is expected, like the `DataView` or typed + * array constructors. + */ + /** * @template {Passable} [T=Passable] * @typedef {Record} CopyRecord diff --git a/packages/patterns/src/keys/checkKey.js b/packages/patterns/src/keys/checkKey.js index 148537f6f7..cb1c8641b2 100644 --- a/packages/patterns/src/keys/checkKey.js +++ b/packages/patterns/src/keys/checkKey.js @@ -560,6 +560,9 @@ const checkKeyInternal = (val, check) => { // A copyArray is a key iff all its children are keys return val.every(checkIt); } + case 'copyBytes': { + return true; + } case 'tagged': { const tag = getTag(val); switch (tag) { diff --git a/packages/patterns/src/keys/compareKeys.js b/packages/patterns/src/keys/compareKeys.js index b8ee5864e6..ea434b39ec 100644 --- a/packages/patterns/src/keys/compareKeys.js +++ b/packages/patterns/src/keys/compareKeys.js @@ -145,6 +145,26 @@ export const compareKeys = (left, right) => { // Thus, if array X is a prefix of array Y, then X is smaller than Y. return compareRank(left.length, right.length); } + case 'copyBytes': { + const leftArray = new Uint8Array(left.slice()); + const rightArray = new Uint8Array(right.slice()); + const byteLen = Math.min(left.byteLength, right.byteLength); + for (let i = 0; i < byteLen; i += 1) { + const leftByte = leftArray[i]; + const rightByte = rightArray[i]; + if (leftByte < rightByte) { + return -1; + } + if (leftByte > rightByte) { + return 1; + } + } + // If all corresponding bytes are the same, + // then according to their lengths. + // Thus, if the data of CopyBytes X is a prefix of + // the data of CopyBytes Y, then X is smaller than Y. + return compareRank(left.byteLength, right.byteLength); + } case 'copyRecord': { // Pareto partial order comparison. const leftNames = recordNames(left); diff --git a/packages/patterns/src/patterns/internal-types.js b/packages/patterns/src/patterns/internal-types.js index 072cac4fe7..16ae52b326 100644 --- a/packages/patterns/src/patterns/internal-types.js +++ b/packages/patterns/src/patterns/internal-types.js @@ -15,6 +15,7 @@ * @template {Passable} [T=Passable] * @typedef {import('@endo/pass-style').CopyArray} CopyArray */ +/** @typedef {import('@endo/pass-style').CopyBytes} CopyBytes */ /** @typedef {import('@endo/pass-style').Checker} Checker */ /** @typedef {import('@endo/marshal').RankCompare} RankCompare */ /** @typedef {import('@endo/marshal').RankCover} RankCover */ diff --git a/packages/patterns/src/patterns/patternMatchers.js b/packages/patterns/src/patterns/patternMatchers.js index 2a22fa6744..66f64a546d 100644 --- a/packages/patterns/src/patterns/patternMatchers.js +++ b/packages/patterns/src/patterns/patternMatchers.js @@ -69,6 +69,7 @@ export const defaultLimits = harden({ numPropertiesLimit: 80, propertyNameLengthLimit: 100, arrayLengthLimit: 10_000, + byteLengthLimit: 100_000, numSetElementsLimit: 10_000, numUniqueBagElementsLimit: 10_000, numMapEntriesLimit: 5000, @@ -376,6 +377,9 @@ const makePatternKit = () => { // patterns return patt.every(checkIt); } + case 'copyBytes': { + return true; + } case 'copyMap': { // A copyMap's keys are keys and therefore already known to be // patterns. @@ -451,6 +455,7 @@ const makePatternKit = () => { case 'bigint': case 'string': case 'symbol': + case 'copyBytes': case 'copySet': case 'copyBag': case 'remotable': { @@ -632,6 +637,10 @@ const makePatternKit = () => { // ]); break; } + case 'copyBytes': { + // TODO implement + throw Fail`getCover of copyBytes not yet implemented`; + } case 'copyRecord': { // XXX this doesn't get along with the world of cover === pair of // strings. In the meantime, fall through to the default which @@ -1172,6 +1181,34 @@ const makePatternKit = () => { getRankCover: () => getPassStyleCover('copyArray'), }); + /** @type {MatchHelper} */ + const matchBytesHelper = Far('match:bytes helper', { + checkMatches: (specimen, [limits = undefined], check) => { + const { byteLengthLimit } = limit(limits); + // prettier-ignore + return ( + checkKind(specimen, 'copyBytes', check) && + // eslint-disable-next-line @endo/restrict-comparison-operands + (/** @type {CopyBytes} */ (specimen).byteLength <= byteLengthLimit || + check( + false, + X`bytes ${specimen} must not be bigger than ${byteLengthLimit}`, + )) + ); + }, + + checkIsWellFormed: (payload, check) => + checkIsWellFormedWithLimit( + payload, + harden([]), + check, + 'match:bytes payload', + ), + + getRankCover: (_matchPayload, _encodePassable) => + getPassStyleCover('string'), + }); + /** @type {MatchHelper} */ const matchSetOfHelper = Far('match:setOf helper', { checkMatches: (specimen, [keyPatt, limits = undefined], check) => { @@ -1526,6 +1563,7 @@ const makePatternKit = () => { 'match:gt': matchGTHelper, 'match:arrayOf': matchArrayOfHelper, + 'match:bytes': matchBytesHelper, 'match:recordOf': matchRecordOfHelper, 'match:setOf': matchSetOfHelper, 'match:bagOf': matchBagOfHelper, @@ -1554,6 +1592,7 @@ const makePatternKit = () => { const SymbolShape = makeTagged('match:symbol', []); const RecordShape = makeTagged('match:recordOf', [AnyShape, AnyShape]); const ArrayShape = makeTagged('match:arrayOf', [AnyShape]); + const BytesShape = makeTagged('match:bytes', []); const SetShape = makeTagged('match:setOf', [AnyShape]); const BagShape = makeTagged('match:bagOf', [AnyShape, AnyShape]); const MapShape = makeTagged('match:mapOf', [AnyShape, AnyShape]); @@ -1632,6 +1671,8 @@ const makePatternKit = () => { limits ? M.recordOf(M.any(), M.any(), limits) : RecordShape, array: (limits = undefined) => limits ? M.arrayOf(M.any(), limits) : ArrayShape, + bytes: (limits = undefined) => + limits ? makeLimitsMatcher('match:bytes', [limits]) : BytesShape, set: (limits = undefined) => (limits ? M.setOf(M.any(), limits) : SetShape), bag: (limits = undefined) => limits ? M.bagOf(M.any(), M.any(), limits) : BagShape, diff --git a/packages/patterns/src/types.js b/packages/patterns/src/types.js index 2c18776e73..b0355a7bed 100644 --- a/packages/patterns/src/types.js +++ b/packages/patterns/src/types.js @@ -17,6 +17,7 @@ export {}; * @template {Passable} [T=Passable] * @typedef {import('@endo/pass-style').CopyArray} CopyArray */ +/** @typedef {import('@endo/pass-style').CopyBytes} CopyBytes */ /** @typedef {import('@endo/pass-style').Checker} Checker */ /** @typedef {import('@endo/marshal').RankCompare} RankCompare */ /** @typedef {import('@endo/marshal').RankCover} RankCover */ @@ -25,7 +26,7 @@ export {}; * @typedef {Passable} Key * * Keys are Passable arbitrarily-nested pass-by-copy containers - * (CopyArray, CopyRecord, CopySet, CopyBag, CopyMap) in which every + * (CopyArray, CopyBytes, CopyRecord, CopySet, CopyBag, CopyMap) in which every * non-container leaf is either a Passable primitive value or a Remotable (a * remotely-accessible object or presence for a remote object), or such leaves * in isolation with no container. @@ -64,7 +65,7 @@ export {}; * @typedef {Passable} Pattern * * Patterns are Passable arbitrarily-nested pass-by-copy containers - * (CopyArray, CopyRecord, CopySet, CopyBag, CopyMap) in which every + * (CopyArray, CopyBytes, CopyRecord, CopySet, CopyBag, CopyMap) in which every * non-container leaf is either a Key or a Matcher, or such leaves in isolation * with no container. * @@ -224,6 +225,7 @@ export {}; * @property {number} numPropertiesLimit * @property {number} propertyNameLengthLimit * @property {number} arrayLengthLimit + * @property {number} byteLengthLimit * @property {number} numSetElementsLimit * @property {number} numUniqueBagElementsLimit * @property {number} numMapEntriesLimit @@ -298,6 +300,9 @@ export {}; * @property {(limits?: Limits) => Matcher} array * Matches any CopyArray, subject to limits. * + * @property {(limits?: Limits) => Matcher} bytes + * Matches any CopyBytes, subject to limits. + * * @property {(limits?: Limits) => Matcher} set * Matches any CopySet, subject to limits. *