Skip to content

Commit

Permalink
Merge pull request #6814 from Agoric/6468-swingstore-getnext
Browse files Browse the repository at this point in the history 
swingstore+kernel: replace getKeys/vatstoreGetAfter with getNext
  • Loading branch information
@mergify
mergify[bot] committed Jan 24, 2023
2 parents d235463 + a3ade8d commit b2a96d4
Show file tree
Hide file tree
Showing 39 changed files with 2,064 additions and 2,804 deletions.
35 changes: 8 additions & 27 deletions packages/SwingSet/src/kernel/deviceTranslator.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,24 +111,10 @@ export function makeDSTranslator(deviceID, deviceName, kernelKeeper) {
return harden(['vatstoreSet', deviceID, key, value]);
}

function translateVatstoreGetAfter(priorKey, lowerBound, upperBound) {
if (priorKey !== '') {
assertValidVatstoreKey(priorKey);
}
assertValidVatstoreKey(lowerBound);
if (upperBound) {
assertValidVatstoreKey(upperBound);
}
kdebug(
`syscall[${deviceID}].vatstoreGetAfter(${priorKey}, ${lowerBound}, ${upperBound})`,
);
return harden([
'vatstoreGetAfter',
deviceID,
priorKey,
lowerBound,
upperBound,
]);
function translateVatstoreGetNextKey(priorKey) {
assertValidVatstoreKey(priorKey);
kdebug(`syscall[${deviceID}].vatstoreGetNextKey(${priorKey})`);
return harden(['vatstoreGetNextKey', deviceID, priorKey]);
}

function translateVatstoreDelete(key) {
Expand Down Expand Up @@ -165,8 +151,8 @@ export function makeDSTranslator(deviceID, deviceName, kernelKeeper) {
return translateVatstoreGet(...args);
case 'vatstoreSet':
return translateVatstoreSet(...args);
case 'vatstoreGetAfter':
return translateVatstoreGetAfter(...args);
case 'vatstoreGetNextKey':
return translateVatstoreGetNextKey(...args);
case 'vatstoreDelete':
return translateVatstoreDelete(...args);
case 'callKernelHook':
Expand Down Expand Up @@ -195,13 +181,8 @@ function makeKRTranslator(deviceID, kernelKeeper) {
return harden(['ok', undefined]);
}
}
case 'vatstoreGetAfter': {
if (resultData) {
assert(Array.isArray(resultData));
return harden(['ok', resultData]);
} else {
return harden(['ok', undefined]);
}
case 'vatstoreGetNextKey': {
return harden(['ok', resultData]);
}
case 'callKernelHook': {
insistCapData(resultData);
Expand Down
169 changes: 23 additions & 146 deletions packages/SwingSet/src/kernel/kernelSyscall.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,32 +40,14 @@ export function makeKernelSyscallHandler(tools) {
return `${vatID}.vs.${key}`;
}

function descopeVatstoreKey(key) {
return key.replace(/^([^.]+)\.vs\.(.+)$/, '$2');
function descopeVatstoreKey(vatID, dbKey) {
const prefix = `${vatID}.vs.`;
assert(dbKey.startsWith(prefix), `${dbKey} must start with ${prefix}`);
return dbKey.slice(prefix.length);
}

let workingPriorKey;
let workingLowerBound;
let workingUpperBound;
/** @type {IterableIterator<string> | undefined} */
let workingKeyIterator;

/** @param {boolean} [done] */
function clearVatStoreIteration(done = false) {
try {
if (
!done &&
workingKeyIterator &&
typeof workingKeyIterator.return === 'function'
) {
workingKeyIterator.return();
}
} finally {
workingKeyIterator = undefined;
workingPriorKey = undefined;
workingLowerBound = undefined;
workingUpperBound = undefined;
}
function vatstoreKeyInRange(vatID, key) {
return key.startsWith(`${vatID}.vs.`);
}

/**
Expand Down Expand Up @@ -93,137 +75,33 @@ export function makeKernelSyscallHandler(tools) {
const actualKey = vatstoreKeyKey(vatID, key);
kernelKeeper.incStat('syscalls');
kernelKeeper.incStat('syscallVatstoreSet');
clearVatStoreIteration();
kvStore.set(actualKey, value);
return OKNULL;
}

/**
* Execute one step of iteration over a range of vatstore keys.
* Get the next vatstore key after 'priorKey' (in lexicographic
* order, as defined by swingstore's getNextKey() function), or
* undefined if this vat's portion of the vatstore has no more keys
*
* @param {string} vatID The vat whose vatstore is being iterated
* @param {string} priorKey The key that was returned by the prior cycle of
* the iteration, or '' if this is the first cycle
* @param {string} lowerBound The lower bound of the iteration range
* @param {string} [upperBound] The uppper bound of the iteration range. If
* omitted, this defaults to a string equivalent to the `lowerBound` string
* with its rightmost character replaced by the lexically next character.
* For example, if `lowerBound` is 'hello', then `upperBound` would default
* to 'hellp')
*
* @returns {['ok', [string, string]|[undefined, undefined]]} A pair of a
* status code and a result value. In the case of this operation, the
* status code is always 'ok'. The result value is a pair of the key and
* value of the first vatstore entry whose key is lexically greater than
* both `priorKey` and `lowerBound`. If there are no such entries or if the
* first such entry has a key that is lexically greater than or equal to
* `upperBound`, then result value is a pair of undefineds instead,
* signalling the end of iteration.
*
* Usage notes:
*
* Iteration is accomplished by repeatedly calling `vatstoreGetAfter` in a
* loop, each time pass the key that was returned in the previous iteration,
* until undefined is returned. While the initial value for `priorKey` is
* conventionally the empty string, in fact any value that is less than
* `lowerBound` may be used to the same effect.
*
* Keys in the vatstore are arbitrary strings, but subsets of the keyspace are
* often organized hierarchically. This iteration API allows simple iteration
* over an explicit key range or iteration over the set of keys with a given
* prefix, depending on how you use it:
*
* Explicitly providing both a lower and an upper bound will enable iteration
* over the key range `lowerBound` <= key < `upperBound`
* @param {string} priorKey A key before the desired key
*
* Providing only the lower bound while letting the upper bound default will
* iterate over all keys that have `lowerBound` as a prefix.
*
* For example, if the stored keys are:
* bar
* baz
* foocount
* foopriority
* joober
* plugh.3
* plugh.47
* plugh.8
* zot
*
* Then the bounds would iterate over the key sequence
* ---------------- -----------------------------------
* 'bar', 'goomba' bar, baz, foocount, foopriority
* 'bar', 'joober' bar, baz, foocount, foopriority
* 'foo' foocount, foopriority
* 'plugh.' plugh.3, plugh.47, plugh.8
* 'baz', 'plugh' baz, foocount, foopriority, joober
* 'bar', '~' bar, baz, foocount, foopriority, joober, plugh.3, plugh.47, plugh.8, zot
* @returns {['ok', string|null]} A pair of a status code and
* a result value. In the case of this operation, the status code
* is always 'ok'. The result value is either a key or null.
*/
function vatstoreGetAfter(vatID, priorKey, lowerBound, upperBound) {
const actualPriorKey = vatstoreKeyKey(vatID, priorKey);
const actualLowerBound = vatstoreKeyKey(vatID, lowerBound);
let actualUpperBound;
if (upperBound) {
actualUpperBound = vatstoreKeyKey(vatID, upperBound);
} else {
const lastChar = String.fromCharCode(
actualLowerBound.slice(-1).charCodeAt(0) + 1,
);
actualUpperBound = `${actualLowerBound.slice(0, -1)}${lastChar}`;
}
function vatstoreGetNextKey(vatID, priorKey) {
const dbPriorKey = vatstoreKeyKey(vatID, priorKey);
kernelKeeper.incStat('syscalls');
kernelKeeper.incStat('syscallVatstoreGetAfter');
/** @type {IteratorResult<string>} */
let nextIter;
// Note that the working key iterator will be invalidated if the parameters
// to `vatstoreGetAfter` don't correspond to the working key iterator's
// belief about what iteration was in progress. In particular, the bounds
// incorporate the vatID. Additionally, when this syscall is used for
// iteration over a collection, the bounds also incorporate the collection
// ID. This ensures that uncoordinated concurrent iterations cannot
// interfere with each other. If such concurrent iterations *do* happen,
// there will be a modest performance cost since the working key iterator
// will have to be regenerated each time, but we expect this to be a rare
// case since the normal use pattern is a single iteration in a loop within
// a single crank.
if (
workingPriorKey === actualPriorKey &&
workingLowerBound === actualLowerBound &&
workingUpperBound === actualUpperBound &&
workingKeyIterator
) {
nextIter = workingKeyIterator.next();
} else {
clearVatStoreIteration();
let startKey;
if (priorKey === '') {
startKey = actualLowerBound;
} else {
startKey = actualPriorKey;
}
assert(actualLowerBound <= startKey);
assert(actualLowerBound < actualUpperBound);
assert(startKey < actualUpperBound);
workingLowerBound = actualLowerBound;
workingUpperBound = actualUpperBound;
workingKeyIterator = kvStore.getKeys(startKey, actualUpperBound);
// @ts-ignore some resolution thinks this is undefined
nextIter = workingKeyIterator.next();
if (!nextIter.done && nextIter.value === actualPriorKey) {
// @ts-ignore some resolution thinks this is undefined
nextIter = workingKeyIterator.next();
kernelKeeper.incStat('syscallVatstoreGetNextKey');
const nextDbKey = kvStore.getNextKey(dbPriorKey);
if (nextDbKey) {
if (vatstoreKeyInRange(vatID, nextDbKey)) {
return harden(['ok', descopeVatstoreKey(vatID, nextDbKey)]);
}
}
if (nextIter.done) {
clearVatStoreIteration(true);
return harden(['ok', [undefined, undefined]]);
} else {
const nextKey = nextIter.value;
const resultValue = kvStore.get(nextKey);
workingPriorKey = nextKey;
const resultKey = descopeVatstoreKey(nextKey);
return harden(['ok', [resultKey, resultValue]]);
}
return harden(['ok', null]);
}

/**
Expand All @@ -237,7 +115,6 @@ export function makeKernelSyscallHandler(tools) {
const actualKey = vatstoreKeyKey(vatID, key);
kernelKeeper.incStat('syscalls');
kernelKeeper.incStat('syscallVatstoreDelete');
clearVatStoreIteration();
kvStore.delete(actualKey);
return OKNULL;
}
Expand Down Expand Up @@ -374,9 +251,9 @@ export function makeKernelSyscallHandler(tools) {
const [_, ...args] = ksc;
return vatstoreSet(...args);
}
case 'vatstoreGetAfter': {
case 'vatstoreGetNextKey': {
const [_, ...args] = ksc;
return vatstoreGetAfter(...args);
return vatstoreGetNextKey(...args);
}
case 'vatstoreDelete': {
const [_, ...args] = ksc;
Expand Down
6 changes: 3 additions & 3 deletions packages/SwingSet/src/kernel/metrics.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ export const KERNEL_STATS_SUM_METRICS = /** @type {const} */ ([
description: 'Total number of SwingSet vatstore set kernel calls',
},
{
key: 'syscallVatstoreGetAfter',
name: 'swingset_syscall_vatstore_getAfter_total',
description: 'Total number of SwingSet vatstore getAfter kernel calls',
key: 'syscallVatstoreGetNextKey',
name: 'swingset_syscall_vatstore_getNextKey_total',
description: 'Total number of SwingSet vatstore getNextKey kernel calls',
},
{
key: 'syscallVatstoreDelete',
Expand Down
17 changes: 7 additions & 10 deletions packages/SwingSet/src/kernel/state/deviceKeeper.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { assert, Fail } from '@agoric/assert';
import { parseKernelSlot } from '../parseKernelSlots.js';
import { makeVatSlot, parseVatSlot } from '../../lib/parseVatSlots.js';
import { insistDeviceID } from '../../lib/id.js';
import { enumeratePrefixedKeys } from './storageHelper.js';

const FIRST_DEVICE_IMPORTED_OBJECT_ID = 10n;
const FIRST_DEVICE_IMPORTED_DEVICE_ID = 20n;
Expand Down Expand Up @@ -188,16 +189,12 @@ export function makeDeviceKeeper(kvStore, deviceID, tools) {
/** @type {Array<[string, string, string]>} */
const res = [];
const prefix = `${deviceID}.c.`;
for (const k of kvStore.getKeys(prefix, `${deviceID}.c/`)) {
// The bounds passed to getKeys() here work because '/' is the next
// character in ASCII after '.'
if (k.startsWith(prefix)) {
const slot = k.slice(prefix.length);
if (!slot.startsWith('k')) {
const devSlot = slot;
const kernelSlot = kvStore.get(k);
res.push([kernelSlot, deviceID, devSlot]);
}
for (const k of enumeratePrefixedKeys(kvStore, prefix)) {
const slot = k.slice(prefix.length);
if (!slot.startsWith('k')) {
const devSlot = slot;
const kernelSlot = kvStore.get(k);
res.push([kernelSlot, deviceID, devSlot]);
}
}
return harden(res);
Expand Down
29 changes: 15 additions & 14 deletions packages/SwingSet/src/kernel/state/kernelKeeper.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,11 @@ import {
import { kdebug } from '../../lib/kdebug.js';
import { KERNEL_STATS_METRICS } from '../metrics.js';
import { makeKernelStats } from './stats.js';
import { getPrefixedValues, deletePrefixedKeys } from './storageHelper.js';
import {
enumeratePrefixedKeys,
getPrefixedValues,
deletePrefixedKeys,
} from './storageHelper.js';

const enableKernelGC = true;

Expand Down Expand Up @@ -801,8 +805,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
// used.

// first, scan for exported objects, which must be orphaned
for (const k of kvStore.getKeys(`${vatID}.c.o+`, `${vatID}.c.o,`)) {
assert(k.startsWith(exportPrefix), k);
for (const k of enumeratePrefixedKeys(kvStore, exportPrefix)) {
// The void for an object exported by a vat will always be of the form
// `o+NN`. The '+' means that the vat exported the object (rather than
// importing it) and therefore the object is owned by (i.e., within) the
Expand All @@ -815,19 +818,17 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
}

// then scan for imported objects, which must be decrefed
for (const k of kvStore.getKeys(`${vatID}.c.o-`, `${vatID}.c.o.`)) {
assert(k.startsWith(importPrefix), k);
for (const k of enumeratePrefixedKeys(kvStore, importPrefix)) {
// abandoned imports: delete the clist entry as if the vat did a
// drop+retire
const kref = kvStore.get(k) || assert.fail('getKeys ensures get');
const kref = kvStore.get(k) || assert.fail('getNextKey ensures get');
const vref = k.slice(`${vatID}.c.`.length);
vatKeeper.deleteCListEntry(kref, vref);
// that will also delete both db keys
}

// now find all orphaned promises, which must be rejected
for (const k of kvStore.getKeys(`${vatID}.c.p`, `${vatID}.c.p.`)) {
assert(k.startsWith(promisePrefix), k);
for (const k of enumeratePrefixedKeys(kvStore, promisePrefix)) {
// The vpid for a promise imported or exported by a vat (and thus
// potentially a promise for which the vat *might* be the decider) will
// always be of the form `p+NN` or `p-NN`. The corresponding vpid->kpid
Expand All @@ -845,7 +846,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
}

// now loop back through everything and delete it all
for (const k of kvStore.getKeys(`${vatID}.`, `${vatID}/`)) {
for (const k of enumeratePrefixedKeys(kvStore, `${vatID}.`)) {
kvStore.delete(k);
}

Expand All @@ -861,7 +862,7 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {
kvStore.set(DYNAMIC_IDS_KEY, JSON.stringify(newDynamicVatIDs));
} else {
kdebug(`removing static vat ${vatID}`);
for (const k of kvStore.getKeys('vat.name.', 'vat.name/')) {
for (const k of enumeratePrefixedKeys(kvStore, 'vat.name.')) {
if (kvStore.get(k) === vatID) {
kvStore.delete(k);
const VAT_NAMES_KEY = 'vat.names';
Expand Down Expand Up @@ -1103,19 +1104,19 @@ export default function makeKernelKeeper(kernelStorage, kernelSlog) {

function getStaticVats() {
const result = [];
for (const k of kvStore.getKeys('vat.name.', 'vat.name/')) {
for (const k of enumeratePrefixedKeys(kvStore, 'vat.name.')) {
const name = k.slice(9);
const vatID = kvStore.get(k) || assert.fail('getKeys ensures get');
const vatID = kvStore.get(k) || assert.fail('getNextKey ensures get');
result.push([name, vatID]);
}
return result;
}

function getDevices() {
const result = [];
for (const k of kvStore.getKeys('device.name.', 'device.name/')) {
for (const k of enumeratePrefixedKeys(kvStore, 'device.name.')) {
const name = k.slice(12);
const deviceID = kvStore.get(k) || assert.fail('getKeys ensures get');
const deviceID = kvStore.get(k) || assert.fail('getNextKey ensures get');
result.push([name, deviceID]);
}
return result;
Expand Down

0 comments on commit b2a96d4

Please sign in to comment.