feat: allow choosing rust wasm contracts serialization format

package.json

@@ -19,7 +19,7 @@
     "mjs:burn:in:hell": "bash mjs-package.sh",
     "build:types": "tsc -b tsconfig.types.json",
     "bundle": "node bundle.js",
-    "build": "yarn build:cjs && yarn build:mjs && yarn mjs:burn:in:hell && yarn build:types && yarn bundle",
+    "build": "yarn run clean && yarn build:cjs && yarn build:mjs && yarn mjs:burn:in:hell && yarn build:types && yarn bundle",
     "format": "prettier --write 'src/**/*.ts'",
     "clean": "rimraf ./lib",
     "lint": "eslint . --ext .ts",

src/contract/deploy/CreateContract.ts

@@ -1,4 +1,5 @@
 import { JWKInterface } from 'arweave/node/lib/wallet';
+import { SerializationFormat } from 'core/modules/StateEvaluator';
 import { SignatureType } from '../../contract/Signature';
 import { Source } from './Source';
 
@@ -18,9 +19,10 @@ export const emptyTransfer: ArTransfer = {
   winstonQty: '0'
 };
 
-export interface CommonContractData {
+export interface CommonContractData<T extends SerializationFormat> {
   wallet: ArWallet | SignatureType;
-  initState: string | Buffer;
+  stateFormat: T;
+  initState: T extends SerializationFormat.JSON ? string : Buffer;
   tags?: Tags;
   transfer?: ArTransfer;
   data?: {
@@ -29,13 +31,13 @@ export interface CommonContractData {
   };
 }
 
-export interface ContractData extends CommonContractData {
+export interface ContractData<T extends SerializationFormat> extends CommonContractData<T> {
   src: string | Buffer;
   wasmSrcCodeDir?: string;
   wasmGlueCode?: string;
 }
 
-export interface FromSrcTxContractData extends CommonContractData {
+export interface FromSrcTxContractData<T extends SerializationFormat> extends CommonContractData<T> {
   srcTxId: string;
 }
 
@@ -44,10 +46,10 @@ export interface ContractDeploy {
   srcTxId?: string;
 }
 
-export interface CreateContract extends Source {
-  deploy(contractData: ContractData, disableBundling?: boolean): Promise<ContractDeploy>;
+export interface CreateContract<T extends SerializationFormat> extends Source {
+  deploy(contractData: ContractData<T>, disableBundling?: boolean): Promise<ContractDeploy>;
 
-  deployFromSourceTx(contractData: FromSrcTxContractData, disableBundling?: boolean): Promise<ContractDeploy>;
+  deployFromSourceTx(contractData: FromSrcTxContractData<T>, disableBundling?: boolean): Promise<ContractDeploy>;
 
   deployBundled(rawDataItem: Buffer): Promise<ContractDeploy>;
 }

src/contract/deploy/impl/DefaultCreateContract.ts

@@ -9,8 +9,10 @@ import { LoggerFactory } from '../../../logging/LoggerFactory';
 import { CreateContract, ContractData, ContractDeploy, FromSrcTxContractData, ArWallet } from '../CreateContract';
 import { SourceData, SourceImpl } from './SourceImpl';
 import { Buffer } from 'redstone-isomorphic';
+import { SerializationFormat } from 'core/modules/StateEvaluator';
+import { exhaustive } from 'utils/utils';
 
-export class DefaultCreateContract implements CreateContract {
+export class DefaultCreateContract<T extends SerializationFormat> implements CreateContract<T> {
   private readonly logger = LoggerFactory.INST.create('DefaultCreateContract');
   private readonly source: SourceImpl;
 
@@ -21,8 +23,11 @@ export class DefaultCreateContract implements CreateContract {
     this.source = new SourceImpl(this.warp);
   }
 
-  async deploy(contractData: ContractData, disableBundling?: boolean): Promise<ContractDeploy> {
-    const { wallet, initState, tags, transfer, data } = contractData;
+  async deploy<T extends SerializationFormat>(
+    contractData: ContractData<T>,
+    disableBundling?: boolean
+  ): Promise<ContractDeploy> {
+    const { wallet, stateFormat, initState, tags, transfer, data } = contractData;
 
     const effectiveUseBundler =
       disableBundling == undefined ? this.warp.definitionLoader.type() == 'warp' : !disableBundling;
@@ -38,6 +43,7 @@ export class DefaultCreateContract implements CreateContract {
       {
         srcTxId: srcTx.id,
         wallet,
+        stateFormat,
         initState,
         tags,
         transfer,
@@ -48,13 +54,13 @@ export class DefaultCreateContract implements CreateContract {
     );
   }
 
-  async deployFromSourceTx(
-    contractData: FromSrcTxContractData,
+  async deployFromSourceTx<T extends SerializationFormat>(
+    contractData: FromSrcTxContractData<T>,
     disableBundling?: boolean,
     srcTx: Transaction = null
   ): Promise<ContractDeploy> {
     this.logger.debug('Creating new contract from src tx');
-    const { wallet, srcTxId, initState, tags, transfer, data } = contractData;
+    const { wallet, srcTxId, stateFormat, initState, tags, transfer, data } = contractData;
     this.signature = new Signature(this.warp, wallet);
     const signer = this.signature.signer;
 
@@ -85,12 +91,34 @@ export class DefaultCreateContract implements CreateContract {
     contractTX.addTag(SmartWeaveTags.SDK, 'RedStone');
     if (data) {
       contractTX.addTag(SmartWeaveTags.CONTENT_TYPE, data['Content-Type']);
+      contractTX.addTag(SmartWeaveTags.INIT_STATE_FORMAT, stateFormat);
       contractTX.addTag(
         SmartWeaveTags.INIT_STATE,
         typeof initState === 'string' ? initState : new TextDecoder().decode(initState)
       );
     } else {
-      contractTX.addTag(SmartWeaveTags.CONTENT_TYPE, 'application/json');
+      let contentType: undefined | string;
+
+      switch (stateFormat) {
+        case SerializationFormat.JSON:
+          contentType = 'application/json';
+          break;
+        case SerializationFormat.MSGPACK:
+          // NOTE: There is still no officially registered Media Type for Messagepack and there are
+          // apparently multiple different versions used in the wild like:
+          // - application/msgpack
+          // - application/x-msgpack
+          // - application/x.msgpack
+          // - [...]
+          // See <https://github.com/msgpack/msgpack/issues/194>. I've decided to use the first one
+          // as it looks like the one that makes the most sense.
+          contentType = 'application/msgpack';
+          break;
+        default:
+          return exhaustive(stateFormat);
+      }
+
+      contractTX.addTag(SmartWeaveTags.CONTENT_TYPE, contentType);
     }
 
     if (this.warp.environment === 'testnet') {

src/core/SmartWeaveTags.ts

@@ -11,6 +11,7 @@ export enum SmartWeaveTags {
   SDK = 'SDK',
   MIN_FEE = 'Min-Fee',
   INIT_STATE = 'Init-State',
+  INIT_STATE_FORMAT = 'Init-State-Format',
   INIT_STATE_TX = 'Init-State-TX',
   INTERACT_WRITE = 'Interact-Write',
   WASM_LANG = 'Wasm-Lang',

src/core/Warp.ts

@@ -16,7 +16,7 @@ import { DefinitionLoader } from './modules/DefinitionLoader';
 import { ExecutorFactory } from './modules/ExecutorFactory';
 import { HandlerApi } from './modules/impl/HandlerExecutorFactory';
 import { InteractionsLoader } from './modules/InteractionsLoader';
-import { EvalStateResult, StateEvaluator } from './modules/StateEvaluator';
+import { EvalStateResult, SerializationFormat, StateEvaluator } from './modules/StateEvaluator';
 import { WarpBuilder } from './WarpBuilder';
 import { WarpPluginType, WarpPlugin, knownWarpPlugins } from './WarpPlugin';
 import { SortKeyCache } from '../cache/SortKeyCache';
@@ -39,7 +39,7 @@ export class Warp {
   /**
    * @deprecated createContract will be a private field, please use its methods directly e.g. await warp.deploy(...)
    */
-  readonly createContract: CreateContract;
+  readonly createContract: CreateContract<SerializationFormat>;
   readonly testing: Testing;
 
   private readonly plugins: Map<WarpPluginType, WarpPlugin<unknown, unknown>> = new Map();
@@ -73,11 +73,17 @@ export class Warp {
     return new HandlerBasedContract<State>(contractTxId, this, callingContract, innerCallData);
   }
 
-  async deploy(contractData: ContractData, disableBundling?: boolean): Promise<ContractDeploy> {
+  async deploy<T extends SerializationFormat>(
+    contractData: ContractData<T>,
+    disableBundling?: boolean
+  ): Promise<ContractDeploy> {
     return await this.createContract.deploy(contractData, disableBundling);
   }
 
-  async deployFromSourceTx(contractData: FromSrcTxContractData, disableBundling?: boolean): Promise<ContractDeploy> {
+  async deployFromSourceTx<T extends SerializationFormat>(
+    contractData: FromSrcTxContractData<T>,
+    disableBundling?: boolean
+  ): Promise<ContractDeploy> {
     return await this.createContract.deployFromSourceTx(contractData, disableBundling);
   }
 

src/core/modules/StateEvaluator.ts

@@ -1,3 +1,7 @@
+import { pack, unpack } from 'msgpackr';
+import stringify from 'safe-stable-stringify';
+import { exhaustive } from 'utils/utils';
+
 import { SortKeyCache, SortKeyCacheResult } from '../../cache/SortKeyCache';
 import { CurrentTx } from '../../contract/Contract';
 import { ExecutionContext } from '../../core/ExecutionContext';
@@ -138,8 +142,38 @@ export class DefaultEvaluationOptions implements EvaluationOptions {
   throwOnInternalWriteError = true;
 
   cacheEveryNInteractions = -1;
+
+  wasmSerializationFormat = SerializationFormat.JSON;
+}
+
+export enum SerializationFormat {
+  JSON = 'application/json',
+  MSGPACK = 'application/msgpack'
 }
 
+export const stringToSerializationFormat = (format: string): SerializationFormat => {
+  const formatTyped = format as SerializationFormat;
+
+  switch (formatTyped) {
+    case SerializationFormat.JSON:
+      return SerializationFormat.JSON;
+    case SerializationFormat.MSGPACK:
+      return SerializationFormat.MSGPACK;
+    default:
+      return exhaustive(formatTyped, `Unsupported serialization format: "${formatTyped}"`);
+  }
+};
+
+export const Serializers = {
+  [SerializationFormat.JSON]: stringify,
+  [SerializationFormat.MSGPACK]: pack
+} as const satisfies Record<SerializationFormat, unknown>;
+
+export const Deserializers = {
+  [SerializationFormat.JSON]: JSON.parse,
+  [SerializationFormat.MSGPACK]: unpack
+} as const satisfies Record<SerializationFormat, unknown>;
+
 // an interface for the contract EvaluationOptions - can be used to change the behaviour of some features.
 export interface EvaluationOptions {
   // whether exceptions from given transaction interaction should be ignored
@@ -214,4 +248,11 @@ export interface EvaluationOptions {
   // force SDK to cache the state after evaluating each N interactions
   // defaults to -1, which effectively turns off this feature
   cacheEveryNInteractions: number;
+
+  /**
+   * What serialization format should be used for the WASM<->JS bridge. Note that changing this
+   * currently only affects Rust smartcontracts. AssemblyScript and Go smartcontracts will always
+   * use JSON. Defaults to JSON.
+   */
+  wasmSerializationFormat: SerializationFormat;
 }

src/core/modules/impl/CacheableStateEvaluator.ts

@@ -6,7 +6,7 @@ import { ExecutionContextModifier } from '../../../core/ExecutionContextModifier
 import { GQLNodeInterface } from '../../../legacy/gqlResult';
 import { LoggerFactory } from '../../../logging/LoggerFactory';
 import { indent } from '../../../utils/utils';
-import { EvalStateResult } from '../StateEvaluator';
+import { EvalStateResult, SerializationFormat } from '../StateEvaluator';
 import { DefaultStateEvaluator } from './DefaultStateEvaluator';
 import { HandlerApi } from './HandlerExecutorFactory';
 import { genesisSortKey } from './LexicographicalInteractionsSorter';
@@ -35,11 +35,12 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator {
     currentTx: CurrentTx[]
   ): Promise<SortKeyCacheResult<EvalStateResult<State>>> {
     const cachedState = executionContext.cachedState;
+    const { wasmSerializationFormat: serializationFormat } = executionContext.evaluationOptions;
     if (cachedState && cachedState.sortKey == executionContext.requestedSortKey) {
       this.cLogger.info(
         `Exact cache hit for sortKey ${executionContext?.contractDefinition?.txId}:${cachedState.sortKey}`
       );
-      executionContext.handler?.initState(cachedState.cachedValue.state);
+      executionContext.handler?.initState(cachedState.cachedValue.state, serializationFormat);
       return cachedState;
     }
 
@@ -71,10 +72,10 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator {
     if (missingInteractions.length == 0) {
       this.cLogger.info(`No missing interactions ${contractTxId}`);
       if (cachedState) {
-        executionContext.handler?.initState(cachedState.cachedValue.state);
+        executionContext.handler?.initState(cachedState.cachedValue.state, serializationFormat);
         return cachedState;
       } else {
-        executionContext.handler?.initState(executionContext.contractDefinition.initState);
+        executionContext.handler?.initState(executionContext.contractDefinition.initState, serializationFormat);
         this.cLogger.debug('Inserting initial state into cache');
         const stateToCache = new EvalStateResult(executionContext.contractDefinition.initState, {}, {});
         // no real sort-key - as we're returning the initial state

src/core/modules/impl/ContractDefinitionLoader.ts

@@ -12,7 +12,8 @@ import { TagsParser } from './TagsParser';
 import { WasmSrc } from './wasm/WasmSrc';
 import { WarpEnvironment } from '../../Warp';
 import { SortKeyCache } from 'cache/SortKeyCache';
-import { unpack } from 'msgpackr';
+import { Deserializers, SerializationFormat, stringToSerializationFormat } from '../StateEvaluator';
+import { exhaustive } from 'utils/utils';
 
 const supportedSrcContentTypes = ['application/javascript', 'application/wasm'];
 
@@ -56,10 +57,7 @@ export class ContractDefinitionLoader implements DefinitionLoader {
     const minFee = this.tagsParser.getTag(contractTx, SmartWeaveTags.MIN_FEE);
     this.logger.debug('Tags decoding', benchmark.elapsed());
     benchmark.reset();
-    // TODO: It should be stored somewhere whether the initial state is stored as a JSON string or
-    // as a MsgPack binary. For the sake of this experiment, we assume it's always MsgPack.
-    // const initState = JSON.parse(await this.evalInitialState(contractTx));
-    const initState = unpack(await this.arweaveWrapper.txData(contractTx.id));
+    const initState = await this.evalInitialState<State>(contractTx);
     this.logger.debug('Parsing src and init state', benchmark.elapsed());
 
     const { src, srcBinary, srcWasmLang, contractType, metadata, srcTx } = await this.loadContractSource(
@@ -123,14 +121,42 @@ export class ContractDefinitionLoader implements DefinitionLoader {
     };
   }
 
-  private async evalInitialState(contractTx: Transaction): Promise<string> {
+  private async evalInitialState<State>(contractTx: Transaction): Promise<State> {
     if (this.tagsParser.getTag(contractTx, SmartWeaveTags.INIT_STATE)) {
-      return this.tagsParser.getTag(contractTx, SmartWeaveTags.INIT_STATE);
+      const format = stringToSerializationFormat(
+        this.tagsParser.getTag(contractTx, SmartWeaveTags.INIT_STATE_FORMAT) ?? 'application/json'
+      );
+      const initState = this.tagsParser.getTag(contractTx, SmartWeaveTags.INIT_STATE);
+
+      switch (format) {
+        case SerializationFormat.JSON:
+          return Deserializers[format](initState);
+        case SerializationFormat.MSGPACK:
+          return Deserializers[format](new TextEncoder().encode(initState));
+        default:
+          exhaustive(format);
+      }
     } else if (this.tagsParser.getTag(contractTx, SmartWeaveTags.INIT_STATE_TX)) {
       const stateTX = this.tagsParser.getTag(contractTx, SmartWeaveTags.INIT_STATE_TX);
-      return this.arweaveWrapper.txDataString(stateTX);
+
+      return this.getInitialStateFromTx(await this.arweave.transactions.get(stateTX));
     } else {
-      return this.arweaveWrapper.txDataString(contractTx.id);
+      return this.getInitialStateFromTx(contractTx);
+    }
+  }
+
+  private async getInitialStateFromTx<State>(tx: Transaction): Promise<State> {
+    const format = stringToSerializationFormat(
+      this.tagsParser.getTag(tx, SmartWeaveTags.CONTENT_TYPE) ?? 'application/json'
+    );
+
+    switch (format) {
+      case SerializationFormat.JSON:
+        return Deserializers[format](await this.arweaveWrapper.txDataString(tx.id));
+      case SerializationFormat.MSGPACK:
+        return Deserializers[format](await this.arweaveWrapper.txData(tx.id));
+      default:
+        exhaustive(format);
     }
   }
 

src/core/modules/impl/DefaultStateEvaluator.ts

@@ -53,16 +53,21 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
     executionContext: ExecutionContext<State, HandlerApi<State>>,
     currentTx: CurrentTx[]
   ): Promise<SortKeyCacheResult<EvalStateResult<State>>> {
-    const { ignoreExceptions, stackTrace, internalWrites, cacheEveryNInteractions } =
-      executionContext.evaluationOptions;
+    const {
+      ignoreExceptions,
+      stackTrace,
+      internalWrites,
+      cacheEveryNInteractions,
+      wasmSerializationFormat: serializationFormat
+    } = executionContext.evaluationOptions;
     const { contract, contractDefinition, sortedInteractions, warp } = executionContext;
 
     let currentState = baseState.state;
     let currentSortKey = null;
     const validity = baseState.validity;
     const errorMessages = baseState.errorMessages;
 
-    executionContext?.handler.initState(currentState);
+    executionContext?.handler.initState(currentState, serializationFormat);
 
     const depth = executionContext.contract.callDepth();
 
@@ -76,7 +81,7 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
     let lastConfirmedTxState: { tx: GQLNodeInterface; state: EvalStateResult<State> } = null;
 
     const missingInteractionsLength = missingInteractions.length;
-    executionContext.handler.initState(currentState);
+    executionContext.handler.initState(currentState, serializationFormat);
 
     const evmSignatureVerificationPlugin = warp.hasPlugin('evm-signature-verification')
       ? warp.loadPlugin<GQLNodeInterface, Promise<boolean>>('evm-signature-verification')
@@ -166,7 +171,7 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
         if (newState !== null) {
           currentState = newState.cachedValue.state;
           // we need to update the state in the wasm module
-          executionContext?.handler.initState(currentState);
+          executionContext?.handler.initState(currentState, serializationFormat);
 
           validity[missingInteraction.id] = newState.cachedValue.validity[missingInteraction.id];
           if (newState.cachedValue.errorMessages?.[missingInteraction.id]) {