descriptor-policy: initial implementation of wallet-policies

See bitcoin/bips#1389

Policies are a restriction on the general variable substitution
mechanism that wally already supports, combined with a shorthand
notation for path derivation.

1  - Key variables can only be named @n where n is an integer
2  - Key variable names must increase sequentially from 0 to n
3  - Key variables names must be followed by '/*', '/**', or '/<m;n>/*'
4  - Key variables can only be serialized BIP32 public keys without paths
5  - All key expressions to substitute must be unique
6  - Al least one key expression must be present
7  - Key variables must appear in the policy in order from 0 to n
     (back-references are allowed for repeated keys)
8  - All key expressions in a policy must be in the form of Key variables
9  - All key expression must share the same solved cardinality (keys
     using '/*', cannot be mixed with those using '/**' or '/<m;n>/*')
10 - The solved cardinality of a policy must be 1 or 2 (e.g. no combo())`.
11 - All repeated references to the same key must use distinct
     derivations.

This initial change implements and tests points 1-4.

This implementation will ignore the whitelisted expression lists given
in the BIP, and instead accept any valid descriptor that doesn't have a
solved cardinality greater than 2. See the above linked github PR discussion
for the rationale behind this decision.

Author: jgriffiths

include/wally_descriptor.h

@@ -15,6 +15,8 @@ struct wally_descriptor;
 #define WALLY_MINISCRIPT_TAPSCRIPT        0x01 /** Tapscript, use x-only pubkeys */
 #define WALLY_MINISCRIPT_ONLY             0x02 /** Only allow miniscript (not descriptor) expressions */
 #define WALLY_MINISCRIPT_REQUIRE_CHECKSUM 0x04 /** Require a checksum to be present */
+#define WALLY_MINISCRIPT_NO_UNUSED        0x08 /** Require all variable substitutions be used */
+#define WALLY_MINISCRIPT_POLICY           0x10 /** Only allow policy @n variable substitution */
 #define WALLY_MINISCRIPT_DEPTH_MASK       0xffff0000 /** Mask for limiting maximum depth */
 #define WALLY_MINISCRIPT_DEPTH_SHIFT      16 /** Shift to convert maximum depth to flags */
 

src/ctest/test_descriptor.c

@@ -51,6 +51,17 @@ static const struct wally_map g_key_map = {
     NULL
 };
 
+static struct wally_map_item g_policy_map_items[] = {
+    { B("@0"), B("xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL") }
+};
+
+static const struct wally_map g_policy_map = {
+    g_policy_map_items,
+    NUM_ELEMS(g_policy_map_items),
+    NUM_ELEMS(g_policy_map_items),
+    NULL
+};
+
 static const uint32_t g_miniscript_index_0 = 0;
 static const uint32_t g_miniscript_index_16 = 0x10;
 
@@ -968,6 +979,32 @@ static const struct descriptor_test {
         "5221038145454b87fc9ec3557478d6eadc2aea290b50f3c469b828abeb542ae8f8849d2102d2b36900396c9282fa14628566582f206a5dd0bcc8d5e892611806cafb0301f052ae",
         "y5pky4r2"
     },
+    /* Wallet policies https://github.com/bitcoin/bips/pull/1389 */
+    {
+        "policy - single asterisk reconciliation",
+        "pkh(mainnet_xpub/*)",
+        WALLY_NETWORK_BITCOIN_MAINNET, 0, 0, 0, NULL, 0,
+        "76a914bb57ca9e62c7084081edc68d2cbc9524a523784288ac",
+        "cp8r8rlg"
+    }, {
+        "policy - single asterisk",
+        "pkh(@0/*)", // Becomes "pkh(mainnet_xpub/*)" i.e. the test case above this
+        WALLY_NETWORK_BITCOIN_MAINNET, 0, 0, 0, NULL, WALLY_MINISCRIPT_POLICY,
+        "76a914bb57ca9e62c7084081edc68d2cbc9524a523784288ac",
+        "cp8r8rlg"
+    }, {
+        "policy - double asterisk",
+        "pkh(@0/**)", // Becomes "pkh(mainnet_xpub/<0;1>/*)"
+        WALLY_NETWORK_BITCOIN_MAINNET, 0, 0, 0, NULL, WALLY_MINISCRIPT_POLICY,
+        "76a9143099ad49dfdd021bf3748f7f858e0d1fa0b4f6f888ac",
+        "ydnzkve4"
+    }, {
+        "policy - multi-path",
+        "pkh(@0/<0;1>/*)", // Becomes "pkh(mainnet_xpub/<0;1>/*)"
+        WALLY_NETWORK_BITCOIN_MAINNET, 0, 0, 0, NULL, WALLY_MINISCRIPT_POLICY,
+        "76a9143099ad49dfdd021bf3748f7f858e0d1fa0b4f6f888ac",
+        "ydnzkve4"
+    },
     /*
      * Misc error cases (code coverage)
      */
@@ -1375,6 +1412,28 @@ static const struct descriptor_test {
         "descriptor - hardened xpub multi-path", /* TODO: Allow setting an xpriv into the descriptor */
         "pkh(mainnet_xpub/<0';1>)",
         WALLY_NETWORK_BITCOIN_MAINNET, 0, 0, 0, NULL, 0, NULL, ""
+    },
+    /* Wallet policy error cases */
+    {
+        "policy errchk - key with path",
+        "pkh(@0/0/*)",
+        WALLY_NETWORK_BITCOIN_MAINNET, 0, 0, 0, NULL,
+        WALLY_MINISCRIPT_POLICY, NULL, ""
+    }, {
+        "policy errchk - missing key postfix",
+        "pkh(@0)",
+        WALLY_NETWORK_BITCOIN_MAINNET, 0, 0, 0, NULL,
+        WALLY_MINISCRIPT_POLICY, NULL, ""
+    }, {
+        "policy errchk - terminal key postfix",
+        "@0",
+        WALLY_NETWORK_BITCOIN_MAINNET, 0, 0, 0, NULL,
+        WALLY_MINISCRIPT_POLICY, NULL, ""
+    }, {
+        "policy errchk - missing key number",
+        "@",
+        WALLY_NETWORK_BITCOIN_MAINNET, 0, 0, 0, NULL,
+        WALLY_MINISCRIPT_POLICY, NULL, ""
     }
 };
 
@@ -1823,10 +1882,12 @@ static bool check_descriptor_to_script(const struct descriptor_test* test)
     int expected_ret, ret, len_ret;
     uint32_t multi_index = 0;
     uint32_t child_num = test->child_num ? *test->child_num : 0, features;
+    const bool is_policy = test->flags & WALLY_MINISCRIPT_POLICY;
+    const struct wally_map *keys = is_policy ? &g_policy_map : &g_key_map;
 
     expected_ret = test->script ? WALLY_OK : WALLY_EINVAL;
 
-    ret = wally_descriptor_parse(test->descriptor, &g_key_map, test->network,
+    ret = wally_descriptor_parse(test->descriptor, keys, test->network,
                                  test->flags, &descriptor);
     if (expected_ret == WALLY_OK || ret == expected_ret) {
         /* For failure cases, we may fail when generating instead of parsing,

src/descriptor.c

@@ -17,7 +17,11 @@
 #define NUM_ELEMS(a) (sizeof(a) / sizeof(a[0]))
 #define MS_FLAGS_ALL (WALLY_MINISCRIPT_TAPSCRIPT | \
         WALLY_MINISCRIPT_ONLY | \
-        WALLY_MINISCRIPT_REQUIRE_CHECKSUM)
+        WALLY_MINISCRIPT_REQUIRE_CHECKSUM | \
+        WALLY_MINISCRIPT_NO_UNUSED | \
+        WALLY_MINISCRIPT_POLICY)
+#define MS_FLAGS_CANONICALIZE (WALLY_MINISCRIPT_REQUIRE_CHECKSUM | \
+        WALLY_MINISCRIPT_NO_UNUSED | WALLY_MINISCRIPT_POLICY)
 
 /* Properties and expressions definition */
 #define TYPE_NONE  0x00
@@ -270,6 +274,7 @@ static const struct addr_ver_t *addr_ver_from_family(
 static const struct ms_builtin_t *builtin_get(const ms_node *node);
 static int generate_script(ms_ctx *ctx, ms_node *node,
                            unsigned char *script, size_t script_len, size_t *written);
+static bool is_valid_policy_map(const struct wally_map *map_in);
 
 /* Wrapper for strtoll */
 static bool strtoll_n(const char *str, size_t str_len, int64_t *v)
@@ -355,34 +360,48 @@ static int generate_checksum(const char *str, size_t str_len, char *checksum_out
     return WALLY_OK;
 }
 
-static inline bool is_identifer_char(char c)
+typedef bool (*is_identifer_fn)(char c);
+
+static bool is_identifer_char(char c)
 {
     return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
 }
+static bool is_policy_start_char(char c) { return c == '@'; }
+static bool is_policy_identifer_char(char c) { return c >= '0' && c <= '9'; }
 
 static int canonicalize(const char *descriptor,
                         const struct wally_map *vars_in, uint32_t flags,
                         char **output)
 {
     const size_t VAR_MAX_NAME_LEN = 16;
+    is_identifer_fn is_id_start = is_identifer_char, is_id_char = is_identifer_char;
     size_t required_len = 0;
     const char *p = descriptor, *start;
     char *out;
 
     if (output)
         *output = NULL;
 
-    if (!descriptor || (flags & ~WALLY_MINISCRIPT_REQUIRE_CHECKSUM) || !output)
+    if (!descriptor || (flags & ~MS_FLAGS_CANONICALIZE) || !output)
         return WALLY_EINVAL;
 
+    if (flags & WALLY_MINISCRIPT_POLICY) {
+        if (!is_valid_policy_map(vars_in))
+            return WALLY_EINVAL; /* Invalid policy variables given */
+        is_id_start = is_policy_start_char;
+        is_id_char = is_policy_identifer_char;
+    }
+
     /* First, find the length of the canonicalized descriptor */
     while (*p && *p != '#') {
-        while (*p && *p != '#' && !is_identifer_char(*p)) {
+        while (*p && *p != '#' && !is_id_start(*p)) {
             ++required_len;
             ++p;
         }
-        start = p;
-        while (is_identifer_char(*p))
+        if (!is_id_start(*p))
+            break;
+        start = p++;
+        while (is_id_char(*p))
             ++p;
         if (p != start) {
             const bool starts_with_digit = *start >= '0' && *start <= '9';
@@ -394,36 +413,60 @@ static int canonicalize(const char *descriptor,
                 const struct wally_map_item *item;
                 item = wally_map_get(vars_in, (unsigned char*)start, lookup_len);
                 required_len += item ? item->value_len : lookup_len;
+                if (item && flags & WALLY_MINISCRIPT_POLICY) {
+                    if (*p++ != '/')
+                        return WALLY_EINVAL;
+                    ++required_len;
+                    if (*p == '<')
+                        continue;
+                    if (*p++ != '*')
+                        return WALLY_EINVAL;
+                    if (*p == '*') {
+                        ++p;
+                        required_len += strlen("<0;1>/*");
+                    } else {
+                        required_len += 1;
+                    }
+                }
             }
         }
     }
 
     if (!*p && (flags & WALLY_MINISCRIPT_REQUIRE_CHECKSUM))
         return WALLY_EINVAL; /* Checksum required but not present */
-
     if (!(*output = wally_malloc(required_len + 1 + DESCRIPTOR_CHECKSUM_LENGTH + 1)))
         return WALLY_ENOMEM;
 
     p = descriptor;
     out = *output;
     while (*p && *p != '#') {
-        while (*p && *p != '#' && !is_identifer_char(*p)) {
+        while (*p && *p != '#' && !is_id_start(*p)) {
             *out++ = *p++;
         }
-        start = p;
-        while (is_identifer_char(*p))
+        if (!is_id_start(*p))
+            break;
+        start = p++;
+        while (is_id_char(*p))
             ++p;
         if (p != start) {
             const bool is_number = *start >= '0' && *start <= '9';
             size_t lookup_len = p - start;
-            if (!vars_in || lookup_len > VAR_MAX_NAME_LEN || is_number) {
+            if (!vars_in || lookup_len > VAR_MAX_NAME_LEN || is_number)
                 memcpy(out, start, lookup_len);
-            } else {
+            else {
                 /* Lookup the potential identifier */
                 const struct wally_map_item *item;
                 item = wally_map_get(vars_in, (unsigned char*)start, lookup_len);
                 lookup_len = item ? item->value_len : lookup_len;
                 memcpy(out, item ? (char *)item->value : start, lookup_len);
+                if (item && flags & WALLY_MINISCRIPT_POLICY) {
+                    if (p[1] == '*' && p[2] == '*') {
+                        out += lookup_len;
+                        lookup_len = strlen("/<0;1>/*");
+                        memcpy(out, "/<0;1>/*", lookup_len);
+                        p += strlen("/**");
+                    }
+                }
             }
             out += lookup_len;
         }
@@ -2455,6 +2498,47 @@ static uint32_t get_max_depth(const char *miniscript, size_t miniscript_len)
     return depth == 1 ? max_depth : 0xffffffff;
 }
 
+static bool is_valid_policy_map(const struct wally_map *map_in)
+{
+    ms_ctx ctx;
+    ms_node* node;
+    int64_t v;
+    size_t i;
+    int ret = WALLY_OK;
+
+    if (!map_in || !map_in->num_items)
+        return WALLY_EINVAL; /* Must contain at least one key expression */
+
+    memset(&ctx, 0, sizeof(ctx));
+
+    for (i = 0; ret == WALLY_OK && i < map_in->num_items; ++i) {
+        const struct wally_map_item *item = &map_in->items[i];
+        if (!item->key || item->key_len < 2 || item->key[0] != '@' ||
+            !strtoll_n((const char *)item->key + 1, item->key_len - 1, &v) || v < 0)
+            ret = WALLY_EINVAL; /* Policy keys can only be @n */
+        else if ((size_t)v != i)
+            ret = WALLY_EINVAL; /* Must be sorted in order from 0-n */
+        else if (!item->value || !item->value_len)
+            ret = WALLY_EINVAL; /* No key value */
+        else if (!(node = wally_calloc(sizeof(*node))))
+            ret = WALLY_EINVAL;
+        else {
+            node->data = (const char*)item->value;
+            node->data_len = item->value_len;
+            if (analyze_miniscript_key(&ctx, 0, node, NULL) != WALLY_OK ||
+                node->kind != KIND_BIP32_PUBLIC_KEY ||
+                node->child_path_len) {
+                ret = WALLY_EINVAL; /* Only BIP32 xpubs are allowed */
+            } else if (ctx.features & (WALLY_MS_IS_MULTIPATH | WALLY_MS_IS_RANGED)) {
+                /* Range or multipath must be part of the expression, not the key */
+                ret = WALLY_EINVAL;
+            }
+        }
+        node_free(node);
+    }
+    return ret == WALLY_OK;
+}
+
 int wally_descriptor_parse(const char *miniscript,
                            const struct wally_map *vars_in,
                            uint32_t network, uint32_t flags,
@@ -2480,8 +2564,7 @@ int wally_descriptor_parse(const char *miniscript,
     ctx->addr_ver = addr_ver;
     ctx->num_variants = 1;
     ctx->num_multipaths = 1;
-    ret = canonicalize(miniscript, vars_in,
-                       flags & WALLY_MINISCRIPT_REQUIRE_CHECKSUM,
+    ret = canonicalize(miniscript, vars_in, flags & MS_FLAGS_CANONICALIZE,
                        &ctx->src);
     if (ret == WALLY_OK) {
         ctx->src_len = strlen(ctx->src);

src/test/test_descriptor.py

@@ -12,6 +12,9 @@
 
 MS_TAP = 0x1  # WALLY_MINISCRIPT_TAPSCRIPT
 MS_ONLY = 0x2 # WALLY_MINISCRIPT_ONLY
+REQUIRE_CHECKSUM = 0x4 # WALLY_MINISCRIPT_REQUIRE_CHECKSUM
+NO_UNUSED = 0x08 # WALLY_MINISCRIPT_NO_UNUSED
+POLICY    = 0x10 # WALLY_MINISCRIPT_POLICY
 
 MS_IS_RANGED = 0x1
 MS_IS_MULTIPATH = 0x2
@@ -26,7 +29,7 @@ def wally_map_from_dict(d):
     m = pointer(wally_map())
     assert(wally_map_init_alloc(len(d.keys()), None, m) == WALLY_OK)
     for k,v in d.items():
-        assert(wally_map_add(m, k, len(k), v, len(v)) == WALLY_OK)
+        assert(wally_map_add(m, utf8(k), len(k), utf8(v), len(v)) == WALLY_OK)
     return m
 
 
@@ -35,10 +38,10 @@ class DescriptorTests(unittest.TestCase):
     def test_parse_and_to_script(self):
         """Test parsing and script generation"""
         keys = wally_map_from_dict({
-            utf8('key_local'): utf8('038bc7431d9285a064b0328b6333f3a20b86664437b6de8f4e26e6bbdee258f048'),
-            utf8('key_remote'): utf8('03a22745365f673e658f0d25eb0afa9aaece858c6a48dfe37a67210c2e23da8ce7'),
-            utf8('key_revocation'): utf8('03b428da420cd337c7208ed42c5331ebb407bb59ffbe3dc27936a227c619804284'),
-            utf8('H'): utf8('d0721279e70d39fb4aa409b52839a0056454e3b5'), # HASH160(key_local)
+            'key_local': '038bc7431d9285a064b0328b6333f3a20b86664437b6de8f4e26e6bbdee258f048',
+            'key_remote': '03a22745365f673e658f0d25eb0afa9aaece858c6a48dfe37a67210c2e23da8ce7',
+            'key_revocation': '03b428da420cd337c7208ed42c5331ebb407bb59ffbe3dc27936a227c619804284',
+            'H': 'd0721279e70d39fb4aa409b52839a0056454e3b5', # HASH160(key_local)
         })
         script, script_len = make_cbuffer('00' * 256 * 2)
 
@@ -278,5 +281,40 @@ def test_features_and_depth(self):
                                      flags | (5 << 16), d)
         self.assertEqual(ret, WALLY_EINVAL)
 
+    def test_policy(self):
+        """Test policy parsing"""
+        FLAG_NO_UNUSED = 0x08
+        FLAG_POLICY    = 0x10
+
+        # Substitution variables
+        xpriv = 'xprvA2YKGLieCs6cWCiczALiH1jzk3VCCS5M1pGQfWPkamCdR9UpBgE2Gb8AKAyVjKHkz8v37avcfRjdcnP19dVAmZrvZQfvTcXXSAiFNQ6tTtU'
+        xpub1 = 'xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL'
+        xpub2 = 'xpub6AHA9hZDN11k2ijHMeS5QqHx2KP9aMBRhTDqANMnwVtdyw2TDYRmF8PjpvwUFcL1Et8Hj59S3gTSMcUQ5gAqTz3Wd8EsMTmF3DChhqPQBnU'
+
+        def make_keys(xpubs):
+            keys = {f'@{i}': xpub for i,xpub in enumerate(xpubs)}
+            return wally_map_from_dict(keys)
+
+        bad_args = [
+            # Raw pubkey
+            [POLICY, ['038bc7431d9285a064b0328b6333f3a20b86664437b6de8f4e26e6bbdee258f048']],
+            # Bip32 private key
+            [POLICY, [xpriv]],
+            # Keys must be in the form of @N
+            [POLICY, {'foo': xpub1}],
+            # Keys must start from 0
+            [POLICY, {'@1': xpub1}],
+            # Keys must be successive integers
+            [POLICY, {'@0': xpub1, '@2': xpub2}],
+            # Keys cannot have child paths
+            [POLICY, {'@0': f'{xpub1}/0'}],
+        ]
+        d = c_void_p()
+        for flags, key_items in bad_args:
+            keys = wally_map_from_dict(key_items) if type(key_items) is dict else make_keys(key_items)
+            ret = wally_descriptor_parse('pkh(@0/*)', keys, NETWORK_BTC_MAIN, POLICY, d)
+            self.assertEqual(ret, WALLY_EINVAL)
+            wally_map_free(keys)
+
 if __name__ == '__main__':
     unittest.main()

src/wasm_package/src/const.js

@@ -120,7 +120,9 @@ export const WALLY_HOST_COMMITMENT_LEN = 32;
 export const WALLY_MAX_OP_RETURN_LEN = 80; /* Maximum length of OP_RETURN data push */
 export const WALLY_MINISCRIPT_DEPTH_MASK = 0xffff0000; /** Mask for limiting maximum depth */
 export const WALLY_MINISCRIPT_DEPTH_SHIFT = 16; /** Shift to convert maximum depth to flags */
+export const WALLY_MINISCRIPT_NO_UNUSED = 0x08; /** Require all variable substitutions be used */
 export const WALLY_MINISCRIPT_ONLY = 0x02; /** Only allow miniscript (not descriptor) expressions */
+export const WALLY_MINISCRIPT_POLICY = 0x10; /** Only allow policy @n variable substitution */
 export const WALLY_MINISCRIPT_REQUIRE_CHECKSUM = 0x04; /** Require a checksum to be present */
 export const WALLY_MINISCRIPT_TAPSCRIPT = 0x01; /** Tapscript, use x-only pubkeys */
 export const WALLY_MS_CANONICAL_NO_CHECKSUM = 0x01; /** Do not include a checksum */