py: Implement positional and keyword args via * and **.

Extends previous implementation with * for function calls to * and **
for both function and method calls.

py/runtime.c

@@ -7,6 +7,7 @@
 #include "mpconfig.h"
 #include "qstr.h"
 #include "obj.h"
+#include "objtuple.h"
 #include "objmodule.h"
 #include "parsenum.h"
 #include "runtime0.h"
@@ -503,6 +504,148 @@ mp_obj_t mp_call_method_n_kw(uint n_args, uint n_kw, const mp_obj_t *args) {
     return mp_call_function_n_kw(args[0], n_args + adjust, n_kw, args + 2 - adjust);
 }
 
+mp_obj_t mp_call_method_n_kw_var(bool have_self, uint n_args_n_kw, const mp_obj_t *args, mp_obj_t pos_seq, mp_obj_t kw_dict) {
+    mp_obj_t fun = *args++;
+    mp_obj_t self = MP_OBJ_NULL;
+    if (have_self) {
+        self = *args++; // may be MP_OBJ_NULL
+    }
+    uint n_args = n_args_n_kw & 0xff;
+    uint n_kw = (n_args_n_kw >> 8) & 0xff;
+
+    DEBUG_OP_printf("call method var (fun=%p, self=%p, n_args=%u, n_kw=%u, args=%p, seq=%p, dict=%p)\n", fun, self, n_args, n_kw, args, pos_seq, kw_dict);
+
+    // We need to create the following array of objects:
+    //     args[0 .. n_args]  unpacked(pos_seq)  args[n_args .. n_args + 2 * n_kw]  unpacked(kw_dict)
+    // TODO: optimize one day to avoid constructing new arg array? Will be hard.
+
+    // The new args array
+    mp_obj_t *args2;
+    uint args2_alloc;
+    uint args2_len = 0;
+
+    // Try to get a hint for the size of the kw_dict
+    uint kw_dict_len = 0;
+    if (kw_dict != MP_OBJ_NULL && MP_OBJ_IS_TYPE(kw_dict, &mp_type_dict)) {
+        kw_dict_len = mp_obj_dict_len(kw_dict);
+    }
+
+    // Extract the pos_seq sequence to the new args array.
+    // Note that it can be arbitrary iterator.
+    if (pos_seq == MP_OBJ_NULL) {
+        // no sequence
+
+        // allocate memory for the new array of args
+        args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len);
+        args2 = m_new(mp_obj_t, args2_alloc);
+
+        // copy the self
+        if (self != MP_OBJ_NULL) {
+            args2[args2_len++] = self;
+        }
+
+        // copy the fixed pos args
+        m_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
+        args2_len += n_args;
+
+    } else if (MP_OBJ_IS_TYPE(pos_seq, &mp_type_tuple) || MP_OBJ_IS_TYPE(pos_seq, &mp_type_list)) {
+        // optimise the case of a tuple and list
+
+        // get the items
+        uint len;
+        mp_obj_t *items;
+        mp_obj_get_array(pos_seq, &len, &items);
+
+        // allocate memory for the new array of args
+        args2_alloc = 1 + n_args + len + 2 * (n_kw + kw_dict_len);
+        args2 = m_new(mp_obj_t, args2_alloc);
+
+        // copy the self
+        if (self != MP_OBJ_NULL) {
+            args2[args2_len++] = self;
+        }
+
+        // copy the fixed and variable position args
+        m_seq_cat(args2 + args2_len, args, n_args, items, len, mp_obj_t);
+        args2_len += n_args + len;
+
+    } else {
+        // generic iterator
+
+        // allocate memory for the new array of args
+        args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len) + 3;
+        args2 = m_new(mp_obj_t, args2_alloc);
+
+        // copy the self
+        if (self != MP_OBJ_NULL) {
+            args2[args2_len++] = self;
+        }
+
+        // copy the fixed position args
+        m_seq_copy(args2 + args2_len, args, n_args, mp_obj_t);
+
+        // extract the variable position args from the iterator
+        mp_obj_t iterable = mp_getiter(pos_seq);
+        mp_obj_t item;
+        while ((item = mp_iternext(iterable)) != MP_OBJ_NULL) {
+            if (args2_len >= args2_alloc) {
+                args2 = m_renew(mp_obj_t, args2, args2_alloc, args2_alloc * 2);
+                args2_alloc *= 2;
+            }
+            args2[args2_len++] = item;
+        }
+    }
+
+    // The size of the args2 array now is the number of positional args.
+    uint pos_args_len = args2_len;
+
+    // Copy the fixed kw args.
+    m_seq_copy(args2 + args2_len, args + n_args, 2 * n_kw, mp_obj_t);
+    args2_len += 2 * n_kw;
+
+    // Extract (key,value) pairs from kw_dict dictionary and append to args2.
+    // Note that it can be arbitrary iterator.
+    if (kw_dict == MP_OBJ_NULL) {
+        // pass
+    } else if (MP_OBJ_IS_TYPE(kw_dict, &mp_type_dict)) {
+        // dictionary
+        mp_map_t *map = mp_obj_dict_get_map(kw_dict);
+        assert(args2_len + 2 * map->used <= args2_alloc); // should have enough, since kw_dict_len is in this case hinted correctly above
+        for (uint i = 0; i < map->alloc; i++) {
+            if (map->table[i].key != MP_OBJ_NULL) {
+                args2[args2_len++] = map->table[i].key;
+                args2[args2_len++] = map->table[i].value;
+            }
+        }
+    } else {
+        // generic mapping
+        // TODO is calling 'items' on the mapping the correct thing to do here?
+        mp_obj_t dest[2];
+        mp_load_method(kw_dict, MP_QSTR_items, dest);
+        mp_obj_t iterable = mp_getiter(mp_call_method_n_kw(0, 0, dest));
+        mp_obj_t item;
+        while ((item = mp_iternext(iterable)) != MP_OBJ_NULL) {
+            if (args2_len + 1 >= args2_alloc) {
+                uint new_alloc = args2_alloc * 2;
+                if (new_alloc < 4) {
+                    new_alloc = 4;
+                }
+                args2 = m_renew(mp_obj_t, args2, args2_alloc, new_alloc);
+                args2_alloc = new_alloc;
+            }
+            mp_obj_t *items;
+            mp_obj_get_array_fixed_n(item, 2, &items);
+            args2[args2_len++] = items[0];
+            args2[args2_len++] = items[1];
+        }
+    }
+
+    mp_obj_t res = mp_call_function_n_kw(fun, pos_args_len, (args2_len - pos_args_len) / 2, args2);
+    m_del(mp_obj_t, args2, args2_alloc);
+
+    return res;
+}
+
 mp_obj_t mp_build_tuple(int n_args, mp_obj_t *items) {
     return mp_obj_new_tuple(n_args, items);
 }

py/runtime.h

@@ -37,6 +37,7 @@ mp_obj_t mp_call_function_2(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2);
 mp_obj_t mp_call_function_n_kw_for_native(mp_obj_t fun_in, uint n_args_kw, const mp_obj_t *args);
 mp_obj_t mp_call_function_n_kw(mp_obj_t fun, uint n_args, uint n_kw, const mp_obj_t *args);
 mp_obj_t mp_call_method_n_kw(uint n_args, uint n_kw, const mp_obj_t *args);
+mp_obj_t mp_call_method_n_kw_var(bool have_self, uint n_args_n_kw, const mp_obj_t *args, mp_obj_t pos_seq, mp_obj_t kw_dict);
 
 mp_obj_t mp_build_tuple(int n_args, mp_obj_t *items);
 mp_obj_t mp_build_list(int n_args, mp_obj_t *items);

py/vm.c

@@ -11,7 +11,6 @@
 #include "bc0.h"
 #include "bc.h"
 #include "objgenerator.h"
-#include "objtuple.h"
 
 // Value stack grows up (this makes it incompatible with native C stack, but
 // makes sure that arguments to functions are in natural order arg1..argN
@@ -672,31 +671,39 @@ unwind_jump:
                         SET_TOP(mp_call_function_n_kw(*sp, unum & 0xff, (unum >> 8) & 0xff, sp + 1));
                         break;
 
-                    case MP_BC_CALL_FUNCTION_VAR: {
+                    case MP_BC_CALL_FUNCTION_VAR:
                         DECODE_UINT;
                         // unum & 0xff == n_positional
                         // (unum >> 8) & 0xff == n_keyword
                         // We have folowing stack layout here:
-                        // arg0 arg1 ... kw0 val0 kw1 val1 ... seq <- TOS
-                        // We need to splice seq after all positional args and before kwargs
-                        // TODO: optimize one day to avoid constructing new arg array? Will be hard.
-                        mp_obj_t seq = POP();
-                        int total_stack_args = (unum & 0xff) + ((unum >> 7) & 0x1fe);
-                        sp -= total_stack_args;
-
-                        // Convert vararg sequence to tuple. Note that it can be arbitrary iterator.
-                        // This is null call for tuple, and TODO: we actually could optimize case of list.
-                        mp_obj_tuple_t *varargs = mp_obj_tuple_make_new(MP_OBJ_NULL, 1, 0, &seq);
+                        // fun arg0 arg1 ... kw0 val0 kw1 val1 ... seq <- TOS
+                        obj1 = POP();
+                        sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe);
+                        SET_TOP(mp_call_method_n_kw_var(false, unum, sp, obj1, MP_OBJ_NULL));
+                        break;
 
-                        int pos_args_len = (unum & 0xff) + varargs->len;
-                        mp_obj_t *args = m_new(mp_obj_t, total_stack_args + varargs->len);
-                        m_seq_cat(args, sp + 1, unum & 0xff, varargs->items, varargs->len, mp_obj_t);
-                        m_seq_copy(args + pos_args_len, sp + (unum & 0xff) + 1, ((unum >> 7) & 0x1fe), mp_obj_t);
+                    case MP_BC_CALL_FUNCTION_KW:
+                        DECODE_UINT;
+                        // unum & 0xff == n_positional
+                        // (unum >> 8) & 0xff == n_keyword
+                        // We have folowing stack layout here:
+                        // fun arg0 arg1 ... kw0 val0 kw1 val1 ... dict <- TOS
+                        obj1 = POP();
+                        sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe);
+                        SET_TOP(mp_call_method_n_kw_var(false, unum, sp, MP_OBJ_NULL, obj1));
+                        break;
 
-                        SET_TOP(mp_call_function_n_kw(*sp, pos_args_len, (unum >> 8) & 0xff, args));
-                        m_del(mp_obj_t, args, total_stack_args + varargs->len);
+                    case MP_BC_CALL_FUNCTION_VAR_KW:
+                        DECODE_UINT;
+                        // unum & 0xff == n_positional
+                        // (unum >> 8) & 0xff == n_keyword
+                        // We have folowing stack layout here:
+                        // fun arg0 arg1 ... kw0 val0 kw1 val1 ... seq dict <- TOS
+                        obj2 = POP();
+                        obj1 = POP();
+                        sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe);
+                        SET_TOP(mp_call_method_n_kw_var(false, unum, sp, obj1, obj2));
                         break;
-                    }
 
                     case MP_BC_CALL_METHOD:
                         DECODE_UINT;
@@ -706,6 +713,40 @@ unwind_jump:
                         SET_TOP(mp_call_method_n_kw(unum & 0xff, (unum >> 8) & 0xff, sp));
                         break;
 
+                    case MP_BC_CALL_METHOD_VAR:
+                        DECODE_UINT;
+                        // unum & 0xff == n_positional
+                        // (unum >> 8) & 0xff == n_keyword
+                        // We have folowing stack layout here:
+                        // fun self arg0 arg1 ... kw0 val0 kw1 val1 ... seq <- TOS
+                        obj1 = POP();
+                        sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 1;
+                        SET_TOP(mp_call_method_n_kw_var(true, unum, sp, obj1, MP_OBJ_NULL));
+                        break;
+
+                    case MP_BC_CALL_METHOD_KW:
+                        DECODE_UINT;
+                        // unum & 0xff == n_positional
+                        // (unum >> 8) & 0xff == n_keyword
+                        // We have folowing stack layout here:
+                        // fun self arg0 arg1 ... kw0 val0 kw1 val1 ... dict <- TOS
+                        obj1 = POP();
+                        sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 1;
+                        SET_TOP(mp_call_method_n_kw_var(true, unum, sp, MP_OBJ_NULL, obj1));
+                        break;
+
+                    case MP_BC_CALL_METHOD_VAR_KW:
+                        DECODE_UINT;
+                        // unum & 0xff == n_positional
+                        // (unum >> 8) & 0xff == n_keyword
+                        // We have folowing stack layout here:
+                        // fun self arg0 arg1 ... kw0 val0 kw1 val1 ... seq dict <- TOS
+                        obj2 = POP();
+                        obj1 = POP();
+                        sp -= (unum & 0xff) + ((unum >> 7) & 0x1fe) + 1;
+                        SET_TOP(mp_call_method_n_kw_var(true, unum, sp, obj1, obj2));
+                        break;
+
                     case MP_BC_RETURN_VALUE:
 unwind_return:
                         while (exc_sp >= exc_stack) {

tests/basics/fun-calldblstar.py

@@ -0,0 +1,17 @@
+# test calling a function with keywords given by **dict
+
+def f(a, b):
+    print(a, b)
+
+f(1, **{'b':2})
+f(1, **{'b':val for val in range(1)})
+
+# test calling a method with keywords given by **dict
+
+class A:
+    def f(self, a, b):
+        print(a, b)
+
+a = A()
+a.f(1, **{'b':2})
+a.f(1, **{'b':val for val in range(1)})

tests/basics/fun-callstar.py

@@ -1,3 +1,5 @@
+# function calls with *pos
+
 def foo(a, b, c):
     print(a, b, c)
 
@@ -11,3 +13,21 @@ foo(1, 2, *[100])
 
 # Iterator
 foo(*range(3))
+
+# method calls with *pos
+
+class A:
+    def foo(self, a, b, c):
+        print(a, b, c)
+
+a = A()
+a.foo(*(1, 2, 3))
+a.foo(1, *(2, 3))
+a.foo(1, 2, *(3,))
+a.foo(1, 2, 3, *())
+
+# Another sequence type
+a.foo(1, 2, *[100])
+
+# Iterator
+a.foo(*range(3))

tests/basics/fun-callstardblstar.py

@@ -0,0 +1,17 @@
+# test calling a function with *tuple and **dict
+
+def f(a, b, c, d):
+    print(a, b, c, d)
+
+f(*(1, 2), **{'c':3, 'd':4})
+f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})
+
+# test calling a method with *tuple and **dict
+
+class A:
+    def f(self, a, b, c, d):
+        print(a, b, c, d)
+
+a = A()
+a.f(*(1, 2), **{'c':3, 'd':4})
+a.f(*(1, 2), **{['c', 'd'][i]:(3 + i) for i in range(2)})