Merge pull request #634 from robarnold/upstream-stable

Allocate ivecs out of the kernel pool

Author: pcwalton

src/comp/back/upcall.rs

@@ -47,6 +47,8 @@ type upcalls =
         ValueRef exit,
         ValueRef malloc,
         ValueRef free,
+        ValueRef shared_malloc,
+        ValueRef shared_free,
         ValueRef mark,
         ValueRef new_str,
         ValueRef dup_str,
@@ -56,7 +58,9 @@ type upcalls =
         ValueRef new_task,
         ValueRef start_task,
         ValueRef ivec_resize,
-        ValueRef ivec_spill);
+        ValueRef ivec_spill,
+        ValueRef ivec_resize_shared,
+        ValueRef ivec_spill_shared);
 
 fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls {
     fn decl(type_names tn, ModuleRef llmod, str name, vec[TypeRef] tys,
@@ -97,6 +101,9 @@ fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls {
              malloc=d("malloc", [T_size_t(), T_ptr(T_tydesc(tn))],
                       T_ptr(T_i8())),
              free=dv("free", [T_ptr(T_i8()), T_int()]),
+             shared_malloc=d("shared_malloc",
+                      [T_size_t(), T_ptr(T_tydesc(tn))], T_ptr(T_i8())),
+             shared_free=dv("shared_free", [T_ptr(T_i8())]),
              mark=d("mark", [T_ptr(T_i8())], T_int()),
              new_str=d("new_str", [T_ptr(T_i8()), T_size_t()],
                        T_ptr(T_str())),
@@ -119,7 +126,11 @@ fn declare_upcalls(type_names tn, ModuleRef llmod) -> @upcalls {
              ivec_resize=d("ivec_resize", [T_ptr(T_opaque_ivec()), T_int()],
                            T_void()),
              ivec_spill=d("ivec_spill", [T_ptr(T_opaque_ivec()), T_int()],
-                          T_void()));
+                          T_void()),
+             ivec_resize_shared=d("ivec_resize_shared",
+                           [T_ptr(T_opaque_ivec()), T_int()], T_void()),
+             ivec_spill_shared=d("ivec_spill_shared",
+                          [T_ptr(T_opaque_ivec()), T_int()], T_void()));
 }
 //
 // Local Variables:

src/comp/middle/trans.rs

@@ -1186,6 +1186,12 @@ fn trans_non_gc_free(&@block_ctxt cx, ValueRef v) -> result {
     ret rslt(cx, C_int(0));
 }
 
+fn trans_shared_free(&@block_ctxt cx, ValueRef v) -> result {
+    cx.build.Call(cx.fcx.lcx.ccx.upcalls.shared_free,
+                  [cx.fcx.lltaskptr, cx.build.PointerCast(v, T_ptr(T_i8()))]);
+    ret rslt(cx, C_int(0));
+}
+
 fn find_scope_cx(&@block_ctxt cx) -> @block_ctxt {
     if (cx.kind != NON_SCOPE_BLOCK) { ret cx; }
     alt (cx.parent) {
@@ -1614,6 +1620,18 @@ fn trans_raw_malloc(&@block_ctxt cx, TypeRef llptr_ty, ValueRef llsize) ->
     ret rslt(cx, cx.build.PointerCast(rval, llptr_ty));
 }
 
+// trans_shared_malloc: expects a type indicating which pointer type we want
+// and a size indicating how much space we want malloc'd.
+fn trans_shared_malloc(&@block_ctxt cx, TypeRef llptr_ty, ValueRef llsize) ->
+   result {
+    // FIXME: need a table to collect tydesc globals.
+
+    auto tydesc = C_null(T_ptr(T_tydesc(cx.fcx.lcx.ccx.tn)));
+    auto rval =
+        cx.build.Call(cx.fcx.lcx.ccx.upcalls.shared_malloc,
+                      [cx.fcx.lltaskptr, llsize, tydesc]);
+    ret rslt(cx, cx.build.PointerCast(rval, llptr_ty));
+}
 
 // trans_malloc_boxed: expects an unboxed type and returns a pointer to enough
 // space for something of that type, along with space for a reference count;
@@ -2133,7 +2151,7 @@ fn maybe_free_ivec_heap_part(&@block_ctxt cx, ValueRef v0, ty::t unit_ty) ->
             auto m = maybe_on_heap_cx.build.InBoundsGEP(stub_ptr, v);
             maybe_on_heap_cx.build.Load(m)
         };
-    auto after_free_cx = trans_non_gc_free(maybe_on_heap_cx, heap_ptr).bcx;
+    auto after_free_cx = trans_shared_free(maybe_on_heap_cx, heap_ptr).bcx;
     after_free_cx.build.Br(next_cx.llbb);
     ret rslt(next_cx, C_nil());
 }
@@ -3673,7 +3691,8 @@ mod ivec {
         {
             auto p =
                 heap_resize_cx.build.PointerCast(v, T_ptr(T_opaque_ivec()));
-            heap_resize_cx.build.Call(cx.fcx.lcx.ccx.upcalls.ivec_resize,
+            auto upcall = cx.fcx.lcx.ccx.upcalls.ivec_resize_shared;
+            heap_resize_cx.build.Call(upcall,
                                       [cx.fcx.lltaskptr, p, new_heap_len]);
         }
         auto heap_ptr_resize =
@@ -3713,7 +3732,8 @@ mod ivec {
         {
             auto p =
                 stack_spill_cx.build.PointerCast(v, T_ptr(T_opaque_ivec()));
-            stack_spill_cx.build.Call(cx.fcx.lcx.ccx.upcalls.ivec_spill,
+            auto upcall = cx.fcx.lcx.ccx.upcalls.ivec_spill_shared;
+            stack_spill_cx.build.Call(upcall,
                                       [cx.fcx.lltaskptr, p, new_stack_len]);
         }
         auto spill_stub =
@@ -3963,7 +3983,7 @@ mod ivec {
                             heap_cx.build.InBoundsGEP(stub_ptr_heap,
                                                       stub_a));
         auto heap_sz = heap_cx.build.Add(llsize_of(llheappartty), lllen);
-        auto rs = trans_raw_malloc(heap_cx, T_ptr(llheappartty), heap_sz);
+        auto rs = trans_shared_malloc(heap_cx, T_ptr(llheappartty), heap_sz);
         auto heap_part = rs.val;
         heap_cx = rs.bcx;
         heap_cx.build.Store(heap_part,
@@ -4100,7 +4120,7 @@ mod ivec {
 
         auto heap_part_sz = on_heap_cx.build.Add(alen,
             llsize_of(T_opaque_ivec_heap_part()));
-        auto rs = trans_raw_malloc(on_heap_cx, T_ptr(llheappartty),
+        auto rs = trans_shared_malloc(on_heap_cx, T_ptr(llheappartty),
                                    heap_part_sz);
         on_heap_cx = rs.bcx;
         auto new_heap_ptr = rs.val;
@@ -6038,7 +6058,7 @@ fn trans_ivec(@block_ctxt bcx, &(@ast::expr)[] args, ast::node_id id) ->
             bcx.build.Store(lllen, bcx.build.InBoundsGEP(llstubptr, stub_a));
 
             auto llheapsz = bcx.build.Add(llsize_of(llheapty), lllen);
-            auto rslt = trans_raw_malloc(bcx, T_ptr(llheapty), llheapsz);
+            auto rslt = trans_shared_malloc(bcx, T_ptr(llheapty), llheapsz);
             bcx = rslt.bcx;
             auto llheapptr = rslt.val;
             bcx.build.Store(llheapptr,

src/lib/ivec.rs

@@ -11,15 +11,16 @@ native "rust-intrinsic" mod rusti {
 }
 
 native "rust" mod rustrt {
-    fn ivec_reserve[T](&mutable T[mutable?] v, uint n);
+    fn ivec_reserve_shared[T](&mutable T[mutable?] v, uint n);
     fn ivec_on_heap[T](&T[] v) -> uint;
     fn ivec_to_ptr[T](&T[] v) -> *T;
-    fn ivec_copy_from_buf[T](&mutable T[mutable?] v, *T ptr, uint count);
+    fn ivec_copy_from_buf_shared[T](&mutable T[mutable?] v,
+                                    *T ptr, uint count);
 }
 
 /// Reserves space for `n` elements in the given vector.
 fn reserve[T](&mutable T[mutable?] v, uint n) {
-    rustrt::ivec_reserve(v, n);
+    rustrt::ivec_reserve_shared(v, n);
 }
 
 fn on_heap[T](&T[] v) -> bool {
@@ -204,7 +205,7 @@ fn all[T](fn(&T)->bool f, &T[] v) -> bool {
 
 mod unsafe {
     fn copy_from_buf[T](&mutable T[] v, *T ptr, uint count) {
-        ret rustrt::ivec_copy_from_buf(v, ptr, count);
+        ret rustrt::ivec_copy_from_buf_shared(v, ptr, count);
     }
 }
 

src/rt/rust_builtin.cpp

@@ -648,6 +648,37 @@ ivec_reserve(rust_task *task, type_desc *ty, rust_ivec *v, size_t n_elems)
     v->alloc = new_alloc;
 }
 
+/**
+ * Preallocates the exact number of bytes in the given interior vector.
+ */
+extern "C" CDECL void
+ivec_reserve_shared(rust_task *task, type_desc *ty, rust_ivec *v,
+                    size_t n_elems)
+{
+    size_t new_alloc = n_elems * ty->size;
+    if (new_alloc <= v->alloc)
+        return;     // Already big enough.
+
+    rust_ivec_heap *heap_part;
+    if (v->fill || !v->payload.ptr) {
+        // On stack; spill to heap.
+        heap_part = (rust_ivec_heap *)task->kernel->malloc(new_alloc +
+                                                           sizeof(size_t));
+        heap_part->fill = v->fill;
+        memcpy(&heap_part->data, v->payload.data, v->fill);
+
+        v->fill = 0;
+        v->payload.ptr = heap_part;
+    } else {
+        // On heap; resize.
+        heap_part = (rust_ivec_heap *)task->kernel->realloc(v->payload.ptr,
+                                                new_alloc + sizeof(size_t));
+        v->payload.ptr = heap_part;
+    }
+
+    v->alloc = new_alloc;
+}
+
 /**
  * Returns true if the given vector is on the heap and false if it's on the
  * stack.
@@ -706,6 +737,35 @@ ivec_copy_from_buf(rust_task *task, type_desc *ty, rust_ivec *v, void *ptr,
     v->payload.ptr->fill = new_size;
 }
 
+/**
+ * Copies elements in an unsafe buffer to the given interior vector. The
+ * vector must have size zero.
+ */
+extern "C" CDECL void
+ivec_copy_from_buf_shared(rust_task *task, type_desc *ty, rust_ivec *v,
+                   void *ptr, size_t count)
+{
+    size_t old_size = get_ivec_size(v);
+    if (old_size) {
+        task->fail(1);
+        return;
+    }
+
+    ivec_reserve_shared(task, ty, v, count);
+
+    size_t new_size = count * ty->size;
+    if (v->fill || !v->payload.ptr) {
+        // On stack.
+        memmove(v->payload.data, ptr, new_size);
+        v->fill = new_size;
+        return;
+    }
+
+    // On heap.
+    memmove(v->payload.ptr->data, ptr, new_size);
+    v->payload.ptr->fill = new_size;
+}
+
 extern "C" CDECL void
 pin_task(rust_task *task) {
     task->pin();

src/rt/rust_kernel.cpp

@@ -206,6 +206,11 @@ rust_kernel::malloc(size_t size) {
     return _region->malloc(size);
 }
 
+void *
+rust_kernel::realloc(void *mem, size_t size) {
+    return _region->realloc(mem, size);
+}
+
 void rust_kernel::free(void *mem) {
     _region->free(mem);
 }

src/rt/rust_kernel.h

@@ -110,6 +110,7 @@ class rust_kernel : public rust_thread {
     virtual ~rust_kernel();
 
     void *malloc(size_t size);
+    void *realloc(void *mem, size_t size);
     void free(void *mem);
 
     // FIXME: this should go away

src/rt/rust_upcall.cpp

@@ -289,6 +289,36 @@ upcall_free(rust_task *task, void* ptr, uintptr_t is_gc) {
     task->free(ptr, (bool) is_gc);
 }
 
+extern "C" CDECL uintptr_t
+upcall_shared_malloc(rust_task *task, size_t nbytes, type_desc *td) {
+    LOG_UPCALL_ENTRY(task);
+    scoped_lock with(task->kernel->scheduler_lock);
+
+    LOG(task, mem,
+                   "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR ")",
+                   nbytes, td);
+    void *p = task->kernel->malloc(nbytes);
+    LOG(task, mem,
+                   "upcall shared_malloc(%" PRIdPTR ", 0x%" PRIxPTR
+                   ") = 0x%" PRIxPTR,
+                   nbytes, td, (uintptr_t)p);
+    return (uintptr_t) p;
+}
+
+/**
+ * Called whenever an object's ref count drops to zero.
+ */
+extern "C" CDECL void
+upcall_shared_free(rust_task *task, void* ptr) {
+    LOG_UPCALL_ENTRY(task);
+    scoped_lock with(task->kernel->scheduler_lock);
+    rust_scheduler *sched = task->sched;
+    DLOG(sched, mem,
+             "upcall shared_free(0x%" PRIxPTR")",
+             (uintptr_t)ptr);
+    task->kernel->free(ptr);
+}
+
 extern "C" CDECL uintptr_t
 upcall_mark(rust_task *task, void* ptr) {
     LOG_UPCALL_ENTRY(task);
@@ -537,6 +567,7 @@ extern "C" CDECL void
 upcall_ivec_resize(rust_task *task,
                    rust_ivec *v,
                    size_t newsz) {
+    LOG_UPCALL_ENTRY(task);
     scoped_lock with(task->kernel->scheduler_lock);
     I(task->sched, !v->fill);
 
@@ -556,6 +587,7 @@ extern "C" CDECL void
 upcall_ivec_spill(rust_task *task,
                   rust_ivec *v,
                   size_t newsz) {
+    LOG_UPCALL_ENTRY(task);
     scoped_lock with(task->kernel->scheduler_lock);
     size_t new_alloc = next_power_of_two(newsz);
 
@@ -569,6 +601,46 @@ upcall_ivec_spill(rust_task *task,
     v->payload.ptr = heap_part;
 }
 
+/**
+ * Resizes an interior vector that has been spilled to the heap.
+ */
+extern "C" CDECL void
+upcall_ivec_resize_shared(rust_task *task,
+                          rust_ivec *v,
+                          size_t newsz) {
+    LOG_UPCALL_ENTRY(task);
+    scoped_lock with(task->kernel->scheduler_lock);
+    I(task->sched, !v->fill);
+
+    size_t new_alloc = next_power_of_two(newsz);
+    rust_ivec_heap *new_heap_part = (rust_ivec_heap *)
+        task->kernel->realloc(v->payload.ptr, new_alloc + sizeof(size_t));
+
+    new_heap_part->fill = newsz;
+    v->alloc = new_alloc;
+    v->payload.ptr = new_heap_part;
+}
+
+/**
+ * Spills an interior vector to the heap.
+ */
+extern "C" CDECL void
+upcall_ivec_spill_shared(rust_task *task,
+                         rust_ivec *v,
+                         size_t newsz) {
+    LOG_UPCALL_ENTRY(task);
+    scoped_lock with(task->kernel->scheduler_lock);
+    size_t new_alloc = next_power_of_two(newsz);
+
+    rust_ivec_heap *heap_part = (rust_ivec_heap *)
+        task->kernel->malloc(new_alloc + sizeof(size_t));
+    heap_part->fill = newsz;
+    memcpy(&heap_part->data, v->payload.data, v->fill);
+
+    v->fill = 0;
+    v->alloc = new_alloc;
+    v->payload.ptr = heap_part;
+}
 //
 // Local Variables:
 // mode: C++

src/rt/rustrt.def.in

@@ -11,8 +11,10 @@ debug_tydesc
 do_gc
 get_time
 ivec_copy_from_buf
+ivec_copy_from_buf_shared
 ivec_on_heap
 ivec_reserve
+ivec_reserve_shared
 ivec_to_ptr
 last_os_error
 nano_time
@@ -59,7 +61,9 @@ upcall_free
 upcall_get_type_desc
 upcall_grow_task
 upcall_ivec_resize
+upcall_ivec_resize_shared
 upcall_ivec_spill
+upcall_ivec_spill_shared
 upcall_kill
 upcall_log_double
 upcall_log_float
@@ -74,6 +78,8 @@ upcall_new_task
 upcall_new_vec
 upcall_recv
 upcall_send
+upcall_shared_malloc
+upcall_shared_free
 upcall_sleep
 upcall_start_task
 upcall_trace_str