Go patch committed: allocate defer records on the stack
diff mbox series

Message ID CAOyqgcUXOdLZ7mDS05sPMEAjywbZ3kV-x_9TCkg-i1e75kozLQ@mail.gmail.com
State New
Headers show
Series
  • Go patch committed: allocate defer records on the stack
Related show

Commit Message

Ian Lance Taylor Aug. 17, 2019, 11:43 p.m. UTC
This Go patch Cherry Zhang allocates defer records on the stack.  When
a defer is executed at most once in a function body, we can allocate
the defer record for it on the stack instead of on the heap.  This
should make defers of this kind (which are very common) faster.  This
is a port of https://golang.org/cl/171758 from the gc repo.
Bootstrapped and ran Go testsuite on x86_64-pc-linux-gnu.  Committed
to mainline.

Ian

Patch
diff mbox series

Index: gcc/go/gofrontend/MERGE
===================================================================
--- gcc/go/gofrontend/MERGE	(revision 274598)
+++ gcc/go/gofrontend/MERGE	(working copy)
@@ -1,4 +1,4 @@ 
-838f926c93898767f0337122725a4f52a1335186
+4b47cadf938caadf563f8d0bb3f7111d06f61752
 
 The first line of this file holds the git revision number of the last
 merge done from the gofrontend repository.
Index: gcc/go/gofrontend/runtime.def
===================================================================
--- gcc/go/gofrontend/runtime.def	(revision 274591)
+++ gcc/go/gofrontend/runtime.def	(working copy)
@@ -287,6 +287,10 @@  DEF_GO_RUNTIME(GO, "__go_go", P2(UINTPTR
 DEF_GO_RUNTIME(DEFERPROC, "runtime.deferproc", P3(BOOLPTR, UINTPTR, POINTER),
 	       R0())
 
+// Defer a function, with stack-allocated defer structure.
+DEF_GO_RUNTIME(DEFERPROCSTACK, "runtime.deferprocStack",
+               P4(POINTER, BOOLPTR, UINTPTR, POINTER), R0())
+
 
 // Convert an empty interface to an empty interface, returning ok.
 DEF_GO_RUNTIME(IFACEE2E2, "runtime.ifaceE2E2", P1(EFACE), R2(EFACE, BOOL))
Index: gcc/go/gofrontend/statements.cc
===================================================================
--- gcc/go/gofrontend/statements.cc	(revision 274169)
+++ gcc/go/gofrontend/statements.cc	(working copy)
@@ -2614,7 +2614,11 @@  Thunk_statement::simplify_statement(Gogo
   if (this->classification() == STATEMENT_GO)
     s = Statement::make_go_statement(call, location);
   else if (this->classification() == STATEMENT_DEFER)
-    s = Statement::make_defer_statement(call, location);
+    {
+      s = Statement::make_defer_statement(call, location);
+      if ((Node::make_node(this)->encoding() & ESCAPE_MASK) == Node::ESCAPE_NONE)
+        s->defer_statement()->set_on_stack();
+    }
   else
     go_unreachable();
 
@@ -3019,13 +3023,45 @@  Defer_statement::do_get_backend(Translat
   Location loc = this->location();
   Expression* ds = context->function()->func_value()->defer_stack(loc);
 
-  Expression* call = Runtime::make_call(Runtime::DEFERPROC, loc, 3,
-					ds, fn, arg);
+  Expression* call;
+  if (this->on_stack_)
+    {
+      if (context->gogo()->debug_optimization())
+        go_debug(loc, "stack allocated defer");
+
+      Type* defer_type = Defer_statement::defer_struct_type();
+      Expression* defer = Expression::make_allocation(defer_type, loc);
+      defer->allocation_expression()->set_allocate_on_stack();
+      defer->allocation_expression()->set_no_zero();
+      call = Runtime::make_call(Runtime::DEFERPROCSTACK, loc, 4,
+                                defer, ds, fn, arg);
+    }
+  else
+    call = Runtime::make_call(Runtime::DEFERPROC, loc, 3,
+                              ds, fn, arg);
   Bexpression* bcall = call->get_backend(context);
   Bfunction* bfunction = context->function()->func_value()->get_decl();
   return context->backend()->expression_statement(bfunction, bcall);
 }
 
+Type*
+Defer_statement::defer_struct_type()
+{
+  Type* ptr_type = Type::make_pointer_type(Type::make_void_type());
+  Type* uintptr_type = Type::lookup_integer_type("uintptr");
+  Type* bool_type = Type::make_boolean_type();
+  return Type::make_builtin_struct_type(9,
+                                        "link", ptr_type,
+                                        "frame", ptr_type,
+                                        "panicStack", ptr_type,
+                                        "_panic", ptr_type,
+                                        "pfn", uintptr_type,
+                                        "arg", ptr_type,
+                                        "retaddr", uintptr_type,
+                                        "makefunccanrecover", bool_type,
+                                        "heap", bool_type);
+}
+
 // Dump the AST representation for defer statement.
 
 void
Index: gcc/go/gofrontend/statements.h
===================================================================
--- gcc/go/gofrontend/statements.h	(revision 274169)
+++ gcc/go/gofrontend/statements.h	(working copy)
@@ -24,6 +24,7 @@  class Expression_statement;
 class Block_statement;
 class Return_statement;
 class Thunk_statement;
+class Defer_statement;
 class Goto_statement;
 class Goto_unnamed_statement;
 class Label_statement;
@@ -403,6 +404,11 @@  class Statement
   Thunk_statement*
   thunk_statement();
 
+  // If this is a defer statement, return it.  Otherwise return NULL.
+  Defer_statement*
+  defer_statement()
+  { return this->convert<Defer_statement, STATEMENT_DEFER>(); }
+
   // If this is a goto statement, return it.  Otherwise return NULL.
   Goto_statement*
   goto_statement()
@@ -1419,15 +1425,26 @@  class Defer_statement : public Thunk_sta
 {
  public:
   Defer_statement(Call_expression* call, Location location)
-    : Thunk_statement(STATEMENT_DEFER, call, location)
+    : Thunk_statement(STATEMENT_DEFER, call, location),
+      on_stack_(false)
   { }
 
+  void
+  set_on_stack()
+  { this->on_stack_ = true; }
+
  protected:
   Bstatement*
   do_get_backend(Translate_context*);
 
   void
   do_dump_statement(Ast_dump_context*) const;
+
+ private:
+  static Type*
+  defer_struct_type();
+
+  bool on_stack_;
 };
 
 // A goto statement.
Index: libgo/go/runtime/mgcmark.go
===================================================================
--- libgo/go/runtime/mgcmark.go	(revision 274169)
+++ libgo/go/runtime/mgcmark.go	(working copy)
@@ -657,6 +657,11 @@  func scanstack(gp *g, gcw *gcWork) {
 		scanstackblock(uintptr(unsafe.Pointer(&gp.context)), unsafe.Sizeof(gp.context), gcw)
 	}
 
+	// Note: in the gc runtime scanstack also scans defer records.
+	// This is necessary as it uses stack objects (a.k.a. stack tracing).
+	// We don't (yet) do stack objects, and regular stack/heap scan
+	// will take care of defer records just fine.
+
 	gp.gcscanvalid = true
 }
 
Index: libgo/go/runtime/panic.go
===================================================================
--- libgo/go/runtime/panic.go	(revision 274169)
+++ libgo/go/runtime/panic.go	(working copy)
@@ -13,6 +13,7 @@  import (
 // themselves, so that the compiler will export them.
 //
 //go:linkname deferproc runtime.deferproc
+//go:linkname deferprocStack runtime.deferprocStack
 //go:linkname deferreturn runtime.deferreturn
 //go:linkname setdeferretaddr runtime.setdeferretaddr
 //go:linkname checkdefer runtime.checkdefer
@@ -124,6 +125,38 @@  func deferproc(frame *bool, pfn uintptr,
 	d.makefunccanrecover = false
 }
 
+// deferprocStack queues a new deferred function with a defer record on the stack.
+// The defer record, d, does not need to be initialized.
+// Other arguments are the same as in deferproc.
+//go:nosplit
+func deferprocStack(d *_defer, frame *bool, pfn uintptr, arg unsafe.Pointer) {
+	gp := getg()
+	if gp.m.curg != gp {
+		// go code on the system stack can't defer
+		throw("defer on system stack")
+	}
+	d.pfn = pfn
+	d.retaddr = 0
+	d.makefunccanrecover = false
+	d.heap = false
+	// The lines below implement:
+	//   d.frame = frame
+	//   d.arg = arg
+	//   d._panic = nil
+	//   d.panicStack = gp._panic
+	//   d.link = gp._defer
+	// But without write barriers. They are writes to the stack so they
+	// don't need a write barrier, and furthermore are to uninitialized
+	// memory, so they must not use a write barrier.
+	*(*uintptr)(unsafe.Pointer(&d.frame)) = uintptr(unsafe.Pointer(frame))
+	*(*uintptr)(unsafe.Pointer(&d.arg)) = uintptr(unsafe.Pointer(arg))
+	*(*uintptr)(unsafe.Pointer(&d._panic)) = 0
+	*(*uintptr)(unsafe.Pointer(&d.panicStack)) = uintptr(unsafe.Pointer(gp._panic))
+	*(*uintptr)(unsafe.Pointer(&d.link)) = uintptr(unsafe.Pointer(gp._defer))
+
+	gp._defer = d
+}
+
 // Allocate a Defer, usually using per-P pool.
 // Each defer must be released with freedefer.
 func newdefer() *_defer {
@@ -155,11 +188,13 @@  func newdefer() *_defer {
 			// Duplicate the tail below so if there's a
 			// crash in checkPut we can tell if d was just
 			// allocated or came from the pool.
+			d.heap = true
 			d.link = gp._defer
 			gp._defer = d
 			return d
 		}
 	}
+	d.heap = true
 	d.link = gp._defer
 	gp._defer = d
 	return d
@@ -179,6 +214,9 @@  func freedefer(d *_defer) {
 	if d.pfn != 0 {
 		freedeferfn()
 	}
+	if !d.heap {
+		return
+	}
 	pp := getg().m.p.ptr()
 	if len(pp.deferpool) == cap(pp.deferpool) {
 		// Transfer half of local cache to the central cache.
Index: libgo/go/runtime/runtime2.go
===================================================================
--- libgo/go/runtime/runtime2.go	(revision 274169)
+++ libgo/go/runtime/runtime2.go	(working copy)
@@ -746,6 +746,12 @@  func extendRandom(r []byte, n int) {
 
 // A _defer holds an entry on the list of deferred calls.
 // If you add a field here, add code to clear it in freedefer.
+// This struct must match the code in Defer_statement::defer_struct_type
+// in the compiler.
+// Some defers will be allocated on the stack and some on the heap.
+// All defers are logically part of the stack, so write barriers to
+// initialize them are not required. All defers must be manually scanned,
+// and for heap defers, marked.
 type _defer struct {
 	// The next entry in the stack.
 	link *_defer
@@ -781,6 +787,9 @@  type _defer struct {
 	// function function will be somewhere in libffi, so __retaddr
 	// is not useful.
 	makefunccanrecover bool
+
+	// Whether the _defer is heap allocated.
+	heap bool
 }
 
 // panics
Index: libgo/go/runtime/stack_test.go
===================================================================
--- libgo/go/runtime/stack_test.go	(nonexistent)
+++ libgo/go/runtime/stack_test.go	(working copy)
@@ -0,0 +1,62 @@ 
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime_test
+
+import "testing"
+
+func TestDeferHeapAndStack(t *testing.T) {
+	P := 4     // processors
+	N := 10000 // iterations
+	D := 200   // stack depth
+
+	if testing.Short() {
+		P /= 2
+		N /= 10
+		D /= 10
+	}
+	c := make(chan bool)
+	for p := 0; p < P; p++ {
+		go func() {
+			for i := 0; i < N; i++ {
+				if deferHeapAndStack(D) != 2*D {
+					panic("bad result")
+				}
+			}
+			c <- true
+		}()
+	}
+	for p := 0; p < P; p++ {
+		<-c
+	}
+}
+
+// deferHeapAndStack(n) computes 2*n
+func deferHeapAndStack(n int) (r int) {
+	if n == 0 {
+		return 0
+	}
+	if n%2 == 0 {
+		// heap-allocated defers
+		for i := 0; i < 2; i++ {
+			defer func() {
+				r++
+			}()
+		}
+	} else {
+		// stack-allocated defers
+		defer func() {
+			r++
+		}()
+		defer func() {
+			r++
+		}()
+	}
+	r = deferHeapAndStack(n - 1)
+	escapeMe(new([1024]byte)) // force some GCs
+	return
+}
+
+// Pass a value to escapeMe to force it to escape.
+var escapeMe = func(x interface{}) {}