JIT inlined function separately if trace is too long

Duplicated JITTed code brings overhead for the instruction cache.
This patch reduces duplication by JITting inlined function
separately if trace is too long because the same function is
JITTed in different root trace or side trace sometimes.
It increases 2%~3% performance of our workload in tracing mode.

Signed-off-by: Wang, Xue   <[email protected]>
Signed-off-by: Yang, Lin A <[email protected]>
Signed-off-by: Su, Tao     <[email protected]>

Author: wxue1

ext/opcache/jit/zend_jit.h

@@ -116,6 +116,7 @@ typedef struct _zend_jit_globals {
 	zend_long   max_recursive_calls;   /* max number of recursive inlined call unrolls */
 	zend_long   max_recursive_returns; /* max number of recursive inlined return unrolls */
 	zend_long   max_polymorphic_calls; /* max number of inlined polymorphic calls */
+    zend_long   max_inline_func_length; /* max length of inlined functions in one trace */
 
 	zend_sym_node *symbols;            /* symbols for disassembler */
 

ext/opcache/jit/zend_jit_internal.h

@@ -359,6 +359,7 @@ zend_constant* ZEND_FASTCALL zend_jit_check_constant(const zval *key);
 	_(RECURSION_EXIT,    "return from recursive function") \
 	_(BLACK_LIST,        "trace blacklisted") \
 	_(INNER_LOOP,        "inner loop")                     /* trace it */ \
+	_(JIT_INLINE_FUNC,   "JIT inlined function and skip current trace") /* trace inlined function */ \
 	_(COMPILED_LOOP,     "compiled loop") \
 	_(TRAMPOLINE,        "trampoline call") \
 	_(BAD_FUNC,          "bad function call") \
@@ -383,8 +384,9 @@ typedef enum _zend_jit_trace_stop {
 #define ZEND_JIT_TRACE_STOP_DONE(ret) \
 	(ret < ZEND_JIT_TRACE_STOP_ERROR)
 
+/* restart to trace an inner loop or inlined function */
 #define ZEND_JIT_TRACE_STOP_REPEAT(ret) \
-	(ret == ZEND_JIT_TRACE_STOP_INNER_LOOP)
+	(ret == ZEND_JIT_TRACE_STOP_INNER_LOOP || ret == ZEND_JIT_TRACE_STOP_JIT_INLINE_FUNC)
 
 #define ZEND_JIT_TRACE_STOP_MAY_RECOVER(ret) \
 	(ret <= ZEND_JIT_TRACE_STOP_COMPILER_ERROR)

ext/opcache/jit/zend_jit_trace.c

@@ -7696,7 +7696,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_root(zend_execute_data *execute_data, const
 				zend_jit_trace_stop_description[stop]);
 		}
 		if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
-		 || zend_jit_trace_is_bad_root(orig_opline, stop, offset)) {
+		 || (stop != ZEND_JIT_TRACE_STOP_JIT_INLINE_FUNC && zend_jit_trace_is_bad_root(orig_opline, stop, offset))) {
 			if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
 				fprintf(stderr, "---- TRACE %d blacklisted\n",
 					trace_num);
@@ -8039,7 +8039,7 @@ int ZEND_FASTCALL zend_jit_trace_hot_side(zend_execute_data *execute_data, uint3
 				zend_jit_trace_stop_description[stop]);
 		}
 		if (!ZEND_JIT_TRACE_STOP_MAY_RECOVER(stop)
-		 || zend_jit_trace_exit_is_bad(parent_num, exit_num)) {
+		 || (stop != ZEND_JIT_TRACE_STOP_JIT_INLINE_FUNC && zend_jit_trace_exit_is_bad(parent_num, exit_num))) {
 			zend_jit_blacklist_trace_exit(parent_num, exit_num);
 			if (JIT_G(debug) & ZEND_JIT_DEBUG_TRACE_BLACKLIST) {
 				fprintf(stderr, "---- EXIT %d/%d blacklisted\n",

ext/opcache/jit/zend_jit_vm_helpers.c

@@ -1062,6 +1062,13 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex,
 
 		trace_flags = ZEND_OP_TRACE_INFO(opline, offset)->trace_flags;
 		if (trace_flags) {
+			/* if inlined functions are too long, stop current tracing and restart a new one */
+			if (trace_buffer[idx-1].op == ZEND_JIT_TRACE_ENTER && idx > JIT_G(max_inline_func_length)) {
+				if (!(trace_flags & ZEND_JIT_TRACE_JITED)) {
+					stop = ZEND_JIT_TRACE_STOP_JIT_INLINE_FUNC;
+					break;
+				}
+			}
 			if (trace_flags & ZEND_JIT_TRACE_JITED) {
 				if (trace_flags & ZEND_JIT_TRACE_START_LOOP) {
 					if ((start & ZEND_JIT_TRACE_START_LOOP) != 0

ext/opcache/zend_accelerator_module.c

@@ -326,6 +326,7 @@ ZEND_INI_BEGIN()
 	STD_PHP_INI_ENTRY("opcache.jit_max_root_traces"           , "1024",                       PHP_INI_SYSTEM, OnUpdateLong,     max_root_traces,       zend_jit_globals, jit_globals)
 	STD_PHP_INI_ENTRY("opcache.jit_max_side_traces"           , "128",                        PHP_INI_SYSTEM, OnUpdateLong,     max_side_traces,       zend_jit_globals, jit_globals)
 	STD_PHP_INI_ENTRY("opcache.jit_max_exit_counters"         , "8192",                       PHP_INI_SYSTEM, OnUpdateLong,     max_exit_counters,     zend_jit_globals, jit_globals)
+	STD_PHP_INI_ENTRY("opcache.jit_max_inline_func_length"    , "16",                         PHP_INI_SYSTEM, OnUpdateLong,     max_inline_func_length,zend_jit_globals, jit_globals)
 	STD_PHP_INI_ENTRY("opcache.jit_hot_loop"                  , "64",                         PHP_INI_SYSTEM, OnUpdateCounter,  hot_loop,              zend_jit_globals, jit_globals)
 	STD_PHP_INI_ENTRY("opcache.jit_hot_func"                  , "127",                        PHP_INI_SYSTEM, OnUpdateCounter,  hot_func,              zend_jit_globals, jit_globals)
 	STD_PHP_INI_ENTRY("opcache.jit_hot_return"                , "8",                          PHP_INI_SYSTEM, OnUpdateCounter,  hot_return,            zend_jit_globals, jit_globals)
@@ -849,6 +850,7 @@ ZEND_FUNCTION(opcache_get_configuration)
 	add_assoc_long(&directives,   "opcache.jit_max_root_traces", JIT_G(max_root_traces));
 	add_assoc_long(&directives,   "opcache.jit_max_side_traces", JIT_G(max_side_traces));
 	add_assoc_long(&directives,   "opcache.jit_prof_threshold", JIT_G(prof_threshold));
+    add_assoc_long(&directives,   "opcache.jit_max_inline_func_length", JIT_G(max_inline_func_length));
 #endif
 
 	add_assoc_zval(return_value, "directives", &directives);