From cfc7017b5a4918c0b52e0658d06dd8cdb714a8b8 Mon Sep 17 00:00:00 2001 From: slaren Date: Tue, 15 Aug 2023 20:53:14 +0200 Subject: [PATCH 01/19] llama : add benchmark example --- .gitignore | 1 + Makefile | 7 +- examples/llama-bench/CMakeLists.txt | 8 + examples/llama-bench/llama-bench.cpp | 694 +++++++++++++++++++++++++++ llama.cpp | 17 +- llama.h | 2 + 6 files changed, 724 insertions(+), 5 deletions(-) create mode 100644 examples/llama-bench/CMakeLists.txt create mode 100755 examples/llama-bench/llama-bench.cpp diff --git a/.gitignore b/.gitignore index 743b8a8b6e091..9c749f1ef6614 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ models-mnt /Pipfile /embd-input-test /libllama.so +/llama-bench build-info.h arm_neon.h compile_commands.json diff --git a/Makefile b/Makefile index 070ae12428206..e05f5cf44c7fd 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Define the default target now so that it is always the first target -BUILD_TARGETS = main quantize quantize-stats perplexity embedding vdot train-text-from-scratch convert-llama2c-to-ggml simple server embd-input-test +BUILD_TARGETS = main quantize quantize-stats perplexity embedding vdot train-text-from-scratch convert-llama2c-to-ggml simple server embd-input-test llama-bench # Binaries only useful for tests TEST_TARGETS = tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0 @@ -345,7 +345,7 @@ libllama.so: llama.o ggml.o $(OBJS) $(CXX) $(CXXFLAGS) -shared -fPIC -o $@ $^ $(LDFLAGS) clean: - rm -vf *.o *.so *.dll main quantize quantize-stats perplexity embedding benchmark-matmult save-load-state server simple vdot train-text-from-scratch convert-llama2c-to-ggml embd-input-test build-info.h $(TEST_TARGETS) + rm -vf *.o *.so *.dll main quantize quantize-stats perplexity embedding benchmark-matmult save-load-state server simple vdot train-text-from-scratch convert-llama2c-to-ggml embd-input-test llama-bench build-info.h $(TEST_TARGETS) # # Examples @@ -391,6 +391,9 @@ train-text-from-scratch: examples/train-text-from-scratch/train-text-from-scratc convert-llama2c-to-ggml: examples/convert-llama2c-to-ggml/convert-llama2c-to-ggml.cpp build-info.h ggml.o llama.o $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) +llama-bench: examples/llama-bench/llama-bench.cpp build-info.h ggml.o llama.o common.o $(OBJS) + $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) + build-info.h: $(wildcard .git/index) scripts/build-info.sh @sh scripts/build-info.sh > $@.tmp @if ! cmp -s $@.tmp $@; then \ diff --git a/examples/llama-bench/CMakeLists.txt b/examples/llama-bench/CMakeLists.txt new file mode 100644 index 0000000000000..7e395afd05f75 --- /dev/null +++ b/examples/llama-bench/CMakeLists.txt @@ -0,0 +1,8 @@ +set(TARGET llama-bench) +add_executable(${TARGET} llama-bench.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_11) +if(TARGET BUILD_INFO) + add_dependencies(${TARGET} BUILD_INFO) +endif() diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp new file mode 100755 index 0000000000000..682b5cec93e40 --- /dev/null +++ b/examples/llama-bench/llama-bench.cpp @@ -0,0 +1,694 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ggml.h" +#include "llama.h" +#include "common.h" +#include "build-info.h" + +// utils +static uint64_t get_time_ns() { + using clock = std::chrono::high_resolution_clock; + return std::chrono::nanoseconds(clock::now().time_since_epoch()).count(); +} + +template +static std::string join(const std::vector& values, const std::string& delim) { + std::ostringstream str; + for (size_t i = 0; i < values.size(); i++) { + str << values[i]; + if (i < values.size() - 1) { + str << delim; + } + } + return str.str(); +} + +template +static std::vector split(const std::string& str, char delim) { + std::vector values; + std::istringstream str_stream(str); + std::string token; + while (std::getline(str_stream, token, delim)) { + T value; + std::istringstream token_stream(token); + token_stream >> value; + values.push_back(value); + } + return values; +} + +template +T avg(const std::vector& v) { + if (v.empty()) { + return 0; + } + T sum = std::accumulate(v.begin(), v.end(), T(0)); + return sum / (T)v.size(); +} + +template +T stddev(const std::vector& v) { + if (v.size() <= 1) { + return 0; + } + T mean = avg(v); + T sq_sum = std::inner_product(v.begin(), v.end(), v.begin(), T(0)); + T stdev = std::sqrt(sq_sum / (T)v.size() - mean * mean); + return stdev; +} + +// command line params +enum output_formats {CSV, JSON, MARKDOWN}; + +struct cmd_params { + std::vector model; + std::vector n_prompt; + std::vector n_gen; + std::vector n_batch; + std::vector f32_kv; + std::vector n_threads; + std::vector n_gpu_layers; + std::vector main_gpu; + std::vector mul_mat_q; + std::vector low_vram; + std::vector> tensor_split; + int reps; + bool verbose; + output_formats output_format; +}; + +static cmd_params cmd_params_defaults = { + /* model */ {"models/7B/ggml-model-q4_0.bin"}, + /* n_prompt */ {0, 512}, + /* n_gen */ {0, 128}, + /* n_batch */ {512}, + /* f32_kv */ {false}, + /* n_threads */ {get_num_physical_cores()}, + /* n_gpu_layers */ {99}, + /* main_gpu */ {0}, + /* mul_mat_q */ {true}, + /* low_vram */ {false}, + /* tensor_split */ {{}}, + /* reps */ 5, + /* verbose */ false, + /* output_format */ MARKDOWN +}; + +static void print_usage(int /* argc */, char ** argv) { + fprintf(stdout, "usage: %s [options]\n", argv[0]); + fprintf(stdout, "\n"); + fprintf(stdout, "options:\n"); + fprintf(stdout, " -h, --help\n"); + fprintf(stdout, " -m, --model (default: %s)\n", join(cmd_params_defaults.model, ",").c_str()); + fprintf(stdout, " -np, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); + fprintf(stdout, " -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str()); + fprintf(stdout, " -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str()); + fprintf(stdout, " --memory-f32 <0|1> (default: %s)\n", join(cmd_params_defaults.f32_kv, ",").c_str()); + fprintf(stdout, " -t, --threads (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str()); + fprintf(stdout, " -ngl N, --n-gpu-layers (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str()); + fprintf(stdout, " -mg i, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str()); + fprintf(stdout, " -lv, --low-vram <0|1> (default: %s)\n", join(cmd_params_defaults.low_vram, ",").c_str()); + fprintf(stdout, " -mmq, --mul-mat-q <0|1> (default: %s)\n", join(cmd_params_defaults.mul_mat_q, ",").c_str()); + fprintf(stdout, " -ts, --tensor_split \n"); + fprintf(stdout, " -r, --repetitions (default: %d)\n", cmd_params_defaults.reps); + fprintf(stdout, " -o, --output (default: %s)\n", cmd_params_defaults.output_format == CSV ? "csv" : cmd_params_defaults.output_format == JSON ? "json" : "md"); + fprintf(stdout, " -v, --verbose (default: %s)\n", cmd_params_defaults.verbose ? "1" : "0"); + fprintf(stdout, "\n"); + fprintf(stdout, "Multiple values can be given for each parameter by separating them with ',' or by repeating the parameter.\n"); + +} + +static cmd_params parse_cmd_params(int argc, char ** argv) { + cmd_params params; + std::string arg; + bool invalid_param = false; + const std::string arg_prefix = "--"; + const char split_delim = ','; + + params.verbose = cmd_params_defaults.verbose; + params.output_format = cmd_params_defaults.output_format; + params.reps = cmd_params_defaults.reps; + + for (int i = 1; i < argc; i++) { + arg = argv[i]; + if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) { + std::replace(arg.begin(), arg.end(), '_', '-'); + } + + if (arg == "-h" || arg == "--help") { + print_usage(argc, argv); + exit(0); + } else if (arg == "-m" || arg == "--model") { + if (++i >= argc) { + invalid_param = true; + break; + } + auto p = split(argv[i], split_delim); + params.model.insert(params.model.end(), p.begin(), p.end()); + } else if (arg == "-np" || arg == "--n-prompt") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_prompt = split(argv[i], split_delim); + } else if (arg == "-n" || arg == "--n-gen") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_gen = split(argv[i], split_delim); + } else if (arg == "-b" || arg == "--batch-size") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_batch = split(argv[i], split_delim); + } else if (arg == "--memory-f32") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.f32_kv = split(argv[i], split_delim); + } else if (arg == "-t" || arg == "--threads") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_threads = split(argv[i], split_delim); + } else if (arg == "-ngl" || arg == "--n-gpu-layers") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.n_gpu_layers = split(argv[i], split_delim); + } else if (arg == "-mg" || arg == "--main-gpu") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.main_gpu = split(argv[i], split_delim); + } else if (arg == "-lv" || arg == "--low-vram") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.low_vram = split(argv[i], split_delim); + } else if (arg == "-mmq" || arg == "--mul-mat-q") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.mul_mat_q = split(argv[i], split_delim); + } else if (arg == "-ts" || arg == "--tensor-split") { + if (++i >= argc) { + invalid_param = true; + break; + } + for (auto ts : split(argv[i], split_delim)) { + // split string by , and / + const std::regex regex{R"([,/]+)"}; + std::sregex_token_iterator it{ts.begin(), ts.end(), regex, -1}; + std::vector split_arg{it, {}}; + GGML_ASSERT(split_arg.size() <= LLAMA_MAX_DEVICES); + + std::array tensor_split; + for (size_t i = 0; i < LLAMA_MAX_DEVICES; ++i) { + if (i < split_arg.size()) { + tensor_split[i] = std::stof(split_arg[i]); + } else { + tensor_split[i] = 0.0f; + } + } + params.tensor_split.push_back(tensor_split); + } + } else if (arg == "-r" || arg == "--repetitions") { + if (++i >= argc) { + invalid_param = true; + break; + } + params.reps = std::stoi(argv[i]); + } else if (arg == "-o" || arg == "--output") { + if (++i >= argc) { + invalid_param = true; + break; + } + if (argv[i] == std::string("csv")) { + params.output_format = CSV; + } else if (argv[i] == std::string("json")) { + params.output_format = JSON; + } else if (argv[i] == std::string("md")) { + params.output_format = MARKDOWN; + } else { + invalid_param = true; + break; + } + } else if (arg == "-v" || arg == "--verbose") { + params.verbose = true; + } else { + invalid_param = true; + break; + } + } + if (invalid_param) { + fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); + print_usage(argc, argv); + exit(1); + } + + // set defaults + if (params.model.empty()) { params.model = cmd_params_defaults.model; } + if (params.n_prompt.empty()) { params.n_prompt = cmd_params_defaults.n_prompt; } + if (params.n_gen.empty()) { params.n_gen = cmd_params_defaults.n_gen; } + if (params.n_batch.empty()) { params.n_batch = cmd_params_defaults.n_batch; } + if (params.f32_kv.empty()) { params.f32_kv = cmd_params_defaults.f32_kv; } + if (params.n_gpu_layers.empty()) { params.n_gpu_layers = cmd_params_defaults.n_gpu_layers; } + if (params.main_gpu.empty()) { params.main_gpu = cmd_params_defaults.main_gpu; } + if (params.mul_mat_q.empty()) { params.mul_mat_q = cmd_params_defaults.mul_mat_q; } + if (params.low_vram.empty()) { params.low_vram = cmd_params_defaults.low_vram; } + if (params.tensor_split.empty()) { params.tensor_split = cmd_params_defaults.tensor_split; } + if (params.n_threads.empty()) { params.n_threads = cmd_params_defaults.n_threads; } + + return params; +} + +struct cmd_params_instance { + std::string model; + int n_prompt; + int n_gen; + int n_batch; + bool f32_kv; + int n_gpu_layers; + int main_gpu; + bool mul_mat_q; + bool low_vram; + std::array tensor_split; + + int n_threads; + + llama_context_params to_llama_params() const { + llama_context_params lparams = llama_context_default_params(); + lparams.n_ctx = n_prompt + n_gen; + lparams.n_batch = n_batch; + lparams.f16_kv = !f32_kv; + lparams.n_gpu_layers = n_gpu_layers; + lparams.main_gpu = main_gpu; + lparams.mul_mat_q = mul_mat_q; + lparams.low_vram = low_vram; + lparams.tensor_split = tensor_split.data(); + + return lparams; + } +}; + +static std::vector get_cmd_params_instances(const cmd_params& params) { + std::vector instances; + + for (const auto& m : params.model) + for (const auto& nb : params.n_batch) + for (const auto& fk : params.f32_kv) + for (const auto& nl : params.n_gpu_layers) + for (const auto& mg : params.main_gpu) + for (const auto& mmq : params.mul_mat_q) + for (const auto& lv : params.low_vram) + for (const auto& ts : params.tensor_split) + for (const auto& nt : params.n_threads) + for (const auto& ng : params.n_gen) + for (const auto& np : params.n_prompt) { + if (np == 0 && ng == 0) continue; // no prompt and no generation + + cmd_params_instance instance; + instance.model = m; + instance.n_prompt = np; + instance.n_gen = ng; + instance.n_batch = nb; + instance.f32_kv = fk; + instance.n_gpu_layers = nl; + instance.main_gpu = mg; + instance.mul_mat_q = mmq; + instance.low_vram = lv; + std::copy(std::begin(ts), std::end(ts), std::begin(instance.tensor_split)); + instance.n_threads = nt; + + instances.push_back(instance); + } + + return instances; +} + +// models params +struct model_params { + std::string filename; + std::string type; + + static const std::vector& get_fields() { + static const std::vector fields = {"filename", "type"}; + return fields; + } + + // TODO: use a map instead + std::vector get_values() const { + return {filename, type}; + } +}; + +static bool ggml_cpu_has_metal() { +#if defined(GGML_USE_METAL) + return true; +#else + return false; +#endif +} + +// backend params +struct backend_params { + std::string build_commit = BUILD_COMMIT; + int build_number = BUILD_NUMBER; + bool cuda = ggml_cpu_has_cublas(); + bool opencl = ggml_cpu_has_clblast(); + bool metal = ggml_cpu_has_metal(); + bool gpu_blas = ggml_cpu_has_gpublas(); + bool blas = ggml_cpu_has_blas(); + int n_batch; + int n_threads; + int n_gpu_layers; + int main_gpu; + bool mul_mat_q; + bool low_vram; + std::array tensor_split; + + std::string get_backend() const { + if (cuda) { + return "CUDA"; + } + if (opencl) { + return "OpenCL"; + } + if (metal) { + return "Metal"; + } + if (gpu_blas) { + return "GPU BLAS"; + } + if (blas) { + return "BLAS"; + } + return "CPU"; + } + + static const std::vector& get_fields() { + static const std::vector fields = {"build_number", "build_commit", "cuda", "opencl", "metal", "gpu_blas", "blas", "n_batch", "n_threads", "n_gpu_layers", "main_gpu", "mul_mat_q", "low_vram", "tensor_split"}; + return fields; + } + + std::vector get_values() const { + std::string tensor_split_str; + int max_nonzero = 0; + for (int i = 0; i < LLAMA_MAX_DEVICES; i++) { + if (tensor_split[i] > 0) { + max_nonzero = i; + } + } + for (int i = 0; i < max_nonzero; i++) { + tensor_split_str += std::to_string(tensor_split[i]); + if (i < max_nonzero - 1) { + tensor_split_str += "/"; + } + } + std::vector values = { + std::to_string(build_number), build_commit, + std::to_string(cuda), std::to_string(opencl), std::to_string(metal), std::to_string(gpu_blas), std::to_string(blas), + std::to_string(n_batch), std::to_string(n_threads), + std::to_string(n_gpu_layers), std::to_string(main_gpu), std::to_string(mul_mat_q), std::to_string(low_vram), tensor_split_str + }; + return values; + } +}; + +// benchmark params +struct bench_params { + int n_prompt = 512; + int n_gen = 128; + int n_batch = 512; + bool f32_kv = true; + + static const std::vector& get_fields() { + static const std::vector fields = {"n_prompt", "n_gen", "n_batch", "f16_kv"}; + return fields; + } + + std::vector get_values() const { + return {std::to_string(n_prompt), std::to_string(n_gen), std::to_string(n_batch), std::to_string(f32_kv)}; + } +}; + +// timing results +struct timing_samples { + std::vector t_ns; + + uint64_t avg() const { + return ::avg(t_ns); + } + + uint64_t stddev() const { + return ::stddev(t_ns); + } + + std::vector get_ts(int n) const { + std::vector ts; + std::transform(t_ns.begin(), t_ns.end(), std::back_inserter(ts), [n](uint64_t t) { return 1e9 * n / t; }); + return ts; + } + + double avg_ts(uint64_t n) const { + return ::avg(get_ts(n)); + } + + double stddev_ts(uint64_t n) const { + return ::stddev(get_ts(n)); + } + + static const std::vector& get_fields() { + static const std::vector fields = {"t_ns"}; + return fields; + } +}; + +struct test { + model_params mparams = {}; + bench_params bparams = {}; + backend_params bkparams = {}; + timing_samples tsamples = {}; + + test(const cmd_params_instance& inst, const llama_model* lmodel, const llama_context* ctx) { + mparams.filename = inst.model; + char buf[128]; + llama_model_type(lmodel, buf, sizeof(buf)); + mparams.type = buf; + + bparams.n_prompt = inst.n_prompt; + bparams.n_gen = inst.n_gen; + bparams.n_batch = inst.n_batch; + bparams.f32_kv = inst.f32_kv; + + bkparams.n_batch = inst.n_batch; + bkparams.n_threads = inst.n_threads; + bkparams.n_gpu_layers = inst.n_gpu_layers; + bkparams.main_gpu = inst.main_gpu; + bkparams.mul_mat_q = inst.mul_mat_q; + bkparams.low_vram = inst.low_vram; + bkparams.tensor_split = inst.tensor_split; + + (void) ctx; + } +}; + +struct printer { + FILE * fout; + virtual void print_header(const cmd_params& params) { (void)params; }; + virtual void print_test(const test & t) = 0; + virtual void print_footer() {}; +}; + +struct csv_printer : public printer { + virtual void print_header(const cmd_params& params) { + std::vector fields; + fields.insert(fields.end(), model_params::get_fields().begin(), model_params::get_fields().end()); + fields.insert(fields.end(), bench_params::get_fields().begin(), bench_params::get_fields().end()); + fields.insert(fields.end(), backend_params::get_fields().begin(), backend_params::get_fields().end()); + fields.insert(fields.end(), timing_samples::get_fields().begin(), timing_samples::get_fields().end()); + fprintf(fout, "%s\n", join(fields, ",").c_str()); + (void) params; + } + + void print_values(const std::vector& values) { + fprintf(fout, "%s", join(values, ",").c_str()); + } + + virtual void print_test(const test & t) { + for (auto t_ns : t.tsamples.t_ns) { + print_values(t.mparams.get_values()); + print_values(t.bparams.get_values()); + print_values(t.bkparams.get_values()); + print_values({std::to_string(t_ns)}); + fprintf(fout, "\n"); + } + } +}; + +struct json_printer : public printer { + void print_fields(const std::vector& fields, const std::vector& values) { + assert(fields.size() == values.size()); + for (size_t i = 0; i < fields.size(); i++) { + fprintf(fout, " \"%s\": \"%s\",\n", fields.at(i).c_str(), values.at(i).c_str()); + } + } + + virtual void print_test(const test & t) { + fprintf(fout, "{\n"); + fprintf(fout, " \"model\": {\n"); + print_fields(model_params::get_fields(), t.mparams.get_values()); + fprintf(fout, " },\n"); + fprintf(fout, " \"benchmark\": {\n"); + print_fields(bench_params::get_fields(), t.bparams.get_values()); + fprintf(fout, " },\n"); + fprintf(fout, " \"backend\": {\n"); + print_fields(backend_params::get_fields(), t.bkparams.get_values()); + fprintf(fout, " },\n"); + fprintf(fout, " \"samples\": {\n"); + fprintf(fout, " \"ns\": [ %s ],\n", join(t.tsamples.t_ns, ", ").c_str()); + fprintf(fout, " \"avg\": %" PRIu64 ",\n", t.tsamples.avg()); + fprintf(fout, " \"stddev\": %" PRIu64 "\n", t.tsamples.stddev()); + fprintf(fout, " }\n"); + fprintf(fout, "}\n"); + } +}; + +struct markdown_printer : public printer { + std::vector fields; + + virtual void print_header(const cmd_params& params) { + // TODO: print all params that have multiple values + fprintf(fout, "| model | backend | n_gpu_layers | n_prompt | n_gen | t/s |\n"); + fprintf(fout, "| ----- | ------- | ------------ | -------- | ----- | --- |\n"); + (void) params; + } + + virtual void print_test(const test & t) { + int n_tokens = t.bparams.n_prompt + t.bparams.n_gen; + fprintf(fout, "| %s | %s | %i | %i | %i | %.2f ± %.2f |\n", + t.mparams.type.c_str(), t.bkparams.get_backend().c_str(), t.bkparams.n_gpu_layers, + t.bparams.n_prompt, t.bparams.n_gen, + t.tsamples.avg_ts(n_tokens), + t.tsamples.stddev_ts(n_tokens)); + } +}; + +void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_batch, int n_threads) { + std::vector tokens(n_batch, llama_token_bos()); + int n_processed = 0; + while (n_processed < n_prompt) { + int n = std::min(n_prompt - n_processed, n_batch); + llama_eval(ctx, tokens.data(), n, n_past + n_processed, n_threads); + n_processed += n; + } +} + +void test_gen(llama_context * ctx, int n_gen, int n_past, int n_threads) { + llama_token token = llama_token_bos(); + for (int i = 0; i < n_gen; i++) { + llama_eval(ctx, &token, 1, n_past + i, n_threads); + } +} + +void llama_null_log_callback(enum llama_log_level level, const char * text, void * user_data) { + (void)level; + (void)text; + (void)user_data; +} + +int main(int argc, char ** argv) { + cmd_params params = parse_cmd_params(argc, argv); + + // initialize llama.cpp + if (!params.verbose) { + llama_log_set(llama_null_log_callback, NULL); + } + bool numa = false; + llama_backend_init(numa); + + // initialize printer + std::unique_ptr p; + switch (params.output_format) { + case CSV: + p.reset(new csv_printer()); + break; + case JSON: + p.reset(new json_printer()); + break; + case MARKDOWN: + p.reset(new markdown_printer()); + break; + } + p->fout = stdout; + p->print_header(params); + + std::vector params_instances = get_cmd_params_instances(params); + + for (const auto & inst : params_instances) { + // TODO: keep the model between tests when possible + llama_context_params lparams = inst.to_llama_params(); + + llama_model * lmodel = llama_load_model_from_file(inst.model.c_str(), lparams); + if (lmodel == NULL) { + fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, inst.model.c_str()); + return 1; + } + + llama_context * ctx = llama_new_context_with_model(lmodel, lparams); + if (ctx == NULL) { + fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, inst.model.c_str()); + llama_free_model(lmodel); + return 1; + } + + test t(inst, lmodel, ctx); + + // warmup run + test_gen(ctx, 1, 0, t.bkparams.n_threads); + + for (int i = 0; i < params.reps; i++) { + uint64_t t_start = get_time_ns(); + if (t.bparams.n_prompt > 0) { + test_prompt(ctx, t.bparams.n_prompt, 0, t.bkparams.n_batch, t.bkparams.n_threads); + } + if (t.bparams.n_gen > 0) { + test_gen(ctx, t.bparams.n_gen, t.bparams.n_prompt, t.bkparams.n_threads); + } + uint64_t t_ns = get_time_ns() - t_start; + t.tsamples.t_ns.push_back(t_ns); + } + + p->print_test(t); + + + llama_print_timings(ctx); + + llama_free(ctx); + llama_free_model(lmodel); + } + + p->print_footer(); + + llama_backend_free(); + + return 0; +} diff --git a/llama.cpp b/llama.cpp index c8ab313d9c116..1ee04cc1d044a 100644 --- a/llama.cpp +++ b/llama.cpp @@ -984,7 +984,7 @@ int64_t llama_time_us() { // model loading // -static const char *llama_file_version_name(llama_file_version version) { +static const char * llama_file_version_name(llama_file_version version) { switch (version) { case LLAMA_FILE_VERSION_GGML: return "'ggml' (old version with low tokenizer quality and no mmap support)"; case LLAMA_FILE_VERSION_GGMF_V1: return "ggmf v1 (old version with no mmap support)"; @@ -996,7 +996,7 @@ static const char *llama_file_version_name(llama_file_version version) { return "unknown"; } -static const char *llama_ftype_name(enum llama_ftype ftype) { +const char * llama_ftype_name(enum llama_ftype ftype) { switch (ftype) { case LLAMA_FTYPE_ALL_F32: return "all F32"; case LLAMA_FTYPE_MOSTLY_F16: return "mostly F16"; @@ -1021,7 +1021,7 @@ static const char *llama_ftype_name(enum llama_ftype ftype) { } } -static const char *llama_model_type_name(e_model type) { +static const char * llama_model_type_name(e_model type) { switch (type) { case MODEL_3B: return "3B"; case MODEL_7B: return "7B"; @@ -1799,6 +1799,13 @@ static bool llama_eval_internal( LLAMA_ASSERT((!tokens && embd) || (tokens && !embd)); + LLAMA_ASSERT(n_tokens > 0); + LLAMA_ASSERT(n_past >= 0); + LLAMA_ASSERT(n_threads > 0); + // TODO: keep the values of n_batch and n_ctx + // LLAMA_ASSERT(n_tokens <= n_batch); + // LLAMA_ASSERT(n_past + n_tokens <= n_ctx); + const int64_t t_start_us = ggml_time_us(); #ifdef GGML_USE_MPI @@ -4179,6 +4186,10 @@ int llama_n_embd(const struct llama_context * ctx) { return ctx->model.hparams.n_embd; } +int llama_model_type(const struct llama_model * model, char * buf, size_t buf_size) { + return snprintf(buf, buf_size, "LLaMA %s %s", llama_model_type_name(model->type), llama_ftype_name(model->hparams.ftype)); +} + int llama_get_vocab_from_model( const struct llama_model * model, const char * * strings, diff --git a/llama.h b/llama.h index 92b474891493e..9d732f914cbb1 100644 --- a/llama.h +++ b/llama.h @@ -351,6 +351,8 @@ extern "C" { LLAMA_API int llama_n_ctx_from_model (const struct llama_model * model); LLAMA_API int llama_n_embd_from_model (const struct llama_model * model); + LLAMA_API int llama_model_type(const struct llama_model * model, char * buf, size_t buf_size); + // Get the vocabulary as output parameters. // Returns number of results. LLAMA_API int llama_get_vocab( From 7ec6158eec46cdc95fcecd6bee51311ae93dcdbf Mon Sep 17 00:00:00 2001 From: slaren Date: Tue, 15 Aug 2023 22:50:38 +0200 Subject: [PATCH 02/19] add to examples CMakeLists.txt --- examples/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index b5d9bb29e6ad8..d53652815e83e 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -45,6 +45,7 @@ else() add_subdirectory(convert-llama2c-to-ggml) add_subdirectory(simple) add_subdirectory(embd-input) + add_subdirectory(llama-bench) if (LLAMA_METAL) add_subdirectory(metal) endif() From 6597d61ad7f981233ad1bf00400d42c5c439f784 Mon Sep 17 00:00:00 2001 From: slaren Date: Tue, 15 Aug 2023 22:55:05 +0200 Subject: [PATCH 03/19] fix msvc build --- examples/llama-bench/llama-bench.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 682b5cec93e40..bc44bf890e9ae 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -372,11 +372,11 @@ static bool ggml_cpu_has_metal() { struct backend_params { std::string build_commit = BUILD_COMMIT; int build_number = BUILD_NUMBER; - bool cuda = ggml_cpu_has_cublas(); - bool opencl = ggml_cpu_has_clblast(); - bool metal = ggml_cpu_has_metal(); - bool gpu_blas = ggml_cpu_has_gpublas(); - bool blas = ggml_cpu_has_blas(); + bool cuda = !!ggml_cpu_has_cublas(); + bool opencl = !!ggml_cpu_has_clblast(); + bool metal = !!ggml_cpu_has_metal(); + bool gpu_blas = !!ggml_cpu_has_gpublas(); + bool blas = !!ggml_cpu_has_blas(); int n_batch; int n_threads; int n_gpu_layers; From 6ab6971242201d5f44534639fac550d3bec94e6e Mon Sep 17 00:00:00 2001 From: slaren Date: Tue, 15 Aug 2023 23:02:07 +0200 Subject: [PATCH 04/19] add missing include --- examples/llama-bench/llama-bench.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index bc44bf890e9ae..edf19f2851052 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include "ggml.h" #include "llama.h" #include "common.h" From 52b94f42c89fa562cead22d982faa045ba464d95 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 16 Aug 2023 00:25:59 +0200 Subject: [PATCH 05/19] add Bessel's correction to stdev calculation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Johannes Gäßler --- examples/llama-bench/llama-bench.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index edf19f2851052..7b75b3082a7d6 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -63,7 +63,7 @@ T stddev(const std::vector& v) { } T mean = avg(v); T sq_sum = std::inner_product(v.begin(), v.end(), v.begin(), T(0)); - T stdev = std::sqrt(sq_sum / (T)v.size() - mean * mean); + T stdev = std::sqrt(sq_sum / (T)(v.size() - 1) - mean * mean * (T)v.size() / (T)(v.size() - 1)); return stdev; } From f2cf01ddd20614ad55a063760ed8773ddd073ab4 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 16 Aug 2023 02:39:15 +0200 Subject: [PATCH 06/19] improve markdown formatting --- examples/llama-bench/llama-bench.cpp | 238 +++++++++++++++++++-------- 1 file changed, 169 insertions(+), 69 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 7b75b3082a7d6..04df8a1059074 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -22,7 +22,7 @@ static uint64_t get_time_ns() { } template -static std::string join(const std::vector& values, const std::string& delim) { +static std::string join(const std::vector & values, const std::string & delim) { std::ostringstream str; for (size_t i = 0; i < values.size(); i++) { str << values[i]; @@ -34,7 +34,7 @@ static std::string join(const std::vector& values, const std::string& delim) } template -static std::vector split(const std::string& str, char delim) { +static std::vector split(const std::string & str, char delim) { std::vector values; std::istringstream str_stream(str); std::string token; @@ -48,7 +48,7 @@ static std::vector split(const std::string& str, char delim) { } template -T avg(const std::vector& v) { +T avg(const std::vector & v) { if (v.empty()) { return 0; } @@ -57,7 +57,7 @@ T avg(const std::vector& v) { } template -T stddev(const std::vector& v) { +T stdev(const std::vector & v) { if (v.size() <= 1) { return 0; } @@ -110,7 +110,7 @@ static void print_usage(int /* argc */, char ** argv) { fprintf(stdout, "options:\n"); fprintf(stdout, " -h, --help\n"); fprintf(stdout, " -m, --model (default: %s)\n", join(cmd_params_defaults.model, ",").c_str()); - fprintf(stdout, " -np, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); + fprintf(stdout, " -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); fprintf(stdout, " -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str()); fprintf(stdout, " -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str()); fprintf(stdout, " --memory-f32 <0|1> (default: %s)\n", join(cmd_params_defaults.f32_kv, ",").c_str()); @@ -155,42 +155,48 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { } auto p = split(argv[i], split_delim); params.model.insert(params.model.end(), p.begin(), p.end()); - } else if (arg == "-np" || arg == "--n-prompt") { + } else if (arg == "-p" || arg == "--n-prompt") { if (++i >= argc) { invalid_param = true; break; } - params.n_prompt = split(argv[i], split_delim); + auto p = split(argv[i], split_delim); + params.n_prompt.insert(params.n_prompt.end(), p.begin(), p.end()); } else if (arg == "-n" || arg == "--n-gen") { if (++i >= argc) { invalid_param = true; break; } - params.n_gen = split(argv[i], split_delim); + auto p = split(argv[i], split_delim); + params.n_gen.insert(params.n_gen.end(), p.begin(), p.end()); } else if (arg == "-b" || arg == "--batch-size") { if (++i >= argc) { invalid_param = true; break; } - params.n_batch = split(argv[i], split_delim); + auto p = split(argv[i], split_delim); + params.n_batch.insert(params.n_batch.end(), p.begin(), p.end()); } else if (arg == "--memory-f32") { if (++i >= argc) { invalid_param = true; break; } - params.f32_kv = split(argv[i], split_delim); + auto p = split(argv[i], split_delim); + params.f32_kv.insert(params.f32_kv.end(), p.begin(), p.end()); } else if (arg == "-t" || arg == "--threads") { if (++i >= argc) { invalid_param = true; break; } - params.n_threads = split(argv[i], split_delim); + auto p = split(argv[i], split_delim); + params.n_threads.insert(params.n_threads.end(), p.begin(), p.end()); } else if (arg == "-ngl" || arg == "--n-gpu-layers") { if (++i >= argc) { invalid_param = true; break; } - params.n_gpu_layers = split(argv[i], split_delim); + auto p = split(argv[i], split_delim); + params.n_gpu_layers.insert(params.n_gpu_layers.end(), p.begin(), p.end()); } else if (arg == "-mg" || arg == "--main-gpu") { if (++i >= argc) { invalid_param = true; @@ -202,21 +208,23 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { invalid_param = true; break; } - params.low_vram = split(argv[i], split_delim); + auto p = split(argv[i], split_delim); + params.low_vram.insert(params.low_vram.end(), p.begin(), p.end()); } else if (arg == "-mmq" || arg == "--mul-mat-q") { if (++i >= argc) { invalid_param = true; break; } - params.mul_mat_q = split(argv[i], split_delim); + auto p = split(argv[i], split_delim); + params.mul_mat_q.insert(params.mul_mat_q.end(), p.begin(), p.end()); } else if (arg == "-ts" || arg == "--tensor-split") { if (++i >= argc) { invalid_param = true; break; } for (auto ts : split(argv[i], split_delim)) { - // split string by , and / - const std::regex regex{R"([,/]+)"}; + // split string by ; and / + const std::regex regex{R"([;/]+)"}; std::sregex_token_iterator it{ts.begin(), ts.end(), regex, -1}; std::vector split_arg{it, {}}; GGML_ASSERT(split_arg.size() <= LLAMA_MAX_DEVICES); @@ -310,20 +318,20 @@ struct cmd_params_instance { } }; -static std::vector get_cmd_params_instances(const cmd_params& params) { +static std::vector get_cmd_params_instances(const cmd_params & params) { std::vector instances; - for (const auto& m : params.model) - for (const auto& nb : params.n_batch) - for (const auto& fk : params.f32_kv) - for (const auto& nl : params.n_gpu_layers) - for (const auto& mg : params.main_gpu) - for (const auto& mmq : params.mul_mat_q) - for (const auto& lv : params.low_vram) - for (const auto& ts : params.tensor_split) - for (const auto& nt : params.n_threads) - for (const auto& ng : params.n_gen) - for (const auto& np : params.n_prompt) { + for (const auto & m : params.model) + for (const auto & nb : params.n_batch) + for (const auto & fk : params.f32_kv) + for (const auto & nl : params.n_gpu_layers) + for (const auto & mg : params.main_gpu) + for (const auto & mmq : params.mul_mat_q) + for (const auto & lv : params.low_vram) + for (const auto & ts : params.tensor_split) + for (const auto & nt : params.n_threads) + for (const auto & ng : params.n_gen) + for (const auto & np : params.n_prompt) { if (np == 0 && ng == 0) continue; // no prompt and no generation cmd_params_instance instance; @@ -350,7 +358,7 @@ struct model_params { std::string filename; std::string type; - static const std::vector& get_fields() { + static const std::vector & get_fields() { static const std::vector fields = {"filename", "type"}; return fields; } @@ -371,22 +379,23 @@ static bool ggml_cpu_has_metal() { // backend params struct backend_params { - std::string build_commit = BUILD_COMMIT; - int build_number = BUILD_NUMBER; - bool cuda = !!ggml_cpu_has_cublas(); - bool opencl = !!ggml_cpu_has_clblast(); - bool metal = !!ggml_cpu_has_metal(); - bool gpu_blas = !!ggml_cpu_has_gpublas(); - bool blas = !!ggml_cpu_has_blas(); + static const std::string build_commit; + static const int build_number; + static const bool cuda; + static const bool opencl; + static const bool metal; + static const bool gpu_blas; + static const bool blas; int n_batch; int n_threads; + bool f32_kv; int n_gpu_layers; int main_gpu; bool mul_mat_q; bool low_vram; std::array tensor_split; - std::string get_backend() const { + static std::string get_backend() { if (cuda) { return "CUDA"; } @@ -405,8 +414,13 @@ struct backend_params { return "CPU"; } - static const std::vector& get_fields() { - static const std::vector fields = {"build_number", "build_commit", "cuda", "opencl", "metal", "gpu_blas", "blas", "n_batch", "n_threads", "n_gpu_layers", "main_gpu", "mul_mat_q", "low_vram", "tensor_split"}; + static const std::vector & get_fields() { + static const std::vector fields = { + "build_number", "build_commit", + "cuda", "opencl", "metal", "gpu_blas", "blas", + "n_batch", "n_threads", "f16_kv", + "n_gpu_layers", "main_gpu", "mul_mat_q", "low_vram", "tensor_split" + }; return fields; } @@ -427,27 +441,33 @@ struct backend_params { std::vector values = { std::to_string(build_number), build_commit, std::to_string(cuda), std::to_string(opencl), std::to_string(metal), std::to_string(gpu_blas), std::to_string(blas), - std::to_string(n_batch), std::to_string(n_threads), + std::to_string(n_batch), std::to_string(n_threads), std::to_string(!f32_kv), std::to_string(n_gpu_layers), std::to_string(main_gpu), std::to_string(mul_mat_q), std::to_string(low_vram), tensor_split_str }; return values; } }; +const std::string backend_params::build_commit = BUILD_COMMIT; +const int backend_params::build_number = BUILD_NUMBER; +const bool backend_params::cuda = !!ggml_cpu_has_cublas(); +const bool backend_params::opencl = !!ggml_cpu_has_clblast(); +const bool backend_params::metal = !!ggml_cpu_has_metal(); +const bool backend_params::gpu_blas = !!ggml_cpu_has_gpublas(); +const bool backend_params::blas = !!ggml_cpu_has_blas(); + // benchmark params struct bench_params { - int n_prompt = 512; - int n_gen = 128; - int n_batch = 512; - bool f32_kv = true; + int n_prompt ; + int n_gen; - static const std::vector& get_fields() { - static const std::vector fields = {"n_prompt", "n_gen", "n_batch", "f16_kv"}; + static const std::vector & get_fields() { + static const std::vector fields = {"n_prompt", "n_gen"}; return fields; } std::vector get_values() const { - return {std::to_string(n_prompt), std::to_string(n_gen), std::to_string(n_batch), std::to_string(f32_kv)}; + return {std::to_string(n_prompt), std::to_string(n_gen)}; } }; @@ -459,8 +479,8 @@ struct timing_samples { return ::avg(t_ns); } - uint64_t stddev() const { - return ::stddev(t_ns); + uint64_t stdev() const { + return ::stdev(t_ns); } std::vector get_ts(int n) const { @@ -474,10 +494,10 @@ struct timing_samples { } double stddev_ts(uint64_t n) const { - return ::stddev(get_ts(n)); + return ::stdev(get_ts(n)); } - static const std::vector& get_fields() { + static const std::vector & get_fields() { static const std::vector fields = {"t_ns"}; return fields; } @@ -489,7 +509,7 @@ struct test { backend_params bkparams = {}; timing_samples tsamples = {}; - test(const cmd_params_instance& inst, const llama_model* lmodel, const llama_context* ctx) { + test(const cmd_params_instance & inst, const llama_model * lmodel, const llama_context * ctx) { mparams.filename = inst.model; char buf[128]; llama_model_type(lmodel, buf, sizeof(buf)); @@ -497,10 +517,9 @@ struct test { bparams.n_prompt = inst.n_prompt; bparams.n_gen = inst.n_gen; - bparams.n_batch = inst.n_batch; - bparams.f32_kv = inst.f32_kv; bkparams.n_batch = inst.n_batch; + bkparams.f32_kv = inst.f32_kv; bkparams.n_threads = inst.n_threads; bkparams.n_gpu_layers = inst.n_gpu_layers; bkparams.main_gpu = inst.main_gpu; @@ -514,13 +533,13 @@ struct test { struct printer { FILE * fout; - virtual void print_header(const cmd_params& params) { (void)params; }; + virtual void print_header(const cmd_params & params) { (void)params; }; virtual void print_test(const test & t) = 0; virtual void print_footer() {}; }; struct csv_printer : public printer { - virtual void print_header(const cmd_params& params) { + virtual void print_header(const cmd_params & params) { std::vector fields; fields.insert(fields.end(), model_params::get_fields().begin(), model_params::get_fields().end()); fields.insert(fields.end(), bench_params::get_fields().begin(), bench_params::get_fields().end()); @@ -530,7 +549,7 @@ struct csv_printer : public printer { (void) params; } - void print_values(const std::vector& values) { + void print_values(const std::vector & values) { fprintf(fout, "%s", join(values, ",").c_str()); } @@ -546,7 +565,7 @@ struct csv_printer : public printer { }; struct json_printer : public printer { - void print_fields(const std::vector& fields, const std::vector& values) { + void print_fields(const std::vector & fields, const std::vector & values) { assert(fields.size() == values.size()); for (size_t i = 0; i < fields.size(); i++) { fprintf(fout, " \"%s\": \"%s\",\n", fields.at(i).c_str(), values.at(i).c_str()); @@ -567,7 +586,7 @@ struct json_printer : public printer { fprintf(fout, " \"samples\": {\n"); fprintf(fout, " \"ns\": [ %s ],\n", join(t.tsamples.t_ns, ", ").c_str()); fprintf(fout, " \"avg\": %" PRIu64 ",\n", t.tsamples.avg()); - fprintf(fout, " \"stddev\": %" PRIu64 "\n", t.tsamples.stddev()); + fprintf(fout, " \"stddev\": %" PRIu64 "\n", t.tsamples.stdev()); fprintf(fout, " }\n"); fprintf(fout, "}\n"); } @@ -576,20 +595,102 @@ struct json_printer : public printer { struct markdown_printer : public printer { std::vector fields; - virtual void print_header(const cmd_params& params) { - // TODO: print all params that have multiple values - fprintf(fout, "| model | backend | n_gpu_layers | n_prompt | n_gen | t/s |\n"); - fprintf(fout, "| ----- | ------- | ------------ | -------- | ----- | --- |\n"); + static int get_field_width(const std::string & field) { + if (field == "model") { + return -30; + } + if (field == "t/s") { + return 15; + } + int width = std::max((int)field.length(), 10); + if (field == "backend") { + return -width; + } + return width; + } + + virtual void print_header(const cmd_params & params) { + fields = { "model", "backend" }; + if (backend_params::get_backend() != "CPU") { + fields.push_back("n_gpu_layers"); + } + if (params.n_batch.size() > 1) { + fields.push_back("n_batch"); + } + if (params.n_threads.size() > 1 || backend_params::get_backend() == "CPU") { + fields.push_back("n_threads"); + } + if (params.f32_kv.size() > 1) { + fields.push_back("f32_kv"); + } + if (params.main_gpu.size() > 1) { + fields.push_back("main_gpu"); + } + if (params.mul_mat_q.size() > 1) { + fields.push_back("mul_mat_q"); + } + if (params.low_vram.size() > 1) { + fields.push_back("low_vram"); + } + if (params.n_prompt.size() > 1 || (params.n_prompt.size() == 1 && params.n_prompt.at(0) != 0)) { + fields.push_back("n_prompt"); + } + if (params.n_gen.size() > 1 || (params.n_gen.size() == 1 && params.n_gen.at(0) != 0)) { + fields.push_back("n_gen"); + } + fields.push_back("t/s"); + + fprintf(fout, "|"); + for (const auto & field : fields) { + fprintf(fout, " %*s |", get_field_width(field), field.c_str()); + } + fprintf(fout, "\n"); + fprintf(fout, "|"); + for (const auto & field: fields) { + int width = get_field_width(field); + fprintf(fout, " %s%s%s |", width < 0 ? ":" : "", std::string(std::abs(width) - 1, '-').c_str(), width > 0 ? ":" : ""); + } + fprintf(fout, "\n"); (void) params; } virtual void print_test(const test & t) { int n_tokens = t.bparams.n_prompt + t.bparams.n_gen; - fprintf(fout, "| %s | %s | %i | %i | %i | %.2f ± %.2f |\n", - t.mparams.type.c_str(), t.bkparams.get_backend().c_str(), t.bkparams.n_gpu_layers, - t.bparams.n_prompt, t.bparams.n_gen, - t.tsamples.avg_ts(n_tokens), - t.tsamples.stddev_ts(n_tokens)); + + std::map vmap; + std::transform(model_params::get_fields().begin(), model_params::get_fields().end(), t.mparams.get_values().begin(), + std::inserter(vmap, vmap.end()), std::make_pair); + std::transform(bench_params::get_fields().begin(), bench_params::get_fields().end(), t.bparams.get_values().begin(), + std::inserter(vmap, vmap.end()), std::make_pair); + std::transform(backend_params::get_fields().begin(), backend_params::get_fields().end(), t.bkparams.get_values().begin(), + std::inserter(vmap, vmap.end()), std::make_pair); + + fprintf(fout, "|"); + for (const auto & field : fields) { + std::string value; + if (field == "model") { + value = t.mparams.type; + } else if (field == "backend") { + value = backend_params::get_backend(); + } else if (field == "t/s") { + char buf[128]; + snprintf(buf, sizeof(buf), "%.2f ± %.2f", t.tsamples.avg_ts(n_tokens), t.tsamples.stddev_ts(n_tokens)); + value = buf; + } else if (vmap.find(field) != vmap.end()) { + value = vmap.at(field); + } else { + assert(false); + exit(1); + } + + int width = get_field_width(field); + if (field == "t/s") { + // HACK: the utf-8 character is 2 bytes + width += 1; + } + fprintf(fout, " %*s |", width, value.c_str()); + } + fprintf(fout, "\n"); } }; @@ -680,7 +781,6 @@ int main(int argc, char ** argv) { p->print_test(t); - llama_print_timings(ctx); llama_free(ctx); From f9bbc6f281d1adaa54b9e054fcb97104179cd064 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 16 Aug 2023 03:56:52 +0200 Subject: [PATCH 07/19] add missing include --- examples/llama-bench/llama-bench.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 04df8a1059074..e40204d8249a7 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include "ggml.h" #include "llama.h" #include "common.h" From 19e9beabb3a866e0f254ceaff5008399cef5a55d Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 16 Aug 2023 15:36:56 +0200 Subject: [PATCH 08/19] print warning is NDEBUG is not defined --- examples/llama-bench/llama-bench.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index e40204d8249a7..2e845a0ffa0f5 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -111,7 +111,7 @@ static void print_usage(int /* argc */, char ** argv) { fprintf(stdout, "options:\n"); fprintf(stdout, " -h, --help\n"); fprintf(stdout, " -m, --model (default: %s)\n", join(cmd_params_defaults.model, ",").c_str()); - fprintf(stdout, " -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); + fprintf(stdout, " -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); fprintf(stdout, " -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str()); fprintf(stdout, " -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str()); fprintf(stdout, " --memory-f32 <0|1> (default: %s)\n", join(cmd_params_defaults.f32_kv, ",").c_str()); @@ -450,12 +450,12 @@ struct backend_params { }; const std::string backend_params::build_commit = BUILD_COMMIT; -const int backend_params::build_number = BUILD_NUMBER; -const bool backend_params::cuda = !!ggml_cpu_has_cublas(); -const bool backend_params::opencl = !!ggml_cpu_has_clblast(); -const bool backend_params::metal = !!ggml_cpu_has_metal(); +const int backend_params::build_number = BUILD_NUMBER; +const bool backend_params::cuda = !!ggml_cpu_has_cublas(); +const bool backend_params::opencl = !!ggml_cpu_has_clblast(); +const bool backend_params::metal = !!ggml_cpu_has_metal(); const bool backend_params::gpu_blas = !!ggml_cpu_has_gpublas(); -const bool backend_params::blas = !!ggml_cpu_has_blas(); +const bool backend_params::blas = !!ggml_cpu_has_blas(); // benchmark params struct bench_params { @@ -719,6 +719,10 @@ void llama_null_log_callback(enum llama_log_level level, const char * text, void } int main(int argc, char ** argv) { +#ifndef NDEBUG + fprintf(stderr, "warning: NDEBUG is not defined, performance may be affected\n"); +#endif + cmd_params params = parse_cmd_params(argc, argv); // initialize llama.cpp From 3e3396e2e5b14a2b302e50fa6cd4526e47ccdc87 Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 16 Aug 2023 15:45:39 +0200 Subject: [PATCH 09/19] remove n_prompt and n_gen from the matrix, use each value separately instead --- examples/llama-bench/llama-bench.cpp | 56 ++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 2e845a0ffa0f5..883e0e89301b4 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -90,8 +90,8 @@ struct cmd_params { static cmd_params cmd_params_defaults = { /* model */ {"models/7B/ggml-model-q4_0.bin"}, - /* n_prompt */ {0, 512}, - /* n_gen */ {0, 128}, + /* n_prompt */ {512}, + /* n_gen */ {128}, /* n_batch */ {512}, /* f32_kv */ {false}, /* n_threads */ {get_num_physical_cores()}, @@ -319,7 +319,7 @@ struct cmd_params_instance { } }; -static std::vector get_cmd_params_instances(const cmd_params & params) { +static std::vector get_cmd_params_instances_int(const cmd_params & params, int n_gen, int n_prompt) { std::vector instances; for (const auto & m : params.model) @@ -330,15 +330,12 @@ static std::vector get_cmd_params_instances(const cmd_param for (const auto & mmq : params.mul_mat_q) for (const auto & lv : params.low_vram) for (const auto & ts : params.tensor_split) - for (const auto & nt : params.n_threads) - for (const auto & ng : params.n_gen) - for (const auto & np : params.n_prompt) { - if (np == 0 && ng == 0) continue; // no prompt and no generation - + for (const auto & nt : params.n_threads) { cmd_params_instance instance; + instance.model = m; - instance.n_prompt = np; - instance.n_gen = ng; + instance.n_prompt = n_prompt; + instance.n_gen = n_gen; instance.n_batch = nb; instance.f32_kv = fk; instance.n_gpu_layers = nl; @@ -350,6 +347,27 @@ static std::vector get_cmd_params_instances(const cmd_param instances.push_back(instance); } + return instances; +} + +static std::vector get_cmd_params_instances(const cmd_params & params) { + std::vector instances; + + for (const auto & n_prompt : params.n_prompt) { + if (n_prompt == 0) { + continue; + } + auto instances_prompt = get_cmd_params_instances_int(params, 0, n_prompt); + instances.insert(instances.end(), instances_prompt.begin(), instances_prompt.end()); + } + + for (const auto & n_gen : params.n_gen) { + if (n_gen == 0) { + continue; + } + auto instances_gen = get_cmd_params_instances_int(params, n_gen, 0); + instances.insert(instances.end(), instances_gen.begin(), instances_gen.end()); + } return instances; } @@ -633,12 +651,7 @@ struct markdown_printer : public printer { if (params.low_vram.size() > 1) { fields.push_back("low_vram"); } - if (params.n_prompt.size() > 1 || (params.n_prompt.size() == 1 && params.n_prompt.at(0) != 0)) { - fields.push_back("n_prompt"); - } - if (params.n_gen.size() > 1 || (params.n_gen.size() == 1 && params.n_gen.at(0) != 0)) { - fields.push_back("n_gen"); - } + fields.push_back("test"); fields.push_back("t/s"); fprintf(fout, "|"); @@ -673,6 +686,17 @@ struct markdown_printer : public printer { value = t.mparams.type; } else if (field == "backend") { value = backend_params::get_backend(); + } else if (field == "test") { + char buf[128]; + if (t.bparams.n_prompt > 0) { + snprintf(buf, sizeof(buf), "pp %d", t.bparams.n_prompt); + } else if (t.bparams.n_gen > 0) { + snprintf(buf, sizeof(buf), "tg %d", t.bparams.n_gen); + } else { + assert(false); + exit(1); + } + value = buf; } else if (field == "t/s") { char buf[128]; snprintf(buf, sizeof(buf), "%.2f ± %.2f", t.tsamples.avg_ts(n_tokens), t.tsamples.stddev_ts(n_tokens)); From 5765f90f5873647e56cd6c30dd79c0f9b96af4de Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 16 Aug 2023 19:24:54 +0200 Subject: [PATCH 10/19] better checks for non-optimized builds --- examples/llama-bench/llama-bench.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 883e0e89301b4..79a04d97cb126 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -88,7 +88,7 @@ struct cmd_params { output_formats output_format; }; -static cmd_params cmd_params_defaults = { +static const cmd_params cmd_params_defaults = { /* model */ {"models/7B/ggml-model-q4_0.bin"}, /* n_prompt */ {512}, /* n_gen */ {128}, @@ -688,9 +688,9 @@ struct markdown_printer : public printer { value = backend_params::get_backend(); } else if (field == "test") { char buf[128]; - if (t.bparams.n_prompt > 0) { + if (t.bparams.n_prompt > 0 && t.bparams.n_gen == 0) { snprintf(buf, sizeof(buf), "pp %d", t.bparams.n_prompt); - } else if (t.bparams.n_gen > 0) { + } else if (t.bparams.n_gen > 0 && t.bparams.n_prompt == 0) { snprintf(buf, sizeof(buf), "tg %d", t.bparams.n_gen); } else { assert(false); @@ -743,8 +743,16 @@ void llama_null_log_callback(enum llama_log_level level, const char * text, void } int main(int argc, char ** argv) { -#ifndef NDEBUG - fprintf(stderr, "warning: NDEBUG is not defined, performance may be affected\n"); +#if !defined(NDEBUG) + fprintf(stderr, "warning: asserts enabled, performance may be affected\n"); +#endif + +#if (defined(_MSC_VER) && defined(_DEBUG)) || (!defined(_MSC_VER) && !defined(__OPTIMIZE__)) + fprintf(stderr, "warning: debug build, performance may be affected\n"); +#endif + +#if defined(__SANITIZE_ADDRESS__) || defined(__SANITIZE_THREAD__) + fprintf(stderr, "warning: sanitizer enabled, performance may be affected\n"); #endif cmd_params params = parse_cmd_params(argc, argv); From 89a70f78e74220a6967df95ed0c25a80510cebdf Mon Sep 17 00:00:00 2001 From: slaren Date: Wed, 16 Aug 2023 22:40:53 +0200 Subject: [PATCH 11/19] llama.cpp : fix MEM_REQ_SCRATCH0 reusing the value of n_ctx of the first call --- examples/llama-bench/llama-bench.cpp | 19 ++++++++++--------- llama.cpp | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 79a04d97cb126..e815969ad2b3a 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -477,7 +477,7 @@ const bool backend_params::blas = !!ggml_cpu_has_blas(); // benchmark params struct bench_params { - int n_prompt ; + int n_prompt; int n_gen; static const std::vector & get_fields() { @@ -630,17 +630,18 @@ struct markdown_printer : public printer { virtual void print_header(const cmd_params & params) { fields = { "model", "backend" }; - if (backend_params::get_backend() != "CPU") { + bool is_cpu_backend = backend_params::get_backend() == "CPU" || backend_params::get_backend() == "BLAS"; + if (!is_cpu_backend) { fields.push_back("n_gpu_layers"); } + if (params.n_threads.size() > 1 || is_cpu_backend) { + fields.push_back("n_threads"); + } if (params.n_batch.size() > 1) { fields.push_back("n_batch"); } - if (params.n_threads.size() > 1 || backend_params::get_backend() == "CPU") { - fields.push_back("n_threads"); - } if (params.f32_kv.size() > 1) { - fields.push_back("f32_kv"); + fields.push_back("f16_kv"); } if (params.main_gpu.size() > 1) { fields.push_back("main_gpu"); @@ -723,9 +724,9 @@ void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_batch, int std::vector tokens(n_batch, llama_token_bos()); int n_processed = 0; while (n_processed < n_prompt) { - int n = std::min(n_prompt - n_processed, n_batch); - llama_eval(ctx, tokens.data(), n, n_past + n_processed, n_threads); - n_processed += n; + int n_tokens = std::min(n_prompt - n_processed, n_batch); + llama_eval(ctx, tokens.data(), n_tokens, n_past + n_processed, n_threads); + n_processed += n_tokens; } } diff --git a/llama.cpp b/llama.cpp index 1ee04cc1d044a..3c7dd9530c21f 100644 --- a/llama.cpp +++ b/llama.cpp @@ -115,9 +115,9 @@ static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * // memory sizes (calculated for n_batch == 512) // -static const std::map & MEM_REQ_SCRATCH0(int n_ctx) +static std::map MEM_REQ_SCRATCH0(int n_ctx) { - static std::map k_sizes = { + std::map k_sizes = { { MODEL_3B, ((size_t) n_ctx / 16ull + 92ull) * MB }, { MODEL_7B, ((size_t) n_ctx / 16ull + 100ull) * MB }, { MODEL_13B, ((size_t) n_ctx / 12ull + 120ull) * MB }, From 314a6b54228c93d4a6fe1ba306712bb8872a9ba3 Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 17 Aug 2023 00:12:40 +0200 Subject: [PATCH 12/19] fix json formatting --- examples/llama-bench/llama-bench.cpp | 49 ++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index e815969ad2b3a..b963ae29bd5fa 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -584,30 +584,51 @@ struct csv_printer : public printer { }; struct json_printer : public printer { + bool first = true; + void print_fields(const std::vector & fields, const std::vector & values) { assert(fields.size() == values.size()); for (size_t i = 0; i < fields.size(); i++) { - fprintf(fout, " \"%s\": \"%s\",\n", fields.at(i).c_str(), values.at(i).c_str()); + fprintf(fout, " \"%s\": \"%s\"", fields.at(i).c_str(), values.at(i).c_str()); + if (i < fields.size() - 1) { + fprintf(fout, ",\n"); + } else { + fprintf(fout, "\n"); + } } } + virtual void print_header(const cmd_params & params) { + fprintf(fout, "[\n"); + (void) params; + } + + virtual void print_footer() { + fprintf(fout, "\n]\n"); + } + virtual void print_test(const test & t) { - fprintf(fout, "{\n"); - fprintf(fout, " \"model\": {\n"); + if (first) { + first = false; + } else { + fprintf(fout, ",\n"); + } + fprintf(fout, " {\n"); + fprintf(fout, " \"model\": {\n"); print_fields(model_params::get_fields(), t.mparams.get_values()); - fprintf(fout, " },\n"); - fprintf(fout, " \"benchmark\": {\n"); + fprintf(fout, " },\n"); + fprintf(fout, " \"benchmark\": {\n"); print_fields(bench_params::get_fields(), t.bparams.get_values()); - fprintf(fout, " },\n"); - fprintf(fout, " \"backend\": {\n"); + fprintf(fout, " },\n"); + fprintf(fout, " \"backend\": {\n"); print_fields(backend_params::get_fields(), t.bkparams.get_values()); - fprintf(fout, " },\n"); - fprintf(fout, " \"samples\": {\n"); - fprintf(fout, " \"ns\": [ %s ],\n", join(t.tsamples.t_ns, ", ").c_str()); - fprintf(fout, " \"avg\": %" PRIu64 ",\n", t.tsamples.avg()); - fprintf(fout, " \"stddev\": %" PRIu64 "\n", t.tsamples.stdev()); - fprintf(fout, " }\n"); - fprintf(fout, "}\n"); + fprintf(fout, " },\n"); + fprintf(fout, " \"samples\": {\n"); + fprintf(fout, " \"ns\": [ %s ],\n", join(t.tsamples.t_ns, ", ").c_str()); + fprintf(fout, " \"avg\": %" PRIu64 ",\n", t.tsamples.avg()); + fprintf(fout, " \"stddev\": %" PRIu64 "\n", t.tsamples.stdev()); + fprintf(fout, " }\n"); + fprintf(fout, " }"); } }; From 67362d9db08e0ba8d704464962b4840d701a464b Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 17 Aug 2023 02:19:52 +0200 Subject: [PATCH 13/19] add sql output --- examples/llama-bench/llama-bench.cpp | 340 ++++++++++++++------------- 1 file changed, 174 insertions(+), 166 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index b963ae29bd5fa..9c654d2815a55 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "ggml.h" #include "llama.h" #include "common.h" @@ -68,8 +69,17 @@ T stdev(const std::vector & v) { return stdev; } +static bool ggml_cpu_has_metal() { +#if defined(GGML_USE_METAL) + return true; +#else + return false; +#endif +} + + // command line params -enum output_formats {CSV, JSON, MARKDOWN}; +enum output_formats {CSV, JSON, MARKDOWN, SQL}; struct cmd_params { std::vector model; @@ -122,7 +132,7 @@ static void print_usage(int /* argc */, char ** argv) { fprintf(stdout, " -mmq, --mul-mat-q <0|1> (default: %s)\n", join(cmd_params_defaults.mul_mat_q, ",").c_str()); fprintf(stdout, " -ts, --tensor_split \n"); fprintf(stdout, " -r, --repetitions (default: %d)\n", cmd_params_defaults.reps); - fprintf(stdout, " -o, --output (default: %s)\n", cmd_params_defaults.output_format == CSV ? "csv" : cmd_params_defaults.output_format == JSON ? "json" : "md"); + fprintf(stdout, " -o, --output (default: %s)\n", cmd_params_defaults.output_format == CSV ? "csv" : cmd_params_defaults.output_format == JSON ? "json" : "md"); fprintf(stdout, " -v, --verbose (default: %s)\n", cmd_params_defaults.verbose ? "1" : "0"); fprintf(stdout, "\n"); fprintf(stdout, "Multiple values can be given for each parameter by separating them with ',' or by repeating the parameter.\n"); @@ -257,6 +267,8 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { params.output_format = JSON; } else if (argv[i] == std::string("md")) { params.output_format = MARKDOWN; + } else if (argv[i] == std::string("sql")) { + params.output_format = SQL; } else { invalid_param = true; break; @@ -372,32 +384,7 @@ static std::vector get_cmd_params_instances(const cmd_param return instances; } -// models params -struct model_params { - std::string filename; - std::string type; - - static const std::vector & get_fields() { - static const std::vector fields = {"filename", "type"}; - return fields; - } - - // TODO: use a map instead - std::vector get_values() const { - return {filename, type}; - } -}; - -static bool ggml_cpu_has_metal() { -#if defined(GGML_USE_METAL) - return true; -#else - return false; -#endif -} - -// backend params -struct backend_params { +struct test { static const std::string build_commit; static const int build_number; static const bool cuda; @@ -405,6 +392,8 @@ struct backend_params { static const bool metal; static const bool gpu_blas; static const bool blas; + std::string model_filename; + std::string model_type; int n_batch; int n_threads; bool f32_kv; @@ -413,6 +402,56 @@ struct backend_params { bool mul_mat_q; bool low_vram; std::array tensor_split; + int n_prompt; + int n_gen; + std::string test_time; + std::vector samples_ns; + + test(const cmd_params_instance & inst, const llama_model * lmodel, const llama_context * ctx) { + model_filename = inst.model; + char buf[128]; + llama_model_type(lmodel, buf, sizeof(buf)); + model_type = buf; + n_batch = inst.n_batch; + n_threads = inst.n_threads; + f32_kv = inst.f32_kv; + n_gpu_layers = inst.n_gpu_layers; + main_gpu = inst.main_gpu; + mul_mat_q = inst.mul_mat_q; + low_vram = inst.low_vram; + tensor_split = inst.tensor_split; + n_prompt = inst.n_prompt; + n_gen = inst.n_gen; + // RFC 3339 date-time format + time_t t = time(NULL); + std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&t)); + test_time = buf; + + (void) ctx; + } + + uint64_t avg_ns() const { + return ::avg(samples_ns); + } + + uint64_t stdev_ns() const { + return ::stdev(samples_ns); + } + + std::vector get_ts() const { + int n_tokens = n_prompt + n_gen; + std::vector ts; + std::transform(samples_ns.begin(), samples_ns.end(), std::back_inserter(ts), [n_tokens](uint64_t t) { return 1e9 * n_tokens / t; }); + return ts; + } + + double avg_ts() const { + return ::avg(get_ts()); + } + + double stdev_ts() const { + return ::stdev(get_ts()); + } static std::string get_backend() { if (cuda) { @@ -435,10 +474,14 @@ struct backend_params { static const std::vector & get_fields() { static const std::vector fields = { - "build_number", "build_commit", + "build_commit", "build_number", "cuda", "opencl", "metal", "gpu_blas", "blas", + "model_filename", "model_type", "n_batch", "n_threads", "f16_kv", - "n_gpu_layers", "main_gpu", "mul_mat_q", "low_vram", "tensor_split" + "n_gpu_layers", "main_gpu", "mul_mat_q", "low_vram", "tensor_split", + "n_prompt", "n_gen", "test_time", + "avg_ns", "stddev_ns", + "avg_ts", "stddev_ts" }; return fields; } @@ -458,97 +501,36 @@ struct backend_params { } } std::vector values = { - std::to_string(build_number), build_commit, + build_commit, std::to_string(build_number), std::to_string(cuda), std::to_string(opencl), std::to_string(metal), std::to_string(gpu_blas), std::to_string(blas), + model_filename, model_type, std::to_string(n_batch), std::to_string(n_threads), std::to_string(!f32_kv), - std::to_string(n_gpu_layers), std::to_string(main_gpu), std::to_string(mul_mat_q), std::to_string(low_vram), tensor_split_str + std::to_string(n_gpu_layers), std::to_string(main_gpu), std::to_string(mul_mat_q), std::to_string(low_vram), tensor_split_str, + std::to_string(n_prompt), std::to_string(n_gen), test_time, + std::to_string(avg_ns()), std::to_string(stdev_ns()), + std::to_string(avg_ts()), std::to_string(stdev_ts()) }; return values; } -}; - -const std::string backend_params::build_commit = BUILD_COMMIT; -const int backend_params::build_number = BUILD_NUMBER; -const bool backend_params::cuda = !!ggml_cpu_has_cublas(); -const bool backend_params::opencl = !!ggml_cpu_has_clblast(); -const bool backend_params::metal = !!ggml_cpu_has_metal(); -const bool backend_params::gpu_blas = !!ggml_cpu_has_gpublas(); -const bool backend_params::blas = !!ggml_cpu_has_blas(); - -// benchmark params -struct bench_params { - int n_prompt; - int n_gen; - - static const std::vector & get_fields() { - static const std::vector fields = {"n_prompt", "n_gen"}; - return fields; - } - std::vector get_values() const { - return {std::to_string(n_prompt), std::to_string(n_gen)}; + std::map get_map() const { + std::map map; + auto fields = get_fields(); + auto values = get_values(); + std::transform(fields.begin(), fields.end(), values.begin(), + std::inserter(map, map.end()), std::make_pair); + return map; } }; -// timing results -struct timing_samples { - std::vector t_ns; - - uint64_t avg() const { - return ::avg(t_ns); - } - - uint64_t stdev() const { - return ::stdev(t_ns); - } +const std::string test::build_commit = BUILD_COMMIT; +const int test::build_number = BUILD_NUMBER; +const bool test::cuda = !!ggml_cpu_has_cublas(); +const bool test::opencl = !!ggml_cpu_has_clblast(); +const bool test::metal = !!ggml_cpu_has_metal(); +const bool test::gpu_blas = !!ggml_cpu_has_gpublas(); +const bool test::blas = !!ggml_cpu_has_blas(); - std::vector get_ts(int n) const { - std::vector ts; - std::transform(t_ns.begin(), t_ns.end(), std::back_inserter(ts), [n](uint64_t t) { return 1e9 * n / t; }); - return ts; - } - - double avg_ts(uint64_t n) const { - return ::avg(get_ts(n)); - } - - double stddev_ts(uint64_t n) const { - return ::stdev(get_ts(n)); - } - - static const std::vector & get_fields() { - static const std::vector fields = {"t_ns"}; - return fields; - } -}; - -struct test { - model_params mparams = {}; - bench_params bparams = {}; - backend_params bkparams = {}; - timing_samples tsamples = {}; - - test(const cmd_params_instance & inst, const llama_model * lmodel, const llama_context * ctx) { - mparams.filename = inst.model; - char buf[128]; - llama_model_type(lmodel, buf, sizeof(buf)); - mparams.type = buf; - - bparams.n_prompt = inst.n_prompt; - bparams.n_gen = inst.n_gen; - - bkparams.n_batch = inst.n_batch; - bkparams.f32_kv = inst.f32_kv; - bkparams.n_threads = inst.n_threads; - bkparams.n_gpu_layers = inst.n_gpu_layers; - bkparams.main_gpu = inst.main_gpu; - bkparams.mul_mat_q = inst.mul_mat_q; - bkparams.low_vram = inst.low_vram; - bkparams.tensor_split = inst.tensor_split; - - (void) ctx; - } -}; struct printer { FILE * fout; @@ -559,11 +541,7 @@ struct printer { struct csv_printer : public printer { virtual void print_header(const cmd_params & params) { - std::vector fields; - fields.insert(fields.end(), model_params::get_fields().begin(), model_params::get_fields().end()); - fields.insert(fields.end(), bench_params::get_fields().begin(), bench_params::get_fields().end()); - fields.insert(fields.end(), backend_params::get_fields().begin(), backend_params::get_fields().end()); - fields.insert(fields.end(), timing_samples::get_fields().begin(), timing_samples::get_fields().end()); + std::vector fields = test::get_fields(); fprintf(fout, "%s\n", join(fields, ",").c_str()); (void) params; } @@ -573,13 +551,7 @@ struct csv_printer : public printer { } virtual void print_test(const test & t) { - for (auto t_ns : t.tsamples.t_ns) { - print_values(t.mparams.get_values()); - print_values(t.bparams.get_values()); - print_values(t.bkparams.get_values()); - print_values({std::to_string(t_ns)}); - fprintf(fout, "\n"); - } + print_values(t.get_values()); } }; @@ -589,12 +561,7 @@ struct json_printer : public printer { void print_fields(const std::vector & fields, const std::vector & values) { assert(fields.size() == values.size()); for (size_t i = 0; i < fields.size(); i++) { - fprintf(fout, " \"%s\": \"%s\"", fields.at(i).c_str(), values.at(i).c_str()); - if (i < fields.size() - 1) { - fprintf(fout, ",\n"); - } else { - fprintf(fout, "\n"); - } + fprintf(fout, " \"%s\": \"%s\",\n", fields.at(i).c_str(), values.at(i).c_str()); } } @@ -614,20 +581,9 @@ struct json_printer : public printer { fprintf(fout, ",\n"); } fprintf(fout, " {\n"); - fprintf(fout, " \"model\": {\n"); - print_fields(model_params::get_fields(), t.mparams.get_values()); - fprintf(fout, " },\n"); - fprintf(fout, " \"benchmark\": {\n"); - print_fields(bench_params::get_fields(), t.bparams.get_values()); - fprintf(fout, " },\n"); - fprintf(fout, " \"backend\": {\n"); - print_fields(backend_params::get_fields(), t.bkparams.get_values()); - fprintf(fout, " },\n"); - fprintf(fout, " \"samples\": {\n"); - fprintf(fout, " \"ns\": [ %s ],\n", join(t.tsamples.t_ns, ", ").c_str()); - fprintf(fout, " \"avg\": %" PRIu64 ",\n", t.tsamples.avg()); - fprintf(fout, " \"stddev\": %" PRIu64 "\n", t.tsamples.stdev()); - fprintf(fout, " }\n"); + print_fields(test::get_fields(), t.get_values()); + fprintf(fout, " \"samples_ns\": [ %s ],\n", join(t.samples_ns, ", ").c_str()); + fprintf(fout, " \"samples_ts\": [ %s ]\n", join(t.get_ts(), ", ").c_str()); fprintf(fout, " }"); } }; @@ -651,7 +607,7 @@ struct markdown_printer : public printer { virtual void print_header(const cmd_params & params) { fields = { "model", "backend" }; - bool is_cpu_backend = backend_params::get_backend() == "CPU" || backend_params::get_backend() == "BLAS"; + bool is_cpu_backend = test::get_backend() == "CPU" || test::get_backend() == "BLAS"; if (!is_cpu_backend) { fields.push_back("n_gpu_layers"); } @@ -691,29 +647,21 @@ struct markdown_printer : public printer { } virtual void print_test(const test & t) { - int n_tokens = t.bparams.n_prompt + t.bparams.n_gen; - - std::map vmap; - std::transform(model_params::get_fields().begin(), model_params::get_fields().end(), t.mparams.get_values().begin(), - std::inserter(vmap, vmap.end()), std::make_pair); - std::transform(bench_params::get_fields().begin(), bench_params::get_fields().end(), t.bparams.get_values().begin(), - std::inserter(vmap, vmap.end()), std::make_pair); - std::transform(backend_params::get_fields().begin(), backend_params::get_fields().end(), t.bkparams.get_values().begin(), - std::inserter(vmap, vmap.end()), std::make_pair); + std::map vmap = t.get_map(); fprintf(fout, "|"); for (const auto & field : fields) { std::string value; if (field == "model") { - value = t.mparams.type; + value = t.model_type; } else if (field == "backend") { - value = backend_params::get_backend(); + value = test::get_backend(); } else if (field == "test") { char buf[128]; - if (t.bparams.n_prompt > 0 && t.bparams.n_gen == 0) { - snprintf(buf, sizeof(buf), "pp %d", t.bparams.n_prompt); - } else if (t.bparams.n_gen > 0 && t.bparams.n_prompt == 0) { - snprintf(buf, sizeof(buf), "tg %d", t.bparams.n_gen); + if (t.n_prompt > 0 && t.n_gen == 0) { + snprintf(buf, sizeof(buf), "pp %d", t.n_prompt); + } else if (t.n_gen > 0 && t.n_prompt == 0) { + snprintf(buf, sizeof(buf), "tg %d", t.n_gen); } else { assert(false); exit(1); @@ -721,7 +669,7 @@ struct markdown_printer : public printer { value = buf; } else if (field == "t/s") { char buf[128]; - snprintf(buf, sizeof(buf), "%.2f ± %.2f", t.tsamples.avg_ts(n_tokens), t.tsamples.stddev_ts(n_tokens)); + snprintf(buf, sizeof(buf), "%.2f ± %.2f", t.avg_ts(), t.stdev_ts()); value = buf; } else if (vmap.find(field) != vmap.end()) { value = vmap.at(field); @@ -741,6 +689,60 @@ struct markdown_printer : public printer { } }; +struct sql_printer : public printer { + static std::string get_field_type(const std::string & field) { + if (field == "build_commit") { + return "TEXT"; + } + if (field == "build_number") { + return "INTEGER"; + } + if (field == "cuda" || field == "opencl" || field == "metal" || field == "gpu_blas" || field == "blas") { + return "INTEGER"; + } + if (field == "model_filename" || field == "model_type") { + return "TEXT"; + } + if (field == "n_batch" || field == "n_threads" || field == "f16_kv" || field == "n_gpu_layers" || field == "main_gpu" || field == "mul_mat_q" || field == "low_vram") { + return "INTEGER"; + } + if (field == "tensor_split") { + return "TEXT"; + } + if (field == "n_prompt" || field == "n_gen") { + return "INTEGER"; + } + if (field == "test_time") { + return "TEXT"; + } + if (field == "avg_ns" || field == "stddev_ns" || field == "avg_ts" || field == "stddev_ts") { + return "REAL"; + } + return "TEXT"; + } + + virtual void print_header(const cmd_params & params) { + std::vector fields = test::get_fields(); + fprintf(fout, "CREATE TABLE IF NOT EXISTS test (\n"); + for (size_t i = 0; i < fields.size(); i++) { + fprintf(fout, " %s %s%s\n", fields.at(i).c_str(), get_field_type(fields.at(i)).c_str(), i < fields.size() - 1 ? "," : ""); + } + fprintf(fout, ");\n"); + fprintf(fout, "\n"); + (void) params; + } + + virtual void print_test(const test & t) { + fprintf(fout, "INSERT INTO test (%s) ", join(test::get_fields(), ", ").c_str()); + fprintf(fout, "VALUES ("); + std::vector values = t.get_values(); + for (size_t i = 0; i < values.size(); i++) { + fprintf(fout, "'%s'%s", values.at(i).c_str(), i < values.size() - 1 ? ", " : ""); + } + fprintf(fout, ");\n"); + } +}; + void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_batch, int n_threads) { std::vector tokens(n_batch, llama_token_bos()); int n_processed = 0; @@ -798,6 +800,12 @@ int main(int argc, char ** argv) { case MARKDOWN: p.reset(new markdown_printer()); break; + case SQL: + p.reset(new sql_printer()); + break; + default: + assert(false); + exit(1); } p->fout = stdout; p->print_header(params); @@ -824,18 +832,18 @@ int main(int argc, char ** argv) { test t(inst, lmodel, ctx); // warmup run - test_gen(ctx, 1, 0, t.bkparams.n_threads); + test_gen(ctx, 1, 0, t.n_threads); for (int i = 0; i < params.reps; i++) { uint64_t t_start = get_time_ns(); - if (t.bparams.n_prompt > 0) { - test_prompt(ctx, t.bparams.n_prompt, 0, t.bkparams.n_batch, t.bkparams.n_threads); + if (t.n_prompt > 0) { + test_prompt(ctx, t.n_prompt, 0, t.n_batch, t.n_threads); } - if (t.bparams.n_gen > 0) { - test_gen(ctx, t.bparams.n_gen, t.bparams.n_prompt, t.bkparams.n_threads); + if (t.n_gen > 0) { + test_gen(ctx, t.n_gen, t.n_prompt, t.n_threads); } uint64_t t_ns = get_time_ns() - t_start; - t.tsamples.t_ns.push_back(t_ns); + t.samples_ns.push_back(t_ns); } p->print_test(t); From cac70312e3922a7901a6d5eac50f889dba1b258a Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 17 Aug 2023 02:50:04 +0200 Subject: [PATCH 14/19] add basic cpu and gpu info (linx/cuda only) --- examples/llama-bench/llama-bench.cpp | 90 ++++++++++++++++++++-------- ggml-cuda.cu | 12 ++++ ggml-cuda.h | 42 ++++++------- 3 files changed, 97 insertions(+), 47 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 9c654d2815a55..c8a940c98789f 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -1,21 +1,26 @@ #include +#include #include #include -#include #include +#include +#include +#include +#include +#include #include +#include #include #include #include -#include -#include -#include -#include -#include + #include "ggml.h" #include "llama.h" #include "common.h" #include "build-info.h" +#ifdef GGML_USE_CUBLAS +#include "ggml-cuda.h" +#endif // utils static uint64_t get_time_ns() { @@ -50,7 +55,7 @@ static std::vector split(const std::string & str, char delim) { } template -T avg(const std::vector & v) { +static T avg(const std::vector & v) { if (v.empty()) { return 0; } @@ -59,7 +64,7 @@ T avg(const std::vector & v) { } template -T stdev(const std::vector & v) { +static T stdev(const std::vector & v) { if (v.size() <= 1) { return 0; } @@ -77,6 +82,50 @@ static bool ggml_cpu_has_metal() { #endif } +static std::string get_cpu_info() { + std::string id; +#ifdef __linux__ + FILE * f = fopen("/proc/cpuinfo", "r"); + if (f) { + char buf[1024]; + while (fgets(buf, sizeof(buf), f)) { + if (strncmp(buf, "model name", 10) == 0) { + char * p = strchr(buf, ':'); + if (p) { + p++; + while (std::isspace(*p)) { + p++; + } + while (std::isspace(p[strlen(p) - 1])) { + p[strlen(p) - 1] = '\0'; + } + id = p; + break; + } + } + } + } +#endif + // TODO: other platforms + return id; +} + +static std::string get_gpu_info(void) { + std::string id; +#ifdef GGML_USE_CUBLAS + int count = ggml_cuda_get_device_count(); + for (int i = 0; i < count; i++) { + char buf[128]; + ggml_cuda_get_device_description(i, buf, sizeof(buf)); + id += buf; + if (i < count - 1) { + id += "/"; + } + } +#endif + // TODO: other backends + return id; +} // command line params enum output_formats {CSV, JSON, MARKDOWN, SQL}; @@ -392,6 +441,8 @@ struct test { static const bool metal; static const bool gpu_blas; static const bool blas; + static const std::string cpu_info; + static const std::string gpu_info; std::string model_filename; std::string model_type; int n_batch; @@ -476,6 +527,7 @@ struct test { static const std::vector fields = { "build_commit", "build_number", "cuda", "opencl", "metal", "gpu_blas", "blas", + "cpu_info", "gpu_info", "model_filename", "model_type", "n_batch", "n_threads", "f16_kv", "n_gpu_layers", "main_gpu", "mul_mat_q", "low_vram", "tensor_split", @@ -503,6 +555,7 @@ struct test { std::vector values = { build_commit, std::to_string(build_number), std::to_string(cuda), std::to_string(opencl), std::to_string(metal), std::to_string(gpu_blas), std::to_string(blas), + cpu_info, gpu_info, model_filename, model_type, std::to_string(n_batch), std::to_string(n_threads), std::to_string(!f32_kv), std::to_string(n_gpu_layers), std::to_string(main_gpu), std::to_string(mul_mat_q), std::to_string(low_vram), tensor_split_str, @@ -530,7 +583,8 @@ const bool test::opencl = !!ggml_cpu_has_clblast(); const bool test::metal = !!ggml_cpu_has_metal(); const bool test::gpu_blas = !!ggml_cpu_has_gpublas(); const bool test::blas = !!ggml_cpu_has_blas(); - +const std::string test::cpu_info = get_cpu_info(); +const std::string test::gpu_info = get_gpu_info(); struct printer { FILE * fout; @@ -691,30 +745,18 @@ struct markdown_printer : public printer { struct sql_printer : public printer { static std::string get_field_type(const std::string & field) { - if (field == "build_commit") { - return "TEXT"; - } if (field == "build_number") { return "INTEGER"; } if (field == "cuda" || field == "opencl" || field == "metal" || field == "gpu_blas" || field == "blas") { return "INTEGER"; } - if (field == "model_filename" || field == "model_type") { - return "TEXT"; - } if (field == "n_batch" || field == "n_threads" || field == "f16_kv" || field == "n_gpu_layers" || field == "main_gpu" || field == "mul_mat_q" || field == "low_vram") { return "INTEGER"; } - if (field == "tensor_split") { - return "TEXT"; - } if (field == "n_prompt" || field == "n_gen") { return "INTEGER"; } - if (field == "test_time") { - return "TEXT"; - } if (field == "avg_ns" || field == "stddev_ns" || field == "avg_ts" || field == "stddev_ts") { return "REAL"; } @@ -743,7 +785,7 @@ struct sql_printer : public printer { } }; -void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_batch, int n_threads) { +static void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_batch, int n_threads) { std::vector tokens(n_batch, llama_token_bos()); int n_processed = 0; while (n_processed < n_prompt) { @@ -753,14 +795,14 @@ void test_prompt(llama_context * ctx, int n_prompt, int n_past, int n_batch, int } } -void test_gen(llama_context * ctx, int n_gen, int n_past, int n_threads) { +static void test_gen(llama_context * ctx, int n_gen, int n_past, int n_threads) { llama_token token = llama_token_bos(); for (int i = 0; i < n_gen; i++) { llama_eval(ctx, &token, 1, n_past + i, n_threads); } } -void llama_null_log_callback(enum llama_log_level level, const char * text, void * user_data) { +static void llama_null_log_callback(enum llama_log_level level, const char * text, void * user_data) { (void)level; (void)text; (void)user_data; diff --git a/ggml-cuda.cu b/ggml-cuda.cu index df0cbe18f96ba..5b415c646e8c6 100644 --- a/ggml-cuda.cu +++ b/ggml-cuda.cu @@ -6469,3 +6469,15 @@ bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_ func(tensor->src[0], tensor->src[1], tensor); return true; } + +int ggml_cuda_get_device_count() { + int device_count; + CUDA_CHECK(cudaGetDeviceCount(&device_count)); + return device_count; +} + +void ggml_cuda_get_device_description(int device, char * description, size_t description_size) { + cudaDeviceProp prop; + CUDA_CHECK(cudaGetDeviceProperties(&prop, device)); + snprintf(description, description_size, "%s", prop.name); +} diff --git a/ggml-cuda.h b/ggml-cuda.h index 72d7afa463d74..cad05f5fa47ab 100644 --- a/ggml-cuda.h +++ b/ggml-cuda.h @@ -8,29 +8,25 @@ extern "C" { #define GGML_CUDA_MAX_DEVICES 16 -void ggml_init_cublas(void); -void ggml_cuda_set_tensor_split(const float * tensor_split); - -void ggml_cuda_mul(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); -bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); -size_t ggml_cuda_mul_mat_get_wsize(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); -void ggml_cuda_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst, void * wdata, size_t wsize); - -// TODO: export these with GGML_API -void * ggml_cuda_host_malloc(size_t size); -void ggml_cuda_host_free(void * ptr); - -void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor); - -void ggml_cuda_free_data(struct ggml_tensor * tensor); -void ggml_cuda_assign_buffers(struct ggml_tensor * tensor); -void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor); -void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor); -void ggml_cuda_set_main_device(int main_device); -void ggml_cuda_set_mul_mat_q(bool mul_mat_q); -void ggml_cuda_set_scratch_size(size_t scratch_size); -void ggml_cuda_free_scratch(void); -bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor); +GGML_API void ggml_init_cublas(void); +GGML_API void * ggml_cuda_host_malloc(size_t size); +GGML_API void ggml_cuda_host_free(void * ptr); + +GGML_API bool ggml_cuda_can_mul_mat(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst); +GGML_API void ggml_cuda_set_tensor_split(const float * tensor_split); +GGML_API void ggml_cuda_transform_tensor(void * data, struct ggml_tensor * tensor); +GGML_API void ggml_cuda_free_data(struct ggml_tensor * tensor); +GGML_API void ggml_cuda_assign_buffers(struct ggml_tensor * tensor); +GGML_API void ggml_cuda_assign_buffers_no_scratch(struct ggml_tensor * tensor); +GGML_API void ggml_cuda_assign_buffers_force_inplace(struct ggml_tensor * tensor); +GGML_API void ggml_cuda_set_main_device(int main_device); +GGML_API void ggml_cuda_set_mul_mat_q(bool mul_mat_q); +GGML_API void ggml_cuda_set_scratch_size(size_t scratch_size); +GGML_API void ggml_cuda_free_scratch(void); +GGML_API bool ggml_cuda_compute_forward(struct ggml_compute_params * params, struct ggml_tensor * tensor); + +GGML_API int ggml_cuda_get_device_count(void); +GGML_API void ggml_cuda_get_device_description(int device, char * description, size_t description_size); #ifdef __cplusplus } From 569dc6f3d07c8635b0ccfafe3678d20ebbca6a13 Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 17 Aug 2023 03:06:07 +0200 Subject: [PATCH 15/19] markdown: also show values that differ from the default --- examples/llama-bench/llama-bench.cpp | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index c8a940c98789f..4e18e60103c9f 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -357,14 +357,13 @@ struct cmd_params_instance { int n_gen; int n_batch; bool f32_kv; + int n_threads; int n_gpu_layers; int main_gpu; bool mul_mat_q; bool low_vram; std::array tensor_split; - int n_threads; - llama_context_params to_llama_params() const { llama_context_params lparams = llama_context_default_params(); lparams.n_ctx = n_prompt + n_gen; @@ -546,9 +545,11 @@ struct test { max_nonzero = i; } } - for (int i = 0; i < max_nonzero; i++) { - tensor_split_str += std::to_string(tensor_split[i]); - if (i < max_nonzero - 1) { + for (int i = 0; i <= max_nonzero; i++) { + char buf[32]; + snprintf(buf, sizeof(buf), "%.2f", tensor_split[i]); + tensor_split_str += buf; + if (i < max_nonzero) { tensor_split_str += "/"; } } @@ -593,6 +594,7 @@ struct printer { virtual void print_footer() {}; }; +// TODO: escape strings struct csv_printer : public printer { virtual void print_header(const cmd_params & params) { std::vector fields = test::get_fields(); @@ -665,24 +667,27 @@ struct markdown_printer : public printer { if (!is_cpu_backend) { fields.push_back("n_gpu_layers"); } - if (params.n_threads.size() > 1 || is_cpu_backend) { + if (params.n_batch.size() > 1 || params.n_threads != cmd_params_defaults.n_threads || is_cpu_backend) { fields.push_back("n_threads"); } - if (params.n_batch.size() > 1) { + if (params.n_batch.size() > 1 || params.n_batch != cmd_params_defaults.n_batch) { fields.push_back("n_batch"); } - if (params.f32_kv.size() > 1) { + if (params.f32_kv.size() > 1 || params.f32_kv != cmd_params_defaults.f32_kv) { fields.push_back("f16_kv"); } - if (params.main_gpu.size() > 1) { + if (params.main_gpu.size() > 1 || params.main_gpu != cmd_params_defaults.main_gpu) { fields.push_back("main_gpu"); } - if (params.mul_mat_q.size() > 1) { + if (params.mul_mat_q.size() > 1 || params.mul_mat_q != cmd_params_defaults.mul_mat_q) { fields.push_back("mul_mat_q"); } - if (params.low_vram.size() > 1) { + if (params.low_vram.size() > 1 || params.low_vram != cmd_params_defaults.low_vram) { fields.push_back("low_vram"); } + if (params.tensor_split.size() > 1 || params.tensor_split != cmd_params_defaults.tensor_split) { + fields.push_back("tensor_split"); + } fields.push_back("test"); fields.push_back("t/s"); From 94218e8adeb76b99a9b76095672242b1ec42a9d5 Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 17 Aug 2023 03:12:03 +0200 Subject: [PATCH 16/19] markdown: add build id --- examples/llama-bench/llama-bench.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 4e18e60103c9f..8b4d6ee7e6937 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -746,6 +746,10 @@ struct markdown_printer : public printer { } fprintf(fout, "\n"); } + + virtual void print_footer() { + fprintf(fout, "\nbuild: %s (%d)\n", test::build_commit.c_str(), test::build_number); + } }; struct sql_printer : public printer { From 9c3866099bc71658fa1d551036f8dbf863ecad39 Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 17 Aug 2023 03:22:19 +0200 Subject: [PATCH 17/19] cleanup --- examples/llama-bench/llama-bench.cpp | 45 +++++++++++++--------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 8b4d6ee7e6937..8ea5148c9c9d8 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -110,7 +110,7 @@ static std::string get_cpu_info() { return id; } -static std::string get_gpu_info(void) { +static std::string get_gpu_info() { std::string id; #ifdef GGML_USE_CUBLAS int count = ggml_cuda_get_device_count(); @@ -589,25 +589,22 @@ const std::string test::gpu_info = get_gpu_info(); struct printer { FILE * fout; - virtual void print_header(const cmd_params & params) { (void)params; }; + virtual void print_header(const cmd_params & params) { (void) params; }; virtual void print_test(const test & t) = 0; - virtual void print_footer() {}; + virtual void print_footer() { }; }; // TODO: escape strings struct csv_printer : public printer { - virtual void print_header(const cmd_params & params) { + void print_header(const cmd_params & params) override { std::vector fields = test::get_fields(); fprintf(fout, "%s\n", join(fields, ",").c_str()); (void) params; } - void print_values(const std::vector & values) { - fprintf(fout, "%s", join(values, ",").c_str()); - } - - virtual void print_test(const test & t) { - print_values(t.get_values()); + void print_test(const test & t) override { + std::vector values = t.get_values(); + fprintf(fout, "%s\n", join(values, ",").c_str()); } }; @@ -621,16 +618,12 @@ struct json_printer : public printer { } } - virtual void print_header(const cmd_params & params) { + void print_header(const cmd_params & params) override { fprintf(fout, "[\n"); (void) params; } - virtual void print_footer() { - fprintf(fout, "\n]\n"); - } - - virtual void print_test(const test & t) { + void print_test(const test & t) override { if (first) { first = false; } else { @@ -642,6 +635,10 @@ struct json_printer : public printer { fprintf(fout, " \"samples_ts\": [ %s ]\n", join(t.get_ts(), ", ").c_str()); fprintf(fout, " }"); } + + void print_footer() override { + fprintf(fout, "\n]\n"); + } }; struct markdown_printer : public printer { @@ -661,7 +658,7 @@ struct markdown_printer : public printer { return width; } - virtual void print_header(const cmd_params & params) { + void print_header(const cmd_params & params) override { fields = { "model", "backend" }; bool is_cpu_backend = test::get_backend() == "CPU" || test::get_backend() == "BLAS"; if (!is_cpu_backend) { @@ -705,7 +702,7 @@ struct markdown_printer : public printer { (void) params; } - virtual void print_test(const test & t) { + void print_test(const test & t) override { std::map vmap = t.get_map(); fprintf(fout, "|"); @@ -747,7 +744,7 @@ struct markdown_printer : public printer { fprintf(fout, "\n"); } - virtual void print_footer() { + void print_footer() override { fprintf(fout, "\nbuild: %s (%d)\n", test::build_commit.c_str(), test::build_number); } }; @@ -772,7 +769,7 @@ struct sql_printer : public printer { return "TEXT"; } - virtual void print_header(const cmd_params & params) { + void print_header(const cmd_params & params) override { std::vector fields = test::get_fields(); fprintf(fout, "CREATE TABLE IF NOT EXISTS test (\n"); for (size_t i = 0; i < fields.size(); i++) { @@ -783,7 +780,7 @@ struct sql_printer : public printer { (void) params; } - virtual void print_test(const test & t) { + void print_test(const test & t) override { fprintf(fout, "INSERT INTO test (%s) ", join(test::get_fields(), ", ").c_str()); fprintf(fout, "VALUES ("); std::vector values = t.get_values(); @@ -812,9 +809,9 @@ static void test_gen(llama_context * ctx, int n_gen, int n_past, int n_threads) } static void llama_null_log_callback(enum llama_log_level level, const char * text, void * user_data) { - (void)level; - (void)text; - (void)user_data; + (void) level; + (void) text; + (void) user_data; } int main(int argc, char ** argv) { From b6c81e28cdaf9b5212f63c9ed86c1ef74b834915 Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 17 Aug 2023 17:28:55 +0200 Subject: [PATCH 18/19] improve formatting --- examples/llama-bench/llama-bench.cpp | 106 +++++++++++++++++++++------ 1 file changed, 82 insertions(+), 24 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 8ea5148c9c9d8..90e74501e3a58 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -537,6 +537,25 @@ struct test { return fields; } + enum field_type {STRING, BOOL, INT, FLOAT}; + + static field_type get_field_type(const std::string & field) { + if (field == "build_number" || field == "n_batch" || field == "n_threads" || + field == "n_gpu_layers" || field == "main_gpu" || + field == "n_prompt" || field == "n_gen" || + field == "avg_ns" || field == "stddev_ns") { + return INT; + } + if (field == "cuda" || field == "opencl" || field == "metal" || field == "gpu_blas" || field == "blas" || + field == "f16_kv" || field == "mul_mat_q" || field == "low_vram") { + return BOOL; + } + if (field == "avg_ts" || field == "stddev_ts") { + return FLOAT; + } + return STRING; + } + std::vector get_values() const { std::string tensor_split_str; int max_nonzero = 0; @@ -594,8 +613,19 @@ struct printer { virtual void print_footer() { }; }; -// TODO: escape strings struct csv_printer : public printer { + static std::string escape_csv(const std::string & field) { + std::string escaped = "\""; + for (auto c : field) { + if (c == '"') { + escaped += "\""; + } + escaped += c; + } + escaped += "\""; + return escaped; + } + void print_header(const cmd_params & params) override { std::vector fields = test::get_fields(); fprintf(fout, "%s\n", join(fields, ",").c_str()); @@ -604,6 +634,7 @@ struct csv_printer : public printer { void print_test(const test & t) override { std::vector values = t.get_values(); + std::transform(values.begin(), values.end(), values.begin(), escape_csv); fprintf(fout, "%s\n", join(values, ",").c_str()); } }; @@ -611,10 +642,32 @@ struct csv_printer : public printer { struct json_printer : public printer { bool first = true; - void print_fields(const std::vector & fields, const std::vector & values) { - assert(fields.size() == values.size()); - for (size_t i = 0; i < fields.size(); i++) { - fprintf(fout, " \"%s\": \"%s\",\n", fields.at(i).c_str(), values.at(i).c_str()); + static std::string escape_json(const std::string & value) { + std::string escaped; + for (auto c : value) { + if (c == '"') { + escaped += "\\\""; + } else if (c == '\\') { + escaped += "\\\\"; + } else if (c <= 0x1f) { + char buf[8]; + snprintf(buf, sizeof(buf), "\\u%04x", c); + escaped += buf; + } else { + escaped += c; + } + } + return escaped; + } + + static std::string format_value(const std::string & field, const std::string & value) { + switch (test::get_field_type(field)) { + case test::STRING: + return "\"" + escape_json(value) + "\""; + case test::BOOL: + return value == "0" ? "false" : "true"; + default: + return value; } } @@ -623,6 +676,13 @@ struct json_printer : public printer { (void) params; } + void print_fields(const std::vector & fields, const std::vector & values) { + assert(fields.size() == values.size()); + for (size_t i = 0; i < fields.size(); i++) { + fprintf(fout, " \"%s\": %s,\n", fields.at(i).c_str(), format_value(fields.at(i), values.at(i)).c_str()); + } + } + void print_test(const test & t) override { if (first) { first = false; @@ -634,6 +694,7 @@ struct json_printer : public printer { fprintf(fout, " \"samples_ns\": [ %s ],\n", join(t.samples_ns, ", ").c_str()); fprintf(fout, " \"samples_ts\": [ %s ]\n", join(t.get_ts(), ", ").c_str()); fprintf(fout, " }"); + fflush(fout); } void print_footer() override { @@ -652,7 +713,8 @@ struct markdown_printer : public printer { return 15; } int width = std::max((int)field.length(), 10); - if (field == "backend") { + + if (test::get_field_type(field) == test::STRING) { return -width; } return width; @@ -696,7 +758,7 @@ struct markdown_printer : public printer { fprintf(fout, "|"); for (const auto & field: fields) { int width = get_field_width(field); - fprintf(fout, " %s%s%s |", width < 0 ? ":" : "", std::string(std::abs(width) - 1, '-').c_str(), width > 0 ? ":" : ""); + fprintf(fout, " %s%s |", std::string(std::abs(width) - 1, '-').c_str(), width > 0 ? ":" : "-"); } fprintf(fout, "\n"); (void) params; @@ -750,30 +812,26 @@ struct markdown_printer : public printer { }; struct sql_printer : public printer { - static std::string get_field_type(const std::string & field) { - if (field == "build_number") { - return "INTEGER"; - } - if (field == "cuda" || field == "opencl" || field == "metal" || field == "gpu_blas" || field == "blas") { - return "INTEGER"; - } - if (field == "n_batch" || field == "n_threads" || field == "f16_kv" || field == "n_gpu_layers" || field == "main_gpu" || field == "mul_mat_q" || field == "low_vram") { - return "INTEGER"; - } - if (field == "n_prompt" || field == "n_gen") { - return "INTEGER"; - } - if (field == "avg_ns" || field == "stddev_ns" || field == "avg_ts" || field == "stddev_ts") { - return "REAL"; + static std::string get_sql_field_type(const std::string & field) { + switch (test::get_field_type(field)) { + case test::STRING: + return "TEXT"; + case test::BOOL: + case test::INT: + return "INTEGER"; + case test::FLOAT: + return "REAL"; + default: + assert(false); + exit(1); } - return "TEXT"; } void print_header(const cmd_params & params) override { std::vector fields = test::get_fields(); fprintf(fout, "CREATE TABLE IF NOT EXISTS test (\n"); for (size_t i = 0; i < fields.size(); i++) { - fprintf(fout, " %s %s%s\n", fields.at(i).c_str(), get_field_type(fields.at(i)).c_str(), i < fields.size() - 1 ? "," : ""); + fprintf(fout, " %s %s%s\n", fields.at(i).c_str(), get_sql_field_type(fields.at(i)).c_str(), i < fields.size() - 1 ? "," : ""); } fprintf(fout, ");\n"); fprintf(fout, "\n"); From df87dd74a5ac1d75ba0ce4fddd56336c3fa32dbf Mon Sep 17 00:00:00 2001 From: slaren Date: Thu, 17 Aug 2023 22:11:40 +0200 Subject: [PATCH 19/19] formatting --- examples/llama-bench/llama-bench.cpp | 47 ++++++++++++++-------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/examples/llama-bench/llama-bench.cpp b/examples/llama-bench/llama-bench.cpp index 90e74501e3a58..266c8eab3b2f6 100755 --- a/examples/llama-bench/llama-bench.cpp +++ b/examples/llama-bench/llama-bench.cpp @@ -391,20 +391,19 @@ static std::vector get_cmd_params_instances_int(const cmd_p for (const auto & lv : params.low_vram) for (const auto & ts : params.tensor_split) for (const auto & nt : params.n_threads) { - cmd_params_instance instance; - - instance.model = m; - instance.n_prompt = n_prompt; - instance.n_gen = n_gen; - instance.n_batch = nb; - instance.f32_kv = fk; - instance.n_gpu_layers = nl; - instance.main_gpu = mg; - instance.mul_mat_q = mmq; - instance.low_vram = lv; - std::copy(std::begin(ts), std::end(ts), std::begin(instance.tensor_split)); - instance.n_threads = nt; - + cmd_params_instance instance = { + /* .model = */ m, + /* .n_prompt = */ n_prompt, + /* .n_gen = */ n_gen, + /* .n_batch = */ nb, + /* .f32_kv = */ fk, + /* .n_threads = */ nt, + /* .n_gpu_layers = */ nl, + /* .main_gpu = */ mg, + /* .mul_mat_q = */ mmq, + /* .low_vram = */ lv, + /* .tensor_split = */ ts, + }; instances.push_back(instance); } return instances; @@ -597,14 +596,14 @@ struct test { }; const std::string test::build_commit = BUILD_COMMIT; -const int test::build_number = BUILD_NUMBER; -const bool test::cuda = !!ggml_cpu_has_cublas(); -const bool test::opencl = !!ggml_cpu_has_clblast(); -const bool test::metal = !!ggml_cpu_has_metal(); -const bool test::gpu_blas = !!ggml_cpu_has_gpublas(); -const bool test::blas = !!ggml_cpu_has_blas(); -const std::string test::cpu_info = get_cpu_info(); -const std::string test::gpu_info = get_gpu_info(); +const int test::build_number = BUILD_NUMBER; +const bool test::cuda = !!ggml_cpu_has_cublas(); +const bool test::opencl = !!ggml_cpu_has_clblast(); +const bool test::metal = !!ggml_cpu_has_metal(); +const bool test::gpu_blas = !!ggml_cpu_has_gpublas(); +const bool test::blas = !!ggml_cpu_has_blas(); +const std::string test::cpu_info = get_cpu_info(); +const std::string test::gpu_info = get_gpu_info(); struct printer { FILE * fout; @@ -721,6 +720,7 @@ struct markdown_printer : public printer { } void print_header(const cmd_params & params) override { + // select fields to print fields = { "model", "backend" }; bool is_cpu_backend = test::get_backend() == "CPU" || test::get_backend() == "BLAS"; if (!is_cpu_backend) { @@ -756,12 +756,11 @@ struct markdown_printer : public printer { } fprintf(fout, "\n"); fprintf(fout, "|"); - for (const auto & field: fields) { + for (const auto & field : fields) { int width = get_field_width(field); fprintf(fout, " %s%s |", std::string(std::abs(width) - 1, '-').c_str(), width > 0 ? ":" : "-"); } fprintf(fout, "\n"); - (void) params; } void print_test(const test & t) override {