How to build and run benchmarks

[ericcurtin]

I would like to run:

folly/test/StringBenchmark.cpp

but can't seem to figure out how to run that single one.

Would add to:

https://github.com/facebook/folly/blob/main/folly/docs/Benchmark.md

if pointed in the right direction.

[leikahing]

You may have resolved your issue in the intervening time, but a couple ways you can do this...

The Really Manual Way

This is assuming you have Folly installed or built, and the following dependencies, if you didn't have the Folly build install for you.

• boost - headers only
• double-conversion - lib and headers
• glog - lib and headers
• gflags - lib and headers
• fmt - lib and headers

Note that I set additional lib directories because I use brew on Mac, you might not need to do this based on what platform you're doing this on.

#!/bin/sh

# I use Mac with homebrew, so this gets all the dependencies
# brew install boost double-conversion fmt glog gflags

BREW_INCLUDE_DIR="$(brew --prefix)/include"
BREW_LIB_DIR="$(brew --prefix)/lib"

# I assume you have the folly source code, set this to the root
FOLLY_SOURCE_DIR="/PATH/TO/FOLLY/SRC/"

# wherever libfolly lives
FOLLY_LIB_DIR="/PATH/TO/FOLLY/LIB/DIR"

# these are the files that will build StringBenchmark.cpp
SRCS="$FOLLY_SOURCE_DIR/folly/test/StringBenchmark.cpp $FOLLY_SOURCE_DIR/folly/Benchmark.cpp"


CC="g++"
CC_FLAGS="-std=c++17 -O3 -Wall -Wextra -DNDEBUG"
LINKER_FLAGS="-lfmt -lfolly -lglog -lgflags -ldouble-conversion"

$CC -L $FOLLY_LIB_DIR -L $BREW_LIB_DIR -I $BREW_INCLUDE_DIR -I $FOLLY_SOURCE_DIR $CC_FLAGS $SRCS $LINKER_FLAGS -o stringbench

Copy this script, set/change the vars/command as needed, and if all goes well, you should produce a file called stringbench

Have cmake Generate the Target

I'm not much of a cmake expert, but you can leverage folly's pre-existing CMakeLists.txt to have it generate the target for building the benchmark for you.

Here's a diff of some lines I added to build the StringBenchmark.cpp code

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 21d24ba6e..5729ee60b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -383,6 +383,14 @@ target_compile_features(folly INTERFACE cxx_generic_lambdas)
 
 target_link_libraries(folly PUBLIC folly_deps)
 
+# let's take advantage of cmake already being aware of dependency paths to build
+# benchmarks 
+add_executable(stringbench folly/test/StringBenchmark.cpp folly/Benchmark.cpp)
+target_link_libraries(stringbench PUBLIC folly)
+apply_folly_compile_options_to_target(stringbench)
+# add on release-level flags manually
+target_compile_options(stringbench PRIVATE "-O3" "-DNDEBUG")
+
 # Test utilities exported for use by downstream projects
 add_library(folly_test_util
   ${FOLLY_DIR}/test/DeterministicSchedule.cpp

With that in your top-level CMakeLists.txt file, you can have cmake generate a new Makefile with a new stringbench target in it.

An example cmake command to use (which is what I'm doing on my machine):

cmake \
  -DCMAKE_INCLUDE_PATH="$(brew --prefix)/include" \
  -DOPENSSL_INCLUDE_DIR="$(brew --prefix [email protected])/include" \
  -DCMAKE_LIBRARY_PATH="$(brew --prefix)/lib" \
  -DOPENSSL_ROOT_DIR="$(brew --prefix [email protected])"

After you run this, the generated Makefile will have a target called stringbench

# if you want to make sure
# make help | grep stringbench

make -j16 stringbench

---

In either case, you should hopefully have a working binary. On my host, running the benchmark produces this on main:

❯ ./stringbench
============================================================================
[...]/folly/folly/test/StringBenchmark.cpp     relative  time/iter   iters/s
============================================================================
libc_tolower                                              489.76ns     2.04M
folly_toLowerAscii                                        153.01ns     6.54M
stringPrintfOutputSize(1)                                 119.56ns     8.36M
stringPrintfOutputSize(4)                                 119.16ns     8.39M
stringPrintfOutputSize(16)                                148.90ns     6.72M
stringPrintfOutputSize(64)                                147.11ns     6.80M
stringPrintfOutputSize(256)                               359.55ns     2.78M
stringPrintfOutputSize(1024)                              376.08ns     2.66M
stringPrintfAppendfBenchmark                               16.49ms     60.65
fmtOutputSize(1)                                           48.60ns    20.58M
fmtOutputSize(4)                                           48.74ns    20.52M
fmtOutputSize(16)                                          75.94ns    13.17M
fmtOutputSize(64)                                          89.13ns    11.22M
fmtOutputSize(256)                                        180.84ns     5.53M
fmtOutputSize(1024)                                       459.00ns     2.18M
fmtAppendfBenchmark                                         2.39ms    418.50
follyFmtOutputSize(1)                                     101.21ns     9.88M
follyFmtOutputSize(4)                                     102.19ns     9.79M
follyFmtOutputSize(16)                                    131.61ns     7.60M
follyFmtOutputSize(64)                                    129.17ns     7.74M
follyFmtOutputSize(256)                                   161.56ns     6.19M
follyFmtOutputSize(1024)                                  142.23ns     7.03M
BM_cEscape                                                189.25us     5.28K
BM_cUnescape                                              166.25us     6.02K
BM_uriEscape                                                1.65us   604.60K
BM_uriUnescape                                            875.01ns     1.14M
BM_unhexlify                                               28.85ps    34.66G
splitOnSingleChar                                          10.41us    96.10K
splitOnSingleCharFixed                                    168.84ns     5.92M
splitOnSingleCharFixedAllowExtra                            0.00fs  Infinity
splitStr                                                   11.67us    85.68K
splitStrFixed                                             356.38ns     2.81M
boost_splitOnSingleChar                                     1.79us   558.44K
joinCharStr                                                 1.44us   693.76K
joinStrStr                                                  1.37us   732.29K
joinInt                                                     2.48us   403.09K

[yfeldblum]

Thank you @leikahing for answering the question, and for answering it so completely.

[ericcurtin]

Yes I forgot to thank you guys, thanks this was very helpful