From 611e14e093edcfb8a8ed71d7a107dc7f30290b51 Mon Sep 17 00:00:00 2001 From: Chris Wolfe Date: Sun, 12 Apr 2020 18:30:16 -0700 Subject: [PATCH 1/8] begin getting the fuzzers built as part of CI; add a runner making it possible to exercise them --- fuzz/CMakeLists.txt | 16 ++++++++++++++++ fuzz/standalone_runner.cc | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 fuzz/CMakeLists.txt create mode 100644 fuzz/standalone_runner.cc diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt new file mode 100644 index 0000000..3513d01 --- /dev/null +++ b/fuzz/CMakeLists.txt @@ -0,0 +1,16 @@ +# https://cmake.org/cmake/help/v3.0/command/add_test.html +# https://pabloariasal.github.io/2018/02/19/its-time-to-do-cmake-right/ +enable_language(CXX) + +include_directories(PUBLIC ${CMAKE_SOURCE_DIR}) + +foreach(FUZZERNAME + tokener_parse_ex_fuzzer) + +add_executable(${FUZZERNAME} ${FUZZERNAME}.cc standalone_runner.cc) + +target_include_directories(${FUZZERNAME} PUBLIC ${PROJECT_BINARY_DIR}) + +target_link_libraries(${FUZZERNAME} PRIVATE json-c) + +endforeach(FUZZERNAME) diff --git a/fuzz/standalone_runner.cc b/fuzz/standalone_runner.cc new file mode 100644 index 0000000..61631bc --- /dev/null +++ b/fuzz/standalone_runner.cc @@ -0,0 +1,25 @@ +#include +#include +#include +#include + +// Forward declare the "fuzz target" interface. +// We deliberately keep this inteface simple and header-free. +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); + +int main(int argc, char **argv) { + for (int i = 1; i < argc; i++) { + std::ifstream in(argv[i]); + in.seekg(0, in.end); + size_t length = in.tellg(); + in.seekg(0, in.beg); + std::cout << "Reading " << length << " bytes from " << argv[i] << std::endl; + // Allocate exactly length bytes so that we reliably catch buffer overflows. + std::vector bytes(length); + in.read(bytes.data(), bytes.size()); + assert(in); + LLVMFuzzerTestOneInput(reinterpret_cast(bytes.data()), + bytes.size()); + std::cout << "Execution successful" << std::endl; + } +} From 858ddac92bb6a98a622c8bfcd8459786c5e26d56 Mon Sep 17 00:00:00 2001 From: Chris Wolfe Date: Sun, 12 Apr 2020 21:23:06 -0700 Subject: [PATCH 2/8] build the fuzzers as part of the normal build process --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b36f1c..d6a1099 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -38,6 +38,7 @@ if (CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING AND (NOT MSVC OR NOT (MSVC_VERSION LESS 1800)) # Tests need at least VS2013 ) add_subdirectory(tests) +add_subdirectory(fuzz) endif() # Set some packaging variables. From f3925dcb52d01abd4386899efede6b894f8a1a69 Mon Sep 17 00:00:00 2001 From: Chris Wolfe Date: Mon, 13 Apr 2020 10:04:53 -0700 Subject: [PATCH 3/8] use the standalone runner when LIB_FUZZING_ENGINE is not set --- fuzz/CMakeLists.txt | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index 3513d01..5ae58d7 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -7,10 +7,25 @@ include_directories(PUBLIC ${CMAKE_SOURCE_DIR}) foreach(FUZZERNAME tokener_parse_ex_fuzzer) -add_executable(${FUZZERNAME} ${FUZZERNAME}.cc standalone_runner.cc) + add_executable(${FUZZERNAME} + ${FUZZERNAME}.cc) -target_include_directories(${FUZZERNAME} PUBLIC ${PROJECT_BINARY_DIR}) + target_include_directories(${FUZZERNAME} + PUBLIC + ${PROJECT_BINARY_DIR}) -target_link_libraries(${FUZZERNAME} PRIVATE json-c) + set_target_properties(${FUZZERNAME} PROPERTIES CXX_STANDARD 11) + # define whether we want to use the standalone runner for our own testing or + # link against libFuzzer populated from env variables + if (NOT DEFINED $ENV{LIB_FUZZING_ENGINE}) + add_library(standalone_runner standalone_runner.cc) + target_link_libraries(${FUZZERNAME} + json-c + standalone_runner) + else() + target_link_libraries(${FUZZERNAME} + json-c + ${LIB_FUZZING_ENGINE}) + endif() endforeach(FUZZERNAME) From 26bf7351b7167940db7c2c72c3550a5bdea88721 Mon Sep 17 00:00:00 2001 From: Chris Wolfe Date: Mon, 13 Apr 2020 10:13:37 -0700 Subject: [PATCH 4/8] use cmakified build and copy fuzzer out --- fuzz/CMakeLists.txt | 33 +++++++++++++-------------------- fuzz/build.sh | 39 +++++++-------------------------------- 2 files changed, 20 insertions(+), 52 deletions(-) diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index 5ae58d7..38517e0 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -3,29 +3,22 @@ enable_language(CXX) include_directories(PUBLIC ${CMAKE_SOURCE_DIR}) +include_directories(PUBLIC ${PROJECT_BINARY_DIR}) + +if(DEFINED ENV{LIB_FUZZING_ENGINE}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} $ENV{LIB_FUZZING_ENGINE}") + set(fuzzing_engine "") +else() + add_library(fuzzing_engine standalone_runner.cc) + set(fuzzing_engine fuzzing_engine) +endif() foreach(FUZZERNAME tokener_parse_ex_fuzzer) - add_executable(${FUZZERNAME} - ${FUZZERNAME}.cc) - - target_include_directories(${FUZZERNAME} - PUBLIC - ${PROJECT_BINARY_DIR}) - - set_target_properties(${FUZZERNAME} PROPERTIES CXX_STANDARD 11) + add_executable(${FUZZERNAME} ${FUZZERNAME}.cc) + target_link_libraries(${FUZZERNAME} + json-c + ${fuzzing_engine}) - # define whether we want to use the standalone runner for our own testing or - # link against libFuzzer populated from env variables - if (NOT DEFINED $ENV{LIB_FUZZING_ENGINE}) - add_library(standalone_runner standalone_runner.cc) - target_link_libraries(${FUZZERNAME} - json-c - standalone_runner) - else() - target_link_libraries(${FUZZERNAME} - json-c - ${LIB_FUZZING_ENGINE}) - endif() endforeach(FUZZERNAME) diff --git a/fuzz/build.sh b/fuzz/build.sh index b6a5cc7..3864015 100755 --- a/fuzz/build.sh +++ b/fuzz/build.sh @@ -14,39 +14,14 @@ # limitations under the License. # ################################################################################ +# this is expected to be run only by oss-fuzz. It will run from $SRC (above json-c) +cp $SRC/json-c/fuzz/*.dict $OUT/ -# This should be run from the top of the json-c source tree. +BUILD="$SRC/json-c/build" -mkdir build -cd build -cmake -DBUILD_SHARED_LIBS=OFF .. +mkdir $BUILD +cd $BUILD +cmake -DCMAKE_C_FLAGS="$CFLAGS" -DCMAKE_CXX_FLAGS="$CXXFLAGS" -DBUILD_SHARED_LIBS=OFF .. make -j$(nproc) +cp fuzz/*_fuzzer $OUT/ -LIB=$(pwd)/libjson-c.a -cd .. - -# These seem to be set externally, but let's assign defaults to -# make it possible to at least partially test this standalone. -: ${SRC:=$(dirname "$0")} -: ${OUT:=$SRC/out} -: ${CXX:=gcc} -: ${CXXFLAGS:=} - -[ -d "$OUT" ] || mkdir "$OUT" -cp $SRC/*.dict $OUT/. - -# XXX this doesn't seem to make much sense, since $SRC is presumably -# the "fuzz" directory, which is _inside_ the json-c repo, rather than -# the other way around, but I'm just preserving existing behavior. -erh -INCS=$SRC/json-c -# Compat when testing standalone -[ -e "${INCS}" ] || ln -s .. "${INCS}" - -set -x -set -v -for f in $SRC/*_fuzzer.cc; do - fuzzer=$(basename "$f" _fuzzer.cc) - $CXX $CXXFLAGS -std=c++11 -I$INCS \ - $SRC/${fuzzer}_fuzzer.cc -o $OUT/${fuzzer}_fuzzer \ - -lFuzzingEngine $LIB -done From 5aa12028c98edcd2281b68c7facfbd02c8172d57 Mon Sep 17 00:00:00 2001 From: Chris Wolfe Date: Fri, 17 Apr 2020 10:09:43 -0700 Subject: [PATCH 5/8] get the fuzzer built and running as part of Ctest --- fuzz/CMakeLists.txt | 1 + fuzz/tests/valid.json | 1 + 2 files changed, 2 insertions(+) create mode 100644 fuzz/tests/valid.json diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index 38517e0..d24714e 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -21,4 +21,5 @@ foreach(FUZZERNAME json-c ${fuzzing_engine}) + add_test(NAME test_${FUZZERNAME} COMMAND ${CMAKE_BINARY_DIR}/fuzz/${FUZZERNAME} ${CMAKE_SOURCE_DIR}/fuzz/tests/valid.json) endforeach(FUZZERNAME) diff --git a/fuzz/tests/valid.json b/fuzz/tests/valid.json new file mode 100644 index 0000000..bde58e7 --- /dev/null +++ b/fuzz/tests/valid.json @@ -0,0 +1 @@ +{"foo":123} From 2380df9a371caa1cd72b032877517a8cf24a0b66 Mon Sep 17 00:00:00 2001 From: Chris Wolfe Date: Fri, 17 Apr 2020 10:27:46 -0700 Subject: [PATCH 6/8] make independent test cases for each of the files in fuzz/test/* --- fuzz/CMakeLists.txt | 19 +++++++++++++++---- fuzz/README.md | 20 ++++++++++++++++++++ fuzz/bad_fuzzer.cc | 9 +++++++++ fuzz/build.sh | 17 +++++++++++++---- 4 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 fuzz/bad_fuzzer.cc diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index d24714e..4489535 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -5,21 +5,32 @@ enable_language(CXX) include_directories(PUBLIC ${CMAKE_SOURCE_DIR}) include_directories(PUBLIC ${PROJECT_BINARY_DIR}) +# This sets up either the standalone runner or the user supplied libfuzzing +# engine. We're using the standalone runner for unit testing alone at the +# moment to make the sure the fuzzer builds and runs appropriately. if(DEFINED ENV{LIB_FUZZING_ENGINE}) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} $ENV{LIB_FUZZING_ENGINE}") - set(fuzzing_engine "") + set(FUZZING_ENGINE "") else() add_library(fuzzing_engine standalone_runner.cc) - set(fuzzing_engine fuzzing_engine) + set(FUZZING_ENGINE fuzzing_engine) endif() foreach(FUZZERNAME + bad_fuzzer tokener_parse_ex_fuzzer) add_executable(${FUZZERNAME} ${FUZZERNAME}.cc) target_link_libraries(${FUZZERNAME} json-c - ${fuzzing_engine}) + ${FUZZING_ENGINE}) - add_test(NAME test_${FUZZERNAME} COMMAND ${CMAKE_BINARY_DIR}/fuzz/${FUZZERNAME} ${CMAKE_SOURCE_DIR}/fuzz/tests/valid.json) + file(GLOB TESTFILES "${CMAKE_SOURCE_DIR}/fuzz/tests/*.json") + foreach(TESTFILE ${TESTFILES}) + get_filename_component(FILENAME ${TESTFILE} NAME_WE) + add_test(NAME test_${FUZZERNAME}_${FILENAME} + COMMAND ${CMAKE_BINARY_DIR}/fuzz/${FUZZERNAME} + ${TESTFILE}) + + endforeach(TESTFILE) endforeach(FUZZERNAME) diff --git a/fuzz/README.md b/fuzz/README.md index 237c1da..8962690 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -4,3 +4,23 @@ This directory contains fuzzers that target [llvm's LibFuzzer](https://llvm.org/docs/LibFuzzer.html). They are built and run automatically by Google's [OSS-Fuzz](https://github.com/google/oss-fuzz/) infrastructure. + +## How do I test or run the fuzzers like oss-fuzz? + +``` +git clone https://github.com/google/oss-fuzz.git +cd oss-fuzz +python infra/helper.py build_image json-c +python infra/helper.py build_fuzzers --sanitizer address --engine libfuzzer --architecture x86_64 json-c +python infra/helper.py run_fuzzer json-c tokener_parse_ex_fuzzer +``` + +## How do I add new unit or regression tests for the fuzzer? + +The tests directory contains json files that can be used to either test the fuzzer itself or be used as regression tests. As long as the files end in `.json`, cmake will pick them up and generate a Ctest test case. If/when oss-fuzz finds a bug with a fuzzer, simply pull that test case into the `./tests` directory and it will serve as a regression test. + +Note - the fuzzers are not being run with sanitizers in this repository's CI at the moment; we're strictly building them here to ensure that they function. + +## How do I reproduce a failure form a fuzzer? + +Use [the steps detailed on OSS-fuzz](https://google.github.io/oss-fuzz/advanced-topics/reproducing/). diff --git a/fuzz/bad_fuzzer.cc b/fuzz/bad_fuzzer.cc new file mode 100644 index 0000000..6d71c43 --- /dev/null +++ b/fuzz/bad_fuzzer.cc @@ -0,0 +1,9 @@ +#include + +#include + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + int *array = new int[100]; + delete [] array; + return array[size]; // BOOM +} diff --git a/fuzz/build.sh b/fuzz/build.sh index 3864015..a723496 100755 --- a/fuzz/build.sh +++ b/fuzz/build.sh @@ -15,13 +15,22 @@ # ################################################################################ # this is expected to be run only by oss-fuzz. It will run from $SRC (above json-c) -cp $SRC/json-c/fuzz/*.dict $OUT/ +cp $SRC/json-c/fuzz/*.dict "$OUT/" BUILD="$SRC/json-c/build" -mkdir $BUILD -cd $BUILD +zip -j "$SRC/corpus.zip" "$SRC/go-fuzz-corpus/json/corpus" + +mkdir "$BUILD" +cd "$BUILD" cmake -DCMAKE_C_FLAGS="$CFLAGS" -DCMAKE_CXX_FLAGS="$CXXFLAGS" -DBUILD_SHARED_LIBS=OFF .. make -j$(nproc) -cp fuzz/*_fuzzer $OUT/ +cp fuzz/*_fuzzer "$OUT/" + +fuzzerFiles=$(find fuzz/ -name "*_fuzzer") +for F in $fuzzerFiles; do + FN=$(basename -- $F) + cp "$SRC/corpus.zip" "$OUT/${FN}_seed_corpus.zip" +done +# for fuzzer in fuzz copy the json corpus as a zip file From 31a8afbf880fb1b397f111055d9819723b570f49 Mon Sep 17 00:00:00 2001 From: Chris Wolfe Date: Fri, 17 Apr 2020 21:50:49 -0700 Subject: [PATCH 7/8] remove bad fuzzer --- fuzz/CMakeLists.txt | 1 - fuzz/bad_fuzzer.cc | 9 --------- fuzz/build.sh | 1 - 3 files changed, 11 deletions(-) delete mode 100644 fuzz/bad_fuzzer.cc diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index 4489535..76a2736 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -17,7 +17,6 @@ else() endif() foreach(FUZZERNAME - bad_fuzzer tokener_parse_ex_fuzzer) add_executable(${FUZZERNAME} ${FUZZERNAME}.cc) diff --git a/fuzz/bad_fuzzer.cc b/fuzz/bad_fuzzer.cc deleted file mode 100644 index 6d71c43..0000000 --- a/fuzz/bad_fuzzer.cc +++ /dev/null @@ -1,9 +0,0 @@ -#include - -#include - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - int *array = new int[100]; - delete [] array; - return array[size]; // BOOM -} diff --git a/fuzz/build.sh b/fuzz/build.sh index a723496..62778b0 100755 --- a/fuzz/build.sh +++ b/fuzz/build.sh @@ -33,4 +33,3 @@ for F in $fuzzerFiles; do FN=$(basename -- $F) cp "$SRC/corpus.zip" "$OUT/${FN}_seed_corpus.zip" done -# for fuzzer in fuzz copy the json corpus as a zip file From 968bdd099da343fd5112589590dc79e6c8f12f69 Mon Sep 17 00:00:00 2001 From: Chris Wolfe Date: Sat, 18 Apr 2020 14:37:17 -0700 Subject: [PATCH 8/8] set the right standard --- fuzz/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index 76a2736..eecf030 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -1,6 +1,7 @@ # https://cmake.org/cmake/help/v3.0/command/add_test.html # https://pabloariasal.github.io/2018/02/19/its-time-to-do-cmake-right/ enable_language(CXX) +set (CMAKE_CXX_STANDARD 11) include_directories(PUBLIC ${CMAKE_SOURCE_DIR}) include_directories(PUBLIC ${PROJECT_BINARY_DIR})