Update libafl_nyx to newest qemu-nyx and nyx-packer, fixing some problems with build_nyx_support.sh. Providing Makefile for bug in qemu-nyx (#2070)

Co-authored-by: ganbarutobi <ganbarutobi@gmail.com>
Co-authored-by: Romain Malmain <romain.malmain@pm.me>

libafl_nyx/Makefile.libxdc

@@ -0,0 +1,54 @@
+CC ?= gcc
+CFLAGS += -Ofast -fPIC -fvisibility=hidden -finline-functions
+LDFLAGS =
+
+ifneq ($(origin NO_LTO), environment)
+	CFLAGS += -flto
+	LDFLAGS += -flto
+endif
+
+PREFIX ?= /usr
+
+ODIR=build
+SDIR=src
+
+_OBJ = cfg.o disassembler.o tnt_cache.o decoder.o libxdc.o mmh3.o trace_cache.o
+OBJ = $(patsubst %,$(ODIR)/%,$(_OBJ))
+
+default: libxdc.so libxdc.a ptdump ptdump_static
+
+$(ODIR)/%.o: $(SDIR)/%.c $(SDIR)/*.h libxdc.h
+	mkdir -p build
+	$(CC) -c -o $@ $< $(CFLAGS)
+
+libxdc.so: $(OBJ)
+	$(CC) $^ -o $@ -shared $(CFLAGS) $(LDFLAGS) -L../capstone_v4/ -l:libcapstone.so.4
+
+libxdc.a: $(OBJ)
+	$(AR) rcs $@ $^
+
+ptdump: libxdc.so test/*.c test/*.h
+	$(CC) test/ptdump.c test/page_cache.c test/helper.c -o build/$@ -Itest/ -I./ -Lbuild/ $(CFLAGS) $(LDFLAGS) -L. -lxdc -L../capstone_v4/ -l:libcapstone.so.4
+
+ptdump_static: libxdc.a test/*.c test/*.h
+	$(CC) test/ptdump.c test/page_cache.c test/helper.c -o build/$@ -Itest/ -I./ $(CFLAGS) $(LDFLAGS) -L. -l:libxdc.a -L../capstone_v4/ -l:libcapstone.a
+
+tester_dyn: libxdc.so test/*.c test/*.h
+	$(CC) test/tester.c test/page_cache.c test/helper.c -o $@ -Itest/ -I./ $(CFLAGS) $(LDFLAGS) -L. -lxdc -L../capstone_v4/ -l:libcapstone.so.4
+
+tester_static: libxdc.a test/*.c test/*.h
+	$(CC) test/tester.c test/page_cache.c test/helper.c -o $@ -Itest/ -I./ $(CFLAGS) $(LDFLAGS) -L. -l:libxdc.a -L../capstone_v4/ -l:libcapstone.a
+
+install: libxdc.so libxdc.a ptdump
+	mkdir -p $(PREFIX)/include $(PREFIX)/lib
+	install -m0644 libxdc.h $(PREFIX)/include/
+	install -m0755 libxdc.so $(PREFIX)/lib/
+	install -m0755 libxdc.a $(PREFIX)/lib/
+	install -m0755 build/ptdump $(PREFIX)/bin/
+
+.PHONY: clean install
+
+clean:
+	rm -f $(ODIR)/*.o build/*
+	rm -f libxdc.so
+	rm -f libxdc.a

libafl_nyx/build_nyx_support.sh

@@ -13,21 +13,23 @@ if [ ! -e ./QEMU-Nyx/.git ]; then
     rm -rf ./QEMU-Nyx
     git clone https://github.com/nyx-fuzz/QEMU-Nyx.git || exit 1
     pushd QEMU-Nyx
-    git reset --hard 80f22f77d6aab14e62bf11c80db4e210bbca5fb5
+    git reset --hard e5e1c4c21ff9c4dc80e6409d4eab47146c6024cd
     popd
 fi
 
 if [ ! -e ./packer/.git ]; then
     rm -rf ./packer
-    git clone https://github.com/syheliel/packer.git || exit 1
+    git clone https://github.com/nyx-fuzz/packer  || exit 1
     pushd packer
-    git reset --hard 86b159bafc0b2ba8feeaa8761a45b6201d34084f
+    git reset --hard bcf3e248b660764f48af54232a3388389a2dfc22
     popd
 fi
 
 git submodule init || exit 1
 echo "[*] initializing QEMU-Nyx submodule"
-git submodule update ./QEMU-Nyx 2>/dev/null # ignore errors
+cd QEMU-Nyx/ || return
+git submodule update --init .
+cd ..
 echo "[*] initializing packer submodule"
 git submodule update ./packer 2>/dev/null # ignore errors
 
@@ -35,6 +37,14 @@ git submodule update ./packer 2>/dev/null # ignore errors
 test -e packer/.git || { echo "[-] packer not checked out, please install git or check your internet connection." ; exit 1 ; }
 test -e QEMU-Nyx/.git || { echo "[-] QEMU-Nyx not checked out, please install git or check your internet connection." ; exit 1 ; }
 
+echo "[*] Checking QEMU-Nyx ..."
+if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then
+    cd QEMU-Nyx/ || return
+    cp ../Makefile.libxdc ./libxdc/Makefile || exit 1
+    ./compile_qemu_nyx.sh lto || exit 1
+    cd ..
+fi
+
 echo "[*] checking packer init.cpio.gz ..."
 if [ ! -f "packer/linux_initramfs/init.cpio.gz" ]; then
     cd packer/linux_initramfs/ || return
@@ -42,14 +52,6 @@ if [ ! -f "packer/linux_initramfs/init.cpio.gz" ]; then
     cd ../../
 fi
 
-
-echo "[*] Checking QEMU-Nyx ..."
-if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then
-    cd QEMU-Nyx/ || return
-    ./compile_qemu_nyx.sh lto || exit 1
-    cd ..
-fi
-
 echo "[+] All done for nyx_mode, enjoy!"
 
 exit 0