Restructure repo and rework Makefiles with proper dependency tracking

fa11a2ac · Antoine Kaufmann · bc9b2e09 · fa11a2ac · fa11a2ac · bc9b2e09
Commit fa11a2ac authored Jan 11, 2021 by Antoine Kaufmann
20 changed files
--- a/.gitignore
+++ b/.gitignore
 *.o
+*.d
 *.a
 *.ko
 .*.cmd
 *.pyc
 __pycache__/
 _vimrc_local.vim
-dummy_nic/dummy_nic
+sims/nic/corundum/obj_dir
-corundum/obj_dir
+sims/nic/corundum/corundum_verilator
-corundum/corundum_verilator
+sims/nic/i40e_bm/i40e_bm
-net_wire/net_wire
+sims/net/wire/net_wire
-net_tap/net_tap
+sims/net/tap/net_tap
-net_switch/net_switch
+sims/net/switch/net_switch
-i40e_bm/i40e_bm
+doc/doxygen
+doc/_build
 images/kernel/kheaders/
 images/mqnic/Module.symvers
 images/mqnic/modules.order
@@ -29,4 +31,4 @@ experiments/local-config.sh
 gem5/
 qemu/
 ns-3/
 experiments/slurm
\ No newline at end of file
--- a/Makefile
+++ b/Makefile
-REPO_BASE= $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+include mk/subdir_pre.mk
+include mk/recipes.mk
-all: \
+base_dir := $(d)./
-	corundum/corundum_verilator \
-	corundum_bm/corundum_bm \
+CFLAGS += -Wall -Wextra -Wno-unused-parameter -O3 -fPIC
-	i40e_bm/i40e_bm \
+CXXFLAGS += -Wall -Wextra -Wno-unused-parameter -O3 -fPIC
-	net_tap/net_tap \
-	net_wire/net_wire \
+VERILATOR = verilator
-	net_switch/net_switch
+VFLAGS = +1364-2005ext+v \
+    -Wno-WIDTH -Wno-PINMISSING -Wno-LITENDIAN -Wno-IMPLICIT -Wno-SELRANGE \
+    -Wno-CASEINCOMPLETE -Wno-UNSIGNED
+$(eval $(call subdir,lib))
+$(eval $(call subdir,sims))
+$(eval $(call subdir,images))
-clean:
-	$(MAKE) -C corundum/ clean
-	$(MAKE) -C corundum_bm/ clean
-	$(MAKE) -C i40e_bm/ clean
-	$(MAKE) -C dummy_nic/ clean
-	$(MAKE) -C net_tap/ clean
-	$(MAKE) -C net_wire/ clean
-	$(MAKE) -C net_switch/ clean
-	$(MAKE) -C nicsim_common/ clean
-	$(MAKE) -C netsim_common/ clean
-	$(MAKE) -C libnicbm/ clean
 help:
 	@echo "Targets:"
@@ -27,78 +23,7 @@ help:
 	@echo "  build-images: prepare prereqs for VMs (images directory)"
 	@echo "  external: clone and build our tools in external repos "
 	@echo "            (qemu, gem5, ns-3)"
-	@echo "  run-experiments: run all experiments (experiments directory,"
-	@echo "                   implies all, build-images, external)"
-####################################
-# Tools in this repo
-corundum/corundum_verilator: nicsim_common/libnicsim_common.a
-	$(MAKE) -C corundum/ all
-corundum_bm/corundum_bm: nicsim_common/libnicsim_common.a libnicbm/libnicbm.a
-	$(MAKE) -C corundum_bm/ all
-i40e_bm/i40e_bm: nicsim_common/libnicsim_common.a libnicbm/libnicbm.a
-	$(MAKE) -C i40e_bm/ all
-dummy_nic/dummy_nic: nicsim_common/libnicsim_common.a
-	$(MAKE) -C dummy_nic all
-net_tap/net_tap: netsim_common/libnetsim_common.a
-	$(MAKE) -C net_tap/
-net_wire/net_wire: netsim_common/libnetsim_common.a
-	$(MAKE) -C net_wire/
-net_switch/net_switch: netsim_common/libnetsim_common.a
-	$(MAKE) -C net_switch/
-nicsim_common/libnicsim_common.a:
-	$(MAKE) -C nicsim_common/
-netsim_common/libnetsim_common.a:
-	$(MAKE) -C netsim_common/
-libnicbm/libnicbm.a:
-	$(MAKE) -C libnicbm
-####################################
-# External dependencies
-external: gem5/ready qemu/ready ns-3/ready
-gem5:
-	git clone git@github.com:FreakyPenguin/gem5-cosim.git gem5
-gem5/ready: gem5
-	+cd gem5 && scons build/X86/gem5.opt -j`nproc`
-	touch gem5/ready
-qemu:
-	git clone git@github.com:FreakyPenguin/qemu-cosim.git qemu
-qemu/ready: qemu
-	+cd qemu && ./configure \
-	    --target-list=x86_64-softmmu \
-	    --disable-werror \
-	    --extra-cflags="-I$(REPO_BASE)/proto" \
-	    --enable-cosim-pci && \
-	  $(MAKE)
-	touch qemu/ready
-ns-3:
-	git clone git@github.com:FreakyPenguin/ns-3-cosim.git ns-3
-ns-3/ready: ns-3 netsim_common/libnetsim_common.a
-	+cd ns-3 && COSIM_PATH=$(REPO_BASE) ./cosim-build.sh configure
-	touch ns-3/ready
-####################################
-# External dependencies
-run-experiments: all build-images external
+.PHONY: help
-	$(MAKE) -C experiments
-build-images:
+include mk/subdir_post.mk
-	$(MAKE) -C images
--- a/corundum/Makefile
+++ b/corundum/Makefile
-CXXFLAGS = -Wall
-VERILATOR = verilator
-VFLAGS = +1364-2005ext+v \
-    -Wno-WIDTH -Wno-PINMISSING -Wno-LITENDIAN -Wno-IMPLICIT -Wno-SELRANGE \
-    -Wno-CASEINCOMPLETE -Wno-UNSIGNED
-SRCS = corundum_verilator.cpp dma.cpp mem.cpp
-mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
-BASEDIR := $(abspath $(dir $(mkfile_path))/../)
-all: corundum_verilator
-obj_dir/Vinterface.cpp: rtl/interface.v
-	$(VERILATOR) $(VFLAGS) --cc -O3 \
-	    -CFLAGS "-I$(BASEDIR)/nicsim_common/include -I$(BASEDIR)/proto -O3 -g -Wall" \
-	    -y rtl \
-	    -y lib/axi/rtl \
-	    -y lib/eth/lib/axis/rtl/ \
-	    -y ./lib/pcie/rtl \
-	    rtl/interface.v --exe $(SRCS) "$(BASEDIR)/nicsim_common/libnicsim_common.a"
-obj_dir/Vinterface: obj_dir/Vinterface.cpp $(SRCS)
-	 $(MAKE) -C obj_dir -f Vinterface.mk
-corundum_verilator: obj_dir/Vinterface
-	cp $< $@
-#obj_dir/%.o: CXXFLAGS += -Wno-all
-#obj_dir/%.o: CPPFLAGS += -I /usr/share/verilator/include/
-#
-#obj_dir/verilated.o: /usr/share/verilator/include/verilated.cpp
-#	$(CXX) $(CXXFLAGS) -c $< -o $@
-#
-#obj_dir/verilated_vcd_c.o: /usr/share/verilator/include/verilated_vcd_c.cpp
-#	$(CXX) $(CXXFLAGS) -c $< -o $@
-#
-#corundum_verilator: corundum_verilator.o obj_dir/verilated.o \
-#	    obj_dir/verilated_vcd_c.o obj_dir/Vinterface.o \
-#	    obj_dir/Vinterface__Syms.o obj_dir/Vinterface__Trace.o \
-#	    obj_dir/Vinterface__Trace__Slow.o
-#	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o $@ $^ $(LDLIBS)
-lightclean:
-	rm -f obj_dir/Vinterface corundum_verilator \
-	    obj_dir/corundum_verilator.o obj_dir/mem.o obj_dir/dma.o
-clean:
-	rm -rf obj_dir corundum_verilator *.o
-.PHONY: all clean lightclean
--- a/corundum_bm/Makefile
+++ b/corundum_bm/Makefile
-CPPFLAGS += -I../nicsim_common/include -I../netsim_common/include -I../proto
-CPPFLAGS += -I../libnicbm/include/
-CFLAGS += -Wall -Wextra -Wno-unused-parameter -O3
-all: corundum_bm tester
-corundum.o: corundum.cc
-	g++ $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
-corundum_bm: corundum_bm.o ../libnicbm/libnicbm.a ../nicsim_common/libnicsim_common.a
-	g++ $^ -o $@
-tester: tester.o ../libnicbm/libnicbm.a ../nicsim_common/libnicsim_common.a
-clean:
-	rm -f *.o corundum_bm tester
--- a/i40e_bm/Makefile
+++ b/i40e_bm/Makefile
-CPPFLAGS += -I../nicsim_common/include -I../proto
-CPPFLAGS += -I../libnicbm/include/
-CXXFLAGS += -Wall -Wextra -Wno-unused-parameter -O3 -g
-LDFLGAS = -g
-OBJS := i40e_bm.o i40e_queues.o i40e_adminq.o i40e_hmc.o i40e_lan.o xsums.o \
-    rss.o logger.o
-all: i40e_bm
-i40e_bm: $(OBJS) ../libnicbm/libnicbm.a ../nicsim_common/libnicsim_common.a
-i40e_bm: CC=$(CXX)
-clean:
-	rm -f *.o i40e_bm
--- a/images/Makefile
+++ b/images/Makefile
-PACKER_VERSION := 1.6.0
-KERNEL_VERSION := 5.4.46
-UBUNTU_IMAGE := output-ubuntu1804/ubuntu1804
-BASE_IMAGE := output-base/base
-NOPAXOS_IMAGE := output-nopaxos/nopaxos
-MTCP_IMAGE := output-mtcp/mtcp
-TAS_IMAGE := output-tas/tas
-IMAGES := $(BASE_IMAGE) $(NOPAXOS_IMAGE) $(MTCP_IMAGE) $(TAS_IMAGE)
-RAW_IMAGES := $(addsuffix .raw,$(IMAGES))
-QEMU_IMG := ../qemu/build/qemu-img
-all: $(IMAGES) $(RAW_IMAGES) vmlinux bzImage mqnic/mqnic.ko
-clean:
-	rm -rf packer packer_cache vmlinux bzImage \
-	    mqnic/mqnic.ko mqnic/*.o mqnic/.*.cmd mqnic/mqnic.mod.c \
-	    mqnic/Module.symvers mqnic/modules.order \
-	    output-ubuntu1804 output-base output-mtcp output-nopaxos \
-	    output-tas \
-	    kernel/linux-$(KERNEL_VERSION)/ kheaders.tar.bz2
-################################################
-# Disk image
-%.raw: %
-	$(QEMU_IMG) convert -f qcow2 -O raw $< $@
-$(UBUNTU_IMAGE): ubuntu1804.json packer scripts/vagrant.sh scripts/sshd.sh \
-    scripts/update.sh scripts/packages.sh scripts/cleanup.sh scripts/preseed.cfg
-	rm -rf output-ubuntu1804
-	./packer-wrap.sh build ubuntu1804.json
-	touch $@
-$(BASE_IMAGE): base.json packer $(UBUNTU_IMAGE) bzImage kheaders.tar.bz2 \
-    scripts/guestinit.sh
-	rm -rf output-base
-	./packer-wrap.sh build base.json
-	touch $@
-$(NOPAXOS_IMAGE): nopaxos.json packer $(BASE_IMAGE) scripts/install-nopaxos.sh \
-    nopaxos.config
-	rm -rf output-nopaxos
-	./packer-wrap.sh build nopaxos.json
-	touch $@
-$(MTCP_IMAGE): mtcp.json packer $(BASE_IMAGE) scripts/install-mtcp.sh
-	rm -rf output-mtcp
-	./packer-wrap.sh build mtcp.json
-	touch $@
-$(TAS_IMAGE): tas.json packer $(BASE_IMAGE) scripts/install-tas.sh
-	rm -rf output-tas
-	./packer-wrap.sh build tas.json
-	touch $@
-packer:
-	wget https://releases.hashicorp.com/packer/$(PACKER_VERSION)/packer_$(PACKER_VERSION)_linux_amd64.zip
-	unzip packer_$(PACKER_VERSION)_linux_amd64.zip
-	rm -f packer_$(PACKER_VERSION)_linux_amd64.zip
-################################################
-# Kernel
-KERNEL_DIR := kernel/linux-$(KERNEL_VERSION)
-$(KERNEL_DIR)/vmlinux: $(KERNEL_DIR)/.config
-	$(MAKE) -C $(KERNEL_DIR)
-	touch $@
-vmlinux: $(KERNEL_DIR)/vmlinux
-	cp kernel/linux-$(KERNEL_VERSION)/vmlinux vmlinux
-	touch $@
-# this dependency is a bit stupid, but not sure how to better do this
-bzImage: $(KERNEL_DIR)/vmlinux
-	cp $(KERNEL_DIR)/arch/x86_64/boot/bzImage bzImage
-	touch $@
-kheaders.tar.bz2:
-	rm -rf ./kernel/kheaders
-	mkdir -p ./kernel/kheaders
-	$(MAKE) -C $(KERNEL_DIR) headers_install INSTALL_HDR_PATH=$(abspath ./kernel/kheaders/usr)
-	$(MAKE) -C $(KERNEL_DIR) modules_install INSTALL_MOD_PATH=$(abspath ./kernel/kheaders)
-	rm -f ./kernel/kheaders/lib/modules/$(KERNEL_VERSION)/build
-	ln -s /usr/src/linux-headers-$(KERNEL_VERSION) \
-	    ./kernel/kheaders/lib/modules/$(KERNEL_VERSION)/build
-	rm -f ./kernel/kheaders/lib/modules/$(KERNEL_VERSION)/source
-	mkdir -p ./kernel/kheaders/usr/src/linux-headers-$(KERNEL_VERSION)
-	cp -r $(KERNEL_DIR)/.config $(KERNEL_DIR)/Makefile \
-	    $(KERNEL_DIR)/Module.symvers $(KERNEL_DIR)/scripts \
-	    $(KERNEL_DIR)/include \
-	    ./kernel/kheaders/usr/src/linux-headers-$(KERNEL_VERSION)/
-	mkdir -p ./kernel/kheaders/usr/src/linux-headers-$(KERNEL_VERSION)/tools/objtool/
-	cp $(KERNEL_DIR)/tools/objtool/objtool \
-	    ./kernel/kheaders/usr/src/linux-headers-$(KERNEL_VERSION)/tools/objtool/
-	mkdir -p ./kernel/kheaders/usr/src/linux-headers-$(KERNEL_VERSION)/arch/x86/
-	cp -r $(KERNEL_DIR)/arch/x86/Makefile \
-	    $(KERNEL_DIR)/arch/x86/Makefile_32.cpu \
-	    $(KERNEL_DIR)/arch/x86/Makefile.um \
-	    $(KERNEL_DIR)/arch/x86/include \
-	    ./kernel/kheaders/usr/src/linux-headers-$(KERNEL_VERSION)/arch/x86
-	cd ./kernel/kheaders && tar cjf ../../$@ .
-kernel/linux-$(KERNEL_VERSION)/.config: kernel/config-$(KERNEL_VERSION)
-	wget https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-$(KERNEL_VERSION).tar.xz
-	tar xf linux-$(KERNEL_VERSION).tar.xz
-	rm -f linux-$(KERNEL_VERSION).tar.xz
-	rm -rf kernel/linux-$(KERNEL_VERSION)
-	mv linux-$(KERNEL_VERSION) kernel/
-	cd kernel/linux-$(KERNEL_VERSION) && patch -p1 < ../linux-$(KERNEL_VERSION)-timers-gem5.patch
-	cp kernel/config-$(KERNEL_VERSION) kernel/linux-$(KERNEL_VERSION)/.config
-################################################
-# mqnic kernel module
-mqnic/mqnic.ko: vmlinux
-	$(MAKE) -C kernel/linux-$(KERNEL_VERSION) M=`pwd`/mqnic/ modules
-	touch $@
--- a/images/rules.mk
+++ b/images/rules.mk
+include mk/subdir_pre.mk
+PACKER_VERSION := 1.6.0
+KERNEL_VERSION := 5.4.46
+UBUNTU_IMAGE := $(d)output-ubuntu1804/ubuntu1804
+BASE_IMAGE := $(d)output-base/base
+NOPAXOS_IMAGE := $(d)output-nopaxos/nopaxos
+MTCP_IMAGE := $(d)output-mtcp/mtcp
+TAS_IMAGE := $(d)output-tas/tas
+IMAGES := $(UBUNTU_IMAGE) $(BASE_IMAGE) $(NOPAXOS_IMAGE) $(MTCP_IMAGE) \
+    $(TAS_IMAGE)
+RAW_IMAGES := $(addsuffix .raw,$(IMAGES))
+img_dir := $(d)
+packer := $(d)packer
+bz_image := $(d)bzImage
+vmlinux := $(d)vmlinux
+kernel_pardir := $(d)kernel
+kernel_dir := $(kernel_pardir)/linux-$(KERNEL_VERSION)
+kheader_dir := $(d)kernel/kheaders
+mqnic_dir := $(d)mqnic
+mqnic_mod := $(mqnic_dir)/mqnic.ko
+build-images: $(IMAGES) $(RAW_IMAGES) $(vmlinux) $(bz_image) $(mqnic_mod)
+################################################
+# Disk image
+%.raw: %
+	$(QEMU_IMG) convert -f qcow2 -O raw $< $@
+$(UBUNTU_IMAGE): $(packer) $(QEMU) $(addprefix $(d),ubuntu1804.json \
+    scripts/vagrant.sh scripts/sshd.sh scripts/update.sh scripts/packages.sh \
+    scripts/cleanup.sh scripts/preseed.cfg)
+	rm -rf $(dir $@)
+	cd $(img_dir) && ./packer-wrap.sh build ubuntu1804.json
+	touch $@
+$(BASE_IMAGE): $(packer) $(QEMU) $(d)base.json $(UBUNTU_IMAGE) $(bz_image) \
+    $(d)kheaders.tar.bz2 $(d)scripts/guestinit.sh
+	rm -rf $(dir $@)
+	cd $(img_dir) && ./packer-wrap.sh build base.json
+	touch $@
+$(NOPAXOS_IMAGE): $(packer) $(QEMU) $(d)nopaxos.json $(BASE_IMAGE) \
+    $(addprefix $(d), scripts/install-nopaxos.sh nopaxos.config)
+	rm -rf $(dir $@)
+	cd $(img_dir) && ./packer-wrap.sh build nopaxos.json
+	touch $@
+$(MTCP_IMAGE): $(packer) $(QEMU) $(d)mtcp.json $(BASE_IMAGE) \
+    $(d)scripts/install-mtcp.sh
+	rm -rf $(dir $@)
+	cd $(img_dir) && ./packer-wrap.sh build mtcp.json
+	touch $@
+$(TAS_IMAGE): $(packer) $(QEMU) $(d)tas.json $(BASE_IMAGE) \
+    $(d)scripts/install-tas.sh
+	rm -rf $(dir $@)
+	cd $(img_dir) && ./packer-wrap.sh build tas.json
+	touch $@
+$(packer):
+	wget -O $(img_dir)packer_$(PACKER_VERSION)_linux_amd64.zip https://releases.hashicorp.com/packer/$(PACKER_VERSION)/packer_$(PACKER_VERSION)_linux_amd64.zip
+	cd $(img_dir) && unzip packer_$(PACKER_VERSION)_linux_amd64.zip
+	rm -f $(img_dir)packer_$(PACKER_VERSION)_linux_amd64.zip
+################################################
+# Kernel
+$(kernel_dir)/vmlinux: $(kernel_dir)/.config
+	$(MAKE) -C $(kernel_dir)
+	touch $@
+$(vmlinux): $(kernel_dir)/vmlinux
+	cp $< $@
+	touch $@
+# this dependency is a bit stupid, but not sure how to better do this
+$(bz_image): $(kernel_dir)/vmlinux
+	cp $(kernel_dir)/arch/x86_64/boot/bzImage $@
+	touch $@
+$(d)kheaders.tar.bz2: $(kernel_dir)/vmlinux
+	rm -rf $(kheader_dir)
+	mkdir -p $(kheader_dir)
+	$(MAKE) -C $(kernel_dir) headers_install INSTALL_HDR_PATH=$(abspath $(kheader_dir)/usr)
+	$(MAKE) -C $(kernel_dir) modules_install INSTALL_MOD_PATH=$(abspath $(kheader_dir))
+	rm -f $(kheader_dir)/lib/modules/$(KERNEL_VERSION)/build
+	ln -s /usr/src/linux-headers-$(KERNEL_VERSION) \
+	    $(kheader_dir)/lib/modules/$(KERNEL_VERSION)/build
+	rm -f $(kheader_dir)/lib/modules/$(KERNEL_VERSION)/source
+	mkdir -p $(kheader_dir)/usr/src/linux-headers-$(KERNEL_VERSION)
+	cp -r $(kernel_dir)/.config $(kernel_dir)/Makefile \
+	    $(kernel_dir)/Module.symvers $(kernel_dir)/scripts \
+	    $(kernel_dir)/include \
+	    $(kheader_dir)/usr/src/linux-headers-$(KERNEL_VERSION)/
+	mkdir -p $(kheader_dir)/usr/src/linux-headers-$(KERNEL_VERSION)/tools/objtool/
+	cp $(kernel_dir)/tools/objtool/objtool \
+	    $(kheader_dir)/usr/src/linux-headers-$(KERNEL_VERSION)/tools/objtool/
+	mkdir -p $(kheader_dir)/usr/src/linux-headers-$(KERNEL_VERSION)/arch/x86/
+	cp -r $(kernel_dir)/arch/x86/Makefile \
+	    $(kernel_dir)/arch/x86/Makefile_32.cpu \
+	    $(kernel_dir)/arch/x86/Makefile.um \
+	    $(kernel_dir)/arch/x86/include \
+	    $(kheader_dir)/usr/src/linux-headers-$(KERNEL_VERSION)/arch/x86
+	cd $(kheader_dir) && tar cjf $(abspath $@) .
+$(kernel_dir)/.config: $(kernel_pardir)/config-$(KERNEL_VERSION)
+	rm -rf $(kernel_dir)
+	wget -O - https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-$(KERNEL_VERSION).tar.xz | \
+	    tar xJf - -C $(kernel_pardir)
+	cd $(kernel_dir) && patch -p1 < ../linux-$(KERNEL_VERSION)-timers-gem5.patch
+	cp $< $@
+################################################
+# mqnic kernel module
+$(mqnic_mod): $(vmlinux)
+	$(MAKE) -C $(kernel_dir) M=$(abspath $(mqnic_dir)) modules
+	touch $@
+CLEAN := $(addprefix $(d), mqnic/mqnic.ko mqnic/*.o mqnic/.*.cmd mqnic/*.mod \
+    mqnic/mqnic.mod.c mqnic/Module.symvers mqnic/modules.order)
+DISTCLEAN := $(kernel_dir) $(packer) $(bz_image) $(vmlinux) $(kheader_dir) \
+    $(foreach i,$(IMAGES),$(dir $(i))) \
+    $(d)packer_cache $(d)kheaders.tar.bz2
+include mk/subdir_post.mk
--- a/netsim_common/include/netsim.h
+++ b/netsim_common/include/netsim.h
--- a/netsim_common/internal.h
+++ b/netsim_common/internal.h
--- a/netsim_common/netsim.c
+++ b/netsim_common/netsim.c
--- a/lib/netif/rules.mk
+++ b/lib/netif/rules.mk
+include mk/subdir_pre.mk
+lib_netsim := $(d)libnetsim_common.a
+lib_netsim_inc := $(d)include/
+OBJS := $(addprefix $(d),netsim.o utils.o)
+$(OBJS): CPPFLAGS := $(CPPFLAGS) -I$(lib_netsim_inc) -I$(lib_proto_inc)
+$(lib_netsim): $(OBJS)
+CLEAN := $(lib_netsim) $(OBJS)
+include mk/subdir_post.mk
--- a/netsim_common/utils.c
+++ b/netsim_common/utils.c
--- a/libnicbm/include/nicbm.h
+++ b/libnicbm/include/nicbm.h
--- a/libnicbm/nicbm.cc
+++ b/libnicbm/nicbm.cc
--- a/lib/nicbm/rules.mk
+++ b/lib/nicbm/rules.mk
+include mk/subdir_pre.mk
+lib_nicbm := $(d)libnicbm.a
+lib_nicbm_inc := $(d)include/
+OBJS := $(addprefix $(d),nicbm.o)
+$(OBJS): CPPFLAGS := $(CPPFLAGS) -I$(lib_nicbm_inc) -I$(lib_proto_inc) \
+    -I$(lib_nicsim_inc)
+$(lib_nicbm): $(OBJS)
+CLEAN := $(lib_nicbm) $(OBJS)
+include mk/subdir_post.mk
--- a/nicsim_common/include/nicsim.h
+++ b/nicsim_common/include/nicsim.h
--- a/nicsim_common/internal.h
+++ b/nicsim_common/internal.h
--- a/nicsim_common/nicsim.c
+++ b/nicsim_common/nicsim.c
--- a/lib/nicif/rules.mk
+++ b/lib/nicif/rules.mk
+include mk/subdir_pre.mk
+lib_nicsim := $(d)libnicsim_common.a
+lib_nicsim_inc := $(d)include/
+OBJS := $(addprefix $(d),nicsim.o utils.o)
+$(OBJS): CPPFLAGS := $(CPPFLAGS) -I$(lib_nicsim_inc) -I$(lib_proto_inc)
+$(lib_nicsim): $(OBJS)
+CLEAN := $(lib_nicsim) $(OBJS)
+include mk/subdir_post.mk
--- a/nicsim_common/utils.c
+++ b/nicsim_common/utils.c