Skip to content

GitLab

"benchmark/vscode:/vscode.git/clone" did not exist on "f46f394f4d4dbe4aae85403dec006199b34d2840"
Commit ea903dda authored by Antoine Kaufmann's avatar Antoine Kaufmann
Browse files

images: use shared packer config for extended images 

Just run different scripts inside. This avodis lots of duplication in
the jsons.
parent 9f5e95cd
Show whitespace changes
Inline Side-by-side
Showing with 127 additions and 355 deletions
.gitignore
View file @ ea903dda
......@@ -24,6 +24,7 @@ images/mqnic/modules.order
images/mqnic/mqnic.mod*
images/vmlinux
images/bzImage
images/input-*
images/output-*
images/packer
images/packer_cache
......
images/base.json deleted 100644 → 0
View file @ 9f5e95cd
{
"_comment": "Build with `packer build ubuntu.json`",
"builders": [
{
"iso_urls": [ "output-ubuntu1804/ubuntu1804" ],
"iso_checksum": "none",
"disk_image": "true",
"use_backing_file": "true",
"headless": "{{ user `headless` }}",
"disable_vnc": "false",
"output_directory": "output-{{ user `vm_name` }}",
"qemuargs": [
[
"-m",
"{{ user `memory` }}"
],
[
"-display",
"none"
],
[
"-machine",
"accel=kvm"
],
[
"-cpu",
"host"
],
[
"-smp",
"cpus={{ user `cpus`}}"
],
[
"-serial",
"mon:stdio"
]
],
"shutdown_command": "echo '{{ user `ssh_password` }}'|sudo -S shutdown -P now",
"ssh_password": "{{ user `ssh_password` }}",
"ssh_timeout": "10000s",
"ssh_username": "{{ user `ssh_username` }}",
"type": "qemu",
"vm_name": "{{ user `vm_name` }}"
}
],
"provisioners": [
{
"type": "file",
"source": "scripts/guestinit.sh",
"destination": "/home/ubuntu/guestinit.sh",
"direction": "upload"
},
{
"type": "file",
"source": "bzImage",
"destination": "/tmp/bzImage",
"direction": "upload"
},
{
"type": "file",
"source": "kernel/config-5.4.46",
"destination": "/tmp/config-5.4.46",
"direction": "upload"
},
{
"type": "file",
"source": "kheaders.tar.bz2",
"destination": "/tmp/kheaders.tar.bz2",
"direction": "upload"
},
{
"type": "shell",
"inline": [
"sudo mv /tmp/bzImage /boot/vmlinuz-5.4.46",
"sudo mv /tmp/config-5.4.46 /boot/config-5.4.46",
"sudo update-grub",
"cd / && sudo tar xf /tmp/kheaders.tar.bz2"
]
},
{
"type": "file",
"source": "m5",
"destination": "/tmp/m5",
"direction": "upload"
},
{
"type": "shell",
"inline": ["sudo mv /tmp/m5 /sbin/m5"]
},
{
"execute_command": "echo '{{ user `ssh_password` }}' | {{.Vars}} sudo -E -S bash '{{.Path}}'",
"scripts": [
"scripts/install-base.sh"
],
"type": "shell"
}
],
"variables": {
"cleanup_pause": "",
"cpus": "4",
"custom_script": ".",
"desktop": "false",
"disk_size": "10000",
"headless": "", "hostname": "vagrant",
"memory": "4096",
"preseed": "preseed.cfg",
"ssh_fullname": "vagrant",
"ssh_password": "ubuntu",
"ssh_username": "ubuntu",
"vagrantfile_template": "",
"version": "0.1.0",
"vm_name": "base"
}
}
images/extended-image.pkr.hcl 0 → 100644
View file @ ea903dda
variable "cpus" {
type = string
default = "4"
}
variable "memory" {
type = string
default = "4096"
}
variable "outname" {
type = string
default = "extended"
}
variable "base_img" {
type = string
default = "base"
}
source "qemu" "autogenerated_1" {
communicator = "ssh"
cpus = "${var.cpus}"
disk_image = true
headless = true
iso_checksum = "none"
iso_url = "output-${var.base_img}/${var.base_img}"
memory = "${var.memory}"
net_device = "virtio-net"
output_directory = "output-${var.outname}"
qemuargs = [["-machine", "pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off"]]
shutdown_command = "sudo shutdown --poweroff --no-wall now"
ssh_password = "ubuntu"
ssh_username = "ubuntu"
use_backing_file = "true"
vm_name = "${var.outname}"
}
build {
sources = ["source.qemu.autogenerated_1"]
provisioner "file" {
direction = "upload"
source = "input-${var.outname}"
destination = "/tmp/input"
}
provisioner "shell" {
execute_command = "{{ .Vars }} sudo -S -E bash '{{ .Path }}'"
scripts = ["scripts/install-${var.outname}.sh", "scripts/cleanup.sh"]
}
}
images/mtcp.json deleted 100644 → 0
View file @ 9f5e95cd
{
"_comment": "Build with `packer build ubuntu.json`",
"builders": [
{
"iso_urls": [ "output-base/base" ],
"iso_checksum": "none",
"disk_image": "true",
"use_backing_file": "true",
"headless": "{{ user `headless` }}",
"disable_vnc": "false",
"output_directory": "output-{{ user `vm_name` }}",
"qemuargs": [
[
"-m",
"{{ user `memory` }}"
],
[
"-display",
"none"
],
[
"-machine",
"accel=kvm"
],
[
"-cpu",
"host"
],
[
"-smp",
"cpus={{ user `cpus`}}"
],
[
"-serial",
"mon:stdio"
]
],
"shutdown_command": "echo '{{ user `ssh_password` }}'|sudo -S shutdown -P now",
"ssh_password": "{{ user `ssh_password` }}",
"ssh_timeout": "10000s",
"ssh_username": "{{ user `ssh_username` }}",
"type": "qemu",
"vm_name": "{{ user `vm_name` }}"
}
],
"provisioners": [
{
"execute_command": "echo '{{ user `ssh_password` }}' | {{.Vars}} sudo -E -S bash '{{.Path}}'",
"scripts": [
"scripts/install-mtcp.sh"
],
"type": "shell"
}
],
"variables": {
"cleanup_pause": "",
"cpus": "4",
"custom_script": ".",
"desktop": "false",
"disk_size": "10000",
"headless": "", "hostname": "vagrant",
"memory": "4096",
"preseed": "preseed.cfg",
"ssh_fullname": "vagrant",
"ssh_password": "ubuntu",
"ssh_username": "ubuntu",
"vagrantfile_template": "",
"version": "0.1.0",
"vm_name": "mtcp"
}
}
images/nopaxos.json deleted 100644 → 0
View file @ 9f5e95cd
{
"_comment": "Build with `packer build ubuntu.json`",
"builders": [
{
"iso_urls": [ "output-base/base" ],
"iso_checksum": "none",
"disk_image": "true",
"use_backing_file": "true",
"headless": "{{ user `headless` }}",
"disable_vnc": "true",
"output_directory": "output-{{ user `vm_name` }}",
"qemuargs": [
[
"-m",
"{{ user `memory` }}"
],
[
"-display",
"none"
],
[
"-machine",
"accel=kvm"
],
[
"-cpu",
"host"
],
[
"-smp",
"cpus={{ user `cpus`}}"
]
],
"shutdown_command": "echo '{{ user `ssh_password` }}'|sudo -S shutdown -P now",
"ssh_password": "{{ user `ssh_password` }}",
"ssh_timeout": "10000s",
"ssh_username": "{{ user `ssh_username` }}",
"type": "qemu",
"vm_name": "{{ user `vm_name` }}"
}
],
"provisioners": [
{
"execute_command": "echo '{{ user `ssh_password` }}' | {{.Vars}} sudo -E -S bash '{{.Path}}'",
"scripts": [
"scripts/install-nopaxos.sh"
],
"type": "shell"
},
{
"type": "file",
"source": "nopaxos.config",
"destination": "/tmp/nopaxos.config",
"direction": "upload"
},
{
"type": "shell",
"inline": ["sudo mv /tmp/nopaxos.config /root/nopaxos.config"]
}
],
"variables": {
"cleanup_pause": "",
"cpus": "4",
"custom_script": ".",
"desktop": "false",
"disk_size": "10000",
"headless": "", "hostname": "vagrant",
"memory": "4096",
"preseed": "preseed.cfg",
"ssh_fullname": "vagrant",
"ssh_password": "ubuntu",
"ssh_username": "ubuntu",
"vagrantfile_template": "",
"version": "0.1.0",
"vm_name": "nopaxos"
}
}
images/packer-wrap.sh
View file @ ea903dda
......@@ -2,6 +2,32 @@
qemupath=`pwd`/../sims/external/qemu/
cores=`nproc`
if [ $cores -gt 32 ]
then
#limit cores to 32
cores=32
fi
mem=$(($cores * 512))
if [ $mem -lt 4096 ]
then
# at least 4G memory
mem=4096
fi
base_img=$1
outname=$2
pkrfile=$3
mkdir -p input-$outname
# add our qemu to $PATH
export PATH="$qemupath:$qemupath/build/:$PATH"
exec ./packer "$@"
./packer build \
-var "cpus=$cores" \
-var "memory=$mem" \
-var "base_img=$base_img" \
-var "outname=$outname" \
${pkrfile}
rm -rf input-$outname
images/rules.mk
View file @ ea903dda
......@@ -22,7 +22,7 @@
include mk/subdir_pre.mk
PACKER_VERSION := 1.6.0
PACKER_VERSION := 1.7.0
KERNEL_VERSION := 5.4.46
UBUNTU_IMAGE := $(d)output-ubuntu1804/ubuntu1804
......@@ -40,9 +40,13 @@ bz_image := $(d)bzImage
vmlinux := $(d)vmlinux
kernel_pardir := $(d)kernel
kernel_dir := $(kernel_pardir)/linux-$(KERNEL_VERSION)
kernel_config := $(kernel_pardir)/config-$(KERNEL_VERSION)
kheader_dir := $(d)kernel/kheaders
kheader_tar := $(d)kheaders.tar.bz2
mqnic_dir := $(d)mqnic
mqnic_mod := $(mqnic_dir)/mqnic.ko
m5_bin := $(d)m5
guest_init := $(d)/scripts/guestinit.sh
build-images: $(IMAGES) $(RAW_IMAGES) $(vmlinux) $(bz_image) $(mqnic_mod)
......@@ -55,35 +59,47 @@ build-images: $(IMAGES) $(RAW_IMAGES) $(vmlinux) $(bz_image) $(mqnic_mod)
$(UBUNTU_IMAGE): $(packer) $(QEMU) $(addprefix $(d),ubuntu1804.json \
scripts/user-data scripts/packages.sh scripts/cleanup.sh)
rm -rf $(dir $@)
cd $(img_dir) && ./packer-wrap.sh build ubuntu1804.json
cd $(img_dir) && ./packer-wrap.sh none ubuntu1804 ubuntu1804.json
touch $@
$(BASE_IMAGE): $(packer) $(QEMU) $(d)base.json $(UBUNTU_IMAGE) $(bz_image) \
$(d)kheaders.tar.bz2 $(d)scripts/guestinit.sh
$(BASE_IMAGE): $(packer) $(QEMU) $(UBUNTU_IMAGE) $(bz_image) $(m5_bin) \
$(kheader_tar) $(guest_init) $(kernel_config) \
$(addprefix $(d), extended-image.pkr.hcl scripts/install-base.sh \
scripts/cleanup.sh)
rm -rf $(dir $@)
cd $(img_dir) && ./packer-wrap.sh build base.json
mkdir -p $(img_dir)/input-base
cp $(m5_bin) $(kheader_tar) $(guest_init) $(bz_image) $(kernel_config) \
$(img_dir)/input-base/
cd $(img_dir) && ./packer-wrap.sh ubuntu1804 base extended-image.pkr.hcl
rm -rf $(img_dir)/input-base
touch $@
$(NOPAXOS_IMAGE): $(packer) $(QEMU) $(d)nopaxos.json $(BASE_IMAGE) \
$(addprefix $(d), scripts/install-nopaxos.sh nopaxos.config)
$(NOPAXOS_IMAGE): $(packer) $(QEMU) $(BASE_IMAGE) \
$(addprefix $(d), extended-image.pkr.hcl scripts/install-nopaxos.sh \
scripts/cleanup.sh nopaxos.config)
rm -rf $(dir $@)
cd $(img_dir) && ./packer-wrap.sh build nopaxos.json
mkdir -p $(img_dir)/input-nopaxos
cp $(img_dir)/nopaxos.config $(img_dir)/input-nopaxos/
cd $(img_dir) && ./packer-wrap.sh base nopaxos extended-image.pkr.hcl
touch $@
$(MTCP_IMAGE): $(packer) $(QEMU) $(d)mtcp.json $(BASE_IMAGE) \
$(d)scripts/install-mtcp.sh
$(MTCP_IMAGE): $(packer) $(QEMU) $(BASE_IMAGE) \
$(addprefix $(d), extended-image.pkr.hcl scripts/install-mtcp.sh \
scripts/cleanup.sh)
rm -rf $(dir $@)
cd $(img_dir) && ./packer-wrap.sh build mtcp.json
cd $(img_dir) && ./packer-wrap.sh base mtcp extended-image.pkr.hcl
touch $@
$(TAS_IMAGE): $(packer) $(QEMU) $(d)tas.json $(BASE_IMAGE) \
$(d)scripts/install-tas.sh
$(TAS_IMAGE): $(packer) $(QEMU) $(BASE_IMAGE) \
$(addprefix $(d), extended-image.pkr.hcl scripts/install-tas.sh \
scripts/cleanup.sh)
rm -rf $(dir $@)
cd $(img_dir) && ./packer-wrap.sh build tas.json
cd $(img_dir) && ./packer-wrap.sh base tas extended-image.pkr.hcl
touch $@
$(packer):
wget -O $(img_dir)packer_$(PACKER_VERSION)_linux_amd64.zip https://releases.hashicorp.com/packer/$(PACKER_VERSION)/packer_$(PACKER_VERSION)_linux_amd64.zip
wget -O $(img_dir)packer_$(PACKER_VERSION)_linux_amd64.zip \
https://releases.hashicorp.com/packer/$(PACKER_VERSION)/packer_$(PACKER_VERSION)_linux_amd64.zip
cd $(img_dir) && unzip packer_$(PACKER_VERSION)_linux_amd64.zip
rm -f $(img_dir)packer_$(PACKER_VERSION)_linux_amd64.zip
......@@ -104,7 +120,7 @@ $(bz_image): $(kernel_dir)/vmlinux
cp $(kernel_dir)/arch/x86_64/boot/bzImage $@
touch $@
$(d)kheaders.tar.bz2: $(kernel_dir)/vmlinux
$(kheader_tar): $(kernel_dir)/vmlinux
rm -rf $(kheader_dir)
mkdir -p $(kheader_dir)
$(MAKE) -C $(kernel_dir) headers_install INSTALL_HDR_PATH=$(abspath $(kheader_dir)/usr)
......@@ -148,7 +164,7 @@ CLEAN := $(addprefix $(d), mqnic/mqnic.ko mqnic/*.o mqnic/.*.cmd mqnic/*.mod \
mqnic/mqnic.mod.c mqnic/Module.symvers mqnic/modules.order)
DISTCLEAN := $(kernel_dir) $(packer) $(bz_image) $(vmlinux) $(kheader_dir) \
$(foreach i,$(IMAGES),$(dir $(i))) \
$(foreach i,$(IMAGES),$(dir $(i)) $(subst output-,input-,$(dir $(i)))) \
$(d)packer_cache $(d)kheaders.tar.bz2
include mk/subdir_post.mk
images/scripts/install-base.sh
View file @ ea903dda
......@@ -2,6 +2,16 @@
set -eux
pushd /tmp/input
mv guestinit.sh /home/ubuntu/guestinit.sh
mv bzImage /boot/vmlinuz-5.4.46
mv config-5.4.46 /boot/
mv m5 /sbin/m5
update-grub
tar xf kheaders.tar.bz2 -C /
popd
rm -rf /tmp/input
apt-get -y install memcached libevent-dev
systemctl disable memcached.service
......
images/scripts/install-nopaxos.sh
View file @ ea903dda
......@@ -12,3 +12,5 @@ mkdir -p /root
git clone https://github.com/nicklijl/simbricks-nopaxos.git /root/nopaxos
cd /root/nopaxos
make -j4
mv /tmp/input/nopaxos.config /root/nopaxos.config
images/tas.json deleted 100644 → 0
View file @ 9f5e95cd
{
"_comment": "Build with `packer build ubuntu.json`",
"builders": [
{
"iso_urls": [ "output-base/base" ],
"iso_checksum": "none",
"disk_image": "true",
"use_backing_file": "true",
"headless": "{{ user `headless` }}",
"disable_vnc": "false",
"output_directory": "output-{{ user `vm_name` }}",
"qemuargs": [
[
"-m",
"{{ user `memory` }}"
],
[
"-display",
"none"
],
[
"-machine",
"accel=kvm"
],
[
"-cpu",
"host"
],
[
"-smp",
"cpus={{ user `cpus`}}"
],
[
"-serial",
"mon:stdio"
]
],
"shutdown_command": "echo '{{ user `ssh_password` }}'|sudo -S shutdown -P now",
"ssh_password": "{{ user `ssh_password` }}",
"ssh_timeout": "10000s",
"ssh_username": "{{ user `ssh_username` }}",
"type": "qemu",
"vm_name": "{{ user `vm_name` }}"
}
],
"provisioners": [
{
"execute_command": "echo '{{ user `ssh_password` }}' | {{.Vars}} sudo -E -S bash '{{.Path}}'",
"scripts": [
"scripts/install-tas.sh"
],
"type": "shell"
}
],
"variables": {
"cleanup_pause": "",
"cpus": "4",
"custom_script": ".",
"desktop": "false",
"disk_size": "10000",
"headless": "", "hostname": "vagrant",
"memory": "4096",
"preseed": "preseed.cfg",
"ssh_fullname": "vagrant",
"ssh_password": "ubuntu",
"ssh_username": "ubuntu",
"vagrantfile_template": "",
"version": "0.1.0",
"vm_name": "tas"
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment