Skip to content

GitLab

"...text-generation-inference.git" did not exist on "90a1d04a2f560df25a2786fcc1f117a05650dd7d"
Commit 9f5e95cd authored by Antoine Kaufmann's avatar Antoine Kaufmann
Browse files

images: build ubuntu1804 using cloud image 

This is a lot faster than manually installing from ISO.
parent 92756010
Hide whitespace changes
Inline Side-by-side
Showing with 65 additions and 316 deletions
images/rules.mk
View file @ 9f5e95cd
...@@ -53,8 +53,7 @@ build-images: $(IMAGES) $(RAW_IMAGES) $(vmlinux) $(bz_image) $(mqnic_mod) ...@@ -53,8 +53,7 @@ build-images: $(IMAGES) $(RAW_IMAGES) $(vmlinux) $(bz_image) $(mqnic_mod)
$(QEMU_IMG) convert -f qcow2 -O raw $< $@ $(QEMU_IMG) convert -f qcow2 -O raw $< $@
$(UBUNTU_IMAGE): $(packer) $(QEMU) $(addprefix $(d),ubuntu1804.json \ $(UBUNTU_IMAGE): $(packer) $(QEMU) $(addprefix $(d),ubuntu1804.json \
scripts/vagrant.sh scripts/sshd.sh scripts/update.sh scripts/packages.sh \ scripts/user-data scripts/packages.sh scripts/cleanup.sh)
scripts/cleanup.sh scripts/preseed.cfg)
rm -rf $(dir $@) rm -rf $(dir $@)
cd $(img_dir) && ./packer-wrap.sh build ubuntu1804.json cd $(img_dir) && ./packer-wrap.sh build ubuntu1804.json
touch $@ touch $@
......
images/scripts/cleanup.sh
View file @ 9f5e95cd
#!/bin/bash -eux #!/bin/bash -eux
SSH_USER=${SSH_USERNAME:-vagrant}
# Make sure udev does not block our network - http://6.ptmc.org/?p=164 # Make sure udev does not block our network - http://6.ptmc.org/?p=164
echo "==> Cleaning up udev rules" echo "==> Cleaning up udev rules"
rm -rf /dev/.udev/ rm -rf /dev/.udev/
...@@ -17,20 +15,6 @@ if [ -d "/var/lib/dhcp" ]; then ...@@ -17,20 +15,6 @@ if [ -d "/var/lib/dhcp" ]; then
rm /var/lib/dhcp/* rm /var/lib/dhcp/*
fi fi
UBUNTU_VERSION=$(lsb_release -sr)
if [[ ${UBUNTU_VERSION} == 16.04 ]] || [[ ${UBUNTU_VERSION} == 16.10 ]]; then
# Modified version of
# https://github.com/cbednarski/packer-ubuntu/blob/master/scripts-1604/vm_cleanup.sh#L9-L15
# Instead of eth0 the interface is now called ens5 to mach the PCI
# slot, so we need to change the networking scripts to enable the
# correct interface.
#
# NOTE: After the machine is rebooted Packer will not be able to reconnect
# (Vagrant will be able to) so make sure this is done in your final
# provisioner.
sed -i "s/ens3/ens5/g" /etc/network/interfaces
fi
# Add delay to prevent "vagrant reload" from failing # Add delay to prevent "vagrant reload" from failing
echo "pre-up sleep 2" >> /etc/network/interfaces echo "pre-up sleep 2" >> /etc/network/interfaces
...@@ -50,7 +34,7 @@ DISK_USAGE_BEFORE_CLEANUP=$(df -h) ...@@ -50,7 +34,7 @@ DISK_USAGE_BEFORE_CLEANUP=$(df -h)
# Remove Bash history # Remove Bash history
unset HISTFILE unset HISTFILE
rm -f /root/.bash_history rm -f /root/.bash_history
rm -f /home/${SSH_USER}/.bash_history rm -f /home/ubuntu/.bash_history
# Clean up log files # Clean up log files
find /var/log -type f | while read f; do echo -ne '' > "${f}"; done; find /var/log -type f | while read f; do echo -ne '' > "${f}"; done;
...@@ -59,49 +43,3 @@ echo "==> Clearing last login information" ...@@ -59,49 +43,3 @@ echo "==> Clearing last login information"
>/var/log/lastlog >/var/log/lastlog
>/var/log/wtmp >/var/log/wtmp
>/var/log/btmp >/var/log/btmp
# NOTE: Shrinking is not part of the build process
# so this will only grow the image...
# # Whiteout root
# count=$(df --sync -kP / | tail -n1 | awk -F ' ' '{print $4}')
# let count--
# dd if=/dev/zero of=/tmp/whitespace bs=1024 count=$count
# rm /tmp/whitespace
# # Whiteout /boot
# count=$(df --sync -kP /boot | tail -n1 | awk -F ' ' '{print $4}')
# let count--
# dd if=/dev/zero of=/boot/whitespace bs=1024 count=$count
# rm /boot/whitespace
# echo '==> Clear out swap and disable until reboot'
# set +e
# swapuuid=$(/sbin/blkid -o value -l -s UUID -t TYPE=swap)
# case "$?" in
# 2|0) ;;
# *) exit 1 ;;
# esac
# set -e
# if [ "x${swapuuid}" != "x" ]; then
# # Whiteout the swap partition to reduce box size
# # Swap is disabled till reboot
# swappart=$(readlink -f /dev/disk/by-uuid/$swapuuid)
# /sbin/swapoff "${swappart}"
# dd if=/dev/zero of="${swappart}" bs=1M || echo "dd exit code $? is suppressed"
# /sbin/mkswap -U "${swapuuid}" "${swappart}"
# fi
# # Zero out the free space to save space in the final image
# dd if=/dev/zero of=/EMPTY bs=1M || echo "dd exit code $? is suppressed"
# rm -f /EMPTY
# # Make sure we wait until all the data is written to disk, otherwise
# # Packer might quite too early before the large files are deleted
# sync
# echo "==> Disk usage before cleanup"
# echo ${DISK_USAGE_BEFORE_CLEANUP}
# echo "==> Disk usage after cleanup"
# df -h
images/scripts/network.sh deleted 100755 → 0
View file @ 92756010
#!/bin/sh
# Disable udev persistent net rules
rm /etc/udev/rules.d/70-persistent-net.rules
mkdir /etc/udev/rules.d/70-persistent-net.rules
rm /lib/udev/rules.d/75-persistent-net-generator.rules
rm -rf /dev/.udev/ /var/lib/dhcp3/*
echo "pre-up sleep 2" >> /etc/network/interfaces
# Disable DNS reverse lookup
echo "UseDNS no" >> /etc/ssh/sshd_config
images/scripts/packages.sh
View file @ 9f5e95cd
DEV_PACKAGES=" #!/bin/bash
build-essential apt-get update
curl mount 1>&2
emacs24-nox apt-get -y install \
htop ntp \
nmon nfs-common \
slurm iperf \
tcpdump netperf \
unzip netcat \
" make \
git \
ESSENTIAL_PACKAGES=" pkg-config \
ntp libevent-dev \
nfs-common libunwind-dev \
iperf autoconf \
netperf automake \
netcat libtool \
make g++ \
git libssl-dev \
pkg-config ethtool \
libevent-dev
libunwind-dev
autoconf
automake
libtool
g++
libssl-dev
ethtool
"
if [[ $INSTALL_DEV_PACKAGES =~ true || $INSTALL_DEV_PACKAGES =~ 1 ||
$INSTALL_DEV_PACKAGES =~ yes ]]; then
apt-get -y install $DEV_PACKAGES
fi
apt-get -y install $ESSENTIAL_PACKAGES
images/scripts/preseed.cfg deleted 100644 → 0
View file @ 92756010
choose-mirror-bin mirror/http/proxy string
d-i debian-installer/framebuffer boolean false
d-i debconf/frontend select noninteractive
d-i base-installer/kernel/override-image string linux-server
d-i clock-setup/utc boolean true
d-i clock-setup/utc-auto boolean true
d-i finish-install/reboot_in_progress note
d-i grub-installer/only_debian boolean true
d-i grub-installer/with_other_os boolean true
d-i partman-auto/method string regular
d-i partman/choose_partition select finish
d-i partman/confirm boolean true
d-i partman/confirm_nooverwrite boolean true
d-i partman/confirm_write_new_label boolean true
d-i pkgsel/include string openssh-server
d-i pkgsel/install-language-support boolean false
d-i pkgsel/update-policy select none
d-i pkgsel/upgrade select full-upgrade
d-i time/zone string UTC
d-i user-setup/allow-password-weak boolean true
d-i user-setup/encrypt-home boolean false
tasksel tasksel/first multiselect standard, ubuntu-server
images/scripts/sshd.sh deleted 100755 → 0
View file @ 92756010
#!/bin/bash -eux
echo "UseDNS no" >> /etc/ssh/sshd_config
images/scripts/update.sh deleted 100755 → 0
View file @ 92756010
#!/bin/bash -eux
# Disable the release upgrader
echo "==> Disabling the release upgrader"
sed -i.bak 's/^Prompt=.*$/Prompt=never/' /etc/update-manager/release-upgrades
echo "==> Checking version of Ubuntu"
. /etc/lsb-release
if [[ $DISTRIB_RELEASE == 16.04 || $DISTRIB_RELEASE == 18.04 ]]; then
echo "==> Disabling periodic apt upgrades"
echo 'APT::Periodic::Enable "0";' >> /etc/apt/apt.conf.d/10periodic
fi
echo "==> Updating list of repositories"
# apt-get update does not actually perform updates, it just downloads and indexes the list of packages
apt-get -y update
if [[ $UPDATE =~ true || $UPDATE =~ 1 || $UPDATE =~ yes ]]; then
echo "==> Performing dist-upgrade (all packages and kernel)"
apt-get -y dist-upgrade --force-yes
reboot
sleep 60
fi
images/scripts/user-data 0 → 100644
View file @ 9f5e95cd
#cloud-config
ssh_pwauth: True
users:
- name: ubuntu
plain_text_passwd: ubuntu
sudo: ALL=(ALL) NOPASSWD:ALL
shell: /bin/bash
lock_passwd: false
images/scripts/vagrant.sh deleted 100755 → 0
View file @ 92756010
#!/bin/bash
date > /etc/vagrant_box_build_time
SSH_USER=${SSH_USERNAME:-vagrant}
SSH_PASS=${SSH_PASSWORD:-vagrant}
SSH_USER_HOME=${SSH_USER_HOME:-/home/${SSH_USER}}
VAGRANT_INSECURE_KEY="ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"
# Packer passes boolean user variables through as '1', but this might change in
# the future, so also check for 'true'.
if [ "$INSTALL_VAGRANT_KEY" = "true" ] || [ "$INSTALL_VAGRANT_KEY" = "1" ]; then
# Create Vagrant user (if not already present)
if ! id -u $SSH_USER >/dev/null 2>&1; then
echo "==> Creating $SSH_USER user"
/usr/sbin/groupadd $SSH_USER
/usr/sbin/useradd $SSH_USER -g $SSH_USER -G sudo -d $SSH_USER_HOME --create-home
echo "${SSH_USER}:${SSH_PASS}" | chpasswd
fi
# Set up sudo
echo "==> Giving ${SSH_USER} sudo powers"
echo "${SSH_USER} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/vagrant
chmod 440 /etc/sudoers.d/vagrant
# Fix stdin not being a tty
if grep -q -E "^mesg n$" /root/.profile && sed -i "s/^mesg n$/tty -s \\&\\& mesg n/g" /root/.profile; then
echo "==> Fixed stdin not being a tty."
fi
echo "==> Installing vagrant key"
mkdir $SSH_USER_HOME/.ssh
chmod 700 $SSH_USER_HOME/.ssh
cd $SSH_USER_HOME/.ssh
# https://raw.githubusercontent.com/mitchellh/vagrant/master/keys/vagrant.pub
echo "${VAGRANT_INSECURE_KEY}" > $SSH_USER_HOME/.ssh/authorized_keys
chmod 600 $SSH_USER_HOME/.ssh/authorized_keys
chown -R $SSH_USER:$SSH_USER $SSH_USER_HOME/.ssh
fi
images/ubuntu1804.json
View file @ 9f5e95cd
{ {
"_comment": "Build with `packer build ubuntu.json`", "builders": [
"builders": [ {
{ "type": "qemu",
"boot_command": [ "communicator": "ssh",
"{{ user `boot_command_prefix` }}", "headless": true,
"/install/vmlinuz noapic ", "disk_image": true,
"file=/floppy/{{ user `preseed` }} ", "cpus": "{{ user `cpus` }}",
"debian-installer={{ user `locale` }} auto locale={{ user `locale` }} kbd-chooser/method=us ", "memory": "{{ user `memory` }}",
"hostname={{ user `hostname` }} ", "iso_checksum": "file:https://cloud-images.ubuntu.com/bionic/current/SHA256SUMS",
"fb=false debconf/frontend=noninteractive ", "iso_url": "https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img",
"keyboard-configuration/modelcode=SKIP ", "qemuargs": [
"keyboard-configuration/layout=USA ", ["-smbios", "type=1,serial=ds=nocloud-net;instance-id=packer;seedfrom=http://{{ .HTTPIP }}:{{ .HTTPPort }}/"],
"keyboard-configuration/variant=USA console-setup/ask_detect=false ", ["-machine","pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off"]
"passwd/user-fullname={{ user `ssh_fullname` }} ", ],
"passwd/user-password={{ user `ssh_password` }} ", "http_directory": "scripts",
"passwd/user-password-again={{ user `ssh_password` }} ", "ssh_username": "ubuntu",
"passwd/username={{ user `ssh_username` }} ", "ssh_password": "ubuntu",
"initrd=/install/initrd.gz -- <enter>" "net_device": "virtio-net",
], "shutdown_command": "sudo shutdown --poweroff --no-wall now",
"disk_size": "{{ user `disk_size` }}", "output_directory": "output-ubuntu1804",
"floppy_files": [ "vm_name": "ubuntu1804"
"scripts/{{ user `preseed` }}" }
], ],
"headless": "{{ user `headless` }}", "provisioners": [
"http_directory": "http", {
"iso_checksum": "{{ user `iso_checksum_type` }}:{{ user `iso_checksum` }}", "type": "shell",
"iso_urls": [ "execute_command": "{{.Vars}} sudo -S -E bash '{{.Path}}'",
"{{ user `iso_url` }}" "scripts": [
], "scripts/packages.sh",
"output_directory": "output-{{ user `vm_name` }}", "scripts/cleanup.sh"
"qemuargs": [
[
"-m",
"{{ user `memory` }}"
],
[
"-display",
"none"
],
[
"-machine",
"accel=kvm"
],
[
"-cpu",
"host"
],
[
"-smp",
"cpus={{ user `cpus`}}"
] ]
], }
"shutdown_command": "echo '{{ user `ssh_password` }}'|sudo -S shutdown -P now", ],
"ssh_password": "{{ user `ssh_password` }}", "variables": {
"ssh_timeout": "10000s", "cpus": "4",
"ssh_username": "{{ user `ssh_username` }}", "memory": "4096"
"type": "qemu",
"vm_name": "{{ user `vm_name` }}"
} }
],
"provisioners": [
{
"environment_vars": [
"UPDATE={{user `update`}}",
"INSTALL_DEV_PACKAGES={{user `install_dev_packages`}}",
"INSTALL_VAGRANT_KEY={{user `install_vagrant_key`}}",
"SSH_USERNAME={{user `ssh_username`}}",
"SSH_PASSWORD={{user `ssh_password`}}",
"http_proxy={{user `http_proxy`}}",
"https_proxy={{user `https_proxy`}}",
"ftp_proxy={{user `ftp_proxy`}}",
"rsync_proxy={{user `rsync_proxy`}}",
"no_proxy={{user `no_proxy`}}"
],
"execute_command": "echo '{{ user `ssh_password` }}' | {{.Vars}} sudo -E -S bash '{{.Path}}'",
"scripts": [
"scripts/vagrant.sh",
"scripts/sshd.sh",
"scripts/update.sh",
"scripts/packages.sh",
"scripts/cleanup.sh"
],
"type": "shell"
}
],
"variables": {
"boot_command_prefix": "<esc><esc><enter><wait>",
"cleanup_pause": "",
"cpus": "4",
"custom_script": ".",
"desktop": "false",
"disk_size": "10000",
"ftp_proxy": "{{env `ftp_proxy`}}",
"headless": "",
"hostname": "vagrant",
"http_proxy": "{{env `http_proxy`}}",
"https_proxy": "{{env `https_proxy`}}",
"install_dev_packages": "false",
"install_vagrant_key": "true",
"iso_checksum": "f5cbb8104348f0097a8e513b10173a07dbc6684595e331cb06f93f385d0aecf6",
"iso_checksum_type": "sha256",
"iso_name": "ubuntu-18.04.6-server-amd64.iso",
"iso_url": "http://cdimage.ubuntu.com/ubuntu/releases/18.04.5/release/ubuntu-18.04.6-server-amd64.iso",
"locale": "en_US",
"memory": "4096",
"no_proxy": "{{env `no_proxy`}}",
"preseed": "preseed.cfg",
"rsync_proxy": "{{env `rsync_proxy`}}",
"ssh_fullname": "vagrant",
"ssh_password": "ubuntu",
"ssh_username": "ubuntu",
"update": "false",
"vagrantfile_template": "",
"version": "0.1.0",
"vm_name": "ubuntu1804"
}
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment