Makefile MicroVPS LINUX CONTAINERS FROM SCRATCH Joshua Hoffman
Makefile MicroVPSLINUX CONTAINERS FROM SCRATCH
Joshua Hoffman
ABOUTLINUX CONTAINERS FROM SCRATCH
DO NOT EXIST
CONCEPT (NOT A THING)
LINUX DISTRO
SANDWICH
LINUX CONTAINERS FROM SCRATCH
POPULAR SANDWICH INGREDIENTS
▸ tomatoes
▸ cucumber
▸ bread
▸ toothpicks
LINUX CONTAINERS FROM SCRATCH
POPULAR CONTAINER INGREDIENTS
▸ kernel namespaces
▸ cgroups
▸ build automation
▸ portable archive
single process full os
?
MICROVPS
LINUX CONTAINERS FROM SCRATCH
MICROVPS REQUIREMENTS
▸ minimal runtime
▸ dedicated network namespace
▸ native package management
▸ automated build
▸ fast iteration cycle
▸ simple deployment/management
STOP!
WHAT PROBLEM ARE YOU TRYING TO SOLVE?
Abraham Lincoln
LEARNING LAB
LINUX CONTAINERS FROM SCRATCH
LAB REQUIREMENTS
▸ 20-50 Virtual Servers
▸ Single Physical Server
▸ Easy Setup and Teardown
LINUX CONTAINERS FROM SCRATCH
LAB VIRTUAL SERVER REQUIREMENTS
▸ dedicated ip
▸ http server
▸ ssh root access
TOOLS
LINUX CONTAINERS FROM SCRATCH
PHILOSOPHY OF RELIABLE SYSTEMS
▸ standard > disruptive
▸ battle tested > new
▸ simple > complex
▸ modular > monolithic
▸ built-in > add-on
LINUX CONTAINERS FROM SCRATCH
CONTAINER BUILDING TOOLS
▸ make
▸ yum
▸ systemd
▸ iproute2
▸ rsync
▸ bridge-utils
SETUP CONTAINER BUILD TOOLING
DEMO
LINUX CONTAINERS FROM SCRATCH
SETUP DEVELOPMENT SYSTEM
▸ Install packages yum -y install bridge-utils rsync iptables-services
▸ Mount the CentOS 7 iso mkdir /mnt/cdrom mount -oloop,ro CentOS-7-x86_64-DVD-1503-01.iso /mnt/cdrom
LINUX CONTAINERS FROM SCRATCH
SETUP DEVELOPMENT SYSTEM
▸ Disable firewalld systemctl stop firewalld systemctl disable firewalld
▸ Disable selinux setenforce 0 sed -ie 's/=enforcing/=permissive/' /etc/sysconfig/selinux
LINUX CONTAINERS FROM SCRATCH
SETUP CONTAINER NETWORKING
▸ Create the file /etc/sysconfig/network-scripts/ifcfg-mvpsbr0 NAME=mvpsbr0 IPADDR=10.100.10.1 NETMASK=255.255.255.0 TYPE=Bridge BOOTPROTO=none DEVICE=mvpsbr0 NM_MANAGED=no ONBOOT=yes
LINUX CONTAINERS FROM SCRATCH
SETUP CONTAINER NETWORKING
▸ Activate the new ethernet bridge ifup mvpsbr0
▸ Verify the configuration ip addr show mvpsbr0
LINUX CONTAINERS FROM SCRATCH
SETUP CONTAINER NETWORKING
▸ Enable IP routing echo “net.ipv4.ip_forward = 1” > /etc/sysctl.d/lcfs.conf sysctl -p /etc/sysctl.d/lcfs.conf
▸ Setup IP masquerading for container network iptables -t nat -A POSTROUTING -s 10.100.10.0/24 -j MASQUERADE iptables-save > /etc/sysconfig/iptables systemctl enable iptables
LINUX CONTAINERS FROM SCRATCH
SETUP DEVELOPMENT SYSTEM
▸ Edit /etc/sysconfig/grub GRUB_CMDLINE_LINUX=“(…truncated…) crashkernel=auto rhgb quiet audit=0”
▸ Rebuild grub configuration grub2-mkconfig -o /boot/grub2/grub.cfg
▸ Reboot
LINUX CONTAINERS FROM SCRATCH
SETUP YUM FOR CONTAINER BUILDING
▸ Create a yum.conf [main] assumeyes=1 keepcache=0 tsflags=nodocs gpgcheck=1 plugins=0 distroverpkg=centos-release reposdir=/dev/null
[cdrom] name=CentOS-7 - Base baseurl=file:///mnt/cdrom gpgcheck=1 gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
LINUX CONTAINERS FROM SCRATCH
CREATE AN EMPTY CONTAINER PROJECT
▸ Make a directory mkdir container1
▸ Make an “fstree” sub-directory mkdir container1/fstree
▸ Add a makefile touch container1/Makefile
LINUX CONTAINERS FROM SCRATCH
PROJECT LAYOUT
▸ project layout microvps/ container1/ fstree/ Makefile container2/ fstree/ Makefile yum.conf
EXPERIMENT #1 CENTOS ‘MINIMAL INSTALL’ + APACHE
DEMO
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
NAME := web1 PACKAGES := '@^Minimal Install' httpd IP_ADDR := 10.100.10.21/24 GATEWAY := 10.100.10.1 ROOTFS := rootfs YUM_CONF := ../yum.conf CENTOS_VER := 7 FSTREE := fstree
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
container: mkdir -vp $(ROOTFS) # install packages yum --config=$(YUM_CONF) \ --installroot=$(abspath $(ROOTFS)) \ --releasever=$(CENTOS_VER) \ install $(PACKAGES) # clean up metadata yum --config=$(YUM_CONF) \ --installroot=$(abspath $(ROOTFS)) \ --releasever=$(CENTOS_VER) \ clean all # install custom files rsync -av $(FSTREE)/ $(ROOTFS)
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
test: # add a network namespace ip netns add $(NAME) # add a linked virtual network device pair ip link add mvps-$(NAME) type veth peer name xmvps-$(NAME) # move one into the namespace ip link set xmvps-$(NAME) netns $(NAME) # add the other to the bridge brctl addif $(BRIDGE) mvps-$(NAME) ip link set mvps-$(NAME) up # rename it ip netns exec $(NAME) ip link set xmvps-$(NAME) name eth0 # configure it ip netns exec $(NAME) ip link set eth0 up ip netns exec $(NAME) ip addr add $(IP_ADDR) dev eth0 ip netns exec $(NAME) ip route add default via $(GATEWAY) # launch it ip netns exec $(NAME) systemd-nspawn -M $(NAME) -D $(ROOTFS) -b || true # remove network namespace ip netns del $(NAME)
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
clean: rm -rf $(ROOTFS)
LINUX CONTAINERS FROM SCRATCH
POPULATE THE FSTREE
fstree/etc/passwd fstree/etc/shadow fstree/etc/group fstree/etc/systemd/system/multi-user.target.wants/httpd.service fstree/var/www/html/index.html
EXPERIMENT #2 REDUCE CENTOS RUNTIME
DEMO
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
container: (…truncated…) # remove systemd links rm -vf $(ROOTFS)/etc/systemd/system/*.wants/* rm -vf $(ROOTFS)/lib/systemd/system/basic.target.wants/* rm -vf $(ROOTFS)/lib/systemd/system/sysinit.target.wants/* rm -vf $(ROOTFS)/lib/systemd/system/sockets.target.wants/*udev* rm -vf $(ROOTFS)/lib/systemd/system/sockets.target.wants/*initctl* rm -vf $(ROOTFS)/lib/systemd/system/local-fs.target.wants/* rm -vf $(ROOTFS)/lib/systemd/system/anaconda.target.wants/* rm -vf $(ROOTFS)/lib/systemd/system/multi-user.target.wants/* rm -vf $(ROOTFS)/etc/systemd/system/default.target # install custom files rsync -av $(FSTREE)/ $(ROOTFS)
LINUX CONTAINERS FROM SCRATCH
POPULATE THE FSTREE
fstree/etc/passwd fstree/etc/shadow fstree/etc/group fstree/etc/systemd/system/multi-user.target.wants/httpd.service fstree/var/www/html/index.html fstree/etc/systemd/system/default.target fstree/etc/systemd/system/httpd.service fstree/etc/systemd/system/multi-user.target.wants/sshd.service fstree/lib/systemd/system/sysinit.target.wants/systemd-tmpfiles-setup.service fstree/lib/systemd/system/sysinit.target.wants/systemd-update-utmp.service
LINUX CONTAINERS FROM SCRATCH
UPDATE THE HTTPD SERVICE FILE
[Unit] Description=The Apache HTTP Server After=network.target remote-fs.target nss-lookup.target Wants=systemd-tmpfiles-setup.service
(…truncated…)
EXPERIMENT #3 DEPLOY, MANAGE WITH SYSTEMD
DEMO
LINUX CONTAINERS FROM SCRATCH
SETUP RUNTIME SYSTEM
▸ Create a directory where containers will be installed mkdir /home/microvps
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
install: $(NAME).conf $(NAME).service mkdir $(INSTALL_PATH)/$(NAME) cp -a $(ROOTFS) $(INSTALL_PATH)/$(NAME)/ cp $(NAME).conf $(INSTALL_PATH)/$(NAME)/ cp $(NAME).service $(INSTALL_PATH)/$(NAME)/ ln -s $(INSTALL_PATH)/$(NAME)/$(NAME).service \ /etc/systemd/system/$(NAME).service
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
$(NAME).conf: printf 'NAME=%s\n' $(NAME) > $@ printf 'ROOTFS=%s\n' "$(INSTALL_PATH)/$(NAME)/$(ROOTFS)" >> $@ printf 'BRIDGE=%s\n' $(BRIDGE) >> $@ printf 'IP_ADDR=%s\n' $(IP_ADDR) >> $@ printf 'GATEWAY=%s\n' $(GATEWAY) >> $@
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
$(NAME).service: systemd.service.in sed -e 's;EnvironmentFile=;EnvironmentFile=$(INSTALL_PATH)/$(NAME)/$(NAME).conf;' \ < systemd.service.in \ > $(NAME).service
LINUX CONTAINERS FROM SCRATCH
CONFIGURE ENVIRONMENT FOR SYSTEMD UNIT
▸ MicroVPS config file NAME=web3 ROOTFS=/home/microvps/web3/rootfs BRIDGE=mvpsbr0 IP_ADDR=10.100.10.23/24 GATEWAY=10.100.10.1
LINUX CONTAINERS FROM SCRATCH
SYSTEMD UNIT FILE
[Unit] Description=MicroVPS Container Server After=network.target
[Service] EnvironmentFile= ExecStartPre=/usr/sbin/ip netns add ${NAME} ExecStartPre=/usr/sbin/ip link add mvps-${NAME} type veth peer name xmvps-${NAME} ExecStartPre=/usr/sbin/ip link set xmvps-${NAME} netns ${NAME} ExecStartPre=/usr/sbin/brctl addif ${BRIDGE} mvps-${NAME} ExecStartPre=/usr/sbin/ip link set mvps-${NAME} up ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip link set xmvps-${NAME} name eth0 ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip link set eth0 up ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip addr add ${IP_ADDR} dev eth0 ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip route add default via ${GATEWAY} ExecStart=/usr/sbin/ip netns exec ${NAME} /usr/bin/systemd-nspawn -M ${NAME} -D ${ROOTFS} -b ExecStopPost=/usr/sbin/ip netns del ${NAME} KillMode=process
LINUX CONTAINERS FROM SCRATCH
SYSTEMD UNIT FILE
[Unit] Description=MicroVPS Container Server After=network.target
[Service] EnvironmentFile= ExecStartPre=/usr/sbin/ip netns add ${NAME} ExecStartPre=/usr/sbin/ip link add mvps-${NAME} type veth peer name xmvps-${NAME} ExecStartPre=/usr/sbin/ip link set xmvps-${NAME} netns ${NAME} ExecStartPre=/usr/sbin/brctl addif ${BRIDGE} mvps-${NAME} ExecStartPre=/usr/sbin/ip link set mvps-${NAME} up ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip link set xmvps-${NAME} name eth0 ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip link set eth0 up ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip addr add ${IP_ADDR} dev eth0 ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip route add default via ${GATEWAY} ExecStart=/usr/sbin/ip netns exec ${NAME} /usr/bin/systemd-nspawn -M ${NAME} -D ${ROOTFS} -b ExecStopPost=/usr/sbin/ip netns del ${NAME} KillMode=process
LINUX CONTAINERS FROM SCRATCH
SYSTEMD UNIT FILE
[Unit] Description=MicroVPS Container Server After=network.target
[Service] EnvironmentFile= ExecStartPre=/usr/sbin/ip netns add ${NAME} ExecStartPre=/usr/sbin/ip link add mvps-${NAME} type veth peer name xmvps-${NAME} ExecStartPre=/usr/sbin/ip link set xmvps-${NAME} netns ${NAME} ExecStartPre=/usr/sbin/brctl addif ${BRIDGE} mvps-${NAME} ExecStartPre=/usr/sbin/ip link set mvps-${NAME} up ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip link set xmvps-${NAME} name eth0 ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip link set eth0 up ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip addr add ${IP_ADDR} dev eth0 ExecStartPre=/usr/sbin/ip netns exec ${NAME} /usr/sbin/ip route add default via ${GATEWAY} ExecStart=/usr/sbin/ip netns exec ${NAME} /usr/bin/systemd-nspawn -M ${NAME} -D ${ROOTFS} -b ExecStopPost=/usr/sbin/ip netns del ${NAME} KillMode=process
EXPERIMENT #4 RESTRICT RESOURCES
DEMO
LINUX CONTAINERS FROM SCRATCH
SYSTEMD UNIT FILE
[Unit] Description=MicroVPS Container Server After=network.target
[Service] MemoryAccounting=yes MemoryLimit=64M (…truncated…)
Q & A
LINUX CONTAINERS FROM SCRATCH
MICROVPS REQUIREMENTS
▸ minimal runtime
▸ dedicated network namespace
▸ native package management
▸ automated build
▸ fast iteration cycle
▸ simple deployment/management
EXPERIMENT #5 BUSYBOX + DROPBEAR
DEMO
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
container: busybox-x86_64 dropbearmulti-x86_64 # create directory structure mkdir -vp $(ROOTFS) mkdir $(ROOTFS)/{etc,root,tmp,bin,sbin,home,usr,var,run,service} mkdir $(ROOTFS)/usr/{bin,sbin,share,service} mkdir $(ROOTFS)/var/{run,log,tmp} mkdir $(ROOTFS)/var/log/{lastlog,udhcpc} mkdir $(ROOTFS)/etc/dropbear chmod 01777 $(ROOTFS)/tmp $(ROOTFS)/var/tmp
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
container: (…truncated…) # install busybox install -m755 busybox-x86_64 $(ROOTFS)/bin/busybox # create busybox links ./$(ROOTFS)/bin/busybox --list-all | \ awk '{print "ln -s /bin/busybox $(ROOTFS)/" $$0}' | sh
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
container: (…truncated…) # install dropbear install -m755 dropbearmulti-x86_64 $(ROOTFS)/usr/sbin/dropbear ln -s ../sbin/dropbear $(ROOTFS)/usr/bin/ssh ln -s ../sbin/dropbear $(ROOTFS)/usr/bin/scp ln -s ../sbin/dropbear $(ROOTFS)/usr/sbin/dropbearkey ln -s ../sbin/dropbear $(ROOTFS)/usr/sbin/dropbearconvert
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
container: (…truncated…) # create dropbear keys ./$(ROOTFS)/usr/sbin/dropbearkey -t rsa -f \ $(ROOTFS)/etc/dropbear/dropbear_rsa_host_key ./$(ROOTFS)/usr/sbin/dropbearkey -t dss -f \ $(ROOTFS)/etc/dropbear/dropbear_dss_host_key ./$(ROOTFS)/usr/sbin/dropbearkey -t ecdsa -f \ $(ROOTFS)/etc/dropbear/dropbear_ecdsa_host_key
LINUX CONTAINERS FROM SCRATCH
CREATE A CONTAINER MAKEFILE
busybox-x86_64: curl -L -o $@ \ http://busybox.net/downloads/binaries/latest/busybox-x86_64
dropbearmulti-x86_64: curl -L -o $@ \ http://landley.net/aboriginal/downloads/binaries/extras/dropbearmulti-x86_64
Q & A