Completely restructure & document the kernel image build (weaveworks#506)

* Completely restructure & document the kernel image build

Co-Authored-By: leigh capili <[email protected]>

Author: luxas

images/kernel/Dockerfile

@@ -5,10 +5,9 @@ RUN git fetch --tags
 RUN git checkout v${KERNEL_VERSION} && \
     make clean && make mrproper
 
-COPY config-${KERNEL_VERSION}${KERNEL_EXTRA} .config
+COPY generated/config-amd64-${KERNEL_VERSION}${KERNEL_EXTRA} .config
 
-RUN make EXTRAVERSION=${KERNEL_EXTRA} LOCALVERSION= olddefconfig && \
-	make EXTRAVERSION=${KERNEL_EXTRA} LOCALVERSION= olddefconfig
+RUN make EXTRAVERSION=${KERNEL_EXTRA} LOCALVERSION= olddefconfig
 
 RUN	make EXTRAVERSION=${KERNEL_EXTRA} LOCALVERSION= -j32
 RUN make EXTRAVERSION=${KERNEL_EXTRA} LOCALVERSION= modules_install

images/kernel/Makefile

@@ -1,11 +1,22 @@
-KERNEL_VERSION ?= 4.19.47
-KERNEL_EXTRA ?= 
+# Check https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/refs/ for updates
+KERNEL_VERSIONS ?= 4.14.166 4.19.97 5.4.13
 
 all: build
-build:
-	docker build -t weaveworks/ignite-kernel:${KERNEL_VERSION}${KERNEL_EXTRA} \
-		--build-arg KERNEL_VERSION=${KERNEL_VERSION} \
-		--build-arg KERNEL_EXTRA=${KERNEL_EXTRA} .
 
-push:
-	docker push weaveworks/ignite-kernel:${KERNEL_VERSION}${KERNEL_EXTRA}
+upgrade: $(addprefix upgrade-,$(KERNEL_VERSIONS))
+	./patch-config.sh
+	for file in generated/*; do \
+		./upgrade-config.sh $$file $$file; done
+
+upgrade-%:
+	for file in upstream/*; do \
+		./upgrade-config.sh $$file versioned/$$(basename $$file)-$*; done
+
+build: $(addprefix build-,$(KERNEL_VERSIONS))
+build-%:
+	docker build -t weaveworks/ignite-kernel:$* \
+		--build-arg KERNEL_VERSION=$* .
+
+push: build $(addprefix push-,$(KERNEL_VERSIONS))
+push-%:
+	docker push weaveworks/ignite-kernel:${KERNEL_VERSION}

images/kernel/README.md

@@ -0,0 +1,47 @@
+# Kernel Images
+
+These kernel OCI images contain the kernel binary (at `/boot/vmlinux`) and supporting modules (in `/lib/modules`)
+for guest VMs ran by Ignite.
+
+## Building the Kernel Images
+
+```console
+$ make
+```
+
+## Versions
+
+All LTS versions starting from 4.14 and above are supported by the Ignite team.
+This means in practice:
+
+- 4.14.x
+- 4.19.x
+- 5.4.x
+
+The exact patch versions may be found in the [Makefile](Makefile).
+The available versions exist in the [stable kernel git tree](https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/refs/).
+
+## Upgrading to a new kernel version
+
+The kernel Makefile has an `upgrade` command that will generate patched kernel configs for each specific version in `KERNEL_VERSIONS`.
+
+The linux kernel source code is checked out in a build container for each target version, and a resulting "olddefconfig" based on each Firecracker recommended base-config in the `upstream/` directory is then output to the `versioned/` directory.
+
+Once `make upgrade-%` has produced a firecracker base-config for all `KERNEL_VERSIONS`, the resulting versioned configs are copied to the `generated/` dir and are patched with ignite specific `./config-patches`. These configs under `generated/` are used for the matching kernel builds.
+
+Run:
+
+```console
+$ make upgrade
+```
+
+after you've upgraded the values in the Makefile.
+
+## Kernel Config Parameters we care about
+
+Some options to the kernel are specifically important for making guest software work.
+
+Please see: [config-patches](config-patches) for what kernel configs we've changed.
+The base kernel config is the MicroVM-optimized config file from the Firecracker team.
+We're storing it in [upstream/config-amd64](upstream/config-amd64). It's available online
+at [firecracker/resources](https://github.com/firecracker-microvm/firecracker/tree/master/resources).

images/kernel/config-patches

@@ -0,0 +1,22 @@
+# In this file, the recipe for patching all kernel configs (all versions & architectures) is
+
+# For making Weave Net work
+CONFIG_DUMMY=y
+# Enable VXLAN support as a module so that e.g. Flannel works
+CONFIG_VXLAN=m
+# Enable support for soft shutdown of amd64 VMs
+# See https://github.com/firecracker-microvm/firecracker/blob/master/docs/api_requests/actions.md#sendctrlaltdel
+CONFIG_KEYBOARD_ATKBD=y
+CONFIG_SERIO=y
+CONFIG_SERIO_I8042=y
+CONFIG_SERIO_LIBPS2=y
+# Make the guest's wall clock not drift
+# https://github.com/firecracker-microvm/firecracker/blob/master/FAQ.md#my-guest-wall-clock-is-drifting-how-can-i-fix-it
+CONFIG_PTP_1588_CLOCK=y
+CONFIG_PTP_1588_CLOCK_KVM=y
+
+# Some patches for keeping network functionalities that Kubernetes needs/might need
+CONFIG_IPVLAN=y
+CONFIG_IPVTAP=y
+CONFIG_TAP=y
+CONFIG_IP_VS_MH=m

images/kernel/config-4.14.123 renamed to images/kernel/generated/config-amd64-4.14.166

@@ -1,6 +1,6 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 4.14.123 Kernel Configuration
+# Linux/x86 4.14.166 Kernel Configuration
 #
 CONFIG_64BIT=y
 CONFIG_X86_64=y
@@ -592,6 +592,9 @@ CONFIG_ARCH_RANDOM=y
 CONFIG_X86_SMAP=y
 # CONFIG_X86_INTEL_MPX is not set
 CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS=y
+CONFIG_X86_INTEL_TSX_MODE_OFF=y
+# CONFIG_X86_INTEL_TSX_MODE_ON is not set
+# CONFIG_X86_INTEL_TSX_MODE_AUTO is not set
 CONFIG_SECCOMP=y
 # CONFIG_HZ_100 is not set
 CONFIG_HZ_250=y
@@ -1313,6 +1316,7 @@ CONFIG_DNS_RESOLVER=m
 # CONFIG_BATMAN_ADV is not set
 CONFIG_OPENVSWITCH=m
 CONFIG_OPENVSWITCH_GRE=m
+CONFIG_OPENVSWITCH_VXLAN=m
 CONFIG_VSOCKETS=y
 CONFIG_VIRTIO_VSOCKETS=y
 CONFIG_VIRTIO_VSOCKETS_COMMON=y
@@ -1837,7 +1841,7 @@ CONFIG_I2C_ALGOBIT=m
 # CONFIG_SPI is not set
 # CONFIG_SPMI is not set
 # CONFIG_HSI is not set
-CONFIG_PPS=m
+CONFIG_PPS=y
 CONFIG_PPS_DEBUG=y
 
 #
@@ -1854,12 +1858,12 @@ CONFIG_PPS_CLIENT_GPIO=m
 #
 # PTP clock support
 #
-CONFIG_PTP_1588_CLOCK=m
+CONFIG_PTP_1588_CLOCK=y
 
 #
 # Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks.
 #
-CONFIG_PTP_1588_CLOCK_KVM=m
+CONFIG_PTP_1588_CLOCK_KVM=y
 # CONFIG_GPIOLIB is not set
 # CONFIG_W1 is not set
 # CONFIG_POWER_AVS is not set
@@ -3260,6 +3264,7 @@ CONFIG_ASSOCIATIVE_ARRAY=y
 CONFIG_HAS_IOMEM=y
 CONFIG_HAS_IOPORT_MAP=y
 CONFIG_HAS_DMA=y
+# CONFIG_SGL_ALLOC is not set
 # CONFIG_DMA_NOOP_OPS is not set
 # CONFIG_DMA_VIRT_OPS is not set
 CONFIG_CPU_RMAP=y

images/kernel/config-4.19.47 renamed to images/kernel/generated/config-amd64-4.19.97

@@ -1,14 +1,15 @@
 #
 # Automatically generated file; DO NOT EDIT.
-# Linux/x86 4.19.47 Kernel Configuration
+# Linux/x86 4.19.97 Kernel Configuration
 #
 
 #
-# Compiler: gcc (Ubuntu 7.4.0-1ubuntu1~18.10) 7.4.0
+# Compiler: gcc (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0
 #
 CONFIG_CC_IS_GCC=y
 CONFIG_GCC_VERSION=70400
 CONFIG_CLANG_VERSION=0
+CONFIG_CC_HAS_ASM_GOTO=y
 CONFIG_IRQ_WORK=y
 CONFIG_BUILDTIME_EXTABLE_SORT=y
 CONFIG_THREAD_INFO_IN_TASK=y
@@ -358,6 +359,9 @@ CONFIG_X86_SMAP=y
 CONFIG_X86_INTEL_UMIP=y
 # CONFIG_X86_INTEL_MPX is not set
 CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS=y
+CONFIG_X86_INTEL_TSX_MODE_OFF=y
+# CONFIG_X86_INTEL_TSX_MODE_ON is not set
+# CONFIG_X86_INTEL_TSX_MODE_AUTO is not set
 CONFIG_SECCOMP=y
 # CONFIG_HZ_100 is not set
 CONFIG_HZ_250=y
@@ -758,7 +762,7 @@ CONFIG_PACKET_DIAG=m
 CONFIG_UNIX=y
 CONFIG_UNIX_DIAG=m
 CONFIG_TLS=m
-CONFIG_TLS_DEVICE=y
+# CONFIG_TLS_DEVICE is not set
 CONFIG_XFRM=y
 CONFIG_XFRM_OFFLOAD=y
 CONFIG_XFRM_ALGO=y
@@ -1338,6 +1342,7 @@ CONFIG_DNS_RESOLVER=m
 # CONFIG_BATMAN_ADV is not set
 CONFIG_OPENVSWITCH=m
 CONFIG_OPENVSWITCH_GRE=m
+CONFIG_OPENVSWITCH_VXLAN=m
 CONFIG_VSOCKETS=y
 CONFIG_VSOCKETS_DIAG=y
 CONFIG_VIRTIO_VSOCKETS=y
@@ -1393,7 +1398,6 @@ CONFIG_LWTUNNEL=y
 CONFIG_LWTUNNEL_BPF=y
 CONFIG_DST_CACHE=y
 CONFIG_GRO_CELLS=y
-CONFIG_SOCK_VALIDATE_XMIT=y
 # CONFIG_NET_DEVLINK is not set
 CONFIG_MAY_USE_DEVLINK=y
 CONFIG_FAILOVER=y
@@ -1871,7 +1875,7 @@ CONFIG_I2C_ALGOBIT=m
 # CONFIG_SPI is not set
 # CONFIG_SPMI is not set
 # CONFIG_HSI is not set
-CONFIG_PPS=m
+CONFIG_PPS=y
 CONFIG_PPS_DEBUG=y
 
 #
@@ -1888,12 +1892,12 @@ CONFIG_PPS_CLIENT_GPIO=m
 #
 # PTP clock support
 #
-CONFIG_PTP_1588_CLOCK=m
+CONFIG_PTP_1588_CLOCK=y
 
 #
 # Enable PHYLIB and NETWORK_PHY_TIMESTAMPING to see the additional clocks.
 #
-CONFIG_PTP_1588_CLOCK_KVM=m
+CONFIG_PTP_1588_CLOCK_KVM=y
 # CONFIG_PINCTRL is not set
 # CONFIG_GPIOLIB is not set
 # CONFIG_W1 is not set
@@ -2165,10 +2169,10 @@ CONFIG_DRM_PANEL_ORIENTATION_QUIRKS=m
 #
 # Frame buffer Devices
 #
-CONFIG_FB=m
-# CONFIG_FIRMWARE_EDID is not set
 CONFIG_FB_CMDLINE=y
 CONFIG_FB_NOTIFY=y
+CONFIG_FB=m
+# CONFIG_FIRMWARE_EDID is not set
 CONFIG_FB_CFB_FILLRECT=m
 CONFIG_FB_CFB_COPYAREA=m
 CONFIG_FB_CFB_IMAGEBLIT=m
@@ -2266,7 +2270,7 @@ CONFIG_HID_GENERIC=m
 # CONFIG_HID_LOGITECH is not set
 # CONFIG_HID_MAGICMOUSE is not set
 # CONFIG_HID_MAYFLASH is not set
-# CONFIG_HID_REDRAGON is not set
+CONFIG_HID_REDRAGON=y
 # CONFIG_HID_MICROSOFT is not set
 # CONFIG_HID_MONTEREY is not set
 # CONFIG_HID_MULTITOUCH is not set
@@ -2404,7 +2408,6 @@ CONFIG_IOMMU_SUPPORT=y
 #
 # CONFIG_RPMSG_QCOM_GLINK_RPM is not set
 # CONFIG_RPMSG_VIRTIO is not set
-# CONFIG_SOUNDWIRE is not set
 
 #
 # SOC (System On Chip) specific Drivers