fix replace wait errors and rate limiting (rancher#30)

This is a decent-sized refactor, primarily in how the controller is setup, that I think enhances readability. Additionally, wrangler has been upgraded which now supports openapi schema creation so norman has been dropped.

Primarily, however, this change is meant to address rancher#25 (and possibly rancher#36) caused by a bug in wrangler that:
- prevented the controller from properly replacing jobs that could not be patched (pod tempalte spec changes)
- prevented the controller from replacing a job generated with the `apply.WithNoDelete` option
- prevented the controller from processing external plan edits in a timely manner (no more long waits due to spurious replace-wait errors)

Fixes rancher#25
Fixes rancher#34

Depends on rancher/wrangler#66

Author: dweomer

Dockerfile.dapper

@@ -39,6 +39,7 @@ ENV DAPPER_OUTPUT ./bin ./dist
 ENV DAPPER_DOCKER_SOCKET true
 ENV HOME ${DAPPER_SOURCE}
 ENV KUBECONFIG /root/.kube/config
+ENV KUBEHOST 172.17.0.1
 ENV GO111MODULE off
 WORKDIR ${DAPPER_SOURCE}
 

Makefile

@@ -10,7 +10,8 @@ TARGETS := $(shell ls scripts)
 $(TARGETS): .dapper
 	./.dapper $@
 
-e2e: | e2e-sonobuoy e2e-verify
+e2e: e2e-sonobuoy
+	$(MAKE) e2e-verify
 
 clean:
 	rm -rvf ./bin ./dist

e2e/cluster/local/images/k3s/Dockerfile

@@ -19,6 +19,7 @@ RUN set -x \
  && k3s --version
 
 COPY scratch/*-${ARCH}.tar /var/lib/rancher/k3s/agent/images/
+#ADD https://github.com/rancher/k3s/releases/download/${K3S_VERSION}/k3s-airgap-images-${ARCH}.tar /var/lib/rancher/k3s/agent/images/
 
 VOLUME /var/lib/cni
 VOLUME /var/lib/kubelet

e2e/cluster/local/scripts/cluster-get-kubeconfig

@@ -5,5 +5,5 @@ set -e -o pipefail
 cd $(dirname $0)/..
 
 mkdir -vp "$(dirname ${KUBECONFIG:="${HOME}/.kube/config"})"
-docker exec local-leader kubectl config view --raw | sed -e 's/127.0.0.1/172.17.0.1/g' > "${KUBECONFIG}"
+docker exec local-leader kubectl config view --raw | sed -e "s/127.0.0.1/${KUBEHOST:=127.0.0.1}/g" > "${KUBECONFIG}"
 #echo "${KUBECONFIG}" >&2

e2e/framework/controller/deployment.go

@@ -8,7 +8,6 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
-	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	e2eframework "k8s.io/kubernetes/test/e2e/framework"
 	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
 )
@@ -59,7 +58,7 @@ func DeploymentWithTolerations(toleration ...corev1.Toleration) DeploymentOption
 func DeploymentDefaultTolerations() DeploymentOption {
 	return DeploymentWithTolerations(
 		corev1.Toleration{
-			Key:      schedulerapi.TaintNodeUnschedulable,
+			Key:      corev1.TaintNodeUnschedulable,
 			Operator: corev1.TolerationOpExists,
 			Effect:   corev1.TaintEffectNoSchedule,
 		},

e2e/framework/framework.go

@@ -7,14 +7,17 @@ import (
 
 	"github.com/onsi/ginkgo"
 	"github.com/rancher/system-upgrade-controller/e2e/framework/controller"
+	upgradeapi "github.com/rancher/system-upgrade-controller/pkg/apis/upgrade.cattle.io"
 	upgradeapiv1 "github.com/rancher/system-upgrade-controller/pkg/apis/upgrade.cattle.io/v1"
-	upgrade "github.com/rancher/system-upgrade-controller/pkg/generated/clientset/versioned"
+	upgradecln "github.com/rancher/system-upgrade-controller/pkg/generated/clientset/versioned"
 	upgradescheme "github.com/rancher/system-upgrade-controller/pkg/generated/clientset/versioned/scheme"
 	"github.com/rancher/wrangler/pkg/condition"
 	appsv1 "k8s.io/api/apps/v1"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
@@ -38,7 +41,7 @@ type Options struct {
 type Client struct {
 	framework.Framework
 
-	UpgradeClientSet *upgrade.Clientset
+	UpgradeClientSet *upgradecln.Clientset
 
 	controllerDeployment     *appsv1.Deployment
 	controllerServiceAccount *corev1.ServiceAccount
@@ -116,8 +119,8 @@ func (c *Client) DeletePlans(options *metav1.DeleteOptions, listOpts metav1.List
 	return c.UpgradeClientSet.UpgradeV1().Plans(c.Namespace.Name).DeleteCollection(options, listOpts)
 }
 
-func (c *Client) PollPlanCondition(name string, cond condition.Cond, interval, timeout time.Duration) (plan *upgradeapiv1.Plan, err error) {
-	return plan, wait.Poll(interval, timeout, func() (bool, error) {
+func (c *Client) WaitForPlanCondition(name string, cond condition.Cond, timeout time.Duration) (plan *upgradeapiv1.Plan, err error) {
+	return plan, wait.Poll(time.Second, timeout, func() (bool, error) {
 		plan, err = c.GetPlan(name, metav1.GetOptions{})
 		if err != nil {
 			return false, err
@@ -126,6 +129,30 @@ func (c *Client) PollPlanCondition(name string, cond condition.Cond, interval, t
 	})
 }
 
+func (c *Client) WaitForPlanJobs(plan *upgradeapiv1.Plan, count int, timeout time.Duration) (jobs []batchv1.Job, err error) {
+	complete := condition.Cond(batchv1.JobComplete)
+	failed := condition.Cond(batchv1.JobFailed)
+
+	labelSelector := labels.SelectorFromSet(labels.Set{
+		upgradeapi.LabelPlan: plan.Name,
+	})
+
+	return jobs, wait.Poll(5*time.Second, timeout, func() (bool, error) {
+		list, err := c.ClientSet.BatchV1().Jobs(plan.Namespace).List(metav1.ListOptions{
+			LabelSelector: labelSelector.String(),
+		})
+		if err != nil {
+			return false, err
+		}
+		for _, item := range list.Items {
+			if failed.IsTrue(&item) || complete.IsTrue(&item) {
+				jobs = append(jobs, item)
+			}
+		}
+		return len(jobs) >= count, nil
+	})
+}
+
 func (c *Client) BeforeEach() {
 	c.beforeFramework()
 	c.Framework.BeforeEach()
@@ -204,7 +231,7 @@ func (c *Client) beforeFramework() {
 		config.NegotiatedSerializer = upgradescheme.Codecs
 	}
 
-	c.UpgradeClientSet, err = upgrade.NewForConfig(config)
+	c.UpgradeClientSet, err = upgradecln.NewForConfig(config)
 	framework.ExpectNoError(err)
 
 	restClient, err := rest.RESTClientFor(config)

e2e/suite/job_generate_test.go

@@ -0,0 +1,62 @@
+package suite_test
+
+import (
+	"time"
+
+	batchv1 "k8s.io/api/batch/v1"
+
+	"github.com/rancher/system-upgrade-controller/e2e/framework"
+	upgradeapiv1 "github.com/rancher/system-upgrade-controller/pkg/apis/upgrade.cattle.io/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("Job Generation", func() {
+	e2e := framework.New("generate")
+
+	When("fails because of a bad plan", func() {
+		var (
+			err  error
+			plan *upgradeapiv1.Plan
+			jobs []batchv1.Job
+		)
+		BeforeEach(func() {
+			plan = e2e.NewPlan("fail-then-succeed-", "library/alpine:3.11", []string{"sh", "-c"}, "exit 1")
+			plan.Spec.Version = "latest"
+			plan.Spec.Concurrency = 1
+			plan.Spec.NodeSelector = &metav1.LabelSelector{
+				MatchExpressions: []metav1.LabelSelectorRequirement{{
+					Key:      "node-role.kubernetes.io/master",
+					Operator: metav1.LabelSelectorOpDoesNotExist,
+				}},
+			}
+			plan, err = e2e.CreatePlan(plan)
+			Expect(err).ToNot(HaveOccurred())
+
+			plan, err = e2e.WaitForPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 30*time.Second)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(upgradeapiv1.PlanLatestResolved.IsTrue(plan)).To(BeTrue())
+
+			jobs, err = e2e.WaitForPlanJobs(plan, 1, 120*time.Second)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(jobs).To(HaveLen(1))
+			Expect(jobs[0].Status.Succeeded).To(BeNumerically("==", 0))
+			Expect(jobs[0].Status.Failed).To(BeNumerically(">=", 1))
+
+			plan, err = e2e.GetPlan(plan.Name, metav1.GetOptions{})
+			Expect(err).ToNot(HaveOccurred())
+
+			plan.Spec.Upgrade.Args = []string{"exit 0"}
+			plan, err = e2e.UpdatePlan(plan)
+			Expect(err).ToNot(HaveOccurred())
+
+			jobs, err = e2e.WaitForPlanJobs(plan, 1, 120*time.Second)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(jobs).To(HaveLen(1))
+		})
+		It("should apply successfully after edit", func() {
+			Expect(jobs).To(HaveLen(1))
+			Expect(jobs[0].Status.Succeeded).To(BeNumerically("==", 1))
+			Expect(jobs[0].Status.Failed).To(BeNumerically("==", 0))
+		})
+	})
+})

e2e/suite/plan_resolve_test.go

@@ -9,19 +9,20 @@ import (
 	upgradeapiv1 "github.com/rancher/system-upgrade-controller/pkg/apis/upgrade.cattle.io/v1"
 )
 
-var _ = Describe("Upgrade", func() {
+var _ = Describe("Plan Resolution", func() {
 	e2e := framework.New("resolve")
 
-	When("plan missing channel and version", func() {
+	When("missing channel and version", func() {
 		var (
 			err  error
-			plan = e2e.NewPlan("missing-", "", nil)
+			plan *upgradeapiv1.Plan
 		)
 		BeforeEach(func() {
+			plan = e2e.NewPlan("missing-", "", nil)
 			plan, err = e2e.CreatePlan(plan)
 			Expect(err).ToNot(HaveOccurred())
 
-			plan, err = e2e.PollPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 2*time.Second, 30*time.Second)
+			plan, err = e2e.WaitForPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 30*time.Second)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("should not resolve", func() {
@@ -31,18 +32,19 @@ var _ = Describe("Upgrade", func() {
 		})
 	})
 
-	When("plan has version", func() {
+	When("has version", func() {
 		var (
 			err  error
-			plan = e2e.NewPlan("version-", "", nil)
+			plan *upgradeapiv1.Plan
 		)
 		BeforeEach(func() {
+			plan = e2e.NewPlan("version-", "", nil)
 			plan.Spec.Version = "test"
 
 			plan, err = e2e.CreatePlan(plan)
 			Expect(err).ToNot(HaveOccurred())
 
-			plan, err = e2e.PollPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 2*time.Second, 30*time.Second)
+			plan, err = e2e.WaitForPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 30*time.Second)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("should resolve", func() {
@@ -53,19 +55,20 @@ var _ = Describe("Upgrade", func() {
 		})
 	})
 
-	When("plan has version with semver+metadata", func() {
+	When("has version with semver+metadata", func() {
 		var (
 			err    error
-			plan   = e2e.NewPlan("version-semver-metadata-", "", nil)
+			plan   *upgradeapiv1.Plan
 			semver = "v1.2.3+test"
 		)
 		BeforeEach(func() {
+			plan = e2e.NewPlan("version-semver-metadata-", "", nil)
 			plan.Spec.Version = semver
 
 			plan, err = e2e.CreatePlan(plan)
 			Expect(err).ToNot(HaveOccurred())
 
-			plan, err = e2e.PollPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 2*time.Second, 30*time.Second)
+			plan, err = e2e.WaitForPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 30*time.Second)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("should resolve", func() {
@@ -78,22 +81,23 @@ var _ = Describe("Upgrade", func() {
 		})
 	})
 
-	When("plan has channel", func() {
+	When("has channel", func() {
 		var (
 			err        error
-			plan       = e2e.NewPlan("channel-", "", nil)
+			plan       *upgradeapiv1.Plan
 			channelSrv *httptest.Server
 			channelTag = "test"
 		)
 		BeforeEach(func() {
 			channelSrv = framework.ChannelServer(path.Join("/local", channelTag))
+			plan = e2e.NewPlan("channel-", "", nil)
 			plan.Spec.Channel = channelSrv.URL
 			Expect(plan.Spec.Channel).ToNot(BeEmpty())
 
 			plan, err = e2e.CreatePlan(plan)
 			Expect(err).ToNot(HaveOccurred())
 
-			plan, err = e2e.PollPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 2*time.Second, 30*time.Second)
+			plan, err = e2e.WaitForPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 30*time.Second)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		AfterEach(func() {
@@ -109,22 +113,23 @@ var _ = Describe("Upgrade", func() {
 		})
 	})
 
-	When("plan has channel with semver+metadata", func() {
+	When("has channel with semver+metadata", func() {
 		var (
 			err        error
-			plan       = e2e.NewPlan("channel-semver-metadata-", "", nil)
+			plan       *upgradeapiv1.Plan
 			channelSrv *httptest.Server
 			channelTag = "v1.2.3+test"
 		)
 		BeforeEach(func() {
 			channelSrv = framework.ChannelServer(path.Join("/local/test", channelTag))
+			plan = e2e.NewPlan("channel-semver-metadata-", "", nil)
 			plan.Spec.Channel = channelSrv.URL
 			Expect(plan.Spec.Channel).ToNot(BeEmpty())
 
 			plan, err = e2e.CreatePlan(plan)
 			Expect(err).ToNot(HaveOccurred())
 
-			plan, err = e2e.PollPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 2*time.Second, 30*time.Second)
+			plan, err = e2e.WaitForPlanCondition(plan.Name, upgradeapiv1.PlanLatestResolved, 30*time.Second)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		AfterEach(func() {

go.mod

@@ -5,40 +5,39 @@ go 1.13
 require (
 	github.com/onsi/ginkgo v1.10.1
 	github.com/onsi/gomega v1.7.0
-	github.com/rancher/norman v0.0.0-20191015045301-1fdd5cbabdd9
-	github.com/rancher/wrangler v0.3.1
+	github.com/rancher/wrangler v0.4.2-0.20200225192203-e5307acd8846
 	github.com/rancher/wrangler-api v0.2.1-0.20191025043713-b1ca9c21825a
 	github.com/sirupsen/logrus v1.4.2
 	github.com/urfave/cli v1.22.2
-	k8s.io/api v0.0.0
-	k8s.io/apimachinery v0.0.0
+	k8s.io/api v0.17.2
+	k8s.io/apimachinery v0.17.2
 	k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
-	k8s.io/kubernetes v1.16.3
+	k8s.io/kubernetes v1.17.2
 )
 
 replace (
-	k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.16.3-k3s.2
-	k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.16.3-k3s.2
-	k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.16.3-k3s.2
-	k8s.io/apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.16.3-k3s.2
-	k8s.io/cli-runtime => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.16.3-k3s.2
-	k8s.io/client-go => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.16.3-k3s.2
-	k8s.io/cloud-provider => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.16.3-k3s.2
-	k8s.io/cluster-bootstrap => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.16.3-k3s.2
-	k8s.io/code-generator => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.16.3-k3s.2
-	k8s.io/component-base => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.16.3-k3s.2
-	k8s.io/cri-api => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.16.3-k3s.2
-	k8s.io/csi-translation-lib => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.16.3-k3s.2
-	k8s.io/kube-aggregator => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.16.3-k3s.2
-	k8s.io/kube-controller-manager => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.16.3-k3s.2
-	k8s.io/kube-proxy => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.16.3-k3s.2
-	k8s.io/kube-scheduler => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.16.3-k3s.2
-	k8s.io/kubectl => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.16.3-k3s.2
-	k8s.io/kubelet => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.16.3-k3s.2
-	k8s.io/kubernetes => github.com/rancher/kubernetes v1.16.3-k3s.2
-	k8s.io/legacy-cloud-providers => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.16.3-k3s.2
-	k8s.io/metrics => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.16.3-k3s.2
-	k8s.io/sample-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.16.3-k3s.2
-	k8s.io/sample-cli-plugin => github.com/rancher/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.16.3-k3s.2
-	k8s.io/sample-controller => github.com/rancher/kubernetes/staging/src/k8s.io/sample-controller v1.16.3-k3s.2
+	k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.17.2-k3s1
+	k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.17.2-k3s1
+	k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.17.2-k3s1
+	k8s.io/apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiserver v1.17.2-k3s1
+	k8s.io/cli-runtime => github.com/rancher/kubernetes/staging/src/k8s.io/cli-runtime v1.17.2-k3s1
+	k8s.io/client-go => github.com/rancher/kubernetes/staging/src/k8s.io/client-go v1.17.2-k3s1
+	k8s.io/cloud-provider => github.com/rancher/kubernetes/staging/src/k8s.io/cloud-provider v1.17.2-k3s1
+	k8s.io/cluster-bootstrap => github.com/rancher/kubernetes/staging/src/k8s.io/cluster-bootstrap v1.17.2-k3s1
+	k8s.io/code-generator => github.com/rancher/kubernetes/staging/src/k8s.io/code-generator v1.17.2-k3s1
+	k8s.io/component-base => github.com/rancher/kubernetes/staging/src/k8s.io/component-base v1.17.2-k3s1
+	k8s.io/cri-api => github.com/rancher/kubernetes/staging/src/k8s.io/cri-api v1.17.2-k3s1
+	k8s.io/csi-translation-lib => github.com/rancher/kubernetes/staging/src/k8s.io/csi-translation-lib v1.17.2-k3s1
+	k8s.io/kube-aggregator => github.com/rancher/kubernetes/staging/src/k8s.io/kube-aggregator v1.17.2-k3s1
+	k8s.io/kube-controller-manager => github.com/rancher/kubernetes/staging/src/k8s.io/kube-controller-manager v1.17.2-k3s1
+	k8s.io/kube-proxy => github.com/rancher/kubernetes/staging/src/k8s.io/kube-proxy v1.17.2-k3s1
+	k8s.io/kube-scheduler => github.com/rancher/kubernetes/staging/src/k8s.io/kube-scheduler v1.17.2-k3s1
+	k8s.io/kubectl => github.com/rancher/kubernetes/staging/src/k8s.io/kubectl v1.17.2-k3s1
+	k8s.io/kubelet => github.com/rancher/kubernetes/staging/src/k8s.io/kubelet v1.17.2-k3s1
+	k8s.io/kubernetes => github.com/rancher/kubernetes v1.17.2-k3s1
+	k8s.io/legacy-cloud-providers => github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.17.2-k3s1
+	k8s.io/metrics => github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.17.2-k3s1
+	k8s.io/sample-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.17.2-k3s1
+	k8s.io/sample-cli-plugin => github.com/rancher/kubernetes/staging/src/k8s.io/sample-cli-plugin v1.17.2-k3s1
+	k8s.io/sample-controller => github.com/rancher/kubernetes/staging/src/k8s.io/sample-controller v1.17.2-k3s1
 )