implement route controller

Signed-off-by: Huamin Chen <[email protected]>
vllm-project · Mar 2, 2025 · d41ffdd · d41ffdd
1 parent 1b26eec
commit d41ffdd
Show file tree

Hide file tree

Showing 24 changed files with 1,197 additions and 273 deletions.
diff --git a/router-controller/.dockerignore b/router-controller/.dockerignore
@@ -0,0 +1,3 @@
+# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
+# Ignore build and test binaries.
+bin/
diff --git a/router-controller/.gitignore b/router-controller/.gitignore
@@ -0,0 +1,27 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+bin/*
+Dockerfile.cross
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Go workspace file
+go.work
+
+# Kubernetes Generated files - skip generated files, except for vendored files
+!vendor/**/zz_generated.*
+
+# editor and IDE paraphernalia
+.idea
+.vscode
+*.swp
+*.swo
+*~
diff --git a/router-controller/.golangci.yml b/router-controller/.golangci.yml
@@ -0,0 +1,47 @@
+run:
+  timeout: 5m
+  allow-parallel-runners: true
+
+issues:
+  # don't skip warning about doc comments
+  # don't exclude the default set of lint
+  exclude-use-default: false
+  # restore some of the defaults
+  # (fill in the rest as needed)
+  exclude-rules:
+    - path: "api/*"
+      linters:
+        - lll
+    - path: "internal/*"
+      linters:
+        - dupl
+        - lll
+linters:
+  disable-all: true
+  enable:
+    - dupl
+    - errcheck
+    - exportloopref
+    - ginkgolinter
+    - goconst
+    - gocyclo
+    - gofmt
+    - goimports
+    - gosimple
+    - govet
+    - ineffassign
+    - lll
+    - misspell
+    - nakedret
+    - prealloc
+    - revive
+    - staticcheck
+    - typecheck
+    - unconvert
+    - unparam
+    - unused
+
+linters-settings:
+  revive:
+    rules:
+      - name: comment-spacings
diff --git a/router-controller/README.md b/router-controller/README.md
@@ -1,18 +1,22 @@
-# router-controller
-// TODO(user): Add simple overview of use/purpose
+# Router Controller
+
+This is the controller for the Router CRD. It is responsible for creating and updating the ConfigMap for the vllm_router.
 
 ## Description
-// TODO(user): An in-depth paragraph about your project and overview of use
+
+The Router Controller is a Kubernetes controller that manages the Router custom resource. It watches for Router resources and creates a ConfigMap for the vllm_router.
 
 ## Getting Started
 
 ### Prerequisites
+
 - go version v1.22.0+
 - docker version 17.03+.
 - kubectl version v1.11.3+.
 - Access to a Kubernetes v1.11.3+ cluster.
 
 ### To Deploy on the cluster
+
 **Build and push your image to the location specified by `IMG`:**
 
 ```sh
@@ -21,7 +25,7 @@ make docker-build docker-push IMG=<some-registry>/router-controller:tag
 
 **NOTE:** This image ought to be published in the personal registry you specified.
 And it is required to have access to pull the image from the working environment.
-Make sure you have the proper permission to the registry if the above commands don’t work.
+Make sure you have the proper permission to the registry if the above commands don't work.
 
 **Install the CRDs into the cluster:**
 
@@ -48,6 +52,7 @@ kubectl apply -k config/samples/
 >**NOTE**: Ensure that the samples has default values to test it out.
 
 ### To Uninstall
+
 **Delete the instances (CRs) from the cluster:**
 
 ```sh
@@ -70,7 +75,9 @@ make undeploy
 
 Following are the steps to build the installer and distribute this project to users.
 
-1. Build the installer for the image built and published in the registry:
+### Build the installer
+
+Build the installer for the image built and published in the registry:
 
 ```sh
 make build-installer IMG=<some-registry>/router-controller:tag
@@ -81,34 +88,86 @@ file in the dist directory. This file contains all the resources built
 with Kustomize, which are necessary to install this project without
 its dependencies.
 
-2. Using the installer
+### Using the installer
 
 Users can just run kubectl apply -f <URL for YAML BUNDLE> to install the project, i.e.:
 
 ```sh
 kubectl apply -f https://raw.githubusercontent.com/<org>/router-controller/<tag or branch>/dist/install.yaml
 ```
 
-## Contributing
-// TODO(user): Add detailed information on how you would like others to contribute to this project
+## StaticRoute CRD
+
+The StaticRoute CRD allows you to configure the vllm_router with static backends and models. The controller reads the CRD and creates a ConfigMap that can be used by the vllm_router with the `--dynamic-config-json` option.
+
+### Example
 
-**NOTE:** Run `make help` for more information on all potential `make` targets
+```yaml
+apiVersion: production-stack.vllm.ai/v1alpha1
+kind: StaticRoute
+metadata:
+  name: staticroute-sample
+spec:
+  # Service discovery method
+  serviceDiscovery: static
 
-More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html)
+  # Routing logic
+  routingLogic: roundrobin
 
-## License
+  # Comma-separated list of backend URLs
+  staticBackends: "http://localhost:9001,http://localhost:9002,http://localhost:9003"
 
-Copyright 2025.
+  # Comma-separated list of model names
+  staticModels: "facebook/opt-125m,meta-llama/Llama-3.1-8B-Instruct,facebook/opt-125m"
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+  # Name of the vllm_router to configure
+  routerRef:
+    kind: Service
+    apiVersion: v1
+    name: vllm-router
+    namespace: default
+
+  # Optional: Name of the ConfigMap to create
+  configMapName: vllm-router-config
+```
+
+### How it works
+
+- The controller watches for StaticRoute resources.
+- When a StaticRoute is created or updated, the controller creates or updates a ConfigMap with the dynamic configuration.
+- The ConfigMap contains a `dynamic_config.json` file with the following structure:
+
+```json
+{
+  "service_discovery": "static",
+  "routing_logic": "roundrobin",
+  "static_backends": "http://localhost:9001,http://localhost:9002,http://localhost:9003",
+  "static_models": "facebook/opt-125m,meta-llama/Llama-3.1-8B-Instruct,facebook/opt-125m"
+}
+```
+
+- The controller checks the health endpoint of the vllm_router services that match the `routerSelector` to verify that the configuration is valid.
+- The vllm_router should be configured to use the ConfigMap with the `--dynamic-config-json` option:
+
+```yaml
+containers:
+- name: vllm-router
+  image: vllm-router:latest
+  args:
+  - "--dynamic-config-json /etc/vllm-router/dynamic_config.json"
+  volumeMounts:
+  - name: config-volume
+    mountPath: /etc/vllm-router
+volumes:
+- name: config-volume
+  configMap:
+    name: vllm-router-config
+```
 
-    http://www.apache.org/licenses/LICENSE-2.0
+### Status
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+The StaticRoute resource has the following status fields:
 
+- `configMapRef`: The name of the ConfigMap that was created.
+- `lastAppliedTime`: The time when the configuration was last applied.
+- `conditions`: A list of conditions that represent the latest available observations of the StaticRoute's state.
diff --git a/router-controller/api/v1alpha1/staticroute_types.go b/router-controller/api/v1alpha1/staticroute_types.go
@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -28,14 +29,87 @@ type StaticRouteSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
-	// Foo is an example field of StaticRoute. Edit staticroute_types.go to remove/update
-	Foo string `json:"foo,omitempty"`
+	// ServiceDiscovery specifies the service discovery method
+	// +kubebuilder:validation:Enum=static
+	// +kubebuilder:default=static
+	ServiceDiscovery string `json:"serviceDiscovery"`
+
+	// RoutingLogic specifies the routing logic to use
+	// +kubebuilder:validation:Enum=roundrobin;least_loaded
+	// +kubebuilder:default=roundrobin
+	RoutingLogic string `json:"routingLogic"`
+
+	// StaticBackends is a comma-separated list of backend URLs
+	// +kubebuilder:validation:Required
+	StaticBackends string `json:"staticBackends"`
+
+	// StaticModels is a comma-separated list of model names
+	// +kubebuilder:validation:Required
+	StaticModels string `json:"staticModels"`
+
+	// RouterRef is a reference to the router service
+	// +optional
+	RouterRef *corev1.ObjectReference `json:"routerRef,omitempty"`
+
+	// RouterSelector is a label selector to identify which vllm_router(s) to configure
+	// +optional
+	// Deprecated: Use RouterRef instead
+	RouterSelector *metav1.LabelSelector `json:"routerSelector,omitempty"`
+
+	// HealthCheck defines the health check configuration for the router
+	// +optional
+	HealthCheck *HealthCheckConfig `json:"healthCheck,omitempty"`
+
+	// ConfigMapName is the name of the ConfigMap to create with the dynamic config
+	// +optional
+	ConfigMapName string `json:"configMapName,omitempty"`
+}
+
+// HealthCheckConfig defines the configuration for health checks
+type HealthCheckConfig struct {
+	// Number of seconds after which the probe times out
+	// +optional
+	// +kubebuilder:default=5
+	// +kubebuilder:validation:Minimum=1
+	TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"`
+
+	// Number of seconds between probe attempts
+	// +optional
+	// +kubebuilder:default=10
+	// +kubebuilder:validation:Minimum=1
+	PeriodSeconds int32 `json:"periodSeconds,omitempty"`
+
+	// Minimum consecutive successes for the probe to be considered successful
+	// +optional
+	// +kubebuilder:default=1
+	// +kubebuilder:validation:Minimum=1
+	SuccessThreshold int32 `json:"successThreshold,omitempty"`
+
+	// Minimum consecutive failures for the probe to be considered failed
+	// +optional
+	// +kubebuilder:default=3
+	// +kubebuilder:validation:Minimum=1
+	FailureThreshold int32 `json:"failureThreshold,omitempty"`
 }
 
 // StaticRouteStatus defines the observed state of StaticRoute
 type StaticRouteStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
+
+	// Conditions represent the latest available observations of the StaticRoute's state
+	// +optional
+	// +patchMergeKey=type
+	// +patchStrategy=merge
+	Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
+
+	// ConfigMapRef is a reference to the created ConfigMap
+	// +optional
+	ConfigMapRef string `json:"configMapRef,omitempty"`
+
+	// LastAppliedTime is the last time the configuration was applied to the router
+	// +optional
+	LastAppliedTime *metav1.Time `json:"lastAppliedTime,omitempty"`
 }
 
 // +kubebuilder:object:root=true