Skip to main content

Round-Robin Balancer

In this guide, you will configure a multi-backend model cluster with RoutingStrategy: "round-robin" to distribute inference load evenly across multiple instances.


Runnable Go Code​

examples/round_robin_balancer/main.go
package main

import (
"context"
"fmt"

"github.com/goroute-ai/ai-gateway-go-docs/examples/demoutil"
"github.com/goroute-ai/ai-gateway-go/pkg/gateway"
)

func main() {
ctx := context.Background()
demoutil.Banner("Stateless Round-Robin Load Balancing")

gw, err := gateway.New(
gateway.WithModelList(
gateway.ModelEntry{
ModelName: "balanced-cluster",
Model: "gemini/gemini-2.5-flash-lite",
RoutingStrategy: "round-robin",
},
gateway.ModelEntry{
ModelName: "balanced-cluster",
Model: "openai/gpt-4o-mini",
RoutingStrategy: "round-robin",
},
),
gateway.WithDemoMocksIfEnvMissing(),
gateway.WithSilentLogs(),
)
if err != nil {
demoutil.Fail("Gateway initialization failed", err)
}
defer gw.Close()

demoutil.Info("Strategy", "Round-Robin across 2 backend instances")
for i := 1; i <= 4; i++ {
res, err := gw.Chat(ctx, "balanced-cluster", "ping", gateway.WithMaxTokens(1))
if err != nil {
demoutil.Fail("Request failed", err)
}
demoutil.Info(fmt.Sprintf("Request #%d", i), fmt.Sprintf("Handled by: %s (%s)", res.Model, res.Provider))
}

demoutil.Success("Load Distribution", "Evenly alternated across available backends")
demoutil.Done()
}