From b4f3ec3343b8d0b5aea14a52fbdfb6d0e12a7fa7 Mon Sep 17 00:00:00 2001 From: KevinMidboe Date: Thu, 16 Jan 2025 00:17:11 +0100 Subject: [PATCH] init --- .gitignore | 1 + Makefile | 26 ++++++++ assets/docker-compose-2.yml | 24 +++++++ assets/docker-compose.yml | 24 +++++++ assets/dynamic.yml | 34 ++++++++++ client/etcd/etcd.go | 129 ++++++++++++++++++++++++++++++++++++ config/config.go | 26 ++++++++ converter/json.go | 18 +++++ converter/yaml.go | 28 ++++++++ generator/docker.go | 65 ++++++++++++++++++ generator/traefik.go | 83 +++++++++++++++++++++++ go.mod | 97 +++++++++++++++++++++++++++ main.go | 66 ++++++++++++++++++ 13 files changed, 621 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 assets/docker-compose-2.yml create mode 100644 assets/docker-compose.yml create mode 100644 assets/dynamic.yml create mode 100644 client/etcd/etcd.go create mode 100644 config/config.go create mode 100644 converter/json.go create mode 100644 converter/yaml.go create mode 100644 generator/docker.go create mode 100644 generator/traefik.go create mode 100644 go.mod create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..48b8bf9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +vendor/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..221ef79 --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +.PHONY: build install server + +PROJECT_NAME=$(shell basename $(CURDIR)) + +## build: build the application +build: + export GO111MODULE="auto"; \ + go mod download; \ + go mod vendor; \ + CGO_ENABLED=0 go build -a -ldflags '-s' -installsuffix cgo -o main main.go + +## install: fetches go modules +install: + export GO111MODULE="on"; \ + go mod tidy; \ + go mod download \ + +## server: runs the server with -race +server: + export GO111MODULE="on"; \ + go run main.go + +## modd: Monitors the directory, and recompiles your app every time a file changes. Also runs tests. +## (To install modd, run: go get github.com/cortesi/modd/cmd/modd) +modd: + modd diff --git a/assets/docker-compose-2.yml b/assets/docker-compose-2.yml new file mode 100644 index 0000000..de8ee6e --- /dev/null +++ b/assets/docker-compose-2.yml @@ -0,0 +1,24 @@ +# version: "3.9" + +services: + webapp: + image: nginx:alpine + container_name: webapp + labels: + - "traefik.enable=true" + - "traefik.http.routers.webapp.rule=Host(`webapp.localhost`)" + - "traefik.http.routers.webapp.entrypoints=web" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + - "traefik.http.routers.webapp.middlewares=redirect-to-https" + - "traefik.http.routers.webapp.tls=true" + - "traefik.http.services.webapp.loadbalancer.server.port=80" + + backend: + image: my-backend-app:latest + container_name: backend + labels: + - "traefik.enable=true" + - "traefik.http.routers.backend.rule=Host(`backend.localhost`)" + - "traefik.http.routers.backend.entrypoints=web" + - "traefik.http.routers.backend.tls=true" + - "traefik.http.services.backend.loadbalancer.server.port=5000" # Port exposed by the backend app diff --git a/assets/docker-compose.yml b/assets/docker-compose.yml new file mode 100644 index 0000000..d168b4c --- /dev/null +++ b/assets/docker-compose.yml @@ -0,0 +1,24 @@ +version: "3.9" + +services: + webapp: + image: nginx:alpine + container_name: webapp + labels: + - "traefik.enable=true" + - "traefik.http.routers.webapp.rule=Host(`webapp.localhost`)" + - "traefik.http.routers.webapp.entrypoints=web" + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + - "traefik.http.routers.webapp.middlewares=redirect-to-https" + - "traefik.http.routers.webapp.tls=true" + - "traefik.http.services.webapp.loadbalancer.server.port=80" + + backend: + image: my-backend-app:latest + container_name: backend + labels: + - "traefik.enable=true" + - "traefik.http.routers.backend.rule=Host(`backend.localhost`)" + - "traefik.http.routers.backend.entrypoints=web" + - "traefik.http.routers.backend.tls=true" + - "traefik.http.services.backend.loadbalancer.server.port=5000" # Port exposed by the backend app diff --git a/assets/dynamic.yml b/assets/dynamic.yml new file mode 100644 index 0000000..364eed8 --- /dev/null +++ b/assets/dynamic.yml @@ -0,0 +1,34 @@ +http: + routers: + request-router: + rule: "Host(`request.test`)" + service: request-service + entryPoints: + - web + services: + request-service: + loadBalancer: + servers: + - url: "http://seasoned.schleppe:5000" + weight: 100 + passHostHeader: true + mirrored-service: + mirroring: + service: example-service + mirrorBody: true + maxBodySize: 1024 + mirrors: + - name: mirror1 + percent: 10 + url: http://example.com + - name: mirror2 + percent: 20 + url: http://example.org + healthCheck: + service: "request-service" + + middlewares: + redirect-to-https: + redirectScheme: + permanent: false + scheme: https diff --git a/client/etcd/etcd.go b/client/etcd/etcd.go new file mode 100644 index 0000000..be64e14 --- /dev/null +++ b/client/etcd/etcd.go @@ -0,0 +1,129 @@ +package etcd + +import ( + "context" + "fmt" + "log" + "log/slog" + "os" + "strings" + "time" + + "github.com/pkg/errors" + etcd "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc/connectivity" +) + +type EtcdManager struct { + client *etcd.Client +} + +type EtcdPacket struct { + Key string + Value string +} + +func NewClient() (*EtcdManager, error) { + fmt.Println("setting up etcd client") + + endpoints := []string{"localhost:2379"} + if value, exists := os.LookupEnv("ETCD_ENDPOINTS"); exists { + endpoints = strings.Split(value, ",") + } + + client, err := etcd.New(etcd.Config{ + Endpoints: endpoints, + DialTimeout: 30 * time.Second, + }) + if err != nil { + return nil, errors.Wrap(err, "could not create etcd manager") + } + + mgr := &EtcdManager{client} + + err = mgr.runWithTimeout(func(e *EtcdManager, ctx context.Context) error { + etcdStatus, err := e.client.Status(ctx, e.client.Endpoints()[0]) + if err != nil { + return fmt.Errorf("could not get etcd status: %v", err) + } + if len(etcdStatus.Errors) > 0 { + return fmt.Errorf("etcd server has errors: %v", etcdStatus.Errors) + } + + connState := e.client.ActiveConnection().GetState() + if connState != connectivity.Ready && connState != connectivity.Idle { + return fmt.Errorf("etcd connection is in unexpecetd state: %s", + e.client.ActiveConnection().GetState()) + } + return err + }) + + if err != nil { + return nil, err + } + + return mgr, nil +} + +func (e *EtcdManager) runWithTimeout(runnable func(e *EtcdManager, ctx context.Context) error) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return runnable(e, ctx) +} + +func runWithTimeoutReturning[R any]( + e *EtcdManager, + runnable func(e *EtcdManager, ctx context.Context) (R, error), +) (R, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return runnable(e, ctx) +} + +func (e *EtcdManager) Put(key string, val string, opts ...etcd.OpOption) error { + log.Println("etcd: Putting key", key, val) + return e.runWithTimeout(func(e *EtcdManager, ctx context.Context) error { + _, err := e.client.Put(ctx, key, val, opts...) + return err + }) +} + +func (e *EtcdManager) Get(key string, opts ...etcd.OpOption) (string, error) { + log.Println("etcd: Getting key", key) + return runWithTimeoutReturning(e, func(e *EtcdManager, ctx context.Context) (string, error) { + resp, err := e.client.Get(ctx, key, opts...) + if err != nil { + return "", nil + } + return string(resp.Kvs[0].Value), nil + }) +} + +func (e *EtcdManager) Remove(key string) error { + log.Println("etcd: Removing key", key) + return e.runWithTimeout(func(e *EtcdManager, ctx context.Context) error { + _, err := e.client.Delete(ctx, key) + return err + }) +} + +func RemoveDuplicatePackets(packets *[]EtcdPacket) { + keys := make(map[string]bool) + list := []EtcdPacket{} + keysRm := []string{} + + for _, entry := range *packets { + if _, ok := keys[entry.Key]; !ok { + keys[entry.Key] = true + list = append(list, entry) + } else { + keysRm = append(keysRm, entry.Key) + } + } + + if len(keysRm) > 0 { + slog.Warn(fmt.Sprintf("keys squashed: %s", keysRm)) + } + + *packets = list +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..7ff237e --- /dev/null +++ b/config/config.go @@ -0,0 +1,26 @@ +package config + +import ( + "log" + + "github.com/joho/godotenv" + "github.com/kelseyhightower/envconfig" +) + +// Config contains environment variables. +type Config struct { + EtcdEndpoint string `envconfig:"ETCD_ENDPOINTS"` +} + +// LoadConfig reads environment variables, populates and returns Config. +func LoadConfig() (*Config, error) { + if err := godotenv.Load(); err != nil { + log.Println("No .env file found") + } + + var c Config + + err := envconfig.Process("", &c) + + return &c, err +} diff --git a/converter/json.go b/converter/json.go new file mode 100644 index 0000000..a6aac1f --- /dev/null +++ b/converter/json.go @@ -0,0 +1,18 @@ +package converter + +import ( + "encoding/json" + "log" + + "github.com/traefik/traefik/v3/pkg/config/dynamic" +) + +func TraefikToJSON(config *dynamic.Configuration) map[string]interface{} { + var data map[string]interface{} + jsonData, _ := json.Marshal(config) + if err := json.Unmarshal(jsonData, &data); err != nil { + log.Printf("failed to unmarshal JSON: %w", err) + } + + return data +} diff --git a/converter/yaml.go b/converter/yaml.go new file mode 100644 index 0000000..babdaad --- /dev/null +++ b/converter/yaml.go @@ -0,0 +1,28 @@ +package converter + +import ( + "os" + + "github.com/traefik/traefik/v3/pkg/config/dynamic" + "gopkg.in/yaml.v3" +) + +func TraefikFromYaml(filePath string) (*dynamic.Configuration, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + var cfg dynamic.Configuration + decoder := yaml.NewDecoder(file) + if err := decoder.Decode(&cfg); err != nil { + return nil, err + } + + return &cfg, nil +} + +func TraefikToYaml(config *dynamic.Configuration) string { + return "" +} diff --git a/generator/docker.go b/generator/docker.go new file mode 100644 index 0000000..08ffd2a --- /dev/null +++ b/generator/docker.go @@ -0,0 +1,65 @@ +package generator + +import ( + "context" + "fmt" + "strings" + + "github.com/compose-spec/compose-go/v2/cli" + "github.com/compose-spec/compose-go/v2/types" + "github.com/kevinmidboe/traefik-etcd-advertiser/client/etcd" +) + +func dockerLabelToEtcdKey(key string) string { + return strings.ReplaceAll(key, ".", "/") +} + +func ParseDockerCompose(composeFilePath string) (*types.Project, error) { + ctx := context.Background() + + options, err := cli.NewProjectOptions( + []string{composeFilePath}, + ) + if err != nil { + return nil, err + } + + project, err := options.LoadProject(ctx) + if err != nil { + return nil, err + } + + return project, nil +} + +func createPacket2(config *types.Project) []etcd.EtcdPacket { + blocks := []etcd.EtcdPacket{} + + fmt.Println("DockerToEtcd") + if config.Services == nil || len(config.Services) < 1 { + fmt.Println("services not found - skipping") + return blocks + } + + for serviceName, _ := range config.Services { + if config.Services[serviceName].Labels == nil { + fmt.Println("sevice, but no labels found - continuing") + continue + } + + for key, val := range config.Services[serviceName].Labels { + blocks = append(blocks, etcd.EtcdPacket{ + Key: dockerLabelToEtcdKey(key), + Value: val, + }) + } + } + + return blocks +} + +func DockerToEtcd(config *types.Project, packetList *[]etcd.EtcdPacket) { + items := createPacket2(config) + + *packetList = append(*packetList, items...) +} diff --git a/generator/traefik.go b/generator/traefik.go new file mode 100644 index 0000000..5896f6d --- /dev/null +++ b/generator/traefik.go @@ -0,0 +1,83 @@ +package generator + +import ( + "fmt" + "log/slog" + "math" + "strconv" + + "github.com/kevinmidboe/traefik-etcd-advertiser/client/etcd" + "github.com/kevinmidboe/traefik-etcd-advertiser/converter" + "github.com/traefik/traefik/v3/pkg/config/dynamic" +) + +const traefikPrefix = "traefik" + +func isKnownGenericType(value interface{}) bool { + switch v := value.(type) { + case string: + return true + case float64: + return true + case bool: + return true + default: + slog.Debug(fmt.Sprintf("found unknown generic %s\n", v)) + } + + return false +} + +func convertToGeneric(value interface{}) string { + switch v := value.(type) { + case string: + return v + case float64: + return fmt.Sprintf("%d", int(math.Floor(v))) + case bool: + return strconv.FormatBool(v) + } + + return "unknown type" +} + +// recursively walks the JSON object and creates internal +// `etcdPackets` per leaf node, returns list of packets. +func createPacket(item interface{}, parentKey string) []etcd.EtcdPacket { + blocks := []etcd.EtcdPacket{} + + switch itemD := item.(type) { + // input is JSON object + case map[string]interface{}: + for key, value := range itemD { + // check for generic value type vs nested object, + // either create block or recursively call obj again + + if isKnownGenericType(value) { + blocks = append(blocks, etcd.EtcdPacket{ + Key: parentKey + "/" + key, + Value: convertToGeneric(value), + }) + } else { + blocks = append(blocks, createPacket(itemD[key], fmt.Sprintf("%s/%s", parentKey, key))...) + } + } + // input is JSON list + case []interface{}: + for i, item := range itemD { + blocks = append(blocks, createPacket(item, parentKey+"/"+strconv.Itoa(i))...) + } + } + + return blocks +} + +func TraefikToEtcd(config *dynamic.Configuration, packetList *[]etcd.EtcdPacket) { + // always convert to json before converting to etcd + data := converter.TraefikToJSON(config) + + // generate list of etcd commands + items := createPacket(data, traefikPrefix) + *packetList = append(*packetList, items...) +} + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3f679a2 --- /dev/null +++ b/go.mod @@ -0,0 +1,97 @@ +module github.com/kevinmidboe/traefik-etcd-advertiser + +go 1.23.0 + +toolchain go1.23.3 + +require ( + github.com/compose-spec/compose-go/v2 v2.4.7 + github.com/joho/godotenv v1.5.1 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/pkg/errors v0.9.1 + github.com/traefik/traefik/v3 v3.3.2 + go.etcd.io/etcd/client/v3 v3.5.17 + google.golang.org/grpc v1.67.1 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.32.0 +) + +require ( + github.com/aws/aws-sdk-go v1.44.327 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-acme/lego/v4 v4.21.0 // indirect + github.com/go-jose/go-jose/v4 v4.0.4 // indirect + github.com/go-kit/log v0.2.1 // indirect + github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-github/v28 v28.1.1 // indirect + github.com/google/go-querystring v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/http-wasm/http-wasm-host-go v0.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-shellwords v1.0.12 // indirect + github.com/miekg/dns v1.1.62 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/rs/zerolog v1.33.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/traefik/paerser v0.2.1 // indirect + github.com/unrolled/render v1.0.2 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + go.etcd.io/etcd/api/v3 v3.5.17 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.17 // indirect + go.opentelemetry.io/otel v1.32.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect + go.opentelemetry.io/otel/log v0.8.0 // indirect + go.opentelemetry.io/otel/metric v1.32.0 // indirect + go.opentelemetry.io/otel/sdk v1.32.0 // indirect + go.opentelemetry.io/otel/sdk/log v0.8.0 // indirect + go.opentelemetry.io/otel/trace v1.32.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.26.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/exp v0.0.0-20241210194714-1829a127f884 // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/tools v0.28.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/protobuf v1.35.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/apimachinery v0.32.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/main.go b/main.go new file mode 100644 index 0000000..2ce9826 --- /dev/null +++ b/main.go @@ -0,0 +1,66 @@ +package main + +import ( + "fmt" + "log" + "os" + "strings" + + "github.com/kevinmidboe/traefik-etcd-advertiser/client/etcd" + "github.com/kevinmidboe/traefik-etcd-advertiser/config" + "github.com/kevinmidboe/traefik-etcd-advertiser/converter" + "github.com/kevinmidboe/traefik-etcd-advertiser/generator" +) + +func getArgvFilename() string { + if len(os.Args) < 2 { + log.Fatalf("Usage: %s \n", os.Args[0]) + } + + filename := os.Args[1] + return filename +} + +func main() { + _, err := config.LoadConfig() + if err != nil { + log.Println("Error from config loader", err) + } + + // setup etcd client + // etcdManager, err := etcd.NewClient() + if err != nil { + panic(err) + } + + var packets []etcd.EtcdPacket + + // parse traefik config from file + filename := getArgvFilename() + if strings.Contains(filename, "docker-compose.yml") { + // build etcd packets from docker-compose config + dockerConfig, err := generator.ParseDockerCompose(filename) + if err != nil { + log.Fatal(err) + } + + // generator.PrintLabels(labels) + fmt.Println("compose") + log.Println(dockerConfig) + // generator.TraefikToEtcd(traefikConfig, &packets) + + } else { + // build etcd packets from traefik config + traefikConfig, err := converter.TraefikFromYaml(filename) + if err != nil { + log.Fatalf("Error loading traefik YAML config file: %v\n", err) + } + + generator.TraefikToEtcd(traefikConfig, &packets) + } + + for _, packet := range packets { + log.Println(packet) + // etcdManager.Put(packet.Key, packet.Value) + } +}