From b56c3d1e29aca35e60f17d2f78a4d4f9ed07bc5b Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sat, 10 Dec 2022 15:27:24 +0100 Subject: [PATCH] App for fetching and uploading images to cloud bucket storage --- .env.example | 3 + .gitignore | 23 ++++++ Makefile | 26 +++++++ README.md | 36 +++++++++ clients/gcs/gcs.go | 83 ++++++++++++++++++++ clients/gcs/types.go | 23 ++++++ cmd/server/main.go | 32 ++++++++ config/config.go | 28 +++++++ go.mod | 35 +++++++++ go.sum | 156 ++++++++++++++++++++++++++++++++++++++ image/image.go | 24 ++++++ server/handler/error.go | 24 ++++++ server/handler/healthz.go | 11 +++ server/handler/images.go | 119 +++++++++++++++++++++++++++++ server/router.go | 19 +++++ server/server.go | 85 +++++++++++++++++++++ util/geturl.go | 8 ++ util/hash.go | 14 ++++ 18 files changed, 749 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 clients/gcs/gcs.go create mode 100644 clients/gcs/types.go create mode 100644 cmd/server/main.go create mode 100644 config/config.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 image/image.go create mode 100644 server/handler/error.go create mode 100644 server/handler/healthz.go create mode 100644 server/handler/images.go create mode 100644 server/router.go create mode 100644 server/server.go create mode 100644 util/geturl.go create mode 100644 util/hash.go diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d878d44 --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +PORT=8000 +HOSTNAME= +GCS_BUCKET= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a546697 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# Editors +/.idea/ +*.vim + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +main + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (comment below to include vendor) +vendor/ + +# Env +.env diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..47149b2 --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +.PHONY: build install server + +PROJECT_NAME=$(shell basename $(CURDIR)) + +## build: build the application +build: + export GO111MODULE="on"; \ + go mod download; \ + go mod vendor; \ + CGO_ENABLED=0 go build -a -ldflags '-s' -installsuffix cgo -o main cmd/server/main.go + +## install: fetches go modules +install: + export GO111MODULE="on"; \ + go mod tidy; \ + go mod download \ + +## server: runs the server with -race +server: + export GO111MODULE="on"; \ + go run cmd/server/main.go + +## modd: Monitors the directory, and recompiles your app every time a file changes. Also runs tests. +## (To install modd, run: go get github.com/cortesi/modd/cmd/modd) +modd: + modd \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..6ba14af --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# Planetposen image proxy + +App for fetching and uploading images to cloud bucket storage. Should also be a on-the-fly image formatting proxy. + +## Setup + +From source: + +1. `git clone https://github.com/kevinmidboe/planetposen-images` +2. `cp .env.example .env` +3. Update variables in `.env` file +4. `make install` + +## Run + +Run api from command line with: + +```bash +make server +``` + +Run as docker container using: + +```bash +(sudo) docker run -d --name planetposen-images -p 8000:8000 \ + -e PORT=8000 + -e HOSTNAME=planet.schleppe.cloud + -e GCS_BUCKET=planetposen-images +``` + +or + +```bash +(sudo) docker run -d --name planetposen-images -p 8000:8000 --env-file .env planetposen-images +``` + diff --git a/clients/gcs/gcs.go b/clients/gcs/gcs.go new file mode 100644 index 0000000..a8f80e3 --- /dev/null +++ b/clients/gcs/gcs.go @@ -0,0 +1,83 @@ +package gcs + +import ( + "cloud.google.com/go/storage" + "context" + "github.com/kevinmidboe/planetposen-images/util" + "google.golang.org/api/iterator" + "path/filepath" + // "errors" + "fmt" + // "github.com/dbmedialab/dearheart/event" + // "github.com/dbmedialab/dearheart/util" + // "path/filepath" + // "time" +) + +// Client represents a GCS client with the functions that *we* need. +type Client interface { + FileWriter(ctx context.Context, filename string) (*storage.Writer, EncodedPath, error) + FileReader(ctx context.Context, path EncodedPath) (*storage.Reader, error) + FileLister(ctx context.Context) ([]string, error) +} + +// NewClient instantiates our default GCS client. +func NewClient(ctx context.Context, bucketName string) (Client, error) { + c, err := storage.NewClient(ctx) + if err != nil { + return nil, err + } + + return &clientImpl{ + BucketName: bucketName, + GCSClient: c, + }, nil +} + +type clientImpl struct { + BucketName string + GCSClient *storage.Client +} + +func (c *clientImpl) FileWriter(ctx context.Context, filename string) (writer *storage.Writer, path EncodedPath, err error) { + extension := filepath.Ext(filename) + rawPath := RawPath(util.Hash(filename) + extension) + bucket := c.GCSClient.Bucket(c.BucketName) + object := bucket.Object(string(rawPath)) + + path = rawPath.Encode() + writer = object.NewWriter(ctx) + return +} + +func (c *clientImpl) FileReader(ctx context.Context, path EncodedPath) (reader *storage.Reader, err error) { + decoded, err := path.Decode() + if err != nil { + return nil, fmt.Errorf("invalid path %s", err) + } + + bucket := c.GCSClient.Bucket(c.BucketName) + object := bucket.Object(string(decoded)) + + reader, err = object.NewReader(ctx) + return +} + +func (c *clientImpl) FileLister(ctx context.Context) (files []string, err error) { + query := &storage.Query{Prefix: ""} + var names []string + bucket := c.GCSClient.Bucket(c.BucketName) + it := bucket.Objects(ctx, query) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("error from query %s", err) + } + names = append(names, attrs.Name) + } + + return names, nil +} diff --git a/clients/gcs/types.go b/clients/gcs/types.go new file mode 100644 index 0000000..7217709 --- /dev/null +++ b/clients/gcs/types.go @@ -0,0 +1,23 @@ +package gcs + +import "encoding/base64" + +// RawPath is a raw path to a file in GCS +type RawPath string + +// EncodedPath is base64-encoded version of RawPath +type EncodedPath string + +// Encode base64-encodes raw paths +func (p RawPath) Encode() EncodedPath { + return EncodedPath(base64.StdEncoding.EncodeToString([]byte(p))) +} + +// Decode base64-decodes encoded paths +func (p EncodedPath) Decode() (RawPath, error) { + decoded, err := base64.StdEncoding.DecodeString(string(p)) + if err != nil { + return "", err + } + return RawPath(decoded), nil +} diff --git a/cmd/server/main.go b/cmd/server/main.go new file mode 100644 index 0000000..7f504f3 --- /dev/null +++ b/cmd/server/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + + "github.com/kevinmidboe/planetposen-images/config" + "github.com/kevinmidboe/planetposen-images/server" + log "github.com/sirupsen/logrus" +) + +func main() { + // log.SetFormatter(logrustic.NewFormatter("planetposen-images")) + + log.Info("Starting...") + + ctx := context.Background() + config, err := config.LoadConfig() + + if err != nil { + log.Fatal(err.Error()) + } + + var s server.Server + + if err := s.Create(ctx, config); err != nil { + log.Fatal(err.Error()) + } + + if err := s.Serve(ctx); err != nil { + log.Fatal(err.Error()) + } +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..b3f601d --- /dev/null +++ b/config/config.go @@ -0,0 +1,28 @@ +// Package config handles environment variables. +package config + +import ( + "github.com/joho/godotenv" + "github.com/kelseyhightower/envconfig" + log "github.com/sirupsen/logrus" +) + +// Config contains environment variables. +type Config struct { + Hostname string `envconfig:"HOSTNAME"` + GCSBucket string `envconfig:"GCS_BUCKET" default:"p"` + Port string `envconfig:"PORT" default:"8000"` +} + +// LoadConfig reads environment variables, populates and returns Config. +func LoadConfig() (*Config, error) { + if err := godotenv.Load(); err != nil { + log.Info("No .env file found") + } + + var c Config + + err := envconfig.Process("", &c) + + return &c, err +} \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3c02466 --- /dev/null +++ b/go.mod @@ -0,0 +1,35 @@ +module github.com/kevinmidboe/planetposen-images + +go 1.19 + +require ( + cloud.google.com/go/storage v1.28.1 + github.com/gorilla/mux v1.8.0 + github.com/joho/godotenv v1.4.0 + github.com/kelseyhightower/envconfig v1.4.0 + github.com/sirupsen/logrus v1.9.0 + google.golang.org/api v0.103.0 +) + +require ( + cloud.google.com/go v0.105.0 // indirect + cloud.google.com/go/compute v1.12.1 // indirect + cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/iam v0.7.0 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect + golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect + golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect + golang.org/x/text v0.4.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect + google.golang.org/grpc v1.50.1 // indirect + google.golang.org/protobuf v1.28.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..990d7e9 --- /dev/null +++ b/go.sum @@ -0,0 +1,156 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= +cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg= +github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c h1:S34D59DS2GWOEwWNt4fYmTcFrtlOgukG2k9WsomZ7tg= +google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/image/image.go b/image/image.go new file mode 100644 index 0000000..9c8ba61 --- /dev/null +++ b/image/image.go @@ -0,0 +1,24 @@ +package image + +import ( + "github.com/kevinmidboe/planetposen-images/util" +) + +// MessageImage is a representation of a single image in the database +type Image struct { + Path string `json:"path"` + URL string `json:"url,omitempty"` +} + +// GetURL gets URL of the image, also in cases where MessageImage.URL is not defined. +func (mi *Image) GetURL(hostname string) string { + if mi.URL != "" { + return mi.URL + } + return util.ImageURL(hostname, mi.Path) +} + +type PostImageData struct { + // PageTitle string + Filename string +} diff --git a/server/handler/error.go b/server/handler/error.go new file mode 100644 index 0000000..0dd9371 --- /dev/null +++ b/server/handler/error.go @@ -0,0 +1,24 @@ +package handler + +import ( + "encoding/json" + "net/http" + + log "github.com/sirupsen/logrus" +) + +// handleError - Logs the error (if shouldLog is true), and outputs the error message (msg) +func handleError(w http.ResponseWriter, err error, msg string, statusCode int, shouldLog bool) { + if shouldLog { + log.WithField("err", err).Error(msg) + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + errorJSON, _ := json.Marshal(struct { + Error string `json:"error"` + }{ + Error: msg, + }) + w.Write(errorJSON) +} \ No newline at end of file diff --git a/server/handler/healthz.go b/server/handler/healthz.go new file mode 100644 index 0000000..3ff0a91 --- /dev/null +++ b/server/handler/healthz.go @@ -0,0 +1,11 @@ +package handler + +import "net/http" + +// Healthz is used for our readiness and liveness probes. +// GET /_healthz +// Responds: 200 +func Healthz(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte(http.StatusText(http.StatusOK))) +} diff --git a/server/handler/images.go b/server/handler/images.go new file mode 100644 index 0000000..45436db --- /dev/null +++ b/server/handler/images.go @@ -0,0 +1,119 @@ +package handler + +import ( + "encoding/json" + "github.com/kevinmidboe/planetposen-images/util" + "strings" + + // "github.com/sirupsen/logrus" + // "encoding/json" + "fmt" + "github.com/kevinmidboe/planetposen-images/clients/gcs" + "github.com/kevinmidboe/planetposen-images/image" + // "github.com/dbmedialab/dearheart/event" + // "github.com/dbmedialab/dearheart/server/internal/serverutils" + "github.com/gorilla/mux" + "io" + "net/http" + "path/filepath" + // "strconv" + // "strings" +) + +// UploadImages takes a request with file form and uploads the content to GCS +func UploadImages(hostname string, gcsClient gcs.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // Get initial protocol data + ctx := r.Context() + + file, fileHeader, err := r.FormFile("file") + if err != nil { + handleError(w, err, "unable to handle file", http.StatusBadRequest, true) + return + } + var maxSize int64 = 10 * 1024 * 1024 * 1024 + if fileHeader.Size > maxSize { + handleError(w, nil, "File sized %d, larger than the max of %d\", fileHeader.Size, maxSize", http.StatusBadRequest, true) + return + } + + filename := strings.ReplaceAll(fileHeader.Filename, "/", "-") + defer file.Close() + + writer, path, err := gcsClient.FileWriter(ctx, filename) + if err != nil { + handleError(w, err, "File unable to write file to gcs", http.StatusServiceUnavailable, true) + return + } + defer writer.Close() + + _, err = io.Copy(writer, file) + if err != nil { + handleError(w, err, "Error copying file to GCS", http.StatusInternalServerError, true) + } + + finalURL := util.ImageURL(hostname, string(path)) + responseStruct := image.Image{ + Path: string(path), + URL: finalURL, + } + + responseData, _ := json.Marshal(responseStruct) + _, _ = w.Write(responseData) + } +} + +// FetchImage gets a single image from GCS +func FetchImage(gcsClient gcs.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + path := gcs.EncodedPath(mux.Vars(r)["path"]) + if path == "" { + handleError(w, nil, "missing image path ", http.StatusBadRequest, true) + return + } + + reader, err := gcsClient.FileReader(ctx, path) + if err != nil { + handleError(w, err, "error from gcs file reader ", http.StatusBadRequest, true) + return + } + defer reader.Close() + + // we can ignore the error, because we've already verified the path decodes + filename, _ := path.Decode() + extension := filepath.Ext(string(filename)) + if extension != "" { + w.Header().Set("Content-Type", fmt.Sprintf("image/%s", extension[1:])) + } + + _, err = io.Copy(w, reader) + if err != nil { + handleError(w, err, "Couldn't copy the file from GCS ", http.StatusInternalServerError, true) + } + } +} + +func ListImages(gcsClient gcs.Client) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + files, err := gcsClient.FileLister(ctx) + if err != nil { + handleError(w, err, "error from gcs file lister ", http.StatusBadRequest, true) + return + } + + w.WriteHeader(http.StatusOK) + responseJSON, _ := json.Marshal(struct { + Message string `json:"message"` + Success bool `json:"success"` + Files []string `json:"files"` + }{ + Message: "Google storage bucket contents", + Success: true, + Files: files, + }) + w.Write(responseJSON) + } +} diff --git a/server/router.go b/server/router.go new file mode 100644 index 0000000..8f09c58 --- /dev/null +++ b/server/router.go @@ -0,0 +1,19 @@ +package server + +import ( + "github.com/kevinmidboe/planetposen-images/server/handler" +) + +const v1API string = "/api/v1" + +func (s *Server) setupRoutes() { + s.Router.HandleFunc("/_healthz", handler.Healthz).Methods("GET").Name("Health") + + api := s.Router.PathPrefix(v1API).Subrouter() + api.HandleFunc("/images", handler.UploadImages(s.Config.Hostname, s.GCSClient)).Methods("POST").Name("UploadImages") + api.HandleFunc("/images", handler.ListImages(s.GCSClient)).Methods("GET").Name("ListImages") + + // Raw image fetcher + api.HandleFunc("/images/{path}", handler.FetchImage(s.GCSClient)).Methods("GET").Name("FetchImage") + +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..070a27a --- /dev/null +++ b/server/server.go @@ -0,0 +1,85 @@ +// Package server provides functionality to easily set up an HTTTP server. +// +// Clients: +// Database +package server + +import ( + "context" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + + "github.com/gorilla/mux" + "github.com/kevinmidboe/planetposen-images/clients/gcs" + "github.com/kevinmidboe/planetposen-images/config" + log "github.com/sirupsen/logrus" +) + +// Server holds the HTTP server, router, config and all clients. +type Server struct { + Config *config.Config + HTTP *http.Server + Router *mux.Router + GCSClient gcs.Client +} + +// Create sets up the HTTP server, router and all clients. +// Returns an error if an error occurs. +func (s *Server) Create(ctx context.Context, config *config.Config) error { + // metrics.RegisterPrometheusCollectors() + + s.Config = config + s.Router = mux.NewRouter() + s.HTTP = &http.Server{ + Addr: fmt.Sprintf(":%s", s.Config.Port), + Handler: s.Router, + } + + gcsClient, err := gcs.NewClient(ctx, config.GCSBucket) + if err != nil { + return err + } + + s.GCSClient = gcsClient + + s.setupRoutes() + + return nil +} + +// Serve tells the server to start listening and serve HTTP requests. +// It also makes sure that the server gracefully shuts down on exit. +// Returns an error if an error occurs. +func (s *Server) Serve(ctx context.Context) error { + // closer, err := trace.InitGlobalTracer(s.Config) + + // if err != nil { + // return err + // } + + // defer closer.Close() + + go func(ctx context.Context, s *Server) { + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + + <-stop + + log.Info("Shutdown signal received") + + if err := s.HTTP.Shutdown(ctx); err != nil { + log.Error(err.Error()) + } + }(ctx, s) + + log.Infof("Ready at: %s", s.Config.Port) + + if err := s.HTTP.ListenAndServe(); err != http.ErrServerClosed { + log.Fatalf(err.Error()) + } + + return nil +} diff --git a/util/geturl.go b/util/geturl.go new file mode 100644 index 0000000..4f14e28 --- /dev/null +++ b/util/geturl.go @@ -0,0 +1,8 @@ +package util + +import "fmt" + +// ImageURL creates imageURL from hostname and image name +func ImageURL(hostname, name string) string { + return fmt.Sprintf("https://%s/api/v1/images/%s", hostname, name) +} diff --git a/util/hash.go b/util/hash.go new file mode 100644 index 0000000..35620dd --- /dev/null +++ b/util/hash.go @@ -0,0 +1,14 @@ +package util + +import ( + "crypto/sha1" + "fmt" +) + +// Hash hashes a string using sha1 +func Hash(s string) string { + hash := sha1.New() + _, _ = hash.Write([]byte(s)) + bs := hash.Sum(nil) + return fmt.Sprintf("%x", bs) +}