From 576a4e15794685788a5d42895d97dc9ad330b40e Mon Sep 17 00:00:00 2001 From: Kevin Midboe Date: Sat, 17 Dec 2022 14:22:56 +0100 Subject: [PATCH] Implemented logger, write to file on disk --- clients/gcs/gcs.go | 9 ++-- cmd/server/main.go | 14 +++-- logger/logger.go | 110 +++++++++++++++++++++++++++++++++++++++ server/handler/error.go | 24 +++++++-- server/handler/images.go | 16 ++++-- server/server.go | 15 +++--- 6 files changed, 160 insertions(+), 28 deletions(-) create mode 100644 logger/logger.go diff --git a/clients/gcs/gcs.go b/clients/gcs/gcs.go index a8f80e3..eb1096a 100644 --- a/clients/gcs/gcs.go +++ b/clients/gcs/gcs.go @@ -3,15 +3,10 @@ package gcs import ( "cloud.google.com/go/storage" "context" + "fmt" "github.com/kevinmidboe/planetposen-images/util" "google.golang.org/api/iterator" "path/filepath" - // "errors" - "fmt" - // "github.com/dbmedialab/dearheart/event" - // "github.com/dbmedialab/dearheart/util" - // "path/filepath" - // "time" ) // Client represents a GCS client with the functions that *we* need. @@ -73,9 +68,11 @@ func (c *clientImpl) FileLister(ctx context.Context) (files []string, err error) if err == iterator.Done { break } + if err != nil { return nil, fmt.Errorf("error from query %s", err) } + names = append(names, attrs.Name) } diff --git a/cmd/server/main.go b/cmd/server/main.go index 7f504f3..6ad5efb 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,31 +2,29 @@ package main import ( "context" - "github.com/kevinmidboe/planetposen-images/config" + log "github.com/kevinmidboe/planetposen-images/logger" "github.com/kevinmidboe/planetposen-images/server" - log "github.com/sirupsen/logrus" ) func main() { - // log.SetFormatter(logrustic.NewFormatter("planetposen-images")) - - log.Info("Starting...") + logger := log.InitLogger() + logger.Info("Starting...") ctx := context.Background() config, err := config.LoadConfig() if err != nil { - log.Fatal(err.Error()) + logger.Fatal(err) } var s server.Server if err := s.Create(ctx, config); err != nil { - log.Fatal(err.Error()) + logger.Fatal(err) } if err := s.Serve(ctx); err != nil { - log.Fatal(err.Error()) + logger.Fatal(err) } } diff --git a/logger/logger.go b/logger/logger.go new file mode 100644 index 0000000..23ae7d5 --- /dev/null +++ b/logger/logger.go @@ -0,0 +1,110 @@ +package logger + +import ( + "github.com/sirupsen/logrus" + "os" + "time" +) + +// Event stores messages to log later, from our standard interface +type Event struct { + id int + message string +} + +// ApplicationLogger enforces specific log message formats +type ApplicationLogger struct { + *logrus.Logger +} + +// NewLogger initializes the standard logger +func InitLogger() *ApplicationLogger { + var baseLogger = logrus.New() + var applicationLogger = &ApplicationLogger{baseLogger} + + f, _ := os.OpenFile("/var/log/planetposen_logs/planetposen-images.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + + applicationLogger.SetOutput(f) + applicationLogger.Formatter = &logrus.JSONFormatter{ + TimestampFormat: time.RFC3339Nano, + FieldMap: logrus.FieldMap{ + logrus.FieldKeyMsg: "message", + logrus.FieldKeyTime: "@timestamp", + }, + } + + return applicationLogger +} + +// Declare variables to store log messages as new Events +var ( + infoMessage = Event{1, "%s"} + invalidArgMessage = Event{2, "Invalid arg: %s"} + invalidArgValueMessage = Event{3, "Invalid value for argument: %s: %v"} + missingArgMessage = Event{4, "Missing arg: %s"} +) + +func (l *ApplicationLogger) Info(argumentName string) { + l.WithFields(logrus.Fields{ + "application": "planetposen-images", + }).Infof(infoMessage.message, argumentName) +} + +func (l *ApplicationLogger) InfoWithPath(message string, path string) { + l.WithFields(logrus.Fields{ + "application": "planetposen-images", + "path": path, + }).Info(message) +} + +func (l *ApplicationLogger) InfoWithFilename(message string, filename string) { + l.WithFields(logrus.Fields{ + "application": "planetposen-images", + "filename": filename, + }).Info(message) +} + +func (l *ApplicationLogger) UploadSuccessMessage(path string, url string) { + l.WithFields(logrus.Fields{ + "application": "planetposen-images", + "path": path, + "url": url, + }).Info("Uploaded and available") +} + +func (l *ApplicationLogger) Error(message string, error error) { + l.WithFields(logrus.Fields{ + "application": "planetposen-images", + "err": error, + }).Error(message) +} + +func (l *ApplicationLogger) Fatal(error error) { + l.WithFields(logrus.Fields{ + "application": "planetposen-images", + "err": error, + }).Fatal("Fatal application exception!") +} + +func (l *ApplicationLogger) GoogleApiError(message string, error error) { + l.WithFields(logrus.Fields{ + "application": "planetposen-images", + "go_package": "cloud.google.com/go/storage", + "err": error, + }).Error(message) +} + +// InvalidArg is a standard error message +func (l *ApplicationLogger) InvalidArg(argumentName string) { + l.Errorf(invalidArgMessage.message, argumentName) +} + +// InvalidArgValue is a standard error message +func (l *ApplicationLogger) InvalidArgValue(argumentName string, argumentValue string) { + l.Errorf(invalidArgValueMessage.message, argumentName, argumentValue) +} + +// MissingArg is a standard error message +func (l *ApplicationLogger) MissingArg(argumentName string) { + l.Errorf(missingArgMessage.message, argumentName) +} diff --git a/server/handler/error.go b/server/handler/error.go index 0dd9371..2a165af 100644 --- a/server/handler/error.go +++ b/server/handler/error.go @@ -2,15 +2,16 @@ package handler import ( "encoding/json" + log "github.com/kevinmidboe/planetposen-images/logger" "net/http" - - log "github.com/sirupsen/logrus" ) +var logger = log.InitLogger() + // handleError - Logs the error (if shouldLog is true), and outputs the error message (msg) func handleError(w http.ResponseWriter, err error, msg string, statusCode int, shouldLog bool) { if shouldLog { - log.WithField("err", err).Error(msg) + logger.Error(msg, err) } w.Header().Set("Content-Type", "application/json") @@ -21,4 +22,19 @@ func handleError(w http.ResponseWriter, err error, msg string, statusCode int, s Error: msg, }) w.Write(errorJSON) -} \ No newline at end of file +} + +func handleGoogleApiError(w http.ResponseWriter, err error, msg string, statusCode int, shouldLog bool) { + if shouldLog { + logger.GoogleApiError(msg, err) + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + errorJSON, _ := json.Marshal(struct { + Error string `json:"error"` + }{ + Error: msg, + }) + w.Write(errorJSON) +} diff --git a/server/handler/images.go b/server/handler/images.go index 45436db..dd12399 100644 --- a/server/handler/images.go +++ b/server/handler/images.go @@ -40,16 +40,17 @@ func UploadImages(hostname string, gcsClient gcs.Client) http.HandlerFunc { filename := strings.ReplaceAll(fileHeader.Filename, "/", "-") defer file.Close() + logger.InfoWithFilename("uploading image with filename", filename) writer, path, err := gcsClient.FileWriter(ctx, filename) if err != nil { - handleError(w, err, "File unable to write file to gcs", http.StatusServiceUnavailable, true) + handleGoogleApiError(w, err, "File unable to write file to gcs", http.StatusServiceUnavailable, true) return } defer writer.Close() _, err = io.Copy(writer, file) if err != nil { - handleError(w, err, "Error copying file to GCS", http.StatusInternalServerError, true) + handleGoogleApiError(w, err, "Error copying file to GCS", http.StatusInternalServerError, true) } finalURL := util.ImageURL(hostname, string(path)) @@ -57,6 +58,7 @@ func UploadImages(hostname string, gcsClient gcs.Client) http.HandlerFunc { Path: string(path), URL: finalURL, } + logger.UploadSuccessMessage(string(path), finalURL) responseData, _ := json.Marshal(responseStruct) _, _ = w.Write(responseData) @@ -68,6 +70,9 @@ func FetchImage(gcsClient gcs.Client) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() path := gcs.EncodedPath(mux.Vars(r)["path"]) + + logger.InfoWithPath("Getting image", string(path)) + if path == "" { handleError(w, nil, "missing image path ", http.StatusBadRequest, true) return @@ -75,7 +80,7 @@ func FetchImage(gcsClient gcs.Client) http.HandlerFunc { reader, err := gcsClient.FileReader(ctx, path) if err != nil { - handleError(w, err, "error from gcs file reader ", http.StatusBadRequest, true) + handleGoogleApiError(w, err, "error from gcs file reader ", http.StatusBadRequest, true) return } defer reader.Close() @@ -87,6 +92,8 @@ func FetchImage(gcsClient gcs.Client) http.HandlerFunc { w.Header().Set("Content-Type", fmt.Sprintf("image/%s", extension[1:])) } + logger.InfoWithFilename("found and returning file from bucket", string(filename)) + _, err = io.Copy(w, reader) if err != nil { handleError(w, err, "Couldn't copy the file from GCS ", http.StatusInternalServerError, true) @@ -96,11 +103,12 @@ func FetchImage(gcsClient gcs.Client) http.HandlerFunc { func ListImages(gcsClient gcs.Client) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + logger.Info("Listing images") ctx := r.Context() files, err := gcsClient.FileLister(ctx) if err != nil { - handleError(w, err, "error from gcs file lister ", http.StatusBadRequest, true) + handleGoogleApiError(w, err, "error from gcs file lister ", http.StatusBadRequest, true) return } diff --git a/server/server.go b/server/server.go index 070a27a..84114d3 100644 --- a/server/server.go +++ b/server/server.go @@ -1,12 +1,14 @@ // Package server provides functionality to easily set up an HTTTP server. // // Clients: -// Database +// +// Database package server import ( "context" "fmt" + log "github.com/kevinmidboe/planetposen-images/logger" "net/http" "os" "os/signal" @@ -15,7 +17,6 @@ import ( "github.com/gorilla/mux" "github.com/kevinmidboe/planetposen-images/clients/gcs" "github.com/kevinmidboe/planetposen-images/config" - log "github.com/sirupsen/logrus" ) // Server holds the HTTP server, router, config and all clients. @@ -26,6 +27,8 @@ type Server struct { GCSClient gcs.Client } +var logger = log.InitLogger() + // Create sets up the HTTP server, router and all clients. // Returns an error if an error occurs. func (s *Server) Create(ctx context.Context, config *config.Config) error { @@ -68,17 +71,17 @@ func (s *Server) Serve(ctx context.Context) error { <-stop - log.Info("Shutdown signal received") + logger.Info("Shutdown signal received") if err := s.HTTP.Shutdown(ctx); err != nil { - log.Error(err.Error()) + logger.Error("Error causing shutdown", err) } }(ctx, s) - log.Infof("Ready at: %s", s.Config.Port) + logger.Info("Ready at: " + s.Config.Port) if err := s.HTTP.ListenAndServe(); err != http.ErrServerClosed { - log.Fatalf(err.Error()) + logger.Fatal(err) } return nil