Skip to content

Commit

Permalink
Use json logs
Browse files Browse the repository at this point in the history
  • Loading branch information
subtle-byte committed Jul 15, 2023
1 parent 6756438 commit 253f0f1
Show file tree
Hide file tree
Showing 19 changed files with 274 additions and 135 deletions.
4 changes: 2 additions & 2 deletions Dockerfile
@@ -1,5 +1,5 @@
# Use rc2 with fix https://github.com/golang/go/commit/08458804fb6591397fe1c58f4e04fd490e70fbcb
FROM golang:1.21rc2
# Use rc3 with fix https://github.com/golang/go/commit/08458804fb6591397fe1c58f4e04fd490e70fbcb
FROM golang:1.21rc3

WORKDIR /src
ADD . .
Expand Down
4 changes: 3 additions & 1 deletion Makefile
Expand Up @@ -6,15 +6,17 @@ stop-db:

# DB is optional, if not provided, the service will be run without cache
run:
JSON_LOGS=false \
DB_CONN="postgres://postgres:password@localhost:54329/?sslmode=disable" \
DEBUG_TOKEN="dt" \
MAX_REPO_SIZE_MB=100 \
MAX_CONCURRENT_WORK=2 \
go run ./cmd/server/main.go
go run ./cmd/server

run-in-docker:
docker build -t ghloc .
docker run --rm -p 8080:8080 \
-e JSON_LOGS=false \
-e DEBUG_TOKEN="dt" \
-e MAX_REPO_SIZE_MB=100 \
-e MAX_CONCURRENT_WORK=2 \
Expand Down
6 changes: 0 additions & 6 deletions README.md
Expand Up @@ -39,9 +39,3 @@ There is also `filter` URL parameter, which has the opposite behavior to `match`
To make the response more compact you can use `pretty=false`, e.g. `/someuser/somerepo?pretty=false`.

There is useful web frontend for this API: https://github.com/pajecawav/ghloc-web (thanks @pajecawav).

## TODO

* Use `context.Context`.
* Add the prioritized tasks-queue for uncached requests? Limited number of the tasks are executed concurrently.
* Investigate impact on performance of the fact the repositories returns large slices.
2 changes: 1 addition & 1 deletion cmd/ghloc/main.go
Expand Up @@ -127,7 +127,7 @@ func runServer(locsForPaths []loc_count.LOCForPath) {
matcher = &matchers[0]
}
statTree := loc_count.BuildStatTree(locsForPaths, filter, matcher)
rest.WriteResponse(w, (*rest.SortedStat)(statTree), true)
rest.WriteResponse(w, r, (*rest.SortedStat)(statTree), true)
})

serverStatic, err := fs.Sub(serverStatic, "server_static")
Expand Down
36 changes: 0 additions & 36 deletions cmd/server/debug_middleware.go

This file was deleted.

41 changes: 31 additions & 10 deletions cmd/server/main.go
@@ -1,9 +1,12 @@
package main

import (
"context"
"fmt"
"log"
"net/http"
"os"
"time"

// _ "net/http/pprof"

Expand All @@ -14,13 +17,15 @@ import (
_ "github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/source/file"
_ "github.com/lib/pq"
"github.com/rs/zerolog"
"github.com/subtle-byte/ghloc/internal/infrastructure/github_files_provider"
"github.com/subtle-byte/ghloc/internal/infrastructure/postgres_loc_cacher"
"github.com/subtle-byte/ghloc/internal/server/github_handler"
github_stat_service "github.com/subtle-byte/ghloc/internal/service/github_stat"
)

type Config struct {
JSONLogs bool `env:"JSON_LOGS" envDefault:"true"`
DebugToken string `env:"DEBUG_TOKEN"`
MaxRepoSizeMB int `env:"MAX_REPO_SIZE_MB,notEmpty"`
MaxConcurrentWork int `env:"MAX_CONCURRENT_WORK,notEmpty"`
Expand All @@ -30,34 +35,50 @@ type Config struct {
var buildTime = "unknown" // will be replaced during building the docker image

func main() {
log.Printf("Starting up the app (build time: %v)\n", buildTime)
zerolog.TimeFieldFormat = time.RFC3339Nano
zerolog.TimestampFunc = func() time.Time {
return time.Now().Round(time.Microsecond).UTC()
}
logger := zerolog.New(os.Stdout).With().Timestamp().Logger()
log.SetFlags(0)
log.SetOutput(logger)

logger.Info().Str("buildTime", buildTime).Msg("Starting up the app")

ctx := context.Background()
ctx = logger.WithContext(ctx)

cfg := &Config{}
if err := env.Parse(cfg); err != nil {
log.Fatalf("Parsing config: %v", err)
logger.Fatal().Err(err).Msg("Error parsing config")
}
if !cfg.JSONLogs {
out := &zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: zerolog.TimeFieldFormat}
logger = logger.Output(out)
}
log.Printf("Debug token is set: %v", cfg.DebugToken != "")
logger.Info().Msgf("Debug token is set: %v", cfg.DebugToken != "")

github := github_files_provider.New(cfg.MaxRepoSizeMB)
db, closeDB, err := connectAndMigrateDB(cfg.DbConnStr)
pg := github_stat_service.LOCCacher(nil)
if err == nil {
defer closeDB()
pg = postgres_loc_cacher.NewPostgres(db)
log.Println("Connected to DB")
pg = postgres_loc_cacher.NewPostgres(ctx, db)
logger.Info().Msg("Connected to DB")
} else {
log.Printf("Error connecting to DB: %v", err)
log.Println("Warning: continue without DB")
logger.Info().Err(err).Msg("Error connecting to DB")
logger.Warn().Msg("Warning: continue without DB")
}
service := github_stat_service.New(pg, github, cfg.MaxConcurrentWork)

router := chi.NewRouter()
router.Use(middleware.RealIP)
middleware.RequestIDHeader = ""
router.Use(middleware.RequestID)
router.Use(middleware.Logger)
router.Use(NewLoggerMiddleware(logger))
router.Use(NewRequestLoggerMiddleware())
router.Use(middleware.Compress(5))
router.Use(middleware.Recoverer)
router.Use(NewRecoverMiddleware())
router.Use(cors.AllowAll().Handler)

router.Get("/", func(w http.ResponseWriter, r *http.Request) {
Expand All @@ -75,6 +96,6 @@ func main() {
router.With(NewDebugMiddleware(cfg.DebugToken)).Route("/debug", func(r chi.Router) {
getStatHandler.RegisterOn(r)
})
fmt.Println("Listening on http://localhost:8080")
logger.Info().Msg("Listening on http://localhost:8080")
http.ListenAndServe(":8080", router)
}
101 changes: 101 additions & 0 deletions cmd/server/middleware.go
@@ -0,0 +1,101 @@
package main

import (
"fmt"
"net/http"
"net/http/httptest"
"runtime/pprof"
"time"

"github.com/go-chi/chi/v5/middleware"
"github.com/rs/zerolog"
"github.com/subtle-byte/ghloc/internal/util"
)

func NewDebugMiddleware(debugToken string) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
if debugToken == "" {
return http.HandlerFunc(http.NotFound)
}
fn := func(w http.ResponseWriter, r *http.Request) {
if r.FormValue("debug_token") == debugToken {
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Disposition", `attachment; filename="profile"`)
if err := pprof.StartCPUProfile(w); err != nil {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.Header().Del("Content-Disposition")
w.Header().Set("X-Go-Pprof", "1")
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "Could not enable CPU profiling: %s\n", err)
return
}
rr := httptest.ResponseRecorder{}
next.ServeHTTP(&rr, r)
pprof.StopCPUProfile()
} else {
http.NotFound(w, r)
}
}
return http.HandlerFunc(fn)
}
}

func NewLoggerMiddleware(logger zerolog.Logger) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
reqID := middleware.GetReqID(ctx)
logger := logger.With().Str("requestId", reqID).Logger()
ctx = logger.WithContext(ctx)
r = r.WithContext(ctx)
next.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}

func NewRequestLoggerMiddleware() func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
logger := *zerolog.Ctx(r.Context())
logger = logger.With().
Str("method", r.Method).
Bool("tls", r.TLS != nil).
Str("host", r.Host).
Str("url", r.RequestURI).
Str("protocol", r.Proto).
Str("from", r.RemoteAddr).Logger()
logger.Info().Msg("New request")
start := time.Now()
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
defer func() {
logger.Info().
Float64("durationSec", time.Since(start).Seconds()).
Int("status", ww.Status()).
Int("responseBytes", ww.BytesWritten()).
Msg("Request finished")
}()
next.ServeHTTP(ww, r)
}
return http.HandlerFunc(fn)
}
}

func NewRecoverMiddleware() func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err != nil {
stack := util.GetStack(1)
zerolog.Ctx(r.Context()).Error().
Any("stack", stack).
Any("panicValue", err).
Msg("Panic recovered")
w.WriteHeader(http.StatusInternalServerError)
}
}()
next.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}
5 changes: 4 additions & 1 deletion go.mod
Expand Up @@ -4,18 +4,21 @@ go 1.19

require (
github.com/caarlos0/env/v9 v9.0.0
github.com/go-chi/chi/v5 v5.0.8
github.com/go-chi/chi/v5 v5.0.10
github.com/go-chi/cors v1.2.1
github.com/golang-migrate/migrate/v4 v4.16.2
github.com/lib/pq v1.10.9
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
github.com/rs/zerolog v1.29.1
github.com/stretchr/testify v1.8.4
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/sys v0.10.0 // indirect
Expand Down
21 changes: 19 additions & 2 deletions go.sum
Expand Up @@ -2,17 +2,19 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc=
github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhui/dktest v0.3.16 h1:i6gq2YQEtcrjKbeJpBkWjE8MmLZPYllcjOFbTZuPDnw=
github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0=
github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk=
github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/golang-migrate/migrate/v4 v4.16.2 h1:8coYbMKUyInrFk1lfGfRovTLAW7PhWp8qQDT2iKfuoA=
github.com/golang-migrate/migrate/v4 v4.16.2/go.mod h1:pfcJX4nPHaVdc5nmdCikFBWtm+UBpiZjRNNsyBbp0/o=
Expand All @@ -23,15 +25,26 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
Expand All @@ -40,6 +53,10 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
Expand Down

0 comments on commit 253f0f1

Please sign in to comment.