refactor: migrate bookmark static pages to new http server

Makefile

@@ -94,3 +94,8 @@ build: clean
 coverage:
 	$(GO) test $(GO_TEST_FLAGS) -coverprofile=coverage.txt ./...
 	$(GO) tool cover -html=coverage.txt
+
+## Run generate accross the project
+.PHONY: generated
+generate:
+	$(GO) generate ./...

docs/Configuration.md

@@ -1,35 +1,53 @@
-Content
----
+# Configuration
 
 <!-- TOC -->
 
-- [Content](#content)
 - [Data Directory](#data-directory)
+- [Webroot](#webroot)
+  - [Nginx](#nginx)
 - [Database](#database)
-    - [MySQL](#mysql)
-    - [PostgreSQL](#postgresql)
+  - [MySQL](#mysql)
+  - [PostgreSQL](#postgresql)
 
 <!-- /TOC -->
 
-Data Directory
----
+## Data Directory
 
 Shiori is designed to work out of the box, but you can change where it stores your bookmarks if you need to.
 
 By default, Shiori saves your bookmarks in one of the following directories:
 
-| Platform |                          Directory                           |
-|----------|--------------------------------------------------------------|
+| Platform | Directory                                                    |
+| -------- | ------------------------------------------------------------ |
 | Linux    | `${XDG_DATA_HOME}/shiori` (default: `~/.local/share/shiori`) |
-| macOS    | `~/Library/Application Support/shiori`                               |
+| macOS    | `~/Library/Application Support/shiori`                       |
 | Windows  | `%LOCALAPPDATA%/shiori`                                      |
 
 If you pass the flag `--portable` to Shiori, your data will be stored  in the `shiori-data` subdirectory alongside the shiori executable.
 
 To specify a custom path, set the `SHIORI_DIR` environment variable.
 
-Database
----
+## Webroot
+
+If you want to serve Shiori behind a reverse proxy, you can set the `SHIORI_WEBROOT` environment variable to the path where Shiori is served, e.g. `/shiori`.
+
+Keep in mind this configuration wont make Shiori accessible from `/shiori` path so you need to setup your reverse proxy accordingly so it can strip the webroot path.
+
+We provide some examples for popular reverse proxies below. Please follow your reverse proxy documentation in order to setup it properly.
+
+### Nginx
+
+Fox nginx, you can use the following configuration as a example. The important part **is the trailing slash in `proxy_pass` directive**:
+
+```nginx
+location /shiori {
+    proxy_pass http://localhost:8080/;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+}
+```
+
+## Database
 
 Shiori uses an SQLite3 database stored in the above data directory by default. If you prefer, you can also use MySQL or PostgreSQL database by setting it in environment variables.
 

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/disintegration/imaging v1.6.2
 	github.com/fatih/color v1.16.0
-	github.com/gin-contrib/gzip v0.0.6
 	github.com/gin-contrib/requestid v0.0.6
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.9.1
@@ -27,16 +26,18 @@ require (
 	github.com/sethvargo/go-envconfig v0.9.0
 	github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92
 	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/afero v1.11.0
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.0
 	github.com/swaggo/swag v1.16.2
 	github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f
-	golang.org/x/crypto v0.15.0
+	go.uber.org/mock v0.3.0
+	golang.org/x/crypto v0.16.0
 	golang.org/x/image v0.14.0
-	golang.org/x/net v0.18.0
-	golang.org/x/term v0.14.0
+	golang.org/x/net v0.19.0
+	golang.org/x/term v0.15.0
 	modernc.org/sqlite v1.27.0
 )
 
@@ -88,7 +89,7 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/arch v0.6.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.15.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect

go.sum

@@ -222,6 +222,8 @@ github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92/go.mod h1:7/OT02F6
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -269,14 +271,16 @@ go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
 go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
+go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc=
 golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
-golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4=
 golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE=
@@ -292,8 +296,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
-golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -315,14 +319,14 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
-golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
-golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

internal/cmd/add.go

@@ -45,7 +45,7 @@
 	excerpt = normalizeSpace(excerpt)
 
 	// Create bookmark item
-	book := model.Bookmark{
+	book := model.BookmarkDTO{
 		URL:           url,
 		Title:         title,
 		Excerpt:       excerpt,
@@ -101,7 +101,7 @@
 				KeepExcerpt: excerpt != "",
 			}
 
-			book, isFatalErr, err = core.ProcessBookmark(request)
+			book, isFatalErr, err = core.ProcessBookmark(deps, request)
 			content.Close()
 
 			if err != nil {

internal/cmd/check.go

@@ -76,7 +76,7 @@
 	for i, book := range bookmarks {
 		wg.Add(1)
 
-		go func(i int, book model.Bookmark) {
+		go func(i int, book model.BookmarkDTO) {
 			// Make sure to finish the WG
 			defer wg.Done()
 

internal/cmd/import.go

@@ -52,7 +52,7 @@
 	defer srcFile.Close()
 
 	// Parse bookmark's file
-	bookmarks := []model.Bookmark{}
+	bookmarks := []model.BookmarkDTO{}
 	mapURL := make(map[string]struct{})
 
 	doc, err := goquery.NewDocumentFromReader(srcFile)
@@ -135,7 +135,7 @@
 		}
 
 		// Add item to list
-		bookmark := model.Bookmark{
+		bookmark := model.BookmarkDTO{
 			URL:      url,
 			Title:    title,
 			Tags:     tags,

internal/cmd/pocket.go

@@ -36,7 +36,7 @@
 	defer srcFile.Close()
 
 	// Parse pocket's file
-	bookmarks := []model.Bookmark{}
+	bookmarks := []model.BookmarkDTO{}
 	mapURL := make(map[string]struct{})
 
 	doc, err := goquery.NewDocumentFromReader(srcFile)
@@ -93,7 +93,7 @@
 		}
 
 		// Add item to list
-		bookmark := model.Bookmark{
+		bookmark := model.BookmarkDTO{
 			URL:      url,
 			Title:    title,
 			Modified: modified.Format(model.DatabaseDateFormat),

internal/cmd/root.go

@@ -8,9 +8,11 @@
 
 	"github.com/go-shiori/shiori/internal/config"
 	"github.com/go-shiori/shiori/internal/database"
+	"github.com/go-shiori/shiori/internal/dependencies"
 	"github.com/go-shiori/shiori/internal/domains"
 	"github.com/go-shiori/shiori/internal/model"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"golang.org/x/net/context"
 )
@@ -47,7 +49,7 @@
 	return rootCmd
 }
 
-func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *config.Dependencies) {
+func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *dependencies.Dependencies) {
 	logger := logrus.New()
 
 	portableMode, _ := cmd.Flags().GetBool("portable")
@@ -96,9 +98,11 @@
 		logger.Warn("Development mode is ENABLED, this will enable some helpers for local development, unsuitable for production environments")
 	}
 
-	dependencies := config.NewDependencies(logger, db, cfg)
-	dependencies.Domains.Auth = domains.NewAccountsDomain(logger, cfg.Http.SecretKey, db)
-	dependencies.Domains.Archiver = domains.NewArchiverDomain(logger, cfg.Storage.DataDir)
+	dependencies := dependencies.NewDependencies(logger, db, cfg)
+	dependencies.Domains.Auth = domains.NewAccountsDomain(dependencies)
+	dependencies.Domains.Archiver = domains.NewArchiverDomain(dependencies)
+	dependencies.Domains.Bookmarks = domains.NewBookmarksDomain(dependencies)
+	dependencies.Domains.Storage = domains.NewStorageDomain(dependencies, afero.NewBasePathFs(afero.NewOsFs(), cfg.Storage.DataDir))
 
 	// Workaround: Get accounts to make sure at least one is present in the database.
 	// If there's no accounts in the database, create the shiori/gopher account the legacy api

internal/cmd/server.go

@@ -37,7 +37,7 @@
 		rootPath, _ := cmd.Flags().GetString("webroot")
 		accessLog, _ := cmd.Flags().GetBool("access-log")
 		serveWebUI, _ := cmd.Flags().GetBool("serve-web-ui")
-		secretKey, _ := cmd.Flags().GetString("secret-key")
+		secretKey, _ := cmd.Flags().GetBytesHex("secret-key")
 
 		cfg, dependencies := initShiori(ctx, cmd)
 
@@ -66,7 +66,10 @@
 
 		dependencies.Log.Infof("Starting Shiori v%s", model.BuildVersion)
 
-		server := http.NewHttpServer(dependencies.Log).Setup(cfg, dependencies)
+		server, err := http.NewHttpServer(dependencies.Log).Setup(cfg, dependencies)
+		if err != nil {
+			dependencies.Log.WithError(err).Fatal("error setting up server")
+		}
 
 		if err := server.Start(ctx); err != nil {
 			dependencies.Log.WithError(err).Fatal("error starting server")

internal/cmd/update.go

@@ -147,7 +147,7 @@
 				book.URL = url
 			}
 
-			go func(i int, book model.Bookmark) {
+			go func(i int, book model.BookmarkDTO) {
 				// Make sure to finish the WG
 				defer wg.Done()
 
@@ -175,7 +175,7 @@
 					LogArchival: logArchival,
 				}
 
-				book, _, err = core.ProcessBookmark(request)
+				book, _, err = core.ProcessBookmark(deps, request)
 				content.Close()
 
 				if err != nil {

internal/cmd/utils.go

@@ -41,7 +41,7 @@
 	return err == nil && tmp.Scheme != "" && tmp.Hostname() != ""
 }
 
-func printBookmarks(bookmarks ...model.Bookmark) {
+func printBookmarks(bookmarks ...model.BookmarkDTO) {
 	for _, bookmark := range bookmarks {
 		// Create bookmark index
 		strBookmarkIndex := fmt.Sprintf("%d. ", bookmark.ID)

internal/config/config.go

@@ -50,7 +50,7 @@ type HttpConfig struct {
 	RootPath   string `env:"HTTP_ROOT_PATH,default=/"`
 	AccessLog  bool   `env:"HTTP_ACCESS_LOG,default=True"`
 	ServeWebUI bool   `env:"HTTP_SERVE_WEB_UI,default=True"`
-	SecretKey  string `env:"HTTP_SECRET_KEY"`
+	SecretKey  []byte `env:"HTTP_SECRET_KEY"`
 	// Fiber Specific
 	BodyLimit                    int           `env:"HTTP_BODY_LIMIT,default=1024"`
 	ReadTimeout                  time.Duration `env:"HTTP_READ_TIMEOUT,default=10s"`
@@ -82,13 +82,13 @@ type Config struct {
 // SetDefaults sets the default values for the configuration
 func (c *HttpConfig) SetDefaults(logger *logrus.Logger) {
 	// Set a random secret key if not set
-	if c.SecretKey == "" {
+	if len(c.SecretKey) == 0 {
 		logger.Warn("SHIORI_HTTP_SECRET_KEY is not set, using random value. This means that all sessions will be invalidated on server restart.")
 		randomUUID, err := uuid.NewV4()
 		if err != nil {
 			logger.WithError(err).Fatal("couldn't generate a random UUID")
 		}
-		c.SecretKey = randomUUID.String()
+		c.SecretKey = []byte(randomUUID.String())
 	}
 }