refactor: migrate bookmark static pages to new http server (#775)

* migrate bookmark content route to new http server

* new archive page

* remove unused go generate comment

* database mock

* utils cleanup

* unused var

* domains refactor and tests

* fixed secret key type

* redirect to login on ui errors

* fixed archive folder with storage domain

* webroot documentation

* some bookmark route tests

* fixed error in bookmark domain for non existant bookmarks

* centralice errors

* add coverage data to unittests

* added tests, refactor storage to use afero

* removed mock to avoid increasing complexity

* using deps to copy files around

* remove config usage (to deps)

* remove handler-ui file

Makefile

@@ -94,3 +94,8 @@ build: clean
 coverage:
 	$(GO) test $(GO_TEST_FLAGS) -coverprofile=coverage.txt ./...
 	$(GO) tool cover -html=coverage.txt
+
+## Run generate accross the project
+.PHONY: generated
+generate:
+	$(GO) generate ./...

docs/Configuration.md

@@ -1,35 +1,53 @@
-Content
----
+# Configuration
 
 <!-- TOC -->
 
-- [Content](#content)
 - [Data Directory](#data-directory)
+- [Webroot](#webroot)
+  - [Nginx](#nginx)
 - [Database](#database)
-    - [MySQL](#mysql)
-    - [PostgreSQL](#postgresql)
+  - [MySQL](#mysql)
+  - [PostgreSQL](#postgresql)
 
 <!-- /TOC -->
 
-Data Directory
----
+## Data Directory
 
 Shiori is designed to work out of the box, but you can change where it stores your bookmarks if you need to.
 
 By default, Shiori saves your bookmarks in one of the following directories:
 
-| Platform |                          Directory                           |
-|----------|--------------------------------------------------------------|
+| Platform | Directory                                                    |
+| -------- | ------------------------------------------------------------ |
 | Linux    | `${XDG_DATA_HOME}/shiori` (default: `~/.local/share/shiori`) |
-| macOS    | `~/Library/Application Support/shiori`                               |
+| macOS    | `~/Library/Application Support/shiori`                       |
 | Windows  | `%LOCALAPPDATA%/shiori`                                      |
 
 If you pass the flag `--portable` to Shiori, your data will be stored  in the `shiori-data` subdirectory alongside the shiori executable.
 
 To specify a custom path, set the `SHIORI_DIR` environment variable.
 
-Database
----
+## Webroot
+
+If you want to serve Shiori behind a reverse proxy, you can set the `SHIORI_WEBROOT` environment variable to the path where Shiori is served, e.g. `/shiori`.
+
+Keep in mind this configuration wont make Shiori accessible from `/shiori` path so you need to setup your reverse proxy accordingly so it can strip the webroot path.
+
+We provide some examples for popular reverse proxies below. Please follow your reverse proxy documentation in order to setup it properly.
+
+### Nginx
+
+Fox nginx, you can use the following configuration as a example. The important part **is the trailing slash in `proxy_pass` directive**:
+
+```nginx
+location /shiori {
+    proxy_pass http://localhost:8080/;
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+}
+```
+
+## Database
 
 Shiori uses an SQLite3 database stored in the above data directory by default. If you prefer, you can also use MySQL or PostgreSQL database by setting it in environment variables.
 

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/disintegration/imaging v1.6.2
 	github.com/fatih/color v1.16.0
-	github.com/gin-contrib/gzip v0.0.6
 	github.com/gin-contrib/requestid v0.0.6
 	github.com/gin-contrib/static v0.0.1
 	github.com/gin-gonic/gin v1.9.1
@@ -27,6 +26,7 @@ require (
 	github.com/sethvargo/go-envconfig v0.9.0
 	github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92
 	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/afero v1.11.0
 	github.com/spf13/cobra v1.8.0
 	github.com/stretchr/testify v1.8.4
 	github.com/swaggo/files v1.0.1

go.sum

@@ -212,6 +212,8 @@ github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92/go.mod h1:7/OT02F6
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

internal/cmd/add.go

@@ -45,7 +45,7 @@ func addHandler(cmd *cobra.Command, args []string) {
 	excerpt = normalizeSpace(excerpt)
 
 	// Create bookmark item
-	book := model.Bookmark{
+	book := model.BookmarkDTO{
 		URL:           url,
 		Title:         title,
 		Excerpt:       excerpt,
@@ -101,7 +101,7 @@ func addHandler(cmd *cobra.Command, args []string) {
 				KeepExcerpt: excerpt != "",
 			}
 
-			book, isFatalErr, err = core.ProcessBookmark(request)
+			book, isFatalErr, err = core.ProcessBookmark(deps, request)
 			content.Close()
 
 			if err != nil {

internal/cmd/check.go

@@ -76,7 +76,7 @@ func checkHandler(cmd *cobra.Command, args []string) {
 	for i, book := range bookmarks {
 		wg.Add(1)
 
-		go func(i int, book model.Bookmark) {
+		go func(i int, book model.BookmarkDTO) {
 			// Make sure to finish the WG
 			defer wg.Done()
 

internal/cmd/import.go

@@ -52,7 +52,7 @@ func importHandler(cmd *cobra.Command, args []string) {
 	defer srcFile.Close()
 
 	// Parse bookmark's file
-	bookmarks := []model.Bookmark{}
+	bookmarks := []model.BookmarkDTO{}
 	mapURL := make(map[string]struct{})
 
 	doc, err := goquery.NewDocumentFromReader(srcFile)
@@ -135,7 +135,7 @@ func importHandler(cmd *cobra.Command, args []string) {
 		}
 
 		// Add item to list
-		bookmark := model.Bookmark{
+		bookmark := model.BookmarkDTO{
 			URL:      url,
 			Title:    title,
 			Tags:     tags,

internal/cmd/pocket.go

@@ -36,7 +36,7 @@ func pocketHandler(cmd *cobra.Command, args []string) {
 	defer srcFile.Close()
 
 	// Parse pocket's file
-	bookmarks := []model.Bookmark{}
+	bookmarks := []model.BookmarkDTO{}
 	mapURL := make(map[string]struct{})
 
 	doc, err := goquery.NewDocumentFromReader(srcFile)
@@ -93,7 +93,7 @@ func pocketHandler(cmd *cobra.Command, args []string) {
 		}
 
 		// Add item to list
-		bookmark := model.Bookmark{
+		bookmark := model.BookmarkDTO{
 			URL:      url,
 			Title:    title,
 			Modified: modified.Format(model.DatabaseDateFormat),

internal/cmd/root.go

@@ -8,9 +8,11 @@ import (
 
 	"github.com/go-shiori/shiori/internal/config"
 	"github.com/go-shiori/shiori/internal/database"
+	"github.com/go-shiori/shiori/internal/dependencies"
 	"github.com/go-shiori/shiori/internal/domains"
 	"github.com/go-shiori/shiori/internal/model"
 	"github.com/sirupsen/logrus"
+	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"golang.org/x/net/context"
 )
@@ -47,7 +49,7 @@ func ShioriCmd() *cobra.Command {
 	return rootCmd
 }
 
-func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *config.Dependencies) {
+func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *dependencies.Dependencies) {
 	logger := logrus.New()
 
 	portableMode, _ := cmd.Flags().GetBool("portable")
@@ -96,9 +98,11 @@ func initShiori(ctx context.Context, cmd *cobra.Command) (*config.Config, *confi
 		logger.Warn("Development mode is ENABLED, this will enable some helpers for local development, unsuitable for production environments")
 	}
 
-	dependencies := config.NewDependencies(logger, db, cfg)
-	dependencies.Domains.Auth = domains.NewAccountsDomain(logger, cfg.Http.SecretKey, db)
-	dependencies.Domains.Archiver = domains.NewArchiverDomain(logger, cfg.Storage.DataDir)
+	dependencies := dependencies.NewDependencies(logger, db, cfg)
+	dependencies.Domains.Auth = domains.NewAccountsDomain(dependencies)
+	dependencies.Domains.Archiver = domains.NewArchiverDomain(dependencies)
+	dependencies.Domains.Bookmarks = domains.NewBookmarksDomain(dependencies)
+	dependencies.Domains.Storage = domains.NewStorageDomain(dependencies, afero.NewBasePathFs(afero.NewOsFs(), cfg.Storage.DataDir))
 
 	// Workaround: Get accounts to make sure at least one is present in the database.
 	// If there's no accounts in the database, create the shiori/gopher account the legacy api

internal/cmd/server.go

@@ -37,7 +37,7 @@ func newServerCommandHandler() func(cmd *cobra.Command, args []string) {
 		rootPath, _ := cmd.Flags().GetString("webroot")
 		accessLog, _ := cmd.Flags().GetBool("access-log")
 		serveWebUI, _ := cmd.Flags().GetBool("serve-web-ui")
-		secretKey, _ := cmd.Flags().GetString("secret-key")
+		secretKey, _ := cmd.Flags().GetBytesHex("secret-key")
 
 		cfg, dependencies := initShiori(ctx, cmd)
 
@@ -66,7 +66,10 @@ func newServerCommandHandler() func(cmd *cobra.Command, args []string) {
 
 		dependencies.Log.Infof("Starting Shiori v%s", model.BuildVersion)
 
-		server := http.NewHttpServer(dependencies.Log).Setup(cfg, dependencies)
+		server, err := http.NewHttpServer(dependencies.Log).Setup(cfg, dependencies)
+		if err != nil {
+			dependencies.Log.WithError(err).Fatal("error setting up server")
+		}
 
 		if err := server.Start(ctx); err != nil {
 			dependencies.Log.WithError(err).Fatal("error starting server")

internal/cmd/update.go

@@ -147,7 +147,7 @@ func updateHandler(cmd *cobra.Command, args []string) {
 				book.URL = url
 			}
 
-			go func(i int, book model.Bookmark) {
+			go func(i int, book model.BookmarkDTO) {
 				// Make sure to finish the WG
 				defer wg.Done()
 
@@ -175,7 +175,7 @@ func updateHandler(cmd *cobra.Command, args []string) {
 					LogArchival: logArchival,
 				}
 
-				book, _, err = core.ProcessBookmark(request)
+				book, _, err = core.ProcessBookmark(deps, request)
 				content.Close()
 
 				if err != nil {

internal/cmd/utils.go

@@ -41,7 +41,7 @@ func isURLValid(s string) bool {
 	return err == nil && tmp.Scheme != "" && tmp.Hostname() != ""
 }
 
-func printBookmarks(bookmarks ...model.Bookmark) {
+func printBookmarks(bookmarks ...model.BookmarkDTO) {
 	for _, bookmark := range bookmarks {
 		// Create bookmark index
 		strBookmarkIndex := fmt.Sprintf("%d. ", bookmark.ID)

internal/config/config.go

@@ -50,7 +50,7 @@ type HttpConfig struct {
 	RootPath   string `env:"HTTP_ROOT_PATH,default=/"`
 	AccessLog  bool   `env:"HTTP_ACCESS_LOG,default=True"`
 	ServeWebUI bool   `env:"HTTP_SERVE_WEB_UI,default=True"`
-	SecretKey  string `env:"HTTP_SECRET_KEY"`
+	SecretKey  []byte `env:"HTTP_SECRET_KEY"`
 	// Fiber Specific
 	BodyLimit                    int           `env:"HTTP_BODY_LIMIT,default=1024"`
 	ReadTimeout                  time.Duration `env:"HTTP_READ_TIMEOUT,default=10s"`
@@ -82,13 +82,13 @@ type Config struct {
 // SetDefaults sets the default values for the configuration
 func (c *HttpConfig) SetDefaults(logger *logrus.Logger) {
 	// Set a random secret key if not set
-	if c.SecretKey == "" {
+	if len(c.SecretKey) == 0 {
 		logger.Warn("SHIORI_HTTP_SECRET_KEY is not set, using random value. This means that all sessions will be invalidated on server restart.")
 		randomUUID, err := uuid.NewV4()
 		if err != nil {
 			logger.WithError(err).Fatal("couldn't generate a random UUID")
 		}
-		c.SecretKey = randomUUID.String()
+		c.SecretKey = []byte(randomUUID.String())
 	}
 }
 

internal/core/ebook.go

@@ -1,22 +1,21 @@
 package core
 
 import (
-	"fmt"
 	"os"
 	fp "path/filepath"
 	"strconv"
 	"strings"
 
 	epub "github.com/go-shiori/go-epub"
+	"github.com/go-shiori/shiori/internal/dependencies"
 	"github.com/go-shiori/shiori/internal/model"
 	"github.com/pkg/errors"
 )
 
 // GenerateEbook receives a `ProcessRequest` and generates an ebook file in the destination path specified.
 // The destination path `dstPath` should include file name with ".epub" extension
 // The bookmark model will be used to update the UI based on whether this function is successful or not.
-func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err error) {
-
+func GenerateEbook(deps *dependencies.Dependencies, req ProcessRequest, dstPath string) (book model.BookmarkDTO, err error) {
 	book = req.Bookmark
 
 	// Make sure bookmark ID is defined
@@ -27,14 +26,14 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err
 	// Get current state of bookmark cheak archive and thumb
 	strID := strconv.Itoa(book.ID)
 
-	imagePath := fp.Join(req.DataDir, "thumb", fmt.Sprintf("%d", book.ID))
-	archivePath := fp.Join(req.DataDir, "archive", fmt.Sprintf("%d", book.ID))
+	bookmarkThumbnailPath := model.GetThumbnailPath(&book)
+	bookmarkArchivePath := model.GetArchivePath(&book)
 
-	if _, err := os.Stat(imagePath); err == nil {
+	if deps.Domains.Storage.FileExists(bookmarkThumbnailPath) {
 		book.ImageURL = fp.Join("/", "bookmark", strID, "thumb")
 	}
 
-	if _, err := os.Stat(archivePath); err == nil {
+	if deps.Domains.Storage.FileExists(bookmarkArchivePath) {
 		book.HasArchive = true
 	}
 
@@ -77,7 +76,7 @@ func GenerateEbook(req ProcessRequest, dstPath string) (book model.Bookmark, err
 	defer tmpFile.Close()
 
 	// If everything go well we move ebook to dstPath
-	err = MoveFileToDestination(dstPath, tmpFile)
+	err = deps.Domains.Storage.WriteFile(dstPath, tmpFile)
 	if err != nil {
 		return book, errors.Wrap(err, "failed move ebook to destination")
 	}