Config: Add “daily” and “weekly” backup schedule options #4243

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/photoprism/restore_albums.go → internal/backup/albums.go

@@ -1,22 +1,81 @@
-package photoprism
+package backup
 
 import (
 	"path/filepath"
 	"regexp"
+	"time"
 
 	"github.com/photoprism/photoprism/internal/entity"
+	"github.com/photoprism/photoprism/internal/get"
 	"github.com/photoprism/photoprism/internal/query"
 	"github.com/photoprism/photoprism/pkg/clean"
 	"github.com/photoprism/photoprism/pkg/fs"
 )
 
+// Albums creates a YAML file backup of all albums.
+func Albums(backupPath string, force bool) (count int, err error) {
+	// Make sure only one backup/restore operation is running at a time.
+	backupAlbumsMutex.Lock()
+	defer backupAlbumsMutex.Unlock()
+
+	// Get albums from database.
+	albums, queryErr := query.Albums(0, 1000000)
+
+	if queryErr != nil {
+		return count, queryErr
+	}
+
+	if !fs.PathExists(backupPath) {
+		backupPath = get.Config().BackupAlbumsPath()
+	}
+
+	log.Debugf("backup: album backups will be stored in %s", clean.Log(backupPath))
+	log.Infof("backup: saving album metadata in YAML backup files")
+
+	var latest time.Time
+
+	// Ignore the last modification timestamp if the force flag is set.
+	if !force {
+		latest = backupAlbumsTime
+	}
+
+	// Save albums to YAML backup files.
+	for _, a := range albums {
+		// Album modification timestamp.
+		changed := a.UpdatedAt
+
+		// Skip albums that have already been saved to YAML backup files.
+		if !force && !backupAlbumsTime.IsZero() && !changed.IsZero() && !backupAlbumsTime.Before(changed) {
+			continue
+		}
+
+		// Remember the lastest modification timestamp.
+		if changed.After(latest) {
+			latest = changed
+		}
+
+		// Write album metadata to YAML backup file.
+		if saveErr := a.SaveBackupYaml(backupPath); saveErr != nil {
+			err = saveErr
+		} else {
+			count++
+		}
+	}
+
+	// Set backupAlbumsTime to latest modification timestamp,
+	// so that already saved albums can be skipped next time.
+	backupAlbumsTime = latest
+
+	return count, err
+}
+
 // RestoreAlbums restores all album YAML file backups.
 func RestoreAlbums(backupPath string, force bool) (count int, result error) {
 	// Make sure only one backup/restore operation is running at a time.
 	backupAlbumsMutex.Lock()
 	defer backupAlbumsMutex.Unlock()
 
-	c := Config()
+	c := get.Config()
 
 	if !c.BackupAlbums() && !force {
 		log.Debugf("albums: metadata backup files are disabled")

internal/backup/albums_test.go

@@ -0,0 +1,35 @@
+package backup
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/photoprism/photoprism/pkg/fs"
+)
+
+func TestAlbums(t *testing.T) {
+	backupPath, err := filepath.Abs("./testdata/albums")
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err = os.MkdirAll(backupPath, fs.ModeDir); err != nil {
+		t.Fatal(err)
+	}
+
+	count, err := Albums(backupPath, true)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	assert.Equal(t, 30, count)
+
+	if err = os.RemoveAll(backupPath); err != nil {
+		t.Fatal(err)
+	}
+}

internal/backup/backup.go

@@ -0,0 +1,31 @@
+/*
+Package backup provides backup and restore functions for databases and albums.
+
+Copyright (c) 2018 - 2024 PhotoPrism UG. All rights reserved.
+
+	This program is free software: you can redistribute it and/or modify
+	it under Version 3 of the GNU Affero General Public License (the "AGPL"):
+	<https://docs.photoprism.app/license/agpl>
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	The AGPL is supplemented by our Trademark and Brand Guidelines,
+	which describe how our Brand Assets may be used:
+	<https://www.photoprism.app/trademark>
+
+Feel free to send an email to hello@photoprism.app if you have questions,
+want to support our work, or just want to say hello.
+
+Additional information can be found in our Developer Guide:
+<https://docs.photoprism.app/developer-guide/>
+*/
+package backup
+
+import (
+	"github.com/photoprism/photoprism/internal/event"
+)
+
+var log = event.Log

internal/backup/backup_test.go

@@ -0,0 +1,29 @@
+package backup
+
+import (
+	"os"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/photoprism/photoprism/internal/config"
+	"github.com/photoprism/photoprism/internal/event"
+	"github.com/photoprism/photoprism/internal/get"
+	"github.com/photoprism/photoprism/internal/photoprism"
+)
+
+func TestMain(m *testing.M) {
+	log = logrus.StandardLogger()
+	log.SetLevel(logrus.TraceLevel)
+	event.AuditLog = log
+
+	c := config.TestConfig()
+	defer c.CloseDb()
+
+	get.SetConfig(c)
+	photoprism.SetConfig(c)
+
+	code := m.Run()
+
+	os.Exit(code)
+}

internal/backup/const.go

@@ -0,0 +1,3 @@
+package backup
+
+const SqlBackupFileNamePattern = "[2-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9].sql"

internal/photoprism/restore_database.go → internal/backup/database.go

@@ -1,4 +1,4 @@
-package photoprism
+package backup
 
 import (
 	"bytes"
@@ -11,22 +11,157 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"time"
+
+	"github.com/dustin/go-humanize/english"
 
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/entity"
+	"github.com/photoprism/photoprism/internal/get"
 	"github.com/photoprism/photoprism/pkg/clean"
 	"github.com/photoprism/photoprism/pkg/fs"
 )
 
-const SqlBackupFileNamePattern = "[2-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9].sql"
+// Database creates a database backup dump with the specified file and path name.
+func Database(backupPath, fileName string, toStdOut, force bool, retain int) (err error) {
+	// Ensure that only one database backup/restore operation is running at a time.
+	backupDatabaseMutex.Lock()
+	defer backupDatabaseMutex.Unlock()
+
+	// Backup action shown in logs.
+	backupAction := "creating"
+
+	// Get configuration.
+	c := get.Config()
+
+	if !toStdOut {
+		if backupPath == "" {
+			backupPath = c.BackupDatabasePath()
+		}
+
+		// Create the backup path if it does not already exist.
+		if err = fs.MkdirAll(backupPath); err != nil {
+			return err
+		}
+
+		// Check if the backup path is writable.
+		if !fs.PathWritable(backupPath) {
+			return fmt.Errorf("backup path is not writable")
+		}
+
+		if fileName == "" {
+			backupFile := time.Now().UTC().Format("2006-01-02") + ".sql"
+			fileName = filepath.Join(backupPath, backupFile)
+		}
+
+		log.Debugf("backup: database backups will be stored in %s", clean.Log(backupPath))
+
+		if _, err = os.Stat(fileName); err == nil && !force {
+			return fmt.Errorf("%s already exists", clean.Log(filepath.Base(fileName)))
+		} else if err == nil {
+			backupAction = "replacing"
+		}
+
+		// Create backup path if not exists.
+		if dir := filepath.Dir(fileName); dir != "." {
+			if err = fs.MkdirAll(dir); err != nil {
+				return err
+			}
+		}
+	}
+
+	var cmd *exec.Cmd
+
+	switch c.DatabaseDriver() {
+	case config.MySQL, config.MariaDB:
+		cmd = exec.Command(
+			c.MariadbDumpBin(),
+			"--protocol", "tcp",
+			"-h", c.DatabaseHost(),
+			"-P", c.DatabasePortString(),
+			"-u", c.DatabaseUser(),
+			"-p"+c.DatabasePassword(),
+			c.DatabaseName(),
+		)
+	case config.SQLite3:
+		if !fs.FileExistsNotEmpty(c.DatabaseFile()) {
+			return fmt.Errorf("sqlite database file %s not found", clean.LogQuote(c.DatabaseFile()))
+		}
+
+		cmd = exec.Command(
+			c.SqliteBin(),
+			c.DatabaseFile(),
+			".dump",
+		)
+	default:
+		return fmt.Errorf("unsupported database type: %s", c.DatabaseDriver())
+	}
+
+	// Write to stdout or file.
+	var f *os.File
+	if toStdOut {
+		log.Infof("backup: sending database backup to stdout")
+		f = os.Stdout
+	} else if f, err = os.OpenFile(fileName, os.O_TRUNC|os.O_RDWR|os.O_CREATE, fs.ModeBackup); err != nil {
+		return fmt.Errorf("failed to create %s (%s)", clean.Log(fileName), err)
+	} else {
+		log.Infof("backup: %s database backup file %s", backupAction, clean.Log(filepath.Base(fileName)))
+		defer f.Close()
+	}
+
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	cmd.Stdout = f
+
+	// Log exact command for debugging in trace mode.
+	log.Trace(cmd.String())
+
+	// Run backup command.
+	if cmdErr := cmd.Run(); cmdErr != nil {
+		if errStr := strings.TrimSpace(stderr.String()); errStr != "" {
+			return errors.New(errStr)
+		}
+
+		return cmdErr
+	}
+
+	// Delete old backups if the number of backup files to keep has been specified.
+	if !toStdOut && backupPath != "" && retain > 0 {
+		files, globErr := filepath.Glob(filepath.Join(regexp.QuoteMeta(backupPath), SqlBackupFileNamePattern))
+
+		if globErr != nil {
+			return globErr
+		}
+
+		if len(files) == 0 {
+			return fmt.Errorf("found no database backup files in %s", backupPath)
+		} else if len(files) <= retain {
+			return nil
+		}
+
+		sort.Strings(files)
+
+		log.Infof("backup: retaining %s", english.Plural(retain, "database backup", "database backups"))
+
+		for i := 0; i < len(files)-retain; i++ {
+			if err = os.Remove(files[i]); err != nil {
+				return err
+			} else {
+				log.Infof("backup: removed database backup file %s", clean.Log(filepath.Base(files[i])))
+			}
+		}
+	}
+
+	return nil
+}
 
 // RestoreDatabase restores the database from a backup file with the specified path and name.
 func RestoreDatabase(backupPath, fileName string, fromStdIn, force bool) (err error) {
 	// Ensure that only one database backup/restore operation is running at a time.
 	backupDatabaseMutex.Lock()
 	defer backupDatabaseMutex.Unlock()
 
-	c := Config()
+	c := get.Config()
 
 	// If empty, use default backup file name.
 	if !fromStdIn {

internal/backup/sync.go

@@ -0,0 +1,12 @@
+package backup
+
+import (
+	"sync"
+	"time"
+)
+
+var (
+	backupDatabaseMutex = sync.Mutex{}
+	backupAlbumsMutex   = sync.Mutex{}
+	backupAlbumsTime    = time.Time{}
+)

internal/backup/testdata/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

internal/commands/backup.go

@@ -9,8 +9,8 @@ import (
 	"github.com/dustin/go-humanize/english"
 	"github.com/urfave/cli"
 
+	"github.com/photoprism/photoprism/internal/backup"
 	"github.com/photoprism/photoprism/internal/config"
-	"github.com/photoprism/photoprism/internal/photoprism"
 	"github.com/photoprism/photoprism/pkg/fs"
 )
 
@@ -101,7 +101,7 @@ func backupAction(ctx *cli.Context) error {
 			fileName = filepath.Join(databasePath, backupFile)
 		}
 
-		if err = photoprism.BackupDatabase(databasePath, fileName, fileName == "-", force, retain); err != nil {
+		if err = backup.Database(databasePath, fileName, fileName == "-", force, retain); err != nil {
 			return fmt.Errorf("failed to create database backup: %w", err)
 		}
 	}
@@ -115,7 +115,7 @@ func backupAction(ctx *cli.Context) error {
 			albumsPath = conf.BackupAlbumsPath()
 		}
 
-		if count, backupErr := photoprism.BackupAlbums(albumsPath, true); backupErr != nil {
+		if count, backupErr := backup.Albums(albumsPath, true); backupErr != nil {
 			return backupErr
 		} else {
 			log.Infof("backup: saved %s", english.Plural(count, "album backup", "album backups"))