Skip to content

Commit

Permalink
feat: get the list of supported extensions from the server #134 (#164)
Browse files Browse the repository at this point in the history
  • Loading branch information
@simulot
simulot committed Feb 19, 2024
1 parent 6791c93 commit a47cca4
Show file tree
Hide file tree
Showing 17 changed files with 264 additions and 193 deletions.
46 changes: 15 additions & 31 deletions browser/files/localassets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"time"

"github.com/simulot/immich-go/browser"
"github.com/simulot/immich-go/helpers/fshelper"
"github.com/simulot/immich-go/immich"
"github.com/simulot/immich-go/immich/metadata"
"github.com/simulot/immich-go/logger"
)
Expand All @@ -19,13 +19,15 @@ type LocalAssetBrowser struct {
fsyss []fs.FS
albums map[string]string
log *logger.Journal
sm immich.SupportedMedia
}

func NewLocalFiles(ctx context.Context, log *logger.Journal, fsyss ...fs.FS) (*LocalAssetBrowser, error) {
func NewLocalFiles(ctx context.Context, log *logger.Journal, sm immich.SupportedMedia, fsyss ...fs.FS) (*LocalAssetBrowser, error) {
return &LocalAssetBrowser{
fsyss: fsyss,
albums: map[string]string{},
log: log,
sm: sm,
}, nil
}

Expand Down Expand Up @@ -78,19 +80,6 @@ func (la *LocalAssetBrowser) handleFolder(ctx context.Context, fsys fs.FS, fileC
return err
}

// fileMap := map[string][]fs.DirEntry{}
// for _, e := range entries {
// if e.IsDir() {
// continue
// }
// ext := path.Ext(e.Name())
// _, err := fshelper.MimeFromExt(ext)
// if strings.ToLower(ext) == ".xmp" || err == nil {
// base := strings.TrimSuffix(e.Name(), ext)
// fileMap[base] = append(fileMap[base], e)
// }
// }

for _, e := range entries {
if e.IsDir() {
continue
Expand All @@ -99,22 +88,18 @@ func (la *LocalAssetBrowser) handleFolder(ctx context.Context, fsys fs.FS, fileC
la.log.AddEntry(fileName, logger.DiscoveredFile, "")
name := e.Name()
ext := strings.ToLower(path.Ext(name))
if fshelper.IsMetadataExt(ext) {
la.log.AddEntry(name, logger.Metadata, "")
continue
} else if fshelper.IsIgnoredExt(ext) {

t := la.sm.TypeFromExt(ext)
switch t {
default:
la.log.AddEntry(fileName, logger.Unsupported, "")
continue
}
m, err := fshelper.MimeFromExt(strings.ToLower(ext))
if err != nil {
la.log.AddEntry(fileName, logger.Unsupported, "")
case immich.TypeSidecar:
la.log.AddEntry(name, logger.Metadata, "")
continue
}
ss := strings.Split(m[0], "/")
if ss[0] == "image" {
case immich.TypeImage:
la.log.AddEntry(name, logger.ScannedImage, "")
} else {
case immich.TypeVideo:
la.log.AddEntry(name, logger.ScannedVideo, "")
}

Expand Down Expand Up @@ -154,7 +139,7 @@ func (la *LocalAssetBrowser) handleFolder(ctx context.Context, fsys fs.FS, fileC
}

func (la *LocalAssetBrowser) checkSidecar(f *browser.LocalAssetFile, entries []fs.DirEntry, dir, name string) bool {
assetBase := baseNames(name)
assetBase := la.baseNames(name)

for _, name := range assetBase {
xmp := name + ".[xX][mM][pP]"
Expand All @@ -176,16 +161,15 @@ func (la *LocalAssetBrowser) checkSidecar(f *browser.LocalAssetFile, entries []f
return false
}

func baseNames(n string) []string {
func (la *LocalAssetBrowser) baseNames(n string) []string {
n = escapeName(n)
names := []string{n}
ext := path.Ext(n)
for {
if ext == "" {
return names
}
_, err := fshelper.MimeFromExt(ext)
if err != nil {
if la.sm.TypeFromExt(ext) == "" {
return names
}
n = strings.TrimSuffix(n, ext)
Expand Down
3 changes: 2 additions & 1 deletion browser/files/localassets_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/kr/pretty"
"github.com/psanford/memfs"
"github.com/simulot/immich-go/browser/files"
"github.com/simulot/immich-go/immich"
"github.com/simulot/immich-go/logger"
)

Expand Down Expand Up @@ -74,7 +75,7 @@ func TestLocalAssets(t *testing.T) {
}
ctx := context.Background()

b, err := files.NewLocalFiles(ctx, logger.NewJournal(logger.NoLog{}), fsys)
b, err := files.NewLocalFiles(ctx, logger.NewJournal(logger.NoLog{}), immich.DefaultSupportedMedia, fsys)
if err != nil {
t.Error(err)
}
Expand Down
68 changes: 29 additions & 39 deletions browser/gp/googlephotos.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/simulot/immich-go/browser"
"github.com/simulot/immich-go/helpers/fshelper"
"github.com/simulot/immich-go/helpers/gen"
"github.com/simulot/immich-go/immich"
"github.com/simulot/immich-go/logger"
)

Expand All @@ -23,15 +24,14 @@ type Takeout struct {
uploaded map[fileKey]any // track files already uploaded
albums map[string]string // tack album names by folder
jnl *logger.Journal
sm immich.SupportedMedia
}

// walkerCatalog collects all directory catalogs
type walkerCatalog map[string]directoryCatalog // by directory in the walker

// directoryCatalog captures all files in a given directory
type directoryCatalog struct {
// isAlbum bool // true when the directory is recognized as an album
// albumTitle string // album title from album's metadata
files map[string]fileInfo // map of fileInfo by base name
}

Expand All @@ -54,17 +54,13 @@ type jsonKey struct {
year int
}

// type fileWalkerPath struct {
// w archwalker.Walker
// p string
// }

func NewTakeout(ctx context.Context, jnl *logger.Journal, fsyss ...fs.FS) (*Takeout, error) {
func NewTakeout(ctx context.Context, jnl *logger.Journal, sm immich.SupportedMedia, fsyss ...fs.FS) (*Takeout, error) {
to := Takeout{
fsyss: fsyss,
jsonByYear: map[jsonKey]*GoogleMetaData{},
albums: map[string]string{},
jnl: jnl,
sm: sm,
}
err := to.passOne(ctx)
if err != nil {
Expand All @@ -82,7 +78,6 @@ func (to *Takeout) passOne(ctx context.Context) error {
to.catalogs = map[fs.FS]walkerCatalog{}
for _, w := range to.fsyss {
to.catalogs[w] = walkerCatalog{}
// wName := "" //w.Name()
err := to.passOneFsWalk(ctx, w)
if err != nil {
return err
Expand Down Expand Up @@ -144,31 +139,26 @@ func (to *Takeout) passOneFsWalk(ctx context.Context, w fs.FS) error {
return nil
}
default:

if fshelper.IsIgnoredExt(ext) {
to.jnl.AddEntry(name, logger.Discarded, "File ignored")
return nil
}

m, err := fshelper.MimeFromExt(ext)
if err != nil {
t := to.sm.TypeFromExt(ext)
switch t {
case immich.TypeUnknown:
to.jnl.AddEntry(name, logger.Unsupported, "")
return nil
}

if strings.Contains(name, "Failed Videos") {
to.jnl.AddEntry(name, logger.FailedVideo, "")
case immich.TypeIgnored:
to.jnl.AddEntry(name, logger.Discarded, "File ignored")
return nil
case immich.TypeVideo:
if strings.Contains(name, "Failed Videos") {
to.jnl.AddEntry(name, logger.FailedVideo, "")
return nil
}
to.jnl.AddEntry(name, logger.ScannedVideo, "")
case immich.TypeImage:
to.jnl.AddEntry(name, logger.ScannedImage, "")
}
dirCatalog.files[base] = fileInfo{
length: int(finfo.Size()),
}
ss := strings.Split(m[0], "/")
if ss[0] == "image" {
to.jnl.AddEntry(name, logger.ScannedImage, "")
} else {
to.jnl.AddEntry(name, logger.ScannedVideo, "")
}
}
to.catalogs[w][dir] = dirCatalog
return nil
Expand All @@ -191,7 +181,7 @@ func (to *Takeout) addJSON(dir, base string, md *GoogleMetaData) {
to.jsonByYear[k] = md
}

type matcherFn func(jsonName string, fileName string) bool
type matcherFn func(jsonName string, fileName string, sm immich.SupportedMedia) bool

// matchers is a list of matcherFn from the most likely to be used to the least one
var matchers = []matcherFn{
Expand Down Expand Up @@ -258,7 +248,7 @@ func (to *Takeout) solvePuzzle() {
l := to.catalogs[w][d]
for f := range l.files {
if l.files[f].md == nil {
if matcher(k.name, f) {
if matcher(k.name, f, to.sm) {
to.jnl.AddEntry(path.Join(d, f), logger.AssociatedMetadata, fmt.Sprintf("%s (%d)", k.name, k.year))
// if not already matched
i := l.files[f]
Expand All @@ -278,7 +268,7 @@ func (to *Takeout) solvePuzzle() {
//
// PXL_20230922_144936660.jpg.json
// PXL_20230922_144936660.jpg
func normalMatch(jsonName string, fileName string) bool {
func normalMatch(jsonName string, fileName string, sm immich.SupportedMedia) bool {
base := strings.TrimSuffix(jsonName, path.Ext(jsonName))
return base == fileName
}
Expand All @@ -294,10 +284,10 @@ func normalMatch(jsonName string, fileName string) bool {
// 😀😃😄😁😆😅😂🤣🥲☺️😊😇🙂🙃😉😌😍🥰😘😗😙😚😋.json
// 😀😃😄😁😆😅😂🤣🥲☺️😊😇🙂🙃😉😌😍🥰😘😗😙😚😋😛.jpg

func matchWithOneCharOmitted(jsonName string, fileName string) bool {
func matchWithOneCharOmitted(jsonName string, fileName string, sm immich.SupportedMedia) bool {
base := strings.TrimSuffix(jsonName, path.Ext(jsonName))
if strings.HasPrefix(fileName, base) {
if fshelper.IsExtensionPrefix(path.Ext(base)) {
if t := sm.TypeFromExt(path.Ext(base)); t == immich.TypeImage || t == immich.TypeVideo {
// Trim only if the EXT is known extension, and not .COVER or .ORIGINAL
base = strings.TrimSuffix(base, path.Ext(base))
}
Expand All @@ -314,7 +304,7 @@ func matchWithOneCharOmitted(jsonName string, fileName string) bool {
//
// Backyard_ceremony_wedding_photography_xxxxxxx_(494).json
// Backyard_ceremony_wedding_photography_xxxxxxx_m(494).jpg
func matchVeryLongNameWithNumber(jsonName string, fileName string) bool {
func matchVeryLongNameWithNumber(jsonName string, fileName string, sm immich.SupportedMedia) bool {
jsonName = strings.TrimSuffix(jsonName, path.Ext(jsonName))

p1JSON := strings.Index(jsonName, "(")
Expand All @@ -340,7 +330,7 @@ func matchVeryLongNameWithNumber(jsonName string, fileName string) bool {
//
// IMG_3479.JPG(2).json
// IMG_3479(2).JPG
func matchDuplicateInYear(jsonName string, fileName string) bool {
func matchDuplicateInYear(jsonName string, fileName string, sm immich.SupportedMedia) bool {
jsonName = strings.TrimSuffix(jsonName, path.Ext(jsonName))
p1JSON := strings.Index(jsonName, "(")
if p1JSON < 1 {
Expand All @@ -363,11 +353,11 @@ func matchDuplicateInYear(jsonName string, fileName string) bool {
// PXL_20220405_090123740.PORTRAIT.jpg
// PXL_20220405_090123740.PORTRAIT-modifié.jpg

func matchEditedName(jsonName string, fileName string) bool {
func matchEditedName(jsonName string, fileName string, sm immich.SupportedMedia) bool {
base := strings.TrimSuffix(jsonName, path.Ext(jsonName))
ext := path.Ext(base)
if ext != "" {
if _, err := fshelper.MimeFromExt(ext); err == nil {
if sm.IsMedia(ext) {
base := strings.TrimSuffix(base, ext)
fname := strings.TrimSuffix(fileName, path.Ext(fileName))
return strings.HasPrefix(fname, base)
Expand All @@ -383,7 +373,7 @@ func matchEditedName(jsonName string, fileName string) bool {
// original_1d4caa6f-16c6-4c3d-901b-9387de10e528_P.jpg
// original_1d4caa6f-16c6-4c3d-901b-9387de10e528_P(1).jpg

func matchForgottenDuplicates(jsonName string, fileName string) bool {
func matchForgottenDuplicates(jsonName string, fileName string, sm immich.SupportedMedia) bool {
jsonName = strings.TrimSuffix(jsonName, path.Ext(jsonName))
fileName = strings.TrimSuffix(fileName, path.Ext(fileName))
if strings.HasPrefix(fileName, jsonName) {
Expand Down Expand Up @@ -431,11 +421,11 @@ func (to *Takeout) passTwoWalk(ctx context.Context, w fs.FS, assetChan chan *bro
dir = strings.TrimSuffix(dir, "/")
ext := strings.ToLower(path.Ext(base))

if fshelper.IsIgnoredExt(ext) {
if to.sm.IsIgnoredExt(ext) {
return nil
}

if _, err := fshelper.MimeFromExt(ext); err != nil {
if !to.sm.IsMedia(ext) {
return nil
}
f, exist := to.catalogs[w][dir].files[base]
Expand Down
14 changes: 9 additions & 5 deletions browser/gp/googlephotos_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package gp

import "testing"
import (
"testing"

"github.com/simulot/immich-go/immich"
)

func Test_matchEditedName(t *testing.T) {
tests := []struct {
Expand Down Expand Up @@ -31,7 +35,7 @@ func Test_matchEditedName(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.fileName, func(t *testing.T) {
if got := matchEditedName(tt.jsonName, tt.fileName); got != tt.want {
if got := matchEditedName(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia); got != tt.want {
t.Errorf("matchEditedName() = %v, want %v", got, tt.want)
}
})
Expand All @@ -57,7 +61,7 @@ func Test_matchVeryLongNameWithNumber(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.fileName, func(t *testing.T) {
if got := matchVeryLongNameWithNumber(tt.jsonName, tt.fileName); got != tt.want {
if got := matchVeryLongNameWithNumber(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia); got != tt.want {
t.Errorf("matchVeryLongNameWithNumber() = %v, want %v", got, tt.want)
}
})
Expand Down Expand Up @@ -86,7 +90,7 @@ func Test_matchDuplicateInYear(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := matchDuplicateInYear(tt.jsonName, tt.fileName); got != tt.want {
if got := matchDuplicateInYear(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia); got != tt.want {
t.Errorf("matchDuplicateInYear() = %v, want %v", got, tt.want)
}
})
Expand Down Expand Up @@ -115,7 +119,7 @@ func Test_matchForgottenDuplicates(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := matchForgottenDuplicates(tt.jsonName, tt.fileName); got != tt.want {
if got := matchForgottenDuplicates(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia); got != tt.want {
t.Errorf("matchDuplicateInYear() = %v, want %v", got, tt.want)
}
})
Expand Down
3 changes: 2 additions & 1 deletion browser/gp/testgp_bigread_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"testing"

"github.com/simulot/immich-go/helpers/fshelper"
"github.com/simulot/immich-go/immich"
"github.com/simulot/immich-go/logger"
)

Expand All @@ -29,7 +30,7 @@ func TestReadBigTakeout(t *testing.T) {
}
cnt := 0
fsyss, err := fshelper.ParsePath(m, true)
to, err := NewTakeout(context.Background(), j, fsyss...)
to, err := NewTakeout(context.Background(), j, immich.DefaultSupportedMedia, fsyss...)
if err != nil {
t.Error(err)
return
Expand Down

0 comments on commit a47cca4

Please sign in to comment.