Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: get the list of supported extensions from the server #134 #164

Merged
merged 1 commit into from
Feb 19, 2024
Merged

feat: get the list of supported extensions from the server #134 #164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
46 changes: 15 additions & 31 deletions browser/files/localassets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"time"

"github.com/simulot/immich-go/browser"
"github.com/simulot/immich-go/helpers/fshelper"
"github.com/simulot/immich-go/immich"
"github.com/simulot/immich-go/immich/metadata"
"github.com/simulot/immich-go/logger"
)
Expand All @@ -19,13 +19,15 @@ type LocalAssetBrowser struct {
fsyss []fs.FS
albums map[string]string
log *logger.Journal
sm immich.SupportedMedia
}

func NewLocalFiles(ctx context.Context, log *logger.Journal, fsyss ...fs.FS) (*LocalAssetBrowser, error) {
func NewLocalFiles(ctx context.Context, log *logger.Journal, sm immich.SupportedMedia, fsyss ...fs.FS) (*LocalAssetBrowser, error) {
return &LocalAssetBrowser{
fsyss: fsyss,
albums: map[string]string{},
log: log,
sm: sm,
}, nil
}

Expand Down Expand Up @@ -78,19 +80,6 @@ func (la *LocalAssetBrowser) handleFolder(ctx context.Context, fsys fs.FS, fileC
return err
}

// fileMap := map[string][]fs.DirEntry{}
// for _, e := range entries {
// if e.IsDir() {
// continue
// }
// ext := path.Ext(e.Name())
// _, err := fshelper.MimeFromExt(ext)
// if strings.ToLower(ext) == ".xmp" || err == nil {
// base := strings.TrimSuffix(e.Name(), ext)
// fileMap[base] = append(fileMap[base], e)
// }
// }

for _, e := range entries {
if e.IsDir() {
continue
Expand All @@ -99,22 +88,18 @@ func (la *LocalAssetBrowser) handleFolder(ctx context.Context, fsys fs.FS, fileC
la.log.AddEntry(fileName, logger.DiscoveredFile, "")
name := e.Name()
ext := strings.ToLower(path.Ext(name))
if fshelper.IsMetadataExt(ext) {
la.log.AddEntry(name, logger.Metadata, "")
continue
} else if fshelper.IsIgnoredExt(ext) {

t := la.sm.TypeFromExt(ext)
switch t {
default:
la.log.AddEntry(fileName, logger.Unsupported, "")
continue
}
m, err := fshelper.MimeFromExt(strings.ToLower(ext))
if err != nil {
la.log.AddEntry(fileName, logger.Unsupported, "")
case immich.TypeSidecar:
la.log.AddEntry(name, logger.Metadata, "")
continue
}
ss := strings.Split(m[0], "/")
if ss[0] == "image" {
case immich.TypeImage:
la.log.AddEntry(name, logger.ScannedImage, "")
} else {
case immich.TypeVideo:
la.log.AddEntry(name, logger.ScannedVideo, "")
}

Expand Down Expand Up @@ -154,7 +139,7 @@ func (la *LocalAssetBrowser) handleFolder(ctx context.Context, fsys fs.FS, fileC
}

func (la *LocalAssetBrowser) checkSidecar(f *browser.LocalAssetFile, entries []fs.DirEntry, dir, name string) bool {
assetBase := baseNames(name)
assetBase := la.baseNames(name)

for _, name := range assetBase {
xmp := name + ".[xX][mM][pP]"
Expand All @@ -176,16 +161,15 @@ func (la *LocalAssetBrowser) checkSidecar(f *browser.LocalAssetFile, entries []f
return false
}

func baseNames(n string) []string {
func (la *LocalAssetBrowser) baseNames(n string) []string {
n = escapeName(n)
names := []string{n}
ext := path.Ext(n)
for {
if ext == "" {
return names
}
_, err := fshelper.MimeFromExt(ext)
if err != nil {
if la.sm.TypeFromExt(ext) == "" {
return names
}
n = strings.TrimSuffix(n, ext)
Expand Down
3 changes: 2 additions & 1 deletion browser/files/localassets_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/kr/pretty"
"github.com/psanford/memfs"
"github.com/simulot/immich-go/browser/files"
"github.com/simulot/immich-go/immich"
"github.com/simulot/immich-go/logger"
)

Expand Down Expand Up @@ -74,7 +75,7 @@ func TestLocalAssets(t *testing.T) {
}
ctx := context.Background()

b, err := files.NewLocalFiles(ctx, logger.NewJournal(logger.NoLog{}), fsys)
b, err := files.NewLocalFiles(ctx, logger.NewJournal(logger.NoLog{}), immich.DefaultSupportedMedia, fsys)
if err != nil {
t.Error(err)
}
Expand Down
68 changes: 29 additions & 39 deletions browser/gp/googlephotos.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/simulot/immich-go/browser"
"github.com/simulot/immich-go/helpers/fshelper"
"github.com/simulot/immich-go/helpers/gen"
"github.com/simulot/immich-go/immich"
"github.com/simulot/immich-go/logger"
)

Expand All @@ -23,15 +24,14 @@ type Takeout struct {
uploaded map[fileKey]any // track files already uploaded
albums map[string]string // tack album names by folder
jnl *logger.Journal
sm immich.SupportedMedia
}

// walkerCatalog collects all directory catalogs
type walkerCatalog map[string]directoryCatalog // by directory in the walker

// directoryCatalog captures all files in a given directory
type directoryCatalog struct {
// isAlbum bool // true when the directory is recognized as an album
// albumTitle string // album title from album's metadata
files map[string]fileInfo // map of fileInfo by base name
}

Expand All @@ -54,17 +54,13 @@ type jsonKey struct {
year int
}

// type fileWalkerPath struct {
// w archwalker.Walker
// p string
// }

func NewTakeout(ctx context.Context, jnl *logger.Journal, fsyss ...fs.FS) (*Takeout, error) {
func NewTakeout(ctx context.Context, jnl *logger.Journal, sm immich.SupportedMedia, fsyss ...fs.FS) (*Takeout, error) {
to := Takeout{
fsyss: fsyss,
jsonByYear: map[jsonKey]*GoogleMetaData{},
albums: map[string]string{},
jnl: jnl,
sm: sm,
}
err := to.passOne(ctx)
if err != nil {
Expand All @@ -82,7 +78,6 @@ func (to *Takeout) passOne(ctx context.Context) error {
to.catalogs = map[fs.FS]walkerCatalog{}
for _, w := range to.fsyss {
to.catalogs[w] = walkerCatalog{}
// wName := "" //w.Name()
err := to.passOneFsWalk(ctx, w)
if err != nil {
return err
Expand Down Expand Up @@ -144,31 +139,26 @@ func (to *Takeout) passOneFsWalk(ctx context.Context, w fs.FS) error {
return nil
}
default:

if fshelper.IsIgnoredExt(ext) {
to.jnl.AddEntry(name, logger.Discarded, "File ignored")
return nil
}

m, err := fshelper.MimeFromExt(ext)
if err != nil {
t := to.sm.TypeFromExt(ext)
switch t {
case immich.TypeUnknown:
to.jnl.AddEntry(name, logger.Unsupported, "")
return nil
}

if strings.Contains(name, "Failed Videos") {
to.jnl.AddEntry(name, logger.FailedVideo, "")
case immich.TypeIgnored:
to.jnl.AddEntry(name, logger.Discarded, "File ignored")
return nil
case immich.TypeVideo:
if strings.Contains(name, "Failed Videos") {
to.jnl.AddEntry(name, logger.FailedVideo, "")
return nil
}
to.jnl.AddEntry(name, logger.ScannedVideo, "")
case immich.TypeImage:
to.jnl.AddEntry(name, logger.ScannedImage, "")
}
dirCatalog.files[base] = fileInfo{
length: int(finfo.Size()),
}
ss := strings.Split(m[0], "/")
if ss[0] == "image" {
to.jnl.AddEntry(name, logger.ScannedImage, "")
} else {
to.jnl.AddEntry(name, logger.ScannedVideo, "")
}
}
to.catalogs[w][dir] = dirCatalog
return nil
Expand All @@ -191,7 +181,7 @@ func (to *Takeout) addJSON(dir, base string, md *GoogleMetaData) {
to.jsonByYear[k] = md
}

type matcherFn func(jsonName string, fileName string) bool
type matcherFn func(jsonName string, fileName string, sm immich.SupportedMedia) bool

// matchers is a list of matcherFn from the most likely to be used to the least one
var matchers = []matcherFn{
Expand Down Expand Up @@ -258,7 +248,7 @@ func (to *Takeout) solvePuzzle() {
l := to.catalogs[w][d]
for f := range l.files {
if l.files[f].md == nil {
if matcher(k.name, f) {
if matcher(k.name, f, to.sm) {
to.jnl.AddEntry(path.Join(d, f), logger.AssociatedMetadata, fmt.Sprintf("%s (%d)", k.name, k.year))
// if not already matched
i := l.files[f]
Expand All @@ -278,7 +268,7 @@ func (to *Takeout) solvePuzzle() {
//
// PXL_20230922_144936660.jpg.json
// PXL_20230922_144936660.jpg
func normalMatch(jsonName string, fileName string) bool {
func normalMatch(jsonName string, fileName string, sm immich.SupportedMedia) bool {
base := strings.TrimSuffix(jsonName, path.Ext(jsonName))
return base == fileName
}
Expand All @@ -294,10 +284,10 @@ func normalMatch(jsonName string, fileName string) bool {
// 😀😃😄😁😆😅😂🤣🥲☺️😊😇🙂🙃😉😌😍🥰😘😗😙😚😋.json
// 😀😃😄😁😆😅😂🤣🥲☺️😊😇🙂🙃😉😌😍🥰😘😗😙😚😋😛.jpg

func matchWithOneCharOmitted(jsonName string, fileName string) bool {
func matchWithOneCharOmitted(jsonName string, fileName string, sm immich.SupportedMedia) bool {
base := strings.TrimSuffix(jsonName, path.Ext(jsonName))
if strings.HasPrefix(fileName, base) {
if fshelper.IsExtensionPrefix(path.Ext(base)) {
if t := sm.TypeFromExt(path.Ext(base)); t == immich.TypeImage || t == immich.TypeVideo {
// Trim only if the EXT is known extension, and not .COVER or .ORIGINAL
base = strings.TrimSuffix(base, path.Ext(base))
}
Expand All @@ -314,7 +304,7 @@ func matchWithOneCharOmitted(jsonName string, fileName string) bool {
//
// Backyard_ceremony_wedding_photography_xxxxxxx_(494).json
// Backyard_ceremony_wedding_photography_xxxxxxx_m(494).jpg
func matchVeryLongNameWithNumber(jsonName string, fileName string) bool {
func matchVeryLongNameWithNumber(jsonName string, fileName string, sm immich.SupportedMedia) bool {
jsonName = strings.TrimSuffix(jsonName, path.Ext(jsonName))

p1JSON := strings.Index(jsonName, "(")
Expand All @@ -340,7 +330,7 @@ func matchVeryLongNameWithNumber(jsonName string, fileName string) bool {
//
// IMG_3479.JPG(2).json
// IMG_3479(2).JPG
func matchDuplicateInYear(jsonName string, fileName string) bool {
func matchDuplicateInYear(jsonName string, fileName string, sm immich.SupportedMedia) bool {
jsonName = strings.TrimSuffix(jsonName, path.Ext(jsonName))
p1JSON := strings.Index(jsonName, "(")
if p1JSON < 1 {
Expand All @@ -363,11 +353,11 @@ func matchDuplicateInYear(jsonName string, fileName string) bool {
// PXL_20220405_090123740.PORTRAIT.jpg
// PXL_20220405_090123740.PORTRAIT-modifié.jpg

func matchEditedName(jsonName string, fileName string) bool {
func matchEditedName(jsonName string, fileName string, sm immich.SupportedMedia) bool {
base := strings.TrimSuffix(jsonName, path.Ext(jsonName))
ext := path.Ext(base)
if ext != "" {
if _, err := fshelper.MimeFromExt(ext); err == nil {
if sm.IsMedia(ext) {
base := strings.TrimSuffix(base, ext)
fname := strings.TrimSuffix(fileName, path.Ext(fileName))
return strings.HasPrefix(fname, base)
Expand All @@ -383,7 +373,7 @@ func matchEditedName(jsonName string, fileName string) bool {
// original_1d4caa6f-16c6-4c3d-901b-9387de10e528_P.jpg
// original_1d4caa6f-16c6-4c3d-901b-9387de10e528_P(1).jpg

func matchForgottenDuplicates(jsonName string, fileName string) bool {
func matchForgottenDuplicates(jsonName string, fileName string, sm immich.SupportedMedia) bool {
jsonName = strings.TrimSuffix(jsonName, path.Ext(jsonName))
fileName = strings.TrimSuffix(fileName, path.Ext(fileName))
if strings.HasPrefix(fileName, jsonName) {
Expand Down Expand Up @@ -431,11 +421,11 @@ func (to *Takeout) passTwoWalk(ctx context.Context, w fs.FS, assetChan chan *bro
dir = strings.TrimSuffix(dir, "/")
ext := strings.ToLower(path.Ext(base))

if fshelper.IsIgnoredExt(ext) {
if to.sm.IsIgnoredExt(ext) {
return nil
}

if _, err := fshelper.MimeFromExt(ext); err != nil {
if !to.sm.IsMedia(ext) {
return nil
}
f, exist := to.catalogs[w][dir].files[base]
Expand Down
14 changes: 9 additions & 5 deletions browser/gp/googlephotos_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package gp

import "testing"
import (
"testing"

"github.com/simulot/immich-go/immich"
)

func Test_matchEditedName(t *testing.T) {
tests := []struct {
Expand Down Expand Up @@ -31,7 +35,7 @@ func Test_matchEditedName(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.fileName, func(t *testing.T) {
if got := matchEditedName(tt.jsonName, tt.fileName); got != tt.want {
if got := matchEditedName(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia); got != tt.want {
t.Errorf("matchEditedName() = %v, want %v", got, tt.want)
}
})
Expand All @@ -57,7 +61,7 @@ func Test_matchVeryLongNameWithNumber(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.fileName, func(t *testing.T) {
if got := matchVeryLongNameWithNumber(tt.jsonName, tt.fileName); got != tt.want {
if got := matchVeryLongNameWithNumber(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia); got != tt.want {
t.Errorf("matchVeryLongNameWithNumber() = %v, want %v", got, tt.want)
}
})
Expand Down Expand Up @@ -86,7 +90,7 @@ func Test_matchDuplicateInYear(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := matchDuplicateInYear(tt.jsonName, tt.fileName); got != tt.want {
if got := matchDuplicateInYear(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia); got != tt.want {
t.Errorf("matchDuplicateInYear() = %v, want %v", got, tt.want)
}
})
Expand Down Expand Up @@ -115,7 +119,7 @@ func Test_matchForgottenDuplicates(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := matchForgottenDuplicates(tt.jsonName, tt.fileName); got != tt.want {
if got := matchForgottenDuplicates(tt.jsonName, tt.fileName, immich.DefaultSupportedMedia); got != tt.want {
t.Errorf("matchDuplicateInYear() = %v, want %v", got, tt.want)
}
})
Expand Down
3 changes: 2 additions & 1 deletion browser/gp/testgp_bigread_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"testing"

"github.com/simulot/immich-go/helpers/fshelper"
"github.com/simulot/immich-go/immich"
"github.com/simulot/immich-go/logger"
)

Expand All @@ -29,7 +30,7 @@ func TestReadBigTakeout(t *testing.T) {
}
cnt := 0
fsyss, err := fshelper.ParsePath(m, true)
to, err := NewTakeout(context.Background(), j, fsyss...)
to, err := NewTakeout(context.Background(), j, immich.DefaultSupportedMedia, fsyss...)
if err != nil {
t.Error(err)
return
Expand Down