simulot
GetAllAssets takes so long, how to skip?
Background:
Happily used immich-go before, to migrate from google, now, my library is at 123646 photos and growing. haven't used immich-go since, but recently wanted to make an automated import script when I plug my SD card.
My script runs the immich-go instead of immich-cli mostly because ur solution is a binary with no dependency hell :D
Issue
Current issue is that I see a lot of GetAllAssets, and they take 2m31s total of just waiting before even uploading begins.
2025-05-30T11:41:10+05:00 RESPONSE 85 GetAllAssets POST https://<REDACTED>/api/search/metadata
Header:
Server : <REDACTED>
X-Immich-Cid : <REDACTED>
X-Powered-By : Express
Alt-Svc : h3=":443"; ma=2592000
Content-Length : 139791
Content-Type : application/json; charset=utf-8
Date : Fri, 30 May 2025 06:41:11 GMT
Etag : "<REDACTED>"
Status: 200 OK
-- response body start --
<REDACTED>
-- response body end --
so...I gotta wait till all of the 85 GetAllAssets with Content-Length : 139791 pass, everytime before the upload starts.
just in case, my command:
immich-go upload from-folder --server=<REDACTED> --api-key=<REDACTED> --no-ui --pause-immich-jobs=false --api-trace --session-tag --manage-heic-jpeg=StackCoverHeic --manage-raw-jpeg=StackCoverJPG /Volumes/Fuji2/DCIM/937_FUJI
Proposed solution
- I have glanced over the -h and did not see if there is a way to put argument to skip the check, but if one exists, it 100% be a solution.
- I can try to implement it myself, but I will require some help understanding what is actually happening, and why the check exists at all, and if it is crucial or unskippable. I can try to investigate myself, but some help would be great
- If it is an easy to implement thing that can be done fast by someone who already knows the codebase and it is more efficient that way, I think it could be done by someone from existing contributors :D
My script runs the immich-go instead of immich-cli mostly because ur solution is a binary with no dependency hell :D
Yeap, that was my motivation for starting this project!
The Immich API is designed to handle duplicates, but there have been edge cases that caused internal errors and connection losses. This is why the list of Immich assets is downloaded upfront.
I'm not sure about the current status of this issue. The upload logic depends on this check. So I would be cautious about removing it.
I had the same problem. The initialization time was unbearable with too many assets and albums.
I found that GetAllAssets and GetImmichAlbums weren't concurrency. After making them concurrently, initialization became very fast.
Of course, this is just a workaround. IMO, the best solution is to set up a local index cache, like the standard Immich client does.
Patch:
diff --git a/app/cmd/upload/run.go b/app/cmd/upload/run.go
index 57cc83b..1b45fb3 100644
--- a/app/cmd/upload/run.go
+++ b/app/cmd/upload/run.go
@@ -215,27 +215,31 @@ func (upCmd *UpCmd) getImmichAlbums(ctx context.Context) error {
return fmt.Errorf("can't get the album list from the server: %w", err)
}
+ wg := sync.WaitGroup{}
+ lock := sync.Mutex{}
select {
case <-ctx.Done():
return ctx.Err()
case <-upCmd.immichAssetsReady:
// Wait for the server's assets to be ready.
for _, a := range serverAlbums {
- select {
- case <-ctx.Done():
- return ctx.Err()
- default:
+ wg.Add(1)
+ go func(a immich.AlbumSimplified) {
+ defer wg.Done()
// Get the album info from the server, with assets.
r, err := upCmd.app.Client().Immich.GetAlbumInfo(ctx, a.ID, false)
if err != nil {
upCmd.app.Log().Error("can't get the album info from the server", "album", a.AlbumName, "err", err)
- continue
+ return
}
ids := make([]string, 0, len(r.Assets))
for _, aa := range r.Assets {
ids = append(ids, aa.ID)
}
+ lock.Lock()
+ defer lock.Unlock()
+
album := assets.NewAlbum(a.ID, a.AlbumName, a.Description)
upCmd.albumsCache.NewCollection(a.AlbumName, album, ids)
upCmd.app.Log().Info("got album from the server", "album", a.AlbumName, "assets", len(r.Assets))
@@ -249,8 +253,9 @@ func (upCmd *UpCmd) getImmichAlbums(ctx context.Context) error {
}
a.Albums = append(a.Albums, album)
}
- }
+ }(a)
}
+ wg.Wait()
}
return nil
}
diff --git a/immich/metadata.go b/immich/metadata.go
index 005775b..c37d635 100644
--- a/immich/metadata.go
+++ b/immich/metadata.go
@@ -2,6 +2,7 @@ package immich
import (
"context"
+ "sync"
)
type searchMetadataResponse struct {
@@ -31,31 +32,79 @@ type SearchMetadataQuery struct {
OriginalFileName string `json:"originalFileName,omitempty"`
}
+type pageResult struct {
+ nextPage int
+ err error
+}
+
func (ic *ImmichClient) callSearchMetadata(ctx context.Context, query *SearchMetadataQuery, filter func(*Asset) error) error {
query.Page = 1
query.Size = 1000
+
+ const maxWorkers = 20
+ currentPage := 1
+
+ filterLock := sync.Mutex{}
+
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
- resp := searchMetadataResponse{}
- err := ic.newServerCall(ctx, EndPointGetAllAssets).do(postRequest("/search/metadata", "application/json", setJSONBody(&query), setAcceptJSON()), responseJSON(&resp))
- if err != nil {
- return err
+ resultChan := make(chan pageResult, maxWorkers)
+ var wg sync.WaitGroup
+
+ for pageNum := currentPage; pageNum < currentPage+maxWorkers; pageNum++ {
+ wg.Add(1)
+ go func(page int) {
+ defer wg.Done()
+
+ pageQuery := *query
+ pageQuery.Page = page
+
+ resp := searchMetadataResponse{}
+ err := ic.newServerCall(ctx, EndPointGetAllAssets).do(postRequest("/search/metadata", "application/json", setJSONBody(&pageQuery), setAcceptJSON()), responseJSON(&resp))
+ if err != nil {
+ resultChan <- pageResult{nextPage: 0, err: err}
+ return
+ }
+
+ for _, a := range resp.Assets.Items {
+ filterLock.Lock()
+ err = filter(a)
+ filterLock.Unlock()
+ if err != nil {
+ resultChan <- pageResult{nextPage: 0, err: err}
+ return
+ }
+ }
+
+ resultChan <- pageResult{nextPage: resp.Assets.NextPage, err: nil}
+ }(pageNum)
}
- for _, a := range resp.Assets.Items {
- err = filter(a)
- if err != nil {
- return err
+ wg.Wait()
+ close(resultChan)
+
+ maxNextPage := 0
+
+ for result := range resultChan {
+ if result.err != nil {
+ return result.err
+ }
+
+ if result.nextPage > 0 {
+ if result.nextPage > maxNextPage {
+ maxNextPage = result.nextPage
+ }
}
}
- if resp.Assets.NextPage == 0 {
+ if maxNextPage <= currentPage {
return nil
}
- query.Page = resp.Assets.NextPage
+
+ currentPage = maxNextPage
}
}
}
