previewgenerator
previewgenerator copied to clipboard
nextcloud
Memory allocation error (with huge CPU usage) with a picture too big
How to use GitHub
- Please use the 👍 reaction to show that you are affected by the same issue.
- Please don't comment if you have no relevant information to add. It's just extra noise for everyone subscribed to this issue.
- Subscribe to receive notifications on status change and new comments.
Steps to reproduce
- Upload a 200Mpx image made with a Samsung S24 Ultra or S25 Ultra
- Try to generate a preview
Expected behaviour
The preview should work
Actual behaviour
Using the OCC command, the response is:
In Generator.php line 366:
[OCP\Files\NotFoundException]
No provider successfully handled the preview generation
Exception trace:
at /var/www/html/lib/private/Preview/Generator.php:366
OC\Preview\Generator->generateProviderPreview() at /var/www/html/lib/private/Preview/Generator.php:313
OC\Preview\Generator->getMaxPreview() at /var/www/html/lib/private/Preview/Generator.php:117
OC\Preview\Generator->generatePreviews() at /var/www/html/lib/private/PreviewManager.php:182
OC\PreviewManager->generatePreviews() at /var/www/html/core/Command/Preview/Generate.php:92
OC\Core\Command\Preview\Generate->execute() at /var/www/html/3rdparty/symfony/console/Command/Command.php:326
Symfony\Component\Console\Command\Command->run() at /var/www/html/3rdparty/symfony/console/Application.php:1078
Symfony\Component\Console\Application->doRunCommand() at /var/www/html/3rdparty/symfony/console/Application.php:324
Symfony\Component\Console\Application->doRun() at /var/www/html/3rdparty/symfony/console/Application.php:175
Symfony\Component\Console\Application->run() at /var/www/html/lib/private/Console/Application.php:187
OC\Console\Application->run() at /var/www/html/console.php:87
require_once() at /var/www/html/occ:33
preview:generate [-s|--size SIZE] [-c|--crop] [-m|--mode MODE] [--] <file>
In Nextcloud logs, I get Memory allocation error: Security limit exceeded: Allocating (size) exceeds the security limit of 536870912 bytes (6.1000)
The CPU usage then stays huge.
Server configuration
Web server: Apache
Database: pgsql
PHP version: 8.3.22
Nextcloud version: Nextcloud AIO v11.1.0
List of activated apps
- activity: 4.0.0
- admin_audit: 1.21.0
- bruteforcesettings: 4.0.0
- calendar: 5.3.3
- cfg_share_links: 7.0.1
- circles: 31.0.0
- cloud_federation_api: 1.14.0
- cloud_py_api: 0.2.0
- comments: 1.21.0
- contacts: 7.1.3
- contactsinteraction: 1.12.0
- cookbook: 0.11.3
- cospend: 3.0.11
- dashboard: 7.11.0
- dav: 1.33.0
- deck: 1.15.1
- duplicatefinder: 1.7.3
- epubviewer: 1.8.0
- federatedfilesharing: 1.21.0
- federation: 1.21.0
- files: 2.3.1
- files_downloadlimit: 4.0.0
- files_external: 1.23.0
- files_fulltextsearch: 31.0.0
- files_pdfviewer: 4.0.0
- files_reminders: 1.4.0
- files_sharing: 1.23.1
- files_trashbin: 1.21.0
- files_versions: 1.24.0
- firstrunwizard: 4.0.0
- fulltextsearch: 31.0.0
- fulltextsearch_elasticsearch: 31.0.0
- geoblocker: 0.5.16
- gpxpod: 7.0.4
- group_everyone: 0.1.17
- logreader: 4.0.0
- lookup_server_connector: 1.19.0
- mail: 5.1.4
- mediadc: 0.4.0
- memories: 7.5.2
- music: 2.2.0
- nextcloud-aio: 0.8.0
- nextcloud_announcements: 3.0.0
- notes: 4.12.1
- notifications: 4.0.0
- notify_push: 1.1.0
- oauth2: 1.19.1
- password_policy: 3.0.0
- photos: 4.0.0-dev.1
- previewgenerator: 5.8.0
- privacy: 3.0.0
- profile: 1.0.0
- provisioning_api: 1.21.0
- recognize: 9.0.1
- recommendations: 4.0.0
- related_resources: 2.0.0
- richdocuments: 8.7.1
- secrets: 2.1.2
- serverinfo: 3.0.0
- settings: 1.14.0
- sharebymail: 1.21.0
- spreed: 21.1.0
- support: 3.0.0
- suspicious_login: 9.0.1
- systemtags: 1.21.1
- tasks: 0.16.1
- text: 5.0.0
- theming: 2.6.1
- twofactor_backupcodes: 1.20.0
- twofactor_totp: 13.0.0-dev.0
- twofactor_webauthn: 2.2.0
- updatenotification: 1.21.0
- user_status: 1.11.0
- viewer: 4.0.0
- weather_status: 1.11.0
- webhook_listeners: 1.2.0
- whiteboard: 1.1.1
- workflowengine: 2.13.0
Nextcloud configuration
{
"system": {
"one-click-instance": true,
"one-click-instance.user-limit": 100,
"memcache.local": "\\OC\\Memcache\\APCu",
"apps_paths": [
{
"path": "\/var\/www\/html\/apps",
"url": "\/apps",
"writable": false
},
{
"path": "\/var\/www\/html\/custom_apps",
"url": "\/custom_apps",
"writable": true
}
],
"appsallowlist": false,
"check_data_directory_permissions": false,
"memcache.distributed": "\\OC\\Memcache\\Redis",
"memcache.locking": "\\OC\\Memcache\\Redis",
"redis": {
"host": "***REMOVED SENSITIVE VALUE***",
"password": "***REMOVED SENSITIVE VALUE***",
"port": 6379
},
"overwritehost": "***REMOVED SENSITIVE VALUE***",
"overwriteprotocol": "https",
"passwordsalt": "***REMOVED SENSITIVE VALUE***",
"secret": "***REMOVED SENSITIVE VALUE***",
"trusted_domains": [
"***REMOVED SENSITIVE VALUE***"
],
"datadirectory": "***REMOVED SENSITIVE VALUE***",
"dbtype": "pgsql",
"version": "31.0.6.2",
"overwrite.cli.url": "***REMOVED SENSITIVE VALUE***",
"dbname": "***REMOVED SENSITIVE VALUE***",
"dbhost": "***REMOVED SENSITIVE VALUE***",
"dbport": "",
"dbtableprefix": "oc_",
"dbuser": "***REMOVED SENSITIVE VALUE***",
"dbpassword": "***REMOVED SENSITIVE VALUE***",
"installed": true,
"instanceid": "***REMOVED SENSITIVE VALUE***",
"maintenance": false,
"updatedirectory": "\/nc-updater",
"loglevel": 0,
"log_type": "file",
"logfile": "\/var\/www\/html\/data\/nextcloud.log",
"log_rotate_size": "10485760",
"log.condition": {
"apps": [
"admin_audit"
]
},
"preview_max_x": 2048,
"preview_max_y": 2048,
"preview_max_filesize_image": -1,
"preview_max_memory": -1,
"jpeg_quality": 60,
"enabledPreviewProviders": {
"0": "OC\\Preview\\Imaginary",
"1": "OC\\Preview\\MarkDown",
"2": "OC\\Preview\\MP3",
"3": "OC\\Preview\\TXT",
"4": "OC\\Preview\\OpenDocument",
"5": "OC\\Preview\\Movie",
"6": "OC\\Preview\\Krita",
"7": "OC\\Preview\\ImaginaryPDF",
"8": "OC\\Preview\\HEIC",
"9": "OC\\Preview\\TIFF",
"23": "OC\\Preview\\ImaginaryPDF"
},
"enable_previews": true,
"upgrade.disable-web": true,
"mail_smtpmode": "smtp",
"trashbin_retention_obligation": "auto, 30",
"versions_retention_obligation": "auto, 30",
"activity_expire_days": "30",
"simpleSignUpLink.shown": false,
"share_folder": "\/Shared",
"one-click-instance.link": "https:\/\/nextcloud.com\/all-in-one\/",
"upgrade.cli-upgrade-link": "https:\/\/github.com\/nextcloud\/all-in-one\/discussions\/2726",
"allow_local_remote_servers": true,
"davstorage.request_timeout": 7200,
"htaccess.RewriteBase": "\/",
"dbpersistent": false,
"files_external_allow_create_new_local": false,
"trusted_proxies": "***REMOVED SENSITIVE VALUE***",
"preview_imaginary_url": "***REMOVED SENSITIVE VALUE***",
"defaultapp": "",
"twofactor_enforced": "false",
"twofactor_enforced_groups": [],
"twofactor_enforced_excluded_groups": [],
"maintenance_window_start": 100,
"preview_imaginary_key": "***REMOVED SENSITIVE VALUE***",
"auth.bruteforce.protection.enabled": true,
"ratelimit.protection.enabled": true,
"default_phone_region": "FR",
"mail_smtpname": "***REMOVED SENSITIVE VALUE***",
"mail_smtppassword": "***REMOVED SENSITIVE VALUE***",
"mail_domain": "***REMOVED SENSITIVE VALUE***",
"mail_from_address": "***REMOVED SENSITIVE VALUE***",
"mail_smtphost": "***REMOVED SENSITIVE VALUE***",
"mail_smtpauth": 1,
"mail_smtpport": "465",
"mail_sendmailmode": "smtp",
"mail_smtpsecure": "ssl",
"documentation_url.server_logs": "https:\/\/github.com\/nextcloud\/all-in-one\/discussions\/5425",
"memories.db.triggers.fcu": true,
"memories.exiftool": "\/var\/www\/html\/custom_apps\/memories\/bin-ext\/exiftool-aarch64-musl",
"memories.vod.path": "\/var\/www\/html\/custom_apps\/memories\/bin-ext\/go-vod-aarch64",
"memories.vod.ffmpeg": "\/usr\/bin\/ffmpeg",
"memories.vod.ffprobe": "\/usr\/bin\/ffprobe",
"updatechecker": false,
"DOMAIN": "***REMOVED SENSITIVE VALUE***"
}
}
The issue seems to be coming from sharp and libvips according to my researchs. It has been discussed on immich:
- https://github.com/immich-app/immich/issues/17500
- https://github.com/immich-app/immich/issues/14316
Here are some MR on libvips that seem to talk about this: https://github.com/libvips/libvips/pull/4398 https://github.com/libvips/libvips/pull/4399 Is this related ?
This is likely a problem with your image library linking against libheif (e.g., Imagick) and setting an insufficient maximum memory limit, not Nextcloud itself. Ref https://github.com/ImageMagick/ImageMagick/issues/8109
@gu1lhem @maxi1134 @CodyTray Could one of you please run the following command on their Nextcloud server/container:
magick --version
If your version is new enough, the problem should be fixed by simply adjusting the policy of Imagick (to allow bigger allocations for pictures). You might also need to increase the maximum allowed memory size for PHP accordingly.
Regarding libvips: I assume you are using the Imaginay container to generate previews? We might be out of luck since this requires updating the library inside the container as the referenced PRs are rather recent. However, the Imaginary project is pretty much dead at this point so you would need to do it yourself or hope for a fork.
@gu1lhem @maxi1134 @CodyTray Could one of you please run the following command on their Nextcloud server/container:
magick --version
maxi1134@diaspora:~$ sudo docker exec -it nextcloud-app-1 /bin/bash
[sudo] password for maxi1134:
root@b85b6a65474d:/var/www/html# magick --version
Version: ImageMagick 7.1.1-43 Q16 x86_64 22550 https://imagemagick.org
Copyright: (C) 1999 ImageMagick Studio LLC
License: https://imagemagick.org/script/license.php
Features: Cipher DPC Modules OpenMP(4.5)
Delegates (built-in): bzlib djvu fftw fontconfig freetype heic jbig jng jp2 jpeg lcms lqr ltdl lzma openexr pangocairo png raw tiff webp wmf x xml zlib zstd
Compiler: gcc (14.2)
Here is the result!
Also, this is my current policy:
<policymap>
<policy domain="Undefined" rights="none"/>
<!-- Set maximum parallel threads. -->
<!-- <policy domain="resource" name="thread" value="2"/> -->
<!-- Set maximum time in seconds. When this limit is exceeded, an exception
is thrown and processing stops. -->
<!-- <policy domain="resource" name="time" value="120"/> -->
<!-- Set maximum number of open pixel cache files. When this limit is
exceeded, any subsequent pixels cached to disk are closed and reopened
on demand. -->
<!-- <policy domain="resource" name="file" value="768"/> -->
<policy domain="resource" name="memory" value="10240MiB"/>
<!-- Set maximum amount of memory map in bytes to allocate for the pixel
cache. When this limit is exceeded, the image pixels are cached to
disk. -->
<policy domain="resource" name="map" value="20480MiB"/>
<!-- Set the maximum width * height of an image that can reside in the pixel
cache memory. Images that exceed the area limit are cached to disk. -->
<policy domain="resource" name="area" value="2560MP"/>
<!-- Set maximum amount of disk space in bytes permitted for use by the pixel
cache. When this limit is exceeded, the pixel cache is not be created
and an exception is thrown. -->
<policy domain="resource" name="disk" value="20GiB"/>
<!-- Set the maximum length of an image sequence. When this limit is
exceeded, an exception is thrown. -->
<!-- <policy domain="resource" name="list-length" value="32"/> -->
<!-- Set the maximum width of an image. When this limit is exceeded, an
exception is thrown. -->
<policy domain="resource" name="width" value="320KP"/>
<!-- Set the maximum height of an image. When this limit is exceeded, an
exception is thrown. -->
<policy domain="resource" name="height" value="320KP"/>
<!-- Periodically yield the CPU for at least the time specified in
milliseconds. -->
<!-- <policy domain="resource" name="throttle" value="2"/> -->
<!-- Do not create temporary files in the default shared directories, instead
specify a private area to store only ImageMagick temporary files. -->
<!-- <policy domain="resource" name="temporary-path" value="/magick/tmp/"/> -->
<!-- Force memory initialization by memory mapping select memory
allocations. -->
<!-- <policy domain="cache" name="memory-map" value="anonymous"/> -->
<!-- Ensure all image data is fully flushed and synchronized to disk. -->
<!-- <policy domain="cache" name="synchronize" value="true"/> -->
<!-- Replace passphrase for secure distributed processing -->
<!-- <policy domain="cache" name="shared-secret" value="secret-passphrase" stealth="true"/> -->
<!-- Do not permit any delegates to execute. -->
<!-- <policy domain="delegate" rights="none" pattern="*"/> -->
<!-- Do not permit any image filters to load. -->
<!-- <policy domain="filter" rights="none" pattern="*"/> -->
<!-- Don't read/write from/to stdin/stdout. -->
<!-- <policy domain="path" rights="none" pattern="-"/> -->
<!-- don't read sensitive paths. -->
<!-- <policy domain="path" rights="none" pattern="/etc/*"/> -->
<!-- Indirect reads are not permitted. -->
<policy domain="path" rights="none" pattern="@*"/>
<!-- These image types are security risks on read, but write is fine -->
<!-- <policy domain="module" rights="write" pattern="{MSL,MVG,PS,SVG,URL,XPS}"/> -->
<!-- This policy sets the number of times to replace content of certain
memory buffers and temporary files before they are freed or deleted. -->
<!-- <policy domain="system" name="shred" value="1"/> -->
<!-- Enable the initialization of buffers with zeros, resulting in a minor
performance penalty but with improved security. -->
<!-- <policy domain="system" name="memory-map" value="anonymous"/> -->
<!-- Set the maximum amount of memory in bytes that are permitted for
allocation requests. -->
<!-- <policy domain="system" name="max-memory-request" value="256MiB"/> -->
<policy domain="delegate" rights="none" pattern="URL" />
<policy domain="delegate" rights="none" pattern="HTTPS" />
<policy domain="delegate" rights="none" pattern="HTTP" />
</policymap>