Ylianst
Issue when downloading files
Describe the bug
when downloading files sometimes though the file option
https://remote.xxxxx.com/devicefile.ashx?c=9L2lf@DyCZbDi7Tvd9BMU0DJQnpajANA6kCar7fztnlVREr0MPsEAfLb9pY2LhoPmdLRdEsvp4TyTz0xOeqvLbgwlhS8S8h76tmX8orC6kTQ1fMpwzxO9SzREktj15EroqTH1cLuLdvI&m=QyWAPAyhLgazjBirnouL0kPckpr7QMX$M8IQdXPpNMmFy5o$mkD@B$BWBcoxDxGu&n=eAT6luLcLrk0Bu@En$FZlKR$HktINc@RxvOd6QNceCxR9nO0z3BhkEPYalIl42Dm&f=C:/Program Files (x86)/nxproxy/log/nxproxy.log
i get an error saying 'i might have a temporary problem or it could have moved.'
To Reproduce Steps to reproduce the behavior:
- Go to the meschentral option on the files and try to download the files
Screenshots If applicable, add screenshots to help explain your problem.
Currently running 1.1.42
Your config.json file
{
"__comment__" : "This is a sample configuration file, edit a section and remove the _ in front of the name. Refer to the user's guide for details.",
"settings": {
"Cert": "remote.domain.com",
"trustedproxy": "192.168.7.245",
"MongoDb": "mongodb://meshcentral-db:27017",
"MongoDbName": "meshcentral",
"_MongoDbChangeStream": true,
"WANonly": true,
"_LANonly": true,
"_Minify": 1,
"_SessionTime": 30,
"_SessionKey": "MyReallySecretPassword1",
"_DbEncryptKey": "MyReallySecretPassword2",
"_DbExpire": {
"events": 1728000,
"powerevents": 864000
},
"plugins": {
"enabled": false
},
"_Port": 4430,
"RedirPort": 800,
"AliasPort": 443,
"_AllowLoginToken": true,
"_AllowFraming": true,
"_WebRTC": false,
"_ClickOnce": false,
"_SelfUpdate": true,
"AgentPing": 300,
"_AgentPong": 60,
"_AgentIdleTimeout": 150,
"MeshErrorLogPath": "/var/log/mesh.log",
"NpmPath": "/usr/local/bin/npm",
"_NpmProxy": "http://1.2.3.4:80",
"_AllowHighQualityDesktop": true,
"_UserAllowedIP": "127.0.0.1,192.168.1.0/24",
"_UserBlockedIP": "127.0.0.1,::1,192.168.0.100",
"_AgentAllowedIP": "192.168.0.100/24",
"_AgentBlockedIP": "127.0.0.1,::1",
"_LocalDiscovery": {
"name": "Local server name",
"info": "Information about this server"
},
"TlsOffload": "127.0.0.1",
"BrowserPong": 20,
"AgentPong": 20,
"_MpsTlsOffload": true,
"_No2FactorAuth": true,
"_WebRtConfig": {
"iceServers": [
{ "urls": "stun:stun.services.mozilla.com" },
{ "urls": "stun:stun.l.google.com:19302" }
]
},
"_AutoBackup": {
"backupIntervalHours": 24,
"keepLastDaysBackup": 10,
"zipPassword": "MyReallySecretPassword3",
"_backupPath": "C:\\backups"
},
"_Redirects": {
"meshcommander": "https://www.meshcommander.com/"
}
},
"domains": {
"": {
"Title": "Domain ",
"Title2": "Remote",
"TitlePicture": "domain.png",
"WelcomePicture": "wallpaper.png",
"WelcomePictureFullScreen": true,
"_UserQuota": 1048576,
"_MeshQuota": 248576,
"_NewAccounts": true,
"_UserNameIsEmail": true,
"_NewAccountEmailDomains": [ "sample.com" ],
"_NewAccountsRights": [ "nonewgroups", "notools" ],
"_Footer": "<a href='https://twitter.com/mytwitter'>Twitter</a>",
"CertUrl": "https://remote.domain.com:443/",
"_PasswordRequirements": { "min": 8, "max": 128, "upper": 1, "lower": 1, "numeric": 1, "nonalpha": 1, "reset": 90, "force2factor": true },
"_AgentNoProxy": true,
"_GeoLocation": true,
"_UserAllowedIP": "127.0.0.1,192.168.1.0/24",
"_UserBlockedIP": "127.0.0.1,::1,192.168.0.100",
"_AgentAllowedIP": "192.168.0.100/24",
"_AgentBlockedIP": "127.0.0.1,::1",
"___UserSessionIdleTimeout__" : "Number of user idle minutes before auto-disconnect",
"_UserSessionIdleTimeout" : 30,
"__UserConsentFlags__" : "Set to: 1 for desktop, 2 for terminal, 3 for files, 7 for all",
"_UserConsentFlags" : 7,
"_Limits": {
"_MaxDevices": 100,
"_MaxUserAccounts": 100,
"_MaxUserSessions": 100,
"_MaxAgentSessions": 100,
"MaxSingleUserSessions": 10
},
"_AmtAcmActivation": {
"log": "amtactivation.log",
"certs": {
"mycertname": {
"certfiles": [ "amtacm-leafcert.crt", "amtacm-intermediate1.crt", "amtacm-intermediate2.crt", "amtacm-rootcert.crt" ],
"keyfile": "amtacm-leafcert.key"
}
}
},
"_Redirects": {
"meshcommander": "https://www.meshcommander.com/"
},
"sessionRecording": {
"onlySelectedUsers": true,
"_onlySelectedUserGroups": true,
"_onlySelectedDeviceGroups": true,
"filepath": "/meshcentral/recordings",
"index": true,
"_maxRecordings": 10,
"maxRecordingDays": 90,
"_maxRecordingSizeMegabytes": 3,
"__protocols__": "Is an array: 1 = Terminal, 2 = Desktop, 5 = Files, 100 = Intel AMT WSMAN, 101 = Intel AMT Redirection, 200 = Messenger",
"protocols": [ 1, 2, 5 ]
},
"_yubikey": { "id": "0000", "secret": "xxxxxxxxxxxxxxxxxxxxx", "_proxy": "http://myproxy.domain.com:80" },
"_httpheaders": { "Strict-Transport-Security": "max-age=360000" },
"_agentConfig": [ "webSocketMaskOverride=1" ]
},
"_customer1": {
"DNS": "customer1.myserver.com",
"Title": "Customer1",
"Title2": "TestServer",
"NewAccounts": 1,
"Auth": "sspi",
"Footer": "Test",
"_CertUrl": "https://192.168.2.106:443/"
},
"info": {
"share": "C:\\ExtraWebSite"
}
},
"_letsencrypt": {
"__comment__": "Go to https://letsdebug.net/ first before trying Let's Encrypt.",
"email": "[email protected]",
"names": "remote.domain.com",
"rsaKeySize": 3072,
"production": false
},
"_peers": {
"serverId": "server1",
"servers": {
"server1": { "url": "wss://192.168.2.133:443/" },
"server2": { "url": "wss://192.168.1.106:443/" }
}
},
"smtp": {
"host": "mail.domain.com.co",
"port": 465,
"from": "[email protected]",
"user": "noreply",
"pass": "test123!",
"tls": true
}
}
Thanks
The error code is showing as 401, which means not authorised, But what's also weird is why the f argument in the url has spaces in it? It shouldn't have spaces?
hi @si458 thanks for the reply, whats odd its that only a few files not all some work fine its some in particular not sure
@killmasta93 you also didn't follow the bug template report correctly so it's going to be hard to reproduce your bug/issues What OS is meshcentral running on What nodejs version are u using Are u using docker or normal npm install? What's the remote computer os? What browser are you using? What local os is ur browser running on?
crap im seeing this too and i dont know whats changed apart from the meshagent updates 👎
crap im seeing this too and i dont know whats changed apart from the meshagent updates 👎
I'm running mine in Docker behind Nginx, is there a way to simulate this issue?
@tomsik-radek I'm not using any reverse proxy and it's happening, from my testing what appears to be happening is the meshagent is sending the file data (binary) too quickly for expressjs to handle it (sounds stupid) But we can't put like a counter or stepping in the agent to pause it because it pipes the file contents directly so the is no way to do this So what we have to do is put the buffer in the expressjs to hold the data before sending it to the web browser of the person who downloaded it I had it partly fixed but when I asked for a 9mb file I got 3.8mb, but that's better than asking for a 9mb file and u got nothing...
hi @si458 sorry for late reply, let me know if there is something on my end i can help with?
@tomsik-radek I'm not using any reverse proxy and it's happening, from my testing what appears to be happening is the meshagent is sending the file data (binary) too quickly for expressjs to handle it (sounds stupid) But we can't put like a counter or stepping in the agent to pause it because it pipes the file contents directly so the is no way to do this So what we have to do is put the buffer in the expressjs to hold the data before sending it to the web browser of the person who downloaded it I had it partly fixed but when I asked for a 9mb file I got 3.8mb, but that's better than asking for a 9mb file and u got nothing...
I'm asking because I did try downloading a 100MB file from the Files tab (I don't normally use it) and had no issues with downloading it. I'm on 1.1.43, deployed via Docker using MongoDB backend.
Edit: Oh, so the issue is downloading from the Files tab on the connected client. In that case I'm also confirming the issue in a Docker install behind Nginx! Edit2: Also the ZIP option seems to be affected
@tomsik-radek I'm not using any reverse proxy and it's happening, from my testing what appears to be happening is the meshagent is sending the file data (binary) too quickly for expressjs to handle it (sounds stupid) But we can't put like a counter or stepping in the agent to pause it because it pipes the file contents directly so the is no way to do this So what we have to do is put the buffer in the expressjs to hold the data before sending it to the web browser of the person who downloaded it I had it partly fixed but when I asked for a 9mb file I got 3.8mb, but that's better than asking for a 9mb file and u got nothing...
I'm asking because I did try downloading a 100MB file from the Files tab (I don't normally use it) and had no issues with downloading it. I'm on 1.1.43, deployed via Docker using MongoDB backend.
Edit: Oh, so the issue is downloading from the Files tab on the connected client. In that case I'm also confirming the issue in a Docker install behind Nginx! Edit2: Also the ZIP option seems to be affected
yeah but whats odd is that its not all only a few files
@tomsik-radek I'm not using any reverse proxy and it's happening, from my testing what appears to be happening is the meshagent is sending the file data (binary) too quickly for expressjs to handle it (sounds stupid) But we can't put like a counter or stepping in the agent to pause it because it pipes the file contents directly so the is no way to do this So what we have to do is put the buffer in the expressjs to hold the data before sending it to the web browser of the person who downloaded it I had it partly fixed but when I asked for a 9mb file I got 3.8mb, but that's better than asking for a 9mb file and u got nothing...
I'm asking because I did try downloading a 100MB file from the Files tab (I don't normally use it) and had no issues with downloading it. I'm on 1.1.43, deployed via Docker using MongoDB backend. Edit: Oh, so the issue is downloading from the Files tab on the connected client. In that case I'm also confirming the issue in a Docker install behind Nginx! Edit2: Also the ZIP option seems to be affected
yeah but whats odd is that its not all only a few files
I'm also confirming it happens in HTTP mode, so it's not an SSL/HTTPS issue
@tomsik-radek @killmasta93 thank you for your findings im still looking into this, i no what the problem is, i just dont know how to fix it? no matter what i try its just not behaving like it should do :(
basically these are the steps
- you ask meshcentral to download a file
devicefile.ashx(GET REQUEST) - your GET request asks meshcentral to ask the remote agent to create a new websocket back to meshcentral
- your remote agent connects to the websocket it was given and then JOINS your GET request session
- once connected your GET request says to remote device over websocket GIVE ME THIS FILE
- the remote agent goes OK here the data ill pipe it over to you over the websocket in 4096 blocks (i think it actually bytes)
- meshcentral then sees that data from the websocket and goes OK pass this to the web browser via its GET request
- your browser downloads the data
- the remote agent goes OK finished, closes the websocket, closes the data, jobs done!
but what appears to be happening is between step 5 and 6, it doesnt ALWAYS pass all the complete data AND sometimes it just JUMPS to step 8 even tho it hadnt read all the data? so this in turn suddenly makes the browser only gets like 1/3 of the actual data
to test this you simply remove the pause/unpause steps inside of meshdevicefile.ashx as this seems to stop it working all together for some reason?
but then dont pipe the data to your browser using obj.res.write,
instead just use a counter and add the length of the data together and it SHOULD total to the size of cmd.size
but it doesnt always total the same size
so its EITHER a websocket/nodejs issue with meshcentral, OR a meshagent compile error maybe???
example output from my debugging of not a full file
RELAY WEBSOCKET 9629696 13213696 object 4096 - meshdevicefile.js:235
RELAY WEBSOCKET 9633792 13213696 object 4096 - meshdevicefile.js:235
RELAY WEBSOCKET 9637888 13213696 object 4096 - meshdevicefile.js:235
RELAY WEBSOCKET 9641984 13213696 object 4096 - meshdevicefile.js:235
RELAY WEBSOCKET 9646080 13213696 object 4096 - meshdevicefile.js:235
CLOSE WEBSOCKET 9646080 13213696
example output from my debugging of a full file
RELAY WEBSOCKET 172916736 172927448 object 4096 - meshdevicefile.js:235
RELAY WEBSOCKET 172920832 172927448 object 4096 - meshdevicefile.js:235
RELAY WEBSOCKET 172924928 172927448 object 4096 - meshdevicefile.js:235
RELAY WEBSOCKET 172927448 172927448 object 2520 - meshdevicefile.js:235
CLOSE WEBSOCKET 172927448 172927448
so just an update on this issue, ive got a workaround working and still doing tests
instead of piping the file which uses createReadStream to avoid a memory overhang, it now opens the file and sends the file in 64k chunks using openSync which might use a little more memory but only when a file is opened
also amazingly this is actually HOW its done in the mobile ui so the mobile ui works but the web ui doesnt,
im getting about 3.8MB/s (35mb/s) transfering a 150MB file now and this is even faster than using rustdesk which only gets like 8mb/s!
im just not sure if i should increase the 64K chunk/buffer or not?
hi @si458 let me know how i can increase it on my end to test it out? thanks
Change from true to false.
"Compression": false,
"WsCompression": false,
"AgentWsCompression": false,
Does the problem go away?
Change from true to false.
"Compression": false, "WsCompression": false, "AgentWsCompression": false,Does the problem go away?
It worked for me, thank you!
I just want to chime in and say that this fixed my issues as well. i wasn't able to download any file, it would after a minute or so time out.
hi @si458 thank you so much yeah it seems that update solved the issue, thanks so much
@killmasta93 I don't scream hallelujah but yippee! Do report back if your you still experience problems tho as its still very strange that it's always worked with compression. Then suddenly stopped working? Unless the new meshagent builds don't like compression anymore? Who knows!