Save_key see exception

[george0st]

I catch exception during writing data to on-line feature store via MLRun see exception (I made performance tests)

Information from log: `> 2021-12-08 12:13:57,616 [error] Pushing error to error stream: Failed to save aggregation for /steuer02/FeatureStore/samplefs/nosql/sets/samplefs-latest//70738475230.13808311698. Response status code was 400: b'\r\n

\r\n\r\n

400 Bad Request

\r\n

---

nginx\r\n\r\n\r\n' Update expression was: fn0=70738475230;fn1=13808311698;fn2=9580326840;fn3=3587069327;fn4=34503427288;fn5=28586460034;fn6=66938916043;fn7=20556937420;fn8=51083396633;fn9=84308629846;fn10=46736315466;fn11=6414738243;fn12=81887600690;fn13=66910387493;fn14=86812830989;fn15=46443402764;fn16=57462523630;fn17=46834943817;fn18=53865626262;fn19=19647742601;fn20=65069655985;fn21=10164917181;fn22=21987890372;fn23=11928772998;fn24=34543022758;fn25=55289296719;fn26=63054015646;fn27=56588244860;fn28=23556836055;fn29=28261945431;fn30=13033113270;fn31=53442511342;fn32=14957966615;fn33=28918223609;fn34=75060029710;fn35=27252828377;fn36=69361079159;fn37=26737276854;fn38=50311139696;fn39=83449083099;fn40=36252376308;fn41=14847593621;fn42=22536253293;fn43=80981105016;fn44=85378445397;fn45=57851813559;fn46=11355937008;fn47=55454561986;fn48=87721634859;fn49=76974805617;fn50=20809035627;fn51=45599881883;fn52=88829786949;fn53=45838294379;fn54=13215248622;fn55=35670259470;fn56=32234347321;fn57=37298127464;fn58=22393212310;fn59=65547816011;fn60=19532604562;fn61=28944067251;fn62=34563626574;fn63=9880902432;fn64=55876871274;fn65=47192690963;fn66=5471977544;fn67=42493804052;fn68=14859625302;fn69=36239441130;fn70=89578410092;fn71=190758210;fn72=15748736671;fn73=52070463607;fn74=54116882831;fn75=83021659235;fn76=2908909623;fn77=9782076349;fn78=80609003113;fn79=20943292651;fn80=73308061464;fn81=68386013094;fn82=3280283174;fn83=24567045060;fn84=77114849706;fn85=69459940965;fn86=75294552801;fn87=37528972340;fn88=62772463504;fn89=9979838322;fn90=86136456888;fn91=67668802981;fn92=24589031331;fn93=76295741821;fn94=21066772198;fn95=31508309601;fn96=65741884494;fn97=29335413105;fn98=65916040683;fn99=52211254160 Traceback (most recent call last): File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/flow.py", line 139, in _do_and_recover return await self._do(event) File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/targets.py", line 773, in _do await self._table._persist(_PersistJob(key, data_to_persist, self._handle_completed, event)) File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 400, in _persist await self._worker_awaitable File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 367, in _persist_worker raise ex File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 342, in _persist_worker completed = await task[1] File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 429, in _safe_process_events raise ex File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 422, in _safe_process_events return await self._internal_persist_key(job.key, job.data) File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 120, in _internal_persist_key self._partitioned_by_key, additional_data_persist) File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/drivers.py", line 160, in _save_key f'Failed to save aggregation for {table_path}/{key}. Response status code was {response.status_code}: {response.body}\n' storey.dtypes.V3ioError: Failed to save aggregation for /steuer02/FeatureStore/samplefs/nosql/sets/samplefs-latest//70738475230.13808311698. Response status code was 400: b'\r\n\r\n\r\n

400 Bad Request

\r\n

---

nginx\r\n\r\n\r\n' Update expression was: fn0=70738475230;fn1=13808311698;fn2=9580326840;fn3=3587069327;fn4=34503427288;fn5=28586460034;fn6=66938916043;fn7=20556937420;fn8=51083396633;fn9=84308629846;fn10=46736315466;fn11=6414738243;fn12=81887600690;fn13=66910387493;fn14=86812830989;fn15=46443402764;fn16=57462523630;fn17=46834943817;fn18=53865626262;fn19=19647742601;fn20=65069655985;fn21=10164917181;fn22=21987890372;fn23=11928772998;fn24=34543022758;fn25=55289296719;fn26=63054015646;fn27=56588244860;fn28=23556836055;fn29=28261945431;fn30=13033113270;fn31=53442511342;fn32=14957966615;fn33=28918223609;fn34=75060029710;fn35=27252828377;fn36=69361079159;fn37=26737276854;fn38=50311139696;fn39=83449083099;fn40=36252376308;fn41=14847593621;fn42=22536253293;fn43=80981105016;fn44=85378445397;fn45=57851813559;fn46=11355937008;fn47=55454561986;fn48=87721634859;fn49=76974805617;fn50=20809035627;fn51=45599881883;fn52=88829786949;fn53=45838294379;fn54=13215248622;fn55=35670259470;fn56=32234347321;fn57=37298127464;fn58=22393212310;fn59=65547816011;fn60=19532604562;fn61=28944067251;fn62=34563626574;fn63=9880902432;fn64=55876871274;fn65=47192690963;fn66=5471977544;fn67=42493804052;fn68=14859625302;fn69=36239441130;fn70=89578410092;fn71=190758210;fn72=15748736671;fn73=52070463607;fn74=54116882831;fn75=83021659235;fn76=2908909623;fn77=9782076349;fn78=80609003113;fn79=20943292651;fn80=73308061464;fn81=68386013094;fn82=3280283174;fn83=24567045060;fn84=77114849706;fn85=69459940965;fn86=75294552801;fn87=37528972340;fn88=62772463504;fn89=9979838322;fn90=86136456888;fn91=67668802981;fn92=24589031331;fn93=76295741821;fn94=21066772198;fn95=31508309601;fn96=65741884494;fn97=29335413105;fn98=65916040683;fn99=52211254160

2021-12-08 12:13:57,616 [error] Pushing error to error stream: Failed to save aggregation for /steuer02/FeatureStore/samplefs/nosql/sets/samplefs-latest//70738475230.13808311698. Response status code was 400: b'\r\n

\r\n\r\n

400 Bad Request

\r\n

---

nginx\r\n\r\n\r\n' Update expression was: fn0=70738475230;fn1=13808311698;fn2=9580326840;fn3=3587069327;fn4=34503427288;fn5=28586460034;fn6=66938916043;fn7=20556937420;fn8=51083396633;fn9=84308629846;fn10=46736315466;fn11=6414738243;fn12=81887600690;fn13=66910387493;fn14=86812830989;fn15=46443402764;fn16=57462523630;fn17=46834943817;fn18=53865626262;fn19=19647742601;fn20=65069655985;fn21=10164917181;fn22=21987890372;fn23=11928772998;fn24=34543022758;fn25=55289296719;fn26=63054015646;fn27=56588244860;fn28=23556836055;fn29=28261945431;fn30=13033113270;fn31=53442511342;fn32=14957966615;fn33=28918223609;fn34=75060029710;fn35=27252828377;fn36=69361079159;fn37=26737276854;fn38=50311139696;fn39=83449083099;fn40=36252376308;fn41=14847593621;fn42=22536253293;fn43=80981105016;fn44=85378445397;fn45=57851813559;fn46=11355937008;fn47=55454561986;fn48=87721634859;fn49=76974805617;fn50=20809035627;fn51=45599881883;fn52=88829786949;fn53=45838294379;fn54=13215248622;fn55=35670259470;fn56=32234347321;fn57=37298127464;fn58=22393212310;fn59=65547816011;fn60=19532604562;fn61=28944067251;fn62=34563626574;fn63=9880902432;fn64=55876871274;fn65=47192690963;fn66=5471977544;fn67=42493804052;fn68=14859625302;fn69=36239441130;fn70=89578410092;fn71=190758210;fn72=15748736671;fn73=52070463607;fn74=54116882831;fn75=83021659235;fn76=2908909623;fn77=9782076349;fn78=80609003113;fn79=20943292651;fn80=73308061464;fn81=68386013094;fn82=3280283174;fn83=24567045060;fn84=77114849706;fn85=69459940965;fn86=75294552801;fn87=37528972340;fn88=62772463504;fn89=9979838322;fn90=86136456888;fn91=67668802981;fn92=24589031331;fn93=76295741821;fn94=21066772198;fn95=31508309601;fn96=65741884494;fn97=29335413105;fn98=65916040683;fn99=52211254160 Traceback (most recent call last): File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/flow.py", line 139, in _do_and_recover return await self._do(event) File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/targets.py", line 773, in _do await self._table._persist(_PersistJob(key, data_to_persist, self._handle_completed, event)) File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 400, in _persist await self._worker_awaitable File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 367, in _persist_worker raise ex File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 342, in _persist_worker completed = await task[1] File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 429, in _safe_process_events raise ex File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 422, in _safe_process_events return await self._internal_persist_key(job.key, job.data) File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/table.py", line 120, in _internal_persist_key self._partitioned_by_key, additional_data_persist) File "/User/.pythonlibs/steuer/lib/python3.7/site-packages/storey/drivers.py", line 160, in _save_key f'Failed to save aggregation for {table_path}/{key}. Response status code was {response.status_code}: {response.body}\n' storey.dtypes.V3ioError: Failed to save aggregation for /steuer02/FeatureStore/samplefs/nosql/sets/samplefs-latest//70738475230.13808311698. Response status code was 400: b'\r\n\r\n\r\n

400 Bad Request

\r\n

---

nginx\r\n\r\n\r\n' Update expression was: fn0=70738475230;fn1=13808311698;fn2=9580326840;fn3=3587069327;fn4=34503427288;fn5=28586460034;fn6=66938916043;fn7=20556937420;fn8=51083396633;fn9=84308629846;fn10=46736315466;fn11=6414738243;fn12=81887600690;fn13=66910387493;fn14=86812830989;fn15=46443402764;fn16=57462523630;fn17=46834943817;fn18=53865626262;fn19=19647742601;fn20=65069655985;fn21=10164917181;fn22=21987890372;fn23=11928772998;fn24=34543022758;fn25=55289296719;fn26=63054015646;fn27=56588244860;fn28=23556836055;fn29=28261945431;fn30=13033113270;fn31=53442511342;fn32=14957966615;fn33=28918223609;fn34=75060029710;fn35=27252828377;fn36=69361079159;fn37=26737276854;fn38=50311139696;fn39=83449083099;fn40=36252376308;fn41=14847593621;fn42=22536253293;fn43=80981105016;fn44=85378445397;fn45=57851813559;fn46=11355937008;fn47=55454561986;fn48=87721634859;fn49=76974805617;fn50=20809035627;fn51=45599881883;fn52=88829786949;fn53=45838294379;fn54=13215248622;fn55=35670259470;fn56=32234347321;fn57=37298127464;fn58=22393212310;fn59=65547816011;fn60=19532604562;fn61=28944067251;fn62=34563626574;fn63=9880902432;fn64=55876871274;fn65=47192690963;fn66=5471977544;fn67=42493804052;fn68=14859625302;fn69=36239441130;fn70=89578410092;fn71=190758210;fn72=15748736671;fn73=52070463607;fn74=54116882831;fn75=83021659235;fn76=2908909623;fn77=9782076349;fn78=80609003113;fn79=20943292651;fn80=73308061464;fn81=68386013094;fn82=3280283174;fn83=24567045060;fn84=77114849706;fn85=69459940965;fn86=75294552801;fn87=37528972340;fn88=62772463504;fn89=9979838322;fn90=86136456888;fn91=67668802981;fn92=24589031331;fn93=76295741821;fn94=21066772198;fn95=31508309601;fn96=65741884494;fn97=29335413105;fn98=65916040683;fn99=52211254160 `