streama
streama copied to clipboard
Published
20 hours ago •
streamaserver
streamaserver
Docker(With Mysql) Stops working after some time(broken mysql connections)
Issue description
Steps to Reproduce
version: '3'
services:
mysql:
image: mysql:5.7
volumes:
- ./data:/var/lib/mysql # CHANGE THIS TO LOCAL DATABASE PATH
expose:
- 3306
environment:
MYSQL_ROOT_PASSWORD: streama_root_password
MYSQL_USER: streama
MYSQL_DATABASE: streama
MYSQL_PASSWORD: streama_password
streama:
image: gkiko/streama:latest
ports:
- 172.17.0.1:8080:8080
volumes:
- ./videos:/data # !!! CHANGE THIS TO YOU LOCAL VIDEO STORE PATH !!!
depends_on:
- mysql
environment:
ACTIVE_PROFILE: mysql
MYSQL_HOST: mysql
MYSQL_PORT: 3306
MYSQL_DB: streama
MYSQL_USER: streama
MYSQL_PASSWORD: streama_password
run for hours/days
Expected Behaviour
Should Fix itself?
Actual Behaviour
Mysql:
2019-12-11T07:58:09.086444Z 11 [Note] Aborted connection 11 to db: 'streama' user: 'streama' host: '172.19.0.3' (Got timeout reading communication packets
2019-12-11T07:58:09.235876Z 7 [Note] Aborted connection 7 to db: 'streama' user: 'streama' host: '172.19.0.3' (Got timeout reading communication packets)
2019-12-11T07:58:09.244754Z 9 [Note] Aborted connection 9 to db: 'streama' user: 'streama' host: '172.19.0.3' (Got timeout reading communication packets)
HTTP Log Streama:
"HTTP Status 500 - Could not roll back Hibernate transaction; nested exception is org.hibernate.TransactionException: Unable to rollback against JDBC Connection"
Caused by: com.mysql.jdbc.exceptions.jdbc4.CommunicationsException: The last packet successfully received from the server was 68,407,268 milliseconds ago. The last packet sent successfully to the server was 68,407,268 milliseconds ago. is longer than the server configured value of 'wait_timeout'. You should consider either expiring and/or testing connection validity before use in your application, increasing the server configured values for client timeouts, or using the Connector/J connection property 'autoReconnect=true' to avoid this problem.
streama_1 | at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
streama_1 | at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
streama_1 | at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
streama_1 | at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
streama_1 | at com.mysql.jdbc.Util.handleNewInstance(Util.java:400)
streama_1 | at com.mysql.jdbc.SQLError.createCommunicationsException(SQLError.java:1038)
streama_1 | at com.mysql.jdbc.MysqlIO.send(MysqlIO.java:3621)
streama_1 | at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2429)
streama_1 | at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2594)
streama_1 | at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2541)
streama_1 | at com.mysql.jdbc.ConnectionImpl.setAutoCommit(ConnectionImpl.java:4882)
streama_1 | at sun.reflect.GeneratedMethodAccessor295.invoke(Unknown Source)
streama_1 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
streama_1 | at java.lang.reflect.Method.invoke(Method.java:498)
streama_1 | at org.apache.tomcat.jdbc.pool.ProxyConnection.invoke(ProxyConnection.java:126)
streama_1 | at org.apache.tomcat.jdbc.pool.JdbcInterceptor.invoke(JdbcInterceptor.java:108)
streama_1 | at org.apache.tomcat.jdbc.pool.DisposableConnectionFacade.invoke(DisposableConnectionFacade.java:81)
streama_1 | at com.sun.proxy.$Proxy88.setAutoCommit(Unknown Source)
streama_1 | at org.springframework.jdbc.datasource.LazyConnectionDataSourceProxy$LazyConnectionInvocationHandler.getTargetConnection(LazyConnectionDataSourceProxy.java:423)
streama_1 | at org.springframework.jdbc.datasource.LazyConnectionDataSourceProxy$LazyConnectionInvocationHandler.invoke(LazyConnectionDataSourceProxy.java:376)
streama_1 | at com.sun.proxy.$Proxy90.prepareStatement(Unknown Source)
streama_1 | at sun.reflect.GeneratedMethodAccessor246.invoke(Unknown Source)
streama_1 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
streama_1 | at java.lang.reflect.Method.invoke(Method.java:498)
streama_1 | at org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy$TransactionAwareInvocationHandler.invoke(TransactionAwareDataSourceProxy.java:240)
streama_1 | at com.sun.proxy.$Proxy90.prepareStatement(Unknown Source)
streama_1 | at org.hibernate.engine.jdbc.internal.StatementPreparerImpl$5.doPrepare(StatementPreparerImpl.java:146)
streama_1 | at org.hibernate.engine.jdbc.internal.StatementPreparerImpl$StatementPreparationTemplate.prepareStatement(StatementPreparerImpl.java:172)
streama_1 | at org.hibernate.engine.jdbc.internal.StatementPreparerImpl.prepareQueryStatement(StatementPreparerImpl.java:148)
streama_1 | at org.hibernate.loader.Loader.prepareQueryStatement(Loader.java:1934)
streama_1 | at org.hibernate.loader.Loader.executeQueryStatement(Loader.java:1903)
streama_1 | at org.hibernate.loader.Loader.executeQueryStatement(Loader.java:1881)
streama_1 | at org.hibernate.loader.Loader.doQuery(Loader.java:925)
streama_1 | at org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(Loader.java:342)
streama_1 | at org.hibernate.loader.Loader.doList(Loader.java:2622)
streama_1 | at org.hibernate.loader.Loader.doList(Loader.java:2605)
streama_1 | at org.hibernate.loader.Loader.listIgnoreQueryCache(Loader.java:2434)
streama_1 | at org.hibernate.loader.Loader.list(Loader.java:2429)
streama_1 | at org.hibernate.loader.criteria.CriteriaLoader.list(CriteriaLoader.java:109)
streama_1 | at org.hibernate.internal.SessionImpl.list(SessionImpl.java:1787)
streama_1 | at org.hibernate.internal.CriteriaImpl.list(CriteriaImpl.java:363)
streama_1 | at org.hibernate.internal.CriteriaImpl.uniqueResult(CriteriaImpl.java:385)
streama_1 | at org.grails.orm.hibernate.AbstractHibernateGormStaticApi$_findWhere_closure15.doCall(AbstractHibernateGormStaticApi.groovy:427)
streama_1 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
streama_1 | at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
streama_1 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
streama_1 | at java.lang.reflect.Method.invoke(Method.java:498)
streama_1 | at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93)
streama_1 | at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325)
streama_1 | at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:294)
streama_1 | at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1024)
streama_1 | at groovy.lang.Closure.call(Closure.java:414)
streama_1 | at org.codehaus.groovy.runtime.ConvertedClosure.invokeCustom(ConvertedClosure.java:54)
streama_1 | at org.codehaus.groovy.runtime.ConversionHandler.invoke(ConversionHandler.java:124)
streama_1 | at com.sun.proxy.$Proxy109.doInHibernate(Unknown Source)
streama_1 | at org.grails.orm.hibernate.GrailsHibernateTemplate.doExecute(GrailsHibernateTemplate.java:286)
streama_1 | at org.grails.orm.hibernate.GrailsHibernateTemplate.execute(GrailsHibernateTemplate.java:230)
streama_1 | at org.grails.orm.hibernate.GrailsHibernateTemplate.execute(GrailsHibernateTemplate.java:116)
streama_1 | at org.grails.orm.hibernate.AbstractHibernateGormStaticApi.findWhere(AbstractHibernateGormStaticApi.groovy:413)
streama_1 | at org.grails.datastore.gorm.GormStaticApi.findWhere(GormStaticApi.groovy:765)
streama_1 | at org.grails.datastore.gorm.GormEntity$Trait$Helper.findWhere(GormEntity.groovy:845)
streama_1 | at org.grails.datastore.gorm.GormEntity$Trait$Helper$findWhere$9.call(Unknown Source)
streama_1 | at streama.User.findWhere(User.groovy)
streama_1 | at streama.User$findWhere$2.call(Unknown Source)
streama_1 | at grails.plugin.springsecurity.userdetails.GormUserDetailsService.$tt__loadUserByUsername(GormUserDetailsService.groovy:60)
streama_1 | at grails.plugin.springsecurity.userdetails.GormUserDetailsService$_loadUserByUsername_closure1.doCall(GormUserDetailsService.groovy)
streama_1 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
streama_1 | at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
streama_1 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
streama_1 | at java.lang.reflect.Method.invoke(Method.java:498)
streama_1 | at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:93)
streama_1 | at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325)
streama_1 | at org.codehaus.groovy.runtime.metaclass.ClosureMetaClass.invokeMethod(ClosureMetaClass.java:294)
streama_1 | at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1024)
streama_1 | at groovy.lang.Closure.call(Closure.java:414)
streama_1 | at groovy.lang.Closure.call(Closure.java:430)
streama_1 | at grails.transaction.GrailsTransactionTemplate$2.doInTransaction(GrailsTransactionTemplate.groovy:96)
streama_1 | at org.springframework.transaction.support.TransactionTemplate.execute(TransactionTemplate.java:133)
streama_1 | at grails.transaction.GrailsTransactionTemplate.execute(GrailsTransactionTemplate.groovy:93)
streama_1 | at grails.plugin.springsecurity.userdetails.GormUserDetailsService.loadUserByUsername(GormUserDetailsService.groovy)
streama_1 | at grails.plugin.springsecurity.userdetails.GormUserDetailsService.loadUserByUsername(GormUserDetailsService.groovy:71)
streama_1 | at org.springframework.security.web.authentication.rememberme.TokenBasedRememberMeServices.processAutoLoginCookie(TokenBasedRememberMeServices.java:123)
streama_1 | at org.springframework.security.web.authentication.rememberme.AbstractRememberMeServices.autoLogin(AbstractRememberMeServices.java:130)
streama_1 | at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:98)
streama_1 | at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
streama_1 | at org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:169)
streama_1 | at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
streama_1 | at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:200)
streama_1 | at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
streama_1 | at grails.plugin.springsecurity.web.authentication.logout.MutableLogoutFilter.doFilter(MutableLogoutFilter.groovy:62)
streama_1 | at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
streama_1 | at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:105)
streama_1 | at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
streama_1 | at grails.plugin.springsecurity.web.SecurityRequestHolderFilter.doFilter(SecurityRequestHolderFilter.groovy:58)
streama_1 | at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
streama_1 | at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:214)
streama_1 | at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:177)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
streama_1 | at org.grails.web.servlet.mvc.GrailsWebRequestFilter.doFilterInternal(GrailsWebRequestFilter.java:77)
streama_1 | at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
streama_1 | at org.grails.web.filters.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:67)
streama_1 | at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
streama_1 | at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
streama_1 | at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
streama_1 | at org.springframework.web.filter.CorsFilter.doFilterInternal(CorsFilter.java:96)
streama_1 | at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
streama_1 | at org.springframework.boot.actuate.autoconfigure.MetricsFilter.doFilterInternal(MetricsFilter.java:106)
streama_1 | at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192)
streama_1 | at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165)
streama_1 | at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198)
streama_1 | at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
streama_1 | at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:474)
streama_1 | at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140)
streama_1 | ... 13 common frames omitted
streama_1 | Caused by: java.net.SocketException: Broken pipe (Write failed)
streama_1 | at java.net.SocketOutputStream.socketWrite0(Native Method)
streama_1 | at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
streama_1 | at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
streama_1 | at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
streama_1 | at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
streama_1 | at com.mysql.jdbc.MysqlIO.send(MysqlIO.java:3603)
streama_1 | ... 129 common frames omitted
It Doesnt seem to Reconnect ever
Environment Information
- Operating System: Linux/Docker
- Streama version: Newest
- Container Version (If Applicable): streama:latest
Confirmed.
ERROR org.hibernate.engine.jdbc.spi.SqlExceptionHelper - The last packet successfully received from the server was 636,630 milliseconds ago. The last packet sent successfully to the server was 636,630 milliseconds ago. is longer than the server configured value of 'wait_timeout'. You should consider either expiring and/or testing connection validity before use in your application, increasing the server configured values for client timeouts, or using the Connector/J connection property 'autoReconnect=true' to avoid this problem.
ERROR org.springframework.transaction.support.TransactionTemplate - Application exception overridden by rollback exception
org.springframework.dao.RecoverableDataAccessException: Hibernate operation: could not extract ResultSet; SQL [n/a]; The last packet successfully received from the server was 620,445 milliseconds ago. The last packet sent successfully to the server was 620,447 milliseconds ago. is longer than the server configured value of 'wait_timeout'. You should consider either expiring and/or testing connection validity before use in your application, increasing the server configured values for client timeouts, or using the Connector/J connection property 'autoReconnect=true' to avoid this problem.; nested exception is com.mysql.jdbc.exceptions.jdbc4.CommunicationsException: The last packet successfully received from the server was 620,445 milliseconds ago. The last packet sent successfully to the server was 620,447 milliseconds ago. is longer than the server configured value of 'wait_timeout'. You should consider either expiring and/or testing connection validity before use in your application, increasing the server configured values for client timeouts, or using the Connector/J connection property 'autoReconnect=true' to avoid this problem.
at org.springframework.jdbc.support.SQLExceptionSubclassTranslator.doTranslate(SQLExceptionSubclassTranslator.java:98)
and on db side
2019-12-16 23:04:12 26 [Warning] Aborted connection 26 to db: 'streama' user: 'streama' host: '10.0.12.8' (Got timeout reading communication packets)
2019-12-16 23:04:28 24 [Warning] Aborted connection 24 to db: 'streama' user: 'streama' host: '10.0.12.8' (Got timeout reading communication packets)
2019-12-16 23:04:29 22 [Warning] Aborted connection 22 to db: 'streama' user: 'streama' host: '10.0.12.8' (Got timeout reading communication packets)
2019-12-16 23:14:49 25 [Warning] Aborted connection 25 to db: 'streama' user: 'streama' host: '10.0.12.8' (Got timeout reading communication packets)
2019-12-16 23:14:49 23 [Warning] Aborted connection 23 to db: 'streama' user: 'streama' host: '10.0.12.8' (Got timeout reading communication packets)
Seems like Mysql is dropping the connection due to inactivity. Try to increase the param "wait_timeout". The default is 28800 seconds (8h), try to increase it to 80h, like this:
mysql:
image: mysql:5.7
volumes:
- ./data:/var/lib/mysql # CHANGE THIS TO LOCAL DATABASE PATH
expose:
- 3306
environment:
MYSQL_ROOT_PASSWORD: streama_root_password
MYSQL_USER: streama
MYSQL_DATABASE: streama
MYSQL_PASSWORD: streama_password
command: [
'--wait_timeout=288000',
]
Seems like Mysql is dropping the connection due to inactivity.
Try to increase the param "wait_timeout". The default is 28800 seconds (8h), try to increase it to 80h, like this:
mysql: image: mysql:5.7 volumes: - ./data:/var/lib/mysql # CHANGE THIS TO LOCAL DATABASE PATH expose: - 3306 environment: MYSQL_ROOT_PASSWORD: streama_root_password MYSQL_USER: streama MYSQL_DATABASE: streama MYSQL_PASSWORD: streama_password command: [ '--wait_timeout=288000', ]
Hello， I wonder what happens after 80h, will the timeout issue still exist ? Many thanks!
