Tendis icon indicating copy to clipboard operation
Tendis copied to clipboard

Published 20 hours ago • verified publisher icon Tencent

drainRspCallback:Connection reset by peer

Open zhaochunxue opened this issue 2 years ago • 2 comments

线上环境使用tendis是2.4.3最新版本的，使用的lpush命令，大约2k/s左右，并没有bigkey，查看了下rocksdb的日志，也没有发生阻写，有很多的慢日志都是2-4s导致客户端超时严重，和下边的日志，慢日志的命令都不大，能帮忙查看一下吗？还挺急的 W0610 08:06:31.369436 261 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:06:31.369606 260 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:06:31.369451 258 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:06:31.369457 264 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:06:31.369611 259 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:06:31.369670 264 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:06:31.369618 260 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:06:31.369719 257 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.921859 259 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.921867 261 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922086 264 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.921895 263 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922129 262 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922149 257 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922128 264 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922196 257 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922087 260 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922168 262 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922191 259 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922168 263 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922304 264 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922308 259 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922330 263 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922395 260 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922453 264 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922514 258 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922575 262 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922667 259 network.cpp:902] drainRspCallback:Connection reset by peer W0610 08:43:17.922750 260 network.cpp:902] drainRspCallback:Connection reset by peer I0610 09:22:43.853833 299 command.cpp:836] bigkey delete:656E67696E5F6167673A343332353731303532383833363432,rcdType:L,size:2143

下边是配置文件

  1. "allow-cross-slot"
  2. "no"
  3. "aof-enabled"
  4. "no"
  5. "aof-psync-num"
  6. "500"
  7. "bind"
  8. ""0.0.0.0""
  9. "binlog-send-batch"
  10. "256"
  11. "binlog-send-bytes"
  12. "16777216"
  13. "binlog-using-defaultcf"
  14. "no"
  15. "binlogdelrange"
  16. "100000"
  17. "binlogfilesecs"
  18. "1200"
  19. "binlogfilesizemb"
  20. "64"
  21. "binlogratelimitmb"
  22. "64"
  23. "checkkeytypeforsetcmd"
  24. "no"
  25. "chunksize"
  26. "16384"
  27. "cluster-enabled"
  28. "yes"
  29. "cluster-migration-barrier"
  30. "1"
  31. "cluster-migration-batch-size"
  32. "10240"
  33. "cluster-migration-binlog-iters"
  34. "10"
  35. "cluster-migration-distance"
  36. "10000"
  37. "cluster-migration-rate-limit"
  38. "32"
  39. "cluster-migration-slots-num-per-task"
  40. "10"
  41. "cluster-migration-timeout"
  42. "5"
  43. "cluster-node-timeout"
  44. "30000"
  45. "cluster-require-full-coverage"
  46. "yes"
  47. "cluster-single-node"
  48. "no"
  49. "cluster-slave-no-failover"
  50. "no"
  51. "cluster-slave-validity-factor"
  52. "10"
  53. "compactrange-after-deleterange"
  54. "no"
  55. "daemon"
  56. "no"
  57. "databases"
  58. "16"
  59. "delcntindexmgr"
  60. "10000"
  61. "deletefilesinrange-for-binlog"
  62. "yes"
  63. "deljobcntindexmgr"
  64. "1"
  65. "dir"
  66. ""./store/db""
  67. "direct-io"
  68. "no"
  69. "domain-enabled"
  70. "no"
  71. "dumpdir"
  72. ""./store/dump""
  73. "executorthreadnum"
  74. "16"
  75. "executorworkpoolsize"
  76. "8"
  77. "force-recovery"
  78. "0"
  79. "fullpushthreadnum"
  80. "4"
  81. "fullreceivethreadnum"
  82. "4"
  83. "garbage-delete-size"
  84. "30"
  85. "garbagedeletethreadnum"
  86. "1"
  87. "generallog"
  88. "no"
  89. "incrpushthreadnum"
  90. "8"
  91. "jeprof-auto-dump"
  92. "no"
  93. "keysdefaultlimit"
  94. "100"
  95. "kvstorecount"
  96. "10"
  97. "lockdbxwaittimeout"
  98. "5"
  99. "lockwaittimeout"
  100. "3600"
  101. "log-error"
  102. "no"
  103. "logdir"
  104. ""./store/log""
  105. "loglevel"
  106. ""notice""
  107. "logrecyclethreadnum"
  108. "4"
  109. "lua-time-limit"
  110. "5000"
  111. "luastatemaxidletime"
  112. "3600000"
  113. "masterauth"
  114. ""***""
  115. "maxbinlogkeepnum"
  116. "10000000"
  117. "maxclients"
  118. "10000"
  119. "migrate-gc-enabled"
  120. "no"
  121. "migrate-snapshot-key-num"
  122. "100000"
  123. "migrate-snapshot-retry-num"
  124. "1000"
  125. "migratereceivethreadnum"
  126. "4"
  127. "migratesenderthreadnum"
  128. "4"
  129. "minbinlogkeepsec"
  130. "604800"
  131. "netbatchsize"
  132. "1048576"
  133. "netbatchtimeoutsec"
  134. "60"
  135. "netiothreadnum"
  136. "8"
  137. "noexpire"
  138. "no"
  139. "pausetimeindexmgr"
  140. "10"
  141. "pidfile"
  142. ""./store/tendisplus.pid""
  143. "port"
  144. "6509"
  145. "proto-max-bulk-len"
  146. "536870912"
  147. "replicate-fix-enabled"
  148. "yes"
  149. "requirepass"
  150. ""***""
  151. "rocks.blockcache_num_shard_bits"
  152. "6"
  153. "rocks.blockcache_strict_capacity_limit"
  154. "no"
  155. "rocks.blockcachemb"
  156. "20480"
  157. "rocks.compaction_deletes_ratio"
  158. "0.000000"
  159. "rocks.compaction_deletes_trigger"
  160. "0"
  161. "rocks.compaction_deletes_window"
  162. "0"
  163. "rocks.compress_type"
  164. ""snappy""
  165. "rocks.disable_wal"
  166. "no"
  167. "rocks.flush_log_at_trx_commit"
  168. "no"
  169. "rocks.level0_compress_enabled"
  170. "no"
  171. "rocks.level1_compress_enabled"
  172. "yes"
  173. "rocks.max_background_jobs"
  174. "2"
  175. "rocks.max_open_files"
  176. "-1"
  177. "rocks.rate_limiter_auto_tuned"
  178. "yes"
  179. "rocks.rate_limiter_fairness"
  180. "10"
  181. "rocks.rate_limiter_rate_bytes_per_sec"
  182. "0"
  183. "rocks.rate_limiter_refill_period_us"
  184. "100000"
  185. "rocks.wal_dir"
  186. """"
  187. "save-min-binlogid"
  188. "yes"
  189. "scancntindexmgr"
  190. "1000"
  191. "scandefaultlimit"
  192. "10"
  193. "scandefaultmaxiteratetimes"
  194. "10000"
  195. "scanjobcntindexmgr"
  196. "1"
  197. "slave-migrate-enabled"
  198. "no"
  199. "slavebinlogkeepnum"
  200. "1"
  201. "slowlog"
  202. ""./store/log/slowlog""
  203. "slowlog-file-enabled"
  204. "yes"
  205. "slowlog-flush-interval"
  206. "1000"
  207. "slowlog-log-slower-than"
  208. "100000"
  209. "slowlog-max-len"
  210. "128"
  211. "storage"
  212. ""rocks""
  213. "timeoutsecbinlogwaitrsp"
  214. "30"
  215. "truncatebinlogintervalms"
  216. "1000"
  217. "truncatebinlognum"
  218. "100000"
  219. "version-increase"
  220. "yes"

zhaochunxue avatar Jun 10 '22 01:06 zhaochunxue

复在issue中，我后续跟进一下。

已发

zhaochunxue avatar Jun 10 '22 02:06 zhaochunxue

我跟进一下

raffertyyu avatar Jun 16 '22 07:06 raffertyyu