EasyLogger
EasyLogger copied to clipboard
Published
20 hours ago •
armink
armink
elog 死锁
配置如下：
/*
* This file is part of the EasyLogger Library.
*
* Copyright (c) 2015, Armink, <[email protected]>
*
* Permission is hereby granted, free of charge, to any person obtaining
* a copy of this software and associated documentation files (the
* 'Software'), to deal in the Software without restriction, including
* without limitation the rights to use, copy, modify, merge, publish,
* distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to
* the following conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*
* Function: It is the configure head file for this library.
* Created on: 2015-07-30
*/
#ifndef _ELOG_CFG_H_
#define _ELOG_CFG_H_
/* enable log output. default open this macro */
#define ELOG_OUTPUT_ENABLE
/* setting static output log level */
#define ELOG_OUTPUT_LVL ELOG_LVL_VERBOSE
/* enable assert check */
#define ELOG_ASSERT_ENABLE
/* buffer size for every line's log */
#define ELOG_LINE_BUF_SIZE 1024000
/* output line number max length */
#define ELOG_LINE_NUM_MAX_LEN 5
/* output filter's tag max length */
#define ELOG_FILTER_TAG_MAX_LEN 16
/* output filter's keyword max length */
#define ELOG_FILTER_KW_MAX_LEN 16
/* output newline sign */
#define ELOG_NEWLINE_SIGN "\n"
/* enable log color */
#define ELOG_COLOR_ENABLE
/* enable asynchronous output mode */
// #define ELOG_ASYNC_OUTPUT_ENABLE
/* the highest output level for async mode, other level will sync output */
#define ELOG_ASYNC_OUTPUT_LVL ELOG_LVL_DEBUG
/* buffer size for asynchronous output mode */
#define ELOG_ASYNC_OUTPUT_BUF_SIZE (ELOG_LINE_BUF_SIZE * 100)
/* each asynchronous output's log which must end with newline sign */
//#define ELOG_ASYNC_LINE_OUTPUT
/* asynchronous output mode using POSIX pthread implementation */
#define ELOG_ASYNC_OUTPUT_USING_PTHREAD
//ELOG_BUF_OUTPUT_ENABLE 缓冲输出模式下日志缓冲区满了才会输出，或者手动调用 elog_flush 方法时输出
#endif /* _ELOG_CFG_H_ */
初始化代码如下：
void log_init(void)
{/* close printf buffer */
setbuf(stdout, NULL);
/* initialize EasyLogger */
elog_init();
/* set EasyLogger log format */
elog_set_fmt(ELOG_LVL_ASSERT, ELOG_FMT_ALL);
elog_set_fmt(ELOG_LVL_ERROR, ELOG_FMT_LVL | ELOG_FMT_TAG | ELOG_FMT_TIME);
elog_set_fmt(ELOG_LVL_WARN, ELOG_FMT_LVL | ELOG_FMT_TAG | ELOG_FMT_TIME);
elog_set_fmt(ELOG_LVL_INFO, ELOG_FMT_LVL | ELOG_FMT_TAG | ELOG_FMT_TIME);
elog_set_fmt(ELOG_LVL_DEBUG, ELOG_FMT_ALL & ~ELOG_FMT_FUNC);
elog_set_fmt(ELOG_LVL_VERBOSE, ELOG_FMT_ALL & ~ELOG_FMT_FUNC);
#ifdef ELOG_COLOR_ENABLE
elog_set_text_color_enabled(true);
#endif
/* start EasyLogger */
elog_start();
}
gdb信息如下:
[~]# ./gdb -p 8793
GNU gdb (GDB) 7.11.1
Copyright (C) 2016 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "arm-buildroot-linux-gnueabihf".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word".
Attaching to process 8793
[New LWP 8794]
[New LWP 8795]
[New LWP 8796]
warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available.
warning: Unable to find libthread_db matching inferior's thread library, thread debugging will not be available.
0x76e47ba4 in __libc_do_syscall () from /lib/libpthread.so.0
(gdb) bt
#0 0x76e47ba4 in __libc_do_syscall () from /lib/libpthread.so.0
#1 0x76e4568c in __lll_lock_wait () from /lib/libpthread.so.0
#2 0x76e415ba in pthread_mutex_lock () from /lib/libpthread.so.0
#3 0x00017184 in elog_port_output_lock () at ../../easylogger/port/elog_port.c:128
#4 0x00016080 in elog_output_lock () at ../../easylogger/src/elog.c:306
#5 0x00016254 in elog_output (level=3 '\003', tag=0x192bc "NO_TAG", file=0x192b4 "db.c",
func=0x197e0 <__FUNCTION__.7622> "db_get_retained_count", line=239,
format=0x19538 "db get retained message count %d") at ../../easylogger/src/elog.c:413
#6 0x0001253c in db_get_retained_count () at db.c:239
#7 0x0001363e in sync_publishRetainedMsg (clientPt=0x2c0d4 <_client>) at sync.c:258
#8 0x0001588a in main (argc=2, argv=0x7e983e44) at transceiver.c:809
(gdb) where
#0 0x76e47ba4 in __libc_do_syscall () from /lib/libpthread.so.0
#1 0x76e4568c in __lll_lock_wait () from /lib/libpthread.so.0
#2 0x76e415ba in pthread_mutex_lock () from /lib/libpthread.so.0
#3 0x00017184 in elog_port_output_lock () at ../../easylogger/port/elog_port.c:128
#4 0x00016080 in elog_output_lock () at ../../easylogger/src/elog.c:306
#5 0x00016254 in elog_output (level=3 '\003', tag=0x192bc "NO_TAG", file=0x192b4 "db.c",
func=0x197e0 <__FUNCTION__.7622> "db_get_retained_count", line=239,
format=0x19538 "db get retained message count %d") at ../../easylogger/src/elog.c:413
#6 0x0001253c in db_get_retained_count () at db.c:239
#7 0x0001363e in sync_publishRetainedMsg (clientPt=0x2c0d4 <_client>) at sync.c:258
#8 0x0001588a in main (argc=2, argv=0x7e983e44) at transceiver.c:809
(gdb)
看了别的 issue 断言的时候要把几个锁关掉，但是整个程序中没有使用到断言。
仔细查看了下代码，发现日志初始化代码运行了两次，也就是log_init()调用了两次，是否和这个有关系？
使用了，就在elog_port.c中增加了一个写入到文件的函数:
static char _log_dir[50] = "/media/tfcard/log/";
static char _file_name[50] = {0};
void log_setdir(const char * dir)
{
bzero(_log_dir, sizeof(_log_dir));
strcpy(_log_dir, dir);
}
void log_setdirname(const char * name)
{
bzero(_file_name, sizeof(_file_name));
strcpy(_file_name, name);
}
static void _file_output(const char *log, size_t size)
{
char log_path[128] = {0};
FILE *fp = NULL;
struct tm tm_t;
time_t t = time(NULL);
localtime_r(&t, &tm_t);
if(_log_dir[strlen(_log_dir) - 1] != '/')
_log_dir[strlen(_log_dir)] = '/';
// printf("-1-- error: %s(error: %d) %d\n", strerror(errno), errno, EAGAIN);
if (access(_log_dir, F_OK)) {
if (mkdir(_log_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)) {
return;
}
}
// printf("--2- error: %s(error: %d) %d\n", strerror(errno), errno, EAGAIN);
snprintf(log_path, sizeof(log_path), "%s%s/", _log_dir, _file_name);
if (access(log_path, F_OK)) {
if (mkdir(log_path, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)) {
return;
}
}
// printf("-3-- error: %s(error: %d) %d\n", strerror(errno), errno, EAGAIN);
snprintf(log_path, sizeof(log_path), "%s%s/%04d%02d%02d.log", _log_dir, _file_name,
tm_t.tm_year + 1900, tm_t.tm_mon + 1, tm_t.tm_mday);
// printf("-4-- error: %s(error: %d) %d\n", strerror(errno), errno, EAGAIN);
if ((fp = fopen(log_path, "a+")) != NULL) {
fprintf(fp, "%.*s", size, log);
fclose(fp);
}
// printf("-5-- error: %s(error: %d) %d\n", strerror(errno), errno, EAGAIN);
}
/**
* output log port interface
*
* @param log output of log
* @param size log size
*/
void elog_port_output(const char *log, size_t size) {
/* output to terminal */
printf("%.*s", size, log);
_file_output(log, size);
}
今天又遇到了
(gdb) bt
#0 0x76e55ba4 in __libc_do_syscall () from /lib/libpthread.so.0
#1 0x76e5368c in __lll_lock_wait () from /lib/libpthread.so.0
#2 0x76e4f5ba in pthread_mutex_lock () from /lib/libpthread.so.0
#3 0x00015c12 in elog_port_output_lock () at ../../easylogger/port/elog_port.c:128
#4 0x00014ea8 in elog_output_lock () at ../../easylogger/src/elog.c:306
#5 0x0001500e in elog_output (level=level@entry=2 '\002', tag=tag@entry=0x18170 "Trans", file=file@entry=0x18160 "transceiver.c",
func=func@entry=0x18158 <__FUNCTION__.9550> "main", line=line@entry=850,
format=format@entry=0x1893c "mqtt is not connected, start connecting....") at ../../easylogger/src/elog.c:413
#6 0x00014b3c in main (argc=<optimized out>, argv=<optimized out>) at transceiver.c:850
我就两个线程，两个线程中同时有在记日志。
可能不是EasyLogger的问题，我发现所有线程都死在了__libc_do_syscall这边，我把EasyLogger去掉了还是如此，包括我内部调用了exit(0)函数，也是没有让进程退出。很让人头疼，不知道大家有没有遇到过这种问题。
Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/libthread_db.so.1".
__libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
43 ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S: No such file or directory.
(gdb) bt
#0 __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
#1 0x76f1468c in __lll_lock_wait (futex=futex@entry=0x23921a8 <mqttasync_mutex_store>, private=0) at ../ports/sysdeps/unix/sysv/linux/arm/nptl/lowlevellock.c:46
#2 0x76f105ba in __GI___pthread_mutex_lock (mutex=0x23921a8 <mqttasync_mutex_store>) at pthread_mutex_lock.c:79
#3 0x000284f8 in Thread_lock_mutex (mutex=0x23921a8 <mqttasync_mutex_store>) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/Thread.c:120
#4 0x00017a9e in MQTTAsync_lock_mutex (amutex=0x23921a8 <mqttasync_mutex_store>) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTAsync.c:445
#5 0x0001d0de in MQTTAsync_isConnected (handle=0x373bf74) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTAsync.c:3122
#6 0x00014992 in sync_isConnected (handle=<optimized out>) at sync.c:168
#7 0x00016452 in main (argc=<optimized out>, argv=<optimized out>) at transceiver.c:836
(gdb) info thread
Id Target Id Frame
* 1 Thread 0x76fc4000 (LWP 13803) "transceiver" __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
2 Thread 0x761c6460 (LWP 13804) "transceiver" 0x76bfddc2 in nanosleep () at ../sysdeps/unix/syscall-template.S:81
3 Thread 0x759c6460 (LWP 13805) "transceiver" __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
4 Thread 0x751c6460 (LWP 13806) "transceiver" __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
5 Thread 0x749a4460 (LWP 13807) "transceiver" __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
6 Thread 0x741a4460 (LWP 13808) "transceiver" __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
(gdb) thread 2
[Switching to thread 2 (Thread 0x761c6460 (LWP 13804))]
#0 0x76bfddc2 in nanosleep () at ../sysdeps/unix/syscall-template.S:81
81 ../sysdeps/unix/syscall-template.S: No such file or directory.
(gdb) bt
#0 0x76bfddc2 in nanosleep () at ../sysdeps/unix/syscall-template.S:81
#1 0x76c1e09a in usleep (useconds=useconds@entry=1000) at ../sysdeps/unix/sysv/linux/usleep.c:32
#2 0x00017404 in _port_send_thread (arg=<optimized out>) at ../port.c:155
#3 0x76f0e5ce in start_thread (arg=0x761c5fa0) at pthread_create.c:314
#4 0x76c226bc in ?? () at ../ports/sysdeps/unix/sysv/linux/arm/nptl/../clone.S:92 from /lib/libc.so.6
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb) thread 3
[Switching to thread 3 (Thread 0x759c6460 (LWP 13805))]
#0 __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
43 ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S: No such file or directory.
(gdb) bt
#0 __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
#1 0x76c2bbb0 in __lll_lock_wait_private (futex=futex@entry=0x76c7a7c8 <main_arena>) at ../ports/sysdeps/unix/sysv/linux/arm/nptl/lowlevellock.c:31
#2 0x76be148a in __GI___libc_malloc (bytes=23, bytes@entry=0) at malloc.c:2884
#3 0x76fa55be in open_path (name=name@entry=0x76c598cc "libgcc_s.so.1", namelen=namelen@entry=14, secure=secure@entry=0, sps=<optimized out>, realname=realname@entry=0x759c4c9c,
fbp=fbp@entry=0x759c4ca4, loader=0x76fc6ab8, whatcode=whatcode@entry=64, found_other_class=found_other_class@entry=0x759c4c9b) at dl-load.c:2038
#4 0x76fa7636 in _dl_map_object (loader=loader@entry=0x76fc6ab8, name=name@entry=0x76c598cc "libgcc_s.so.1", type=type@entry=2, trace_mode=0, mode=mode@entry=-1879048191, nsid=0)
at dl-load.c:2290
#5 0x76faf3ee in dl_open_worker (a=0x759c510b) at dl-open.c:224
#6 0x76fac482 in _dl_catch_error (objname=0x759c510c, objname@entry=0x759c5124, errstring=0x759c5110, errstring@entry=0x759c5128, mallocedp=0x759c510b,
mallocedp@entry=0x759c5123, operate=0x76faf36d <dl_open_worker>, args=0x759c5114, args@entry=0x759c512c) at dl-error.c:187
#7 0x76faefa0 in _dl_open (file=0x76c598cc "libgcc_s.so.1", mode=-2147483647, caller_dlopen=0x76c2d669 <init+20>, nsid=<optimized out>, argc=2, argv=0x7ed31e44, env=0x7ed31e50)
at dl-open.c:650
#8 0x76c47454 in do_dlopen (ptr=0x759c5378) at dl-libc.c:87
#9 0x76fac482 in _dl_catch_error (objname=0x759c5360, errstring=0x759c5364, mallocedp=0x759c535f, operate=0x76c47419 <do_dlopen>, args=0x759c5378) at dl-error.c:187
#10 0x76c474e6 in dlerror_run (operate=<optimized out>, args=args@entry=0x759c5378) at dl-libc.c:46
#11 0x76c47532 in __GI___libc_dlopen_mode (name=<optimized out>, mode=<optimized out>) at dl-libc.c:163
#12 0x76c2d668 in init () at ../ports/sysdeps/arm/backtrace.c:45
#13 0x76f134f0 in __GI___pthread_once (once_control=0x76c7c5fc <once>, init_routine=0x76c2d655 <init>) at ../ports/sysdeps/unix/sysv/linux/arm/nptl/pthread_once.c:76
#14 0x76c2d710 in __GI___backtrace (array=array@entry=0x759c53f0, size=size@entry=64) at ../ports/sysdeps/arm/backtrace.c:98
#15 0x76ba10a6 in backtrace_and_maps (do_abort=do_abort@entry=2, written=<optimized out>, fd=fd@entry=2) at ../sysdeps/unix/sysv/linux/libc_fatal.c:49
#16 0x76bd76b0 in __libc_message (do_abort=2, fmt=<optimized out>) at ../sysdeps/posix/libc_fatal.c:172
#17 0x76bde540 in malloc_printerr (action=3, str=0x76c5d0dc "malloc(): memory corruption", ptr=<optimized out>) at malloc.c:4991
#18 0x76bdf92e in _int_malloc (av=av@entry=0x76c7a7c8 <main_arena>, bytes=bytes@entry=40) at malloc.c:3444
#19 0x76be13f8 in __GI___libc_malloc (bytes=40) at malloc.c:2888
#20 0x76e55750 in json_object () from /usr/lib/libjansson.so.4
#21 0x76e53e14 in ?? () from /usr/lib/libjansson.so.4
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb) thread 4
[Switching to thread 4 (Thread 0x751c6460 (LWP 13806))]
#0 __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
43 ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S: No such file or directory.
(gdb) bt
#0 __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
#1 0x76f1468c in __lll_lock_wait (futex=futex@entry=0x76fc951c <_rtld_local+1220>, private=0) at ../ports/sysdeps/unix/sysv/linux/arm/nptl/lowlevellock.c:46
#2 0x76f105fa in __GI___pthread_mutex_lock (mutex=0x76fc951c <_rtld_local+1220>) at pthread_mutex_lock.c:114
#3 0x76fac8a6 in _dl_fini () at dl-fini.c:156
#4 0x76bb3934 in __run_exit_handlers (status=status@entry=0, listp=0x76c7a4c8 <__exit_funcs>, run_list_atexit=run_list_atexit@entry=true) at exit.c:82
#5 0x76bb396c in __GI_exit (status=status@entry=0) at exit.c:104
#6 0x00014ed4 in _walking_dead_thread (param=<optimized out>) at transceiver.c:791
#7 0x76f0e5ce in start_thread (arg=0x751c5fa0) at pthread_create.c:314
#8 0x76c226bc in ?? () at ../ports/sysdeps/unix/sysv/linux/arm/nptl/../clone.S:92 from /lib/libc.so.6
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb) thread 5
[Switching to thread 5 (Thread 0x749a4460 (LWP 13807))]
#0 __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
43 ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S: No such file or directory.
(gdb) bt
#0 __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
#1 0x76c2bbb0 in __lll_lock_wait_private (futex=futex@entry=0x76c7a7c8 <main_arena>) at ../ports/sysdeps/unix/sysv/linux/arm/nptl/lowlevellock.c:31
#2 0x76bdf1de in _int_free (av=0x76c7a7c8 <main_arena>, p=0x376c6a0, have_lock=0) at malloc.c:3940
#3 0x0002a6b0 in Internal_heap_unlink (file=0x32030 "/home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTProtocolClient.c", line=232, p=0x376c6f4)
at /home/guojq/bench/paho.mqtt.c-1.3.0/src/Heap.c:247
#4 0x0002a728 in myfree (file=0x32030 "/home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTProtocolClient.c", line=232, p=0x376c6f4) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/Heap.c:269
#5 0x00020856 in MQTTProtocol_storePublication (publish=0x738006fc, len=0x749a3cf0) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTProtocolClient.c:232
#6 0x00020722 in MQTTProtocol_createMessage (publish=0x738006fc, mm=0x749a3de8, qos=1, retained=0) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTProtocolClient.c:192
#7 0x00020652 in MQTTProtocol_startPublish (pubclient=0x373c314, publish=0x738006fc, qos=1, retained=0, mm=0x749a3de8)
at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTProtocolClient.c:159
#8 0x00019b62 in MQTTAsync_processCommand () at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTAsync.c:1502
#9 0x0001a502 in MQTTAsync_sendThread (n=0x0) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTAsync.c:1795
#10 0x76f0e5ce in start_thread (arg=0x749a3fa0) at pthread_create.c:314
#11 0x76c226bc in ?? () at ../ports/sysdeps/unix/sysv/linux/arm/nptl/../clone.S:92 from /lib/libc.so.6
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb) thread 6
[Switching to thread 6 (Thread 0x741a4460 (LWP 13808))]
#0 __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
43 ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S: No such file or directory.
(gdb) bt
#0 __libc_do_syscall () at ../ports/sysdeps/unix/sysv/linux/arm/libc-do-syscall.S:43
#1 0x76f1468c in __lll_lock_wait (futex=futex@entry=0x23921a8 <mqttasync_mutex_store>, private=0) at ../ports/sysdeps/unix/sysv/linux/arm/nptl/lowlevellock.c:46
#2 0x76f105ba in __GI___pthread_mutex_lock (mutex=0x23921a8 <mqttasync_mutex_store>) at pthread_mutex_lock.c:79
#3 0x000284f8 in Thread_lock_mutex (mutex=0x23921a8 <mqttasync_mutex_store>) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/Thread.c:120
#4 0x00017a9e in MQTTAsync_lock_mutex (amutex=0x23921a8 <mqttasync_mutex_store>) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTAsync.c:445
#5 0x0001e218 in MQTTAsync_cycle (sock=0x741a3de4, timeout=1000, rc=0x741a3de8) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTAsync.c:3681
#6 0x0001ae7c in MQTTAsync_receiveThread (n=0x373bf74) at /home/guojq/bench/paho.mqtt.c-1.3.0/src/MQTTAsync.c:2076
#7 0x76f0e5ce in start_thread (arg=0x741a3fa0) at pthread_create.c:314
#8 0x76c226bc in ?? () at ../ports/sysdeps/unix/sysv/linux/arm/nptl/../clone.S:92 from /lib/libc.so.6
Backtrace stopped: previous frame identical to this frame (corrupt stack?)
(gdb)