tls-client icon indicating copy to clipboard operation
tls-client copied to clipboard

Published 20 hours ago • verified publisher icon bogdanfinn

[Bug]: Crash after servals hour on concurrency

Open iamat8080 opened this issue 1 year ago • 9 comments

TLS client version

v1.3.12

System information

windows 10 amd ryzen 7 3700x

Issue description

github.com/bogdanfinn/fhttp.setRequestCancel.func4()
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/client.go:398 +0x8b
created by github.com/bogdanfinn/fhttp.setRequestCancel
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/client.go:397 +0x44a
goroutine 439378 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x181b7c2e2e8, 0x72)
        C:/Program Files/Go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc3afeae5b260f336?, 0xc005b4b6c0?, 0x0)
        C:/Program Files/Go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.execIO(0xc004a09418, 0xe04580)
        C:/Program Files/Go/src/internal/poll/fd_windows.go:175 +0xf7
internal/poll.(*FD).Read(0xc004a09400, {0xc001098000, 0x1500, 0x1500})
        C:/Program Files/Go/src/internal/poll/fd_windows.go:436 +0x2b8
net.(*netFD).Read(0xc004a09400, {0xc001098000?, 0xc001098005?, 0x36?})
        C:/Program Files/Go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00022f2d8, {0xc001098000?, 0x0?, 0x0?})
        C:/Program Files/Go/src/net/net.go:183 +0x45
github.com/bogdanfinn/utls.(*atLeastReader).Read(0xc0044d6678, {0xc001098000?, 0xc0044d6678?, 0x0?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:791 +0x3d
bytes.(*Buffer).ReadFrom(0xc001127430, {0xe74660, 0xc0044d6678})
        C:/Program Files/Go/src/bytes/buffer.go:202 +0x98
github.com/bogdanfinn/utls.(*Conn).readFromUntil(0xc001127180, {0x181b72aa0f8?, 0xc00022f2d8}, 0xc000bf4000?)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:813 +0xe5
github.com/bogdanfinn/utls.(*Conn).readRecordOrCCS(0xc001127180, 0x0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:620 +0x116
github.com/bogdanfinn/utls.(*Conn).readRecord(...)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:586
github.com/bogdanfinn/utls.(*Conn).Read(0xc001127180, {0xc004104000, 0x1000, 0xc003ee5e00?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:1300 +0x16f
bufio.(*Reader).Read(0xc002ec1b60, {0xc0036b6658, 0x9, 0x30?})
        C:/Program Files/Go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0xe740e0, 0xc002ec1b60}, {0xc0036b6658, 0x9, 0x9}, 0x9)
        C:/Program Files/Go/src/io/io.go:332 +0x9a
io.ReadFull(...)
        C:/Program Files/Go/src/io/io.go:351
github.com/bogdanfinn/fhttp/http2.readFrameHeader({0xc0036b6658?, 0x9?, 0xcd1ca2?}, {0xe740e0?, 0xc002ec1b60?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:238 +0x6e
github.com/bogdanfinn/fhttp/http2.(*Framer).ReadFrame(0xc0036b6620)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:494 +0x95
github.com/bogdanfinn/fhttp/http2.(*clientConnReadLoop).run(0xc005b4bfa0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2100 +0x170
github.com/bogdanfinn/fhttp/http2.(*ClientConn).readLoop(0xc002c6c1a0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2021 +0x79
created by github.com/bogdanfinn/fhttp/http2.(*Transport).newClientConn
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:860 +0xc92

Steps to reproduce / Code Sample

type request struct {
	client       tls_client.HttpClient
	Thread       int
	userdata     string
        data string
}

func init_(thread int) request {
	tl, _ := tls_client.NewHttpClient(nil, option()...)
	return request{
		client:       tl,
		Thread:       thread,
		userdata:     userdataitter(),
	}
}

func (ms *request) Today() bool {
	url := "https://api.mydomain.com/data"
	payload := fmt.Sprintf("data=today", ms.userdata)
	req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(payload))
	if err != nil {
		PrintLogF(err.Error(), ms.Thread)
		return false
	}

	req.Header = bheader
	resp, err := ms.client.Do(req)
	if err != nil {
		ms.clientChanger()
		PrintLogE(err.Error(), ms.Thread)
		return false
	}
        defer resp.Body.Close()

	readBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		PrintLogF(err.Error(), ms.Thread)
		return false
	}

	var data map[string]interface{}
	err = json.Unmarshal(readBytes, &data)
	if err != nil {
		PrintLogF("Error unmarshalling JSON:"+err.Error(), ms.Thread)
		return false, false
	}
        ms.data = data

	return true

}

func (ms *request) postdata() bool {

	url := "https://api.seconddomain.com/postdata"

	req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(ms.data))
	if err != nil {
		PrintLogF(err.Error(), ms.Thread)
		return false
	}

	req.Header = ms.headerstream
	resp, err := ms.client.Do(req)
	if err != nil {
		PrintLogE(err.Error(), ms.Thread)
		time.Sleep(5 * time.Second)
		return false
	}
        defer resp.Body.Close()
	if resp.StatusCode == 201 {
		return true
	} else {
		readBytes, err := io.ReadAll(resp.Body)
		if err != nil {
			PrintLogF(err.Error(), ms.Thread)
			return false
		}
		PrintLogM(fmt.Sprintf("status : %d dat : %s", resp.StatusCode, readBytes), ms.Thread)
	}
	return false
}

func run(Threads int) bool {
	self := init_(Threads)
	for i := 0; i < 3; i++{
		anid := self.Today()
		if !anid{
			if i == 2{
				PrintLogF(fmt.Sprintf("FAIL %s", self.userdata), self.Thread)
				self.client.CloseIdleConnections()
				return false
			}
			continue
		}
		break
	}
	for i := 0; i < 3; i++{
		anid := self.postdata()
		if !anid{
			if i == 2{
				PrintLogF(fmt.Sprintf("FAIL POST %s", self.userdata), self.Thread)
				self.client.CloseIdleConnections()
				return false
			}
			continue
		}
		break
	}
	self.client.CloseIdleConnections()

	return true
}

func main() {
	
	threads := CurrentConfig.Threads // Define the number of threads
	//threads = 1

	var wg sync.WaitGroup
	wg.Add(threads)

	for i := 1; i <= threads; i++ {
		go func(threadNum int) {
			defer wg.Done()
			run(threadNum)
		}(i)
		time.Sleep(500 * time.Millisecond)
	}
	wg.Wait()
	PrintLog("DONE", 0)
}

iamat8080 avatar Aug 15 '23 05:08 iamat8080

@iamat8080 the code does not reproduce your issue.

bogdanfinn avatar Aug 15 '23 05:08 bogdanfinn

updated code it crashed after serval hours of running not able to figure out why and where the bug @bogdanfinn

iamat8080 avatar Aug 15 '23 05:08 iamat8080 

0x95
github.com/bogdanfinn/fhttp/http2.(*clientConnReadLoop).run(0xc003469fa0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2100 +0x170
github.com/bogdanfinn/fhttp/http2.(*ClientConn).readLoop(0xc006b13a00)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2021 +0x79
created by github.com/bogdanfinn/fhttp/http2.(*Transport).newClientConn
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:860 +0xc92

goroutine 774853 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x1ad6700f8e0, 0x72)
        C:/Program Files/Go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xd81159c9a0d3a647?, 0xc002c976c0?, 0x0)
        C:/Program Files/Go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.execIO(0xc000ca0018, 0x8a4540)
        C:/Program Files/Go/src/internal/poll/fd_windows.go:175 +0xf7
internal/poll.(*FD).Read(0xc000ca0000, {0xc00547b500, 0x1500, 0x1500})
        C:/Program Files/Go/src/internal/poll/fd_windows.go:436 +0x2b8
net.(*netFD).Read(0xc000ca0000, {0xc00547b500?, 0xc00547b505?, 0x36?})
        C:/Program Files/Go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005dc378, {0xc00547b500?, 0x0?, 0x0?})
        C:/Program Files/Go/src/net/net.go:183 +0x45
github.com/bogdanfinn/utls.(*atLeastReader).Read(0xc004cc36e0, {0xc00547b500?, 0xc004cc36e0?, 0x0?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:791 +0x3d
bytes.(*Buffer).ReadFrom(0xc003027eb0, {0x914600, 0xc004cc36e0})
        C:/Program Files/Go/src/bytes/buffer.go:202 +0x98
github.com/bogdanfinn/utls.(*Conn).readFromUntil(0xc003027c00, {0x1ad40a440d8?, 0xc0005dc378}, 0xc00504f000?)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:813 +0xe5
github.com/bogdanfinn/utls.(*Conn).readRecordOrCCS(0xc003027c00, 0x0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:620 +0x116
github.com/bogdanfinn/utls.(*Conn).readRecord(...)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:586
github.com/bogdanfinn/utls.(*Conn).Read(0xc003027c00, {0xc006ee9000, 0x1000, 0xc009dcd200?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:1300 +0x16f
bufio.(*Reader).Read(0xc0090fed20, {0xc0059be8f8, 0x9, 0x30?})
        C:/Program Files/Go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x914080, 0xc0090fed20}, {0xc0059be8f8, 0x9, 0x9}, 0x9)
        C:/Program Files/Go/src/io/io.go:332 +0x9a
io.ReadFull(...)
        C:/Program Files/Go/src/io/io.go:351
github.com/bogdanfinn/fhttp/http2.readFrameHeader({0xc0059be8f8?, 0x9?, 0x771ca2?}, {0x914080?, 0xc0090fed20?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:238 +0x6e
github.com/bogdanfinn/fhttp/http2.(*Framer).ReadFrame(0xc0059be8c0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:494 +0x95
github.com/bogdanfinn/fhttp/http2.(*clientConnReadLoop).run(0xc002c97fa0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2100 +0x170
github.com/bogdanfinn/fhttp/http2.(*ClientConn).readLoop(0xc0091fd040)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2021 +0x79
created by github.com/bogdanfinn/fhttp/http2.(*Transport).newClientConn
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:860 +0xc92

this time it's different

iamat8080 avatar Aug 17 '23 05:08 iamat8080

hello @gngpp I am using windows

iamat8080 avatar Aug 22 '23 06:08 iamat8080

updated code it crashed after serval hours of running not able to figure out why and where the bug @bogdanfinn

Getting the same exact issue, works perfectly fine but crashes after a few hours of running.

github.com/bogdanfinn/fhttp.setRequestCancel.func4() C:/Users/Administrator/go/pkg/mod/github.com/bogdanfinn/[email protected]/client.go:398 +0x8b created by github.com/bogdanfinn/fhttp.setRequestCancel C:/Users/Administrator/go/pkg/mod/github.com/bogdanfinn/[email protected]/client.go:397 +0x44a

KhushC-03 avatar Aug 26 '23 11:08 KhushC-03

Still no code provided which reproduces the issue and can be compiled.

Bildschirmfoto vom 2023-08-28 09-44-30

bogdanfinn avatar Aug 28 '23 07:08 bogdanfinn 

package main

import (
	"fmt"
	"io"
	"sync"
	"time"

	http "github.com/bogdanfinn/fhttp"
	tls_client "github.com/bogdanfinn/tls-client"
)

type re struct {
	client       tls_client.HttpClient
	Thread       int
	userdata     string
	data string
}

func init__(thread int) re {
	tl, _ := tls_client.NewHttpClient(nil, option()...)
	return re{
		client:       tl,
		Thread:       thread,
		userdata:     userdataitter(),
	}
}

var gg = http.Header{
	"accept":             {"application/json"},
	"Cache-Control":      {"no-cache"},
	"user-agent":         {"android/8.1.7.1053/NapsterGlobal"},
	"Accept-Encoding":    {"gzip, deflate"},
	"Origin": {"https://www.ssense.com"},
	"Referer":       {"https://www.ssense.com/"},
	"Connection":         {"Close"},
	http.HeaderOrderKey: {
		"accept",
		"Accept-Encoding",
		"Cache-Control",
		"Origin",
		"Referer",
		"User-Agent",
		}}

func (ms *re) Today() bool {
	url := "https://www.ssense.com/en-us/men"
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return false
	}

	req.Header = gg
	resp, err := ms.client.Do(req)
	if err != nil {
		return false
	}
	defer resp.Body.Close()

	readBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		return false
	}
	fmt.Println(readBytes)

	return true

}

func run(Threads int) bool {
	self := init__(Threads)
	for i := 0; i < 3; i++{
		anid := self.Today()
		if !anid{
			if i == 2{
				self.client.CloseIdleConnections()
				return false
			}
			continue
		}
		break
	}
	self.client.CloseIdleConnections()

	return true
}

func main() {

	threads := CurrentConfig.Threads // Define the number of threads
	//threads = 1

	var wg sync.WaitGroup
	wg.Add(threads)

	for i := 1; i <= threads; i++ {
		go func(threadNum int) {
			defer wg.Done()
			run(threadNum)
		}(i)
		time.Sleep(500 * time.Millisecond)
	}
	wg.Wait()

}

it's sample code I didn't add another post request where I scarpe and post to DB using API

If you format any bot like this with a high number of threads and in an endless loop running for a while it might crash or it would be great if you guide me on what to do I'll test debug on my side and provide you with all logs else I have to write different code using different domains and APIs which can be similar to this one

PS: I believe running in an endless loop with few posts and get requests can reproduce the crash @bogdanfinn

iamat8080 avatar Aug 30 '23 06:08 iamat8080

Brother, I also encountered the same problem as you, have you solved it? It will crash after reaching millions of requests.

huaibaoyy1 avatar Feb 16 '24 11:02 huaibaoyy1