CycleTLS
CycleTLS copied to clipboard
Danny-Dasilva
Connection Keeping Feature
Hi, this lib is awesome! But there are cases where user need to keep the connection alive. Seem clcleTls does not provide such API. I'am wondering if this feature could be added. I already wrote a simple solution, perhaps I can contribute code to this lib? But I might need to talk'bout the API details with you guys, co's I didn't use some of you code, I'm not sure if this is ok :)
type FixedJa3RoutingRoundTripper struct {
mu sync.Mutex
userAgent string
tlsHelloSpec *utls.ClientHelloSpec
helloClientId *utls.ClientHelloID
proxy string // e.g. http://localhost:8888
timeout time.Duration
hostPort2Delegate map[string]*singleConnTransport // www.baidu.com:443 -> RoundTripper
}
func NewFixedJa3RoutingRoundTripper(
userAgent string,
tlsHelloSpec *utls.ClientHelloSpec, helloClientId *utls.ClientHelloID,
proxy string, timeout time.Duration) *FixedJa3RoutingRoundTripper {
roundTripper := new(FixedJa3RoutingRoundTripper)
roundTripper.userAgent = userAgent
roundTripper.tlsHelloSpec = tlsHelloSpec
roundTripper.helloClientId = helloClientId
roundTripper.proxy = proxy
roundTripper.timeout = timeout
roundTripper.hostPort2Delegate = make(map[string]*singleConnTransport)
return roundTripper
}
func (rt *FixedJa3RoutingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
if len(req.Header.Get("User-Agent")) == 0 {
req.Header.Set("User-Agent", rt.userAgent)
}
rt.mu.Lock() // TODO: performance issue
delegate, err := rt.routeToDelegate(req)
rt.mu.Unlock()
if err != nil {
return nil, err
}
if delegate == nil {
return nil, errors.New("didn't find matching delegate")
}
return delegate.RoundTrip(req)
}
func (rt *FixedJa3RoutingRoundTripper) routeToDelegate(req *http.Request) (http.RoundTripper, error) {
hostAndPort := GetHostAndPort(req)
if delegate, ok := rt.hostPort2Delegate[hostAndPort]; ok {
return delegate, nil
}
builder := &transportBuilder{rt: rt, req: req}
builder.build()
if builder.transport != nil && builder.err == nil { // success
rt.hostPort2Delegate[hostAndPort] = builder.transport
return builder.transport, builder.err
} else {
return nil, builder.err
}
}
type transportBuilder struct {
rt *FixedJa3RoutingRoundTripper
req *http.Request
transport *singleConnTransport
err error
}
func (b *transportBuilder) build() {
if IsHttp(b.req) {
b.buildHttpTransport()
} else {
b.buildHttpsTransport()
}
}
func (b *transportBuilder) buildHttpTransport() {
conn, err := b.establishRawTcpConnection()
if err != nil {
b.err = err
return
}
transport := &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
return conn, nil
},
MaxIdleConns: 1,
}
b.transport = &singleConnTransport{
conn: conn,
roundTripper: transport,
}
}
func (b *transportBuilder) buildHttpsTransport() {
conn, err := b.establishRawTcpConnection()
defer func() {
if err != nil {
b.transport = nil
b.err = err
}
}()
if err != nil {
b.err = err
return
}
tlsConn, err := b.performTlsHandshake(conn)
if err != nil {
return
}
switch tlsConn.ConnectionState().NegotiatedProtocol {
case http2.NextProtoTLS:
h2Transport := &http2.Transport{
DialTLSContext: func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) {
return tlsConn, nil
},
}
b.transport = &singleConnTransport{
conn: tlsConn,
roundTripper: h2Transport,
}
default:
httpTransport := &http.Transport{
DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
return tlsConn, nil
},
}
b.transport = &singleConnTransport{
conn: tlsConn,
roundTripper: httpTransport,
}
}
}
func (b *transportBuilder) establishRawTcpConnection() (net.Conn, error) {
hasProxy := len(b.rt.proxy) > 0
if hasProxy {
return b.connectViaProxy()
} else { // direct connection
return net.Dial("tcp", GetHostAndPort(b.req))
}
}
func (b *transportBuilder) connectViaProxy() (conn net.Conn, err error) {
defer func() {
if err != nil && conn != nil {
_ = conn.Close()
}
}()
proxyUrl, err := url.Parse(b.rt.proxy)
if err != nil {
return
}
proxyHostAndPort := proxyUrl.Host // could be like localhost:8888 or just localhost
proxyPort := proxyUrl.Port()
if len(proxyPort) == 0 {
if proxyUrl.Scheme == "https" {
proxyPort = "443"
} else if proxyUrl.Scheme == "http" {
proxyPort = "80"
} else {
err = errors.New("only support http scheme proxies")
return
}
proxyPort = proxyHostAndPort + ":" + proxyPort
}
d := &net.Dialer{Timeout: b.rt.timeout}
conn, err = d.Dial("tcp", proxyHostAndPort)
hostAndPort := GetHostAndPort(b.req)
connectReq := (&http.Request{
Method: "CONNECT",
URL: &url.URL{Host: hostAndPort},
Host: hostAndPort,
ProtoMinor: 1,
ProtoMajor: 1,
})
if proxyUrl.User != nil {
username := proxyUrl.User.Username()
password, _ := proxyUrl.User.Password()
connectReq.Header = make(http.Header)
if len(username) > 0 && len(password) > 0 {
auth := username + ":" + password
basicAuth := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
connectReq.Header.Set("Proxy-Authorization", basicAuth)
}
}
if err = connectReq.Write(conn); err != nil {
return
}
resp, err := http.ReadResponse(bufio.NewReader(conn), connectReq)
if err != nil {
return
}
if resp.StatusCode != http.StatusOK {
return nil, errors.New("error connection to remote server")
}
return
}
func (b *transportBuilder) performTlsHandshake(rawConn net.Conn) (tlsConn *utls.UConn, err error) {
defer func() {
if err != nil {
_ = rawConn.Close()
}
}()
host, _, err := net.SplitHostPort(GetHostAndPort(b.req))
if err != nil {
return
}
if b.rt.tlsHelloSpec != nil {
tlsConn = utls.UClient(rawConn, &utls.Config{
ServerName: host,
InsecureSkipVerify: true,
}, utls.HelloCustom) // TODO: build a randomized.
if err = tlsConn.ApplyPreset(b.rt.tlsHelloSpec); err != nil {
return
}
} else if b.rt.helloClientId != nil {
tlsConn = utls.UClient(rawConn, &utls.Config{
ServerName: host,
InsecureSkipVerify: true,
}, *b.rt.helloClientId) // TODO: build a randomized.
}
if err = tlsConn.Handshake(); err != nil {
return
}
return
}
type singleConnTransport struct {
conn net.Conn
roundTripper http.RoundTripper
}
func (scRt *singleConnTransport) RoundTrip(req *http.Request) (*http.Response, error) {
return scRt.roundTripper.RoundTrip(req)
}
What do you mean by keeping the connection alive ? What's your use case ? WebSocket ? HTTP/2 Streams ? HTTP/1.1 Pipelining ?
For normal http requests, just keep the underling tcp connection alive, so that later http requests are issued via this tcp connection. For now, cycletls just creates a new http.Client for every http request, and after that request is done, this client is discarded, and hence, the underlying connection is also closed. My use case is related to some spider issue, so I cannot tell the details, some websites would do strict ban if constantly establishing and closing tcp connections with the server.
@neverusedname feel free to make a PR with this. Fasthttp works by reusing and caching existing connections so perhaps implementing something like that would be ideal. Establishing a new client connection on every request isn't ideal
Thanks, I will give it a shot. I am not sure how to reuse the go's nethttp transport's tls connection reuse logic, ran n into a little concurrency issue yersterday, might need a little help if necessay :)
@Danny-Dasilva Hi, I just came up with an idea :)
Currenlty cycletls logic is to customize http.Client Transport field to route a request to a proper http.Transport instance and let the routed transport do the RoundTrip job.
While http.Transport, support connection management, it uses a persistentConn object, which may have an alternate Transport field alt set if the conn is https, and uses that alt to do the RoundTrip job. But it only does this if the return conn is a tls.Connection.
So I am thinging, now that we customized dialTls, maybe we can modify the fhttp/transport.go and add a patch to set the alt field of pconn if pconn.conn is of utls.Uconn type, so that we can resuse http.Tranport connection management capability?
sounds good, fasthttp does this in a similar manner storing outgoing requests and retrieving them if subsequent connections are made. Your approach should work
https://github.com/valyala/fasthttp/blob/8ecfc989d9c4c6c30232a6465ec17d5c6d85fc5f/client.go#L2422-L2425
https://github.com/valyala/fasthttp/blob/8ecfc989d9c4c6c30232a6465ec17d5c6d85fc5f/client.go#L2481-L2519
Hi, I've almost done with implementing connection keeping feature, how can I push my code to the repo and create PR :), I modified both fhttp and cycletls project :)
Hi, I've almost done with implementing connection keeping feature, how can I push my code to the repo and create PR :), I modified both fhttp and cycletls project :)
Hello, you can refer to these Github documentations.
https://docs.github.com/en/get-started/quickstart/contributing-to-projects
https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request
I just found it trivial to apply changes on my forked repo, co's I been modifing code from the main branch's last commit point, anyhow, I will try.
I just found it trivial to apply changes on my forked repo, co's I been modifing code from the main branch's last commit point, anyhow, I will try.
This is the way to use Github. If your branch doesn't appear in "Creating a Pull Request", try creating a new branch from your main branch. Open a terminal in your local fork repository and type the following.
git checkout -b http-keep-alive
This command create a new branch called http-keep-alive based on the current branch and switch to it.
git push --set-upstream origin http-keep-alive
This command pushes the current local branch to the remote 'http-keep-alive' and sets up the branch remotely if it doesn't exist.
Then go to the main repository (not the one you forked), click on "Pull Requests", then "New pull request".
No, I meant if i forked new repos, I may need to manually apply changes to the focked repo, co's I modified code directly on the cloned repo which I din't fork, that would be trivial, I may have to eyeball the changes, and copy and paste the modifed files from the original repo to my forked repo, though I did create a new branch. But your reply reminds me to set the origin to my forked repo, that would save a lot of the work, thanks :). btw, be nice with the picture, plz :)
No, I meant if i forked new repos, I may need to manually apply changes to the focked repo, co's I modified code directly on the cloned repo which I din't fork, that would be trivial, I may have to eyeball the changes, and copy and paste the modifed files from the original repo to my forked repo, though I did create a new branch. But your reply reminds me to set the origin to my forked repo, that would save a lot of the work, thanks :). btw, be nice with the picture, plz :)
I wouldn't have explained if I'd wanted to be mean, I just wanted to share an amusing image. In that case you can do. StackOverflow - Cloning a repo from someone else's Github and pushing it to a repo on my Github
git init
git add .
git commit -m "your commit message"
git remote add origin "SSH or HTTPS link of your forked repo"
git push --set-upstream origin http-keep-alive
@RealAlphabet Thanks, your advice for forkking repos worked! I've already forked repos and issued 2 PRS :)
@Danny-Dasilva Hi, i've created a PR, and it's been like 2 weeks. Any feedbacks, do I need to modify my code to meet some PR requirements?
