User-agent switching doesn't work with Proxy

[soanni]

Hi there, thank you for your amazing job, it's really a great framework! I'm scratching my head several days but can't understand what's wrong ... I'm using HTTP Forward Proxy (Squid) and User-Agent switching from Colly extensions but in the Squid logs i can see that User-Agent header is default Golang-user-agent-1.1. However in OnRequest hook i can see that user-agent switching does happens, moreover the User-Agent is still custom even in http_backend.go.Do() method.

func (h *httpBackend) Do(request *http.Request, bodySize int) (*Response, error) {
        fmt.Println(request) // I'm logging to check the User-Agent is custom
	res, err := h.Client.Do(request)

But on Squid side i see 'Golang-user-agent-1.1' for every request. I suspect that smth happens with User-Agent header on net/http side particularly when you use Proxy. The code is below (i tried with ProxySwicther extension but still no luck)

       c := colly.NewCollector(
		colly.Debugger(&debug.LogDebugger{}),
		colly.Async(true),
	)
	c.WithTransport(&http.Transport{
		Proxy: func(pr *http.Request) (*url.URL, error){
			parsedU, err := url.Parse(viper.GetString("squid"))
			if err != nil {
				return nil, err
			}
			return parsedU, nil
		},
    	        DisableKeepAlives: true,
	})

	extensions.RandomUserAgent(c)

	c.OnRequest(func(r *colly.Request) {
		log.Println("Visiting", r.URL)
		log.Println("UserAgent", r.Headers.Get("User-Agent"))
	})

[soanni]

project is dead?

[asciimoo]

Hmm.. Interesting, do you get the same result if you set the proxy with Collector.SetProxy()?

[makelove]

Hmm.. Interesting, do you get the same result if you set the proxy with Collector.SetProxy()?

yes

package main
import (
	"fmt"
	"log"
	"net/http"
	"github.com/gocolly/colly"
)

func main() {
	url := "https://httpbin.org/ip"
	c := colly.NewCollector(
		colly.AllowURLRevisit(),
		colly.Async(true),
	)
	c.UserAgent = "curl/7.54.0"
	c.WithTransport(&http.Transport{ 
		DisableKeepAlives: true, 
	})

	c.OnRequest(func(r *colly.Request) {
		proxy := r.Ctx.Get("proxy")
		c.SetProxy(proxy) //Not working when colly.Async(true),
		log.Println("OnRequest proxy:", proxy)
	})
	c.OnResponse(func(r *colly.Response) {
		log.Println("OnResponse")
		// log.Println("r.Request.ProxyURL", r.Request.ProxyURL) 
		// log.Println("OnResponse Visited", r.Request.URL)

		log.Println(string(r.Body[:]))
		proxy := r.Ctx.Get("proxy")// alway is same one, the Last proxy
		fmt.Println("OnResponse proxy:", proxy)
		fmt.Println("------------")
	})
	c.OnError(func(r *colly.Response, err error) {
		log.Println("OnError ", r.StatusCode, err)
		proxy := r.Ctx.Get("proxy")
		fmt.Println("OnError proxy:", proxy)

		fmt.Println("------------")
	})

	for idx, proxy := range proxy_list {
		fmt.Println(idx, proxy)
		var ctx = colly.NewContext()
		ctx.Put("proxy", proxy)
		c.Request("GET", url, nil, ctx, nil) 
	}
	c.Wait()
}

[makelove]

I think your design of colly have some problem

Why can't setup proxy on every single Request like Scrapy ? it is very easy to use

[asciimoo]

@makelove good idea, would you like to work on it?

[littlecluster]

User Agent and Proxy switching is working fine for me with the below set up. I did have some trouble getting this working though - I cannot get proxy rotation to work without DisableKeepAlives=True - would it be worth updating the documentation for this?

type httpBin struct {
	Headers struct {
		UserAgent string \`json:"User-Agent"\`
	} \`json:"headers"\`
	Origin string \`json:"origin"\`
}

func main() {
	// Instantiate the collector
	c := colly.NewCollector(

		// apply collector settings
		colly.AllowURLRevisit(),
		colly.Async(true), // testing async settings
	)

	// add random user agent extention
	extensions.RandomUserAgent(c)

	// load proxies into round robin switcher
	rp, err := proxy.RoundRobinProxySwitcher(proxies.GetAll()...) // list of proxy strings
	if err != nil {
		log.Fatal(err)
	}

	// if using async then disable transport keep alives
	c.WithTransport(&http.Transport{
		Proxy:             rp,
		DisableKeepAlives: true, // must be true
	})

	// Print the response
	c.OnResponse(func(r *colly.Response) {
		obj := httpBin{}
		err := json.Unmarshal(r.Body, &obj)
		if err != nil {
			log.Fatal(err)
		}

		fmt.Printf("%s: %s\n", obj.Origin, obj.Headers.UserAgent)
	})

	// create a request queue with 2 consumer threads
	q, _ := queue.New(
		2, // Number of consumer threads
		&queue.InMemoryQueueStorage{MaxSize: 10000}, // Use default queue storage
	)

	for i := 0; i < 100; i++ {
		// Add URLs to the queue
		q.AddURL("https://httpbin.org/get")
	}
	// Consume URLs
	q.Run(c)

	// wait re async
	c.Wait()
}