ferret
ferret copied to clipboard
Published
20 hours ago •
MontFerret
MontFerret
What is the right way to send HttpHeader in FQL ?
Hi,
I am trying to send Proxy-Authorization header as below.
LET proxy_header = {"Proxy-Authorization": ["Basic e40b7d5eff464a4fb51efed2d1a19a24"]}
LET doc = DOCUMENT("https://example.com", { driver: "cdp", headers: proxy_header})
But Ferret is crashing with the following error. What could be the issue?
fatal error: fault
[signal 0xc0000005 code=0x0 addr=0x47ed50 pc=0x8412aa]
goroutine 1 [running]:
runtime.throw(0xf230c1, 0x5)
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/panic.go:1116 +0x79 fp=0xc000214e78 sp=0xc000214e48 pc=0x868e19
runtime.sigpanic()
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/signal_windows.go:249 +0x24f fp=0xc000214ea8 sp=0xc000214e78 pc=0x87c14f
runtime.evacuated(...)
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/map.go:203
runtime.mapiternext(0xc000214fc8)
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/map.go:876 +0x4aa fp=0xc000214f28 sp=0xc000214ea8 pc=0x8412aa
runtime.mapiterinit(0xec87a0, 0xc000215318, 0xc000214fc8)
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/map.go:843 +0x1da fp=0xc000214f48 sp=0xc000214f28 pc=0x840d1a
github.com/MontFerret/ferret/pkg/drivers.HTTPHeaders.MarshalJSON(0xc000215318, 0xec87a0, 0xc000215318, 0x209b6cbc090, 0xc000215318, 0x30002)
/home/runner/work/ferret/ferret/pkg/drivers/headers.go:111 +0x9d fp=0xc000215038 sp=0xc000214f48 pc=0xa7aa7d
github.com/wI2L/jettison.encodeJSONMarshaler(0xec87a0, 0xc000215318, 0xc000749000, 0x0, 0x1000, 0xff9220, 0xc0000a40b8, 0xf3d23d, 0x23, 0x5, ...)
/home/runner/go/pkg/mod/github.com/w!i2!l/[email protected]/encode.go:692 +0x79 fp=0xc0002150f0 sp=0xc000215038 pc=0xa489f9
github.com/wI2L/jettison.encodeMarshaler(0xc000215318, 0xc000749000, 0x0, 0x1000, 0xff9220, 0xc0000a40b8, 0xf3d23d, 0x23, 0x5, 0x80, ...)
/home/runner/go/pkg/mod/github.com/w!i2!l/[email protected]/encode.go:668 +0x182 fp=0xc0002151c8 sp=0xc0002150f0 pc=0xa48262
github.com/wI2L/jettison.newJSONMarshalerInstr.func1(0xc000215318, 0xc000749000, 0x0, 0x1000, 0xff9220, 0xc0000a40b8, 0xf3d23d, 0x23, 0x5, 0x80, ...)
/home/runner/go/pkg/mod/github.com/w!i2!l/[email protected]/instruction.go:241 +0xc8 fp=0xc000215280 sp=0xc0002151c8 pc=0xa53468
github.com/wI2L/jettison.wrapInlineInstr.func1(0xc000936ab0, 0xc000749003, 0x0, 0x1000, 0xff9220, 0xc0000a40b8, 0xf3d23d, 0x23, 0x5, 0x80, ...)
/home/runner/go/pkg/mod/github.com/w!i2!l/[email protected]/instruction.go:406 +0xa6 fp=0xc000215318 sp=0xc000215280 pc=0xa53e06
github.com/wI2L/jettison.marshalJSON(0xec87a0, 0xc000936ab0, 0xff9220, 0xc0000a40b8, 0xf3d23d, 0x23, 0x5, 0x80, 0x0, 0x0, ...)
/home/runner/go/pkg/mod/github.com/w!i2!l/[email protected]/json.go:167 +0x103 fp=0xc0002153f8 sp=0xc000215318 pc=0xa4dac3
github.com/wI2L/jettison.MarshalOpts(0xec87a0, 0xc000936ab0, 0xc000215608, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/runner/go/pkg/mod/github.com/w!i2!l/[email protected]/json.go:142 +0x20a fp=0xc000215598 sp=0xc0002153f8 pc=0xa4d78a
github.com/MontFerret/ferret/pkg/drivers/cdp/network.(*Manager).SetHeaders(0xc00003e1c0, 0xff9260, 0xc0005b5740, 0xc000936ab0, 0x0, 0x0)
/home/runner/work/ferret/ferret/pkg/drivers/cdp/network/manager.go:185 +0x10a fp=0xc000215630 sp=0xc000215598 pc=0xcc56ea
github.com/MontFerret/ferret/pkg/drivers/cdp.LoadHTMLPage(0xff9260, 0xc0005b5740, 0xc0000a03c0, 0xc00002e231, 0x3f, 0x0, 0x0, 0x0, 0x0, 0xc000936ab0, ...)
/home/runner/work/ferret/ferret/pkg/drivers/cdp/page.go:83 +0x2fe fp=0xc000215768 sp=0xc000215630 pc=0xcc943e
github.com/MontFerret/ferret/pkg/drivers/cdp.(*Driver).Open(0xc0002c9e80, 0xff9260, 0xc0005b5740, 0xc00002e231, 0x3f, 0x0, 0x0, 0x0, 0x0, 0xc000936ab0, ...)
/home/runner/work/ferret/ferret/pkg/drivers/cdp/driver.go:65 +0x2d8 fp=0xc000215860 sp=0xc000215768 pc=0xcc75b8
github.com/MontFerret/ferret/pkg/stdlib/html.Open(0xff9260, 0xc0005b5740, 0xc00092cd00, 0x2, 0x2, 0x0, 0x0, 0x0, 0x0)
/home/runner/work/ferret/ferret/pkg/stdlib/html/document.go:75 +0x399 fp=0xc000215a20 sp=0xc000215860 pc=0xcd2099
github.com/MontFerret/ferret/pkg/runtime/expressions.(*FunctionCallExpression).Exec(0xc0002c9800, 0xff92a0, 0xc0009369c0, 0xc00092cce0, 0xc00069a1e0, 0x0, 0x0, 0x0)
/home/runner/work/ferret/ferret/pkg/runtime/expressions/func_call.go:58 +0x2a2 fp=0xc000215aa0 sp=0xc000215a20 pc=0xd11262
github.com/MontFerret/ferret/pkg/runtime/expressions.(*VariableDeclarationExpression).Exec(0xc00092c600, 0xff92a0, 0xc0009369c0, 0xc00092cce0, 0x1000280, 0x14d4220, 0x0, 0x0)
/home/runner/work/ferret/ferret/pkg/runtime/expressions/variable.go:48 +0x5d fp=0xc000215af0 sp=0xc000215aa0 pc=0xd11f5d
github.com/MontFerret/ferret/pkg/runtime/expressions.(*BodyExpression).Exec(0xc000936030, 0xff92a0, 0xc0009369c0, 0xc00092cce0, 0xc0005b56e0, 0x0, 0x94e439, 0x14a1d20)
/home/runner/work/ferret/ferret/pkg/runtime/expressions/body.go:41 +0xb6 fp=0xc000215b58 sp=0xc000215af0 pc=0xd0ee36
github.com/MontFerret/ferret/pkg/runtime.(*Program).Run(0xc0009367e0, 0xff91e0, 0xc0002c9f00, 0xc000431d18, 0x3, 0x3, 0x0, 0x0, 0x0, 0x0, ...)
/home/runner/work/ferret/ferret/pkg/runtime/program.go:95 +0x202 fp=0xc000215c08 sp=0xc000215b58 pc=0xa61c62
github.com/MontFerret/ferret/cli.Exec(0xc000216000, 0x4cc, 0x0, 0x0, 0xc000155800, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/runner/work/ferret/ferret/cli/exec.go:62 +0x451 fp=0xc000215d60 sp=0xc000215c08 pc=0xd80891
github.com/MontFerret/ferret/cli.ExecFile(0xc0000a4080, 0xa, 0x0, 0x0, 0xc000155800, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/runner/work/ferret/ferret/cli/exec.go:24 +0x128 fp=0xc000215dd0 sp=0xc000215d60 pc=0xd80428
main.main()
/home/runner/work/ferret/ferret/main.go:191 +0x332 fp=0xc000215f88 sp=0xc000215dd0 pc=0xd88b72
runtime.main()
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/proc.go:204 +0x209 fp=0xc000215fe0 sp=0xc000215f88 pc=0x86b5c9
runtime.goexit()
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc000215fe8 sp=0xc000215fe0 pc=0x89b181
goroutine 6 [syscall]:
os/signal.signal_recv(0x0)
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/sigqueue.go:147 +0xa9
os/signal.loop()
/opt/hostedtoolcache/go/1.15.5/x64/src/os/signal/signal_unix.go:23 +0x29
created by os/signal.Notify.func1.1
/opt/hostedtoolcache/go/1.15.5/x64/src/os/signal/signal.go:150 +0x4b
goroutine 7 [chan receive]:
github.com/MontFerret/ferret/cli.Exec.func1(0xc0005b5680, 0xc00005e810, 0xc0005b5560)
/home/runner/work/ferret/ferret/cli/exec.go:49 +0x3b
created by github.com/MontFerret/ferret/cli.Exec
/home/runner/work/ferret/ferret/cli/exec.go:47 +0x35d
goroutine 10 [select]:
net/http.(*persistConn).readLoop(0xc0004370e0)
/opt/hostedtoolcache/go/1.15.5/x64/src/net/http/transport.go:2161 +0x9cc
created by net/http.(*Transport).dialConn
/opt/hostedtoolcache/go/1.15.5/x64/src/net/http/transport.go:1708 +0xcd7
goroutine 11 [select]:
net/http.(*persistConn).writeLoop(0xc0004370e0)
/opt/hostedtoolcache/go/1.15.5/x64/src/net/http/transport.go:2340 +0x134
created by net/http.(*Transport).dialConn
/opt/hostedtoolcache/go/1.15.5/x64/src/net/http/transport.go:1709 +0xcfc
goroutine 38 [IO wait]:
internal/poll.runtime_pollWait(0x209b6d86440, 0x72, 0xff0fa0)
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/netpoll.go:222 +0x65
internal/poll.(*pollDesc).wait(0xc00019e1b8, 0x72, 0x13ab300, 0x0, 0x0)
/opt/hostedtoolcache/go/1.15.5/x64/src/internal/poll/fd_poll_runtime.go:87 +0x4c
internal/poll.execIO(0xc00019e018, 0xf82dc0, 0x908001, 0x909acd, 0x60473a5b)
/opt/hostedtoolcache/go/1.15.5/x64/src/internal/poll/fd_windows.go:175 +0x105
internal/poll.(*FD).Read(0xc00019e000, 0xc0004d2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/opt/hostedtoolcache/go/1.15.5/x64/src/internal/poll/fd_windows.go:441 +0x2ff
net.(*netFD).Read(0xc00019e000, 0xc0004d2000, 0x1000, 0x1000, 0x86ba65, 0xc0004dbb58, 0x893080)
/opt/hostedtoolcache/go/1.15.5/x64/src/net/fd_posix.go:55 +0x56
net.(*conn).Read(0xc000006058, 0xc0004d2000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/opt/hostedtoolcache/go/1.15.5/x64/src/net/net.go:182 +0x95
net/http.(*persistConn).Read(0xc000478000, 0xc0004d2000, 0x1000, 0x1000, 0xc0004cc060, 0xc0004dbc58, 0x835b95)
/opt/hostedtoolcache/go/1.15.5/x64/src/net/http/transport.go:1887 +0x7c
bufio.(*Reader).fill(0xc0004d0000)
/opt/hostedtoolcache/go/1.15.5/x64/src/bufio/bufio.go:101 +0x10a
bufio.(*Reader).Peek(0xc0004d0000, 0x1, 0x0, 0x0, 0x1, 0x0, 0xc000944300)
/opt/hostedtoolcache/go/1.15.5/x64/src/bufio/bufio.go:139 +0x56
net/http.(*persistConn).readLoop(0xc000478000)
/opt/hostedtoolcache/go/1.15.5/x64/src/net/http/transport.go:2040 +0x1ba
created by net/http.(*Transport).dialConn
/opt/hostedtoolcache/go/1.15.5/x64/src/net/http/transport.go:1708 +0xcd7
goroutine 39 [select]:
net/http.(*persistConn).writeLoop(0xc000478000)
/opt/hostedtoolcache/go/1.15.5/x64/src/net/http/transport.go:2340 +0x134
created by net/http.(*Transport).dialConn
/opt/hostedtoolcache/go/1.15.5/x64/src/net/http/transport.go:1709 +0xcfc
goroutine 23 [IO wait]:
internal/poll.runtime_pollWait(0x209b6d86358, 0x72, 0xff0fa0)
/opt/hostedtoolcache/go/1.15.5/x64/src/runtime/netpoll.go:222 +0x65
internal/poll.(*pollDesc).wait(0xc00019e438, 0x72, 0x13ab300, 0x0, 0x0)
/opt/hostedtoolcache/go/1.15.5/x64/src/internal/poll/fd_poll_runtime.go:87 +0x4c
internal/poll.execIO(0xc00019e298, 0xf82dc0, 0xf04201, 0x198, 0xebd280)
/opt/hostedtoolcache/go/1.15.5/x64/src/internal/poll/fd_windows.go:175 +0x105
internal/poll.(*FD).Read(0xc00019e280, 0xc00078a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/opt/hostedtoolcache/go/1.15.5/x64/src/internal/poll/fd_windows.go:441 +0x2ff
net.(*netFD).Read(0xc00019e280, 0xc00078a000, 0x1000, 0x1000, 0xc00056fb10, 0x92bf96, 0xc0005b8868)
/opt/hostedtoolcache/go/1.15.5/x64/src/net/fd_posix.go:55 +0x56
net.(*conn).Read(0xc000006048, 0xc00078a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/opt/hostedtoolcache/go/1.15.5/x64/src/net/net.go:182 +0x95
bufio.(*Reader).fill(0xc0004d0060)
/opt/hostedtoolcache/go/1.15.5/x64/src/bufio/bufio.go:101 +0x10a
bufio.(*Reader).Peek(0xc0004d0060, 0x2, 0xc0005cd005, 0x4, 0x1000, 0xc00056fc68, 0x83a685)
/opt/hostedtoolcache/go/1.15.5/x64/src/bufio/bufio.go:139 +0x56
github.com/gorilla/websocket.(*Conn).read(0xc0005b8580, 0x2, 0x0, 0x0, 0xee6440, 0x4, 0xfef6a0)
/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:370 +0x47
github.com/gorilla/websocket.(*Conn).advanceFrame(0xc0005b8580, 0xfef920, 0xc000088080, 0x0)
/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:798 +0x6b
github.com/gorilla/websocket.(*Conn).NextReader(0xc0005b8580, 0xc000088030, 0xc000088030, 0x201, 0x0, 0xfef920)
/home/runner/go/pkg/mod/github.com/gorilla/[email protected]/conn.go:980 +0x9b
github.com/mafredri/cdp/rpcc.(*wsReadWriteCloser).Read(0xc000558280, 0xc000958000, 0x200, 0x200, 0xc00019c090, 0x83, 0x90)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/socket.go:37 +0x4f
encoding/json.(*Decoder).refill(0xc0005b8840, 0x0, 0x0)
/opt/hostedtoolcache/go/1.15.5/x64/src/encoding/json/stream.go:165 +0xf8
encoding/json.(*Decoder).readValue(0xc0005b8840, 0x0, 0x0, 0x90)
/opt/hostedtoolcache/go/1.15.5/x64/src/encoding/json/stream.go:140 +0x1ff
encoding/json.(*Decoder).Decode(0xc0005b8840, 0xe3bee0, 0xc000930000, 0x83, 0x90)
/opt/hostedtoolcache/go/1.15.5/x64/src/encoding/json/stream.go:63 +0x93
github.com/mafredri/cdp/rpcc.(*jsonCodec).ReadResponse(0xc00062c1f0, 0xc000930000, 0xc00019c090, 0x83)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/conn.go:245 +0x4c
github.com/mafredri/cdp/rpcc.(*Conn).recv(0xc0000a0280, 0xc00062c210, 0xc00062c200)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/conn.go:333 +0xde
created by github.com/mafredri/cdp/rpcc.DialContext
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/conn.go:197 +0x37b
goroutine 24 [select]:
github.com/mafredri/cdp/rpcc.(*streamClient).watch(0xc000334180)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/stream.go:171 +0x115
created by github.com/mafredri/cdp/rpcc.newStreamClient
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/stream.go:142 +0x1bf
goroutine 25 [select]:
github.com/mafredri/cdp/rpcc.(*streamClient).watch(0xc000334200)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/stream.go:171 +0x115
created by github.com/mafredri/cdp/rpcc.newStreamClient
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/stream.go:142 +0x1bf
goroutine 26 [select]:
github.com/mafredri/cdp/session.(*Manager).watch(0xc00024a440, 0xc0005582a0, 0xc0005542a0, 0xc0004d01e0, 0xc0004d00c0)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/session/manager.go:103 +0x265
created by github.com/mafredri/cdp/session.NewManager
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/session/manager.go:192 +0x1a5
goroutine 30 [select]:
github.com/mafredri/cdp/session.(*session).ReadResponse(0xc00024a2c0, 0xc000930050, 0x7, 0xc000720340)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/session/session.go:43 +0x10a
github.com/mafredri/cdp/rpcc.(*Conn).recv(0xc0000a03c0, 0xc00005e220, 0xc00005e210)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/conn.go:333 +0xde
created by github.com/mafredri/cdp/rpcc.DialContext
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/conn.go:197 +0x37b
goroutine 76 [select]:
github.com/mafredri/cdp/rpcc.(*streamClient).watch(0xc000334300)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/stream.go:171 +0x115
created by github.com/mafredri/cdp/rpcc.newStreamClient
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/stream.go:142 +0x1bf
goroutine 75 [select]:
github.com/mafredri/cdp/rpcc.(*streamClient).watch(0xc000334100)
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/stream.go:171 +0x115
created by github.com/mafredri/cdp/rpcc.newStreamClient
/home/runner/go/pkg/mod/github.com/mafredri/[email protected]/rpcc/stream.go:142 +0x1bf
goroutine 77 [runnable]:
github.com/MontFerret/ferret/pkg/drivers/cdp/events.(*SourceCollection).Get(0xc0004c8210, 0x1, 0x3, 0x2, 0x0, 0x0)
/home/runner/work/ferret/ferret/pkg/drivers/cdp/events/sources.go:54 +0x145
github.com/MontFerret/ferret/pkg/drivers/cdp/events.(*Loop).run(0xc0004c8150, 0xff91e0, 0xc00003e180)
/home/runner/work/ferret/ferret/pkg/drivers/cdp/events/loop.go:87 +0x2f2
created by github.com/MontFerret/ferret/pkg/drivers/cdp/events.(*Loop).Run
/home/runner/work/ferret/ferret/pkg/drivers/cdp/events/loop.go:24 +0x5a
I have tried the following as well. That didn't help either.
LET proxy_header = {"Proxy-Authorization": "Basic e40b7d5eff464a4fb51efed2d1a19a24"}
Hi there! I am experiencing some problems with headers, and I was wondering if they could be somehow related to this issue.
I am able to set headers correctly like (omitted some headers for brevity):
LET headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Pragma': 'no-cache',
'Cookie': 'value...'
}
LET doc = DOCUMENT(url, { headers: headers })
The problems arises when I try to do the same with the CDP driver:
LET doc = DOCUMENT(url, { driver: "cdp", headers: headers })
Leaving the driver as it is and removing the headers from the line above works just fine; the combination of both breaks the execution.
The error I see in the logs is:
net::ERR_INVALID_ARGUMENT: DOCUMENT("my url...", {driver:"cdp",headers:headers}) at 10:10
Thanks in advance, regards
P.S. Having a blast with Ferret so far! I just rewrote a python scraper in ~25 LOC instead of 250+ :)
Hey, I'm happy that you are enjoying using Ferret!
Could you tell me what Chrome/Chromium version are you using?
Hi @ziflex, I am working with Chrome Version 92.0.4515.107 (Official Build) (x86_64) on macOS Big Sur 11.5 (Darwin kernel 20.6.0)