panjf2000
[Feature]: support ProxyProtocol
Description of new feature
支持Proxy Protocol协议，gnet.Option中添加控制PP协议开关
Scenarios for new feature
我的服务部署在4层LB后面，4LB不支持开启客户端地址保持，后端服务无法获取源IP和源端口，但4LB支持配置Proxy Protocol协议，需要后端适配
Breaking changes or not?
No
Code snippets (optional)
Alternatives for new feature
None.
Additional context (optional)
None.
🤖 Non-English text detected, translating ...
Are there any specific examples that can further illustrate your needs?
你的四层LB跟你的后端服务建立连接之后，PP包只会发一次，而且是在正式数据包之前。
不管是建立连接后马上给你发一个PP包，还是等有正式数据包之后连同 PP包+正式数据包 一起转发，在gnet实现中，你只能在OnTraffic中去解析，因为即使建连之后马上发一个PP包，OnOpen也无法读到。
所以问题就变成了，怎么在OnTraffic中先解析PP包(一次)，然后正常处理正式数据包。
下面的实现比较丑，但能解决问题。
type MyConnContext struct {
parsed bool
realRemoteAddress string
}
func (s *DemoServer) OnTraffic(c gnet.Conn) gnet.Action {
conn := c.Context().(*MyConnContext)
for {
if !conn.parsed {
// parse pp
ppPacket := parsePP(c gnet.Conn)
conn.realRemoteAddress = ppPacket.GetSourceAddress()
conn.parsed = true
} else {
// your normal logic
}
}
}
parsePP就是你解析PP包的过程：
var (
sigV2 = []byte{'\x0D', '\x0A', '\x0D', '\x0A', '\x00', '\x0D', '\x0A', '\x51', '\x55', '\x49', '\x54', '\x0A'}
)
func parsePP(c gnet.Conn) (*V2, error) {
b1, err := c.Peek(1)
if err != nil {
return nil, err
}
if bytes.Equal(b1[:1], sigV2[:1]) {
totalLenBytes, err := c.Peek(MinV2Len)
if err != nil {
return nil, err
}
length := int(binary.BigEndian.Uint16(totalLenBytes[14:16]))
totalLen := 12 + 1 + 1 + 2 + length
buf, err := c.Peek(totalLen)
if err != nil {
return nil, err
}
if len(buf) < totalLen {
return nil, fmt.Errorf("not complete")
}
n, err := c.Discard(totalLen)
if err != nil {
return nil, fmt.Errorf("discard total packet error: %w", err)
}
if n != len(buf) {
return nil, fmt.Errorf("not discard enough data")
}
v2 := new(V2)
if err = v2.IDecode(buf); err != nil {
return nil, fmt.Errorf("pp deal error: %w", err)
}
return v2, nil
}
return nil, fmt.Errorf("not pp packet")
}
const (
MinV2Len = 16
IPV4Len = 4 + 4 + 2 + 2
IPV6Len = 16 + 16 + 2 + 2
)
var ErrInvalidPudLength = errors.New("invalid pp length")
type V2 struct {
Version byte
Command ProtocolVersionAndCommand
TransportProtocol AddressFamilyAndProtocol
Signature string
AddressFamilyAndProtocol AddressFamilyAndProtocol
Length int
_addr4
_addr6
}
type _ports struct {
// 2 字节
SrcPort uint16
// 2 字节
DstPort uint16
}
type _addr4 struct {
// 4 字节
Src []byte
// 4 字节
Dst []byte
_ports
}
type _addr6 struct {
// 16 字节
Src []byte
// 16 字节
Dst []byte
_ports
}
func (p *V2) IDecode(data []byte) error {
if len(data) < MinV2Len {
return ErrInvalidPudLength
}
p.Signature = string(data[:12])
p.Version = 2
p.Command = ProtocolVersionAndCommand(data[12])
p.TransportProtocol = AddressFamilyAndProtocol(data[13])
p.Length = int(binary.BigEndian.Uint16(data[14:16]))
if len(data) < MinV2Len+p.Length {
return ErrInvalidPudLength
}
if p.TransportProtocol.IsIPv4() {
if p.Length != IPV4Len {
return ErrInvalidPudLength
}
p._addr4.Src = data[16:20]
p._addr4.Dst = data[20:24]
p._addr4.SrcPort = binary.BigEndian.Uint16(data[24:26])
p._addr4.DstPort = binary.BigEndian.Uint16(data[26:28])
return nil
}
if p.TransportProtocol.IsIPv6() {
if p.Length != IPV6Len {
return ErrInvalidPudLength
}
p._addr6.Src = data[16:32]
p._addr6.Dst = data[32:48]
p._addr4.SrcPort = binary.BigEndian.Uint16(data[48:50])
p._addr4.DstPort = binary.BigEndian.Uint16(data[50:52])
return nil
}
return ErrInvalidPudLength
}
func (p *V2) GetSourceAddress() string {
if p.TransportProtocol.IsIPv4() {
return net.IP(p._addr4.Src).String()
}
return net.IP(p._addr6.Src).String()
}
// ProtocolVersionAndCommand represents the command in proxy protocol v2.
type ProtocolVersionAndCommand byte
// AddressFamilyAndProtocol represents address family and transport protocol.
type AddressFamilyAndProtocol byte
// IsIPv4 returns true if the address family is IPv4 (AF_INET4), false otherwise.
func (ap AddressFamilyAndProtocol) IsIPv4() bool {
return ap&0xF0 == 0x10
}
// IsIPv6 returns true if the address family is IPv6 (AF_INET6), false otherwise.
func (ap AddressFamilyAndProtocol) IsIPv6() bool {
return ap&0xF0 == 0x20
}
🤖 Non-English text detected, translating ...
After your layer 4 LB is connected to your backend service, the PP packet will only be sent once, and before the official data packet.
Whether it is to send you a PP packet immediately after establishing the connection, or to forward it together with the PP packet + official packet after establishing the connection, in the gnet implementation, you can only parse it in OnTraffic, because even if you send a PP packet immediately after establishing the connection, OnOpen cannot be read.
So the problem becomes, how to first parse PP packets (once) in OnTraffic, and then process the official data packets normally.
The following implementation is ugly, but it can solve the problem.
type MyConnContext struct {
parsed bool
realRemoteAddress string
}
func (s *DemoServer) OnTraffic(c gnet.Conn) gnet.Action {
conn := c.Context().(*MyConnContext)
for {
if !conn.parsed {
// parse pp
ppPacket := parsePP(c gnet.Conn)
conn.realRemoteAddress = ppPacket.GetSourceAddress()
conn.parsed = true
} else {
// your normal logic
}
}
}
parsePP is the process of parsing PP packages:
var (
sigV2 = []byte{'\x0D', '\x0A', '\x0D', '\x0A', '\x00', '\x0D', '\x0A', '\x51', '\x55', '\x49', '\x54', '\x0A'}
)
func parsePP(c gnet.Conn) (*V2, error) {
b1, err := c.Peek(1)
if err != nil {
return nil, err
}
if bytes.Equal(b1[:1], sigV2[:1]) {
totalLenBytes, err := c.Peek(MinV2Len)
if err != nil {
return nil, err
}
length := int(binary.BigEndian.Uint16(totalLenBytes[14:16]))
totalLen := 12 + 1 + 1 + 2 + length
buf, err := c.Peek(totalLen)
if err != nil {
return nil, err
}
if len(buf) < totalLen {
return nil, fmt.Errorf("not complete")
}
n, err := c.Discard(totalLen)
if err != nil {
return nil, fmt.Errorf("discard total packet error: %w", err)
}
if n != len(buf) {
return nil, fmt.Errorf("not discard enough data")
}
v2 := new(V2)
if err = v2.IDecode(buf); err != nil {
return nil, fmt.Errorf("pp deal error: %w", err)
}
return v2, nil
}
return nil, fmt.Errorf("not pp packet")
}
const (
MinV2Len = 16
IPV4Len = 4 + 4 + 2 + 2
IPV6Len = 16 + 16 + 2 + 2
)
var ErrInvalidPudLength = errors.New("invalid pp length")
type V2 struct {
Version byte
Command ProtocolVersionAndCommand
TransportProtocol AddressFamilyAndProtocol
Signature string
AddressFamilyAndProtocol AddressFamilyAndProtocol
Length int
_addr4
_addr6
}
type _ports struct {
// 2 bytes
SrcPort uint16
// 2 bytes
DstPort uint16
}
type _addr4 struct {
// 4 bytes
Src []byte
// 4 bytes
Dst []byte
_ports
}
type _addr6 struct {
// 16 bytes
Src []byte
// 16 bytes
Dst []byte
_ports
}
func (p *V2) IDecode(data []byte) error {
if len(data) < MinV2Len {
return ErrInvalidPudLength
}
p.Signature = string(data[:12])
p.Version = 2
p.Command = ProtocolVersionAndCommand(data[12])
p.TransportProtocol = AddressFamilyAndProtocol(data[13])
p.Length = int(binary.BigEndian.Uint16(data[14:16]))
if len(data) < MinV2Len+p.Length {
return ErrInvalidPudLength
}
if p.TransportProtocol.IsIPv4() {
if p.Length != IPV4Len {
return ErrInvalidPudLength
}
p._addr4.Src = data[16:20]
p._addr4.Dst = data[20:24]
p._addr4.SrcPort = binary.BigEndian.Uint16(data[24:26])
p._addr4.DstPort = binary.BigEndian.Uint16(data[26:28])
return nil
}
if p.TransportProtocol.IsIPv6() {
if p.Length != IPV6Len {
return ErrInvalidPudLength
}
p._addr6.Src = data[16:32]
p._addr6.Dst = data[32:48]
p._addr4.SrcPort = binary.BigEndian.Uint16(data[48:50])
p._addr4.DstPort = binary.BigEndian.Uint16(data[50:52])
return nil
}
return ErrInvalidPudLength
}
func (p *V2) GetSourceAddress() string {
if p.TransportProtocol.IsIPv4() {
return net.IP(p._addr4.Src).String()
}
return net.IP(p._addr6.Src).String()
}
// ProtocolVersionAndCommand represents the command in proxy protocol v2.
type ProtocolVersionAndCommand byte
// AddressFamilyAndProtocol represents address family and transport protocol.
type AddressFamilyAndProtocol byte
// IsIPv4 returns true if the address family is IPv4 (AF_INET4), false otherwise.
func (ap AddressFamilyAndProtocol) IsIPv4() bool {
return ap&0xF0 == 0x10
}
// IsIPv6 returns true if the address family is IPv6 (AF_INET6), false otherwise.
func (ap AddressFamilyAndProtocol) IsIPv6() bool {
return ap&0xF0 == 0x20
}
不管是建立连接后马上给你发一个PP包，还是等有正式数据包之后连同 PP包+正式数据包 一起转发，在gnet实现中，你只能在OnTraffic中去解析，因为即使建连之后马上发一个PP包，OnOpen也无法读到。
其实你可以调用 c.Fd() 返回 fd，然后通过系统调用 syscall.Read 读这个 PP 包，不过 fd 是非阻塞的，所以你需要用 for 循环一直到读到完整的 PP 包。
所以问题就变成了，怎么在OnTraffic中先解析PP包(一次)，然后正常处理正式数据包。
如果要在 OnTraffic 里处理，你可以用 sync.Once
🤖 Non-English text detected, translating ...
Whether it is to send you a PP packet immediately after establishing the connection, or to forward it together with the PP packet + the official data packet, in gnet implementation, you can only parse it in OnTraffic, because even if you send a PP packet immediately after establishing the connection, OnOpen cannot read it.
In fact, you can call c.Fd() to return fd, and then read the PP package through the system call syscall.Read, but fd is non-blocking, so you need to loop with for until you read the complete PP package.
So the problem becomes, how to first parse PP packets (once) in OnTraffic, and then process the official data packets normally.
If you want to handle it in OnTraffic, you can use sync.Once
This issue is marked as stale because it has been open for 30 days with no activity.
You should take one of the following actions:
- Manually close this issue if it is no longer relevant
- Comment if you have more information to share
This issue will be automatically closed in 7 days if no further activity occurs.
![github-actions[bot] avatar](https://avatars.githubusercontent.com/in/15368?v=4 "Published by a pub.dev verified publisher"){kind=small}
This issue was closed because it has been inactive for 7 days since being marked as stale.
If you believe this is a false alarm, please leave a comment for it or open a new issue, you can also reopen this issue directly if you have permission.