How to get response headers only, without the body?

[quantimnot]

My Use Case

I needed to calculate the storage requirements for a very large set of archives hosted on a website. The archives are retrieved with a POST method, so I can't use HEAD.

My Solution

Make the POST call and close the connection after the response headers are received.

I didn't see a means to accomplish this with std/httpclient or chronos/apps/http/httpclient. So this is what I did:

include chronos/apps/http/httpclient

proc getHeaderOnlyResponse(req: HttpClientRequestRef): Future[HttpTable] {.
     async.} =
  var buffer: array[HttpMaxHeadersSize, byte]
  let bytesRead =
    try:
      await req.connection.reader.readUntil(addr buffer[0],
                                            len(buffer), HeadersMark).wait(
                                            req.session.headersTimeout)
    except CancelledError as exc:
      raise exc
    except AsyncTimeoutError:
      raiseHttpReadError("Reading response headers timed out")
    except AsyncStreamError:
      raiseHttpReadError("Could not read response headers")

  ## Process response headers.
  let resp = parseResponse(buffer, false)
  if resp.failed():
    raiseHttpReadError("Invalid headers received")

  let headers =
    block:
      var res = HttpTable.init()
      for key, value in resp.headers(buffer):
        res.add(key, value)
      if res.count(ContentTypeHeader) > 1:
        raiseHttpReadError("Invalid headers received, too many `Content-Type`")
      if res.count(ContentLengthHeader) > 1:
        raiseHttpReadError("Invalid headers received, too many `Content-Length`")
      if res.count(TransferEncodingHeader) > 1:
        raiseHttpReadError("Invalid headers received, too many `Transfer-Encoding`")
      res

  waitFor req.closeWait
  return headers


proc fetchHeaders*(request: HttpClientRequestRef): Future[HttpTable] {.
     async.} =
  doAssert(request.state == HttpReqRespState.Ready,
           "Request's state is " & $request.state)
  let connection =
    try:
      await request.session.acquireConnection(request.address)
    except CancelledError as exc:
      request.setError(newHttpInterruptError())
      raise exc
    except HttpError as exc:
      request.setError(exc)
      raise exc

  request.connection = connection

  try:
    let headers = request.prepareRequest()
    request.connection.state = HttpClientConnectionState.RequestHeadersSending
    request.state = HttpReqRespState.Open
    await request.connection.writer.write(headers)
    request.connection.state = HttpClientConnectionState.RequestHeadersSent
    request.connection.state = HttpClientConnectionState.RequestBodySending
    if len(request.buffer) > 0:
      await request.connection.writer.write(request.buffer)
    request.connection.state = HttpClientConnectionState.RequestBodySent
    request.state = HttpReqRespState.Finished
  except CancelledError as exc:
    request.setError(newHttpInterruptError())
    raise exc
  except AsyncStreamError as exc:
    let error = newHttpWriteError("Could not send request headers")
    request.setError(error)
    raise error

  let resp =
    try:
      await request.getHeaderOnlyResponse()
    except CancelledError as exc:
      request.setError(newHttpInterruptError())
      raise exc
    except HttpError as exc:
      request.setError(exc)
      raise exc
  return resp

This probably isn't a common use case and worth adding to the implementation, so I'm just dropping it here in case someone finds it useful.

[cheatfate]

Here i want to show how to obtain response object, which can be used to obtain response headers.

import chronos/apps/http/httpclient

proc getClient() {.async.} =
  var session = HttpSessionRef.new({HttpClientFlag.NoVerifyHost,
                                    HttpClientFlag.NoVerifyServerName},
                                    maxRedirections = 10)
  var request =
    block:
      let res = HttpClientRequestRef.new(session, "https://httpbin.org/get",
                                         meth = MethodGet)
      if res.isErr():
        echo "ERROR: ", res.error()
        quit(1)
      res.get()
  # Here we obtain `response` object.
  var response = await request.send()
  echo "HTTP RESPONSE STATUS CODE = ", response.status
  echo "HTTP RESPONSE HEADES: "
  echo $response.headers
  await response.closeWait()
  await request.closeWait()
  await session.closeWait()

proc postClient() {.async.} =
  var session = HttpSessionRef.new({HttpClientFlag.NoVerifyHost,
                                    HttpClientFlag.NoVerifyServerName},
                                    maxRedirections = 10)
  var request =
    block:
      let res = HttpClientRequestRef.new(session, "https://httpbin.org/post",
                                         meth = MethodPost)
      if res.isErr():
        echo "ERROR: ", res.error()
        quit(1)
      res.get()
  request.headers.set("Content-type", "application/json")
  request.headers.set("Accept", "application/json")
  request.headers.set("Transfer-encoding", "chunked")
  var writer = await request.open()
  await writer.write("{\"data\": \"data\"}")
  await writer.finish()
  await writer.closeWait()
  # Here we obtain `response` object.
  var response = await request.finish()
  echo "HTTP RESPONSE STATUS CODE = ", response.status
  echo "HTTP RESPONSE HEADES: "
  echo $response.headers
  await response.closeWait()
  await request.closeWait()
  await session.closeWait()

when isMainModule:
  waitFor getClient()
  waitFor postClient()

As you can see there no need to dive deep into chronos internals.

[cheatfate]

And this is main difference between std/httpclient and chronos.httpclient because in both samples above, body has not been received yet, only response headers are obtained from the network.

[arnetheduck]

@cheatfate how about we create an examples/ folder and put a few of these simple cases in there? along with https://forum.nim-lang.org/t/7964#52137 and perhaps a TCP ping/pong server