historify_request doesnt work properly for multipart requests in a multi-threaded environment

[drmaples]

I am intermittently seeing the wrong requests in httpretty.latest_requests because of the way historify_request either appends or overwrites the end of the latest_requests list.

This bug is exposed by using multithreaded code via futures. Overwriting the last element in cls.latest_requests is not safe in a multi-threaded environment.

There needs to be a way to index the requests in latest_requests and overwrite/append the appropriate one.

Below is some sample code that demonstrates this, it may take several times of running the test in order for the test to fail.

Output

########################################
<HTTPrettyRequest("", total_headers=7, body_length=9)>
Host: 127.0.0.1:666
Content-Length: 10
Accept-Encoding: gzip, deflate
Accept: */*
User-Agent: python-requests/2.3.0 CPython/2.7.7 Darwin/13.3.0
_request_number: 2
MY_HEADER: SomeHeader2222


<HTTPrettyRequest("", total_headers=7, body_length=10)>
Host: 127.0.0.1:666
Content-Length: 10
Accept-Encoding: gzip, deflate
Accept: */*
User-Agent: python-requests/2.3.0 CPython/2.7.7 Darwin/13.3.0
_request_number: 2
MY_HEADER: SomeHeader2222
########################################

F
======================================================================
FAIL: test_historify_request_breaks_for_multipart_requests (test_bad_historify_request.TestMyAsyncSender)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/private/tmp/darrell_env/lib/python2.7/site-packages/httpretty/core.py", line 1029, in wrapper
    return test(*args, **kw)
  File "/private/tmp/darrell_env/test_bad_historify_request.py", line 79, in test_historify_request_breaks_for_multipart_requests
    self.assertEquals('SomeHeader111', fake_reqs[0].headers['MY_HEADER'])
AssertionError: u'SomeHeader111' != 'SomeHeader2222'

Sample Code

#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import unicode_literals

'''
Python 2.7.7

futures==2.1.6
httpretty==0.8.3
nose==1.3.3
requests==2.3.0
'''

from unittest import TestCase

import requests
import futures
import httpretty


class MyAsyncSender(object):
    def __init__(self, base_url):
        self.base_url = base_url

    def send_async(self, request_data_list):
        def response_hook(r, **kw):
            print('Async Request #{} response status_code={}\ninput:\n{}\noutput:\n{}\n'.format(r.request.headers['_request_number'], r.request.body, r.status_code, repr(r.content)))

        sess = requests.Session()
        sess.mount('http://', requests.adapters.HTTPAdapter(max_retries=3))
        sess.mount('https://', requests.adapters.HTTPAdapter(max_retries=3))

        futs = []
        with futures.ThreadPoolExecutor(max_workers=10) as executor:
            for n, (api_call, data) in enumerate(request_data_list):
                headers = {
                    'MY_HEADER': str(api_call),
                    '_request_number': str(n + 1),
                }

                future = executor.submit(sess.post, self.base_url, data=data, headers=headers, timeout=500, hooks={'response': response_hook})
                futs.append(future)

            futures.wait(futs, timeout=30)

        return [fut.result() for fut in futs]


class TestMyAsyncSender(TestCase):
    def setUp(self):
        self.mas = MyAsyncSender('http://127.0.0.1:666/foobar')

    @httpretty.activate
    def test_historify_request_breaks_for_multipart_requests(self):
        server_responses = [
            ('bla', 401),
            (None, 200)
        ]
        httpretty.register_uri(httpretty.POST, self.mas.base_url, responses=[httpretty.Response(body=body, status=status) for body, status in server_responses])

        # httplib thinks unicode is a file since its not an instance of str, makes a multipart request
        # see here: http://hg.python.org/cpython/file/07bbc2e20aa4/Lib/httplib.py#l737
        urls_and_data = [
            ('SomeHeader111', u'some-xml1'),
            ('SomeHeader2222', u'some-xml22'),
        ]
        responses = self.mas.send_async(urls_and_data)

        fake_reqs = sorted(httpretty.HTTPretty.latest_requests, key=lambda r: r.headers['_request_number'])
        self.assertEquals(2, len(fake_reqs))

        print '#' * 40
        for r in fake_reqs:
            print r
            print r.headers
            print
        print '#' * 40

        self.assertEquals('SomeHeader111', fake_reqs[0].headers['MY_HEADER'])
        self.assertEquals('SomeHeader2222', fake_reqs[1].headers['MY_HEADER'])

        self.assertEquals(2, len(responses))

    @httpretty.activate
    def test_historify_request_works_for_normal_requests(self):
        server_responses = [
            ('bla', 401),
            (None, 200)
        ]
        httpretty.register_uri(httpretty.POST, self.mas.base_url, responses=[httpretty.Response(body=body, status=status) for body, status in server_responses])

        #
        urls_and_data = [
            ('SomeHeader111', str('some-xml1')),
            ('SomeHeader2222', str('some-xml22')),
        ]
        responses = self.mas.send_async(urls_and_data)

        fake_reqs = sorted(httpretty.HTTPretty.latest_requests, key=lambda r: r.headers['_request_number'])
        self.assertEquals(2, len(fake_reqs))

        print '#' * 40
        for r in fake_reqs:
            print r
            print r.headers
            print
        print '#' * 40

        self.assertEquals('SomeHeader111', fake_reqs[0].headers['MY_HEADER'])
        self.assertEquals('SomeHeader2222', fake_reqs[1].headers['MY_HEADER'])

        self.assertEquals(2, len(responses))