Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
title: "Use a separate Curl handle per thread in PycurlClient so concurrent requests can't overwrite each other's URL, body, or headers"
semver_level: patch
jira_tickets_closed:
- RUN_DEVSDK-3704
---
60 changes: 38 additions & 22 deletions stripe/_http_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -871,8 +871,6 @@ def __init__(
_lib = pycurl

self.pycurl = _lib
# Initialize this within the object so that we can reuse connections.
self._curl = _lib.Curl()

self._parsed_proxy = {}
# need to urlparse the proxy, since PyCurl
Expand All @@ -886,6 +884,20 @@ def __init__(
# but we know value to be a string because all the value types on Proxy_ are strings.
self._parsed_proxy[scheme] = urlparse(cast(str, value))

def _get_curl(self):
"""
Returns this thread's Curl handle, creating it on first use.

A libcurl easy handle must never be used by more than one thread at a
time, so each thread gets its own. Reusing a handle within a thread
lets us keep its established connections.
"""
curl = getattr(self._thread_local, "curl", None)
if curl is None:
curl = self.pycurl.Curl()
self._thread_local.curl = curl
return curl

def parse_headers(self, data):
if "\r\n" not in data:
return {}
Expand Down Expand Up @@ -944,48 +956,49 @@ def _request_internal(
# pristine state, but notably reset() doesn't reset connections, so we
# still get to take advantage of those by virtue of re-using the same
# object.
self._curl.reset()
curl = self._get_curl()
curl.reset()

proxy = self._get_proxy(url)
if proxy:
if proxy.hostname:
self._curl.setopt(self.pycurl.PROXY, proxy.hostname)
curl.setopt(self.pycurl.PROXY, proxy.hostname)
if proxy.port:
self._curl.setopt(self.pycurl.PROXYPORT, proxy.port)
curl.setopt(self.pycurl.PROXYPORT, proxy.port)
if proxy.username or proxy.password:
self._curl.setopt(
curl.setopt(
self.pycurl.PROXYUSERPWD,
"%s:%s" % (proxy.username, proxy.password),
)

if method == "get":
self._curl.setopt(self.pycurl.HTTPGET, 1)
curl.setopt(self.pycurl.HTTPGET, 1)
elif method == "post":
self._curl.setopt(self.pycurl.POST, 1)
self._curl.setopt(self.pycurl.POSTFIELDS, post_data)
curl.setopt(self.pycurl.POST, 1)
curl.setopt(self.pycurl.POSTFIELDS, post_data)
else:
self._curl.setopt(self.pycurl.CUSTOMREQUEST, method.upper())
curl.setopt(self.pycurl.CUSTOMREQUEST, method.upper())

# pycurl doesn't like unicode URLs
self._curl.setopt(self.pycurl.URL, url)
curl.setopt(self.pycurl.URL, url)

self._curl.setopt(self.pycurl.WRITEFUNCTION, b.write)
self._curl.setopt(self.pycurl.HEADERFUNCTION, rheaders.write)
self._curl.setopt(self.pycurl.NOSIGNAL, 1)
self._curl.setopt(self.pycurl.CONNECTTIMEOUT, 30)
self._curl.setopt(self.pycurl.TIMEOUT, 80)
curl.setopt(self.pycurl.WRITEFUNCTION, b.write)
curl.setopt(self.pycurl.HEADERFUNCTION, rheaders.write)
curl.setopt(self.pycurl.NOSIGNAL, 1)
curl.setopt(self.pycurl.CONNECTTIMEOUT, 30)
curl.setopt(self.pycurl.TIMEOUT, 80)
_assert_no_header_injection(headers)
self._curl.setopt(
curl.setopt(
self.pycurl.HTTPHEADER,
["%s: %s" % (k, v) for k, v in dict(headers).items()],
)
if self._verify_ssl_certs:
self._curl.setopt(self.pycurl.CAINFO, stripe.ca_bundle_path)
curl.setopt(self.pycurl.CAINFO, stripe.ca_bundle_path)
else:
self._curl.setopt(self.pycurl.SSL_VERIFYHOST, False)
curl.setopt(self.pycurl.SSL_VERIFYHOST, False)

try:
self._curl.perform()
curl.perform()
except self.pycurl.error as e:
self._handle_request_error(e)

Expand All @@ -995,7 +1008,7 @@ def _request_internal(
else:
rcontent = b.getvalue().decode("utf-8")

rcode = self._curl.getinfo(self.pycurl.RESPONSE_CODE)
rcode = curl.getinfo(self.pycurl.RESPONSE_CODE)
headers = self.parse_headers(rheaders.getvalue().decode("utf-8"))

return rcontent, rcode, headers
Expand Down Expand Up @@ -1044,7 +1057,10 @@ def _get_proxy(self, url) -> Optional["ParseResult"]:
return None

def close(self):
pass
curl = getattr(self._thread_local, "curl", None)
if curl is not None:
curl.close()
self._thread_local.curl = None


class UrllibClient(HTTPClient):
Expand Down
114 changes: 114 additions & 0 deletions tests/test_http_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import base64
import json
import sys
import threading
from typing import Any, Callable, Dict, List, Optional
from unittest.mock import MagicMock, Mock, call, patch

Expand Down Expand Up @@ -1166,6 +1167,119 @@ def test_request_rejects_header_injection(
with pytest.raises(ValueError):
make_request("get", self.valid_url, headers, None)

@pytest.fixture
def created_curls(self, mocked_request_lib):
curls: List[Mock] = []

def make_curl():
curl = Mock()
curl.getinfo.return_value = 200
curls.append(curl)
return curl

mocked_request_lib.Curl = Mock(side_effect=make_curl)
return curls

@staticmethod
def setopt_values(curl, option):
return [
c.args[1]
for c in curl.setopt.call_args_list
if c.args[0] == option
]

def test_uses_one_curl_handle_per_thread(
self, mocked_request_lib, created_curls
):
client = self.REQUEST_CLIENT(proxy=self.PROXY, _lib=mocked_request_lib)

client.request("get", self.valid_url, {})
client.request("get", self.valid_url, {})
assert len(created_curls) == 1

errors: List[BaseException] = []

def request_in_thread():
try:
client.request("get", self.valid_url, {})
except BaseException as e:
errors.append(e)

thread = threading.Thread(target=request_in_thread)
thread.start()
thread.join(timeout=5)

assert errors == []
assert len(created_curls) == 2
assert created_curls[0] is not created_curls[1]

def test_concurrent_requests_do_not_share_curl_state(
self, mocked_request_lib
):
client = self.REQUEST_CLIENT(proxy=self.PROXY, _lib=mocked_request_lib)
url_a = "https://api.stripe.com/v1/payouts"
url_b = "https://api.stripe.com/v1/transfers"
a_performing = threading.Event()
release_a = threading.Event()
errors: List[BaseException] = []

# Hold request A inside perform() until request B has fully run. If the
# threads shared a handle, B's settings would replace A's mid-request.
def block_first_perform():
if not a_performing.is_set():
a_performing.set()
assert release_a.wait(timeout=5)

def request_a():
try:
client.request(
"post", url_a, {"Authorization": "Bearer sk_test_A"}, "a=1"
)
except BaseException as e:
errors.append(e)

curl_a = Mock()
curl_a.getinfo.return_value = 200
curl_a.perform.side_effect = block_first_perform
curl_b = Mock()
curl_b.getinfo.return_value = 200
mocked_request_lib.Curl = Mock(side_effect=[curl_a, curl_b])

thread_a = threading.Thread(target=request_a)
thread_a.start()
assert a_performing.wait(timeout=5)

client.request(
"post", url_b, {"Authorization": "Bearer sk_test_B"}, "b=2"
)
release_a.set()
thread_a.join(timeout=5)

assert errors == []
lib = mocked_request_lib
assert self.setopt_values(curl_a, lib.URL) == [url_a]
assert self.setopt_values(curl_a, lib.POSTFIELDS) == ["a=1"]
assert self.setopt_values(curl_a, lib.HTTPHEADER) == [
["Authorization: Bearer sk_test_A"]
]
assert self.setopt_values(curl_b, lib.URL) == [url_b]
assert self.setopt_values(curl_b, lib.POSTFIELDS) == ["b=2"]
assert self.setopt_values(curl_b, lib.HTTPHEADER) == [
["Authorization: Bearer sk_test_B"]
]

def test_close_closes_this_threads_curl_handle(
self, mocked_request_lib, created_curls
):
client = self.REQUEST_CLIENT(proxy=self.PROXY, _lib=mocked_request_lib)
client.request("get", self.valid_url, {})

client.close()

created_curls[0].close.assert_called_once_with()
client.request("get", self.valid_url, {})
assert len(created_curls) == 2


class TestPycurlClientHttpProxy(TestPycurlClient):
PROXY = "http://user:withPwd@slap:8888/"
Expand Down
Loading