Start a real http server instead of mocking libs (#1938)

* Start a real http server instead of mocking libs

tests/conftest.py

@@ -1,5 +1,7 @@
-import os
 import json
+import os
+import socket
+from threading import Thread
 
 import pytest
 import jsonschema
@@ -14,6 +16,17 @@
 except ImportError:
     eventlet = None
 
+try:
+    # Python 2
+    import BaseHTTPServer
+
+    HTTPServer = BaseHTTPServer.HTTPServer
+    BaseHTTPRequestHandler = BaseHTTPServer.BaseHTTPRequestHandler
+except Exception:
+    # Python 3
+    from http.server import BaseHTTPRequestHandler, HTTPServer
+
+
 import sentry_sdk
 from sentry_sdk._compat import iteritems, reraise, string_types
 from sentry_sdk.envelope import Envelope
@@ -561,3 +574,30 @@ def __ne__(self, test_obj):
 def teardown_profiling():
     yield
     teardown_profiler()
+
+
+class MockServerRequestHandler(BaseHTTPRequestHandler):
+    def do_GET(self):  # noqa: N802
+        # Process an HTTP GET request and return a response with an HTTP 200 status.
+        self.send_response(200)
+        self.end_headers()
+        return
+
+
+def get_free_port():
+    s = socket.socket(socket.AF_INET, type=socket.SOCK_STREAM)
+    s.bind(("localhost", 0))
+    _, port = s.getsockname()
+    s.close()
+    return port
+
+
+def create_mock_http_server():
+    # Start a mock server to test outgoing http requests
+    mock_server_port = get_free_port()
+    mock_server = HTTPServer(("localhost", mock_server_port), MockServerRequestHandler)
+    mock_server_thread = Thread(target=mock_server.serve_forever)
+    mock_server_thread.setDaemon(True)
+    mock_server_thread.start()
+
+    return mock_server_port

tests/integrations/stdlib/test_httplib.py

@@ -1,10 +1,8 @@
 import platform
-import sys
 import random
-import responses
-import pytest
+import sys
 
-from sentry_sdk.consts import MATCH_ALL
+import pytest
 
 try:
     # py3
@@ -25,25 +23,29 @@
 except ImportError:
     import mock  # python < 3.3
 
+
 from sentry_sdk import capture_message, start_transaction
+from sentry_sdk.consts import MATCH_ALL
 from sentry_sdk.tracing import Transaction
 from sentry_sdk.integrations.stdlib import StdlibIntegration
 
+from tests.conftest import create_mock_http_server
 
-def test_crumb_capture(sentry_init, capture_events):
-    sentry_init(integrations=[StdlibIntegration()])
+PORT = create_mock_http_server()
 
-    url = "http://example.com/"
-    responses.add(responses.GET, url, status=200)
 
+def test_crumb_capture(sentry_init, capture_events):
+    sentry_init(integrations=[StdlibIntegration()])
     events = capture_events()
 
-    response = urlopen(url)
-    assert response.getcode() == 200
+    url = "http://localhost:{}/some/random/url".format(PORT)
+    urlopen(url)
+
     capture_message("Testing!")
 
     (event,) = events
     (crumb,) = event["breadcrumbs"]["values"]
+
     assert crumb["type"] == "http"
     assert crumb["category"] == "httplib"
     assert crumb["data"] == {
@@ -62,14 +64,11 @@ def before_breadcrumb(crumb, hint):
         return crumb
 
     sentry_init(integrations=[StdlibIntegration()], before_breadcrumb=before_breadcrumb)
-
-    url = "http://example.com/"
-    responses.add(responses.GET, url, status=200)
-
     events = capture_events()
 
+    url = "http://localhost:{}/some/random/url".format(PORT)
     response = urlopen(url)
-    assert response.getcode() == 200
+
     capture_message("Testing!")
 
     (event,) = events
@@ -113,7 +112,7 @@ def test_httplib_misuse(sentry_init, capture_events, request):
     sentry_init()
     events = capture_events()
 
-    conn = HTTPSConnection("httpstat.us", 443)
+    conn = HTTPConnection("localhost", PORT)
 
     # make sure we release the resource, even if the test fails
     request.addfinalizer(conn.close)
@@ -138,7 +137,7 @@ def test_httplib_misuse(sentry_init, capture_events, request):
     assert crumb["type"] == "http"
     assert crumb["category"] == "httplib"
     assert crumb["data"] == {
-        "url": "https://httpstat.us/200",
+        "url": "http://localhost:{}/200".format(PORT),
         "method": "GET",
         "status_code": 200,
         "reason": "OK",