Update contributing guide. Update tests to use default client fixtures.

Author: fkdosilovic

docs/source/contributors_guide/contributing.rst

@@ -47,24 +47,47 @@ You'll see a different output since the documentation is build with
 Testing
 -------
 
-If you implemented a feature or fixed a bug, please add tests for it.
+.. warning::
+    If you are implementing features or fixing bugs, you are expected to have
+    all of the three API keys (ATD, Sulu, and RapidAPI) setup and set in you
+    environment variables - ``JUDGE0_SULU_API_KEY``, ``JUDGE0_RAPID_API_KEY``,
+    and ``JUDGE0_ATD_API_KEY``.
 
-Unfortunately, at the moment you cannot run full test suite because it requires
-access to API keys for all implemented API hubs (ATD, Sulu, and RapidAPI) and
-a private Judge0 instance. To partially address this situation, you can run and
-test your implemented feature and tests locally and use the GitHub CI pipeline
-to run the full test suite.
+Every bug fix or new feature should have tests for it. The tests are located in
+the ``tests`` directory and are written using `pytest <https://docs.pytest.org/en/stable/>`_.
 
-To run the tests locally, you can use the following command:
+While working with the tests, you should use the following fixtures:
 
-.. code-block:: console
+* ``default_ce_client`` - a client, chosen based on the environment variables set, that uses the CE flavor of the client.
+* ``default_extra_ce_client`` - a client, chosen based on the environment variables set, that uses the Extra CE flavor of the client.
 
-    $ pytest -svv tests -k '<test_name>'
+The ``default_ce_client`` and ``default_extra_ce_client`` are fixtures that
+return a client based on the environment variables set. This enables you to
+run the full test suite locally, but also to run the tests on the CI pipeline
+without changing the tests.
 
-To make the test compatible with the CI pipeline, you should use one of the
-client fixtures:
+You can use the fixtures in your tests like this:
 
 .. code-block:: python
     
     def test_my_test(request):
-        client = request.getfixturevalue("judge0_ce_client") # or judge0_extra_ce_client
+        client = request.getfixturevalue("default_ce_client") # or default_extra_ce_client
+
+To run the tests locally, you can use the following command:
+
+.. code-block:: console
+
+    $ pytest -svv tests -k '<test_name>'
+
+This will enable you to run a single test, without incurring the cost of
+running the full test suite. If you want to run the full test suite, you can
+use the following command:
+
+.. code-block:: console
+
+    $ pytest -svv tests
+
+or you can create a draft PR and let the CI pipeline run the tests for you.
+The CI pipeline will run the tests on every PR, using a private instance
+of Judge0, so you can be sure that your changes are not breaking the existing
+functionality.

tests/conftest.py

@@ -13,23 +13,31 @@ def judge0_ce_client():
     api_key = os.getenv("JUDGE0_TEST_API_KEY")
     api_key_header = os.getenv("JUDGE0_TEST_API_KEY_HEADER")
     endpoint = os.getenv("JUDGE0_TEST_CE_ENDPOINT")
-    client = clients.Client(
-        endpoint=endpoint,
-        auth_headers={api_key_header: api_key},
-    )
-    return client
+
+    if api_key is None or api_key_header is None or endpoint is None:
+        return None
+    else:
+        client = clients.Client(
+            endpoint=endpoint,
+            auth_headers={api_key_header: api_key},
+        )
+        return client
 
 
 @pytest.fixture(scope="session")
 def judge0_extra_ce_client():
     api_key = os.getenv("JUDGE0_TEST_API_KEY")
     api_key_header = os.getenv("JUDGE0_TEST_API_KEY_HEADER")
     endpoint = os.getenv("JUDGE0_TEST_EXTRA_CE_ENDPOINT")
-    client = clients.Client(
-        endpoint=endpoint,
-        auth_headers={api_key_header: api_key},
-    )
-    return client
+
+    if api_key is None or api_key_header is None or endpoint is None:
+        return None
+    else:
+        client = clients.Client(
+            endpoint=endpoint,
+            auth_headers={api_key_header: api_key},
+        )
+        return client
 
 
 @pytest.fixture(scope="session")
@@ -63,12 +71,44 @@ def rapid_extra_ce_client():
 @pytest.fixture(scope="session")
 def sulu_ce_client():
     api_key = os.getenv("JUDGE0_SULU_API_KEY")
+    if api_key is None:
+        pytest.fail(
+            "Sulu API key is not available for testing. Make sure to have "
+            "JUDGE0_SULU_API_KEY in your environment variables."
+        )
+
     client = clients.SuluJudge0CE(api_key)
     return client
 
 
 @pytest.fixture(scope="session")
 def sulu_extra_ce_client():
     api_key = os.getenv("JUDGE0_SULU_API_KEY")
+    if api_key is None:
+        pytest.fail(
+            "Sulu API key is not available for testing. Make sure to have "
+            "JUDGE0_SULU_API_KEY in your environment variables."
+        )
+
     client = clients.SuluJudge0ExtraCE(api_key)
     return client
+
+
+@pytest.fixture(scope="session")
+def default_ce_client(judge0_ce_client, sulu_ce_client):
+    if judge0_ce_client is not None:
+        return judge0_ce_client
+    if sulu_ce_client is not None:
+        return sulu_ce_client
+
+    pytest.fail("No default CE client available for testing.")
+
+
+@pytest.fixture(scope="session")
+def default_extra_ce_client(judge0_extra_ce_client, sulu_extra_ce_client):
+    if judge0_extra_ce_client is not None:
+        return judge0_extra_ce_client
+    if sulu_extra_ce_client is not None:
+        return sulu_extra_ce_client
+
+    pytest.fail("No default Extra CE client available for testing.")

tests/test_api_test_cases.py

@@ -207,7 +207,7 @@ def test_single_test_case_in_iterable(self, test_cases, stdin, expected_output):
 def test_test_cases_from_run(
     source_code_or_submissions, test_cases, expected_status, request
 ):
-    client = request.getfixturevalue("judge0_ce_client")
+    client = request.getfixturevalue("default_ce_client")
 
     if isinstance(source_code_or_submissions, str):
         submissions = judge0.run(
@@ -254,7 +254,7 @@ def test_test_cases_from_run(
     ],
 )
 def test_no_test_cases(submissions, expected_status, request):
-    client = request.getfixturevalue("judge0_ce_client")
+    client = request.getfixturevalue("default_ce_client")
 
     submissions = judge0.run(
         client=client,
@@ -269,7 +269,7 @@ def test_no_test_cases(submissions, expected_status, request):
 
 @pytest.mark.parametrize("n_submissions", [42, 84])
 def test_batched_test_cases(n_submissions, request):
-    client = request.getfixturevalue("judge0_ce_client")
+    client = request.getfixturevalue("default_ce_client")
     submissions = [
         Submission(source_code=f"print({i})", expected_output=f"{i}")
         for i in range(n_submissions)

tests/test_submission.py

@@ -52,7 +52,7 @@ def test_from_json():
 
 
 def test_status_before_and_after_submission(request):
-    client = request.getfixturevalue("judge0_ce_client")
+    client = request.getfixturevalue("default_ce_client")
     submission = Submission(source_code='print("Hello World!")')
 
     assert submission.status is None
@@ -65,7 +65,7 @@ def test_status_before_and_after_submission(request):
 
 
 def test_is_done(request):
-    client = request.getfixturevalue("judge0_ce_client")
+    client = request.getfixturevalue("default_ce_client")
     submission = Submission(source_code='print("Hello World!")')
 
     assert submission.status is None
@@ -77,7 +77,7 @@ def test_is_done(request):
 
 
 def test_language_before_and_after_execution(request):
-    client = request.getfixturevalue("judge0_ce_client")
+    client = request.getfixturevalue("default_ce_client")
     code = """\
     public class Main {
         public static void main(String[] args) {
@@ -97,7 +97,7 @@ def test_language_before_and_after_execution(request):
 
 
 def test_language_executable(request):
-    client = request.getfixturevalue("judge0_ce_client")
+    client = request.getfixturevalue("default_ce_client")
     code = b64decode(
         "f0VMRgIBAQAAAAAAAAAAAAIAPgABAAAAAABAAAAAAABAAAAAAAAAAEAQAAAAAAAAAAAAAEAAOAABAEAABAADAAEAAAAFAAAAABAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAJQAAAAAAAAAlAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADHAjVANsAG+GABAAInHDwUx/41HPA8FAGhlbGxvLCB3b3JsZAoALnNoc3RydGFiAC50ZXh0AC5yb2RhdGEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACwAAAAEAAAAGAAAAAAAAAAAAQAAAAAAAABAAAAAAAAAXAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABEAAAABAAAAAgAAAAAAAAAYAEAAAAAAABgQAAAAAAAADQAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAABAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAAlEAAAAAAAABkAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAA"  # noqa: E501
     )