diff --git a/documentation/src/docs/asciidoc/release-notes/release-notes-5.13.0-M3.adoc b/documentation/src/docs/asciidoc/release-notes/release-notes-5.13.0-M3.adoc index 0fc3363128ec..02e6d302fb22 100644 --- a/documentation/src/docs/asciidoc/release-notes/release-notes-5.13.0-M3.adoc +++ b/documentation/src/docs/asciidoc/release-notes/release-notes-5.13.0-M3.adoc @@ -105,11 +105,13 @@ to start reporting discovery issues. via a configuration parameter. Please also see the <<../user-guide/index.adoc#extensions-keeping-state-autocloseable-migration, migration note>> for third-party extensions wanting to support both JUnit 5.13 and earlier versions. - * `java.util.Locale` arguments are now converted according to the IETF BCP 47 language tag format. See the <<../user-guide/index.adoc#writing-tests-parameterized-tests-argument-conversion-implicit, User Guide>> for details. +* Avoid reporting potentially misleading validation exception for `@ParameterizedClass` + test classes and `@ParameterizedTest` methods as suppressed exception for earlier + failures. [[release-notes-5.13.0-M3-junit-vintage]] === JUnit Vintage diff --git a/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/ClassTemplateInvocationContextProvider.java b/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/ClassTemplateInvocationContextProvider.java index 51a2edada45b..0af499b57f14 100644 --- a/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/ClassTemplateInvocationContextProvider.java +++ b/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/ClassTemplateInvocationContextProvider.java @@ -90,6 +90,8 @@ public interface ClassTemplateInvocationContextProvider extends Extension { * invoked; never {@code null} * @return a {@code Stream} of {@code ClassTemplateInvocationContext} * instances for the invocation of the class template; never {@code null} + * @throws TemplateInvocationValidationException if a validation fails when + * while providing or closing the {@link java.util.stream.Stream}. * @see #supportsClassTemplate * @see ExtensionContext */ diff --git a/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TemplateInvocationValidationException.java b/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TemplateInvocationValidationException.java new file mode 100644 index 000000000000..94cad7ab8677 --- /dev/null +++ b/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TemplateInvocationValidationException.java @@ -0,0 +1,35 @@ +/* + * Copyright 2015-2025 the original author or authors. + * + * All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v2.0 which + * accompanies this distribution and is available at + * + * https://www.eclipse.org/legal/epl-v20.html + */ + +package org.junit.jupiter.api.extension; + +import static org.apiguardian.api.API.Status.EXPERIMENTAL; + +import org.apiguardian.api.API; +import org.junit.platform.commons.JUnitException; + +/** + * {@code TemplateInvocationValidationException} is an exception thrown by a + * {@link TestTemplateInvocationContextProvider} or + * {@link ClassTemplateInvocationContextProvider} if a validation fails when + * while providing or closing {@link java.util.stream.Stream} of invocation + * contexts. + * + * @since 5.13 + */ +@API(status = EXPERIMENTAL, since = "5.13") +public class TemplateInvocationValidationException extends JUnitException { + + private static final long serialVersionUID = 1L; + + public TemplateInvocationValidationException(String message) { + super(message); + } +} diff --git a/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TestTemplateInvocationContextProvider.java b/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TestTemplateInvocationContextProvider.java index 31076236ecae..f4a64479cde9 100644 --- a/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TestTemplateInvocationContextProvider.java +++ b/junit-jupiter-api/src/main/java/org/junit/jupiter/api/extension/TestTemplateInvocationContextProvider.java @@ -86,6 +86,8 @@ public interface TestTemplateInvocationContextProvider extends Extension { * to be invoked; never {@code null} * @return a {@code Stream} of {@code TestTemplateInvocationContext} * instances for the invocation of the test template method; never {@code null} + * @throws TemplateInvocationValidationException if a validation fails when + * while providing or closing the {@link java.util.stream.Stream}. * @see #supportsTestTemplate * @see ExtensionContext */ diff --git a/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/descriptor/TemplateExecutor.java b/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/descriptor/TemplateExecutor.java index 64988963685f..b2537fb226c9 100644 --- a/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/descriptor/TemplateExecutor.java +++ b/junit-jupiter-engine/src/main/java/org/junit/jupiter/engine/descriptor/TemplateExecutor.java @@ -19,8 +19,10 @@ import org.junit.jupiter.api.extension.Extension; import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.TemplateInvocationValidationException; import org.junit.jupiter.engine.execution.JupiterEngineExecutionContext; import org.junit.jupiter.engine.extension.ExtensionRegistry; +import org.junit.platform.commons.util.ExceptionUtils; import org.junit.platform.commons.util.Preconditions; import org.junit.platform.engine.TestDescriptor; import org.junit.platform.engine.UniqueId; @@ -52,11 +54,24 @@ private void executeForProvider(P provider, AtomicInteger invocationIndex, int initialValue = invocationIndex.get(); - try (Stream stream = provideContexts(provider, extensionContext)) { + Stream stream = provideContexts(provider, extensionContext); + try { stream.forEach(invocationContext -> createInvocationTestDescriptor(invocationContext, invocationIndex.incrementAndGet()) // .ifPresent(testDescriptor -> execute(dynamicTestExecutor, testDescriptor))); } + catch (Throwable t) { + try { + stream.close(); + } + catch (TemplateInvocationValidationException ignore) { + // ignore exceptions from close() to avoid masking the original failure + } + throw ExceptionUtils.throwAsUncheckedException(t); + } + finally { + stream.close(); + } Preconditions.condition( invocationIndex.get() != initialValue || mayReturnZeroContexts(provider, extensionContext), diff --git a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedInvocationContextProvider.java b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedInvocationContextProvider.java index ffa324c13df6..77e5a98059b7 100644 --- a/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedInvocationContextProvider.java +++ b/junit-jupiter-params/src/main/java/org/junit/jupiter/params/ParameterizedInvocationContextProvider.java @@ -17,6 +17,7 @@ import java.util.stream.Stream; import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.TemplateInvocationValidationException; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.ArgumentsProvider; import org.junit.jupiter.params.provider.ArgumentsSource; @@ -47,12 +48,20 @@ protected Stream provideInvocationContexts(ExtensionContext extensionContext, invocationCount.incrementAndGet(); return declarationContext.createInvocationContext(formatter, arguments, invocationCount.intValue()); }) - .onClose(() -> - Preconditions.condition(invocationCount.get() > 0 || declarationContext.isAllowingZeroInvocations(), - () -> String.format("Configuration error: You must configure at least one set of arguments for this @%s", declarationContext.getAnnotationName()))); + .onClose(() -> validateInvokedAtLeastOnce(invocationCount.get(),declarationContext )); // @formatter:on } + private static void validateInvokedAtLeastOnce(long invocationCount, + ParameterizedDeclarationContext declarationContext) { + if (invocationCount == 0 && !declarationContext.isAllowingZeroInvocations()) { + String message = String.format( + "Configuration error: You must configure at least one set of arguments for this @%s", + declarationContext.getAnnotationName()); + throw new TemplateInvocationValidationException(message); + } + } + private static List collectArgumentSources(ParameterizedDeclarationContext declarationContext) { List argumentsSources = findRepeatableAnnotations(declarationContext.getAnnotatedElement(), ArgumentsSource.class); diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedClassIntegrationTests.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedClassIntegrationTests.java index 5fa99e90480b..6e678900084e 100644 --- a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedClassIntegrationTests.java +++ b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedClassIntegrationTests.java @@ -40,6 +40,7 @@ import static org.junit.platform.testkit.engine.EventConditions.started; import static org.junit.platform.testkit.engine.EventConditions.test; import static org.junit.platform.testkit.engine.EventConditions.uniqueId; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; import static org.junit.platform.testkit.engine.TestExecutionResultConditions.suppressed; @@ -69,6 +70,7 @@ import org.junit.jupiter.api.extension.AnnotatedElementContext; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.TemplateInvocationValidationException; import org.junit.jupiter.engine.AbstractJupiterTestEngineTests; import org.junit.jupiter.engine.Constants; import org.junit.jupiter.engine.descriptor.ClassTemplateInvocationTestDescriptor; @@ -368,8 +370,9 @@ void failsWhenInvocationIsRequiredButNoArgumentSetsAreProvided() { var results = executeTestsForClass(ForbiddenZeroInvocationsTestCase.class); results.containerEvents().assertThatEvents() // - .haveExactly(1, event(finishedWithFailure(message( - "Configuration error: You must configure at least one set of arguments for this @ParameterizedClass")))); + .haveExactly(1, + event(finishedWithFailure(instanceOf(TemplateInvocationValidationException.class), message( + "Configuration error: You must configure at least one set of arguments for this @ParameterizedClass")))); } @Test diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestExtensionTests.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestExtensionTests.java index 24dba2894cd7..9eb006afbbc9 100644 --- a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestExtensionTests.java +++ b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestExtensionTests.java @@ -36,6 +36,7 @@ import org.junit.jupiter.api.extension.ExecutableInvoker; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.MediaType; +import org.junit.jupiter.api.extension.TemplateInvocationValidationException; import org.junit.jupiter.api.extension.TestInstances; import org.junit.jupiter.api.function.ThrowingConsumer; import org.junit.jupiter.api.parallel.ExecutionMode; @@ -145,7 +146,7 @@ void throwsExceptionWhenParameterizedTestIsNotInvokedAtLeastOnce() { extensionContextWithAnnotatedTestMethod); // cause the stream to be evaluated stream.toArray(); - var exception = assertThrows(JUnitException.class, stream::close); + var exception = assertThrows(TemplateInvocationValidationException.class, stream::close); assertThat(exception).hasMessage( "Configuration error: You must configure at least one set of arguments for this @ParameterizedTest"); diff --git a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java index 58ae0dd33a73..be726c17fba1 100644 --- a/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java +++ b/jupiter-tests/src/test/java/org/junit/jupiter/params/ParameterizedTestIntegrationTests.java @@ -24,7 +24,6 @@ import static org.junit.jupiter.engine.discovery.JupiterUniqueIdBuilder.uniqueIdForTestTemplateMethod; import static org.junit.jupiter.params.converter.DefaultArgumentConverter.DEFAULT_LOCALE_CONVERSION_FORMAT_PROPERTY_NAME; import static org.junit.jupiter.params.provider.Arguments.arguments; -import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectIteration; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectUniqueId; @@ -88,6 +87,8 @@ import org.junit.jupiter.api.extension.ParameterResolutionException; import org.junit.jupiter.api.extension.ParameterResolver; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.extension.TemplateInvocationValidationException; +import org.junit.jupiter.engine.AbstractJupiterTestEngineTests; import org.junit.jupiter.engine.JupiterTestEngine; import org.junit.jupiter.params.ParameterizedTestIntegrationTests.RepeatableSourcesTestCase.Action; import org.junit.jupiter.params.aggregator.AggregateWith; @@ -112,8 +113,8 @@ import org.junit.jupiter.params.support.ParameterDeclarations; import org.junit.platform.commons.PreconditionViolationException; import org.junit.platform.commons.util.ClassUtils; -import org.junit.platform.engine.DiscoverySelector; import org.junit.platform.engine.TestDescriptor; +import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.testkit.engine.EngineExecutionResults; import org.junit.platform.testkit.engine.EngineTestKit; import org.junit.platform.testkit.engine.Event; @@ -123,7 +124,7 @@ /** * @since 5.0 */ -class ParameterizedTestIntegrationTests { +class ParameterizedTestIntegrationTests extends AbstractJupiterTestEngineTests { private final Locale originalLocale = Locale.getDefault(Locale.Category.FORMAT); @@ -395,7 +396,7 @@ void executesLifecycleMethods() { LifecycleTestCase.lifecycleEvents.clear(); LifecycleTestCase.testMethods.clear(); - var results = execute(selectClass(LifecycleTestCase.class)); + var results = executeTestsForClass(LifecycleTestCase.class); results.allEvents().assertThatEvents() // .haveExactly(1, event(test("test1"), displayName("[1] argument=foo"), finishedWithFailure(message("foo")))) // @@ -456,8 +457,9 @@ void failsWhenInvocationIsRequiredButNoArgumentSetsAreProvided() { var results = execute(ZeroInvocationsTestCase.class, "testThatRequiresInvocations", String.class); results.containerEvents().assertThatEvents() // - .haveExactly(1, event(finishedWithFailure(message( - "Configuration error: You must configure at least one set of arguments for this @ParameterizedTest")))); + .haveExactly(1, + event(finishedWithFailure(instanceOf(TemplateInvocationValidationException.class), message( + "Configuration error: You must configure at least one set of arguments for this @ParameterizedTest")))); } @Test @@ -500,12 +502,20 @@ void executesWithIso639LocaleConversionFormat() { results.allEvents().assertStatistics(stats -> stats.started(4).succeeded(4)); } - private EngineExecutionResults execute(DiscoverySelector... selectors) { - return EngineTestKit.engine(new JupiterTestEngine()).selectors(selectors).execute(); - } + @Test + void reportsExceptionInStaticInitializersWithoutInvocationCountValidation() { + var results = executeTestsForClass(ExceptionInStaticInitializerTestCase.class); - private EngineExecutionResults execute(Class testClass, String methodName, Class... methodParameterTypes) { - return execute(selectMethod(testClass, methodName, ClassUtils.nullSafeToString(methodParameterTypes))); + var failure = results.containerEvents().stream() // + .filter(finishedWithFailure()::matches) // + .findAny() // + .orElseThrow(); + + var throwable = failure.getRequiredPayload(TestExecutionResult.class).getThrowable().orElseThrow(); + + assertThat(throwable) // + .isInstanceOf(ExceptionInInitializerError.class) // + .hasNoSuppressedExceptions(); } private EngineExecutionResults execute(Map configurationParameters, Class testClass, @@ -520,6 +530,10 @@ private EngineExecutionResults execute(String methodName, Class... methodPara return execute(TestCase.class, methodName, methodParameterTypes); } + private EngineExecutionResults execute(Class testClass, String methodName, Class... methodParameterTypes) { + return executeTests(selectMethod(testClass, methodName, ClassUtils.nullSafeToString(methodParameterTypes))); + } + /** * @since 5.4 */ @@ -947,7 +961,7 @@ void duplicateMethodNames() { // other words, we're not really testing the support for @RepeatedTest // and @TestFactory, but their presence also contributes to the bug // reported in #3001. - ParameterizedTestIntegrationTests.this.execute(selectClass(DuplicateMethodNamesMethodSourceTestCase.class))// + executeTestsForClass(DuplicateMethodNamesMethodSourceTestCase.class)// .testEvents()// .assertStatistics(stats -> stats.started(8).failed(0).finished(8)); } @@ -1366,7 +1380,7 @@ void closeAutoCloseableArgumentsAfterTestDespiteEarlyFailure() { @Test void executesTwoIterationsBasedOnIterationAndUniqueIdSelector() { var methodId = uniqueIdForTestTemplateMethod(TestCase.class, "testWithThreeIterations(int)"); - var results = execute(selectUniqueId(appendTestTemplateInvocationSegment(methodId, 3)), + var results = executeTests(selectUniqueId(appendTestTemplateInvocationSegment(methodId, 3)), selectIteration(selectMethod(TestCase.class, "testWithThreeIterations", "int"), 1)); results.allEvents().assertThatEvents() // @@ -2648,4 +2662,24 @@ void test(AutoCloseableArgument autoCloseable) { } } + static class ExceptionInStaticInitializerTestCase { + + static { + //noinspection ConstantValue + if (true) + throw new RuntimeException("boom"); + } + + private static Stream getArguments() { + return Stream.of("foo", "bar"); + } + + @ParameterizedTest + @MethodSource("getArguments") + void test(String value) { + fail("should not be called: " + value); + } + + } + }