EvoMaster
EvoMaster copied to clipboard
EMResearch
Generated test cases fail due to 'checkInstrumentation'
the generated test for scout-api
package em.external.se.devscout.scoutapi;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
import java.util.Map;
import java.util.List;
import static org.evomaster.client.java.controller.api.EMTestUtils.*;
import org.evomaster.client.java.controller.SutHandler;
import io.restassured.RestAssured;
import static io.restassured.RestAssured.given;
import io.restassured.response.ValidatableResponse;
import static org.hamcrest.Matchers.*;
import io.restassured.config.JsonConfig;
import io.restassured.path.json.config.JsonPathConfig;
import static io.restassured.config.RedirectConfig.redirectConfig;
import static org.evomaster.client.java.controller.contentMatchers.NumberMatcher.*;
import static org.evomaster.client.java.controller.contentMatchers.StringMatcher.*;
import static org.evomaster.client.java.controller.contentMatchers.SubStringMatcher.*;
import static org.evomaster.client.java.controller.expect.ExpectationHandler.expectationHandler;
import org.evomaster.client.java.controller.expect.ExpectationHandler;
import io.restassured.path.json.JsonPath;
import java.util.Arrays;
/**
* This file was automatically generated by EvoMaster on 2024-04-17T09:46:11.478+08:00[Asia/Shanghai]
* <br>
* The generated test suite contains 3 tests
* <br>
* Covered targets: 21
* <br>
* Used time: 0h 1m 58s
* <br>
* Needed budget for current results: 100%
* <br>
* This file contains one example of each category of fault. The test cases in this file are a subset of the set of test cases likely to indicate faults.
*/
public class EM_RDMIO_BASELINE_False_False_0_90_True_True_True_0_ConArchive_100_True_100_1_Test_fault_representatives {
private static final SutHandler controller = new em.external.se.devscout.scoutapi.ExternalEvoMasterController("/Users/lixin/Workplace/Java/EMB/jdk_8_maven/em/external/rest/scout-api/src/main/resources/scout-api-sut.jar").setJaCoCo("/Users/lixin/Downloads/jacoco-0.8.12/lib/jacocoagent.jar","/Users/lixin/Downloads/jacoco-0.8.12/lib/jacococli.jar","/Users/lixin/Downloads/jacoco.exec",8899);
private static String baseUrlOfSut;
/** [ems] - expectations master switch - is the variable that activates/deactivates expectations individual test cases
* by default, expectations are turned off. The variable needs to be set to [true] to enable expectations
*/
private static boolean ems = false;
/**
* sco - supported code oracle - checking that the response status code is among those supported according to the schema
*/
private static boolean sco = false;
@BeforeClass
public static void initClass() {
controller.setupForGeneratedTest();
// ((ExternalEvoMasterController) controller).setInstrumentation(true);
baseUrlOfSut = controller.startSut();
controller.registerOrExecuteInitSqlCommandsIfNeeded();
assertNotNull(baseUrlOfSut);
RestAssured.enableLoggingOfRequestAndResponseIfValidationFails();
RestAssured.useRelaxedHTTPSValidation();
RestAssured.urlEncodingEnabled = false;
RestAssured.config = RestAssured.config()
.jsonConfig(JsonConfig.jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))
.redirect(redirectConfig().followRedirects(false));
}
@AfterClass
public static void tearDown() {
controller.stopSut();
}
@Before
public void initTest() {
controller.resetDatabase(null);
controller.resetStateOfSUT();
}
/**
* [test_0_with500] is a part of 1 or more clusters, as defined by the selected clustering options.
* ErrorText_0
* LastLine_0
*/
@Test(timeout = 60000)
public void test_0_with500() throws Exception {
given().accept("*/*")
.header("x-EMextraHeader123", "")
.get(baseUrlOfSut + "/api/v2/activities?" +
"attrs=j3lXBmEjhG0&" +
"text=NNElQ7&" +
"ages=sJAb89zaoaubuKUg&" +
"durations=D4k2ttrHsWfLa&" +
"my_favourites=false&" +
"ratings_count_min=137&" +
"ratings_average_min=0.6247422789970475&" +
"random=1147563612")
.then()
.statusCode(500) // se/devscout/scoutapi/resource/ActivityResourceV2_77_all
.assertThat()
.contentType("application/json")
.body("'code'", numberMatches(500.0))
.body("'message'", containsString("API currently does not support filtering on your own favourites."));
}
/**
* [test_1_with500] is a part of 1 or more clusters, as defined by the selected clustering options.
* ErrorText_0
* LastLine_0
*/
@Test(timeout = 60000)
public void test_1_with500() throws Exception {
given().accept("*/*")
.header("x-EMextraHeader123", "")
.get(baseUrlOfSut + "/api/v1/media_files/834/file?EMextraParam123=_EM_5_XYZ_")
.then()
.statusCode(500) // se/devscout/scoutapi/resource/MediaFileResource_260_downloadFile
.assertThat()
.contentType("application/json")
.body("'code'", numberMatches(500.0));
}
@Test(timeout = 60000)
public void test_2() throws Exception {
ExpectationHandler expectationHandler = expectationHandler();
ValidatableResponse res_0 = given().accept("*/*")
.header("x-EMextraHeader123", "")
.contentType("application/json")
.body(" { " +
" \"id\": 833, " +
" \"name\": \"_EM_14_XYZ_\", " +
" \"date_published\": \"1949-11-03T18:39:42\", " +
" \"date_created\": \"1965-09-23T21:17:10\", " +
" \"date_updated\": \"1941-02-22T12:24:52\", " +
" \"description_introduction\": \"_EM_16_XYZ_\", " +
" \"description_prepare\": \"qIwX\", " +
" \"description_main\": \"OzqItqwK67\", " +
" \"description_safety\": \"_EM_17_XYZ_\", " +
" \"age_min\": 55, " +
" \"participants_max\": 66, " +
" \"time_max\": 997, " +
" \"source\": \"_EM_18_XYZ_\", " +
" \"media_files\": [] " +
" } ")
.patch(baseUrlOfSut + "/api/v2/activities/833")
.then()
.statusCode(401)
.assertThat()
.contentType("text/plain")
.body(containsString("Credentials are required to access this resource."));
expectationHandler.expect(ems)
.that(sco, Arrays.asList(200).contains(res_0.extract().statusCode()));
}
}
If there are stacktraces of the errors, please paste them here.
java.lang.IllegalStateException: Instrumentation is not active
at org.evomaster.client.java.controller.ExternalSutController.checkInstrumentation(ExternalSutController.java:523)
at org.evomaster.client.java.controller.ExternalSutController.setExecutingInitSql(ExternalSutController.java:475)
at org.evomaster.client.java.controller.internal.SutController.registerOrExecuteInitSqlCommandsIfNeeded(SutController.java:446)
at em.external.se.devscout.scoutapi.EM_RDMIO_BASELINE_False_False_0_90_True_True_True_0_ConArchive_100_True_100_1_Test_fault_representatives.initClass(EM_RDMIO_BASELINE_False_False_0_90_True_True_True_0_ConArchive_100_True_100_1_Test_fault_representatives.java:63)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24)
at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
at org.junit.runners.ParentRunner.run(ParentRunner.java:363)
at org.junit.runner.JUnitCore.run(JUnitCore.java:137)
at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:69)
at com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38)
at com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11)
at com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35)
at com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:232)
at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:55)
I discovered that the database initialization script is executed by ServerController, which is not initialized when externalSutController.instrumentation is set to false.
So, I added ((ExternalSutController) controller).setInstrumentation(true); to the method initClass in the test, and the test ran successfully.
the code that initializes ServerController in ExternalSutController
public final String startSut() {
SimpleLogger.info("Going to start the SUT");
.......
if (this.instrumentation) {
if (this.serverController == null) {
this.serverController = new ServerController();
}
...
}
the code that executes the database initialization script in ExternalSutController
public final void setExecutingInitSql(boolean executingInitSql) {
this.checkInstrumentation();
this.serverController.setExecutingInitSql(executingInitSql);
ExecutionTracer.setExecutingInitSql(executingInitSql);
}
Other important info:
- version of EvoMaster (EM) used
- 2.0.0
- how EM is run (eg, if from JAR or from one of its OS installers)
- run generated tests of scout-api from EMB in IDEA
- version of applicable runtimes (eg, JVM, NodeJS and .Net). For Java, can paste the output of
java --versionopenjdk version "1.8.0_352" OpenJDK Runtime Environment (Zulu 8.66.0.15-CA-macos-aarch64) (build 1.8.0_352-b08) OpenJDK 64-Bit Server VM (Zulu 8.66.0.15-CA-macos-aarch64) (build 25.352-b08, mixed mode) - command-line options used to run EM
hi @Lx09 , thanks for reporting this bug.
@man-zhang do you think you have sometime to look at it?
thanks @man-zhang
this should now be fixed in snapshot version on master branch
will close once we make new release