From 7dc59a6098a724cdd0fcd79bf1b000f472d469cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janis=20Da=CC=88hne?= <janis.daehne@informatik.uni-halle.de>
Date: Wed, 24 May 2023 11:18:38 +0200
Subject: [PATCH] - added support for adt structural induction checker (test)
---
api.php | 40 +++-
constants.php | 3 +-
...nalCheckerStructuralInductionTest_func.php | 212 ++++++++++++++++++
helpers.php | 2 +-
readme.md | 5 +
5 files changed, 257 insertions(+), 5 deletions(-)
create mode 100644 do_externalCheckerStructuralInductionTest_func.php
diff --git a/api.php b/api.php
index 4be47da..86a0558 100644
--- a/api.php
+++ b/api.php
@@ -446,7 +446,8 @@ foreach ($testCases as $test) {
$hasTriedToCompile = TRUE;
}
- } else if ($arg_command === $s_command_regexTest) {
+ }
+ else if ($arg_command === $s_command_regexTest) {
require_once './do_regexTest_func.php';
@@ -479,7 +480,8 @@ foreach ($testCases as $test) {
addToDebugTimings("after regex test");
- } else if ($arg_command === $s_command_blackBoxTest || $arg_command === $s_command_justRunTest || $arg_command === $s_command_compareFilesTest || $arg_command === $s_command_externalCheckerTest) {
+ }
+ else if ($arg_command === $s_command_blackBoxTest || $arg_command === $s_command_justRunTest || $arg_command === $s_command_compareFilesTest || $arg_command === $s_command_externalCheckerTest || $arg_command === $s_command_externalCheckerStructuralInductionTest) {
if ($isDebug) {
$time_pre = microtime(true);
@@ -571,7 +573,12 @@ foreach ($testCases as $test) {
$arg_DiskSpaceLimit = $test[$s_test_diskSpaceLimitInKb];
- if ($compileCmd === NULL || $execCmd === NULL) {
+ addToDebugTimings("needCompilation2 " . $compileCmd);
+ addToDebugTimings("execCmd " . $execCmd);
+
+ # return error if we have no cmd to execute
+ # there is a special case when compile command is null and we are running a structural induction test (this does not need compile command)
+ if (($compileCmd === NULL && $arg_command !== $s_command_externalCheckerStructuralInductionTest) || $execCmd === NULL) {
$result = create_test_result($status_code_DbIssue, "could not find compile command or execute command (unknown plang?)", $testId,
FALSE, FALSE, NULL,
NULL, NULL, NULL,
@@ -622,6 +629,11 @@ foreach ($testCases as $test) {
$needCompilation = !$hasTriedToCompile;
+ # this test does not compile, only run/check
+ if ($arg_command === $s_command_externalCheckerStructuralInductionTest) {
+ $needCompilation = FALSE;
+ }
+
if ($needCompilation === FALSE && $compileResult[$s_hasCompiled] === FALSE) {
//if the compile fail ... all other tests would give use main file not found...
@@ -684,6 +696,28 @@ foreach ($testCases as $test) {
addToDebugTimings("after external checker test");
+ }
+ else if ($arg_command === $s_command_externalCheckerStructuralInductionTest) {
+ require_once './do_externalCheckerStructuralInductionTest_func.php';
+
+ addToDebugTimings("before external checker structural induction test");
+
+ try {
+ $_result = do_externalCheckerStructualInductionTest($conn, $traceString, $allUserSourceCode, $arg_mainFileNameWithExtension, $test, $fullWorkingDirPath,
+ $min_timeout, $min_compileTimeout, $min_memoryLimit, $min_diskSpaceLimit,
+ $compileCmd, $execCmd, $sourceFileExtensions, False,
+ $requestDistinctionString,
+ $showTestRunnerDebugOutput,
+ $compilerOptions,
+ $arg_characterLimitProtocol,
+ $arg_maxPoints
+ );
+ } catch (Exception $e) {
+ //critical error, abort execution (all other tests are likely to fail)
+ output($status_code_InternalServerError, "error executing external checker structural induction test: " . $e->getMessage());
+ goto handleCleanup;
+ }
+
}
else {
require_once './do_blackBoxTest_func.php';
diff --git a/constants.php b/constants.php
index fc15a83..75db990 100644
--- a/constants.php
+++ b/constants.php
@@ -101,10 +101,11 @@ $s_arg_maxNumberOfTestsWithOneRequest = 'maxNumberOfTestsWithOneRequest';
$s_command_compile = 'compileTest';
$s_command_blackBoxTest = 'blackBoxTest';
$s_command_externalCheckerTest = 'externalCheckerTest';
+$s_command_externalCheckerStructuralInductionTest= 'externalCheckerStructuralInductionTest';
$s_command_compareFilesTest = 'compareFilesTest';
$s_command_regexTest = 'regexTest';
$s_command_justRunTest = 'justRunTest';
-$s_supportedCommands = [$s_command_compile, $s_command_blackBoxTest, $s_command_externalCheckerTest, $s_command_compareFilesTest, $s_command_regexTest, $s_command_justRunTest];
+$s_supportedCommands = [$s_command_compile, $s_command_blackBoxTest, $s_command_externalCheckerTest, $s_command_externalCheckerStructuralInductionTest, $s_command_compareFilesTest, $s_command_regexTest, $s_command_justRunTest];
# db table columns
diff --git a/do_externalCheckerStructuralInductionTest_func.php b/do_externalCheckerStructuralInductionTest_func.php
new file mode 100644
index 0000000..daa9548
--- /dev/null
+++ b/do_externalCheckerStructuralInductionTest_func.php
@@ -0,0 +1,212 @@
+<?php
+
+require_once __DIR__ . '/protocolHeaderPartParser.php';
+
+/**
+ * runs the given black box test against the given files
+ * @param $conn PDO
+ * @param $traceString string
+ * @param $sourceCode string the user program source code as one string
+ * @param string $mainFileNameWithExtension the main file to compile (with extension)
+ * @param $test
+ * @param $fullWorkingDirPath
+ * @param int|string $timeout the timeout in MS (can be string from db or int from default values)
+ * @param int|string $compileTimeout the compile timeout in MS (can be string from db or int from default values)
+ * @param int|string $memoryLimit the memory limit in kb (can be string from db or int from default values)
+ * @param int|string $diskSpaceLimit the max disk space the program can write to (can be string from db or int from default values)
+ * @param string $compileCmd the command to execute to compile the file(s)
+ * @param string $execCmd the command to execute to run the test
+ * @param $sourceFileExtensions array the list with the file extensions from the source files for the p language
+ * @param $needCompilation bool true: should compile sources, false: not (this assumes the sources are already compiled)
+ * @param $justRun true: just run test, false: normal back box test
+ * @param $uniqueSessionId string the unique session id for the test runner to handle sandbox file system management
+ * @param $showTestRunnerDebugOutput 1: show internal debug output from the test runner (included in the test response)
+ * @param $compilerOptions string the options for the compiler
+ * @param $arg_characterLimitProtocol
+ * @param $maxPoints int
+ * @return array the result
+ * @internal param array $allFiles all files to copy in teh dir to use
+ * Format: array[int] => {fileName: string, fileContent: string}
+ */
+function do_externalCheckerStructualInductionTest($conn, $traceString, $sourceCode, $mainFileNameWithExtension, $test, $fullWorkingDirPath,
+ $timeout, $compileTimeout, $memoryLimit, $diskSpaceLimit,
+ $compileCmd, $execCmd, $sourceFileExtensions, $needCompilation,
+ $uniqueSessionId,
+ $showTestRunnerDebugOutput,
+ $compilerOptions,
+ $arg_characterLimitProtocol,
+ $arg_maxPoints
+
+
+)
+{
+
+ global $isDebug;
+ global $config;
+ global $s_command_externalCheckerStructuralInductionTest;
+ global $s_test_content;
+ global $s_test_allAssets;
+ global $s_return_val;
+ global $s_output;
+ global $s_user_program_exit_code;
+ global $s_runnerVersion;
+ global $s_timeForCompiling;
+ global $s_timeForUserProgram;
+ global $s_timeoutInMsUsed;
+ global $s_compileTimeoutInMsUsed;
+ global $s_output_without_header_part;
+ global $s_characterLimitExceeded;
+ global $s_points;
+
+
+ $command_to_execute_string = $s_command_externalCheckerStructuralInductionTest;
+
+
+ # compile is done in the runner
+ $commandToExecute = $execCmd;
+ $output = [];
+ $return_var = -1;
+
+ $testContent = $test[$s_test_content];
+ $allAssets = $test[$s_test_allAssets];
+
+ addToDebugTimings("before creating test assets");
+
+ # create the assets for the current test
+ if (createTestAssets($allAssets, $fullWorkingDirPath) === FALSE) {
+ //error message is in createFiles function
+ return array(
+ $s_return_val => 100,
+ $s_output => 'could not create asset(s)',
+ $s_user_program_exit_code => NULL,
+ $s_runnerVersion => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
+ );
+ }
+ addToDebugTimings("after creating test assets");
+
+
+ $longCmd = $config['runner']
+ . ' ' . escapeshellarg($command_to_execute_string) # arg[0] the test method
+ . ' ' . escapeshellarg($fullWorkingDirPath) # arg[1] dir path
+ . ' ' . escapeshellarg($mainFileNameWithExtension) # arg[2] file path
+ . ' ' . escapeshellarg($compileCmd) # arg[3] compile command
+
+ . ' ' . escapeshellarg($commandToExecute) # arg[4] command to execute the test
+ . ' ' . escapeshellarg($timeout) # arg[5] the timeout in ms
+ . ' ' . escapeshellarg($memoryLimit) # arg[6] the memory limit
+ . ' ' . escapeshellarg($diskSpaceLimit) # arg[7] the disk limit
+ . ' ' . escapeshellarg(implode(',', $sourceFileExtensions)) # arg[8] the source file extensions
+
+
+ . ' ' . escapeshellarg($arg_characterLimitProtocol) # arg[9] character limit for the test protocol, output cut after the limit
+ . ' ' . escapeshellarg($compilerOptions) # arg[10] options for the compiler
+ . ' ' . escapeshellarg($arg_maxPoints) # arg[11] (int) the max points we expect (but checker can give mor or less than that)
+
+ . ' ' . escapeshellarg(($showTestRunnerDebugOutput === TRUE ? 1 : 0)) # arg[12] showTestRunnerDebugOutput
+ . ' ' . escapeshellarg($uniqueSessionId) # arg[13] a unique session id
+
+ ;
+
+
+
+ addToDebugTimings("longCmd: " . $longCmd);
+
+ require_once './transactionHelper.php';
+ addTraceLogToDb($conn, $config, $traceString, $longCmd, $sourceCode);
+
+ if ($isDebug) {
+ debug("using runner: " . $config['runner']);
+ debug("full command: " . $longCmd);
+ }
+
+ # the test content is passed in as stdin (because might be too long for a console argument
+ # e.g. windows its 32bit ca. 2.000 kb...
+
+ $pipesDescriptor = array(
+ 0 => array('pipe', 'r'), # stdin is a pipe that the child will read from
+ 1 => array('pipe', 'w'), # stdout is a pipe that the child will write to
+ 2 => array("pipe", "w") # stderr
+ );
+
+
+ if ($isDebug) {
+ $time_pre = microtime(true);
+ }
+
+
+ $env = $config['environmentVarsParsed'];
+
+ addToDebugTimings("starting runner");
+
+ # without bypass_shell it won't work on windows
+ $process = proc_open($longCmd, $pipesDescriptor, $pipes, $fullWorkingDirPath, $env, array('bypass_shell' => TRUE));
+
+// $state = proc_get_status($process);
+// warn('open pid: ' . $state['pid']);
+
+ if (is_resource($process)) {
+ fwrite($pipes[0], $testContent);
+ fclose($pipes[0]);
+
+
+ $output = stream_get_contents($pipes[1]);
+ fclose($pipes[1]);
+
+ $errorOutput = stream_get_contents($pipes[2]);
+ fclose($pipes[2]);
+
+ $return_var = proc_close($process);
+ }
+
+ # we cannot use exec because this does no let us write to the stdin of the process
+ #exec($longCmd, $output, $return_var);
+
+ if ($isDebug) {
+ $time_post = microtime(true);
+ $exec_time = ($time_post - $time_pre) * 1000; //in ms
+ debug("time to run the black-box-tests: " . $exec_time);
+ }
+
+ if ($isDebug && (isset($errorOutput) && trim($errorOutput) !== '')) {
+ debug("error during execution of the (blackbox) test runner: " . $errorOutput);
+ }
+
+ addToDebugTimings("finished runner");
+
+
+ # the test assets are removed when the dir gets removed
+ # also the test runner could renamed/added/deleted some files
+
+ $headerPart = parseHeaderPart($output);
+
+ //$output
+
+ #file_put_contents($fullWorkingDirPath . 'xyz.txt', $output);
+
+ # in case the test runner didn't even run we may need to cast (vals from db)
+ if (gettype($timeout) === 'string') {
+ $timeout = intval($timeout, 10);
+ }
+
+ if (gettype($compileTimeout) === 'string') {
+ $compileTimeout = intval($compileTimeout, 10);
+ }
+
+ addToDebugTimings("finished parsing runner header");
+ addToDebugTimings("headerPart " . json_encode($headerPart));
+
+ return array(
+ $s_return_val => $return_var,
+ $s_output => $headerPart[$s_output_without_header_part],
+ $s_user_program_exit_code => $headerPart[$s_user_program_exit_code],
+ $s_runnerVersion => $headerPart[$s_runnerVersion],
+ $s_timeForCompiling => $headerPart[$s_timeForCompiling],
+ $s_timeForUserProgram => $headerPart[$s_timeForUserProgram],
+ $s_timeoutInMsUsed => $timeout,
+ $s_compileTimeoutInMsUsed => $compileTimeout,
+ $s_characterLimitExceeded => $headerPart[$s_characterLimitExceeded],
+ $s_points => $headerPart[$s_points]
+ );
+}
diff --git a/helpers.php b/helpers.php
index 3d4968f..d16d1cd 100644
--- a/helpers.php
+++ b/helpers.php
@@ -15,7 +15,7 @@ function formatOutput($output) {
$formattedOutput = $output;
# backend requires output to be a string... so convert it if needed
- if (count($formattedOutput) === 0) {
+ if (strlen($formattedOutput) === 0) {
$formattedOutput = '';
} else if (is_array($output)) {
$formattedOutput = join($responseNewLineString, $formattedOutput);
diff --git a/readme.md b/readme.md
index e5955e7..5c18c9a 100644
--- a/readme.md
+++ b/readme.md
@@ -85,6 +85,11 @@ The commands will probably require the users files (code). This files can be acc
- default is true, is specified in the test content as `evaluate:true/false`
- only used for external checkers
+* **49** - contains the compiler options (if any or empty string)
+ - can be used for external checkers
+ - compiler options are added automatically after all commands
+ - if this is used the compiler options are not appended (only inserted once)
+
* **#50** - contains the timeout in ms
* **#51** - contains the memory limit in kb
* **#52** - contains the disk space limit in kb (the program can write to)
--
GitLab