diff --git a/api.php b/api.php
index a8fa6f3d38d397d0821fad7485578366990e8d81..51e2564bead4614500f2bc21f3329c3d9df08597 100644
--- a/api.php
+++ b/api.php
@@ -278,9 +278,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
- $s_programExitCode => $default_not_executed_user_program_exit_code, //the default value if we not executed the user program
+
+ $s_programExitCode => NULL,
+ $s_runnerVersion => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
+
$s_testResultCode => 100, # this is the result from the test runner program
- $s_protocol => '' # the conversation protocol
+ $s_protocol => '', # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
@@ -298,9 +305,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
- $s_programExitCode => $default_not_executed_user_program_exit_code,
+
+ $s_programExitCode => NULL,
+ $s_runnerVersion => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
+
$s_testResultCode => 100, # this is the result from the test runner program
- $s_protocol => '' # the conversation protocol
+ $s_protocol => '', # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
@@ -316,9 +330,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
- $s_programExitCode => $default_not_executed_user_program_exit_code,
+
+ $s_programExitCode => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
+ $s_runnerVersion => NULL,
+
$s_testResultCode => 100, # this is the result from the test runner program
- $s_protocol => '' # the conversation protocol
+ $s_protocol => '', # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
@@ -345,9 +366,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => $_result[$s_return_val] === 0,
$s_hasCompiled => $_result[$s_return_val] !== 50, // all other errors returned before this line
- $s_programExitCode => $default_not_executed_user_program_exit_code,
+
+ $s_programExitCode => $_result[$s_user_program_exit_code],
+ $s_timeForCompiling => $_result[$s_timeForCompiling],
+ $s_timeForUserProgram => $_result[$s_timeForUserProgram],
+ $s_runnerVersion => $_result[$s_runnerVersion],
+
$s_testResultCode => $_result[$s_return_val], # this is the result from the test runner program
- $s_protocol => formatOutput($_result[$s_output]) # the conversation protocol
+ $s_protocol => formatOutput($_result[$s_output]), # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
@@ -381,9 +409,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => $_result[$s_return_val] === 0,
$s_hasCompiled => $_result[$s_return_val] !== 50, // all other errors returned before this line
- $s_programExitCode => $default_not_executed_user_program_exit_code,
+
+ $s_programExitCode => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
+ $s_runnerVersion => NULL,
+
$s_testResultCode => $_result[$s_return_val], # this is the result from the test runner program
- $s_protocol => formatOutput($_result[$s_output]) # the conversation protocol
+ $s_protocol => formatOutput($_result[$s_output]), # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
@@ -400,9 +435,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
- $s_programExitCode => $default_not_executed_user_program_exit_code,
+
+ $s_programExitCode => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
+ $s_runnerVersion => NULL,
+
$s_testResultCode => 100, # this is the result from the test runner program
- $s_protocol => '' # the conversation protocol
+ $s_protocol => '', # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
@@ -419,9 +461,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
- $s_programExitCode => $default_not_executed_user_program_exit_code,
+
+ $s_programExitCode => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
+ $s_runnerVersion => NULL,
+
$s_testResultCode => 100, # this is the result from the test runner program
- $s_protocol => '' # the conversation protocol
+ $s_protocol => '', # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
@@ -438,9 +487,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
- $s_programExitCode => $default_not_executed_user_program_exit_code,
+
+ $s_programExitCode => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
+ $s_runnerVersion => NULL,
+
$s_testResultCode => 100, # this is the result from the test runner program
- $s_protocol => '' # the conversation protocol
+ $s_protocol => '', # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
@@ -457,9 +513,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
- $s_programExitCode => $default_not_executed_user_program_exit_code,
+
+ $s_programExitCode => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
+ $s_runnerVersion => NULL,
+
$s_testResultCode => 100, # this is the result from the test runner program
- $s_protocol => '' # the conversation protocol
+ $s_protocol => '', # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
@@ -575,9 +638,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => $_result[$s_return_val] === 0,
$s_hasCompiled => $_result[$s_return_val] !== 50, // all other errors returned before this line
+
$s_programExitCode => $_result[$s_user_program_exit_code],
+ $s_timeForCompiling => $_result[$s_timeForCompiling],
+ $s_timeForUserProgram => $_result[$s_timeForUserProgram],
+ $s_runnerVersion => $_result[$s_runnerVersion],
+
$s_testResultCode => $_result[$s_return_val], # this is the result from the test runner program
- $s_protocol => formatOutput($_result[$s_output]) # the conversation protocol
+ $s_protocol => formatOutput($_result[$s_output]), # the conversation protocol
+
+ $s_output_test_server_version => $versionString,
);
if ($needCompilation) {
diff --git a/constants.php b/constants.php
index 10c589bbe29f638426ab20b67f98317a268bd687..16be586927243ab83605d429163682aa49617dec 100644
--- a/constants.php
+++ b/constants.php
@@ -3,7 +3,7 @@
# variables
//use this to know which version we published
-$versionString = '1.1.2';
+$versionString = '2.0.0';
$rootDirNameToUseString = 'work';
$isDebug = false; //logs debug to STDOUT
@@ -39,6 +39,8 @@ $s_arg_hardMemoryLimitInKb = 'hardMemoryLimitInKb';
$s_arg_hardDiskSpaceLimitInKb = 'hardDiskSpaceLimitInKb';
$s_arg_maxParallelTests = 'maxParallelTests';
+$s_output_test_server_version = 'testServerVersion';
+
$timeZone = date_default_timezone_get(); #'Europe/Berlin'; #date_default_timezone_get(); //'Europe/Berlin'
@@ -69,7 +71,6 @@ $status_code_InternalServerError = 100;
$s_output = 'output'; # the output from a test method
$s_return_val = 'return_val'; # the return value from a test method (the runner exit code)
-$s_user_program_exit_code = 'exit_code'; # the return value from the user program (exit code) 0 is the default value (e.g. if we have no exit code)
$default_not_executed_user_program_exit_code = 0;
@@ -126,6 +127,17 @@ $s_test_memoryLimitInKb = 'memoryLimitInKb'; # the memory limit for the test
$s_test_diskSpaceLimitInKb = 'maxDiskSpaceInKb'; # the max dis space the test can write to
$s_test_allAssets = 'allAssets'; # the asset files
+
+$exitCodeOutputPrefix = 'exit:';
+
+$s_user_program_exit_code = 'exit';
+$s_runnerVersion = 'runnerVersion';
+$s_timeForCompiling = 'timeForCompiling';
+$s_timeForUserProgram = 'timeForUserProgram';
+$s_output_without_header_part = 'stripedOutput';
+
+$s_header_part_separator = "---\n";
+
/**
* outputs a message via echo (if debug is enabled outputs the message to stdout)
* @param $responseCode int the response code to send
diff --git a/do_blackBoxTest_func.php b/do_blackBoxTest_func.php
index 5688ae93875405ebe60e8f19a6a44fe0a2b5bad1..1842c84c3da49e453525272649703d6677f6c226 100644
--- a/do_blackBoxTest_func.php
+++ b/do_blackBoxTest_func.php
@@ -1,5 +1,6 @@
<?php
+require_once __DIR__ . '/protocolHeaderPartParser.php';
/**
* runs the given black box test against the given files
@@ -39,20 +40,15 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
global $config;
global $s_command_blackBoxTest;
global $s_command_justRunTest;
- global $s_fullWorkingDirPath;
- global $status_code_FileSystemIssue;
- global $s_test_id;
global $s_test_content;
- global $s_passed;
- global $s_hasCompiled;
- global $s_programExitCode;
- global $s_testResultCode;
- global $s_protocol;
- global $responseNewLineString;
global $s_test_allAssets;
global $s_return_val;
- global $s_user_program_exit_code;
global $s_output;
+ global $s_user_program_exit_code;
+ global $s_runnerVersion;
+ global $s_timeForCompiling;
+ global $s_timeForUserProgram;
+ global $s_output_without_header_part;
$command_to_execute_string = $s_command_blackBoxTest;
@@ -76,28 +72,31 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
return array(
$s_return_val => 100,
$s_output => 'could not create asset(s)',
- $s_user_program_exit_code => 0
+ $s_user_program_exit_code => NULL,
+ $s_runnerVersion => NULL,
+ $s_timeForCompiling => NULL,
+ $s_timeForUserProgram => NULL,
);
}
$longCmd = $config['runner']
- . ' ' . $command_to_execute_string # arg[0] the test method
- . ' "' . $fullWorkingDirPath . '"' # arg[1] dir path
- . ' ' . $mainFileNameWithExtension # arg[2] file path
- . ' "' . $compileCmd . '"' # arg[3] command to execute # currently ignored because we call compile in php...
-
- . ' "' . $commandToExecute . '"' # arg[4] command to execute the test
- . ' ' . $timeout # arg[5] the timeout
- . ' ' . $memoryLimit # arg[6] the memory limit
- . ' ' . $diskSpaceLimit # arg[7] the disk limit
- . ' "' . implode(',', $sourceFileExtensions) . '"' # arg[8] the source file extensions
- . ' ' . ($needCompilation ? 'true' : 'false') # arg[9] true: compile program new, false: not
- . ' "' . $maxLinesToRead . '"' # arg[10] max lines to read from the user program (inclusive)
- . ' "' . $maxErrLinesToRead . '"' # arg[11] max lines to read from the user program stderr (inclusive)
- . ' "' . $maxLinesToWrite . '"' # arg[12] max lines to write to the user program (inclusive)
- . ' "' . $uniqueSessionId . '"' # arg[13] a unique session id
- . ' "' . ($showTestRunnerDebugOutput === TRUE ? 1 : 0) . '"' # arg[14] showTestRunnerDebugOutput
+ . ' ' . $command_to_execute_string # arg[0] the test method
+ . ' "' . $fullWorkingDirPath . '"' # arg[1] dir path
+ . ' ' . $mainFileNameWithExtension # arg[2] file path
+ . ' "' . $compileCmd . '"' # arg[3] command to execute # currently ignored because we call compile in php...
+
+ . ' "' . $commandToExecute . '"' # arg[4] command to execute the test
+ . ' ' . $timeout # arg[5] the timeout
+ . ' ' . $memoryLimit # arg[6] the memory limit
+ . ' ' . $diskSpaceLimit # arg[7] the disk limit
+ . ' "' . implode(',', $sourceFileExtensions) . '"' # arg[8] the source file extensions
+ . ' ' . ($needCompilation ? 'true' : 'false') # arg[9] true: compile program new, false: not
+ . ' "' . $maxLinesToRead . '"' # arg[10] max lines to read from the user program (inclusive)
+ . ' "' . $maxErrLinesToRead . '"' # arg[11] max lines to read from the user program stderr (inclusive)
+ . ' "' . $maxLinesToWrite . '"' # arg[12] max lines to write to the user program (inclusive)
+ . ' "' . $uniqueSessionId . '"' # arg[13] a unique session id
+ . ' "' . ($showTestRunnerDebugOutput === TRUE ? 1 : 0) . '"' # arg[14] showTestRunnerDebugOutput
;
@@ -152,7 +151,7 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
debug("time to run the black-box-tests: " . $exec_time);
}
- if ($isDebug && (isset($errorOutput) && trim($errorOutput)!=='')) {
+ if ($isDebug && (isset($errorOutput) && trim($errorOutput) !== '')) {
debug("error during execution of the (blackbox) test runner: " . $errorOutput);
}
@@ -160,23 +159,7 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
# the test assets are removed when the dir gets removed
# also the test runner could renamed/added/deleted some files
- //extract the program exit code from the protocol if any/possible else pretend 0
- $userProgramExitCode = 0;
- //the test runner uses \n
- $maybeExitOutput = substr($output, 0, strpos($output, "\n"));
-
- if(isset($maybeExitOutput)) {
- $exitCodeOutputPrefix = 'exit:';
-
- $checkString = substr($maybeExitOutput, 0, strlen($exitCodeOutputPrefix));
-
- //user output as > as start
- if ($checkString === $exitCodeOutputPrefix) {
- $exitCodeString = substr($maybeExitOutput, strlen($exitCodeOutputPrefix));
-
- $userProgramExitCode = intval($exitCodeString, 10);
- }
- }
+ $headerPart = parseHeaderPart($output);
//$output
@@ -184,7 +167,10 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
return array(
$s_return_val => $return_var,
- $s_output => $output,
- $s_user_program_exit_code => $userProgramExitCode
+ $s_output => $headerPart[$s_output_without_header_part],
+ $s_user_program_exit_code => $headerPart[$s_user_program_exit_code],
+ $s_runnerVersion => $headerPart[$s_runnerVersion],
+ $s_timeForCompiling => $headerPart[$s_timeForCompiling],
+ $s_timeForUserProgram => $headerPart[$s_timeForUserProgram],
);
}
diff --git a/do_compareFilesTest_func.php b/do_compareFilesTest_func.php
index ca5020693b58e0a044cde3d1b296e9e0feb2dbf8..ca838070f2dbbcec4f70215f33cd301437f73bd6 100644
--- a/do_compareFilesTest_func.php
+++ b/do_compareFilesTest_func.php
@@ -1,5 +1,6 @@
<?php
+require_once __DIR__ . '/protocolHeaderPartParser.php';
/**
* runs the given black box test against the given files
@@ -36,23 +37,17 @@ function do_compareFilesTest($mainFileNameWithExtension, $test, $fullWorkingDirP
global $isDebug;
global $config;
global $s_command_compareFilesTest;
- global $s_fullWorkingDirPath;
- global $status_code_FileSystemIssue;
- global $s_test_id;
global $s_test_content;
- global $s_passed;
- global $s_hasCompiled;
- global $s_programExitCode;
- global $s_testResultCode;
- global $s_protocol;
- global $responseNewLineString;
global $s_test_allAssets;
global $s_return_val;
- global $s_user_program_exit_code;
global $s_output;
global $s_fileName;
global $s_fileHash;
-
+ global $s_user_program_exit_code;
+ global $s_runnerVersion;
+ global $s_timeForCompiling;
+ global $s_timeForUserProgram;
+ global $s_output_without_header_part;
$command_to_execute_string = $s_command_compareFilesTest;
@@ -166,23 +161,7 @@ function do_compareFilesTest($mainFileNameWithExtension, $test, $fullWorkingDirP
# the test assets are removed when the dir gets removed
# also the test runner could renamed/added/deleted some files
- //extract the program exit code from the protocol if any/possible else pretend 0
- $userProgramExitCode = 0;
- //the test runner uses \n
- $maybeExitOutput = substr($output, 0, strpos($output, "\n"));
-
- if (isset($maybeExitOutput)) {
- $exitCodeOutputPrefix = 'exit:';
-
- $checkString = substr($maybeExitOutput, 0, strlen($exitCodeOutputPrefix));
-
- //user output as > as start
- if ($checkString === $exitCodeOutputPrefix) {
- $exitCodeString = substr($maybeExitOutput, strlen($exitCodeOutputPrefix));
-
- $userProgramExitCode = intval($exitCodeString, 10);
- }
- }
+ $headerPart = parseHeaderPart($output);
//$output
@@ -190,7 +169,10 @@ function do_compareFilesTest($mainFileNameWithExtension, $test, $fullWorkingDirP
return array(
$s_return_val => $return_var,
- $s_output => $output,
- $s_user_program_exit_code => $userProgramExitCode
+ $s_output => $headerPart[$s_output_without_header_part],
+ $s_user_program_exit_code => $headerPart[$s_user_program_exit_code],
+ $s_runnerVersion => $headerPart[$s_runnerVersion],
+ $s_timeForCompiling => $headerPart[$s_timeForCompiling],
+ $s_timeForUserProgram => $headerPart[$s_timeForUserProgram],
);
}
diff --git a/do_compileTest_func.php b/do_compileTest_func.php
index a512888bdbfa2992376947926de8a6fe50b077c7..927bdc3ce33fb1f1c5b0d6e30991aab997777ea3 100644
--- a/do_compileTest_func.php
+++ b/do_compileTest_func.php
@@ -1,6 +1,7 @@
<?php
include_once './constants.php';
+require_once __DIR__ . '/protocolHeaderPartParser.php';
/**
* compiles the given main file (if a compile error occurs then an answer is sent)
@@ -26,6 +27,11 @@ function do_compile($mainFileNameWithExtension, $fullWorkingDirPath, $compileCmd
global $s_output;
global $s_return_val;
global $s_command_compile;
+ global $s_user_program_exit_code;
+ global $s_runnerVersion;
+ global $s_timeForCompiling;
+ global $s_timeForUserProgram;
+ global $s_output_without_header_part;
$command_to_execute_string = $s_command_compile;
@@ -70,8 +76,14 @@ function do_compile($mainFileNameWithExtension, $fullWorkingDirPath, $compileCmd
debug("error during execution of the (compile) test runner: " . $errorOutput);
}
+ $headerPart = parseHeaderPart($output);
+
return array(
$s_return_val => $return_var,
- $s_output => $output
+ $s_output => $headerPart[$s_output_without_header_part],
+ $s_user_program_exit_code => $headerPart[$s_user_program_exit_code],
+ $s_runnerVersion => $headerPart[$s_runnerVersion],
+ $s_timeForCompiling => $headerPart[$s_timeForCompiling],
+ $s_timeForUserProgram => $headerPart[$s_timeForUserProgram],
);
}
diff --git a/do_regexTest_func.php b/do_regexTest_func.php
index 85ecd0cbfea38484fbd89a640ef7fd629da94be7..828ddb4c83471457df7acb53388171c0aa14e597 100644
--- a/do_regexTest_func.php
+++ b/do_regexTest_func.php
@@ -11,7 +11,7 @@
* @param $showTestRunnerDebugOutput true show internal debug output from the test runner (included in the test response)
* @return array
*/
-function do_regexTest_json($allFiles, $testContent, $showTestRunnerDebugOutput)
+function do_regexTest_json_DEPRECATED($allFiles, $testContent, $showTestRunnerDebugOutput)
{
global $availableRegexModes;
diff --git a/protocolHeaderPartParser.php b/protocolHeaderPartParser.php
new file mode 100644
index 0000000000000000000000000000000000000000..9a4b39f3ad595b4499b3d34a9877c617d9ffe8fa
--- /dev/null
+++ b/protocolHeaderPartParser.php
@@ -0,0 +1,82 @@
+<?php
+
+function parseHeaderPart($output)
+{
+
+ global $s_header_part_separator;
+ global $s_user_program_exit_code;
+ global $s_runnerVersion;
+ global $s_timeForCompiling;
+ global $s_timeForUserProgram;
+ global $s_output_without_header_part;
+
+
+
+ //if we have no header tuples we might have only so ---\n
+ //this is because we cannot use $headerPos -1
+ $headerPos = strpos($output, $s_header_part_separator);
+ $headerPartText = substr($output, 0, $headerPos);
+ $stripedOutput = substr($output, $headerPos + strlen($s_header_part_separator));
+
+ $exit = NULL;
+ $runnerVersion = NULL;
+ $timeForCompiling = NULL;
+ $timeForUserProgram = NULL;
+
+
+ $headerPartLines = explode("\n", $headerPartText);
+
+ //empty lines will be ignored because no case matches
+
+ foreach ($headerPartLines as $line) {
+
+ if (strpos($line, $s_user_program_exit_code) === 0) {
+
+ $val = getTuple($line);
+ if ($val === null) continue; //nothing would change...
+
+ $exit = intval($val, 10);
+
+ } else if (strpos($line, $s_runnerVersion) === 0) {
+
+ $val = getTuple($line);
+ if ($val === null) continue;
+
+ $runnerVersion = $val;
+
+ } else if (strpos($line, $s_timeForCompiling) === 0) {
+
+ $val = getTuple($line);
+ if ($val === null) continue;
+
+ $timeForCompiling = intval($val, 10);
+
+ } else if (strpos($line, $s_timeForUserProgram) === 0) {
+
+ $val = getTuple($line);
+ if ($val === null) continue;
+
+ $timeForUserProgram = intval($val, 10);
+ }
+ }
+
+
+ return array(
+ $s_user_program_exit_code => $exit,
+ $s_runnerVersion => $runnerVersion,
+ $s_timeForCompiling => $timeForCompiling,
+ $s_timeForUserProgram => $timeForUserProgram,
+ $s_output_without_header_part => $stripedOutput,
+ );
+}
+
+function getTuple($tupleString)
+{
+ $tuple = explode(':', $tupleString);
+
+ if (count($tuple) !== 2) return NULL;
+
+ return $tuple[1] === "null"
+ ? NULL
+ : $tuple[1];
+}
\ No newline at end of file
diff --git a/readme.md b/readme.md
index 6d9aee9f72368789c276197e1d12d4000df478cf..5bd60d1ba9565b9372988e723a2898629af0791f 100644
--- a/readme.md
+++ b/readme.md
@@ -64,21 +64,23 @@ The request is a json object so the request should be from mime type application
"allFiles": [
{
"fileName": "Main.java",
- "fileContent: "public class Main {\npublic static void main(String[] args) {\nSystem.out.println("echo");\n}\n}"
+ "fileContent": "public class Main {\npublic static void main(String[] args) {\nSystem.out.println(\"echo\");\n}\n}"
}
],
"tests": [
- "command": "blackBoxTest",
- "testContents": [
- {
- "id": 0,
- "content": "> Hello World\n!",
- "assets": []
- }
- ],
- "timeoutInMs": 1000,
- "memoryLimitInKb": 1000,
- "maxDiskSpaceInKb": 1000
+ {
+ "command": "blackBoxTest",
+ "testContents": [
+ {
+ "id": 0,
+ "content": "> Hello World\n!",
+ "assets": []
+ }
+ ],
+ "timeoutInMs": 1000,
+ "memoryLimitInKb": 1000,
+ "maxDiskSpaceInKb": 1000
+ }
]
}
```
@@ -90,11 +92,10 @@ A json array representing the responses. If an errors occurs before or after the
**In case some server error happened and no test could be executed then just one object is returned:**
```json
-[
- {
- "responseCode": 0,
- "message": "some error"
-]
+{
+ "responseCode": 0,
+ "message": "some error"
+}
```
If the tests could be executed or to code to execute the tests is reached then the response will be an array for with an response object for every test
@@ -114,9 +115,13 @@ If the tests could be executed or to code to execute the tests is reached then t
"id": 0,
"passed": true,
"hasCompiled": true,
- "programExitCode": 0,
+ "programExitCode": null,
+ "timeForCompiling": null,
+ "timeForUserProgram": null,
+ "runnerVersion": "1.0.0",
"testResultCode": 0,
- "protocol": "can be multiline \n\r string"
+ "protocol": "can be multiline \n\r string",
+ "testServerVersion": "1.0.0",
}
]
}
@@ -137,8 +142,15 @@ other fields
* **hasCompiled**: (bool) true: test has compiled (if the test needs compilation, if not always true), false: not [actually this can be determined by checking testResultCode === (compile error return code from the test runner)]
* **testResultCode**: (int) the exit code of the test runner [see Test Runner Return Codes](##%20Return-Codes)
-* **programExitCode**: (int) the exit code of the users program **[CURRENTLY NOT SET, ALWAYS 0!]**
-* **protocol**: (string) a string representing the output (could contain the error message. New line chars are encoded with **\\\r\\\n** (because json encode automaticall encodes \r\n to \\\n\\\n)
+* **programExitCode**: (int | null) the exit code of the users program, null if the user program was not (fully) executed e.g. timeout hit
+* **timeForCompiling**: (int | null) the time for compiling the user program or null if not compiled
+* **timeForUserProgram**: (int | null) the time for executing the user program or null if not (fully) executed
+
+* **runnerVersion**: (string | null) the test-runner version or null if the test-runner was not executed
+
+* **protocol**: (string) a string representing the output (could contain the error message. New line chars are encoded with **\\\r\\\n** (because json encode automaticall encodes \r\n to \\\n\\\n)
+
+* **testServerVersion**: (string) the test-server version
## Response Codes from the test server
@@ -152,15 +164,56 @@ other fields
**For explanation about the different test types see below**
-### Example
-```json
-{
- "passed": true,
- "hasCompiled": true,
- "programExitCode": 0,
- "testResultCode": 0,
- "protocol": "can be multiline \n\r string"
-}
+## Protocols
+
+The Test-Runner uses `\n` new lines
+
+All protocols consists out of 2 parts
+
+the first part is the header part and contains some misc data about the test run e.g. test-runner version...
+
+the seconds part is the real conversion protocol between the test-runner and the user program
+
+the two parts are separated by a `---\n` line
+
+
+**the header part including the `---\n`** must be removed from the protocol!
+
+### Protocol Header part
+
+the header part has the same schema for all tests
+
+every information is on a new line and is a tuple where key and value is separated by `:`
+
+there are the following information possible (the tuple keys) (the keys are not case sensitive!)
+
+- `exit` (number) the user program exit code or `null` if it din't finish e.g. timeout hit
+- `runnerVersion` (string) the test runner version
+- `timeForCompiling` (int) the time for compiling in ms or `null` if compiling was not done or finished
+- `timeForUserProgram` (int) the time for running the user program or `null` if the execution was not done or finished
+
+after the header part the line `---\n` is followed, indicating the end of the header part
+
+all values can be `null`!
+
+the order of the tuples is not important
+
+note that the tuple key and values must not contain the separator char `:` or the `\n` char or `---\n`
+
+```
+exit:0
+runnerVersion:1.0.0
+timeForCompiling:null
+timeForUserProgram:null
+---
+```
+
+```
+exit:null
+runnerVersion:1.0.0
+timeForCompiling:1000
+timeForUserProgram:null
+---
```
## Protocol Output Compile Test
@@ -172,15 +225,14 @@ when a compile error occurred then the output of the compiler
## Protocol Output for Just Run Test
-Actually the program is run without any input...
+Actually the program is run without any input (to stdin)...
-1. Line: contains the arguments passed to the users program, starting with a $ sign (only if arguments were present)
-all other lines (meaning different by the first character):
+all lines (meaning different by the first character):
+* '**$**' are the arguments passed to the users program, starting with a $ sign (only if arguments were present)
* '**>**' indicating a line that was read from the users program
* '**EOT**' indicates that the test ended here (written after the program terminated)
this is useful when some output is received after the test has finished
-* '**exit:**' the user program exit code (can be empty if the user program didn't end e.g. timeout)
## Protocol Output Regex Test
1. Line: the used mode e.g. used mode: blacklist
@@ -190,19 +242,10 @@ the next lines contains the test results for every file
## Protocol Output Black Box Test
-The Test-Runner uses `\n` new lines
-1./2. Line can be the user program exit code (so the test server can easier get it)
-but only if the user program actually ran and finished
-* '**exit:**' the user program exit code (can be empty if the user program didn't end e.g. timeout) if any this must be the first line in the protocol (to find it faster)
- **the test runner must not output the a line starting with `exit:` else the test server might get the wrong exit code**
- * the number is the expected exit code (defaults to 0 e.g. when there is no exit code because of timeout)
-
-* '**e**' is the expected user program exit code
- * this must be the first or second line
- * the first if we have no program exit code
- * the second if we have a user program exit code
- * not that this is only true for protocol lines produced by the test runner (not e.g. test content input lines to the test runner)
+1 Line can be the expected user program exit code
+but only if the user program actually ran and finished
+* '**e**' is the expected user program exit code followed by a number (e.g. `e0`)
all other lines
* '**$**' are the arguments passed to the users program, starting with a $ sign (only if arguments were present)
@@ -222,7 +265,6 @@ when a compile error occurred then the output of the compiler
## Protocol Output Compare files Test
```
-exit:[X]
EOT
ccompare [fileName1WithExtension] [fileName2WithExtension] (ignoreCase) (ignoreLeadingWhitespaces) (ignoreTrailingWhitespaces) (ignoreAllEmptyOrWhitespacesOnlyLines)
c[result]