Skip to content
Snippets Groups Projects
Commit 13f3c05c authored by Janis Daniel Dähne's avatar Janis Daniel Dähne
Browse files

- added support for compile, execution time measurement 

- changed test runner protocol
parent 33d0f63e
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
api.php
+ 89
− 19
View file @ 13f3c05c
......@@ -278,9 +278,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
$s_programExitCode => $default_not_executed_user_program_exit_code, //the default value if we not executed the user program
$s_programExitCode => NULL,
$s_runnerVersion => NULL,
$s_timeForCompiling => NULL,
$s_timeForUserProgram => NULL,
$s_testResultCode => 100, # this is the result from the test runner program
$s_protocol => '' # the conversation protocol
$s_protocol => '', # the conversation protocol
$s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
......@@ -298,9 +305,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
$s_programExitCode => $default_not_executed_user_program_exit_code,
$s_programExitCode => NULL,
$s_runnerVersion => NULL,
$s_timeForCompiling => NULL,
$s_timeForUserProgram => NULL,
$s_testResultCode => 100, # this is the result from the test runner program
$s_protocol => '' # the conversation protocol
$s_protocol => '', # the conversation protocol
$s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
......@@ -316,9 +330,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
$s_programExitCode => $default_not_executed_user_program_exit_code,
$s_programExitCode => NULL,
$s_timeForCompiling => NULL,
$s_timeForUserProgram => NULL,
$s_runnerVersion => NULL,
$s_testResultCode => 100, # this is the result from the test runner program
$s_protocol => '' # the conversation protocol
$s_protocol => '', # the conversation protocol
$s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
......@@ -345,9 +366,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => $_result[$s_return_val] === 0,
$s_hasCompiled => $_result[$s_return_val] !== 50, // all other errors returned before this line
$s_programExitCode => $default_not_executed_user_program_exit_code,
$s_programExitCode => $_result[$s_user_program_exit_code],
$s_timeForCompiling => $_result[$s_timeForCompiling],
$s_timeForUserProgram => $_result[$s_timeForUserProgram],
$s_runnerVersion => $_result[$s_runnerVersion],
$s_testResultCode => $_result[$s_return_val], # this is the result from the test runner program
$s_protocol => formatOutput($_result[$s_output]) # the conversation protocol
$s_protocol => formatOutput($_result[$s_output]), # the conversation protocol
$s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
......@@ -381,9 +409,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => $_result[$s_return_val] === 0,
$s_hasCompiled => $_result[$s_return_val] !== 50, // all other errors returned before this line
$s_programExitCode => $default_not_executed_user_program_exit_code,
$s_programExitCode => NULL,
$s_timeForCompiling => NULL,
$s_timeForUserProgram => NULL,
$s_runnerVersion => NULL,
$s_testResultCode => $_result[$s_return_val], # this is the result from the test runner program
$s_protocol => formatOutput($_result[$s_output]) # the conversation protocol
$s_protocol => formatOutput($_result[$s_output]), # the conversation protocol
$s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
......@@ -400,9 +435,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
$s_programExitCode => $default_not_executed_user_program_exit_code,
$s_programExitCode => NULL,
$s_timeForCompiling => NULL,
$s_timeForUserProgram => NULL,
$s_runnerVersion => NULL,
$s_testResultCode => 100, # this is the result from the test runner program
$s_protocol => '' # the conversation protocol
$s_protocol => '', # the conversation protocol
$s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
......@@ -419,9 +461,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
$s_programExitCode => $default_not_executed_user_program_exit_code,
$s_programExitCode => NULL,
$s_timeForCompiling => NULL,
$s_timeForUserProgram => NULL,
$s_runnerVersion => NULL,
$s_testResultCode => 100, # this is the result from the test runner program
$s_protocol => '' # the conversation protocol
$s_protocol => '', # the conversation protocol
$s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
......@@ -438,9 +487,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
$s_programExitCode => $default_not_executed_user_program_exit_code,
$s_programExitCode => NULL,
$s_timeForCompiling => NULL,
$s_timeForUserProgram => NULL,
$s_runnerVersion => NULL,
$s_testResultCode => 100, # this is the result from the test runner program
$s_protocol => '' # the conversation protocol
$s_protocol => '', # the conversation protocol
$s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
......@@ -457,9 +513,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => FALSE,
$s_hasCompiled => FALSE, // all other errors returned before this line
$s_programExitCode => $default_not_executed_user_program_exit_code,
$s_programExitCode => NULL,
$s_timeForCompiling => NULL,
$s_timeForUserProgram => NULL,
$s_runnerVersion => NULL,
$s_testResultCode => 100, # this is the result from the test runner program
$s_protocol => '' # the conversation protocol
$s_protocol => '', # the conversation protocol
$s_output_test_server_version => $versionString,
);
array_push($testResults, $result);
continue;
......@@ -575,9 +638,16 @@ foreach ($testCases as $test) {
$s_test_id => $testId,
$s_passed => $_result[$s_return_val] === 0,
$s_hasCompiled => $_result[$s_return_val] !== 50, // all other errors returned before this line
$s_programExitCode => $_result[$s_user_program_exit_code],
$s_timeForCompiling => $_result[$s_timeForCompiling],
$s_timeForUserProgram => $_result[$s_timeForUserProgram],
$s_runnerVersion => $_result[$s_runnerVersion],
$s_testResultCode => $_result[$s_return_val], # this is the result from the test runner program
$s_protocol => formatOutput($_result[$s_output]) # the conversation protocol
$s_protocol => formatOutput($_result[$s_output]), # the conversation protocol
$s_output_test_server_version => $versionString,
);
if ($needCompilation) {
......
constants.php
+ 14
− 2
View file @ 13f3c05c
......@@ -3,7 +3,7 @@
# variables
//use this to know which version we published
$versionString = '1.1.2';
$versionString = '2.0.0';
$rootDirNameToUseString = 'work';
$isDebug = false; //logs debug to STDOUT
......@@ -39,6 +39,8 @@ $s_arg_hardMemoryLimitInKb = 'hardMemoryLimitInKb';
$s_arg_hardDiskSpaceLimitInKb = 'hardDiskSpaceLimitInKb';
$s_arg_maxParallelTests = 'maxParallelTests';
$s_output_test_server_version = 'testServerVersion';
$timeZone = date_default_timezone_get(); #'Europe/Berlin'; #date_default_timezone_get(); //'Europe/Berlin'
......@@ -69,7 +71,6 @@ $status_code_InternalServerError = 100;
$s_output = 'output'; # the output from a test method
$s_return_val = 'return_val'; # the return value from a test method (the runner exit code)
$s_user_program_exit_code = 'exit_code'; # the return value from the user program (exit code) 0 is the default value (e.g. if we have no exit code)
$default_not_executed_user_program_exit_code = 0;
......@@ -126,6 +127,17 @@ $s_test_memoryLimitInKb = 'memoryLimitInKb'; # the memory limit for the test
$s_test_diskSpaceLimitInKb = 'maxDiskSpaceInKb'; # the max dis space the test can write to
$s_test_allAssets = 'allAssets'; # the asset files
$exitCodeOutputPrefix = 'exit:';
$s_user_program_exit_code = 'exit';
$s_runnerVersion = 'runnerVersion';
$s_timeForCompiling = 'timeForCompiling';
$s_timeForUserProgram = 'timeForUserProgram';
$s_output_without_header_part = 'stripedOutput';
$s_header_part_separator = "---\n";
/**
* outputs a message via echo (if debug is enabled outputs the message to stdout)
* @param $responseCode int the response code to send
......
do_blackBoxTest_func.php
+ 33
− 47
View file @ 13f3c05c
<?php
require_once __DIR__ . '/protocolHeaderPartParser.php';
/**
* runs the given black box test against the given files
......@@ -39,20 +40,15 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
global $config;
global $s_command_blackBoxTest;
global $s_command_justRunTest;
global $s_fullWorkingDirPath;
global $status_code_FileSystemIssue;
global $s_test_id;
global $s_test_content;
global $s_passed;
global $s_hasCompiled;
global $s_programExitCode;
global $s_testResultCode;
global $s_protocol;
global $responseNewLineString;
global $s_test_allAssets;
global $s_return_val;
global $s_user_program_exit_code;
global $s_output;
global $s_user_program_exit_code;
global $s_runnerVersion;
global $s_timeForCompiling;
global $s_timeForUserProgram;
global $s_output_without_header_part;
$command_to_execute_string = $s_command_blackBoxTest;
......@@ -76,28 +72,31 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
return array(
$s_return_val => 100,
$s_output => 'could not create asset(s)',
$s_user_program_exit_code => 0
$s_user_program_exit_code => NULL,
$s_runnerVersion => NULL,
$s_timeForCompiling => NULL,
$s_timeForUserProgram => NULL,
);
}
$longCmd = $config['runner']
. ' ' . $command_to_execute_string # arg[0] the test method
. ' "' . $fullWorkingDirPath . '"' # arg[1] dir path
. ' ' . $mainFileNameWithExtension # arg[2] file path
. ' "' . $compileCmd . '"' # arg[3] command to execute # currently ignored because we call compile in php...
. ' "' . $commandToExecute . '"' # arg[4] command to execute the test
. ' ' . $timeout # arg[5] the timeout
. ' ' . $memoryLimit # arg[6] the memory limit
. ' ' . $diskSpaceLimit # arg[7] the disk limit
. ' "' . implode(',', $sourceFileExtensions) . '"' # arg[8] the source file extensions
. ' ' . ($needCompilation ? 'true' : 'false') # arg[9] true: compile program new, false: not
. ' "' . $maxLinesToRead . '"' # arg[10] max lines to read from the user program (inclusive)
. ' "' . $maxErrLinesToRead . '"' # arg[11] max lines to read from the user program stderr (inclusive)
. ' "' . $maxLinesToWrite . '"' # arg[12] max lines to write to the user program (inclusive)
. ' "' . $uniqueSessionId . '"' # arg[13] a unique session id
. ' "' . ($showTestRunnerDebugOutput === TRUE ? 1 : 0) . '"' # arg[14] showTestRunnerDebugOutput
. ' ' . $command_to_execute_string # arg[0] the test method
. ' "' . $fullWorkingDirPath . '"' # arg[1] dir path
. ' ' . $mainFileNameWithExtension # arg[2] file path
. ' "' . $compileCmd . '"' # arg[3] command to execute # currently ignored because we call compile in php...
. ' "' . $commandToExecute . '"' # arg[4] command to execute the test
. ' ' . $timeout # arg[5] the timeout
. ' ' . $memoryLimit # arg[6] the memory limit
. ' ' . $diskSpaceLimit # arg[7] the disk limit
. ' "' . implode(',', $sourceFileExtensions) . '"' # arg[8] the source file extensions
. ' ' . ($needCompilation ? 'true' : 'false') # arg[9] true: compile program new, false: not
. ' "' . $maxLinesToRead . '"' # arg[10] max lines to read from the user program (inclusive)
. ' "' . $maxErrLinesToRead . '"' # arg[11] max lines to read from the user program stderr (inclusive)
. ' "' . $maxLinesToWrite . '"' # arg[12] max lines to write to the user program (inclusive)
. ' "' . $uniqueSessionId . '"' # arg[13] a unique session id
. ' "' . ($showTestRunnerDebugOutput === TRUE ? 1 : 0) . '"' # arg[14] showTestRunnerDebugOutput
;
......@@ -152,7 +151,7 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
debug("time to run the black-box-tests: " . $exec_time);
}
if ($isDebug && (isset($errorOutput) && trim($errorOutput)!=='')) {
if ($isDebug && (isset($errorOutput) && trim($errorOutput) !== '')) {
debug("error during execution of the (blackbox) test runner: " . $errorOutput);
}
......@@ -160,23 +159,7 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
# the test assets are removed when the dir gets removed
# also the test runner could renamed/added/deleted some files
//extract the program exit code from the protocol if any/possible else pretend 0
$userProgramExitCode = 0;
//the test runner uses \n
$maybeExitOutput = substr($output, 0, strpos($output, "\n"));
if(isset($maybeExitOutput)) {
$exitCodeOutputPrefix = 'exit:';
$checkString = substr($maybeExitOutput, 0, strlen($exitCodeOutputPrefix));
//user output as > as start
if ($checkString === $exitCodeOutputPrefix) {
$exitCodeString = substr($maybeExitOutput, strlen($exitCodeOutputPrefix));
$userProgramExitCode = intval($exitCodeString, 10);
}
}
$headerPart = parseHeaderPart($output);
//$output
......@@ -184,7 +167,10 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath,
return array(
$s_return_val => $return_var,
$s_output => $output,
$s_user_program_exit_code => $userProgramExitCode
$s_output => $headerPart[$s_output_without_header_part],
$s_user_program_exit_code => $headerPart[$s_user_program_exit_code],
$s_runnerVersion => $headerPart[$s_runnerVersion],
$s_timeForCompiling => $headerPart[$s_timeForCompiling],
$s_timeForUserProgram => $headerPart[$s_timeForUserProgram],
);
}
do_compareFilesTest_func.php
+ 12
− 30
View file @ 13f3c05c
<?php
require_once __DIR__ . '/protocolHeaderPartParser.php';
/**
* runs the given black box test against the given files
......@@ -36,23 +37,17 @@ function do_compareFilesTest($mainFileNameWithExtension, $test, $fullWorkingDirP
global $isDebug;
global $config;
global $s_command_compareFilesTest;
global $s_fullWorkingDirPath;
global $status_code_FileSystemIssue;
global $s_test_id;
global $s_test_content;
global $s_passed;
global $s_hasCompiled;
global $s_programExitCode;
global $s_testResultCode;
global $s_protocol;
global $responseNewLineString;
global $s_test_allAssets;
global $s_return_val;
global $s_user_program_exit_code;
global $s_output;
global $s_fileName;
global $s_fileHash;
global $s_user_program_exit_code;
global $s_runnerVersion;
global $s_timeForCompiling;
global $s_timeForUserProgram;
global $s_output_without_header_part;
$command_to_execute_string = $s_command_compareFilesTest;
......@@ -166,23 +161,7 @@ function do_compareFilesTest($mainFileNameWithExtension, $test, $fullWorkingDirP
# the test assets are removed when the dir gets removed
# also the test runner could renamed/added/deleted some files
//extract the program exit code from the protocol if any/possible else pretend 0
$userProgramExitCode = 0;
//the test runner uses \n
$maybeExitOutput = substr($output, 0, strpos($output, "\n"));
if (isset($maybeExitOutput)) {
$exitCodeOutputPrefix = 'exit:';
$checkString = substr($maybeExitOutput, 0, strlen($exitCodeOutputPrefix));
//user output as > as start
if ($checkString === $exitCodeOutputPrefix) {
$exitCodeString = substr($maybeExitOutput, strlen($exitCodeOutputPrefix));
$userProgramExitCode = intval($exitCodeString, 10);
}
}
$headerPart = parseHeaderPart($output);
//$output
......@@ -190,7 +169,10 @@ function do_compareFilesTest($mainFileNameWithExtension, $test, $fullWorkingDirP
return array(
$s_return_val => $return_var,
$s_output => $output,
$s_user_program_exit_code => $userProgramExitCode
$s_output => $headerPart[$s_output_without_header_part],
$s_user_program_exit_code => $headerPart[$s_user_program_exit_code],
$s_runnerVersion => $headerPart[$s_runnerVersion],
$s_timeForCompiling => $headerPart[$s_timeForCompiling],
$s_timeForUserProgram => $headerPart[$s_timeForUserProgram],
);
}
do_compileTest_func.php
+ 13
− 1
View file @ 13f3c05c
<?php
include_once './constants.php';
require_once __DIR__ . '/protocolHeaderPartParser.php';
/**
* compiles the given main file (if a compile error occurs then an answer is sent)
......@@ -26,6 +27,11 @@ function do_compile($mainFileNameWithExtension, $fullWorkingDirPath, $compileCmd
global $s_output;
global $s_return_val;
global $s_command_compile;
global $s_user_program_exit_code;
global $s_runnerVersion;
global $s_timeForCompiling;
global $s_timeForUserProgram;
global $s_output_without_header_part;
$command_to_execute_string = $s_command_compile;
......@@ -70,8 +76,14 @@ function do_compile($mainFileNameWithExtension, $fullWorkingDirPath, $compileCmd
debug("error during execution of the (compile) test runner: " . $errorOutput);
}
$headerPart = parseHeaderPart($output);
return array(
$s_return_val => $return_var,
$s_output => $output
$s_output => $headerPart[$s_output_without_header_part],
$s_user_program_exit_code => $headerPart[$s_user_program_exit_code],
$s_runnerVersion => $headerPart[$s_runnerVersion],
$s_timeForCompiling => $headerPart[$s_timeForCompiling],
$s_timeForUserProgram => $headerPart[$s_timeForUserProgram],
);
}
do_regexTest_func.php
+ 1
− 1
View file @ 13f3c05c
......@@ -11,7 +11,7 @@
* @param $showTestRunnerDebugOutput true show internal debug output from the test runner (included in the test response)
* @return array
*/
function do_regexTest_json($allFiles, $testContent, $showTestRunnerDebugOutput)
function do_regexTest_json_DEPRECATED($allFiles, $testContent, $showTestRunnerDebugOutput)
{
global $availableRegexModes;
......
protocolHeaderPartParser.php 0 → 100644
+ 82
− 0
View file @ 13f3c05c
<?php
function parseHeaderPart($output)
{
global $s_header_part_separator;
global $s_user_program_exit_code;
global $s_runnerVersion;
global $s_timeForCompiling;
global $s_timeForUserProgram;
global $s_output_without_header_part;
//if we have no header tuples we might have only so ---\n
//this is because we cannot use $headerPos -1
$headerPos = strpos($output, $s_header_part_separator);
$headerPartText = substr($output, 0, $headerPos);
$stripedOutput = substr($output, $headerPos + strlen($s_header_part_separator));
$exit = NULL;
$runnerVersion = NULL;
$timeForCompiling = NULL;
$timeForUserProgram = NULL;
$headerPartLines = explode("\n", $headerPartText);
//empty lines will be ignored because no case matches
foreach ($headerPartLines as $line) {
if (strpos($line, $s_user_program_exit_code) === 0) {
$val = getTuple($line);
if ($val === null) continue; //nothing would change...
$exit = intval($val, 10);
} else if (strpos($line, $s_runnerVersion) === 0) {
$val = getTuple($line);
if ($val === null) continue;
$runnerVersion = $val;
} else if (strpos($line, $s_timeForCompiling) === 0) {
$val = getTuple($line);
if ($val === null) continue;
$timeForCompiling = intval($val, 10);
} else if (strpos($line, $s_timeForUserProgram) === 0) {
$val = getTuple($line);
if ($val === null) continue;
$timeForUserProgram = intval($val, 10);
}
}
return array(
$s_user_program_exit_code => $exit,
$s_runnerVersion => $runnerVersion,
$s_timeForCompiling => $timeForCompiling,
$s_timeForUserProgram => $timeForUserProgram,
$s_output_without_header_part => $stripedOutput,
);
}
function getTuple($tupleString)
{
$tuple = explode(':', $tupleString);
if (count($tuple) !== 2) return NULL;
return $tuple[1] === "null"
? NULL
: $tuple[1];
}
\ No newline at end of file
readme.md
+ 89
− 47
View file @ 13f3c05c
......@@ -64,21 +64,23 @@ The request is a json object so the request should be from mime type application
"allFiles": [
{
"fileName": "Main.java",
"fileContent: "public class Main {\npublic static void main(String[] args) {\nSystem.out.println("echo");\n}\n}"
"fileContent": "public class Main {\npublic static void main(String[] args) {\nSystem.out.println(\"echo\");\n}\n}"
}
],
"tests": [
"command": "blackBoxTest",
"testContents": [
{
"id": 0,
"content": "> Hello World\n!",
"assets": []
}
],
"timeoutInMs": 1000,
"memoryLimitInKb": 1000,
"maxDiskSpaceInKb": 1000
{
"command": "blackBoxTest",
"testContents": [
{
"id": 0,
"content": "> Hello World\n!",
"assets": []
}
],
"timeoutInMs": 1000,
"memoryLimitInKb": 1000,
"maxDiskSpaceInKb": 1000
}
]
}
```
......@@ -90,11 +92,10 @@ A json array representing the responses. If an errors occurs before or after the
**In case some server error happened and no test could be executed then just one object is returned:**
```json
[
{
"responseCode": 0,
"message": "some error"
]
{
"responseCode": 0,
"message": "some error"
}
```
If the tests could be executed or to code to execute the tests is reached then the response will be an array for with an response object for every test
......@@ -114,9 +115,13 @@ If the tests could be executed or to code to execute the tests is reached then t
"id": 0,
"passed": true,
"hasCompiled": true,
"programExitCode": 0,
"programExitCode": null,
"timeForCompiling": null,
"timeForUserProgram": null,
"runnerVersion": "1.0.0",
"testResultCode": 0,
"protocol": "can be multiline \n\r string"
"protocol": "can be multiline \n\r string",
"testServerVersion": "1.0.0",
}
]
}
......@@ -137,8 +142,15 @@ other fields
* **hasCompiled**: (bool) true: test has compiled (if the test needs compilation, if not always true), false: not [actually this can be determined by checking testResultCode === (compile error return code from the test runner)]
* **testResultCode**: (int) the exit code of the test runner [see Test Runner Return Codes](##%20Return-Codes)
* **programExitCode**: (int) the exit code of the users program **[CURRENTLY NOT SET, ALWAYS 0!]**
* **protocol**: (string) a string representing the output (could contain the error message. New line chars are encoded with **\\\r\\\n** (because json encode automaticall encodes \r\n to \\\n\\\n)
* **programExitCode**: (int | null) the exit code of the users program, null if the user program was not (fully) executed e.g. timeout hit
* **timeForCompiling**: (int | null) the time for compiling the user program or null if not compiled
* **timeForUserProgram**: (int | null) the time for executing the user program or null if not (fully) executed
* **runnerVersion**: (string | null) the test-runner version or null if the test-runner was not executed
* **protocol**: (string) a string representing the output (could contain the error message. New line chars are encoded with **\\\r\\\n** (because json encode automaticall encodes \r\n to \\\n\\\n)
* **testServerVersion**: (string) the test-server version
## Response Codes from the test server
......@@ -152,15 +164,56 @@ other fields
**For explanation about the different test types see below**
### Example
```json
{
"passed": true,
"hasCompiled": true,
"programExitCode": 0,
"testResultCode": 0,
"protocol": "can be multiline \n\r string"
}
## Protocols
The Test-Runner uses `\n` new lines
All protocols consists out of 2 parts
the first part is the header part and contains some misc data about the test run e.g. test-runner version...
the seconds part is the real conversion protocol between the test-runner and the user program
the two parts are separated by a `---\n` line
**the header part including the `---\n`** must be removed from the protocol!
### Protocol Header part
the header part has the same schema for all tests
every information is on a new line and is a tuple where key and value is separated by `:`
there are the following information possible (the tuple keys) (the keys are not case sensitive!)
- `exit` (number) the user program exit code or `null` if it din't finish e.g. timeout hit
- `runnerVersion` (string) the test runner version
- `timeForCompiling` (int) the time for compiling in ms or `null` if compiling was not done or finished
- `timeForUserProgram` (int) the time for running the user program or `null` if the execution was not done or finished
after the header part the line `---\n` is followed, indicating the end of the header part
all values can be `null`!
the order of the tuples is not important
note that the tuple key and values must not contain the separator char `:` or the `\n` char or `---\n`
```
exit:0
runnerVersion:1.0.0
timeForCompiling:null
timeForUserProgram:null
---
```
```
exit:null
runnerVersion:1.0.0
timeForCompiling:1000
timeForUserProgram:null
---
```
## Protocol Output Compile Test
......@@ -172,15 +225,14 @@ when a compile error occurred then the output of the compiler
## Protocol Output for Just Run Test
Actually the program is run without any input...
Actually the program is run without any input (to stdin)...
1. Line: contains the arguments passed to the users program, starting with a $ sign (only if arguments were present)
all other lines (meaning different by the first character):
all lines (meaning different by the first character):
* '**$**' are the arguments passed to the users program, starting with a $ sign (only if arguments were present)
* '**>**' indicating a line that was read from the users program
* '**EOT**' indicates that the test ended here (written after the program terminated)
this is useful when some output is received after the test has finished
* '**exit:**' the user program exit code (can be empty if the user program didn't end e.g. timeout)
## Protocol Output Regex Test
1. Line: the used mode e.g. used mode: blacklist
......@@ -190,19 +242,10 @@ the next lines contains the test results for every file
## Protocol Output Black Box Test
The Test-Runner uses `\n` new lines
1./2. Line can be the user program exit code (so the test server can easier get it)
but only if the user program actually ran and finished
* '**exit:**' the user program exit code (can be empty if the user program didn't end e.g. timeout) if any this must be the first line in the protocol (to find it faster)
**the test runner must not output the a line starting with `exit:` else the test server might get the wrong exit code**
* the number is the expected exit code (defaults to 0 e.g. when there is no exit code because of timeout)
* '**e**' is the expected user program exit code
* this must be the first or second line
* the first if we have no program exit code
* the second if we have a user program exit code
* not that this is only true for protocol lines produced by the test runner (not e.g. test content input lines to the test runner)
1 Line can be the expected user program exit code
but only if the user program actually ran and finished
* '**e**' is the expected user program exit code followed by a number (e.g. `e0`)
all other lines
* '**$**' are the arguments passed to the users program, starting with a $ sign (only if arguments were present)
......@@ -222,7 +265,6 @@ when a compile error occurred then the output of the compiler
## Protocol Output Compare files Test
```
exit:[X]
EOT
ccompare [fileName1WithExtension] [fileName2WithExtension] (ignoreCase) (ignoreLeadingWhitespaces) (ignoreTrailingWhitespaces) (ignoreAllEmptyOrWhitespacesOnlyLines)
c[result]
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment