From 0d5ac39c32750cc3d8b2b5bb3bec9bff150089c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janis=20Daniel=20Da=CC=88hne?= <janis.daehne2@student.uni-halle.de> Date: Sun, 17 Nov 2019 21:16:05 +0100 Subject: [PATCH] - added trace string support to trace who ran which code and when --- .gitignore | 3 +- api.php | 50 +++++++++++++++++-- bootstrap.php | 1 + config.json | 27 ----------- config_example.json | 19 ++++++++ constants.php | 9 +++- do_blackBoxTest_func.php | 9 +++- do_compareFilesTest_func.php | 8 ++- do_compileTest_func.php | 7 ++- readme.md | 43 ++++++++++++++++- stats/private.php | 1 + transactionHelper.php | 94 +++++++++++++++++++++++++++++++++++- worker/traceLog.php | 57 ++++++++++++++++++++++ 13 files changed, 290 insertions(+), 38 deletions(-) delete mode 100644 config.json create mode 100644 config_example.json create mode 100644 worker/traceLog.php diff --git a/.gitignore b/.gitignore index 169f60b..f1c22d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ timestemp.json -.idea/ \ No newline at end of file +.idea/ +config.json \ No newline at end of file diff --git a/api.php b/api.php index d3c51a9..8dfe299 100644 --- a/api.php +++ b/api.php @@ -4,6 +4,8 @@ // Turn off all error reporting error_reporting(0); +//the request requires a $s_requestStaticSecret to be set equal to the configures $s_requestStaticSecret in the config + /* * 2016 - 2017 MIT Janis Dähne */ @@ -110,6 +112,33 @@ if ($hardDiskSpaceLimitInKb === NULL) { exit(1); } +if (!isset($config[$s_arg_insertTraceLogs])) { + output($status_code_InternalServerError, 'could not find insert trace log setting in config'); + exit(1); +} + +$insertTraceLogs = $config[$s_arg_insertTraceLogs]; + +if ($insertTraceLogs === NULL) { + output($status_code_InternalServerError, 'could not read insert trace log from config'); + exit(1); +} + +$dbTraceTableName = $config[$s_arg_dbTraceTableName]; + +if ($dbTraceTableName === NULL) { + output($status_code_InternalServerError, 'could not read the trace table from config'); + exit(1); +} + + +if (!isset($body[$s_requestTraceString])) { + output($status_code_InternalServerError, 'request trace string was not found in the request'); + exit(1); +} +$traceString = $body[$s_requestTraceString]; + + $requestDistinctionString = NULL; # try to get the request distinction key/number if (!isset($body[$s_requestDistinctionNumber])) { @@ -246,6 +275,21 @@ if (createFiles($arg_allFiles, $fullWorkingDirPath) === FALSE) { exit(1); } + +# create a user program source code variable (for logging) + +$allUserSourceCode = ''; + +foreach ($arg_allFiles as $file) { + + $_fileName = $file[$s_fileName]; + $_fileContent = $file[$s_fileContent]; + + $allUserSourceCode = $allUserSourceCode . "---" . $_fileName . "---\n" . $_fileContent . "\n\n"; +} + + + if ($isDebug) { debug("all files created!"); } @@ -364,7 +408,7 @@ foreach ($testCases as $test) { } try { - $_result = do_compile($arg_mainFileNameWithExtension, $fullWorkingDirPath, $min_compileTimeout, + $_result = do_compile($conn, $traceString, $allUserSourceCode, $arg_mainFileNameWithExtension, $fullWorkingDirPath, $min_compileTimeout, $compileCmd, $sourceFileExtensions, $requestDistinctionString, $showTestRunnerDebugOutput, $compilerOptions, $arg_characterLimitProtocol); } catch (Exception $e) { @@ -557,7 +601,7 @@ foreach ($testCases as $test) { require_once './do_compareFilesTest_func.php'; try { - $_result = do_compareFilesTest($arg_mainFileNameWithExtension, $test, $fullWorkingDirPath, + $_result = do_compareFilesTest($conn, $traceString, $allUserSourceCode, $arg_mainFileNameWithExtension, $test, $fullWorkingDirPath, $min_timeout, $min_compileTimeout, $min_memoryLimit, $min_diskSpaceLimit, $compileCmd, $execCmd, $sourceFileExtensions, $needCompilation, $requestDistinctionString, @@ -574,7 +618,7 @@ foreach ($testCases as $test) { require_once './do_blackBoxTest_func.php'; try { - $_result = do_blackBoxTest($arg_mainFileNameWithExtension, $test, $fullWorkingDirPath, + $_result = do_blackBoxTest($conn, $traceString, $allUserSourceCode, $arg_mainFileNameWithExtension, $test, $fullWorkingDirPath, $min_timeout, $min_compileTimeout, $min_memoryLimit, $min_diskSpaceLimit, $compileCmd, $execCmd, $sourceFileExtensions, $needCompilation, ($arg_command === $s_command_justRunTest), diff --git a/bootstrap.php b/bootstrap.php index 7b91ffd..2af96f9 100644 --- a/bootstrap.php +++ b/bootstrap.php @@ -64,6 +64,7 @@ $config[$s_arg_hardTimeoutInMs] = $row[$s_arg_hardTimeoutInMs]; $config[$s_arg_hardCompileTimeoutInMs] = $row[$s_arg_hardCompileTimeoutInMs]; $config[$s_arg_hardMemoryLimitInKb] = $row[$s_arg_hardMemoryLimitInKb]; $config[$s_arg_hardDiskSpaceLimitInKb] = $row[$s_arg_hardDiskSpaceLimitInKb]; +$config[$s_arg_insertTraceLogs] = $row[$s_arg_insertTraceLogs]; $config['maxParallelTests'] = $row['maxParallelTests']; $config['maxNumberOfTestsWithOneRequest'] = $row['maxNumberOfTestsWithOneRequest']; diff --git a/config.json b/config.json deleted file mode 100644 index 39a374a..0000000 --- a/config.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "workingDirFullPath": "/Users/janis/Documents/Test/", - "hardGlobalTimeoutInMs": 2000, - "hardMemoryLimitInKb": 2000, - "hardMaxDiskSpaceLimitInKb": 2000, - "maxParallelTests": 10, - "maxNumberOfTestsWithOneRequest": 15, - "runner": "\"/Users/janis/Misc/Java/JavaVirtualMachines/jdk1.8.0_144.jdk/Contents/Home/bin/java\" -Dfile.encoding=UTF8 -cp \"/Users/janis/Documents/Projects/SyndromAll/Syndrom_DefaultTestRunnerFullThreaded/out\" Main", - "showTestRunnerDebugOutput": 1, - "maxLinesToRead": 100, - "maxErrLinesToRead": 100, - "maxLinesToWrite": 100, - - "dbServer": "localhost:8889", - "dbName": "syndromTestDb", - "dbUser": "tester", - "dbPw": "mysqlpassword2", - "dbPLangTableName": "plangs", - "dbTransactionTableName": "transactions", - - "useConfigFromDb": true, - "dbConfigServer": "localhost:8889", - "dbConfigName": "syndromTestDb", - "dbConfigUser": "tester", - "dbConfigPw": "mysqlpassword2", - "dbConfigTableName": "config" -} \ No newline at end of file diff --git a/config_example.json b/config_example.json new file mode 100644 index 0000000..67ea510 --- /dev/null +++ b/config_example.json @@ -0,0 +1,19 @@ +{ + + "dbServer": "localhost:8889", + "dbName": "syndromTestDb", + "dbUser": "tester", + "dbPw": "mysqlpassword2", + "dbPLangTableName": "plangs", + "dbTransactionTableName": "transactions", + "dbTraceTableName": "traceLog", + + "insertTraceLogs": true, + + "useConfigFromDb": true, + "dbConfigServer": "localhost:8889", + "dbConfigName": "syndromTestDb", + "dbConfigUser": "tester", + "dbConfigPw": "mysqlpassword2", + "dbConfigTableName": "config" +} \ No newline at end of file diff --git a/constants.php b/constants.php index bb72c60..cd2aa7c 100644 --- a/constants.php +++ b/constants.php @@ -3,7 +3,7 @@ # variables //use this to know which version we published -$versionString = '2.4.0'; +$versionString = '2.5.0'; $rootDirNameToUseString = 'work'; $isDebug = false; //logs debug to STDOUT @@ -33,15 +33,19 @@ $s_timestampFile = 'timestemp.json'; $s_lastRunDate = 'lastRunDate'; $s_lastRunDir = 'lastRunDir'; +$s_arg_dbTraceTableName = 'dbTraceTableName'; + $s_arg_dbTransactionTableName = 'dbTransactionTableName'; $s_arg_hardTimeoutInMs = 'hardTimeoutInMs'; $s_arg_hardCompileTimeoutInMs = 'hardCompileTimeoutInMs'; $s_arg_hardMemoryLimitInKb = 'hardMemoryLimitInKb'; $s_arg_hardDiskSpaceLimitInKb = 'hardDiskSpaceLimitInKb'; $s_arg_maxParallelTests = 'maxParallelTests'; +$s_arg_insertTraceLogs = 'insertTraceLogs'; //true/false $s_output_test_server_version = 'testServerVersion'; + $timeZone = date_default_timezone_get(); #'Europe/Berlin'; #date_default_timezone_get(); //'Europe/Berlin' @@ -118,8 +122,11 @@ $s_arg_characterLimitProtocol = 'characterLimitProtocol'; # the character limit # a number to help the this server to separate two requests in case they arrive at the same time (and micro time would return the same value) $s_requestDistinctionNumber = 'requestDistinctionString'; +$s_requestTraceString = 'traceString'; # end variables +$s_requestDeleteTraceLogOlderThanMinutes = 'deleteTraceLogOlderThanMinutes'; + # TEST OBJECT field names $s_test_id = 'testId'; diff --git a/do_blackBoxTest_func.php b/do_blackBoxTest_func.php index abf6f2f..31149c9 100644 --- a/do_blackBoxTest_func.php +++ b/do_blackBoxTest_func.php @@ -4,6 +4,9 @@ require_once __DIR__ . '/protocolHeaderPartParser.php'; /** * runs the given black box test against the given files + * @param $conn PDO + * @param $traceString string + * @param $sourceCode string the user program source code as one string * @param string $mainFileNameWithExtension the main file to compile (with extension) * @param $test * @param $fullWorkingDirPath @@ -24,7 +27,7 @@ require_once __DIR__ . '/protocolHeaderPartParser.php'; * @internal param array $allFiles all files to copy in teh dir to use * Format: array[int] => {fileName: string, fileContent: string} */ -function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath, +function do_blackBoxTest($conn, $traceString, $sourceCode, $mainFileNameWithExtension, $test, $fullWorkingDirPath, $timeout, $compileTimeout, $memoryLimit, $diskSpaceLimit, $compileCmd, $execCmd, $sourceFileExtensions, $needCompilation, $justRun, @@ -104,6 +107,10 @@ function do_blackBoxTest($mainFileNameWithExtension, $test, $fullWorkingDirPath, ; + require_once './transactionHelper.php'; + addTraceLogToDb($conn, $config, $traceString, $longCmd, $sourceCode); + + if ($isDebug) { debug("using runner: " . $config['runner']); debug("full command: " . $longCmd); diff --git a/do_compareFilesTest_func.php b/do_compareFilesTest_func.php index f43c3f4..ff02eae 100644 --- a/do_compareFilesTest_func.php +++ b/do_compareFilesTest_func.php @@ -4,6 +4,9 @@ require_once __DIR__ . '/protocolHeaderPartParser.php'; /** * runs the given black box test against the given files + * @param $conn PDO + * @param $traceString string + * @param $sourceCode string the user program source code as one string * @param string $mainFileNameWithExtension the main file to compile (with extension) * @param $test * @param $fullWorkingDirPath @@ -23,7 +26,7 @@ require_once __DIR__ . '/protocolHeaderPartParser.php'; * @internal param array $allFiles all files to copy in teh dir to use * Format: array[int] => {fileName: string, fileContent: string} */ -function do_compareFilesTest($mainFileNameWithExtension, $test, $fullWorkingDirPath, +function do_compareFilesTest($conn, $traceString, $sourceCode, $mainFileNameWithExtension, $test, $fullWorkingDirPath, $timeout, $compileTimeout, $memoryLimit, $diskSpaceLimit, $compileCmd, $execCmd, $sourceFileExtensions, $needCompilation, $uniqueSessionId, @@ -106,6 +109,9 @@ function do_compareFilesTest($mainFileNameWithExtension, $test, $fullWorkingDirP ; + require_once './transactionHelper.php'; + addTraceLogToDb($conn, $config, $traceString, $longCmd, $sourceCode); + if ($isDebug) { debug("using runner: " . $config['runner']); debug("full command: " . $longCmd); diff --git a/do_compileTest_func.php b/do_compileTest_func.php index 3076e15..da51206 100644 --- a/do_compileTest_func.php +++ b/do_compileTest_func.php @@ -5,6 +5,9 @@ require_once __DIR__ . '/protocolHeaderPartParser.php'; /** * compiles the given main file (if a compile error occurs then an answer is sent) + * @param $conn PDO + * @param $traceString string + * @param $sourceCode string the user program source code as one string * @param string $mainFileNameWithExtension the main file to compile (with extension) * @param $fullWorkingDirPath string the working dir with the source files * @param int|string $compileTimeout the compile timeout in MS (can be string from db or int from default values) @@ -19,7 +22,7 @@ require_once __DIR__ . '/protocolHeaderPartParser.php'; * @internal param array $allFiles all files to copy in teh dir to use * Format: array[int] => {fileName: string, fileContent: string} */ -function do_compile($mainFileNameWithExtension, $fullWorkingDirPath, +function do_compile($conn, $traceString, $sourceCode, $mainFileNameWithExtension, $fullWorkingDirPath, $compileTimeout, $compileCmd, $sourceFileExtensions, $uniqueSessionId, @@ -61,6 +64,8 @@ function do_compile($mainFileNameWithExtension, $fullWorkingDirPath, . ' ' . escapeshellarg($characterLimitProtocol) # arg[9] character limit for the test protocol, output cut after the limit ; + require_once './transactionHelper.php'; + addTraceLogToDb($conn, $config, $traceString, $longCmd, $sourceCode); $env = $config['environmentVarsParsed']; // exec($longCmd, $output, $return_var); diff --git a/readme.md b/readme.md index d75044b..d93158a 100644 --- a/readme.md +++ b/readme.md @@ -16,6 +16,11 @@ we change the dirs every changeDirIntervalInS s so in case some run is not fully **Test Runner** This is the program that gets executed from the Test Server in order to test the users program. (Also called Runner) + +## Before you start + +you need to copy the file `config_example.json` to `config.json` and adjust the settings in it (if needed) + ## default workflow * The Server receives a request @@ -56,6 +61,7 @@ The request is a json object so the request should be from mime type application * **memoryLimitInKb**: (int) the max memory in kb the users program can use * **maxDiskSpaceInKb**: (int) the max disk space in kb the users program can write to * **allFiles**: (array) the files to use +* **traceString**: (string) a string to trace back the source code in the backend (in case a process won't finish we can find the source code with this) ### example ```json @@ -402,6 +408,26 @@ id | rootDirName | testType where '1480443681' ist the seconds-part if microtime() timestamp '0-04701800' is the microtime part of microtime() and '1' ist the request distinction number + +# Trace Log table + +the trace table is used as a log to trace back running/ran processes and source code in the backend + +Columns + +- id - an auto incremented id +- dateTime - the date and time when the entry was added (to later easily delete old entries) +- traceString - the trace string to trace back the source code in the backend +- processString - the process string that was executed (to identify the process later) +- sourceCode - the executed source code of all files + - as this can get longer... for mysql mediumtext is recommended + +#### Example +id | dateTime | traceString | processString | sourceCode +--- | --- | --- | --- | --- +2 | 2019-11-17 19:54:58 | justRunProgram_executingUser:1_solution::userId:1_releaseId:261_plangId:1_mainFileId:84762 | cmd to execute | ... + + ## Config Table Columns (the same as in the local config except the db specific settings) @@ -445,6 +471,7 @@ A json object with the following properties (order does not matter) * **dbUser** : (string) the name of the database user to use for quering the database * **dbPw** : (string) the password of the database user * **dbTransactionTableName** : (string) the table used for the transaction (to ensure the maxParallelTests) +* **dbTraceTableName** : (string) the trace log table name * **useConfigFromDb**: (bool) true: use the config in the database table, false: use the local config file, the `dbConfig*` (and `dbConfigTableName`) settings are used to connect the the mysql db and read the config settings * **dbConfigServer**: (string) -''- but for the config table @@ -455,6 +482,7 @@ A json object with the following properties (order does not matter) the following settings can be set through the db ()`useConfigFromDb` and connected settings) + * **workingDirFullPath** : (string) the path to the directory where the user programs will be tested/saved/executed * **hardTimeoutInMs** : (int) the global timeout for all programming languages * **hardCompileTimeoutInMs** : (int) the global compile timeout for all programming languages @@ -475,6 +503,8 @@ the following settings can be set through the db ()`useConfigFromDb` and connect * **maxErrLinesToRead**: (int) the total (inclusive) number of lines to read from the user program error stream (stderr) (-x or 0 for unlimited) * **maxLinesToWrite**: (int) the total (inclusive) number of lines to write to the user program (-x or 0 for unlimited) +* **insertTraceLogs:** (bool) true: trace log entries will be inserted into the trace table, false: not inserted + * **environmentVars** : (string, \r\n or \n separated, key value pairs are separated by = OR NULL) the environment variables for the test runner. All environment vars are cleared and only these are set (even if this is the empty string). The special delimiters `\n` and `=` must not be used inside keys or values. The test runner is called and these environment variables are set. You can use `"path""` to escape paths with whitespaces @@ -838,4 +868,15 @@ The Runner needs to return result code (exit code) to indicate the result of the ## Mac phpmyadmin -on mac make sure apache is running via `sudo apachectl start` \ No newline at end of file +on mac make sure apache is running via `sudo apachectl start` + + +## Routes + + +POST +- `/api.php` is the main entry point to run tests +- `stats/public.php` returns public available information about the test server +- `stats/private.php` returns private available information about the test server + +- `worker/traceLog.php` can be used to delete old trace log entries \ No newline at end of file diff --git a/stats/private.php b/stats/private.php index e2ae124..205db36 100644 --- a/stats/private.php +++ b/stats/private.php @@ -18,6 +18,7 @@ $response = array( $s_arg_hardCompileTimeoutInMs => $config[$s_arg_hardCompileTimeoutInMs], $s_arg_hardDiskSpaceLimitInKb => $config[$s_arg_hardDiskSpaceLimitInKb], $s_arg_hardMemoryLimitInKb => $config[$s_arg_hardMemoryLimitInKb], + $s_arg_insertTraceLogs => $config[$s_arg_insertTraceLogs], "maxParallelTests" => $config["maxParallelTests"], "maxNumberOfTestsWithOneRequest" => $config["maxNumberOfTestsWithOneRequest"], diff --git a/transactionHelper.php b/transactionHelper.php index 0fc6f2c..b0cd90b 100644 --- a/transactionHelper.php +++ b/transactionHelper.php @@ -1,7 +1,7 @@ <?php -include_once __DIR__.'/constants.php'; -include_once __DIR__.'/sqlHelper.php'; +include_once __DIR__ . '/constants.php'; +include_once __DIR__ . '/sqlHelper.php'; /** * registers/starts a test run (a transaction) @@ -110,3 +110,93 @@ function endTransaction($conn, $config, $transactionKey, $rootDirName) exit(1); } } + +/** + * @param $conn PDO the db connection + * @param $config + * @param $traceString string the trace string from the request + * @param $processString string this should be the process name or arguments to find it in process managers + * @param $sourceCode string all user program source code for from request + */ +function addTraceLogToDb($conn, $config, $traceString, $processString, $sourceCode) +{ + + global $s_arg_dbTraceTableName; + global $status_code_DbIssue; + global $s_arg_insertTraceLogs; + + + //0 = never delete AND never insert + //-x = never delete + $insertTraceLogs = boolval($config[$s_arg_insertTraceLogs]); + + if ($insertTraceLogs === TRUE) { + + debug(strlen($sourceCode)); + + $tableName = $config[$s_arg_dbTraceTableName]; + + $sql = "INSERT INTO `" . $tableName . "` (traceString, processString, sourceCode) VALUES (:traceString, :processString, :sourceCode);"; + $params = array( + ':traceString' => $traceString, + ':processString' => $processString, + ':sourceCode' => $sourceCode + ); + + if (executeSql($conn, $sql, $params) === NULL) { + //some error + output($status_code_DbIssue, 'could not add trace log'); + exit(1); + } + } + +} + +/** + * @param $conn PDO + * @param $config + * @param $s_deleteTraceLogOlderThanMinutes string caller must check if this is a number or not + */ +function removeOldTraceLogs($conn, $config, $s_deleteTraceLogOlderThanMinutes) { + + global $s_arg_dbTraceTableName; + global $status_code_DbIssue; + + $tableName = $config[$s_arg_dbTraceTableName]; + $deleteTraceLogOlderThanMinutes = intval($s_deleteTraceLogOlderThanMinutes); //gives 0 on error + + //$deleteTraceLogOlderThanMinutes = 0 or -x -> never delete + + //delete old entries... this has a performance impact... maybe move this to a cron job or something?? + if (is_numeric($s_deleteTraceLogOlderThanMinutes)) { + + if ($deleteTraceLogOlderThanMinutes > 0) { + + $sql = "DELETE FROM `" . $tableName . "` WHERE dateTime < (NOW() - INTERVAL :olderThan MINUTE);"; + $params = array( + ':olderThan' => $deleteTraceLogOlderThanMinutes + ); + + if (executeSql($conn, $sql, $params) === NULL) { + //some error + output($status_code_DbIssue, 'could not remove old trace logs'); + exit(1); + } + } + else if ($deleteTraceLogOlderThanMinutes === 0){ + + $sql = "DELETE FROM `" . $tableName . "`;"; + $params = array( + ':olderThan' => $deleteTraceLogOlderThanMinutes + ); + + if (executeSql($conn, $sql, $params) === NULL) { + //some error + output($status_code_DbIssue, 'could not remove all old trace logs'); + exit(1); + } + } + + } + +} diff --git a/worker/traceLog.php b/worker/traceLog.php new file mode 100644 index 0000000..8affac5 --- /dev/null +++ b/worker/traceLog.php @@ -0,0 +1,57 @@ +<?php + +// requires a json body with a property called $s_requestDeleteTraceLogOlderThanMinutes which must be an number +// if this is 0 - delete all trace logs +// if this is -x delete no trace logs +// if this is X>0 delete old trace logs that are older than X minutes + +//the request also requires a $s_requestStaticSecret to be set equal to the configures $s_requestStaticSecret in the config + + +//NOTE: only post is allowed!! + + + +$config = require __DIR__.'/../bootstrap.php'; + +require_once __DIR__.'/../constants.php'; + + +require_once __DIR__.'/../transactionHelper.php'; + + +$entityBody = file_get_contents('php://input'); + +$body = json_decode($entityBody, TRUE); + +if ($body === NULL) { + output($status_code_ArgumentInvalidOrMissing, "could not decode request body"); + exit(1); +} + + +if (!isset($body[$s_requestDeleteTraceLogOlderThanMinutes])) { + output($status_code_ArgumentInvalidOrMissing, "argument '" . $s_requestDeleteTraceLogOlderThanMinutes . "' missing"); + exit(1); +} + +$deleteTraceLogOlderThanMinutes = $body[$s_requestDeleteTraceLogOlderThanMinutes]; + +if (is_numeric($deleteTraceLogOlderThanMinutes) === FALSE) { + output($status_code_InternalServerError, $s_requestDeleteTraceLogOlderThanMinutes . "was not numeric"); + exit(1); +} + + +$conn = connectToDb($config['dbServer'], $config['dbName'], $config['dbUser'], $config['dbPw']); + +if ($conn === NULL) { + output($status_code_DbIssue, 'could not connect to db'); + exit(1); +} + + +removeOldTraceLogs($conn, $config, $deleteTraceLogOlderThanMinutes); + + +output($status_code_Ok, "", null); -- GitLab