From 973c904c9ba00076685b8d3311e96e2e3c5ab457 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Janis=20Daniel=20Da=CC=88hne?=
<janis.daehne2@student.uni-halle.de>
Date: Wed, 12 Jun 2019 16:17:37 +0200
Subject: [PATCH] - added worker to delete unused files - renamed schedulers to
workers
---
src/ClientServer/Config/AppConfiguration.cs | 23 ++++++++++++++++---
src/ClientServer/Config/Constants.cs | 6 ++++-
.../Core/Exercises/ReleaseController.cs | 3 ++-
.../Core/Exercises/SubmissionController.cs | 4 ++--
.../Models/Users/SystemSetting.cs | 4 +++-
src/ClientServer/Program.cs | 14 +++++++----
.../ReleaseWorker.cs} | 12 ++++++----
.../SubmissionAssessmentWorker.cs} | 3 ++-
8 files changed, 51 insertions(+), 18 deletions(-)
rename src/ClientServer/{Schedulers/ReleaseScheduler.cs => Workers/ReleaseWorker.cs} (94%)
rename src/ClientServer/{Schedulers/SubmissionAssessmentScheduler.cs => Workers/SubmissionAssessmentWorker.cs} (99%)
diff --git a/src/ClientServer/Config/AppConfiguration.cs b/src/ClientServer/Config/AppConfiguration.cs
index 0638679..7917e82 100644
--- a/src/ClientServer/Config/AppConfiguration.cs
+++ b/src/ClientServer/Config/AppConfiguration.cs
@@ -2,6 +2,8 @@ using System;
using System.Collections.Generic;
using System.Linq;
using ClientServer.Controllers.Core.Login;
+using ClientServer.Schedulers;
+using ClientServer.Workers;
using Microsoft.Extensions.Configuration;
namespace ClientServer.Helpers
@@ -38,7 +40,7 @@ namespace ClientServer.Helpers
/// <summary>
/// true: this is a slave is used to boost performance ... more concurrent requests
- /// false: a master, a master does the opening and closing of releases ... <see cref="ClientServer.Schedulers.ReleaseScheduler"/>
+ /// false: a master, a master does the opening and closing of releases ... <see cref="ReleaseWorker"/>
///
/// if we use multiple nodes as master there could be issues with the auto assessment?!?! not yet tested
/// because we cannot ensure that one release is handled by only one node...
@@ -127,7 +129,7 @@ namespace ClientServer.Helpers
///
/// you probably only want this to init the database with the default/basic data
/// </summary>
- public static bool IsInitControllerEnabled = false;
+ public static bool IsInitControllerEnabled = true;
/// <summary>
/// true: all cookies are send with secure flag set
@@ -140,10 +142,13 @@ namespace ClientServer.Helpers
public static string DebugOriginUrl = "http://localhost:8080";
/// <summary>
- /// used for cors to enable corss site requests, see https://spring.io/understanding/CORS
+ /// used for cors to enable cross site requests, see https://spring.io/understanding/CORS
/// </summary>
public static List<string> DeployOriginUrls = new List<string>();
+
+ public static int DeleteFilesWorkerIntervalInMinutes = 60;
+
//5 mb?
public static long MaxUploadFileSizeInByte = 1024 * 1024 * 5;
@@ -366,6 +371,18 @@ namespace ClientServer.Helpers
overwriteCount++;
}
}
+
+ if (string.IsNullOrWhiteSpace(ConfigurationRoot[Constants.DeleteFilesWorkerIntervalInMinutesKey]) == false)
+ {
+ var temp = ConfigurationRoot[Constants.DeleteFilesWorkerIntervalInMinutesKey];
+ int tempValue;
+ if (int.TryParse(temp, out tempValue))
+ {
+ //keep the setting
+ AppConfiguration.DeleteFilesWorkerIntervalInMinutes = tempValue;
+ overwriteCount++;
+ }
+ }
Console.WriteLine($"[INFO] {overwriteCount} app settings overwritten by appsettings.json");
}
diff --git a/src/ClientServer/Config/Constants.cs b/src/ClientServer/Config/Constants.cs
index 3acea41..160a69b 100644
--- a/src/ClientServer/Config/Constants.cs
+++ b/src/ClientServer/Config/Constants.cs
@@ -187,13 +187,17 @@ namespace ClientServer.Helpers
/// used for cors to enable corss site requests, see https://spring.io/understanding/CORS
/// </summary>
public static string DeployOriginUrlsKey = "DeployOriginUrls";
-
+
+
+ public static string DeleteFilesWorkerIntervalInMinutesKey = "DeleteFilesWorkerIntervalInMinutes";
public static string MaxUploadFileSizeInByteKey = "MaxUploadFileSizeInByte";
+
+
//--- some core constants ---
/// <summary>
diff --git a/src/ClientServer/Controllers/Core/Exercises/ReleaseController.cs b/src/ClientServer/Controllers/Core/Exercises/ReleaseController.cs
index beeeb60..a6fdd5b 100644
--- a/src/ClientServer/Controllers/Core/Exercises/ReleaseController.cs
+++ b/src/ClientServer/Controllers/Core/Exercises/ReleaseController.cs
@@ -12,6 +12,7 @@ using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.EntityFrameworkCore;
using System.Threading.Tasks;
+using ClientServer.Workers;
namespace ClientServer.Controllers.Core.Exercises
{
@@ -643,7 +644,7 @@ namespace ClientServer.Controllers.Core.Exercises
/// <summary>
/// returns true: all ok, false: some invalid values
/// also see
- /// <see cref="ReleaseScheduler.NextRun"/>
+ /// <see cref="ReleaseWorker.NextRun"/>
/// </summary>
/// <param name="release"></param>
/// <returns></returns>
diff --git a/src/ClientServer/Controllers/Core/Exercises/SubmissionController.cs b/src/ClientServer/Controllers/Core/Exercises/SubmissionController.cs
index 2f7f378..1bbfd87 100644
--- a/src/ClientServer/Controllers/Core/Exercises/SubmissionController.cs
+++ b/src/ClientServer/Controllers/Core/Exercises/SubmissionController.cs
@@ -2034,7 +2034,7 @@ namespace ClientServer.Controllers.Core.Exercises
var testingController = new TestingController(_context);
var success =
- await SubmissionAssessmentScheduler.RunForRelease(release, _context, testingController, false, true, systemSettings);
+ await SubmissionAssessmentWorker.RunForRelease(release, _context, testingController, false, true, systemSettings);
if (!success)
{
@@ -2146,7 +2146,7 @@ namespace ClientServer.Controllers.Core.Exercises
ExerciseReleaseId = release.Id,
};
- var success = await SubmissionAssessmentScheduler.RunSingleAutomaticAssessment(release, solutionPk,
+ var success = await SubmissionAssessmentWorker.RunSingleAutomaticAssessment(release, solutionPk,
_context,
testingController, systemSettings);
diff --git a/src/ClientServer/Models/Users/SystemSetting.cs b/src/ClientServer/Models/Users/SystemSetting.cs
index 5d54654..058721b 100644
--- a/src/ClientServer/Models/Users/SystemSetting.cs
+++ b/src/ClientServer/Models/Users/SystemSetting.cs
@@ -16,6 +16,8 @@ namespace ClientServer.Models.Users
/// some settings e.g. <see cref="DefaultGroupRoleId"/> cannot be set because there need to be data inserted to
/// the db first before this can be set... so this is set during the initial creation of the data
/// (<see cref="Init.Get"/>)
+ ///
+ /// settings in this class can be configured to runtime through the ui
/// </summary>
public class SystemSetting
{
@@ -117,7 +119,7 @@ namespace ClientServer.Models.Users
/// <summary>
/// the number of tests that can be run with one request on the submit test server
/// everything that uses the submit test server should make packages/buckets with request so that all tests will be run but with more than 1 request
- /// e.g. <see cref="SubmissionAssessmentScheduler.RunAutomaticAssessment"/> uses this to run all tests with multiple requests (if needed)
+ /// e.g. <see cref="SubmissionAssessmentWorker.RunAutomaticAssessment"/> uses this to run all tests with multiple requests (if needed)
/// this number must be less or equal to the number of allowed MaxNumberOfTestsWithOneRequest on the (submit) test server!
///
/// <see cref="ResponseCode.PartialOk"/> will be returned if there were more requested
diff --git a/src/ClientServer/Program.cs b/src/ClientServer/Program.cs
index 00723bb..497b12e 100644
--- a/src/ClientServer/Program.cs
+++ b/src/ClientServer/Program.cs
@@ -1,9 +1,11 @@
using System;
using System.Collections.Generic;
using System.IO;
+using System.Threading;
using ClientServer.Db;
using ClientServer.Helpers;
using ClientServer.Schedulers;
+using ClientServer.Workers;
using Microsoft.AspNetCore.Hosting;
using Microsoft.EntityFrameworkCore;
@@ -35,12 +37,16 @@ namespace ClientServer
if (AppConfiguration.IsSlave == false)
{
- ReleaseScheduler.NextRun(); //this is sync (because fast)
- ReleaseScheduler.Start();
+ ReleaseWorker.NextRun(); //this is sync (because fast)
+ ReleaseWorker.Start();
//this is async
- SubmissionAssessmentScheduler.StartNextNewThread(-1, true); //rerun the ones we don't finished e.g. because we shut down the server
- SubmissionAssessmentScheduler.Start();
+ SubmissionAssessmentWorker.StartNextNewThread(-1, true); //rerun the ones we don't finished e.g. because we shut down the server
+ SubmissionAssessmentWorker.Start();
+
+ //delete all files that are (still) in the file system but not in the db
+ FileWorker.NextRun(); //this is sync (because fast)
+ FileWorker.Start();
}
host.Run();
diff --git a/src/ClientServer/Schedulers/ReleaseScheduler.cs b/src/ClientServer/Workers/ReleaseWorker.cs
similarity index 94%
rename from src/ClientServer/Schedulers/ReleaseScheduler.cs
rename to src/ClientServer/Workers/ReleaseWorker.cs
index 0a9ff2d..c73d4e7 100644
--- a/src/ClientServer/Schedulers/ReleaseScheduler.cs
+++ b/src/ClientServer/Workers/ReleaseWorker.cs
@@ -1,13 +1,13 @@
using System;
-using ClientServer.Db;
using System.Linq;
using System.Reactive.Linq;
-using System.Threading.Tasks;
+using System.Threading;
+using ClientServer.Db;
using ClientServer.Helpers;
using ClientServer.Models.Exercises.Release;
using Microsoft.EntityFrameworkCore;
-namespace ClientServer.Schedulers
+namespace ClientServer.Workers
{
/// <summary>
/// a worker to check for exercise release changes
@@ -15,7 +15,7 @@ namespace ClientServer.Schedulers
///
/// it's also better in a worker because when we do this in a controller there might be 2 equal requets which would start release 2 times...
/// </summary>
- public static class ReleaseScheduler
+ public static class ReleaseWorker
{
/// <summary>
@@ -31,7 +31,7 @@ namespace ClientServer.Schedulers
Observable
.Interval(TimeSpan.FromMinutes(1))
- //.Interval(TimeSpan.FromSeconds(20)) //for debug
+// .Interval(TimeSpan.FromSeconds(22)) //for debug
.Subscribe((min) => { NextRun(); });
}
@@ -40,6 +40,8 @@ namespace ClientServer.Schedulers
{
//now = DateTime.SpecifyKind(now, DateTimeKind.Unspecified);
+
+// Console.WriteLine($"release: {Thread.CurrentThread.ManagedThreadId}");
//times in the db are stored as utc dateTimes
var now = DateTime.Now.ToUniversalTime();
diff --git a/src/ClientServer/Schedulers/SubmissionAssessmentScheduler.cs b/src/ClientServer/Workers/SubmissionAssessmentWorker.cs
similarity index 99%
rename from src/ClientServer/Schedulers/SubmissionAssessmentScheduler.cs
rename to src/ClientServer/Workers/SubmissionAssessmentWorker.cs
index 7504f2f..bb78ed9 100644
--- a/src/ClientServer/Schedulers/SubmissionAssessmentScheduler.cs
+++ b/src/ClientServer/Workers/SubmissionAssessmentWorker.cs
@@ -21,10 +21,11 @@ namespace ClientServer.Schedulers
/// <summary>
/// no longer a worker because assessment is only possible when a release gets unreleased... (manually or by the release scheduler)
/// </summary>
- public static class SubmissionAssessmentScheduler
+ public static class SubmissionAssessmentWorker
{
/// <summary>
/// the number of the run
+ /// this will cycle from max int to min int which is ok as long as every request has a unique id (positive or negative)
/// </summary>
private static int runNumber = 0;
--
GitLab