Skip to content
Snippets Groups Projects
Commit 8461ec93 authored by DesireeWyrzylala's avatar DesireeWyrzylala
Browse files

evaluate kshape

parent 46e92375
No related branches found
No related tags found
No related merge requests found
Showing with 31597 additions and 1412 deletions
bilder/evaluation/auc-pr/anomaly/kShapeAD_groups_auc-pr_anomaly.png

54.6 KiB

bilder/evaluation/auc-pr/groups/kShapeAD_groups_auc-pr.png

69.9 KiB

bilder/evaluation/vus-pr/anomalien/kShapeAD_groups_vus-pr_anomaly.png

54.4 KiB

bilder/evaluation/vus-pr/groups/kShapeAD_groups_vus-pr.png

70.5 KiB

This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
%% Cell type:code id: tags:
``` python
import numpy as np
import sys
import pathlib
sys.path.append(str(pathlib.Path.absolute)+ '../../../../../')
from src.utils.slidingWindows import find_length_rank
from src.run_model_wrapper import preprocess_data
from src.models.sofia_modelle.KNN import KNN
from src.evaluation import get_metrics
from SAND import SAND
```
%% Output
CUDA available: True
cuDNN version: 90100
%% Cell type:code id: tags:
``` python
#Hyperparamater
params = {'n_neighbors': 50, 'method': 'largest', 'periodicity':1}
params = {'periodicity': 1}
```
%% Cell type:code id: tags:
``` python
#Fit Funktion
def run_Sub_KNN(data, n_neighbors=10, method='largest', periodicity=1, n_jobs=1):
def run_KShapeAD(data, periodicity=1):
slidingWindow = find_length_rank(data, rank=periodicity)
clf = KNN(slidingWindow=slidingWindow, n_neighbors=n_neighbors,method=method, n_jobs=n_jobs)
clf.fit(data)
clf = SAND(pattern_length=slidingWindow, subsequence_length=4*(slidingWindow))
clf.fit(X=data.squeeze(),online=False, overlaping_rate=int(1.5*slidingWindow))
score = clf.decision_scores_
return score.ravel()
```
%% Cell type:code id: tags:
``` python
#define paths to data and for storing files
data_folders = '../../../data/train/temp/'
```
%% Cell type:code id: tags:
``` python
#Ouptput: [{file_name:str,train_data:list,data:list, label:list, sliding_window:int}]
dataList = preprocess_data(data_folders)
```
%% Cell type:code id: tags:
``` python
results = dataList.copy()
#train a model for each file and store metrics, file_name and hyperparameters
for idx in range(len(dataList)):
time_series = dataList[idx]
file_name = time_series['file_name']
print(file_name)
data = time_series['data']
label = np.array(time_series['label'])
train_data = time_series['train_data']
#scores = run_LOF(train_data,data, **params)
scores = run_Sub_KNN(data, **params)
scores = run_KShapeAD(data, **params)
slidingWindow = time_series['sliding_window']
metrics = get_metrics(scores, labels=label, slidingWindow=slidingWindow)
results[idx]['metrics']= metrics.copy()
```
%% Output
257_TAO_id_1_Environment_tr_500_1st_3.csv
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\metrics\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.
_warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
258_TAO_id_2_Environment_tr_500_1st_4.csv
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\metrics\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.
_warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
259_TAO_id_3_Environment_tr_500_1st_7.csv
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\utils\deprecation.py:151: FutureWarning: 'force_all_finite' was renamed to 'ensure_all_finite' in 1.6 and will be removed in 1.8.
warnings.warn(
c:\Users\desiw\AppData\Local\Programs\Python\Python311\Lib\site-packages\sklearn\metrics\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.
_warn_prf(average, modifier, f"{metric.capitalize()} is", len(result))
%% Cell type:code id: tags:
``` python
mean_auc_pr = 0
mean_vus_pr = 0
for element in results:
file = element['file_name']
metrics = element['metrics']
auc_pr = metrics['AUC-PR']
vus_pr = metrics['VUS-PR']
print(f'Datei: {file}, AUC-PR: {auc_pr}, VUS-PR: {vus_pr}')
mean_auc_pr += auc_pr
mean_vus_pr += vus_pr
print('Durchschnittlicher AUC-PR: ', (mean_auc_pr/len(results)))
print('Durchschnittlicher VUS-PR: ', (mean_vus_pr/len(results)))
```
%% Output
Datei: 257_TAO_id_1_Environment_tr_500_1st_3.csv, AUC-PR: 0.12329675188802566, VUS-PR: 0.9079681057410124
Datei: 258_TAO_id_2_Environment_tr_500_1st_4.csv, AUC-PR: 0.15584484586202274, VUS-PR: 0.9463314499613164
Datei: 259_TAO_id_3_Environment_tr_500_1st_7.csv, AUC-PR: 0.03747524809411396, VUS-PR: 0.19297952994706424
Durchschnittlicher AUC-PR: 0.1055389486147208
Durchschnittlicher VUS-PR: 0.682426361883131
Datei: 257_TAO_id_1_Environment_tr_500_1st_3.csv, AUC-PR: 0.0983782436258824, VUS-PR: 0.8871743329644867
Datei: 258_TAO_id_2_Environment_tr_500_1st_4.csv, AUC-PR: 0.14906529874605562, VUS-PR: 0.9406478652985084
Datei: 259_TAO_id_3_Environment_tr_500_1st_7.csv, AUC-PR: 0.04592060635184418, VUS-PR: 0.2505436799139267
Durchschnittlicher AUC-PR: 0.09778804957459407
Durchschnittlicher VUS-PR: 0.6927886260589741
%% Cell type:code id: tags:
``` python
results[0]['metrics'].keys()
```
%% Output
dict_keys(['AUC-PR', 'AUC-ROC', 'VUS-PR', 'VUS-ROC', 'Standard-F1', 'PA-F1', 'Event-based-F1', 'R-based-F1', 'Affiliation-F', 'Precision', 'Recall'])
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment