diff --git a/CHANGELOG.md b/CHANGELOG.md index 1237a14..90b8f13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ Changelog ========= +Version 0.88.2 +-------------- +* changed combine speaker results to show speakers not samples + Version 0.88.1 -------------- * added obligatory scatter plot for regression diff --git a/nkululeko/constants.py b/nkululeko/constants.py index 43b9c4a..7c6b4cf 100644 --- a/nkululeko/constants.py +++ b/nkululeko/constants.py @@ -1,2 +1,2 @@ -VERSION="0.88.1" +VERSION="0.88.2" SAMPLING_RATE = 16000 diff --git a/nkululeko/experiment.py b/nkululeko/experiment.py index 8f01db8..3d5d76d 100644 --- a/nkululeko/experiment.py +++ b/nkululeko/experiment.py @@ -107,8 +107,7 @@ def load_datasets(self): # print keys/column dbs = ",".join(list(self.datasets.keys())) labels = self.util.config_val("DATA", "labels", False) - auto_labels = list( - next(iter(self.datasets.values())).df[self.target].unique()) + auto_labels = list(next(iter(self.datasets.values())).df[self.target].unique()) if labels: self.labels = ast.literal_eval(labels) self.util.debug(f"Using target labels (from config): {labels}") @@ -158,8 +157,7 @@ def fill_tests(self): data.split() data.prepare_labels() self.df_test = pd.concat( - [self.df_test, self.util.make_segmented_index( - data.df_test)] + [self.df_test, self.util.make_segmented_index(data.df_test)] ) self.df_test.is_labeled = data.is_labeled self.df_test.got_gender = self.got_gender @@ -260,8 +258,7 @@ def fill_train_and_tests(self): test_cats = self.df_test[self.target].unique() else: # if there is no target, copy a dummy label - self.df_test = self._add_random_target( - self.df_test).astype("str") + self.df_test = self._add_random_target(self.df_test).astype("str") train_cats = self.df_train[self.target].unique() # print(f"df_train: {pd.DataFrame(self.df_train[self.target])}") # print(f"train_cats with target {self.target}: {train_cats}") @@ -269,8 +266,7 @@ def fill_train_and_tests(self): if type(test_cats) == np.ndarray: self.util.debug(f"Categories test (nd.array): {test_cats}") else: - self.util.debug( - f"Categories test (list): {list(test_cats)}") + self.util.debug(f"Categories test (list): {list(test_cats)}") if type(train_cats) == np.ndarray: self.util.debug(f"Categories train (nd.array): {train_cats}") else: @@ -293,8 +289,7 @@ def fill_train_and_tests(self): target_factor = self.util.config_val("DATA", "target_divide_by", False) if target_factor: - self.df_test[self.target] = self.df_test[self.target] / \ - float(target_factor) + self.df_test[self.target] = self.df_test[self.target] / float(target_factor) self.df_train[self.target] = self.df_train[self.target] / float( target_factor ) @@ -317,16 +312,14 @@ def _add_random_target(self, df): def plot_distribution(self, df_labels): """Plot the distribution of samples and speaker per target class and biological sex""" plot = Plots() - sample_selection = self.util.config_val( - "EXPL", "sample_selection", "all") + sample_selection = self.util.config_val("EXPL", "sample_selection", "all") plot.plot_distributions(df_labels) if self.got_speaker: plot.plot_distributions_speaker(df_labels) def extract_test_feats(self): self.feats_test = pd.DataFrame() - feats_name = "_".join(ast.literal_eval( - glob_conf.config["DATA"]["tests"])) + feats_name = "_".join(ast.literal_eval(glob_conf.config["DATA"]["tests"])) feats_types = self.util.config_val_list("FEATS", "type", ["os"]) self.feature_extractor = FeatureExtractor( self.df_test, feats_types, feats_name, "test" @@ -343,8 +336,7 @@ def extract_feats(self): """ df_train, df_test = self.df_train, self.df_test - feats_name = "_".join(ast.literal_eval( - glob_conf.config["DATA"]["databases"])) + feats_name = "_".join(ast.literal_eval(glob_conf.config["DATA"]["databases"])) self.feats_test, self.feats_train = pd.DataFrame(), pd.DataFrame() feats_types = self.util.config_val("FEATS", "type", "os") # Ensure feats_types is always a list of strings @@ -385,8 +377,7 @@ def extract_feats(self): f"test feats ({self.feats_test.shape[0]}) != test labels" f" ({self.df_test.shape[0]})" ) - self.df_test = self.df_test[self.df_test.index.isin( - self.feats_test.index)] + self.df_test = self.df_test[self.df_test.index.isin(self.feats_test.index)] self.util.warn(f"new test labels shape: {self.df_test.shape[0]}") self._check_scale() @@ -401,8 +392,7 @@ def augment(self): """Augment the selected samples.""" from nkululeko.augmenting.augmenter import Augmenter - sample_selection = self.util.config_val( - "AUGMENT", "sample_selection", "all") + sample_selection = self.util.config_val("AUGMENT", "sample_selection", "all") if sample_selection == "all": df = pd.concat([self.df_train, self.df_test]) elif sample_selection == "train": @@ -497,8 +487,7 @@ def random_splice(self): """ from nkululeko.augmenting.randomsplicer import Randomsplicer - sample_selection = self.util.config_val( - "AUGMENT", "sample_selection", "all") + sample_selection = self.util.config_val("AUGMENT", "sample_selection", "all") if sample_selection == "all": df = pd.concat([self.df_train, self.df_test]) elif sample_selection == "train": @@ -519,8 +508,7 @@ def analyse_features(self, needs_feats): plot_feats = eval( self.util.config_val("EXPL", "feature_distributions", "False") ) - sample_selection = self.util.config_val( - "EXPL", "sample_selection", "all") + sample_selection = self.util.config_val("EXPL", "sample_selection", "all") # get the data labels if sample_selection == "all": df_labels = pd.concat([self.df_train, self.df_test]) @@ -583,8 +571,7 @@ def analyse_features(self, needs_feats): for scat_target in scat_targets: if self.util.is_categorical(df_labels[scat_target]): for scatter in scatters: - plots.scatter_plot( - df_feats, df_labels, scat_target, scatter) + plots.scatter_plot(df_feats, df_labels, scat_target, scatter) else: self.util.debug( f"{self.name}: binning continuous variable to categories" @@ -669,15 +656,15 @@ def plot_confmat_per_speaker(self, function): ) return best = self.get_best_report(self.reports) - # if not best.is_classification: - # best.continuous_to_categorical() - truths = best.truths - preds = best.preds + if best.is_classification: + truths = best.truths + preds = best.preds + else: + truths = best.truths_cont + preds = best.preds_cont speakers = self.df_test.speaker.values - print(f"{len(truths)} {len(preds)} {len(speakers) }") - df = pd.DataFrame( - data={"truth": truths, "pred": preds, "speaker": speakers}) - plot_name = "result_combined_per_speaker" + df = pd.DataFrame(data={"truths": truths, "preds": preds, "speakers": speakers}) + plot_name = f"{self.util.get_exp_name()}_speakercombined_{function}" self.util.debug( f"plotting speaker combination ({function}) confusion matrix to" f" {plot_name}" @@ -692,13 +679,13 @@ def print_best_model(self): def demo(self, file, is_list, outfile): model = self.runmgr.get_best_model() - labelEncoder = None + lab_enc = None try: - labelEncoder = self.label_encoder + lab_enc = self.label_encoder except AttributeError: pass demo = Demo_predictor( - model, file, is_list, self.feature_extractor, labelEncoder, outfile + model, file, is_list, self.feature_extractor, lab_enc, outfile ) demo.run_demo() diff --git a/nkululeko/reporting/reporter.py b/nkululeko/reporting/reporter.py index 472a71f..33334bd 100644 --- a/nkululeko/reporting/reporter.py +++ b/nkululeko/reporting/reporter.py @@ -34,23 +34,24 @@ class Reporter: - def __set_measure(self): + def _set_metric(self): if self.util.exp_is_classification(): - self.MEASURE = "UAR" - self.result.measure = self.MEASURE + self.metric = "uar" + self.METRIC = "UAR" + self.result.metric = self.METRIC self.is_classification = True else: self.is_classification = False - self.measure = self.util.config_val("MODEL", "measure", "mse") - if self.measure == "mse": - self.MEASURE = "MSE" - self.result.measure = self.MEASURE - elif self.measure == "mae": - self.MEASURE = "MAE" - self.result.measure = self.MEASURE - elif self.measure == "ccc": - self.MEASURE = "CCC" - self.result.measure = self.MEASURE + self.metric = self.util.config_val("MODEL", "measure", "mse") + if self.metric == "mse": + self.METRIC = "MSE" + self.result.metric = self.METRIC + elif self.metric == "mae": + self.METRIC = "MAE" + self.result.metric = self.METRIC + elif self.metric == "ccc": + self.METRIC = "CCC" + self.result.metric = self.METRIC def __init__(self, truths, preds, run, epoch, probas=None): """Initialization with ground truth und predictions vector. @@ -70,60 +71,70 @@ def __init__(self, truths, preds, run, epoch, probas=None): self.result = Result(0, 0, 0, 0, "unknown") self.run = run self.epoch = epoch - self.__set_measure() + self._set_metric() self.filenameadd = "" self.cont_to_cat = False if len(self.truths) > 0 and len(self.preds) > 0: if self.util.exp_is_classification(): - uar, (upper, lower) = evaluate_with_conf_int( - self.preds, - unweighted_average_recall, - self.truths, - num_bootstraps=1000, - alpha=5, + uar, upper, lower = self._get_test_result( + self.truths, self.preds, "uar" ) self.result.test = uar self.result.set_upper_lower(upper, lower) self.result.loss = 1 - accuracy(self.truths, self.preds) else: # regression experiment - if self.measure == "mse": - test_result, (upper, lower) = evaluate_with_conf_int( - self.preds, - mean_squared_error, - self.truths, - num_bootstraps=1000, - alpha=5, - ) - elif self.measure == "mae": - test_result, (upper, lower) = evaluate_with_conf_int( - self.preds, - mean_absolute_error, - self.truths, - num_bootstraps=1000, - alpha=5, - ) - elif self.measure == "ccc": - test_result, (upper, lower) = evaluate_with_conf_int( - self.preds, - concordance_cc, - self.truths, - num_bootstraps=1000, - alpha=5, - ) - - if math.isnan(self.result.test): - self.util.debug(f"Truth: {self.truths}") - self.util.debug(f"Predict.: {self.preds}") - self.util.debug("Result is NAN: setting to -1") - self.result.test = -1 - else: - self.util.error(f"unknown measure: {self.measure}") - + # keep the original values for further use, they will be binned later + self.truths_cont = self.truths + self.preds_cont = self.preds + test_result, upper, lower = self._get_test_result( + self.truths, self.preds, self.metric + ) self.result.test = test_result self.result.set_upper_lower(upper, lower) # train and loss are being set by the model - # print out the class probilities + + def _get_test_result(self, truths, preds, metric): + if metric == "uar": + test_result, (upper, lower) = evaluate_with_conf_int( + preds, + unweighted_average_recall, + truths, + num_bootstraps=1000, + alpha=5, + ) + elif metric == "mse": + test_result, (upper, lower) = evaluate_with_conf_int( + preds, + mean_squared_error, + truths, + num_bootstraps=1000, + alpha=5, + ) + elif metric == "mae": + test_result, (upper, lower) = evaluate_with_conf_int( + preds, + mean_absolute_error, + truths, + num_bootstraps=1000, + alpha=5, + ) + elif metric == "ccc": + test_result, (upper, lower) = evaluate_with_conf_int( + preds, + concordance_cc, + truths, + num_bootstraps=1000, + alpha=5, + ) + if math.isnan(test_result): + self.util.debug(f"Truth: {self.truths}") + self.util.debug(f"Predict.: {self.preds}") + self.util.debug("Result is NAN: setting to -1") + test_result = -1 + else: + self.util.error(f"unknown metric: {self.metric}") + return test_result, upper, lower def print_probabilities(self): """Print the probabilities per class to a file in the store.""" @@ -195,31 +206,49 @@ def plot_confmatrix(self, plot_name, epoch=None): def plot_per_speaker(self, result_df, plot_name, function): """Plot a confusion matrix with the mode category per speakers. + If the function is mode and the values continuous, bin first + Args: result_df: a pandas dataframe with columns: preds, truths and speaker. plot_name: name for the figure. function: either mode or mean. """ - speakers = result_df.speaker.unique() - pred = np.zeros(0) - truth = np.zeros(0) + if function == "mode" and not self.is_classification: + truths, preds = result_df["truths"].values, result_df["preds"].values + truths, preds = self.util._bin_distributions(truths, preds) + result_df["truths"], result_df["preds"] = truths, preds + speakers = result_df.speakers.unique() + preds_speakers = np.zeros(0) + truths_speakers = np.zeros(0) for s in speakers: - s_df = result_df[result_df.speaker == s] - mode = s_df.pred.mode().iloc[-1] - mean = s_df.pred.mean() + s_df = result_df[result_df.speakers == s] + s_truth = s_df.truths.iloc[0] + s_pred = None if function == "mode": - s_df.pred = mode + s_pred = s_df.preds.mode().iloc[-1] elif function == "mean": - s_df.pred = mean + s_pred = s_df.preds.mean() else: - self.util.error(f"unkown function {function}") - pred = np.append(pred, s_df.pred.values) - truth = np.append(truth, s_df["truth"].values) - if not (self.is_classification or self.cont_to_cat): - bins = ast.literal_eval(glob_conf.config["DATA"]["bins"]) - truth = np.digitize(truth, bins) - 1 - pred = np.digitize(pred, bins) - 1 - self._plot_confmat(truth, pred.astype("int"), plot_name, 0) + self.util.error(f"unknown function {function}") + preds_speakers = np.append(preds_speakers, s_pred) + truths_speakers = np.append(truths_speakers, s_truth) + test_result, upper, lower = self._get_test_result( + result_df.truths.values, result_df.preds.values, self.metric + ) + test_result = Result(test_result, None, None, None, self.METRIC) + test_result.set_upper_lower(upper, lower) + result_msg = f"Speaker combination result: {test_result.test_result_str()}" + self.util.debug(result_msg) + if function == "mean": + truths_speakers, preds_speakers = self.util._bin_distributions( + truths_speakers, preds_speakers + ) + self._plot_confmat( + truths_speakers, + preds_speakers.astype("int"), + plot_name, + test_result=test_result, + ) def _plot_scatter(self, truths, preds, plot_name, epoch=None): # print(truths) @@ -227,13 +256,10 @@ def _plot_scatter(self, truths, preds, plot_name, epoch=None): if epoch is None: epoch = self.epoch fig_dir = self.util.get_path("fig_dir") - fig = plt.figure() # figsize=[5, 5] - pcc = pearsonr(self.truths, self.preds)[0] - - reg_res = f"{self.result.test:.3f} {self.MEASURE}" - - plt.scatter(truths, preds, cmap="Blues") + reg_res = self.result.test_result_str() + fig = plt.figure() + plt.scatter(truths, preds) plt.xlabel("truth") plt.ylabel("prediction") @@ -258,11 +284,11 @@ def _plot_scatter(self, truths, preds, plot_name, epoch=None): ) ) - def _plot_confmat(self, truths, preds, plot_name, epoch=None): - # print(truths) - # print(preds) + def _plot_confmat(self, truths, preds, plot_name, epoch=None, test_result=None): if epoch is None: epoch = self.epoch + if test_result is None: + test_result = self.result fig_dir = self.util.get_path("fig_dir") labels = glob_conf.labels fig = plt.figure() # figsize=[5, 5] @@ -295,12 +321,15 @@ def _plot_confmat(self, truths, preds, plot_name, epoch=None): reg_res = "" if not self.is_classification: - reg_res = f"{self.result.test:.3f} {self.MEASURE}" + reg_res = f"{test_result.test_result_str()}" + self.util.debug( + f"Best result at epoch {epoch}: {test_result.test_result_str()}" + ) - uar_str = str(int(uar * 1000) / 1000.0)[1:] - acc_str = str(int(acc * 1000) / 1000.0)[1:] - up_str = str(int(upper * 1000) / 1000.0)[1:] - low_str = str(int(lower * 1000) / 1000.0)[1:] + uar_str = self.util.to_3_digits_str(uar) + acc_str = self.util.to_3_digits_str(acc) + up_str = self.util.to_3_digits_str(upper) + low_str = self.util.to_3_digits_str(lower) if epoch != 0: plt.title( @@ -427,7 +456,7 @@ def plot_epoch_progression_finetuned(self, df): ax = df.plot() fig = ax.figure plt.xlabel("epochs") - plt.ylabel(f"{self.MEASURE}") + plt.ylabel(f"{self.METRIC}") plot_path = f"{fig_dir}{plot_name}.{self.format}" plt.savefig(plot_path) self.util.debug(f"plotted epoch progression to {plot_path}") @@ -464,7 +493,7 @@ def plot_epoch_progression(self, reports, out_name): plt.plot(losses, "black", label="losses") plt.plot(losses_eval, "grey", label="losses_eval") plt.xlabel("epochs") - plt.ylabel(f"{self.MEASURE}") + plt.ylabel(f"{self.METRIC}") plt.legend() plt.savefig(f"{fig_dir}{out_name}.{self.format}") plt.close() diff --git a/nkululeko/reporting/result.py b/nkululeko/reporting/result.py index 40ec995..7970878 100644 --- a/nkululeko/reporting/result.py +++ b/nkululeko/reporting/result.py @@ -1,13 +1,15 @@ # result.py +from nkululeko.utils.util import Util class Result: - def __init__(self, test, train, loss, loss_eval, measure): + def __init__(self, test, train, loss, loss_eval, metric): self.test = test self.train = train self.loss = loss self.loss_eval = loss_eval - self.measure = measure + self.metric = metric + self.util = Util("Result") def get_result(self): return self.test @@ -18,10 +20,16 @@ def set_upper_lower(self, upper, lower): self.lower = lower def get_test_result(self): - return f"test: {self.test:.3f} {self.measure}" + return f"test: {self.test:.3f} {self.metric}" def to_string(self): return ( - f"test: {self.test} {self.measure}, train:" - f" {self.train} {self.measure}, loss: {self.loss}, eval-loss: {self.loss_eval}" + f"test: {self.test} {self.metric}, train:" + f" {self.train} {self.metric}, loss: {self.loss}, eval-loss: {self.loss_eval}" ) + + def test_result_str(self): + result_s = self.util.to_3_digits_str(self.test) + up_str = self.util.to_3_digits_str(self.upper) + low_str = self.util.to_3_digits_str(self.lower) + return f"{self.metric}: {result_s} ({up_str}/{low_str})" diff --git a/nkululeko/utils/util.py b/nkululeko/utils/util.py index e0c7b52..dbf036e 100644 --- a/nkululeko/utils/util.py +++ b/nkululeko/utils/util.py @@ -50,9 +50,7 @@ def __init__(self, caller=None, has_config=True): self.got_data_roots = False def get_path(self, entry): - """ - This method allows the user to get the directory path for the given argument. - """ + """This method allows the user to get the directory path for the given argument.""" if self.config is None: # If no configuration file is provided, use default paths if entry == "fig_dir": @@ -139,15 +137,11 @@ def is_categorical(self, pd_series): ) def get_name(self): - """ - Get the name of the experiment - """ + """Get the name of the experiment.""" return self.config["EXP"]["name"] def get_exp_dir(self): - """ - Get the experiment directory - """ + """Get the experiment directory.""" root = os.path.join(self.config["EXP"]["root"], "") name = self.config["EXP"]["name"] dir_name = f"{root}{name}" @@ -176,15 +170,11 @@ def _get_value_descript(self, section, name): return "" def get_data_name(self): - """ - Get a string as name from all databases that are useed - """ + """Get a string as name from all databases that are useed.""" return "_".join(ast.literal_eval(self.config["DATA"]["databases"])) def get_feattype_name(self): - """ - Get a string as name from all feature sets that are used - """ + """Get a string as name from all feature sets that are used.""" return "_".join(ast.literal_eval(self.config["FEATS"]["type"])) def get_exp_name(self, only_train=False, only_data=False): @@ -303,9 +293,9 @@ def get_labels(self): return ast.literal_eval(self.config["DATA"]["labels"]) def continuous_to_categorical(self, series): - """ - discretize a categorical variable. - uses the labels and bins from the ini if present + """Discretize a categorical variable. + + Uses the labels and bins from the ini if present :param series: a pandas series :return a pandas series with discretized values as categories @@ -321,11 +311,23 @@ def continuous_to_categorical(self, series): labels = ["0_low", "1_middle", "2_high"] result = np.digitize(series, bins) - 1 result = pd.Series(result) - for i, l in enumerate(labels): - result = result.replace(i, str(l)) + for i, lab in enumerate(labels): + result = result.replace(i, str(lab)) result = result.astype("category") return result + def _bin_distributions(self, truths, preds): + try: + bins = ast.literal_eval(self.config["DATA"]["bins"]) + except KeyError: + # if no binning is given, simply take three bins, based on truth + b1 = np.quantile(truths, 0.33) + b2 = np.quantile(truths, 0.66) + bins = [-1000000, b1, b2, 1000000] + truths = np.digitize(truths, bins) - 1 + preds = np.digitize(preds, bins) - 1 + return truths, preds + def print_best_results(self, best_reports): res_dir = self.get_res_dir() # go one level up above the "run" level @@ -416,5 +418,10 @@ def high_is_good(self): self.error(f"unknown measure: {measure}") def to_3_digits(self, x): + """Given a float, return this to 3 digits.""" x = float(x) return (int(x * 1000)) / 1000.0 + + def to_3_digits_str(self, x): + """Given a float, return this to 3 digits as string without integer number.""" + return str(self.to_3_digits(x))[1:] diff --git a/tests/exp_agedb_agender_mlp.ini b/tests/exp_agedb_agender_mlp.ini index c2511d2..fd437d7 100644 --- a/tests/exp_agedb_agender_mlp.ini +++ b/tests/exp_agedb_agender_mlp.ini @@ -28,3 +28,4 @@ patience = 3 [PLOT] best_model = True epoch_progression = True +combine_per_speaker = mode diff --git a/tests/exp_agedb_class_os_xgb.ini b/tests/exp_agedb_class_os_xgb.ini index 194003c..7da2b48 100644 --- a/tests/exp_agedb_class_os_xgb.ini +++ b/tests/exp_agedb_class_os_xgb.ini @@ -20,3 +20,5 @@ type = ['os'] scale = standard [MODEL] type = xgb +[PLOT] +combine_per_speaker = mode diff --git a/tests/exp_agedb_os_mlp.ini b/tests/exp_agedb_os_mlp.ini index 3e7efe7..007852b 100644 --- a/tests/exp_agedb_os_mlp.ini +++ b/tests/exp_agedb_os_mlp.ini @@ -27,3 +27,4 @@ patience = 5 [PLOT] best_model = True epoch_progression = True +combine_per_speaker = mode diff --git a/tests/exp_emodb_os_xgb.ini b/tests/exp_emodb_os_xgb.ini index 50fdb54..398ad01 100644 --- a/tests/exp_emodb_os_xgb.ini +++ b/tests/exp_emodb_os_xgb.ini @@ -12,7 +12,7 @@ emodb.test_tables = ['emotion.categories.test.gold_standard'] emodb.train_tables = ['emotion.categories.train.gold_standard'] labels = ['anger', 'happiness'] target = emotion -no_reuse = True +#no_reuse = True [FEATS] type = ['os'] store_format = csv @@ -20,3 +20,5 @@ scale = standard no_reuse = True [MODEL] type = xgb +[PLOT] +combine_per_speaker = mean