@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsCategorical(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    df.transform(validationData);
    BootstrapAggregating.TrainingParameters param = new BootstrapAggregating.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    BootstrapAggregating instance = new BootstrapAggregating(dbName, conf, param);
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = new DummyXYMinMaxNormalizer(dbName, conf);
    instance = new BootstrapAggregating(dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsCategorical(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = MLBuilder.create(new DummyXYMinMaxNormalizer.TrainingParameters(), dbName, conf);
    df.fit_transform(trainingData);
    df.transform(validationData);
    BootstrapAggregating.TrainingParameters param = new BootstrapAggregating.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    BootstrapAggregating instance = MLBuilder.create(param, dbName, conf);
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = MLBuilder.load(DummyXYMinMaxNormalizer.class, dbName, conf);
    instance = MLBuilder.load(BootstrapAggregating.class, dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("testKFoldCrossValidation");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.regressionMixed(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    MatrixLinearRegression.TrainingParameters param = new MatrixLinearRegression.TrainingParameters();
    LinearRegressionMetrics vm = new KFoldValidator<>(LinearRegressionMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 1;
    double result = vm.getRSquare();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("testKFoldCrossValidation");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.regressionMixed(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = MLBuilder.create(new DummyXYMinMaxNormalizer.TrainingParameters(), dbName, conf);
    df.fit_transform(trainingData);
    MatrixLinearRegression.TrainingParameters param = new MatrixLinearRegression.TrainingParameters();
    LinearRegressionMetrics vm = new KFoldValidator<>(LinearRegressionMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 1;
    double result = vm.getRSquare();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}

@Override
public VM validate(Dataframe dataset, TrainingParameters trainingParameters) {
    int n = dataset.size();
    if (k <= 0 || n <= k) {
        throw new IllegalArgumentException("Invalid number of folds.");
    }
    int foldSize = n / k;
    Integer[] ids = new Integer[n];
    int j = 0;
    for (Integer rId : dataset.index()) {
        ids[j] = rId;
        ++j;
    }
    PHPMethods.shuffle(ids);
    AbstractModeler modeler = Trainable.newInstance(trainingParameters, "kfold_" + RandomGenerator.getThreadLocalRandomUnseeded().nextLong(), conf);
    List<VM> validationMetricsList = new LinkedList<>();
    for (int fold = 0; fold < k; ++fold) {
        logger.info("Kfold {}", fold);
        FlatDataList foldTrainingIds = new FlatDataList(new ArrayList<>(n - foldSize));
        FlatDataList foldValidationIds = new FlatDataList(new ArrayList<>(foldSize));
        for (int i = 0; i < n; ++i) {
            boolean isInValidationFoldRange = false;
            if (fold * foldSize <= i && i < (fold + 1) * foldSize) {
                isInValidationFoldRange = true;
            }
            if (isInValidationFoldRange) {
                foldValidationIds.add(ids[i]);
            } else {
                foldTrainingIds.add(ids[i]);
            }
        }
        if (k == 1) {
            foldTrainingIds = foldValidationIds;
        }
        Dataframe trainingData = dataset.getSubset(foldTrainingIds);
        modeler.fit(trainingData);
        trainingData.delete();
        Dataframe validationData = dataset.getSubset(foldValidationIds);
        modeler.predict(validationData);
        VM entrySample = ValidationMetrics.newInstance(vmClass, validationData);
        validationData.delete();
        validationMetricsList.add(entrySample);
    }
    modeler.delete();
    VM avgValidationMetrics = ValidationMetrics.newInstance(vmClass, validationMetricsList);
    return avgValidationMetrics;
}

@Override
public VM validate(Dataframe dataset, TrainingParameters trainingParameters) {
    int n = dataset.size();
    if (k <= 0 || n <= k) {
        throw new IllegalArgumentException("Invalid number of folds.");
    }
    int foldSize = n / k;
    Integer[] ids = new Integer[n];
    int j = 0;
    for (Integer rId : dataset.index()) {
        ids[j] = rId;
        ++j;
    }
    PHPMethods.shuffle(ids);
    AbstractModeler modeler = MLBuilder.create(trainingParameters, "kfold_" + RandomGenerator.getThreadLocalRandomUnseeded().nextLong(), conf);
    List<VM> validationMetricsList = new LinkedList<>();
    for (int fold = 0; fold < k; ++fold) {
        logger.info("Kfold {}", fold);
        FlatDataList foldTrainingIds = new FlatDataList(new ArrayList<>(n - foldSize));
        FlatDataList foldValidationIds = new FlatDataList(new ArrayList<>(foldSize));
        for (int i = 0; i < n; ++i) {
            boolean isInValidationFoldRange = false;
            if (fold * foldSize <= i && i < (fold + 1) * foldSize) {
                isInValidationFoldRange = true;
            }
            if (isInValidationFoldRange) {
                foldValidationIds.add(ids[i]);
            } else {
                foldTrainingIds.add(ids[i]);
            }
        }
        if (k == 1) {
            foldTrainingIds = foldValidationIds;
        }
        Dataframe trainingData = dataset.getSubset(foldTrainingIds);
        modeler.fit(trainingData);
        trainingData.delete();
        Dataframe validationData = dataset.getSubset(foldValidationIds);
        modeler.predict(validationData);
        VM entrySample = ValidationMetrics.newInstance(vmClass, validationData);
        validationData.delete();
        validationMetricsList.add(entrySample);
    }
    modeler.delete();
    VM avgValidationMetrics = ValidationMetrics.newInstance(vmClass, validationMetricsList);
    return avgValidationMetrics;
}

@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.regressionNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    XYMinMaxNormalizer df = new XYMinMaxNormalizer(dbName, conf, new XYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    df.transform(validationData);
    MatrixLinearRegression instance = new MatrixLinearRegression(dbName, conf, new MatrixLinearRegression.TrainingParameters());
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = new XYMinMaxNormalizer(dbName, conf);
    instance = new MatrixLinearRegression(dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    for (Record r : validationData) {
        assertEquals(TypeInference.toDouble(r.getY()), TypeInference.toDouble(r.getYPredicted()), Constants.DOUBLE_ACCURACY_HIGH);
    }
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.regressionNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    XYMinMaxNormalizer df = MLBuilder.create(new XYMinMaxNormalizer.TrainingParameters(), dbName, conf);
    df.fit_transform(trainingData);
    df.transform(validationData);
    MatrixLinearRegression instance = MLBuilder.create(new MatrixLinearRegression.TrainingParameters(), dbName, conf);
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = MLBuilder.load(XYMinMaxNormalizer.class, dbName, conf);
    instance = MLBuilder.load(MatrixLinearRegression.class, dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    for (Record r : validationData) {
        assertEquals(TypeInference.toDouble(r.getY()), TypeInference.toDouble(r.getYPredicted()), Constants.DOUBLE_ACCURACY_HIGH);
    }
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

public static <T extends Trainable, TP extends Parameterizable> T create(TP trainingParameters, String dbName, Configuration conf) {
    try {
        Class<T> aClass = (Class<T>) trainingParameters.getClass().getEnclosingClass();
        Constructor<T> constructor = aClass.getDeclaredConstructor(String.class, Configuration.class, trainingParameters.getClass());
        constructor.setAccessible(true);
        return constructor.newInstance(dbName, conf, trainingParameters);
    } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException ex) {
        throw new RuntimeException(ex);
    }
}


@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.regressionNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    df.transform(validationData);
    NLMS.TrainingParameters param = new NLMS.TrainingParameters();
    param.setTotalIterations(1600);
    param.setL1(0.00000001);
    NLMS instance = new NLMS(dbName, conf, param);
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = new DummyXYMinMaxNormalizer(dbName, conf);
    instance = new NLMS(dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    for (Record r : validationData) {
        assertEquals(TypeInference.toDouble(r.getY()), TypeInference.toDouble(r.getYPredicted()), Constants.DOUBLE_ACCURACY_HIGH);
    }
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.regressionNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = MLBuilder.create(new DummyXYMinMaxNormalizer.TrainingParameters(), dbName, conf);
    df.fit_transform(trainingData);
    df.transform(validationData);
    NLMS.TrainingParameters param = new NLMS.TrainingParameters();
    param.setTotalIterations(1600);
    param.setL1(0.00000001);
    NLMS instance = MLBuilder.create(param, dbName, conf);
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = MLBuilder.load(DummyXYMinMaxNormalizer.class, dbName, conf);
    instance = MLBuilder.load(NLMS.class, dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    for (Record r : validationData) {
        assertEquals(TypeInference.toDouble(r.getY()), TypeInference.toDouble(r.getYPredicted()), Constants.DOUBLE_ACCURACY_HIGH);
    }
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("testKFoldCrossValidation");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.housingNumerical(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    PCA.TrainingParameters featureSelectorParameters = new PCA.TrainingParameters();
    featureSelectorParameters.setMaxDimensions(trainingData.xColumnSize() - 1);
    featureSelectorParameters.setWhitened(false);
    featureSelectorParameters.setVariancePercentageThreshold(0.99999995);
    PCA featureSelector = new PCA(dbName, conf, featureSelectorParameters);
    featureSelector.fit_transform(trainingData);
    featureSelector.delete();
    NLMS.TrainingParameters param = new NLMS.TrainingParameters();
    param.setTotalIterations(500);
    param.setL1(0.001);
    param.setL2(0.001);
    LinearRegressionMetrics vm = new KFoldValidator<>(LinearRegressionMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 0.7748106446239166;
    double result = vm.getRSquare();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("testKFoldCrossValidation");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.housingNumerical(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = MLBuilder.create(new DummyXYMinMaxNormalizer.TrainingParameters(), dbName, conf);
    df.fit_transform(trainingData);
    PCA.TrainingParameters featureSelectorParameters = new PCA.TrainingParameters();
    featureSelectorParameters.setMaxDimensions(trainingData.xColumnSize() - 1);
    featureSelectorParameters.setWhitened(false);
    featureSelectorParameters.setVariancePercentageThreshold(0.99999995);
    PCA featureSelector = MLBuilder.create(featureSelectorParameters, dbName, conf);
    featureSelector.fit_transform(trainingData);
    featureSelector.delete();
    NLMS.TrainingParameters param = new NLMS.TrainingParameters();
    param.setTotalIterations(500);
    param.setL1(0.001);
    param.setL2(0.001);
    LinearRegressionMetrics vm = new KFoldValidator<>(LinearRegressionMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 0.7748106446239166;
    double result = vm.getRSquare();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}


@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.regressionNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    StepwiseRegression.TrainingParameters param = new StepwiseRegression.TrainingParameters();
    param.setAout(0.05);
    MatrixLinearRegression.TrainingParameters trainingParams = new MatrixLinearRegression.TrainingParameters();
    param.setRegressionTrainingParameters(trainingParams);
    StepwiseRegression instance = new StepwiseRegression(dbName, conf, param);
    instance.fit(trainingData);
    df.denormalize(trainingData);
    instance.close();
    df.close();
    df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.transform(validationData);
    instance = new StepwiseRegression(dbName, conf);
    instance.predict(validationData);
    df.denormalize(validationData);
    for (Record r : validationData) {
        Assert.assertEquals(TypeInference.toDouble(r.getY()), TypeInference.toDouble(r.getYPredicted()), Constants.DOUBLE_ACCURACY_HIGH);
    }
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.regressionNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = MLBuilder.create(new DummyXYMinMaxNormalizer.TrainingParameters(), dbName, conf);
    df.fit_transform(trainingData);
    StepwiseRegression.TrainingParameters param = new StepwiseRegression.TrainingParameters();
    param.setAout(0.05);
    MatrixLinearRegression.TrainingParameters trainingParams = new MatrixLinearRegression.TrainingParameters();
    param.setRegressionTrainingParameters(trainingParams);
    StepwiseRegression instance = MLBuilder.create(param, dbName, conf);
    instance.fit(trainingData);
    df.denormalize(trainingData);
    instance.close();
    df.close();
    df = MLBuilder.load(DummyXYMinMaxNormalizer.class, dbName, conf);
    df.transform(validationData);
    instance = MLBuilder.load(StepwiseRegression.class, dbName, conf);
    instance.predict(validationData);
    df.denormalize(validationData);
    for (Record r : validationData) {
        Assert.assertEquals(TypeInference.toDouble(r.getY()), TypeInference.toDouble(r.getYPredicted()), Constants.DOUBLE_ACCURACY_HIGH);
    }
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}


@Test
public void testTrainAndValidate() {
    logger.info("testTrainAndValidate");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    Modeler.TrainingParameters trainingParameters = new Modeler.TrainingParameters();
    trainingParameters.setModelerClass(MultinomialNaiveBayes.class);
    MultinomialNaiveBayes.TrainingParameters modelTrainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    modelTrainingParameters.setMultiProbabilityWeighted(true);
    trainingParameters.setModelerTrainingParameters(modelTrainingParameters);
    trainingParameters.setDataTransformerClass(DummyXMinMaxNormalizer.class);
    DummyXMinMaxNormalizer.TrainingParameters dtParams = new DummyXMinMaxNormalizer.TrainingParameters();
    trainingParameters.setDataTransformerTrainingParameters(dtParams);
    trainingParameters.setFeatureSelectorClass(null);
    trainingParameters.setFeatureSelectorTrainingParameters(null);
    Modeler instance = new Modeler(dbName, conf, trainingParameters);
    instance.fit(trainingData);
    instance.close();
    instance = new Modeler(dbName, conf);
    instance.predict(trainingData);
    ClassificationMetrics vm = new ClassificationMetrics(trainingData);
    double expResult2 = 0.8;
    assertEquals(expResult2, vm.getMacroF1(), Constants.DOUBLE_ACCURACY_HIGH);
    instance.close();
    logger.info("validate");
    instance = new Modeler(dbName, conf);
    instance.predict(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testTrainAndValidate() {
    logger.info("testTrainAndValidate");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    Modeler.TrainingParameters trainingParameters = new Modeler.TrainingParameters();
    MultinomialNaiveBayes.TrainingParameters modelTrainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    modelTrainingParameters.setMultiProbabilityWeighted(true);
    trainingParameters.setModelerTrainingParameters(modelTrainingParameters);
    DummyXMinMaxNormalizer.TrainingParameters dtParams = new DummyXMinMaxNormalizer.TrainingParameters();
    trainingParameters.setDataTransformerTrainingParameters(dtParams);
    trainingParameters.setFeatureSelectorTrainingParameters(null);
    Modeler instance = new Modeler(dbName, conf, trainingParameters);
    instance.fit(trainingData);
    instance.close();
    instance = new Modeler(dbName, conf);
    instance.predict(trainingData);
    ClassificationMetrics vm = new ClassificationMetrics(trainingData);
    double expResult2 = 0.8;
    assertEquals(expResult2, vm.getMacroF1(), Constants.DOUBLE_ACCURACY_HIGH);
    instance.close();
    logger.info("validate");
    instance = new Modeler(dbName, conf);
    instance.predict(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

public void predict(Dataframe newData) {
    logger.info("predict()");
    Modeler.TrainingParameters trainingParameters = knowledgeBase.getTrainingParameters();
    Configuration conf = knowledgeBase.getConf();
    Class dtClass = trainingParameters.getDataTransformerClass();
    boolean transformData = (dtClass != null);
    if (transformData) {
        if (dataTransformer == null) {
            dataTransformer = Trainable.<AbstractTransformer>newInstance(dtClass, dbName, conf);
        }
        setParallelized(dataTransformer);
        dataTransformer.transform(newData);
    }
    Class fsClass = trainingParameters.getFeatureSelectorClass();
    boolean selectFeatures = (fsClass != null);
    if (selectFeatures) {
        if (featureSelector == null) {
            featureSelector = Trainable.<AbstractFeatureSelector>newInstance(fsClass, dbName, conf);
        }
        setParallelized(featureSelector);
        featureSelector.transform(newData);
    }
    if (modeler == null) {
        Class mlClass = trainingParameters.getModelerClass();
        modeler = Trainable.<AbstractModeler>newInstance(mlClass, dbName, conf);
    }
    setParallelized(modeler);
    modeler.predict(newData);
    if (transformData) {
        dataTransformer.denormalize(newData);
    }
}

public void predict(Dataframe newData) {
    logger.info("predict()");
    Modeler.TrainingParameters trainingParameters = knowledgeBase.getTrainingParameters();
    Configuration conf = knowledgeBase.getConf();
    AbstractTrainer.AbstractTrainingParameters dtParams = trainingParameters.getDataTransformerTrainingParameters();
    boolean transformData = dtParams != null;
    if (transformData) {
        if (dataTransformer == null) {
            dataTransformer = Trainable.newInstance(dtParams.getTClass(), dbName, conf);
        }
        setParallelized(dataTransformer);
        dataTransformer.transform(newData);
    }
    AbstractTrainer.AbstractTrainingParameters fsParams = trainingParameters.getFeatureSelectorTrainingParameters();
    boolean selectFeatures = fsParams != null;
    if (selectFeatures) {
        if (featureSelector == null) {
            featureSelector = Trainable.newInstance(fsParams.getTClass(), dbName, conf);
        }
        setParallelized(featureSelector);
        featureSelector.transform(newData);
    }
    if (modeler == null) {
        modeler = Trainable.newInstance(trainingParameters.getModelerTrainingParameters().getTClass(), dbName, conf);
    }
    setParallelized(modeler);
    modeler.predict(newData);
    if (transformData) {
        dataTransformer.denormalize(newData);
    }
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    Adaboost.TrainingParameters param = new Adaboost.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    param.setWeakClassifierClass(MultinomialNaiveBayes.class);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    ;
    double expResult = 0.6923992673992675;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    Adaboost.TrainingParameters param = new Adaboost.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    ;
    double expResult = 0.6923992673992675;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}

@Override
public VM validate(Dataframe dataset, TrainingParameters trainingParameters) {
    int n = dataset.size();
    if (k <= 0 || n <= k) {
        throw new IllegalArgumentException("Invalid number of folds.");
    }
    int foldSize = n / k;
    Integer[] ids = new Integer[n];
    int j = 0;
    for (Integer rId : dataset.index()) {
        ids[j] = rId;
        ++j;
    }
    PHPMethods.shuffle(ids);
    Class<? extends AbstractModeler> aClass = null;
    try {
        String className = trainingParameters.getClass().getCanonicalName();
        aClass = (Class<? extends AbstractModeler>) Class.forName(className.substring(0, className.lastIndexOf('.')));
    } catch (ClassNotFoundException e) {
        throw new IllegalArgumentException(e);
    }
    AbstractModeler modeler = Trainable.newInstance(aClass, "kfold_" + RandomGenerator.getThreadLocalRandomUnseeded().nextLong(), conf, trainingParameters);
    List<VM> validationMetricsList = new LinkedList<>();
    for (int fold = 0; fold < k; ++fold) {
        logger.info("Kfold {}", fold);
        FlatDataList foldTrainingIds = new FlatDataList(new ArrayList<>(n - foldSize));
        FlatDataList foldValidationIds = new FlatDataList(new ArrayList<>(foldSize));
        for (int i = 0; i < n; ++i) {
            boolean isInValidationFoldRange = false;
            if (fold * foldSize <= i && i < (fold + 1) * foldSize) {
                isInValidationFoldRange = true;
            }
            if (isInValidationFoldRange) {
                foldValidationIds.add(ids[i]);
            } else {
                foldTrainingIds.add(ids[i]);
            }
        }
        if (k == 1) {
            foldTrainingIds = foldValidationIds;
        }
        Dataframe trainingData = dataset.getSubset(foldTrainingIds);
        modeler.fit(trainingData);
        trainingData.delete();
        Dataframe validationData = dataset.getSubset(foldValidationIds);
        modeler.predict(validationData);
        VM entrySample = ValidationMetrics.newInstance(vmClass, validationData);
        validationData.delete();
        validationMetricsList.add(entrySample);
    }
    modeler.delete();
    VM avgValidationMetrics = ValidationMetrics.newInstance(vmClass, validationMetricsList);
    return avgValidationMetrics;
}

@Override
public VM validate(Dataframe dataset, TrainingParameters trainingParameters) {
    int n = dataset.size();
    if (k <= 0 || n <= k) {
        throw new IllegalArgumentException("Invalid number of folds.");
    }
    int foldSize = n / k;
    Integer[] ids = new Integer[n];
    int j = 0;
    for (Integer rId : dataset.index()) {
        ids[j] = rId;
        ++j;
    }
    PHPMethods.shuffle(ids);
    AbstractModeler modeler = Trainable.newInstance(trainingParameters, "kfold_" + RandomGenerator.getThreadLocalRandomUnseeded().nextLong(), conf);
    List<VM> validationMetricsList = new LinkedList<>();
    for (int fold = 0; fold < k; ++fold) {
        logger.info("Kfold {}", fold);
        FlatDataList foldTrainingIds = new FlatDataList(new ArrayList<>(n - foldSize));
        FlatDataList foldValidationIds = new FlatDataList(new ArrayList<>(foldSize));
        for (int i = 0; i < n; ++i) {
            boolean isInValidationFoldRange = false;
            if (fold * foldSize <= i && i < (fold + 1) * foldSize) {
                isInValidationFoldRange = true;
            }
            if (isInValidationFoldRange) {
                foldValidationIds.add(ids[i]);
            } else {
                foldTrainingIds.add(ids[i]);
            }
        }
        if (k == 1) {
            foldTrainingIds = foldValidationIds;
        }
        Dataframe trainingData = dataset.getSubset(foldTrainingIds);
        modeler.fit(trainingData);
        trainingData.delete();
        Dataframe validationData = dataset.getSubset(foldValidationIds);
        modeler.predict(validationData);
        VM entrySample = ValidationMetrics.newInstance(vmClass, validationData);
        validationData.delete();
        validationMetricsList.add(entrySample);
    }
    modeler.delete();
    VM avgValidationMetrics = ValidationMetrics.newInstance(vmClass, validationMetricsList);
    return avgValidationMetrics;
}

@Test
public void testValidate() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsCategorical(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    df.transform(validationData);
    Adaboost.TrainingParameters param = new Adaboost.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    param.setWeakClassifierClass(MultinomialNaiveBayes.class);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    Adaboost instance = new Adaboost(dbName, conf, param);
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = new DummyXYMinMaxNormalizer(dbName, conf);
    instance = new Adaboost(dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testValidate() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsCategorical(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    df.transform(validationData);
    Adaboost.TrainingParameters param = new Adaboost.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    Adaboost instance = new Adaboost(dbName, conf, param);
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = new DummyXYMinMaxNormalizer(dbName, conf);
    instance = new Adaboost(dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

public void setWeakClassifierClass(Class<? extends AbstractClassifier> weakClassifierClass) {
    this.weakClassifierClass = weakClassifierClass;
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    BootstrapAggregating.TrainingParameters param = new BootstrapAggregating.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    param.setWeakClassifierClass(MultinomialNaiveBayes.class);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 0.6609432234432234;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    BootstrapAggregating.TrainingParameters param = new BootstrapAggregating.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 0.6609432234432234;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}

@Test
public void testValidate() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsCategorical(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    df.transform(validationData);
    BootstrapAggregating.TrainingParameters param = new BootstrapAggregating.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    param.setWeakClassifierClass(MultinomialNaiveBayes.class);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    BootstrapAggregating instance = new BootstrapAggregating(dbName, conf, param);
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = new DummyXYMinMaxNormalizer(dbName, conf);
    instance = new BootstrapAggregating(dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testValidate() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsCategorical(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    df.transform(validationData);
    BootstrapAggregating.TrainingParameters param = new BootstrapAggregating.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    BootstrapAggregating instance = new BootstrapAggregating(dbName, conf, param);
    instance.fit(trainingData);
    instance.close();
    df.close();
    df = new DummyXYMinMaxNormalizer(dbName, conf);
    instance = new BootstrapAggregating(dbName, conf);
    instance.predict(validationData);
    df.denormalize(trainingData);
    df.denormalize(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}


@Test
public void testSelectFeatures() {
    logger.info("selectFeatures");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.featureSelectorCategorical(conf, 1000);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    ChisquareSelect.TrainingParameters param = new ChisquareSelect.TrainingParameters();
    param.setRareFeatureThreshold(2);
    param.setMaxFeatures(5);
    param.setIgnoringNumericalFeatures(false);
    param.setALevel(0.05);
    ChisquareSelect instance = new ChisquareSelect(dbName, conf, param);
    instance.fit_transform(trainingData);
    instance.close();
    instance = new ChisquareSelect(dbName, conf);
    instance.transform(validationData);
    Set<Object> expResult = new HashSet<>(Arrays.asList("high_paid", "has_boat", "has_luxury_car", "has_butler", "has_pool"));
    Set<Object> result = trainingData.getXDataTypes().keySet();
    assertEquals(expResult, result);
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testSelectFeatures() {
    logger.info("selectFeatures");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.featureSelectorCategorical(conf, 1000);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    ChisquareSelect.TrainingParameters param = new ChisquareSelect.TrainingParameters();
    param.setRareFeatureThreshold(2);
    param.setMaxFeatures(5);
    param.setALevel(0.05);
    ChisquareSelect instance = new ChisquareSelect(dbName, conf, param);
    instance.fit_transform(trainingData);
    instance.close();
    instance = new ChisquareSelect(dbName, conf);
    instance.transform(validationData);
    Set<Object> expResult = new HashSet<>(Arrays.asList("high_paid", "has_boat", "has_luxury_car", "has_butler", "has_pool"));
    Set<Object> result = trainingData.getXDataTypes().keySet();
    assertEquals(expResult, result);
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

public void setIgnoringNumericalFeatures(boolean ignoringNumericalFeatures) {
    this.ignoringNumericalFeatures = ignoringNumericalFeatures;
}


@Test
public void testSelectFeatures() {
    logger.info("selectFeatures");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.featureSelectorCategorical(conf, 1000);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    MutualInformation.TrainingParameters param = new MutualInformation.TrainingParameters();
    param.setRareFeatureThreshold(2);
    param.setMaxFeatures(5);
    param.setIgnoringNumericalFeatures(false);
    MutualInformation instance = new MutualInformation(dbName, conf, param);
    instance.fit_transform(trainingData);
    instance.close();
    instance = new MutualInformation(dbName, conf);
    instance.transform(validationData);
    Set<Object> expResult = new HashSet<>(Arrays.asList("high_paid", "has_boat", "has_luxury_car", "has_butler", "has_pool"));
    Set<Object> result = trainingData.getXDataTypes().keySet();
    assertEquals(expResult, result);
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testSelectFeatures() {
    logger.info("selectFeatures");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.featureSelectorCategorical(conf, 1000);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    MutualInformation.TrainingParameters param = new MutualInformation.TrainingParameters();
    param.setRareFeatureThreshold(2);
    param.setMaxFeatures(5);
    MutualInformation instance = new MutualInformation(dbName, conf, param);
    instance.fit_transform(trainingData);
    instance.close();
    instance = new MutualInformation(dbName, conf);
    instance.transform(validationData);
    Set<Object> expResult = new HashSet<>(Arrays.asList("high_paid", "has_boat", "has_luxury_car", "has_butler", "has_pool"));
    Set<Object> result = trainingData.getXDataTypes().keySet();
    assertEquals(expResult, result);
    instance.delete();
    trainingData.delete();
    validationData.delete();
}


@Test
public void testValidate() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.regressionNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    StepwiseRegression.TrainingParameters param = new StepwiseRegression.TrainingParameters();
    param.setAout(0.05);
    param.setRegressionClass(MatrixLinearRegression.class);
    MatrixLinearRegression.TrainingParameters trainingParams = new MatrixLinearRegression.TrainingParameters();
    param.setRegressionTrainingParameters(trainingParams);
    StepwiseRegression instance = new StepwiseRegression(dbName, conf, param);
    instance.fit(trainingData);
    df.denormalize(trainingData);
    instance.close();
    df.close();
    df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.transform(validationData);
    instance = new StepwiseRegression(dbName, conf);
    instance.predict(validationData);
    df.denormalize(validationData);
    for (Record r : validationData) {
        Assert.assertEquals(TypeInference.toDouble(r.getY()), TypeInference.toDouble(r.getYPredicted()), Constants.DOUBLE_ACCURACY_HIGH);
    }
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Test
public void testValidate() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.regressionNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf, new DummyXYMinMaxNormalizer.TrainingParameters());
    df.fit_transform(trainingData);
    StepwiseRegression.TrainingParameters param = new StepwiseRegression.TrainingParameters();
    param.setAout(0.05);
    MatrixLinearRegression.TrainingParameters trainingParams = new MatrixLinearRegression.TrainingParameters();
    param.setRegressionTrainingParameters(trainingParams);
    StepwiseRegression instance = new StepwiseRegression(dbName, conf, param);
    instance.fit(trainingData);
    df.denormalize(trainingData);
    instance.close();
    df.close();
    df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.transform(validationData);
    instance = new StepwiseRegression(dbName, conf);
    instance.predict(validationData);
    df.denormalize(validationData);
    for (Record r : validationData) {
        Assert.assertEquals(TypeInference.toDouble(r.getY()), TypeInference.toDouble(r.getYPredicted()), Constants.DOUBLE_ACCURACY_HIGH);
    }
    df.delete();
    instance.delete();
    trainingData.delete();
    validationData.delete();
}

@Override
public void delete() {
    if (mlregressor != null) {
        mlregressor.delete();
        mlregressor = null;
    }
    super.delete();
}

@Override
public void delete() {
    loadRegressor().delete();
    super.delete();
}


@Test
public void testTrainAndValidate() {
    logger.info("testTrainAndValidate");
    Configuration conf = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String dbName = this.getClass().getSimpleName();
    Modeler.TrainingParameters trainingParameters = new Modeler.TrainingParameters();
    MultinomialNaiveBayes.TrainingParameters modelTrainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    modelTrainingParameters.setMultiProbabilityWeighted(true);
    trainingParameters.setModelerTrainingParameters(modelTrainingParameters);
    DummyXMinMaxNormalizer.TrainingParameters dtParams = new DummyXMinMaxNormalizer.TrainingParameters();
    trainingParameters.setDataTransformerTrainingParameters(dtParams);
    trainingParameters.setFeatureSelectorTrainingParameters(null);
    Modeler instance = new Modeler(dbName, conf, trainingParameters);
    instance.fit(trainingData);
    instance.close();
    instance = new Modeler(dbName, conf);
    instance.predict(trainingData);
    ClassificationMetrics vm = new ClassificationMetrics(trainingData);
    double expResult2 = 0.8;
    assertEquals(expResult2, vm.getMacroF1(), Constants.DOUBLE_ACCURACY_HIGH);
    instance.close();
    instance = new Modeler(dbName, conf);
    instance.predict(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    instance.delete();
    trainingData.delete();
    validationData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("testKFoldCrossValidation");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    Adaboost.TrainingParameters param = new Adaboost.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    ;
    double expResult = 0.6923992673992675;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("testKFoldCrossValidation");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    BootstrapAggregating.TrainingParameters param = new BootstrapAggregating.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 0.6609432234432234;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    BernoulliNaiveBayes instance = new BernoulliNaiveBayes(dbName, conf);
    BernoulliNaiveBayes.TrainingParameters param = new BernoulliNaiveBayes.TrainingParameters();
    ClassificationMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    double expResult = 0.6631318681318682;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    BernoulliNaiveBayes.TrainingParameters param = new BernoulliNaiveBayes.TrainingParameters();
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 0.6631318681318682;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}

@Override
public VM validate(Dataframe dataset, TrainingParameters trainingParameters) {
    int n = dataset.size();
    if (k <= 0 || n <= k) {
        throw new IllegalArgumentException("Invalid number of folds.");
    }
    int foldSize = n / k;
    Integer[] ids = new Integer[n];
    int j = 0;
    for (Integer rId : dataset.index()) {
        ids[j] = rId;
        ++j;
    }
    PHPMethods.shuffle(ids);
    Class<? extends AbstractModeler> aClass = null;
    try {
        String className = trainingParameters.getClass().getCanonicalName();
        aClass = (Class<? extends AbstractModeler>) Class.forName(className.substring(0, className.lastIndexOf('.')));
    } catch (ClassNotFoundException e) {
        throw new IllegalArgumentException(e);
    }
    AbstractModeler modeler = Trainable.newInstance(aClass, "kfold_" + System.nanoTime(), conf);
    List<VM> validationMetricsList = new LinkedList<>();
    for (int fold = 0; fold < k; ++fold) {
        logger.info("Kfold {}", fold);
        FlatDataList foldTrainingIds = new FlatDataList(new ArrayList<>(n - foldSize));
        FlatDataList foldValidationIds = new FlatDataList(new ArrayList<>(foldSize));
        for (int i = 0; i < n; ++i) {
            boolean isInValidationFoldRange = false;
            if (fold * foldSize <= i && i < (fold + 1) * foldSize) {
                isInValidationFoldRange = true;
            }
            if (isInValidationFoldRange) {
                foldValidationIds.add(ids[i]);
            } else {
                foldTrainingIds.add(ids[i]);
            }
        }
        if (k == 1) {
            foldTrainingIds = foldValidationIds;
        }
        Dataframe trainingData = dataset.getSubset(foldTrainingIds);
        modeler.fit(trainingData, (AbstractTrainer.AbstractTrainingParameters) trainingParameters);
        trainingData.delete();
        Dataframe validationData = dataset.getSubset(foldValidationIds);
        modeler.predict(validationData);
        VM entrySample = ValidationMetrics.newInstance(vmClass, validationData);
        validationData.delete();
        validationMetricsList.add(entrySample);
    }
    modeler.delete();
    VM avgValidationMetrics = ValidationMetrics.newInstance(vmClass, validationMetricsList);
    return avgValidationMetrics;
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    BinarizedNaiveBayes instance = new BinarizedNaiveBayes(dbName, conf);
    BinarizedNaiveBayes.TrainingParameters param = new BinarizedNaiveBayes.TrainingParameters();
    ClassificationMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    double expResult = 0.6631318681318682;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    BinarizedNaiveBayes.TrainingParameters param = new BinarizedNaiveBayes.TrainingParameters();
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 0.6631318681318682;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    MaximumEntropy instance = new MaximumEntropy(dbName, conf);
    MaximumEntropy.TrainingParameters param = new MaximumEntropy.TrainingParameters();
    param.setTotalIterations(10);
    ClassificationMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    double expResult = 0.6051098901098901;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    MaximumEntropy.TrainingParameters param = new MaximumEntropy.TrainingParameters();
    param.setTotalIterations(10);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 0.6051098901098901;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    MultinomialNaiveBayes instance = new MultinomialNaiveBayes(dbName, conf);
    MultinomialNaiveBayes.TrainingParameters param = new MultinomialNaiveBayes.TrainingParameters();
    param.setMultiProbabilityWeighted(true);
    ClassificationMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    double expResult = 0.6631318681318682;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    MultinomialNaiveBayes.TrainingParameters param = new MultinomialNaiveBayes.TrainingParameters();
    param.setMultiProbabilityWeighted(true);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 0.6631318681318682;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.winesOrdinal(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXMinMaxNormalizer df = new DummyXMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXMinMaxNormalizer.TrainingParameters());
    OrdinalRegression instance = new OrdinalRegression(dbName, conf);
    OrdinalRegression.TrainingParameters param = new OrdinalRegression.TrainingParameters();
    param.setTotalIterations(100);
    param.setL2(0.001);
    ClassificationMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    df.denormalize(trainingData);
    double expResult = 0.9823403146614675;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.winesOrdinal(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXMinMaxNormalizer df = new DummyXMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXMinMaxNormalizer.TrainingParameters());
    OrdinalRegression.TrainingParameters param = new OrdinalRegression.TrainingParameters();
    param.setTotalIterations(100);
    param.setL2(0.001);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 0.9823403146614675;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    XMinMaxNormalizer df = new XMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new XMinMaxNormalizer.TrainingParameters());
    SoftMaxRegression instance = new SoftMaxRegression(dbName, conf);
    SoftMaxRegression.TrainingParameters param = new SoftMaxRegression.TrainingParameters();
    param.setTotalIterations(30);
    param.setL1(0.0001);
    param.setL2(0.0001);
    ClassificationMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    df.denormalize(trainingData);
    double expResult = 0.7557492507492508;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    XMinMaxNormalizer df = new XMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new XMinMaxNormalizer.TrainingParameters());
    SoftMaxRegression.TrainingParameters param = new SoftMaxRegression.TrainingParameters();
    param.setTotalIterations(30);
    param.setL1(0.0001);
    param.setL2(0.0001);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 0.7557492507492508;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    SupportVectorMachine instance = new SupportVectorMachine(dbName, conf);
    SupportVectorMachine.TrainingParameters param = new SupportVectorMachine.TrainingParameters();
    param.getSvmParameter().kernel_type = svm_parameter.LINEAR;
    ClassificationMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    double expResult = 0.6473992673992675;
    double result = vm.getMacroF1();
    Assert.assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    SupportVectorMachine.TrainingParameters param = new SupportVectorMachine.TrainingParameters();
    param.getSvmParameter().kernel_type = svm_parameter.LINEAR;
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 0.5861704961704961;
    double result = vm.getMacroF1();
    Assert.assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.gaussianClusters(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    GaussianDPMM instance = new GaussianDPMM(dbName, conf);
    GaussianDPMM.TrainingParameters param = new GaussianDPMM.TrainingParameters();
    param.setAlpha(0.01);
    param.setMaxIterations(100);
    param.setInitializationMethod(GaussianDPMM.TrainingParameters.Initialization.ONE_CLUSTER_PER_RECORD);
    param.setKappa0(0);
    param.setNu0(1);
    param.setMu0(new double[] { 0.0, 0.0 });
    param.setPsi0(new double[][] { { 1.0, 0.0 }, { 0.0, 1.0 } });
    ClusteringMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    double expResult = 1.0;
    double result = vm.getPurity();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.gaussianClusters(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    GaussianDPMM.TrainingParameters param = new GaussianDPMM.TrainingParameters();
    param.setAlpha(0.01);
    param.setMaxIterations(100);
    param.setInitializationMethod(GaussianDPMM.TrainingParameters.Initialization.ONE_CLUSTER_PER_RECORD);
    param.setKappa0(0);
    param.setNu0(1);
    param.setMu0(new double[] { 0.0, 0.0 });
    param.setPsi0(new double[][] { { 1.0, 0.0 }, { 0.0, 1.0 } });
    ClusteringMetrics vm = new KFoldValidator<>(ClusteringMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 1.0;
    double result = vm.getPurity();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.heartDiseaseClusters(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXYMinMaxNormalizer.TrainingParameters());
    HierarchicalAgglomerative instance = new HierarchicalAgglomerative(dbName, conf);
    HierarchicalAgglomerative.TrainingParameters param = new HierarchicalAgglomerative.TrainingParameters();
    param.setDistanceMethod(HierarchicalAgglomerative.TrainingParameters.Distance.EUCLIDIAN);
    param.setLinkageMethod(HierarchicalAgglomerative.TrainingParameters.Linkage.COMPLETE);
    param.setMinClustersThreshold(2);
    param.setMaxDistanceThreshold(Double.MAX_VALUE);
    ClusteringMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    df.denormalize(trainingData);
    double expResult = 0.7666666666666667;
    double result = vm.getPurity();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.heartDiseaseClusters(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXYMinMaxNormalizer.TrainingParameters());
    HierarchicalAgglomerative.TrainingParameters param = new HierarchicalAgglomerative.TrainingParameters();
    param.setDistanceMethod(HierarchicalAgglomerative.TrainingParameters.Distance.EUCLIDIAN);
    param.setLinkageMethod(HierarchicalAgglomerative.TrainingParameters.Linkage.COMPLETE);
    param.setMinClustersThreshold(2);
    param.setMaxDistanceThreshold(Double.MAX_VALUE);
    ClusteringMetrics vm = new KFoldValidator<>(ClusteringMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 0.7666666666666667;
    double result = vm.getPurity();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.heartDiseaseClusters(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXYMinMaxNormalizer.TrainingParameters());
    Kmeans instance = new Kmeans(dbName, conf);
    Kmeans.TrainingParameters param = new Kmeans.TrainingParameters();
    param.setK(2);
    param.setMaxIterations(200);
    param.setInitializationMethod(Kmeans.TrainingParameters.Initialization.FORGY);
    param.setDistanceMethod(Kmeans.TrainingParameters.Distance.EUCLIDIAN);
    param.setWeighted(false);
    param.setCategoricalGamaMultiplier(1.0);
    param.setSubsetFurthestFirstcValue(2.0);
    ClusteringMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    df.denormalize(trainingData);
    double expResult = 0.7888888888888889;
    double result = vm.getPurity();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.heartDiseaseClusters(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXYMinMaxNormalizer.TrainingParameters());
    Kmeans.TrainingParameters param = new Kmeans.TrainingParameters();
    param.setK(2);
    param.setMaxIterations(200);
    param.setInitializationMethod(Kmeans.TrainingParameters.Initialization.FORGY);
    param.setDistanceMethod(Kmeans.TrainingParameters.Distance.EUCLIDIAN);
    param.setWeighted(false);
    param.setCategoricalGamaMultiplier(1.0);
    param.setSubsetFurthestFirstcValue(2.0);
    ClusteringMetrics vm = new KFoldValidator<>(ClusteringMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 0.7888888888888889;
    double result = vm.getPurity();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.multinomialClusters(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    MultinomialDPMM instance = new MultinomialDPMM(dbName, conf);
    MultinomialDPMM.TrainingParameters param = new MultinomialDPMM.TrainingParameters();
    param.setAlpha(0.01);
    param.setMaxIterations(100);
    param.setInitializationMethod(MultinomialDPMM.TrainingParameters.Initialization.ONE_CLUSTER_PER_RECORD);
    param.setAlphaWords(1);
    ClusteringMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    double expResult = 1.0;
    double result = vm.getPurity();
    Assert.assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.multinomialClusters(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    MultinomialDPMM.TrainingParameters param = new MultinomialDPMM.TrainingParameters();
    param.setAlpha(0.01);
    param.setMaxIterations(100);
    param.setInitializationMethod(MultinomialDPMM.TrainingParameters.Initialization.ONE_CLUSTER_PER_RECORD);
    param.setAlphaWords(1);
    ClusteringMetrics vm = new KFoldValidator<>(ClusteringMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 1.0;
    double result = vm.getPurity();
    Assert.assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    Adaboost instance = new Adaboost(dbName, conf);
    Adaboost.TrainingParameters param = new Adaboost.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    param.setWeakClassifierClass(MultinomialNaiveBayes.class);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    double expResult = 0.6923992673992675;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    Adaboost.TrainingParameters param = new Adaboost.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    param.setWeakClassifierClass(MultinomialNaiveBayes.class);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    ;
    double expResult = 0.6923992673992675;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    BootstrapAggregating instance = new BootstrapAggregating(dbName, conf);
    BootstrapAggregating.TrainingParameters param = new BootstrapAggregating.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    param.setWeakClassifierClass(MultinomialNaiveBayes.class);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    double expResult = 0.6609432234432234;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.carsNumeric(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    BootstrapAggregating.TrainingParameters param = new BootstrapAggregating.TrainingParameters();
    param.setMaxWeakClassifiers(5);
    param.setWeakClassifierClass(MultinomialNaiveBayes.class);
    MultinomialNaiveBayes.TrainingParameters trainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    trainingParameters.setMultiProbabilityWeighted(true);
    param.setWeakClassifierTrainingParameters(trainingParameters);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, k).validate(trainingData, param);
    double expResult = 0.6609432234432234;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.regressionMixed(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXYMinMaxNormalizer.TrainingParameters());
    MatrixLinearRegression instance = new MatrixLinearRegression(dbName, conf);
    MatrixLinearRegression.TrainingParameters param = new MatrixLinearRegression.TrainingParameters();
    LinearRegressionMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    df.denormalize(trainingData);
    double expResult = 1;
    double result = vm.getRSquare();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.regressionMixed(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXYMinMaxNormalizer.TrainingParameters());
    MatrixLinearRegression.TrainingParameters param = new MatrixLinearRegression.TrainingParameters();
    LinearRegressionMetrics vm = new KFoldValidator<>(LinearRegressionMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 1;
    double result = vm.getRSquare();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}


@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.housingNumerical(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXYMinMaxNormalizer.TrainingParameters());
    PCA featureSelector = new PCA(dbName, conf);
    PCA.TrainingParameters featureSelectorParameters = new PCA.TrainingParameters();
    featureSelectorParameters.setMaxDimensions(trainingData.xColumnSize() - 1);
    featureSelectorParameters.setWhitened(false);
    featureSelectorParameters.setVariancePercentageThreshold(0.99999995);
    featureSelector.fit_transform(trainingData, featureSelectorParameters);
    featureSelector.delete();
    NLMS instance = new NLMS(dbName, conf);
    NLMS.TrainingParameters param = new NLMS.TrainingParameters();
    param.setTotalIterations(500);
    param.setL1(0.001);
    param.setL2(0.001);
    LinearRegressionMetrics vm = instance.kFoldCrossValidation(trainingData, param, k);
    df.denormalize(trainingData);
    double expResult = 0.7748106446239166;
    double result = vm.getRSquare();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    instance.delete();
    trainingData.delete();
}

@Test
public void testKFoldCrossValidation() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    int k = 5;
    Dataframe[] data = Datasets.housingNumerical(conf);
    Dataframe trainingData = data[0];
    data[1].delete();
    String dbName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = new DummyXYMinMaxNormalizer(dbName, conf);
    df.fit_transform(trainingData, new DummyXYMinMaxNormalizer.TrainingParameters());
    PCA featureSelector = new PCA(dbName, conf);
    PCA.TrainingParameters featureSelectorParameters = new PCA.TrainingParameters();
    featureSelectorParameters.setMaxDimensions(trainingData.xColumnSize() - 1);
    featureSelectorParameters.setWhitened(false);
    featureSelectorParameters.setVariancePercentageThreshold(0.99999995);
    featureSelector.fit_transform(trainingData, featureSelectorParameters);
    featureSelector.delete();
    NLMS.TrainingParameters param = new NLMS.TrainingParameters();
    param.setTotalIterations(500);
    param.setL1(0.001);
    param.setL2(0.001);
    LinearRegressionMetrics vm = new KFoldValidator<>(LinearRegressionMetrics.class, conf, k).validate(trainingData, param);
    df.denormalize(trainingData);
    double expResult = 0.7748106446239166;
    double result = vm.getRSquare();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    df.delete();
    trainingData.delete();
}


@Test
public void testValidate() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    String dbName = this.getClass().getSimpleName();
    Map<Object, URI> dataset = new HashMap<>();
    try {
        dataset.put("negative", this.getClass().getClassLoader().getResource("datasets/sentimentAnalysis.neg.txt").toURI());
        dataset.put("positive", this.getClass().getClassLoader().getResource("datasets/sentimentAnalysis.pos.txt").toURI());
    } catch (UncheckedIOException | URISyntaxException ex) {
        logger.warn("Unable to download datasets, skipping test.");
        throw new RuntimeException(ex);
    }
    UniqueWordSequenceExtractor wsExtractor = new UniqueWordSequenceExtractor(new UniqueWordSequenceExtractor.Parameters());
    Dataframe trainingData = Dataframe.Builder.parseTextFiles(dataset, wsExtractor, conf);
    LatentDirichletAllocation lda = new LatentDirichletAllocation(dbName, conf);
    LatentDirichletAllocation.TrainingParameters trainingParameters = new LatentDirichletAllocation.TrainingParameters();
    trainingParameters.setMaxIterations(15);
    trainingParameters.setAlpha(0.01);
    trainingParameters.setBeta(0.01);
    trainingParameters.setK(25);
    lda.fit(trainingData, trainingParameters);
    lda.predict(trainingData);
    Dataframe reducedTrainingData = new Dataframe(conf);
    for (Record r : trainingData) {
        reducedTrainingData.add(new Record(r.getYPredictedProbabilities(), r.getY()));
    }
    SoftMaxRegression smr = new SoftMaxRegression(dbName, conf);
    SoftMaxRegression.TrainingParameters tp = new SoftMaxRegression.TrainingParameters();
    tp.setLearningRate(1.0);
    tp.setTotalIterations(50);
    ClassificationMetrics vm = smr.kFoldCrossValidation(reducedTrainingData, tp, 1);
    double expResult = 0.6843125117743629;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    smr.delete();
    lda.delete();
    reducedTrainingData.delete();
    trainingData.delete();
}

@Test
public void testValidate() {
    logger.info("validate");
    Configuration conf = Configuration.getConfiguration();
    String dbName = this.getClass().getSimpleName();
    Map<Object, URI> dataset = new HashMap<>();
    try {
        dataset.put("negative", this.getClass().getClassLoader().getResource("datasets/sentimentAnalysis.neg.txt").toURI());
        dataset.put("positive", this.getClass().getClassLoader().getResource("datasets/sentimentAnalysis.pos.txt").toURI());
    } catch (UncheckedIOException | URISyntaxException ex) {
        logger.warn("Unable to download datasets, skipping test.");
        throw new RuntimeException(ex);
    }
    UniqueWordSequenceExtractor wsExtractor = new UniqueWordSequenceExtractor(new UniqueWordSequenceExtractor.Parameters());
    Dataframe trainingData = Dataframe.Builder.parseTextFiles(dataset, wsExtractor, conf);
    LatentDirichletAllocation lda = new LatentDirichletAllocation(dbName, conf);
    LatentDirichletAllocation.TrainingParameters trainingParameters = new LatentDirichletAllocation.TrainingParameters();
    trainingParameters.setMaxIterations(15);
    trainingParameters.setAlpha(0.01);
    trainingParameters.setBeta(0.01);
    trainingParameters.setK(25);
    lda.fit(trainingData, trainingParameters);
    lda.predict(trainingData);
    Dataframe reducedTrainingData = new Dataframe(conf);
    for (Record r : trainingData) {
        reducedTrainingData.add(new Record(r.getYPredictedProbabilities(), r.getY()));
    }
    SoftMaxRegression.TrainingParameters tp = new SoftMaxRegression.TrainingParameters();
    tp.setLearningRate(1.0);
    tp.setTotalIterations(50);
    ClassificationMetrics vm = new KFoldValidator<>(ClassificationMetrics.class, conf, 1).validate(reducedTrainingData, tp);
    double expResult = 0.6843125117743629;
    double result = vm.getMacroF1();
    assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH);
    lda.delete();
    reducedTrainingData.delete();
    trainingData.delete();
}


@Test
public void testTrainAndValidate() {
    logger.info("testTrainAndValidate");
    Configuration configuration = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsNumeric(configuration);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String storageName = this.getClass().getSimpleName();
    Modeler.TrainingParameters trainingParameters = new Modeler.TrainingParameters();
    MultinomialNaiveBayes.TrainingParameters modelTrainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    modelTrainingParameters.setMultiProbabilityWeighted(true);
    trainingParameters.setModelerTrainingParameters(modelTrainingParameters);
    DummyXMinMaxNormalizer.TrainingParameters dtParams = new DummyXMinMaxNormalizer.TrainingParameters();
    trainingParameters.setDataTransformerTrainingParameters(dtParams);
    trainingParameters.setFeatureSelectorTrainingParameters(null);
    Modeler instance = MLBuilder.create(trainingParameters, configuration);
    instance.fit(trainingData);
    instance.save(storageName);
    instance.close();
    instance = MLBuilder.load(Modeler.class, storageName, configuration);
    instance.predict(trainingData);
    ClassificationMetrics vm = new ClassificationMetrics(trainingData);
    double expResult2 = 0.8;
    assertEquals(expResult2, vm.getMacroF1(), Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.close();
    instance.close();
    instance = MLBuilder.load(Modeler.class, storageName, configuration);
    instance.predict(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    instance.delete();
    validationData.close();
}

@Test
public void testTrainAndValidate() {
    logger.info("testTrainAndValidate");
    Configuration configuration = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsNumeric(configuration);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String storageName = this.getClass().getSimpleName();
    Modeler.TrainingParameters trainingParameters = new Modeler.TrainingParameters();
    MinMaxScaler.TrainingParameters nsParams = new MinMaxScaler.TrainingParameters();
    trainingParameters.setNumericalScalerTrainingParameters(nsParams);
    CornerConstraintsEncoder.TrainingParameters ceParams = new CornerConstraintsEncoder.TrainingParameters();
    trainingParameters.setCategoricalEncoderTrainingParameters(ceParams);
    trainingParameters.setFeatureSelectorTrainingParameters(null);
    MultinomialNaiveBayes.TrainingParameters modelTrainingParameters = new MultinomialNaiveBayes.TrainingParameters();
    modelTrainingParameters.setMultiProbabilityWeighted(true);
    trainingParameters.setModelerTrainingParameters(modelTrainingParameters);
    Modeler instance = MLBuilder.create(trainingParameters, configuration);
    instance.fit(trainingData);
    instance.save(storageName);
    instance.close();
    instance = MLBuilder.load(Modeler.class, storageName, configuration);
    instance.predict(trainingData);
    ClassificationMetrics vm = new ClassificationMetrics(trainingData);
    double expResult2 = 0.8;
    assertEquals(expResult2, vm.getMacroF1(), Constants.DOUBLE_ACCURACY_HIGH);
    trainingData.close();
    instance.close();
    instance = MLBuilder.load(Modeler.class, storageName, configuration);
    instance.predict(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    instance.delete();
    validationData.close();
}

public void predict(Dataframe newData) {
    logger.info("predict()");
    initBundle();
    bundle.setParallelized(isParallelized());
    AbstractTransformer dataTransformer = (AbstractTransformer) bundle.get(DT_KEY);
    if (dataTransformer != null) {
        dataTransformer.transform(newData);
    }
    AbstractFeatureSelector featureSelector = (AbstractFeatureSelector) bundle.get(FS_KEY);
    if (featureSelector != null) {
        featureSelector.transform(newData);
    }
    AbstractModeler modeler = (AbstractModeler) bundle.get(ML_KEY);
    modeler.predict(newData);
    if (dataTransformer != null) {
        dataTransformer.denormalize(newData);
    }
}


@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration configuration = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsCategorical(configuration);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String storageName = this.getClass().getSimpleName();
    DummyXYMinMaxNormalizer df = MLBuilder.create(new DummyXYMinMaxNormalizer.TrainingParameters(), configuration);
    df.fit_transform(trainingData);
    df.save(storageName);
    MultinomialNaiveBayes.TrainingParameters param = new MultinomialNaiveBayes.TrainingParameters();
    param.setMultiProbabilityWeighted(true);
    MultinomialNaiveBayes instance = MLBuilder.create(param, configuration);
    instance.fit(trainingData);
    instance.save(storageName);
    df.denormalize(trainingData);
    trainingData.close();
    instance.close();
    df.close();
    df = MLBuilder.load(DummyXYMinMaxNormalizer.class, storageName, configuration);
    instance = MLBuilder.load(MultinomialNaiveBayes.class, storageName, configuration);
    df.transform(validationData);
    instance.predict(validationData);
    df.denormalize(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    df.delete();
    instance.delete();
    validationData.close();
}

@Test
public void testPredict() {
    logger.info("testPredict");
    Configuration configuration = Configuration.getConfiguration();
    Dataframe[] data = Datasets.carsCategorical(configuration);
    Dataframe trainingData = data[0];
    Dataframe validationData = data[1];
    String storageName = this.getClass().getSimpleName();
    MinMaxScaler.TrainingParameters nsParams = new MinMaxScaler.TrainingParameters();
    nsParams.setScaleResponse(true);
    MinMaxScaler numericalScaler = MLBuilder.create(nsParams, configuration);
    numericalScaler.fit_transform(trainingData);
    numericalScaler.save(storageName);
    CornerConstraintsEncoder.TrainingParameters ceParams = new CornerConstraintsEncoder.TrainingParameters();
    CornerConstraintsEncoder categoricalEncoder = MLBuilder.create(ceParams, configuration);
    categoricalEncoder.fit_transform(trainingData);
    categoricalEncoder.save(storageName);
    MultinomialNaiveBayes.TrainingParameters param = new MultinomialNaiveBayes.TrainingParameters();
    param.setMultiProbabilityWeighted(true);
    MultinomialNaiveBayes instance = MLBuilder.create(param, configuration);
    instance.fit(trainingData);
    instance.save(storageName);
    trainingData.close();
    instance.close();
    numericalScaler.close();
    categoricalEncoder.close();
    numericalScaler = MLBuilder.load(MinMaxScaler.class, storageName, configuration);
    categoricalEncoder = MLBuilder.load(CornerConstraintsEncoder.class, storageName, configuration);
    instance = MLBuilder.load(MultinomialNaiveBayes.class, storageName, configuration);
    numericalScaler.transform(validationData);
    categoricalEncoder.transform(validationData);
    instance.predict(validationData);
    Map<Integer, Object> expResult = new HashMap<>();
    Map<Integer, Object> result = new HashMap<>();
    for (Map.Entry<Integer, Record> e : validationData.entries()) {
        Integer rId = e.getKey();
        Record r = e.getValue();
        expResult.put(rId, r.getY());
        result.put(rId, r.getYPredicted());
    }
    assertEquals(expResult, result);
    numericalScaler.delete();
    categoricalEncoder.delete();
    instance.delete();
    validationData.close();
}

public void fit_transform(Dataframe trainingData) {
    fit(trainingData);
    transform(trainingData);
}