001package es.ucm.fdi.gaia.jcolibri.evaluation.evaluators;
002
003import org.apache.logging.log4j.LogManager;
004
005import es.ucm.fdi.gaia.jcolibri.casebase.CachedLinealCaseBase;
006import es.ucm.fdi.gaia.jcolibri.cbraplications.StandardCBRApplication;
007import es.ucm.fdi.gaia.jcolibri.cbrcore.CBRCase;
008import es.ucm.fdi.gaia.jcolibri.cbrcore.CBRCaseBase;
009import es.ucm.fdi.gaia.jcolibri.evaluation.EvaluationReport;
010import es.ucm.fdi.gaia.jcolibri.evaluation.Evaluator;
011import es.ucm.fdi.gaia.jcolibri.exception.ExecutionException;
012import es.ucm.fdi.gaia.jcolibri.util.ProgressController;
013
014import java.util.ArrayList;
015import java.util.Collection;
016import java.util.Date;
017import java.util.List;
018
019/**
020 * This method splits the case base in two sets: one used for testing where each
021 * case is used as query, and another that acts as normal case base. This
022 * process is performed serveral times.
023 * 
024 * @author Juan A. Recio GarcĂ­a - GAIA http://gaia.fdi.ucm.es
025 * @version 2.0
026 */
027public class HoldOutEvaluator extends Evaluator {
028
029        protected StandardCBRApplication app;
030
031        public void init(StandardCBRApplication cbrApp) {
032                
033                report = new EvaluationReport();
034                app = cbrApp;
035                try {
036                        app.configure();
037                } catch (ExecutionException e) {
038                        LogManager.getLogger().error(e);
039                }
040        }
041
042        /**
043         * Performs the Hold-Out evaluation. 
044         * @param testPercent Percent of the case base used as queries. The case base is splited randomly in each repetition.
045         * @param repetitions Number of repetitions. 
046         */
047        public void HoldOut(int testPercent, int repetitions) {
048
049                try {
050
051                        // Obtain the time
052                        long t = (new Date()).getTime();
053                        int numberOfCycles = 0;
054
055                        // Run the precycle to load the case base
056                        LogManager.getLogger().info("Running precycle()");
057                        CBRCaseBase caseBase = app.preCycle();
058
059                        if (!(caseBase instanceof CachedLinealCaseBase))
060                                LogManager.getLogger().warn("Evaluation should be executed using a cached case base");
061
062                        List<CBRCase> originalCases = new ArrayList<>(caseBase.getCases());
063                        
064                        int totalSteps = ((originalCases.size() * testPercent) / 100);
065                        totalSteps = totalSteps*repetitions;
066                        ProgressController.init(getClass(),"Hold Out Evaluation", totalSteps);
067                        
068                        // For each repetition
069                        for (int rep = 0; rep < repetitions; rep++) {
070
071                                List<CBRCase> querySet = new ArrayList<>();
072                                List<CBRCase> caseBaseSet = new ArrayList<>();
073
074                                // Split the case base
075                                splitCaseBase(originalCases, querySet, caseBaseSet, testPercent);
076
077                                // Clear the caseBase
078                                caseBase.forgetCases(originalCases);
079
080                                // Set the cases that acts as case base in this repetition
081                                caseBase.learnCases(caseBaseSet);
082
083                                // Run cycle for each case in querySet
084                                for (CBRCase c : querySet) {
085                                        // Run the cycle
086                                        LogManager.getLogger().info("Running cycle() " + numberOfCycles);
087                                        app.cycle(c);
088
089                                        ProgressController.step(getClass());
090                                        numberOfCycles++;
091                                }
092                        }
093
094                        ProgressController.finish(getClass());
095                        
096                        //Revert case base to original state
097                        caseBase.forgetCases(originalCases);
098                        caseBase.learnCases(originalCases);
099                        
100                        // Run the poscycle to finish the application
101                        LogManager.getLogger().info("Running postcycle()");
102                        app.postCycle();
103
104                        t = (new Date()).getTime() - t;
105
106                        // Obtain and complete the evaluation result
107                        report.setTotalTime(t);
108                        report.setNumberOfCycles(numberOfCycles);
109
110                } catch (Exception e) {
111                        LogManager.getLogger().error(e);
112                }
113
114        }
115
116        /**
117         * Splits the case base in two sets: queries and case base
118         * @param holeCaseBase Complete original case base
119         * @param querySet Output param where queries are stored
120         * @param casebaseSet Output param where case base is stored
121         * @param testPercent Percentage of cases used as queries
122         */
123        protected void splitCaseBase(Collection<CBRCase> holeCaseBase, List<CBRCase> querySet, List<CBRCase> casebaseSet, int testPercent) {
124                querySet.clear();
125                casebaseSet.clear();
126
127                int querySetSize = (holeCaseBase.size() * testPercent) / 100;
128                casebaseSet.addAll(holeCaseBase);
129
130                for (int i = 0; i < querySetSize; i++) {
131                        int random = (int) (Math.random() * casebaseSet.size());
132                        CBRCase _case = casebaseSet.get(random);
133                        casebaseSet.remove(random);
134                        querySet.add(_case);
135                }
136        }
137
138}