001/**
002 * DetailedHoldOutEvaluator.java
003 * jCOLIBRI2 framework. 
004 * @author Juan A. Recio-Garc�a.
005 * @author Lisa Cummins.
006 * GAIA - Group for Artificial Intelligence Applications
007 * http://gaia.fdi.ucm.es
008 * 23/07/2007
009 */
010package es.ucm.fdi.gaia.jcolibri.extensions.maintenance_evaluation.evaluators;
011
012import org.apache.logging.log4j.LogManager;
013
014import es.ucm.fdi.gaia.jcolibri.casebase.CachedLinealCaseBase;
015import es.ucm.fdi.gaia.jcolibri.cbrcore.CBRCase;
016import es.ucm.fdi.gaia.jcolibri.cbrcore.CBRCaseBase;
017import es.ucm.fdi.gaia.jcolibri.extensions.maintenance_evaluation.MaintenanceEvaluator;
018import es.ucm.fdi.gaia.jcolibri.util.ProgressController;
019
020import java.util.ArrayList;
021import java.util.Date;
022import java.util.List;
023
024/**
025 * This evaluation splits the case base in two sets: a training set and a test set. 
026 * The training set is used as a case-base. It is maintained and then the cases in the
027 * test set are used as queries to evaluate the training set. 
028 * This process is performed serveral times.
029 * 
030 * @author Lisa Cummins.
031 * @author Juan A. Recio Garc�a - GAIA http://gaia.fdi.ucm.es
032 */
033public class MaintenanceHoldOutEvaluator extends MaintenanceEvaluator
034{
035        /**
036         * Performs the Hold-Out evaluation. 
037         * @param testPercent percentage of the case base used as queries. 
038         * The case base is split randomly in each repetition.
039         * @param repetitions number of repetitions. 
040         */
041        public void HoldOut(int testPercent, int repetitions) {
042                try {
043                        // Obtain the time
044                        long t = (new Date()).getTime();
045                        int numberOfCycles = 0;
046
047                        // Run the precycle to load the case base
048                        LogManager.getLogger().info("Running precycle()");
049                        CBRCaseBase caseBase = app.preCycle();
050
051                        if (!(caseBase instanceof CachedLinealCaseBase))
052                                LogManager.getLogger().warn("Evaluation should be executed using a cached case base");
053
054                        List<CBRCase> originalCases = new ArrayList<CBRCase>(
055                                        caseBase.getCases());
056                        
057                        int totalSteps = ((originalCases.size() * testPercent) / 100);
058                        totalSteps = totalSteps*repetitions;
059                        ProgressController.init(getClass(),"Hold Out Evaluation", totalSteps);
060
061                        // For each repetition
062                        for (int rep = 0; rep < repetitions; rep++) 
063                        {       List<CBRCase> querySet = new ArrayList<CBRCase>();
064                                prepareCases(originalCases, querySet, testPercent, caseBase);
065                                
066                                // Run cycle for each case in querySet
067                                for (CBRCase c : querySet) 
068                                {       // Run the cycle
069                                        LogManager.getLogger().info("Running cycle() " + numberOfCycles);
070                        
071                                        app.cycle(c);
072                                        
073                                        ProgressController.step(getClass());
074                                        numberOfCycles++;
075                                }
076                        }
077
078                        ProgressController.finish(getClass());
079                        
080                        //Revert case base to original state
081                        caseBase.forgetCases(originalCases);
082                        caseBase.learnCases(originalCases);
083                        
084                        // Run the poscycle to finish the application
085                        LogManager.getLogger().info("Running postcycle()");
086                        app.postCycle();
087
088                        t = (new Date()).getTime() - t;
089
090                        // Obtain and complete the evaluation result
091                        report.setTotalTime(t);
092                        report.setNumberOfCycles(numberOfCycles);
093
094                } catch (Exception e) {
095                        System.out.println(e);
096                }
097        }
098        
099        /**
100         * Prepares the cases for evaluation by setting up test and training sets
101         * @param originalCases Complete original set of cases
102         * @param querySet Where queries are to be stored
103         * @param testPercent Percentage of cases used as queries
104         * @param caseBase The case base
105         */
106        protected void prepareCases(List<CBRCase> originalCases, List<CBRCase> querySet,
107                int testPercent, CBRCaseBase caseBase)
108        {       
109                ArrayList<CBRCase> caseBaseSet = new ArrayList<CBRCase>();
110
111                // Split the case base
112                splitCaseBase(originalCases, querySet, caseBaseSet, testPercent);
113
114                // Clear the caseBase
115                caseBase.forgetCases(originalCases);
116
117                // Set the cases that acts as case base in this repetition
118                caseBase.learnCases(caseBaseSet);
119                
120                if(this.simConfig != null && this.editMethod != null)
121                {       // Perform maintenance on this case base
122                        editCaseBase(caseBase);
123                }
124        }
125
126        /**
127         * Splits the case base in two sets: queries and case base
128         * 
129         * @param wholeCaseBase
130         *            Complete original case base
131         * @param querySet
132         *            Output param where queries are stored
133         * @param casebaseSet
134         *            Output param where case base is stored
135         * @param testPercent
136         *            Percentage of cases used as queries
137         */
138        public static void splitCaseBase(List<CBRCase> wholeCaseBase,
139                List<CBRCase> querySet, List<CBRCase> casebaseSet, int testPercent) 
140        {       querySet.clear();
141                casebaseSet.clear();
142
143                int querySetSize = (wholeCaseBase.size() * testPercent) / 100;
144                casebaseSet.addAll(wholeCaseBase);
145
146                for (int i = 0; i < querySetSize; i++) 
147                {       int random = (int) (Math.random() * casebaseSet.size());
148                        CBRCase _case = casebaseSet.get(random);
149                        casebaseSet.remove(random);
150                        querySet.add(_case);
151                }
152        }
153}