001/*
002 * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016
017package org.tribuo.classification.xgboost;
018
019import com.oracle.labs.mlrg.olcut.config.Config;
020import com.oracle.labs.mlrg.olcut.config.PropertyException;
021import com.oracle.labs.mlrg.olcut.provenance.Provenance;
022import org.tribuo.Dataset;
023import org.tribuo.ImmutableFeatureMap;
024import org.tribuo.ImmutableOutputInfo;
025import org.tribuo.Trainer;
026import org.tribuo.classification.Label;
027import org.tribuo.common.xgboost.XGBoostModel;
028import org.tribuo.common.xgboost.XGBoostTrainer;
029import org.tribuo.provenance.ModelProvenance;
030import org.tribuo.provenance.TrainerProvenance;
031import org.tribuo.provenance.impl.TrainerProvenanceImpl;
032import ml.dmlc.xgboost4j.java.Booster;
033import ml.dmlc.xgboost4j.java.XGBoost;
034import ml.dmlc.xgboost4j.java.XGBoostError;
035
036import java.time.OffsetDateTime;
037import java.util.Collections;
038import java.util.Map;
039import java.util.function.Function;
040import java.util.logging.Level;
041import java.util.logging.Logger;
042
043/**
044 * A {@link Trainer} which wraps the XGBoost training procedure.
045 * <p>
046 * This only exposes a few of XGBoost's training parameters.
047 * <p>
048 * It uses pthreads outside of the JVM to parallelise the computation.
049 * <p>
050 * See:
051 * <pre>
052 * Chen T, Guestrin C.
053 * "XGBoost: A Scalable Tree Boosting System"
054 * Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, 2016.
055 * </pre>
056 * and for the original algorithm:
057 * <pre>
058 * Friedman JH.
059 * "Greedy Function Approximation: a Gradient Boosting Machine"
060 * Annals of statistics, 2001.
061 * </pre>
062 * <p>
063 * N.B.: XGBoost4J wraps the native C implementation of xgboost that links to various C libraries, including libgomp
064 * and glibc (on Linux). If you're running on Alpine, which does not natively use glibc, you'll need to install glibc
065 * into the container.
066 * On the macOS binary on Maven Central is compiled without
067 * OpenMP support, meaning that XGBoost is single threaded on macOS. You can recompile the macOS binary with
068 * OpenMP support after installing libomp from homebrew if necessary.
069 */
070public final class XGBoostClassificationTrainer extends XGBoostTrainer<Label> {
071
072    private static final Logger logger = Logger.getLogger(XGBoostClassificationTrainer.class.getName());
073
074    @Config(description="Evaluation metric to use. The default value is set based on the objective function, so this can be usually left blank.")
075    private String evalMetric = "";
076
077    /**
078     * Create an XGBoost trainer.
079     *
080     * @param numTrees Number of trees to boost.
081     */
082    public XGBoostClassificationTrainer(int numTrees) {
083        this(numTrees, 0.3, 0, 6, 1, 1, 1, 1, 0, 4, true, Trainer.DEFAULT_SEED);
084    }
085
086    /**
087     * Create an XGBoost trainer.
088     *
089     * @param numTrees Number of trees to boost.
090     * @param numThreads Number of threads to use.
091     * @param silent Silence the training output text.
092     */
093    public XGBoostClassificationTrainer(int numTrees, int numThreads, boolean silent) {
094        this(numTrees, 0.3, 0, 6, 1, 1, 1, 1, 0, numThreads, silent, Trainer.DEFAULT_SEED);
095    }
096
097    /**
098     * Create an XGBoost trainer.
099     *
100     * @param numTrees Number of trees to boost.
101     * @param eta Step size shrinkage parameter (default 0.3, range [0,1]).
102     * @param gamma Minimum loss reduction to make a split (default 0, range
103     * [0,inf]).
104     * @param maxDepth Maximum tree depth (default 6, range [1,inf]).
105     * @param minChildWeight Minimum sum of instance weights needed in a leaf
106     * (default 1, range [0, inf]).
107     * @param subsample Subsample size for each tree (default 1, range (0,1]).
108     * @param featureSubsample Subsample features for each tree (default 1,
109     * range (0,1]).
110     * @param lambda L2 regularization term on weights (default 1).
111     * @param alpha L1 regularization term on weights (default 0).
112     * @param nThread Number of threads to use (default 4).
113     * @param silent Silence the training output text.
114     * @param seed RNG seed.
115     */
116    public XGBoostClassificationTrainer(int numTrees, double eta, double gamma, int maxDepth, double minChildWeight, double subsample, double featureSubsample, double lambda, double alpha, int nThread, boolean silent, long seed) {
117        super(numTrees,eta,gamma,maxDepth,minChildWeight,subsample,featureSubsample,lambda,alpha,nThread,silent,seed);
118        postConfig();
119    }
120
121    /**
122     * Create an XGBoost trainer.
123     *
124     * @param boosterType The base learning algorithm.
125     * @param treeMethod The tree building algorithm if using a tree booster.
126     * @param numTrees Number of trees to boost.
127     * @param eta Step size shrinkage parameter (default 0.3, range [0,1]).
128     * @param gamma Minimum loss reduction to make a split (default 0, range
129     * [0,inf]).
130     * @param maxDepth Maximum tree depth (default 6, range [1,inf]).
131     * @param minChildWeight Minimum sum of instance weights needed in a leaf
132     * (default 1, range [0, inf]).
133     * @param subsample Subsample size for each tree (default 1, range (0,1]).
134     * @param featureSubsample Subsample features for each tree (default 1,
135     * range (0,1]).
136     * @param lambda L2 regularization term on weights (default 1).
137     * @param alpha L1 regularization term on weights (default 0).
138     * @param nThread Number of threads to use (default 4).
139     * @param verbosity Set the logging verbosity of the native library.
140     * @param seed RNG seed.
141     */
142    public XGBoostClassificationTrainer(BoosterType boosterType, TreeMethod treeMethod, int numTrees, double eta, double gamma, int maxDepth, double minChildWeight, double subsample, double featureSubsample, double lambda, double alpha, int nThread, LoggingVerbosity verbosity, long seed) {
143        super(boosterType,treeMethod,numTrees,eta,gamma,maxDepth,minChildWeight,subsample,featureSubsample,lambda,alpha,nThread,verbosity,seed);
144        postConfig();
145    }
146
147    /**
148     * This gives direct access to the XGBoost parameter map.
149     * <p>
150     * It lets you pick things that we haven't exposed like dropout trees, binary classification etc.
151     * <p>
152     * This sidesteps the validation that Tribuo provides for the hyperparameters, and so can produce unexpected results.
153     * @param numTrees Number of trees to boost.
154     * @param parameters A map from string to object, where object can be Number or String.
155     */
156    public XGBoostClassificationTrainer(int numTrees, Map<String,Object> parameters) {
157        super(numTrees,parameters);
158        postConfig();
159    }
160
161    /**
162     * For olcut.
163     */
164    protected XGBoostClassificationTrainer() { }
165
166    /**
167     * Used by the OLCUT configuration system, and should not be called by external code.
168     */
169    @Override
170    public void postConfig() {
171        super.postConfig();
172        parameters.put("objective", "multi:softprob");
173        if (!evalMetric.isEmpty()) {
174            parameters.put("eval_metric", evalMetric);
175        }
176        if (!overrideParameters.isEmpty()) {
177            String value = overrideParameters.get("objective");
178            switch (value) {
179                case "multi:softprob":
180                case "multi:softmax":
181                case "binary:logistic":
182                case "binary:logitraw":
183                case "binary:hinge":
184                    break;
185                default:
186                    throw new PropertyException("","overrideParameters","The objective in overrideParameters must be a valid classification objective.");
187            }
188        }
189    }
190
191    @Override
192    public synchronized XGBoostModel<Label> train(Dataset<Label> examples) {
193        return train(examples, Collections.emptyMap());
194    }
195
196    @Override
197    public synchronized XGBoostModel<Label> train(Dataset<Label> examples, Map<String, Provenance> runProvenance) {
198        return train(examples, runProvenance, INCREMENT_INVOCATION_COUNT);
199    }
200
201    @Override
202    public synchronized XGBoostModel<Label> train(Dataset<Label> examples, Map<String, Provenance> runProvenance, int invocationCount) {
203        if (examples.getOutputInfo().getUnknownCount() > 0) {
204            throw new IllegalArgumentException("The supplied Dataset contained unknown Outputs, and this Trainer is supervised.");
205        }
206        ImmutableFeatureMap featureMap = examples.getFeatureIDMap();
207        ImmutableOutputInfo<Label> outputInfo = examples.getOutputIDInfo();
208        if(invocationCount != INCREMENT_INVOCATION_COUNT) {
209            setInvocationCount(invocationCount);
210        }
211        TrainerProvenance trainerProvenance = getProvenance();
212        trainInvocationCounter++;
213        Map<String,Object> curParams = overrideParameters.isEmpty() ? copyParams(parameters) : copyParams(overrideParameters);
214        curParams.put("num_class", outputInfo.size());
215        Booster model;
216        Function<Label,Float> responseExtractor = (Label l) -> (float) outputInfo.getID(l);
217        try {
218            DMatrixTuple<Label> trainingData = convertExamples(examples, featureMap, responseExtractor);
219            model = XGBoost.train(trainingData.data, curParams, numTrees, Collections.emptyMap(), null, null);
220        } catch (XGBoostError e) {
221            logger.log(Level.SEVERE, "XGBoost threw an error", e);
222            throw new IllegalStateException(e);
223        }
224
225        ModelProvenance provenance = new ModelProvenance(XGBoostModel.class.getName(), OffsetDateTime.now(), examples.getProvenance(), trainerProvenance, runProvenance);
226        XGBoostModel<Label> xgModel = createModel("xgboost-classification-model", provenance, featureMap, outputInfo, Collections.singletonList(model), new XGBoostClassificationConverter());
227
228        return xgModel;
229    }
230
231    @Override
232    public TrainerProvenance getProvenance() {
233        return new TrainerProvenanceImpl(this);
234    }
235}