001/* 002 * Copyright (c) 2021, Oracle and/or its affiliates. 003 * 004 * Licensed under the 2-clause BSD license. 005 * 006 * Redistribution and use in source and binary forms, with or without 007 * modification, are permitted provided that the following conditions are met: 008 * 009 * 1. Redistributions of source code must retain the above copyright notice, 010 * this list of conditions and the following disclaimer. 011 * 012 * 2. Redistributions in binary form must reproduce the above copyright notice, 013 * this list of conditions and the following disclaimer in the documentation 014 * and/or other materials provided with the distribution. 015 * 016 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 017 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 018 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 019 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 020 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 021 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 022 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 023 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 024 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 025 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 026 * POSSIBILITY OF SUCH DAMAGE. 027 */ 028 029package com.oracle.labs.mlrg.olcut.config.protobuf; 030 031import com.google.protobuf.TextFormat; 032import com.oracle.labs.mlrg.olcut.config.ConfigurationData; 033import com.oracle.labs.mlrg.olcut.config.ConfigurationManager; 034import com.oracle.labs.mlrg.olcut.config.PropertyException; 035import com.oracle.labs.mlrg.olcut.config.SerializedObject; 036import com.oracle.labs.mlrg.olcut.config.io.ConfigLoader; 037import com.oracle.labs.mlrg.olcut.config.io.ConfigLoaderException; 038import com.oracle.labs.mlrg.olcut.config.io.URLLoader; 039import com.oracle.labs.mlrg.olcut.config.property.GlobalProperties; 040import com.oracle.labs.mlrg.olcut.config.property.ListProperty; 041import com.oracle.labs.mlrg.olcut.config.property.MapProperty; 042import com.oracle.labs.mlrg.olcut.config.property.SimpleProperty; 043import com.oracle.labs.mlrg.olcut.config.protobuf.protos.ComponentProto; 044import com.oracle.labs.mlrg.olcut.config.protobuf.protos.ConfigFileProto; 045import com.oracle.labs.mlrg.olcut.config.protobuf.protos.ConfigProto; 046import com.oracle.labs.mlrg.olcut.config.protobuf.protos.PropertyListProto; 047import com.oracle.labs.mlrg.olcut.config.protobuf.protos.PropertyMapProto; 048import com.oracle.labs.mlrg.olcut.config.protobuf.protos.SerializedObjectProto; 049import com.oracle.labs.mlrg.olcut.util.IOUtil; 050 051import java.io.BufferedReader; 052import java.io.File; 053import java.io.IOException; 054import java.io.InputStream; 055import java.io.InputStreamReader; 056import java.net.MalformedURLException; 057import java.net.URL; 058import java.nio.charset.StandardCharsets; 059import java.security.AccessController; 060import java.security.PrivilegedAction; 061import java.util.ArrayList; 062import java.util.HashMap; 063import java.util.List; 064import java.util.Map; 065import java.util.logging.Level; 066import java.util.logging.Logger; 067 068/** 069 * A loader for configuration data stored in protobuf format. 070 */ 071public final class ProtoLoader implements ConfigLoader { 072 073 private static final Logger logger = Logger.getLogger(ProtoLoader.class.getName()); 074 075 private final URLLoader parent; 076 077 private final Map<String, ConfigurationData> rpdMap; 078 079 private final Map<String, ConfigurationData> existingRPD; 080 081 private final Map<String, SerializedObject> serializedObjects; 082 083 private final GlobalProperties globalProperties; 084 085 private final boolean parseTextFormat; 086 087 /** 088 * Constructs a configuration loader for protobuf data. 089 * @param parent The URL loader for this instance of the configuration manager. 090 * @param rpdMap The configuration data for the current instance. 091 * @param existingRPD Any pre-existing configuration data which may be inherited. 092 * @param serializedObjects The current set of serialized objects. 093 * @param globalProperties The global properties. 094 * @param parseTextFormat Should this loader parse text protobufs or binary ones. 095 */ 096 public ProtoLoader(URLLoader parent, Map<String, ConfigurationData> rpdMap, 097 Map<String, ConfigurationData> existingRPD, 098 Map<String, SerializedObject> serializedObjects, 099 GlobalProperties globalProperties, boolean parseTextFormat) { 100 this.parent = parent; 101 this.rpdMap = rpdMap; 102 this.existingRPD = existingRPD; 103 this.serializedObjects = serializedObjects; 104 this.globalProperties = globalProperties; 105 this.parseTextFormat = parseTextFormat; 106 } 107 108 @Override 109 public void load(URL url) throws ConfigLoaderException { 110 AccessController.doPrivileged((PrivilegedAction<Void>) 111 () -> { 112 String workingDir = ""; 113 if (url.getProtocol().equals("file")) { 114 workingDir = new File(url.getFile()).getParent(); 115 } else if (IOUtil.isDisallowedProtocol(url)) { 116 throw new ConfigLoaderException("Unable to load configurations from URLs with protocol: " + url.getProtocol()); 117 } 118 try (InputStream is = url.openStream()) { 119 ConfigProto proto; 120 if (parseTextFormat) { 121 ConfigProto.Builder protoBuilder = ConfigProto.newBuilder(); 122 BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); 123 TextFormat.getParser().merge(reader,protoBuilder); 124 proto = protoBuilder.build(); 125 } else { 126 proto = ConfigProto.parseFrom(is); 127 } 128 parseConfigProto(workingDir,proto); 129 } catch (IOException e) { 130 throw new ConfigLoaderException(e, e.getMessage()); 131 } 132 return null; 133 } 134 ); 135 } 136 137 @Override 138 public void load(InputStream stream) throws ConfigLoaderException { 139 try { 140 ConfigProto proto; 141 if (parseTextFormat) { 142 ConfigProto.Builder protoBuilder = ConfigProto.newBuilder(); 143 BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); 144 TextFormat.getParser().merge(reader, protoBuilder); 145 proto = protoBuilder.build(); 146 } else { 147 proto = ConfigProto.parseFrom(stream); 148 } 149 parseConfigProto("", proto); 150 } catch (IOException e) { 151 String msg = "Error while parsing input: " + e.getMessage(); 152 throw new ConfigLoaderException(e, msg); 153 } 154 } 155 156 /** 157 * Parses the config proto into this loader. 158 * @param workingDir The current working directory if known. 159 * @param proto The proto to parse. 160 */ 161 private void parseConfigProto(String workingDir, ConfigProto proto) { 162 // Parse global properties 163 for (Map.Entry<String,String> e : proto.getPropertiesMap().entrySet()) { 164 try { 165 globalProperties.setValue(e.getKey(), e.getValue()); 166 } catch (PropertyException ex) { 167 throw new ConfigLoaderException("Invalid global property name: " + e.getKey()); 168 } 169 } 170 171 // Parse additional files 172 for (ConfigFileProto file : proto.getConfigFileList()) { 173 String name = file.getName(); 174 String value = file.getValue(); 175 if (name.isEmpty() || value.isEmpty()) { 176 throw new ConfigLoaderException("File element must have " 177 + "'name' and 'value' attributes, found " + name + ", " + value); 178 } 179 try { 180 URL newURL = ConfigurationManager.class.getResource(value); 181 if (newURL == null) { 182 File newFile = new File(value); 183 if (!newFile.isAbsolute()) { 184 newFile = new File(workingDir,value); 185 } 186 newURL = newFile.toURI().toURL(); 187 } 188 parent.addURL(newURL); 189 } catch (MalformedURLException ex) { 190 throw new ConfigLoaderException(ex, "Incorrectly formatted file element " + name + " with value " + value); 191 } 192 } 193 194 // Parse serialized objects 195 for (SerializedObjectProto serObj : proto.getSerializedObjectList()) { 196 String name = serObj.getName(); 197 String type = serObj.getType(); 198 String location = serObj.getLocation(); 199 if (name.isEmpty() || type.isEmpty() || location.isEmpty()) { 200 throw new ConfigLoaderException("Serialized element must have 'name', 'type' and 'location' elements, found " + serObj.toString()); 201 } 202 serializedObjects.put(name, new SerializedObject(name, location, type)); 203 } 204 205 // Parse components 206 for (ComponentProto component : proto.getComponentsList()) { 207 parseComponentProto(component); 208 } 209 } 210 211 /** 212 * Parses the component proto into this loader. 213 * @param component The component to parse. 214 */ 215 private void parseComponentProto(ComponentProto component) { 216 String name = component.getName(); 217 String type = component.getType(); 218 String override = component.getOverride(); 219 // 220 // Check for a badly formed component tag. 221 if (name.isEmpty() || (type.isEmpty() && override.isEmpty())) { 222 throw new ConfigLoaderException("Component element must specify " 223 + "'name' and either 'type' or 'inherit' attributes, found " + component); 224 } 225 226 boolean exportable = component.getExportable(); 227 boolean importable = component.getImportable(); 228 if (!exportable && component.hasLeaseTime()) { 229 throw new ConfigLoaderException("lease timeout " + component.getLeaseTime() + 230 " specified for component '" + name + "' that does not have export set"); 231 } else if (component.hasLeaseTime() && component.getLeaseTime() < 0) { 232 throw new ConfigLoaderException("lease timeout " 233 + component.getLeaseTime() + " must be greater than 0, for component " + name); 234 } 235 long leaseTime = component.hasLeaseTime() ? component.getLeaseTime() : ConfigurationData.DEFAULT_LEASE_TIME; 236 String entriesName = component.hasEntries() ? component.getEntries() : null; 237 String serializedForm = component.hasSerialized() ? component.getSerialized() : null; 238 239 ConfigurationData rpd; 240 if (!override.isEmpty()) { 241 // 242 // If we're overriding an existing type, then we should pull 243 // its property set, copy it and override it. Note that we're 244 // not doing any type checking here, so it's possible to specify 245 // a type for override that is incompatible with the specified 246 // properties. If that's the case, then things might get 247 // really weird. We'll log an override with a specified type 248 // just in case. 249 ConfigurationData spd = rpdMap.get(override); 250 if (spd == null) { 251 spd = existingRPD.get(override); 252 if (spd == null) { 253 throw new ConfigLoaderException("Override for undefined component: " 254 + override + ", with name " + name); 255 } 256 } 257 if (!type.isEmpty() && !type.equals(spd.getClassName())) { 258 logger.log(Level.FINE, String.format("Overriding component %s with component %s, new type is %s overridden type was %s", 259 spd.getName(), name, type, spd.getClassName())); 260 } 261 if (type.isEmpty()) { 262 type = spd.getClassName(); 263 } 264 rpd = new ConfigurationData(name, type, spd.getProperties(), serializedForm, entriesName, exportable, importable, leaseTime); 265 } else { 266 if (rpdMap.get(name) != null) { 267 throw new ConfigLoaderException("duplicate definition for " 268 + name); 269 } 270 rpd = new ConfigurationData(name, type, serializedForm, entriesName, exportable, importable, leaseTime); 271 } 272 273 // Read out simple properties 274 for (Map.Entry<String,String> e : component.getPropertiesMap().entrySet()) { 275 rpd.add(e.getKey(), new SimpleProperty(e.getValue())); 276 } 277 278 // Read out map properties 279 for (PropertyMapProto e : component.getMapPropertyList()) { 280 // Must be map 281 Map<String, SimpleProperty> mapOutput = new HashMap<>(); 282 for (Map.Entry<String,String> mapProp : e.getElementsMap().entrySet()) { 283 mapOutput.put(mapProp.getKey(), new SimpleProperty(mapProp.getValue())); 284 } 285 rpd.add(e.getName(), new MapProperty(mapOutput)); 286 } 287 288 // Read out list properties 289 for (PropertyListProto e : component.getListPropertyList()) { 290 List<SimpleProperty> listOutput = new ArrayList<>(); 291 List<Class<?>> classListOutput = new ArrayList<>(); 292 293 for (String item : e.getItemList()) { 294 listOutput.add(new SimpleProperty(item)); 295 } 296 297 for (String listType : e.getTypeList()) { 298 try { 299 classListOutput.add(Class.forName(listType)); 300 } catch (ClassNotFoundException cnfe) { 301 throw new ConfigLoaderException("Unable to find class " 302 + listType + " in component '" + name + "', propertylist '" + e.getName() + "'"); 303 } 304 } 305 306 ListProperty listProp; 307 if (classListOutput.isEmpty()) { 308 listProp = new ListProperty(listOutput); 309 } else { 310 listProp = new ListProperty(listOutput,classListOutput); 311 } 312 rpd.add(e.getName(), listProp); 313 } 314 rpdMap.put(rpd.getName(),rpd); 315 } 316}