001/* 002 * Copyright (c) 2021, Oracle and/or its affiliates. 003 * 004 * Licensed under the 2-clause BSD license. 005 * 006 * Redistribution and use in source and binary forms, with or without 007 * modification, are permitted provided that the following conditions are met: 008 * 009 * 1. Redistributions of source code must retain the above copyright notice, 010 * this list of conditions and the following disclaimer. 011 * 012 * 2. Redistributions in binary form must reproduce the above copyright notice, 013 * this list of conditions and the following disclaimer in the documentation 014 * and/or other materials provided with the distribution. 015 * 016 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 017 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 018 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 019 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 020 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 021 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 022 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 023 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 024 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 025 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 026 * POSSIBILITY OF SUCH DAMAGE. 027 */ 028 029package com.oracle.labs.mlrg.olcut.config.protobuf; 030 031import com.google.protobuf.InvalidProtocolBufferException; 032import com.google.protobuf.Message; 033import com.google.protobuf.TextFormat; 034import com.oracle.labs.mlrg.olcut.config.protobuf.protos.ListProvenanceProto; 035import com.oracle.labs.mlrg.olcut.config.protobuf.protos.MapProvenanceProto; 036import com.oracle.labs.mlrg.olcut.config.protobuf.protos.ObjectProvenanceProto; 037import com.oracle.labs.mlrg.olcut.config.protobuf.protos.RootProvenanceProto; 038import com.oracle.labs.mlrg.olcut.config.protobuf.protos.SimpleProvenanceProto; 039import com.oracle.labs.mlrg.olcut.provenance.io.FlatMarshalledProvenance; 040import com.oracle.labs.mlrg.olcut.provenance.io.ListMarshalledProvenance; 041import com.oracle.labs.mlrg.olcut.provenance.io.MapMarshalledProvenance; 042import com.oracle.labs.mlrg.olcut.provenance.io.ObjectMarshalledProvenance; 043import com.oracle.labs.mlrg.olcut.provenance.io.ProvenanceSerialization; 044import com.oracle.labs.mlrg.olcut.provenance.io.SimpleMarshalledProvenance; 045import com.oracle.labs.mlrg.olcut.util.MutableLong; 046import com.oracle.labs.mlrg.olcut.util.Pair; 047 048import java.io.BufferedOutputStream; 049import java.io.IOException; 050import java.io.InputStream; 051import java.io.InputStreamReader; 052import java.io.PrintWriter; 053import java.nio.charset.StandardCharsets; 054import java.nio.file.Files; 055import java.nio.file.Path; 056import java.util.ArrayList; 057import java.util.Base64; 058import java.util.HashMap; 059import java.util.List; 060import java.util.Map; 061 062/** 063 * Class for serializing and deserializing provenances to/from protobufs. 064 * <p> 065 * Uses {@link Base64} to encode and decode binary protobufs into and out of Strings if requested, though it 066 * is preferable to use protobuf's built-in text format. 067 */ 068public final class ProtoProvenanceSerialization implements ProvenanceSerialization { 069 070 private static final Base64.Encoder base64Encoder = Base64.getEncoder(); 071 private static final Base64.Decoder base64Decoder = Base64.getDecoder(); 072 073 private final boolean textFormat; 074 075 /** 076 * Construct a ProtoProvenanceSerialization. 077 * 078 * @param textFormat Output a pbtxt. 079 */ 080 public ProtoProvenanceSerialization(boolean textFormat) { 081 this.textFormat = textFormat; 082 } 083 084 @Override 085 public String getFileExtension() { 086 return textFormat ? "pbtxt" : "pb"; 087 } 088 089 @Override 090 public List<ObjectMarshalledProvenance> deserializeFromFile(Path path) throws IOException { 091 try { 092 InputStream is = Files.newInputStream(path); 093 RootProvenanceProto proto; 094 if (textFormat) { 095 RootProvenanceProto.Builder protoBuilder = RootProvenanceProto.newBuilder(); 096 TextFormat.getParser().merge(new InputStreamReader(is,StandardCharsets.UTF_8), protoBuilder); 097 proto = protoBuilder.build(); 098 } else { 099 proto = RootProvenanceProto.parseFrom(is); 100 } 101 return deserializeFromProto(proto); 102 } catch (InvalidProtocolBufferException | TextFormat.ParseException e) { 103 throw new IllegalArgumentException("Failed to parse protobuf",e); 104 } 105 } 106 107 @Override 108 public List<ObjectMarshalledProvenance> deserializeFromString(String input) { 109 try { 110 RootProvenanceProto proto; 111 if (textFormat) { 112 RootProvenanceProto.Builder protoBuilder = RootProvenanceProto.newBuilder(); 113 TextFormat.getParser().merge(input, protoBuilder); 114 proto = protoBuilder.build(); 115 } else { 116 byte[] bytes = base64Decoder.decode(input); 117 proto = RootProvenanceProto.parseFrom(bytes); 118 } 119 return deserializeFromProto(proto); 120 } catch (InvalidProtocolBufferException | TextFormat.ParseException e) { 121 throw new IllegalArgumentException("Failed to parse protobuf",e); 122 } 123 } 124 125 /** 126 * Deserialize the marshalled provenances from the supplied protobuf. 127 * @param proto The protobuf to deserialize. 128 * @return The marshalled provenances encoded in the protobuf. 129 */ 130 public List<ObjectMarshalledProvenance> deserializeFromProto(RootProvenanceProto proto) { 131 int totalProtos = proto.getLmpCount() + proto.getMmpCount() + proto.getOmpCount() + proto.getSmpCount(); 132 Message[] messages = new Message[totalProtos]; 133 // Unpack into map 134 for (ObjectProvenanceProto p : proto.getOmpList()) { 135 int curIndex = p.getIndex(); 136 if (messages[curIndex] != null) { 137 throw new IllegalArgumentException("Invalid protobuf found, index " + curIndex + " collided, found '" + p.toString() + " and " + messages[curIndex].toString()); 138 } else { 139 messages[curIndex] = p; 140 } 141 } 142 for (SimpleProvenanceProto p : proto.getSmpList()) { 143 int curIndex = p.getIndex(); 144 if (messages[curIndex] != null) { 145 throw new IllegalArgumentException("Invalid protobuf found, index " + curIndex + " collided, found '" + p.toString() + " and " + messages[curIndex].toString()); 146 } else { 147 messages[curIndex] = p; 148 } 149 } 150 for (MapProvenanceProto p : proto.getMmpList()) { 151 int curIndex = p.getIndex(); 152 if (messages[curIndex] != null) { 153 throw new IllegalArgumentException("Invalid protobuf found, index " + curIndex + " collided, found '" + p.toString() + " and " + messages[curIndex].toString()); 154 } else { 155 messages[curIndex] = p; 156 } 157 } 158 for (ListProvenanceProto p : proto.getLmpList()) { 159 int curIndex = p.getIndex(); 160 if (messages[curIndex] != null) { 161 throw new IllegalArgumentException("Invalid protobuf found, index " + curIndex + " collided, found '" + p.toString() + " and " + messages[curIndex].toString()); 162 } else { 163 messages[curIndex] = p; 164 } 165 } 166 167 List<ObjectMarshalledProvenance> outputList = new ArrayList<>(); 168 169 // Deserialize the ObjectMarshalledProvenances 170 for (ObjectProvenanceProto p : proto.getOmpList()) { 171 Map<String,FlatMarshalledProvenance> provMap = new HashMap<>(); 172 for (Map.Entry<String,Integer> e : p.getValuesMap().entrySet()) { 173 FlatMarshalledProvenance fmp = dispatchMessage(messages,e.getValue()); 174 provMap.put(e.getKey(),fmp); 175 } 176 177 ObjectMarshalledProvenance omp = new ObjectMarshalledProvenance(p.getObjectName(),provMap,p.getObjectClassName(),p.getProvenanceClassName()); 178 outputList.add(omp); 179 } 180 181 return outputList; 182 } 183 184 /** 185 * Decode the FlatMarshalledProvenance specified by this message index. 186 * @param messages The messages. 187 * @param index The current index. 188 * @return The FlatMarshalledProvenance specified by the index. 189 */ 190 private static FlatMarshalledProvenance dispatchMessage(Message[] messages, int index) { 191 Message curMessage = messages[index]; 192 if (curMessage instanceof SimpleProvenanceProto) { 193 return decodeSMP((SimpleProvenanceProto) curMessage); 194 } else if (curMessage instanceof ListProvenanceProto) { 195 return decodeLMP(messages, (ListProvenanceProto) curMessage); 196 } else if (curMessage instanceof MapProvenanceProto) { 197 return decodeMMP(messages, (MapProvenanceProto) curMessage); 198 } else { 199 throw new IllegalStateException("Invalid protobuf, a message index points to an ObjectMarshalledProvenance"); 200 } 201 } 202 203 /** 204 * Decodes a SimpleProvenanceProto into a SimpleMarshalledProvenance. 205 * @param proto The proto to decode. 206 * @return The decoded provenance. 207 */ 208 private static SimpleMarshalledProvenance decodeSMP(SimpleProvenanceProto proto) { 209 return new SimpleMarshalledProvenance(proto.getKey(),proto.getValue(),proto.getProvenanceClassName(),proto.getIsReference(),proto.getAdditional()); 210 } 211 212 /** 213 * Decodes a ListProvenanceProto into a ListMarshalledProvenance. 214 * @param messages The messages. 215 * @param proto The proto to decode. 216 * @return The decoded list provenance. 217 */ 218 private static ListMarshalledProvenance decodeLMP(Message[] messages, ListProvenanceProto proto) { 219 List<FlatMarshalledProvenance> list = new ArrayList<>(); 220 221 for (Integer i : proto.getValuesList()) { 222 list.add(dispatchMessage(messages,i)); 223 } 224 225 return new ListMarshalledProvenance(list); 226 } 227 228 /** 229 * Decodes a MapProvenanceProto into a MapMarshalledProvenance. 230 * @param messages The messages. 231 * @param proto The proto to decode. 232 * @return The decoded map provenance. 233 */ 234 private static MapMarshalledProvenance decodeMMP(Message[] messages, MapProvenanceProto proto) { 235 Map<String,FlatMarshalledProvenance> map = new HashMap<>(); 236 237 for (Map.Entry<String,Integer> e : proto.getValuesMap().entrySet()) { 238 map.put(e.getKey(), dispatchMessage(messages,e.getValue())); 239 } 240 241 return new MapMarshalledProvenance(map); 242 } 243 244 /** 245 * Serialize the marshalled provenances into a protobuf. 246 * @param marshalledProvenances The provenances to serialize. 247 * @return A protobuf encoding all the provenances. 248 */ 249 public RootProvenanceProto serializeToProto(List<ObjectMarshalledProvenance> marshalledProvenances) { 250 RootProvenanceProto.Builder builder = RootProvenanceProto.newBuilder(); 251 252 // linearizing the provenances 253 MutableLong counter = new MutableLong(0); 254 for (ObjectMarshalledProvenance omp : marshalledProvenances) { 255 convertProvenance(builder, counter, omp); 256 } 257 258 return builder.build(); 259 } 260 261 /** 262 * Encodes the ObjectMarshalledProvenance into the RootProvenanceProto. 263 * @param builder The proto builder. 264 * @param counter The index counter. 265 * @param omp The provenance to encode. 266 */ 267 private static void convertProvenance(RootProvenanceProto.Builder builder, MutableLong counter, ObjectMarshalledProvenance omp) { 268 ObjectProvenanceProto.Builder ompBuilder = ObjectProvenanceProto.newBuilder(); 269 270 ompBuilder.setIndex(counter.intValue()); 271 counter.increment(); 272 273 ompBuilder.setObjectName(omp.getName()); 274 ompBuilder.setObjectClassName(omp.getObjectClassName()); 275 ompBuilder.setProvenanceClassName(omp.getProvenanceClassName()); 276 277 for (Map.Entry<String, FlatMarshalledProvenance> e : omp.getMap().entrySet()) { 278 int count = dispatchFMP(builder, counter, e.getValue()); 279 ompBuilder.putValues(e.getKey(),count); 280 } 281 282 builder.addOmp(ompBuilder.build()); 283 } 284 285 /** 286 * Dispatch on the type of FlatMarshalledProvenance. 287 * @param builder The root proto builder. 288 * @param counter The provenance counter. 289 * @param fmp The FMP to dispatch on. 290 */ 291 private static int dispatchFMP(RootProvenanceProto.Builder builder, MutableLong counter, FlatMarshalledProvenance fmp) { 292 if (fmp instanceof SimpleMarshalledProvenance) { 293 SimpleMarshalledProvenance smp = (SimpleMarshalledProvenance) fmp; 294 return encodeSMP(builder,counter,smp); 295 } else if (fmp instanceof ListMarshalledProvenance) { 296 ListMarshalledProvenance lmp = (ListMarshalledProvenance) fmp; 297 return encodeLMP(builder,counter,lmp); 298 } else if (fmp instanceof MapMarshalledProvenance) { 299 MapMarshalledProvenance mmp = (MapMarshalledProvenance) fmp; 300 return encodeMMP(builder,counter,mmp); 301 } else { 302 throw new RuntimeException("Should not reach here, unexpected FlatMarshalledProvenance subclass " + fmp.getClass()); 303 } 304 } 305 306 /** 307 * Encodes the SimpleMarshalledProvenance into the RootProvenanceProto. 308 * @param builder The proto builder. 309 * @param counter The index counter. 310 * @param smp The provenance to encode. 311 * @return The index of this SimpleMarshalledProvenance 312 */ 313 private static int encodeSMP(RootProvenanceProto.Builder builder, MutableLong counter, SimpleMarshalledProvenance smp) { 314 SimpleProvenanceProto.Builder smpBuilder = SimpleProvenanceProto.newBuilder(); 315 316 int curIndex = counter.intValue(); 317 smpBuilder.setIndex(curIndex); 318 counter.increment(); 319 320 smpBuilder.setKey(smp.getKey()); 321 smpBuilder.setValue(smp.getValue()); 322 smpBuilder.setAdditional(smp.getAdditional()); 323 smpBuilder.setProvenanceClassName(smp.getProvenanceClassName()); 324 smpBuilder.setIsReference(smp.isReference()); 325 326 builder.addSmp(smpBuilder.build()); 327 return curIndex; 328 } 329 330 /** 331 * Encodes the ListMarshalledProvenance into the RootProvenanceProto. 332 * @param builder The proto builder. 333 * @param counter The index counter. 334 * @param lmp The provenance to encode. 335 * @return The index of this ListMarshalledProvenance 336 */ 337 private static int encodeLMP(RootProvenanceProto.Builder builder, MutableLong counter, ListMarshalledProvenance lmp) { 338 ListProvenanceProto.Builder lmpBuilder = ListProvenanceProto.newBuilder(); 339 340 int curIndex = counter.intValue(); 341 lmpBuilder.setIndex(curIndex); 342 counter.increment(); 343 344 for (FlatMarshalledProvenance fmp : lmp) { 345 lmpBuilder.addValues(dispatchFMP(builder,counter,fmp)); 346 } 347 348 builder.addLmp(lmpBuilder.build()); 349 return curIndex; 350 } 351 352 /** 353 * Encodes the MapMarshalledProvenance into the RootProvenanceProto. 354 * @param builder The proto builder. 355 * @param counter The index counter. 356 * @param mmp The provenance to encode. 357 * @return The index of this MapMarshalledProvenance 358 */ 359 private static int encodeMMP(RootProvenanceProto.Builder builder, MutableLong counter, MapMarshalledProvenance mmp) { 360 MapProvenanceProto.Builder mmpBuilder = MapProvenanceProto.newBuilder(); 361 362 int curIndex = counter.intValue(); 363 mmpBuilder.setIndex(curIndex); 364 counter.increment(); 365 366 for (Pair<String,FlatMarshalledProvenance> p : mmp) { 367 mmpBuilder.putValues(p.getA(),dispatchFMP(builder,counter,p.getB())); 368 } 369 370 builder.addMmp(mmpBuilder.build()); 371 return curIndex; 372 } 373 374 @Override 375 public String serializeToString(List<ObjectMarshalledProvenance> marshalledProvenances) { 376 RootProvenanceProto proto = serializeToProto(marshalledProvenances); 377 378 if (textFormat) { 379 return proto.toString(); 380 } else { 381 return base64Encoder.encodeToString(proto.toByteArray()); 382 } 383 } 384 385 @Override 386 public void serializeToFile(List<ObjectMarshalledProvenance> marshalledProvenances, Path path) throws IOException { 387 RootProvenanceProto proto = serializeToProto(marshalledProvenances); 388 389 if (textFormat) { 390 try (PrintWriter writer = new PrintWriter(Files.newBufferedWriter(path))) { 391 writer.println(proto.toString()); 392 } 393 } else { 394 try (BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(path))) { 395 proto.writeTo(bos); 396 } 397 } 398 } 399}