001 /*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2014 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
019 */
020 package org.sonar.batch.scan;
021
022 import com.google.common.annotations.VisibleForTesting;
023 import com.google.common.collect.Lists;
024 import org.apache.commons.io.IOUtils;
025 import org.apache.commons.io.filefilter.AndFileFilter;
026 import org.apache.commons.io.filefilter.FileFileFilter;
027 import org.apache.commons.io.filefilter.IOFileFilter;
028 import org.apache.commons.io.filefilter.WildcardFileFilter;
029 import org.apache.commons.lang.StringUtils;
030 import org.slf4j.Logger;
031 import org.slf4j.LoggerFactory;
032 import org.sonar.api.CoreProperties;
033 import org.sonar.api.batch.bootstrap.ProjectDefinition;
034 import org.sonar.api.batch.bootstrap.ProjectReactor;
035 import org.sonar.batch.bootstrap.TaskProperties;
036
037 import javax.annotation.CheckForNull;
038 import javax.annotation.Nullable;
039
040 import java.io.File;
041 import java.io.FileFilter;
042 import java.io.FileInputStream;
043 import java.io.IOException;
044 import java.text.MessageFormat;
045 import java.util.ArrayList;
046 import java.util.List;
047 import java.util.Map;
048 import java.util.Map.Entry;
049 import java.util.Properties;
050
051 /**
052 * Class that creates a project definition based on a set of properties.
053 */
054 public class ProjectReactorBuilder {
055
056 private static final String INVALID_VALUE_OF_X_FOR_Y = "Invalid value of {0} for {1}";
057
058 private static final Logger LOG = LoggerFactory.getLogger(ProjectReactorBuilder.class);
059
060 /**
061 * @since 4.1 but not yet exposed in {@link CoreProperties}
062 */
063 private static final String MODULE_KEY_PROPERTY = "sonar.moduleKey";
064
065 protected static final String PROPERTY_PROJECT_BASEDIR = "sonar.projectBaseDir";
066 private static final String PROPERTY_PROJECT_BUILDDIR = "sonar.projectBuildDir";
067 private static final String PROPERTY_MODULES = "sonar.modules";
068
069 /**
070 * New properties, to be consistent with Sonar naming conventions
071 *
072 * @since 1.5
073 */
074 private static final String PROPERTY_SOURCES = "sonar.sources";
075 private static final String PROPERTY_TESTS = "sonar.tests";
076 private static final String PROPERTY_BINARIES = "sonar.binaries";
077 private static final String PROPERTY_LIBRARIES = "sonar.libraries";
078
079 /**
080 * Array of all mandatory properties required for a project without child.
081 */
082 private static final String[] MANDATORY_PROPERTIES_FOR_SIMPLE_PROJECT = {
083 PROPERTY_PROJECT_BASEDIR, CoreProperties.PROJECT_KEY_PROPERTY, CoreProperties.PROJECT_NAME_PROPERTY,
084 CoreProperties.PROJECT_VERSION_PROPERTY, PROPERTY_SOURCES
085 };
086
087 /**
088 * Array of all mandatory properties required for a project with children.
089 */
090 private static final String[] MANDATORY_PROPERTIES_FOR_MULTIMODULE_PROJECT = {PROPERTY_PROJECT_BASEDIR, CoreProperties.PROJECT_KEY_PROPERTY,
091 CoreProperties.PROJECT_NAME_PROPERTY, CoreProperties.PROJECT_VERSION_PROPERTY};
092
093 /**
094 * Array of all mandatory properties required for a child project before its properties get merged with its parent ones.
095 */
096 protected static final String[] MANDATORY_PROPERTIES_FOR_CHILD = {MODULE_KEY_PROPERTY, CoreProperties.PROJECT_NAME_PROPERTY};
097
098 /**
099 * Properties that must not be passed from the parent project to its children.
100 */
101 private static final List<String> NON_HERITED_PROPERTIES_FOR_CHILD = Lists.newArrayList(PROPERTY_PROJECT_BASEDIR, CoreProperties.WORKING_DIRECTORY, PROPERTY_MODULES,
102 CoreProperties.PROJECT_DESCRIPTION_PROPERTY);
103
104 private TaskProperties props;
105 private File rootProjectWorkDir;
106
107 public ProjectReactorBuilder(TaskProperties props) {
108 this.props = props;
109 }
110
111 public ProjectReactor execute() {
112 Properties bootstrapProperties = new Properties();
113 bootstrapProperties.putAll(props.properties());
114 ProjectDefinition rootProject = defineProject(bootstrapProperties, null);
115 rootProjectWorkDir = rootProject.getWorkDir();
116 defineChildren(rootProject);
117 cleanAndCheckProjectDefinitions(rootProject);
118 return new ProjectReactor(rootProject);
119 }
120
121 protected ProjectDefinition defineProject(Properties properties, @Nullable ProjectDefinition parent) {
122 if (properties.containsKey(PROPERTY_MODULES)) {
123 checkMandatoryProperties(properties, MANDATORY_PROPERTIES_FOR_MULTIMODULE_PROJECT);
124 } else {
125 checkMandatoryProperties(properties, MANDATORY_PROPERTIES_FOR_SIMPLE_PROJECT);
126 }
127 File baseDir = new File(properties.getProperty(PROPERTY_PROJECT_BASEDIR));
128 final String projectKey = properties.getProperty(CoreProperties.PROJECT_KEY_PROPERTY);
129 File workDir;
130 if (parent == null) {
131 validateDirectories(properties, baseDir, projectKey);
132 workDir = initRootProjectWorkDir(baseDir);
133 } else {
134 workDir = initModuleWorkDir(baseDir, properties);
135 }
136
137 return ProjectDefinition.create().setProperties(properties)
138 .setBaseDir(baseDir)
139 .setWorkDir(workDir)
140 .setBuildDir(initModuleBuildDir(baseDir, properties));
141 }
142
143 @VisibleForTesting
144 protected File initRootProjectWorkDir(File baseDir) {
145 String workDir = props.property(CoreProperties.WORKING_DIRECTORY);
146 if (StringUtils.isBlank(workDir)) {
147 return new File(baseDir, CoreProperties.WORKING_DIRECTORY_DEFAULT_VALUE);
148 }
149
150 File customWorkDir = new File(workDir);
151 if (customWorkDir.isAbsolute()) {
152 return customWorkDir;
153 }
154 return new File(baseDir, customWorkDir.getPath());
155 }
156
157 @VisibleForTesting
158 protected File initModuleWorkDir(File moduleBaseDir, Properties moduleProperties) {
159 String workDir = moduleProperties.getProperty(CoreProperties.WORKING_DIRECTORY);
160 if (StringUtils.isBlank(workDir)) {
161 String cleanKey = StringUtils.deleteWhitespace(moduleProperties.getProperty(CoreProperties.PROJECT_KEY_PROPERTY));
162 cleanKey = StringUtils.replace(cleanKey, ":", "_");
163 return new File(rootProjectWorkDir, cleanKey);
164 }
165
166 File customWorkDir = new File(workDir);
167 if (customWorkDir.isAbsolute()) {
168 return customWorkDir;
169 }
170 return new File(moduleBaseDir, customWorkDir.getPath());
171 }
172
173 @CheckForNull
174 private File initModuleBuildDir(File moduleBaseDir, Properties moduleProperties) {
175 String buildDir = moduleProperties.getProperty(PROPERTY_PROJECT_BUILDDIR);
176 if (StringUtils.isBlank(buildDir)) {
177 return null;
178 }
179
180 File customBuildDir = new File(buildDir);
181 if (customBuildDir.isAbsolute()) {
182 return customBuildDir;
183 }
184 return new File(moduleBaseDir, customBuildDir.getPath());
185 }
186
187 private void defineChildren(ProjectDefinition parentProject) {
188 Properties parentProps = parentProject.getProperties();
189 if (parentProps.containsKey(PROPERTY_MODULES)) {
190 for (String module : getListFromProperty(parentProps, PROPERTY_MODULES)) {
191 Properties moduleProps = extractModuleProperties(module, parentProps);
192 ProjectDefinition childProject = loadChildProject(parentProject, moduleProps, module);
193 // check the uniqueness of the child key
194 checkUniquenessOfChildKey(childProject, parentProject);
195 // the child project may have children as well
196 defineChildren(childProject);
197 // and finally add this child project to its parent
198 parentProject.addSubProject(childProject);
199 }
200 }
201 }
202
203 protected ProjectDefinition loadChildProject(ProjectDefinition parentProject, Properties moduleProps, String moduleId) {
204 final File baseDir;
205 if (moduleProps.containsKey(PROPERTY_PROJECT_BASEDIR)) {
206 baseDir = resolvePath(parentProject.getBaseDir(), moduleProps.getProperty(PROPERTY_PROJECT_BASEDIR));
207 setProjectBaseDir(baseDir, moduleProps, moduleId);
208 } else {
209 baseDir = new File(parentProject.getBaseDir(), moduleId);
210 setProjectBaseDir(baseDir, moduleProps, moduleId);
211 }
212
213 setModuleKeyAndNameIfNotDefined(moduleProps, moduleId, parentProject.getKey());
214
215 // and finish
216 checkMandatoryProperties(moduleProps, MANDATORY_PROPERTIES_FOR_CHILD);
217 validateDirectories(moduleProps, baseDir, moduleId);
218
219 mergeParentProperties(moduleProps, parentProject.getProperties());
220
221 return defineProject(moduleProps, parentProject);
222 }
223
224 @VisibleForTesting
225 protected static Properties toProperties(File propertyFile) {
226 Properties propsFromFile = new Properties();
227 FileInputStream fileInputStream = null;
228 try {
229 fileInputStream = new FileInputStream(propertyFile);
230 propsFromFile.load(fileInputStream);
231 } catch (IOException e) {
232 throw new IllegalStateException("Impossible to read the property file: " + propertyFile.getAbsolutePath(), e);
233 } finally {
234 IOUtils.closeQuietly(fileInputStream);
235 }
236 // Trim properties
237 for (String propKey : propsFromFile.stringPropertyNames()) {
238 propsFromFile.setProperty(propKey, StringUtils.trim(propsFromFile.getProperty(propKey)));
239 }
240 return propsFromFile;
241 }
242
243 @VisibleForTesting
244 protected static void setModuleKeyAndNameIfNotDefined(Properties childProps, String moduleId, String parentKey) {
245 if (!childProps.containsKey(MODULE_KEY_PROPERTY)) {
246 if (!childProps.containsKey(CoreProperties.PROJECT_KEY_PROPERTY)) {
247 childProps.put(MODULE_KEY_PROPERTY, parentKey + ":" + moduleId);
248 } else {
249 String childKey = childProps.getProperty(CoreProperties.PROJECT_KEY_PROPERTY);
250 childProps.put(MODULE_KEY_PROPERTY, parentKey + ":" + childKey);
251 }
252 }
253 if (!childProps.containsKey(CoreProperties.PROJECT_NAME_PROPERTY)) {
254 childProps.put(CoreProperties.PROJECT_NAME_PROPERTY, moduleId);
255 }
256 // For backward compatibility with ProjectDefinition
257 childProps.put(CoreProperties.PROJECT_KEY_PROPERTY, childProps.getProperty(MODULE_KEY_PROPERTY));
258 }
259
260 @VisibleForTesting
261 protected static void checkUniquenessOfChildKey(ProjectDefinition childProject, ProjectDefinition parentProject) {
262 for (ProjectDefinition definition : parentProject.getSubProjects()) {
263 if (definition.getKey().equals(childProject.getKey())) {
264 throw new IllegalStateException("Project '" + parentProject.getKey() + "' can't have 2 modules with the following key: " + childProject.getKey());
265 }
266 }
267 }
268
269 protected static void setProjectBaseDir(File baseDir, Properties childProps, String moduleId) {
270 if (!baseDir.isDirectory()) {
271 throw new IllegalStateException("The base directory of the module '" + moduleId + "' does not exist: " + baseDir.getAbsolutePath());
272 }
273 childProps.put(PROPERTY_PROJECT_BASEDIR, baseDir.getAbsolutePath());
274 }
275
276 @VisibleForTesting
277 protected static void checkMandatoryProperties(Properties props, String[] mandatoryProps) {
278 StringBuilder missing = new StringBuilder();
279 for (String mandatoryProperty : mandatoryProps) {
280 if (!props.containsKey(mandatoryProperty)) {
281 if (missing.length() > 0) {
282 missing.append(", ");
283 }
284 missing.append(mandatoryProperty);
285 }
286 }
287 String moduleKey = StringUtils.defaultIfBlank(props.getProperty(MODULE_KEY_PROPERTY), props.getProperty(CoreProperties.PROJECT_KEY_PROPERTY));
288 if (missing.length() != 0) {
289 throw new IllegalStateException("You must define the following mandatory properties for '" + (moduleKey == null ? "Unknown" : moduleKey) + "': " + missing);
290 }
291 }
292
293 protected static void validateDirectories(Properties props, File baseDir, String projectId) {
294 if (!props.containsKey(PROPERTY_MODULES)) {
295 // SONARPLUGINS-2285 Not an aggregator project so we can validate that paths are correct if defined
296
297 // We need to resolve patterns that may have been used in "sonar.libraries"
298 for (String pattern : getListFromProperty(props, PROPERTY_LIBRARIES)) {
299 File[] files = getLibraries(baseDir, pattern);
300 if (files == null || files.length == 0) {
301 LOG.error(MessageFormat.format(INVALID_VALUE_OF_X_FOR_Y, PROPERTY_LIBRARIES, projectId));
302 throw new IllegalStateException("No files nor directories matching '" + pattern + "' in directory " + baseDir);
303 }
304 }
305
306 // Check sonar.tests
307 String[] testPaths = getListFromProperty(props, PROPERTY_TESTS);
308 checkExistenceOfPaths(projectId, baseDir, testPaths, PROPERTY_TESTS);
309
310 // Check sonar.binaries
311 String[] binDirs = getListFromProperty(props, PROPERTY_BINARIES);
312 checkExistenceOfDirectories(projectId, baseDir, binDirs, PROPERTY_BINARIES);
313 }
314 }
315
316 @VisibleForTesting
317 protected static void cleanAndCheckProjectDefinitions(ProjectDefinition project) {
318 if (project.getSubProjects().isEmpty()) {
319 cleanAndCheckModuleProperties(project);
320 } else {
321 cleanAndCheckAggregatorProjectProperties(project);
322
323 // clean modules properties as well
324 for (ProjectDefinition module : project.getSubProjects()) {
325 cleanAndCheckProjectDefinitions(module);
326 }
327 }
328 }
329
330 @VisibleForTesting
331 protected static void cleanAndCheckModuleProperties(ProjectDefinition project) {
332 Properties properties = project.getProperties();
333
334 // We need to check the existence of source directories
335 String[] sourcePaths = getListFromProperty(properties, PROPERTY_SOURCES);
336 checkExistenceOfPaths(project.getKey(), project.getBaseDir(), sourcePaths, PROPERTY_SOURCES);
337
338 // And we need to resolve patterns that may have been used in "sonar.libraries"
339 List<String> libPaths = Lists.newArrayList();
340 for (String pattern : getListFromProperty(properties, PROPERTY_LIBRARIES)) {
341 for (File file : getLibraries(project.getBaseDir(), pattern)) {
342 libPaths.add(file.getAbsolutePath());
343 }
344 }
345 properties.remove(PROPERTY_LIBRARIES);
346 properties.put(PROPERTY_LIBRARIES, StringUtils.join(libPaths, ","));
347 }
348
349 @VisibleForTesting
350 protected static void cleanAndCheckAggregatorProjectProperties(ProjectDefinition project) {
351 Properties properties = project.getProperties();
352
353 // SONARPLUGINS-2295
354 String[] sourceDirs = getListFromProperty(properties, PROPERTY_SOURCES);
355 for (String path : sourceDirs) {
356 File sourceFolder = resolvePath(project.getBaseDir(), path);
357 if (sourceFolder.isDirectory()) {
358 LOG.warn("/!\\ A multi-module project can't have source folders, so '{}' won't be used for the analysis. " +
359 "If you want to analyse files of this folder, you should create another sub-module and move them inside it.",
360 sourceFolder.toString());
361 }
362 }
363
364 // "aggregator" project must not have the following properties:
365 properties.remove(PROPERTY_SOURCES);
366 properties.remove(PROPERTY_TESTS);
367 properties.remove(PROPERTY_BINARIES);
368 properties.remove(PROPERTY_LIBRARIES);
369
370 // and they don't need properties related to their modules either
371 Properties clone = (Properties) properties.clone();
372 List<String> moduleIds = Lists.newArrayList(getListFromProperty(properties, PROPERTY_MODULES));
373 for (Entry<Object, Object> entry : clone.entrySet()) {
374 String key = (String) entry.getKey();
375 if (isKeyPrefixedByModuleId(key, moduleIds)) {
376 properties.remove(key);
377 }
378 }
379 }
380
381 @VisibleForTesting
382 protected static void mergeParentProperties(Properties childProps, Properties parentProps) {
383 List<String> moduleIds = Lists.newArrayList(getListFromProperty(parentProps, PROPERTY_MODULES));
384 for (Map.Entry<Object, Object> entry : parentProps.entrySet()) {
385 String key = (String) entry.getKey();
386 if (!childProps.containsKey(key)
387 && !NON_HERITED_PROPERTIES_FOR_CHILD.contains(key)
388 && !isKeyPrefixedByModuleId(key, moduleIds)) {
389 childProps.put(entry.getKey(), entry.getValue());
390 }
391 }
392 }
393
394 private static boolean isKeyPrefixedByModuleId(String key, List<String> moduleIds) {
395 for (String moduleId : moduleIds) {
396 if (key.startsWith(moduleId + ".")) {
397 return true;
398 }
399 }
400 return false;
401 }
402
403 @VisibleForTesting
404 protected static Properties extractModuleProperties(String module, Properties properties) {
405 Properties moduleProps = new Properties();
406 String propertyPrefix = module + ".";
407 int prefixLength = propertyPrefix.length();
408 for (Map.Entry<Object, Object> entry : properties.entrySet()) {
409 String key = (String) entry.getKey();
410 if (key.startsWith(propertyPrefix)) {
411 moduleProps.put(key.substring(prefixLength), entry.getValue());
412 }
413 }
414 return moduleProps;
415 }
416
417 @VisibleForTesting
418 protected static void checkExistenceOfDirectories(String moduleRef, File baseDir, String[] dirPaths, String propName) {
419 for (String path : dirPaths) {
420 File sourceFolder = resolvePath(baseDir, path);
421 if (!sourceFolder.isDirectory()) {
422 LOG.error(MessageFormat.format(INVALID_VALUE_OF_X_FOR_Y, propName, moduleRef));
423 throw new IllegalStateException("The folder '" + path + "' does not exist for '" + moduleRef +
424 "' (base directory = " + baseDir.getAbsolutePath() + ")");
425 }
426 }
427
428 }
429
430 @VisibleForTesting
431 protected static void checkExistenceOfPaths(String moduleRef, File baseDir, String[] paths, String propName) {
432 for (String path : paths) {
433 File sourceFolder = resolvePath(baseDir, path);
434 if (!sourceFolder.exists()) {
435 LOG.error(MessageFormat.format(INVALID_VALUE_OF_X_FOR_Y, propName, moduleRef));
436 throw new IllegalStateException("The folder '" + path + "' does not exist for '" + moduleRef +
437 "' (base directory = " + baseDir.getAbsolutePath() + ")");
438 }
439 }
440
441 }
442
443 /**
444 * Returns files matching specified pattern.
445 */
446 @VisibleForTesting
447 protected static File[] getLibraries(File baseDir, String pattern) {
448 final int i = Math.max(pattern.lastIndexOf('/'), pattern.lastIndexOf('\\'));
449 final String dirPath, filePattern;
450 if (i == -1) {
451 dirPath = ".";
452 filePattern = pattern;
453 } else {
454 dirPath = pattern.substring(0, i);
455 filePattern = pattern.substring(i + 1);
456 }
457 List<IOFileFilter> filters = new ArrayList<IOFileFilter>();
458 if (pattern.indexOf('*') >= 0) {
459 filters.add(FileFileFilter.FILE);
460 }
461 filters.add(new WildcardFileFilter(filePattern));
462 File dir = resolvePath(baseDir, dirPath);
463 File[] files = dir.listFiles((FileFilter) new AndFileFilter(filters));
464 if (files == null) {
465 files = new File[0];
466 }
467 return files;
468 }
469
470 protected static File resolvePath(File baseDir, String path) {
471 File file = new File(path);
472 if (!file.isAbsolute()) {
473 try {
474 file = new File(baseDir, path).getCanonicalFile();
475 } catch (IOException e) {
476 throw new IllegalStateException("Unable to resolve path \"" + path + "\"", e);
477 }
478 }
479 return file;
480 }
481
482 /**
483 * Transforms a comma-separated list String property in to a array of trimmed strings.
484 *
485 * This works even if they are separated by whitespace characters (space char, EOL, ...)
486 *
487 */
488 static String[] getListFromProperty(Properties properties, String key) {
489 return StringUtils.stripAll(StringUtils.split(properties.getProperty(key, ""), ','));
490 }
491
492 }