001 /*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2014 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
019 */
020 package org.sonar.batch.index;
021
022 import com.google.common.collect.Lists;
023 import com.google.common.collect.Maps;
024 import com.google.common.collect.Sets;
025 import org.apache.commons.lang.ObjectUtils;
026 import org.apache.commons.lang.StringUtils;
027 import org.slf4j.Logger;
028 import org.slf4j.LoggerFactory;
029 import org.sonar.api.batch.Event;
030 import org.sonar.api.batch.SonarIndex;
031 import org.sonar.api.batch.bootstrap.ProjectDefinition;
032 import org.sonar.api.database.model.Snapshot;
033 import org.sonar.api.design.Dependency;
034 import org.sonar.api.measures.CoreMetrics;
035 import org.sonar.api.measures.Measure;
036 import org.sonar.api.measures.MeasuresFilter;
037 import org.sonar.api.measures.MeasuresFilters;
038 import org.sonar.api.measures.Metric;
039 import org.sonar.api.measures.MetricFinder;
040 import org.sonar.api.resources.Directory;
041 import org.sonar.api.resources.File;
042 import org.sonar.api.resources.Project;
043 import org.sonar.api.resources.ProjectLink;
044 import org.sonar.api.resources.Qualifiers;
045 import org.sonar.api.resources.Resource;
046 import org.sonar.api.resources.ResourceUtils;
047 import org.sonar.api.resources.Scopes;
048 import org.sonar.api.rules.Rule;
049 import org.sonar.api.rules.Violation;
050 import org.sonar.api.scan.filesystem.PathResolver;
051 import org.sonar.api.utils.SonarException;
052 import org.sonar.api.violations.ViolationQuery;
053 import org.sonar.batch.ProjectTree;
054 import org.sonar.batch.issue.DeprecatedViolations;
055 import org.sonar.batch.issue.ModuleIssues;
056 import org.sonar.batch.scan.measure.MeasureCache;
057 import org.sonar.core.component.ComponentKeys;
058 import org.sonar.core.component.ScanGraph;
059
060 import javax.annotation.CheckForNull;
061 import javax.annotation.Nullable;
062
063 import java.util.ArrayList;
064 import java.util.Collection;
065 import java.util.Collections;
066 import java.util.Date;
067 import java.util.HashMap;
068 import java.util.Iterator;
069 import java.util.List;
070 import java.util.Map;
071 import java.util.Set;
072
073 public class DefaultIndex extends SonarIndex {
074
075 private static final Logger LOG = LoggerFactory.getLogger(DefaultIndex.class);
076
077 private PersistenceManager persistence;
078 private MetricFinder metricFinder;
079 private final ScanGraph graph;
080
081 // caches
082 private Project currentProject;
083 private Map<Resource, Bucket> buckets = Maps.newHashMap();
084 private Map<String, Bucket> bucketsByDeprecatedKey = Maps.newHashMap();
085 private Set<Dependency> dependencies = Sets.newHashSet();
086 private Map<Resource, Map<Resource, Dependency>> outgoingDependenciesByResource = Maps.newHashMap();
087 private Map<Resource, Map<Resource, Dependency>> incomingDependenciesByResource = Maps.newHashMap();
088 private ProjectTree projectTree;
089 private final DeprecatedViolations deprecatedViolations;
090 private ModuleIssues moduleIssues;
091 private final MeasureCache measureCache;
092
093 private ResourceKeyMigration migration;
094
095 public DefaultIndex(PersistenceManager persistence, ProjectTree projectTree, MetricFinder metricFinder,
096 ScanGraph graph, DeprecatedViolations deprecatedViolations, ResourceKeyMigration migration, MeasureCache measureCache) {
097 this.persistence = persistence;
098 this.projectTree = projectTree;
099 this.metricFinder = metricFinder;
100 this.graph = graph;
101 this.deprecatedViolations = deprecatedViolations;
102 this.migration = migration;
103 this.measureCache = measureCache;
104 }
105
106 public void start() {
107 Project rootProject = projectTree.getRootProject();
108 if (StringUtils.isNotBlank(rootProject.getKey())) {
109 doStart(rootProject);
110 }
111 }
112
113 void doStart(Project rootProject) {
114 Bucket bucket = new Bucket(rootProject);
115 addBucket(rootProject, bucket);
116 migration.checkIfMigrationNeeded(rootProject);
117 persistence.saveProject(rootProject, null);
118 currentProject = rootProject;
119
120 for (Project module : rootProject.getModules()) {
121 addModule(rootProject, module);
122 }
123 }
124
125 private void addBucket(Resource resource, Bucket bucket) {
126 buckets.put(resource, bucket);
127 if (StringUtils.isNotBlank(resource.getDeprecatedKey())) {
128 bucketsByDeprecatedKey.put(resource.getDeprecatedKey(), bucket);
129 }
130 }
131
132 private void addModule(Project parent, Project module) {
133 ProjectDefinition parentDefinition = projectTree.getProjectDefinition(parent);
134 java.io.File parentBaseDir = parentDefinition.getBaseDir();
135 ProjectDefinition moduleDefinition = projectTree.getProjectDefinition(module);
136 java.io.File moduleBaseDir = moduleDefinition.getBaseDir();
137 module.setPath(new PathResolver().relativePath(parentBaseDir, moduleBaseDir));
138 addResource(module);
139 for (Project submodule : module.getModules()) {
140 addModule(module, submodule);
141 }
142 }
143
144 @Override
145 public Project getProject() {
146 return currentProject;
147 }
148
149 public void setCurrentProject(Project project, ModuleIssues moduleIssues) {
150 this.currentProject = project;
151
152 // the following components depend on the current module, so they need to be reloaded.
153 this.moduleIssues = moduleIssues;
154 }
155
156 /**
157 * Keep only project stuff
158 */
159 public void clear() {
160 Iterator<Map.Entry<Resource, Bucket>> it = buckets.entrySet().iterator();
161 while (it.hasNext()) {
162 Map.Entry<Resource, Bucket> entry = it.next();
163 Resource resource = entry.getKey();
164 if (!ResourceUtils.isSet(resource)) {
165 entry.getValue().clear();
166 it.remove();
167 }
168 }
169
170 Set<Dependency> projectDependencies = getDependenciesBetweenProjects();
171 dependencies.clear();
172 incomingDependenciesByResource.clear();
173 outgoingDependenciesByResource.clear();
174 for (Dependency projectDependency : projectDependencies) {
175 projectDependency.setId(null);
176 registerDependency(projectDependency);
177 }
178 }
179
180 @CheckForNull
181 @Override
182 public Measure getMeasure(Resource resource, org.sonar.api.batch.measure.Metric<?> metric) {
183 return getMeasures(resource, MeasuresFilters.metric(metric));
184 }
185
186 @CheckForNull
187 @Override
188 public <M> M getMeasures(Resource resource, MeasuresFilter<M> filter) {
189 // Reload resource so that effective key is populated
190 Resource indexedResource = getResource(resource);
191 if (indexedResource == null) {
192 return null;
193 }
194 Iterable<Measure> unfiltered;
195 if (filter instanceof MeasuresFilters.MetricFilter) {
196 // optimization
197 unfiltered = measureCache.byMetric(indexedResource, ((MeasuresFilters.MetricFilter<M>) filter).filterOnMetricKey());
198 } else {
199 unfiltered = measureCache.byResource(indexedResource);
200 }
201 Collection<Measure> all = new ArrayList<Measure>();
202 if (unfiltered != null) {
203 for (Measure measure : unfiltered) {
204 all.add(measure);
205 }
206 }
207 return filter.filter(all);
208 }
209
210 @Override
211 public Measure addMeasure(Resource resource, Measure measure) {
212 Bucket bucket = getBucket(resource);
213 if (bucket != null) {
214 Metric metric = metricFinder.findByKey(measure.getMetricKey());
215 if (metric == null) {
216 throw new SonarException("Unknown metric: " + measure.getMetricKey());
217 }
218 measure.setMetric(metric);
219 if (measureCache.contains(resource, measure)
220 // Hack for SONAR-5212
221 && !measure.getMetric().equals(CoreMetrics.TESTS)) {
222 throw new SonarException("Can not add the same measure twice on " + resource + ": " + measure);
223 }
224 measureCache.put(resource, measure);
225 }
226 return measure;
227 }
228
229 //
230 //
231 //
232 // DEPENDENCIES
233 //
234 //
235 //
236
237 @Override
238 public Dependency addDependency(Dependency dependency) {
239 Dependency existingDep = getEdge(dependency.getFrom(), dependency.getTo());
240 if (existingDep != null) {
241 return existingDep;
242 }
243
244 Dependency parentDependency = dependency.getParent();
245 if (parentDependency != null) {
246 addDependency(parentDependency);
247 }
248
249 if (registerDependency(dependency)) {
250 persistence.saveDependency(currentProject, dependency, parentDependency);
251 }
252 return dependency;
253 }
254
255 boolean registerDependency(Dependency dependency) {
256 Bucket fromBucket = doIndex(dependency.getFrom());
257 Bucket toBucket = doIndex(dependency.getTo());
258
259 if (fromBucket != null && toBucket != null) {
260 dependencies.add(dependency);
261 registerOutgoingDependency(dependency);
262 registerIncomingDependency(dependency);
263 return true;
264 }
265 return false;
266 }
267
268 private void registerOutgoingDependency(Dependency dependency) {
269 Map<Resource, Dependency> outgoingDeps = outgoingDependenciesByResource.get(dependency.getFrom());
270 if (outgoingDeps == null) {
271 outgoingDeps = new HashMap<Resource, Dependency>();
272 outgoingDependenciesByResource.put(dependency.getFrom(), outgoingDeps);
273 }
274 outgoingDeps.put(dependency.getTo(), dependency);
275 }
276
277 private void registerIncomingDependency(Dependency dependency) {
278 Map<Resource, Dependency> incomingDeps = incomingDependenciesByResource.get(dependency.getTo());
279 if (incomingDeps == null) {
280 incomingDeps = new HashMap<Resource, Dependency>();
281 incomingDependenciesByResource.put(dependency.getTo(), incomingDeps);
282 }
283 incomingDeps.put(dependency.getFrom(), dependency);
284 }
285
286 @Override
287 public Set<Dependency> getDependencies() {
288 return dependencies;
289 }
290
291 public Dependency getEdge(Resource from, Resource to) {
292 Map<Resource, Dependency> map = outgoingDependenciesByResource.get(from);
293 if (map != null) {
294 return map.get(to);
295 }
296 return null;
297 }
298
299 public boolean hasEdge(Resource from, Resource to) {
300 return getEdge(from, to) != null;
301 }
302
303 public Set<Resource> getVertices() {
304 return buckets.keySet();
305 }
306
307 public Collection<Dependency> getOutgoingEdges(Resource from) {
308 Map<Resource, Dependency> deps = outgoingDependenciesByResource.get(from);
309 if (deps != null) {
310 return deps.values();
311 }
312 return Collections.emptyList();
313 }
314
315 public Collection<Dependency> getIncomingEdges(Resource to) {
316 Map<Resource, Dependency> deps = incomingDependenciesByResource.get(to);
317 if (deps != null) {
318 return deps.values();
319 }
320 return Collections.emptyList();
321 }
322
323 Set<Dependency> getDependenciesBetweenProjects() {
324 Set<Dependency> result = Sets.newLinkedHashSet();
325 for (Dependency dependency : dependencies) {
326 if (ResourceUtils.isSet(dependency.getFrom()) || ResourceUtils.isSet(dependency.getTo())) {
327 result.add(dependency);
328 }
329 }
330 return result;
331 }
332
333 //
334 //
335 //
336 // VIOLATIONS
337 //
338 //
339 //
340
341 /**
342 * {@inheritDoc}
343 */
344 @Override
345 public List<Violation> getViolations(ViolationQuery violationQuery) {
346 Resource resource = violationQuery.getResource();
347 if (resource == null) {
348 throw new IllegalArgumentException("A resource must be set on the ViolationQuery in order to search for violations.");
349 }
350
351 if (!Scopes.isHigherThanOrEquals(resource, Scopes.FILE)) {
352 return Collections.emptyList();
353 }
354
355 Bucket bucket = buckets.get(resource);
356 if (bucket == null) {
357 return Collections.emptyList();
358 }
359
360 List<Violation> violations = deprecatedViolations.get(bucket.getResource().getEffectiveKey());
361 if (violationQuery.getSwitchMode() == ViolationQuery.SwitchMode.BOTH) {
362 return violations;
363 }
364
365 List<Violation> filteredViolations = Lists.newArrayList();
366 for (Violation violation : violations) {
367 if (isFiltered(violation, violationQuery.getSwitchMode())) {
368 filteredViolations.add(violation);
369 }
370 }
371 return filteredViolations;
372 }
373
374 private static boolean isFiltered(Violation violation, ViolationQuery.SwitchMode mode) {
375 return mode == ViolationQuery.SwitchMode.BOTH || isSwitchOff(violation, mode) || isSwitchOn(violation, mode);
376 }
377
378 private static boolean isSwitchOff(Violation violation, ViolationQuery.SwitchMode mode) {
379 return mode == ViolationQuery.SwitchMode.OFF && violation.isSwitchedOff();
380 }
381
382 private static boolean isSwitchOn(Violation violation, ViolationQuery.SwitchMode mode) {
383 return mode == ViolationQuery.SwitchMode.ON && !violation.isSwitchedOff();
384 }
385
386 @Override
387 public void addViolation(Violation violation, boolean force) {
388 Resource resource = violation.getResource();
389 if (resource == null) {
390 violation.setResource(currentProject);
391 } else if (!Scopes.isHigherThanOrEquals(resource, Scopes.FILE)) {
392 throw new IllegalArgumentException("Violations are only supported on files, directories and project");
393 }
394
395 Rule rule = violation.getRule();
396 if (rule == null) {
397 LOG.warn("Rule is null. Ignoring violation {}", violation);
398 return;
399 }
400
401 Bucket bucket = getBucket(resource);
402 if (bucket == null) {
403 LOG.warn("Resource is not indexed. Ignoring violation {}", violation);
404 return;
405 }
406
407 // keep a limitation (bug?) of deprecated violations api : severity is always
408 // set by sonar. The severity set by plugins is overridden.
409 // This is not the case with issue api.
410 violation.setSeverity(null);
411
412 violation.setResource(bucket.getResource());
413 moduleIssues.initAndAddViolation(violation);
414 }
415
416 //
417 //
418 //
419 // LINKS
420 //
421 //
422 //
423
424 @Override
425 public void addLink(ProjectLink link) {
426 persistence.saveLink(currentProject, link);
427 }
428
429 @Override
430 public void deleteLink(String key) {
431 persistence.deleteLink(currentProject, key);
432 }
433
434 //
435 //
436 //
437 // EVENTS
438 //
439 //
440 //
441
442 @Override
443 public List<Event> getEvents(Resource resource) {
444 // currently events are not cached in memory
445 return persistence.getEvents(resource);
446 }
447
448 @Override
449 public void deleteEvent(Event event) {
450 persistence.deleteEvent(event);
451 }
452
453 @Override
454 public Event addEvent(Resource resource, String name, String description, String category, Date date) {
455 Event event = new Event(name, description, category);
456 event.setDate(date);
457 event.setCreatedAt(new Date());
458
459 persistence.saveEvent(resource, event);
460 return null;
461 }
462
463 @Override
464 public void setSource(Resource reference, String source) {
465 Bucket bucket = getBucket(reference);
466 if (bucket != null) {
467 persistence.setSource(reference, source);
468 }
469 }
470
471 @Override
472 public String getSource(Resource resource) {
473 return persistence.getSource(resource);
474 }
475
476 /**
477 * Does nothing if the resource is already registered.
478 */
479 @Override
480 public Resource addResource(Resource resource) {
481 Bucket bucket = doIndex(resource);
482 return bucket != null ? bucket.getResource() : null;
483 }
484
485 @Override
486 @CheckForNull
487 public <R extends Resource> R getResource(@Nullable R reference) {
488 Bucket bucket = getBucket(reference);
489 if (bucket != null) {
490 return (R) bucket.getResource();
491 }
492 return null;
493 }
494
495 @Override
496 public List<Resource> getChildren(Resource resource) {
497 List<Resource> children = Lists.newLinkedList();
498 Bucket bucket = getBucket(resource);
499 if (bucket != null) {
500 for (Bucket childBucket : bucket.getChildren()) {
501 children.add(childBucket.getResource());
502 }
503 }
504 return children;
505 }
506
507 @Override
508 public Resource getParent(Resource resource) {
509 Bucket bucket = getBucket(resource);
510 if (bucket != null && bucket.getParent() != null) {
511 return bucket.getParent().getResource();
512 }
513 return null;
514 }
515
516 @Override
517 public boolean index(Resource resource) {
518 Bucket bucket = doIndex(resource);
519 return bucket != null;
520 }
521
522 private Bucket doIndex(Resource resource) {
523 if (resource.getParent() != null) {
524 doIndex(resource.getParent());
525 }
526 return doIndex(resource, resource.getParent());
527 }
528
529 @Override
530 public boolean index(Resource resource, Resource parentReference) {
531 Bucket bucket = doIndex(resource, parentReference);
532 return bucket != null;
533 }
534
535 private Bucket doIndex(Resource resource, Resource parentReference) {
536 Bucket bucket = getBucket(resource);
537 if (bucket != null) {
538 return bucket;
539 }
540
541 if (StringUtils.isBlank(resource.getKey())) {
542 LOG.warn("Unable to index a resource without key " + resource);
543 return null;
544 }
545
546 Resource parent = null;
547 if (!ResourceUtils.isLibrary(resource)) {
548 // a library has no parent
549 parent = (Resource) ObjectUtils.defaultIfNull(parentReference, currentProject);
550 }
551
552 Bucket parentBucket = getBucket(parent);
553 if (parentBucket == null && parent != null) {
554 LOG.warn("Resource ignored, parent is not indexed: " + resource);
555 return null;
556 }
557
558 resource.setEffectiveKey(ComponentKeys.createEffectiveKey(currentProject, resource));
559 bucket = new Bucket(resource).setParent(parentBucket);
560 addBucket(resource, bucket);
561
562 Resource parentSnapshot = parentBucket != null ? parentBucket.getResource() : null;
563 Snapshot snapshot = persistence.saveResource(currentProject, resource, parentSnapshot);
564 if (ResourceUtils.isPersistable(resource) && !Qualifiers.LIBRARY.equals(resource.getQualifier())) {
565 graph.addComponent(resource, snapshot);
566 }
567
568 return bucket;
569 }
570
571 @Override
572 public boolean isExcluded(@Nullable Resource reference) {
573 return false;
574 }
575
576 @Override
577 public boolean isIndexed(@Nullable Resource reference, boolean acceptExcluded) {
578 return getBucket(reference) != null;
579 }
580
581 /**
582 * Should support 2 situations
583 * 1) key = new key and deprecatedKey = old key : this is the standard use case in a perfect world
584 * 2) key = null and deprecatedKey = oldKey : this is for plugins that are using deprecated constructors of
585 * {@link File} and {@link Directory}
586 */
587 private Bucket getBucket(@Nullable Resource reference) {
588 if (reference == null) {
589 return null;
590 }
591 if (StringUtils.isNotBlank(reference.getKey())) {
592 return buckets.get(reference);
593 }
594 if (StringUtils.isNotBlank(reference.getDeprecatedKey())) {
595 // Fallback to use deprecated key
596 Bucket bucket = bucketsByDeprecatedKey.get(reference.getDeprecatedKey());
597 if (bucket != null) {
598 // Fix reference resource
599 reference.setKey(bucket.getResource().getKey());
600 reference.setPath(bucket.getResource().getPath());
601 LOG.debug("Resource {} was found using deprecated key. Please update your plugin.", reference);
602 return bucket;
603 }
604 }
605 return null;
606 }
607
608 }