001 /*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2014 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
019 */
020 package org.sonar.batch.scan.filesystem;
021
022 import com.google.common.collect.Sets;
023 import org.apache.commons.io.FileUtils;
024 import org.apache.commons.io.filefilter.FileFilterUtils;
025 import org.apache.commons.io.filefilter.HiddenFileFilter;
026 import org.apache.commons.io.filefilter.IOFileFilter;
027 import org.slf4j.Logger;
028 import org.slf4j.LoggerFactory;
029 import org.sonar.api.BatchComponent;
030 import org.sonar.api.batch.bootstrap.ProjectDefinition;
031 import org.sonar.api.batch.fs.InputDir;
032 import org.sonar.api.batch.fs.InputFile;
033 import org.sonar.api.batch.fs.InputFileFilter;
034 import org.sonar.api.batch.fs.internal.DefaultInputDir;
035 import org.sonar.api.batch.fs.internal.DeprecatedDefaultInputFile;
036 import org.sonar.api.scan.filesystem.PathResolver;
037 import org.sonar.api.utils.MessageException;
038
039 import java.io.File;
040 import java.util.ArrayList;
041 import java.util.Collection;
042 import java.util.HashSet;
043 import java.util.List;
044 import java.util.Set;
045 import java.util.concurrent.Callable;
046 import java.util.concurrent.ExecutionException;
047 import java.util.concurrent.ExecutorService;
048 import java.util.concurrent.Executors;
049 import java.util.concurrent.Future;
050
051 /**
052 * Index input files into {@link InputPathCache}.
053 */
054 public class FileIndexer implements BatchComponent {
055
056 private static final Logger LOG = LoggerFactory.getLogger(FileIndexer.class);
057
058 private static final IOFileFilter DIR_FILTER = FileFilterUtils.and(HiddenFileFilter.VISIBLE, FileFilterUtils.notFileFilter(FileFilterUtils.prefixFileFilter(".")));
059 private static final IOFileFilter FILE_FILTER = HiddenFileFilter.VISIBLE;
060
061 private final List<InputFileFilter> filters;
062 private final InputPathCache fileCache;
063 private final boolean isAggregator;
064 private final ExclusionFilters exclusionFilters;
065 private final InputFileBuilderFactory inputFileBuilderFactory;
066
067 public FileIndexer(List<InputFileFilter> filters, ExclusionFilters exclusionFilters, InputFileBuilderFactory inputFileBuilderFactory,
068 InputPathCache cache, ProjectDefinition def) {
069 this(filters, exclusionFilters, inputFileBuilderFactory, cache, !def.getSubProjects().isEmpty());
070 }
071
072 private FileIndexer(List<InputFileFilter> filters, ExclusionFilters exclusionFilters, InputFileBuilderFactory inputFileBuilderFactory,
073 InputPathCache cache, boolean isAggregator) {
074 this.filters = filters;
075 this.exclusionFilters = exclusionFilters;
076 this.inputFileBuilderFactory = inputFileBuilderFactory;
077 this.fileCache = cache;
078 this.isAggregator = isAggregator;
079 }
080
081 void index(DefaultModuleFileSystem fileSystem) {
082 if (isAggregator) {
083 // No indexing for an aggregator module
084 return;
085 }
086 LOG.info("Index files");
087 exclusionFilters.prepare();
088
089 Progress progress = new Progress(fileCache.filesByModule(fileSystem.moduleKey()), fileCache.dirsByModule(fileSystem.moduleKey()));
090
091 InputFileBuilder inputFileBuilder = inputFileBuilderFactory.create(fileSystem);
092 indexFiles(fileSystem, progress, inputFileBuilder, fileSystem.sources(), InputFile.Type.MAIN);
093 indexFiles(fileSystem, progress, inputFileBuilder, fileSystem.tests(), InputFile.Type.TEST);
094
095 indexAllConcurrently(progress);
096
097 // Populate FS in a synchronous way because PersistIt Exchange is not concurrent
098 for (InputFile indexed : progress.indexed) {
099 fileSystem.add(indexed);
100 }
101 for (InputDir indexed : progress.indexedDir) {
102 fileSystem.add(indexed);
103 }
104
105 // Remove paths that have been removed since previous indexation
106 for (InputFile removed : progress.removed) {
107 fileCache.remove(fileSystem.moduleKey(), removed);
108 }
109 for (InputDir removed : progress.removedDir) {
110 fileCache.remove(fileSystem.moduleKey(), removed);
111 }
112
113 LOG.info(String.format("%d files indexed", progress.count()));
114
115 }
116
117 private void indexAllConcurrently(Progress progress) {
118 ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() + 1);
119 try {
120 List<Future<Void>> all = executor.invokeAll(progress.indexingTasks);
121 for (Future<Void> future : all) {
122 future.get();
123 }
124 } catch (InterruptedException e) {
125 throw new IllegalStateException("FileIndexer was interrupted", e);
126 } catch (ExecutionException e) {
127 Throwable cause = e.getCause();
128 if (cause instanceof RuntimeException) {
129 throw (RuntimeException) cause;
130 } else {
131 throw new IllegalStateException("Error during file indexing", e);
132 }
133 }
134 executor.shutdown();
135 }
136
137 private void indexFiles(DefaultModuleFileSystem fileSystem, Progress progress, InputFileBuilder inputFileBuilder, List<File> sources, InputFile.Type type) {
138 for (File dirOrFile : sources) {
139 if (dirOrFile.isDirectory()) {
140 indexDirectory(inputFileBuilder, fileSystem, progress, dirOrFile, type);
141 } else {
142 indexFile(inputFileBuilder, fileSystem, progress, dirOrFile, type);
143 }
144 }
145 }
146
147 private void indexDirectory(InputFileBuilder inputFileBuilder, DefaultModuleFileSystem fileSystem, Progress status, File dirToIndex, InputFile.Type type) {
148 Collection<File> files = FileUtils.listFiles(dirToIndex, FILE_FILTER, DIR_FILTER);
149 for (File file : files) {
150 indexFile(inputFileBuilder, fileSystem, status, file, type);
151 }
152 }
153
154 private void indexFile(InputFileBuilder inputFileBuilder, DefaultModuleFileSystem fileSystem, Progress progress, File sourceFile, InputFile.Type type) {
155 DeprecatedDefaultInputFile inputFile = inputFileBuilder.create(sourceFile);
156 if (inputFile != null && exclusionFilters.accept(inputFile, type)) {
157 indexFile(inputFileBuilder, fileSystem, progress, inputFile, type);
158 }
159 }
160
161 private void indexFile(final InputFileBuilder inputFileBuilder, final DefaultModuleFileSystem fs,
162 final Progress status, final DeprecatedDefaultInputFile inputFile, final InputFile.Type type) {
163
164 Callable<Void> task = new Callable<Void>() {
165
166 @Override
167 public Void call() throws Exception {
168 InputFile completedFile = inputFileBuilder.complete(inputFile, type);
169 if (completedFile != null && accept(completedFile)) {
170 status.markAsIndexed(inputFile);
171 File parentDir = inputFile.file().getParentFile();
172 String relativePath = new PathResolver().relativePath(fs.baseDir(), parentDir);
173 if (relativePath != null) {
174 DefaultInputDir inputDir = new DefaultInputDir(relativePath);
175 inputDir.setFile(parentDir);
176 inputDir.setKey(new StringBuilder().append(fs.moduleKey()).append(":").append(inputDir.relativePath()).toString());
177 status.markAsIndexed(inputDir);
178 }
179 }
180 return null;
181 }
182 };
183 status.planForIndexing(task);
184 }
185
186 private boolean accept(InputFile inputFile) {
187 // InputFileFilter extensions
188 for (InputFileFilter filter : filters) {
189 if (!filter.accept(inputFile)) {
190 return false;
191 }
192 }
193 return true;
194 }
195
196 private static class Progress {
197 private final Set<InputFile> removed;
198 private final Set<InputDir> removedDir;
199 private final Set<InputFile> indexed;
200 private final Set<InputDir> indexedDir;
201 private final List<Callable<Void>> indexingTasks;
202
203 Progress(Iterable<InputFile> removed, Iterable<InputDir> removedDir) {
204 this.removed = Sets.newHashSet(removed);
205 this.removedDir = Sets.newHashSet(removedDir);
206 this.indexed = new HashSet<InputFile>();
207 this.indexedDir = new HashSet<InputDir>();
208 this.indexingTasks = new ArrayList<Callable<Void>>();
209 }
210
211 void planForIndexing(Callable<Void> indexingTask) {
212 this.indexingTasks.add(indexingTask);
213 }
214
215 synchronized void markAsIndexed(InputFile inputFile) {
216 if (indexed.contains(inputFile)) {
217 throw MessageException.of("File " + inputFile + " can't be indexed twice. Please check that inclusion/exclusion patterns produce "
218 + "disjoint sets for main and test files");
219 }
220 removed.remove(inputFile);
221 indexed.add(inputFile);
222 }
223
224 synchronized void markAsIndexed(InputDir inputDir) {
225 removedDir.remove(inputDir);
226 indexedDir.add(inputDir);
227 }
228
229 int count() {
230 return indexed.size();
231 }
232 }
233
234 }