1 package fr.ifremer.tutti.service.genericformat;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27 import fr.ifremer.tutti.persistence.ProgressionModel;
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.nuiton.jaxx.application.ApplicationIOUtil;
31 import org.nuiton.jaxx.application.ApplicationTechnicalException;
32 import org.nuiton.util.ZipUtil;
33
34 import java.io.File;
35 import java.io.IOException;
36 import java.io.LineNumberReader;
37 import java.io.Serializable;
38 import java.nio.charset.Charset;
39 import java.nio.file.Files;
40 import java.nio.file.Path;
41 import java.util.EnumMap;
42 import java.util.EnumSet;
43 import java.util.LinkedHashSet;
44 import java.util.Set;
45
46 import static org.nuiton.i18n.I18n.t;
47
48
49
50
51
52
53
54 public class GenericFormatArchive implements Serializable {
55
56 private static final long serialVersionUID = 1L;
57
58
59 private static final Log log = LogFactory.getLog(GenericFormatArchive.class);
60
61 private enum ArchiveMode {
62 IMPORT,
63 EXPORT
64 }
65
66 private enum ArchiveFilePath {
67
68 PROTOCOL(false, "protocol.tuttiProtocol"),
69 REFERENTIAL_GEAR(false, "temporaryGears.csv"),
70 REFERENTIAL_PERSON(false, "temporaryPersons.csv"),
71 REFERENTIAL_SPECIES(false, "temporarySpecies.csv"),
72 REFERENTIAL_VESSEL(false, "temporaryVessels.csv"),
73 SAMPLE_CATEGORY(true, "sampleCategory.csv"),
74 DATA_SURVEY(true, "survey.csv"),
75 DATA_GEAR_CARACTERISTIC(true, "gearCaracteristics.csv"),
76 DATA_OPERATION(true, "operation.csv"),
77 DATA_PARAMETER(true, "parameter.csv"),
78 DATA_CATCH(true, "catch.csv"),
79 DATA_SPECIES(false, "species.csv"),
80 DATA_MARINE_LITTER(true, "marineLitter.csv"),
81 DATA_ACCIDENTAL_CATCH(true, "accidentalCatch.csv"),
82 DATA_INDIVIDUAL_OBSERVATION(true, "individualObservation.csv"),
83 DATA_ATTACHMENTS(true, "attachments.csv");
84
85 private final boolean mandatory;
86
87 private final String filename;
88
89 ArchiveFilePath(boolean mandatory, String filename) {
90 this.mandatory = mandatory;
91 this.filename = filename;
92 }
93
94 public boolean isMandatory() {
95 return mandatory;
96 }
97
98 public String getFilename() {
99 return filename;
100 }
101
102 }
103
104 private final File archiveFile;
105
106 private final File workingDirectory;
107
108 private final ArchiveMode archiveMode;
109
110 private final EnumMap<ArchiveFilePath, Integer> countLines;
111
112 private final EnumSet<ArchiveFilePath> missingPaths;
113
114 public static GenericFormatArchive forImport(File archiveFile, File tempDirectory) {
115
116 try {
117
118 File workingDirectory = Files.createTempDirectory(tempDirectory.toPath(), "genericImport").toFile();
119 return new GenericFormatArchive(ArchiveMode.IMPORT, archiveFile, workingDirectory);
120
121 } catch (IOException e) {
122 throw new ApplicationTechnicalException("Could not create generic format import archive", e);
123 }
124
125 }
126
127 public static GenericFormatArchive forExport(File archiveFile, File tempDirectory) {
128
129 try {
130
131 File workingDirectory = Files.createTempDirectory(tempDirectory.toPath(), "genericExport").toFile();
132 return new GenericFormatArchive(ArchiveMode.EXPORT, archiveFile, workingDirectory);
133
134 } catch (IOException e) {
135 throw new ApplicationTechnicalException("Could not create generic format export archive", e);
136 }
137
138 }
139
140 public static GenericFormatArchive forExportFromWorkingDirectory(File archiveFile, File workingDirectory) {
141
142 return new GenericFormatArchive(ArchiveMode.EXPORT, archiveFile, workingDirectory);
143
144 }
145
146 public File getWorkingDirectoryPath() {
147 return workingDirectory;
148 }
149
150 public Path getSampleCategoryModelPath() {
151 return getPath(ArchiveFilePath.SAMPLE_CATEGORY);
152 }
153
154 public Path getAttachmentFilePath() {
155 return getPath(ArchiveFilePath.DATA_ATTACHMENTS);
156 }
157
158 public Path getAttachmentDataPath() {
159 return workingDirectory.toPath().resolve("meas_files");
160 }
161
162 public boolean isProtocolExists() {
163 return Files.exists(getProtocolPath());
164 }
165
166 public boolean isTemporaryReferentialGearsPathExists() {
167 return Files.exists(getTemporaryReferentialGearsPath());
168 }
169
170 public boolean isTemporaryReferentialPersonsPathExists() {
171 return Files.exists(getTemporaryReferentialPersonsPath());
172 }
173
174 public boolean isTemporaryReferentialSpeciesPathExists() {
175 return Files.exists(getTemporaryReferentialSpeciesPath());
176 }
177
178 public boolean isTemporaryReferentialVesselsPathExists() {
179 return Files.exists(getTemporaryReferentialVesselsPath());
180 }
181
182 public Path getProtocolPath() {
183 return getPath(ArchiveFilePath.PROTOCOL);
184 }
185
186 public Path getTemporaryReferentialGearsPath() {
187 return getPath(ArchiveFilePath.REFERENTIAL_GEAR);
188 }
189
190 public Path getTemporaryReferentialPersonsPath() {
191 return getPath(ArchiveFilePath.REFERENTIAL_PERSON);
192 }
193
194 public Path getTemporaryReferentialSpeciesPath() {
195 return getPath(ArchiveFilePath.REFERENTIAL_SPECIES);
196 }
197
198 public Path getTemporaryReferentialVesselsPath() {
199 return getPath(ArchiveFilePath.REFERENTIAL_VESSEL);
200 }
201
202 public Path getSurveyPath() {
203 return getPath(ArchiveFilePath.DATA_SURVEY);
204 }
205
206 public Path getGearCaracteristicsPath() {
207 return getPath(ArchiveFilePath.DATA_GEAR_CARACTERISTIC);
208 }
209
210 public Path getOperationPath() {
211 return getPath(ArchiveFilePath.DATA_OPERATION);
212 }
213
214 public Path getIndividualObservationPath() {
215 return getPath(ArchiveFilePath.DATA_INDIVIDUAL_OBSERVATION);
216 }
217
218 public Path getSpeciesPath() {
219 return getPath(ArchiveFilePath.DATA_SPECIES);
220 }
221
222 public Path getCatchPath() {
223 return getPath(ArchiveFilePath.DATA_CATCH);
224 }
225
226 public Path getAccidentalCatchPath() {
227 return getPath(ArchiveFilePath.DATA_ACCIDENTAL_CATCH);
228 }
229
230 public Path getParameterPath() {
231 return getPath(ArchiveFilePath.DATA_PARAMETER);
232 }
233
234 public Path getMarineLitterPath() {
235 return getPath(ArchiveFilePath.DATA_MARINE_LITTER);
236 }
237
238 public int getSampleCategoryLineCount() {
239 return countImportLines(ArchiveFilePath.SAMPLE_CATEGORY);
240 }
241
242 public int getSurveyLineCount() {
243 return countImportLines(ArchiveFilePath.DATA_SURVEY);
244 }
245
246 public int getGearCaracteristicsPathLineCount() {
247 return countImportLines(ArchiveFilePath.DATA_GEAR_CARACTERISTIC);
248 }
249
250 public int getOperationPathLineCount() {
251 return countImportLines(ArchiveFilePath.DATA_OPERATION);
252 }
253
254 public int getIndividualObservationPathLineCount() {
255 return countImportLines(ArchiveFilePath.DATA_INDIVIDUAL_OBSERVATION);
256 }
257
258 public int getSpeciesPathLineCount() {
259 return countImportLines(ArchiveFilePath.DATA_SPECIES);
260 }
261
262 public int getCatchPathLineCount() {
263 return countImportLines(ArchiveFilePath.DATA_CATCH);
264 }
265
266 public int getAccidentalCatchPathLineCount() {
267 return countImportLines(ArchiveFilePath.DATA_ACCIDENTAL_CATCH);
268 }
269
270 public int getAttachemntsPathLineCount() {
271 return countImportLines(ArchiveFilePath.DATA_ATTACHMENTS);
272 }
273
274 public int getParameterPathLineCount() {
275 return countImportLines(ArchiveFilePath.DATA_PARAMETER);
276 }
277
278 public int getMarineLitterPathLineCount() {
279 return countImportLines(ArchiveFilePath.DATA_MARINE_LITTER);
280 }
281
282 public int getTemporaryReferentialGearLineCount() {
283 return countImportLines(ArchiveFilePath.REFERENTIAL_GEAR);
284 }
285
286 public int getTemporaryReferentialPersonLineCount() {
287 return countImportLines(ArchiveFilePath.REFERENTIAL_PERSON);
288 }
289
290 public int getTemporaryReferentialSpeciesLineCount() {
291 return countImportLines(ArchiveFilePath.REFERENTIAL_SPECIES);
292 }
293
294 public int getTemporaryReferentialVesselLineCount() {
295 return countImportLines(ArchiveFilePath.REFERENTIAL_VESSEL);
296 }
297
298
299 public void validateArchiveLayout() throws GenericFormatArchiveInvalidLayoutException {
300
301 Set<String> errors = new LinkedHashSet<>();
302
303 for (ArchiveFilePath archiveFilePath : ArchiveFilePath.values()) {
304
305 if (archiveFilePath.isMandatory() && missingPaths.contains(archiveFilePath)) {
306
307
308 if (log.isErrorEnabled()) {
309 log.error("Mandatory entry " + archiveFilePath.getFilename() + " not found.");
310 }
311 errors.add(t("tutti.service.genericFormat.importError.missArchiveFile", archiveFilePath.getFilename()));
312
313 }
314 }
315
316 if (!errors.isEmpty()) {
317
318 throw new GenericFormatArchiveInvalidLayoutException(this, errors);
319
320 }
321
322 }
323
324 public void createZip(ProgressionModel progressionModel) {
325
326 if (progressionModel != null) {
327
328 progressionModel.increments(t("tutti.service.genericFormat.export.buildZip", archiveFile));
329
330 }
331
332 ApplicationIOUtil.zip(workingDirectory, archiveFile, t("tutti.service.genericFormat.export.zip.error", archiveFile));
333
334 }
335
336 public File extractAttachment(String path) {
337
338 Path resolve = getAttachmentDataPath().resolve(path);
339 return resolve.toFile();
340
341 }
342
343 protected GenericFormatArchive(ArchiveMode archiveMode, File archiveFile, File workingDirectory) {
344
345 this.archiveFile = archiveFile;
346 this.archiveMode = archiveMode;
347 this.countLines = new EnumMap<>(ArchiveFilePath.class);
348
349 if (log.isInfoEnabled()) {
350 log.info("Archive zip file: " + archiveFile);
351 log.info("Archive working directory: " + workingDirectory);
352 log.info("Archive mode: " + archiveMode);
353 }
354 if (isImport()) {
355
356 this.workingDirectory = extractArchive(archiveFile, workingDirectory);
357 this.missingPaths = computeMissingPaths();
358
359 } else {
360
361 this.workingDirectory = workingDirectory;
362 this.missingPaths = null;
363
364 }
365
366 }
367
368 protected File extractArchive(File archiveFile, File workingDirectory) {
369
370 try {
371 ZipUtil.uncompress(archiveFile, workingDirectory);
372 } catch (IOException e) {
373 throw new ApplicationTechnicalException("Could not explode zipfile " + archiveFile, e);
374 }
375 File[] files = workingDirectory.listFiles();
376 if (files == null || files.length != 1) {
377 throw new ApplicationTechnicalException("Archive should contains only one directory");
378 }
379 return files[0];
380
381 }
382
383 protected boolean isImport() {
384 return ArchiveMode.IMPORT == archiveMode;
385 }
386
387 protected boolean isExport() {
388 return ArchiveMode.EXPORT == archiveMode;
389 }
390
391 protected Path getPath(ArchiveFilePath archiveFilePath) {
392
393 String filename = archiveFilePath.getFilename();
394
395 return workingDirectory.toPath().resolve(filename);
396
397 }
398
399 protected EnumSet<ArchiveFilePath> computeMissingPaths() {
400
401 EnumSet<ArchiveFilePath> result = EnumSet.noneOf(ArchiveFilePath.class);
402
403 for (ArchiveFilePath archiveFilePath : ArchiveFilePath.values()) {
404
405 Path path = getPath(archiveFilePath);
406
407 if (log.isDebugEnabled()) {
408 log.debug("Check if entry " + archiveFilePath + " exists.");
409 }
410
411 if (!Files.exists(path)) {
412
413 if (log.isInfoEnabled()) {
414 log.info("Entry " + archiveFilePath + " not found.");
415 }
416
417 result.add(archiveFilePath);
418
419 } else {
420
421 if (log.isInfoEnabled()) {
422 log.info("Entry " + archiveFilePath + " found.");
423 }
424
425 }
426
427 }
428
429 return result;
430
431 }
432
433 protected int countImportLines(ArchiveFilePath archiveFilePath) {
434
435 Integer result = countLines.get(archiveFilePath);
436 if (result == null) {
437
438 Path path = getPath(archiveFilePath);
439
440 if (path == null) {
441
442 result = 0;
443 countLines.put(archiveFilePath, 0);
444
445 } else {
446
447 int nbLines = countLines(path);
448 result = nbLines - 1;
449 countLines.put(archiveFilePath, result);
450
451 }
452
453 }
454
455 return result;
456
457 }
458
459 protected static int countLines(Path path) {
460
461 try (LineNumberReader reader = new LineNumberReader(Files.newBufferedReader(path, Charset.forName("UTF-8")))) {
462 String lastLine = null;
463 String line;
464 while ((line = reader.readLine()) != null) {
465 lastLine = line;
466 }
467 int cnt = reader.getLineNumber();
468 if (lastLine != null && lastLine.trim().isEmpty()) {
469 cnt--;
470 }
471 return cnt;
472 } catch (IOException e) {
473 throw new ApplicationTechnicalException("Could not read file " + path.toFile().getName(), e);
474 }
475
476 }
477
478 }