1 package fr.ifremer.tutti.service.protocol;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 import com.google.common.base.Preconditions;
26 import com.google.common.collect.ArrayListMultimap;
27 import com.google.common.collect.Multimap;
28 import com.google.common.io.Files;
29 import fr.ifremer.tutti.persistence.entities.protocol.CalcifiedPiecesSamplingDefinition;
30 import fr.ifremer.tutti.persistence.entities.protocol.CalcifiedPiecesSamplingDefinitions;
31 import fr.ifremer.tutti.persistence.entities.protocol.Rtp;
32 import fr.ifremer.tutti.persistence.entities.protocol.SpeciesProtocol;
33 import fr.ifremer.tutti.persistence.entities.protocol.SpeciesProtocols;
34 import fr.ifremer.tutti.persistence.entities.protocol.TuttiProtocol;
35 import fr.ifremer.tutti.persistence.entities.protocol.TuttiProtocols;
36 import fr.ifremer.tutti.persistence.entities.referential.Caracteristic;
37 import fr.ifremer.tutti.persistence.entities.referential.Species;
38 import fr.ifremer.tutti.service.AbstractTuttiService;
39 import org.apache.commons.collections4.CollectionUtils;
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.nuiton.csv.Export;
43 import org.nuiton.csv.Import;
44 import org.nuiton.csv.ImportRuntimeException;
45 import org.nuiton.jaxx.application.ApplicationTechnicalException;
46 import org.nuiton.util.beans.Binder;
47 import org.nuiton.util.beans.BinderFactory;
48
49 import java.io.BufferedWriter;
50 import java.io.File;
51 import java.io.Reader;
52 import java.nio.charset.StandardCharsets;
53 import java.util.ArrayList;
54 import java.util.Collection;
55 import java.util.Comparator;
56 import java.util.HashSet;
57 import java.util.LinkedHashMap;
58 import java.util.LinkedHashSet;
59 import java.util.List;
60 import java.util.Map;
61 import java.util.Set;
62 import java.util.function.Function;
63 import java.util.stream.Collectors;
64
65 import static org.nuiton.i18n.I18n.t;
66
67
68
69
70
71
72
73 public class ProtocolImportExportService extends AbstractTuttiService {
74
75
76 private static final Log log = LogFactory.getLog(ProtocolImportExportService.class);
77
78 public void exportProtocol(TuttiProtocol protocol, File file) {
79 TuttiProtocols.toFile(protocol, file);
80 }
81
82 public TuttiProtocol importProtocol(File file) {
83 return TuttiProtocols.fromFile(file);
84 }
85
86 public List<Species> importProtocolSpecies(File file,
87 TuttiProtocol protocol,
88 Map<String, Caracteristic> caracteristicMap,
89 Map<String, Species> speciesMap) {
90
91 if (log.isInfoEnabled()) {
92 log.info("Will import protocol [" + protocol.getName() + "] species from file: " + file);
93 }
94
95 List<Species> result = new ArrayList<>();
96
97 Map<Integer, SpeciesProtocol> ids = new LinkedHashMap<>();
98
99 if (!protocol.isSpeciesEmpty()) {
100
101
102
103 for (SpeciesProtocol speciesProtocol : protocol.getSpecies()) {
104 ids.put(speciesProtocol.getSpeciesReferenceTaxonId(), speciesProtocol);
105 }
106
107 }
108
109 Map<Integer, SpeciesProtocol> benthosIds = new LinkedHashMap<>();
110
111 if (!protocol.isBenthosEmpty()) {
112
113
114
115 for (SpeciesProtocol speciesProtocol : protocol.getBenthos()) {
116 benthosIds.put(speciesProtocol.getSpeciesReferenceTaxonId(), speciesProtocol);
117 }
118 }
119
120 try (Reader reader = Files.newReader(file, StandardCharsets.UTF_8)) {
121
122 SpeciesRowModel csvModel = SpeciesRowModel.forImport(getCsvSeparator(), caracteristicMap, speciesMap);
123 try (Import<SpeciesRow> importer = Import.newImport(csvModel, reader)) {
124
125 Binder<SpeciesRow, SpeciesProtocol> binder =
126 BinderFactory.newBinder(SpeciesRow.class, SpeciesProtocol.class);
127
128 for (SpeciesRow bean : importer) {
129
130 Species species = bean.getSpecies();
131 Integer id = species.getReferenceTaxonId();
132
133 SpeciesProtocol sp = benthosIds.get(id);
134 if (sp != null) {
135 result.add(species);
136
137 } else {
138
139 sp = ids.get(id);
140 if (sp == null) {
141
142
143 sp = SpeciesProtocols.newSpeciesProtocol();
144 }
145 binder.copy(bean, sp);
146 sp.setMandatorySampleCategoryId(new ArrayList<>(bean.getMandatorySampleCategoryId()));
147
148 cleanRptData(sp);
149
150 ids.put(id, sp);
151 }
152 }
153 }
154
155 List<SpeciesProtocol> values = new ArrayList<>(ids.values());
156 protocol.setSpecies(values);
157
158 return result;
159
160 } catch (ImportRuntimeException e) {
161 throw e;
162 } catch (Exception e) {
163 throw new ApplicationTechnicalException(t("tutti.service.protocol.import.species.error", protocol.getName(), file), e);
164 }
165
166 }
167
168
169
170
171
172
173
174
175 public List<Species> importProtocolBenthos(File file,
176 TuttiProtocol protocol,
177 Map<String, Caracteristic> caracteristicMap,
178 Map<String, Species> speciesMap) {
179
180 if (log.isInfoEnabled()) {
181 log.info("Will import protocol [" + protocol.getName() + "] species from file: " + file);
182 }
183
184 List<Species> result = new ArrayList<>();
185
186 Map<Integer, SpeciesProtocol> ids = new LinkedHashMap<>();
187
188 if (!protocol.isBenthosEmpty()) {
189
190
191
192 for (SpeciesProtocol speciesProtocol : protocol.getBenthos()) {
193 ids.put(speciesProtocol.getSpeciesReferenceTaxonId(), speciesProtocol);
194 }
195 }
196
197 Map<Integer, SpeciesProtocol> speciesIds = new LinkedHashMap<>();
198
199 if (!protocol.isSpeciesEmpty()) {
200
201
202
203 for (SpeciesProtocol speciesProtocol : protocol.getSpecies()) {
204 speciesIds.put(speciesProtocol.getSpeciesReferenceTaxonId(), speciesProtocol);
205 }
206 }
207
208 try (Reader reader = Files.newReader(file, StandardCharsets.UTF_8)) {
209
210 SpeciesRowModel csvModel = SpeciesRowModel.forImport(getCsvSeparator(), caracteristicMap, speciesMap);
211 try (Import<SpeciesRow> importer = Import.newImport(csvModel, reader)) {
212
213 Binder<SpeciesRow, SpeciesProtocol> binder =
214 BinderFactory.newBinder(SpeciesRow.class, SpeciesProtocol.class);
215
216 for (SpeciesRow bean : importer) {
217
218 Species species = bean.getSpecies();
219 Integer id = species.getReferenceTaxonId();
220
221 SpeciesProtocol sp = speciesIds.get(id);
222 if (sp != null) {
223 result.add(species);
224
225 } else {
226
227 sp = ids.get(id);
228 if (sp == null) {
229
230
231 sp = SpeciesProtocols.newSpeciesProtocol();
232 }
233 binder.copy(bean, sp);
234
235 cleanRptData(sp);
236
237 ids.put(id, sp);
238 }
239 }
240
241 List<SpeciesProtocol> values = new ArrayList<>(ids.values());
242 protocol.setBenthos(values);
243
244 }
245
246 } catch (ImportRuntimeException e) {
247 throw e;
248 } catch (Exception e) {
249 throw new ApplicationTechnicalException(t("tutti.service.protocol.import.benthos.error", protocol.getName(), file), e);
250 }
251
252 return result;
253 }
254
255 public void exportProtocolSpecies(File file,
256 List<SpeciesProtocol> protocol,
257 Map<String, Caracteristic> caracteristicMap,
258 Map<String, Species> speciesMap) {
259 if (log.isInfoEnabled()) {
260 log.info("Will export species to file: " + file);
261 }
262
263 List<SpeciesRow> rows;
264
265 if (CollectionUtils.isEmpty(protocol)) {
266
267 rows = null;
268
269 } else {
270
271 rows = protocol.stream().map(new SpeciesProtocolToSpeciesRowFunction(caracteristicMap, speciesMap)).collect(Collectors.toList());
272
273 }
274
275 try (BufferedWriter writer = Files.newWriter(file, StandardCharsets.UTF_8)) {
276
277 SpeciesRowModel csvModel = SpeciesRowModel.forExport(getCsvSeparator());
278 Export export = Export.newExport(csvModel, rows);
279 export.write(writer);
280
281 } catch (Exception e) {
282 throw new ApplicationTechnicalException(t("tutti.service.protocol.export.species.error", file), e);
283 }
284 }
285
286 public void exportProtocolBenthos(File file,
287 List<SpeciesProtocol> protocol,
288 Map<String, Caracteristic> caracteristicMap,
289 Map<String, Species> speciesMap) {
290 if (log.isInfoEnabled()) {
291 log.info("Will export benthos to file: " + file);
292 }
293
294 List<SpeciesRow> rows;
295
296 if (CollectionUtils.isEmpty(protocol)) {
297
298 rows = null;
299
300 } else {
301
302 rows = protocol.stream().map(new SpeciesProtocolToSpeciesRowFunction(caracteristicMap, speciesMap)).collect(Collectors.toList());
303
304 }
305
306 try (BufferedWriter writer = Files.newWriter(file, StandardCharsets.UTF_8)) {
307
308 SpeciesRowModel csvModel = SpeciesRowModel.forExport(getCsvSeparator());
309 Export export = Export.newExport(csvModel, rows);
310 export.write(writer);
311
312 } catch (Exception e) {
313 throw new ApplicationTechnicalException(t("tutti.service.protocol.export.benthos.error", file), e);
314 }
315 }
316
317
318
319
320
321
322
323
324 public Set<Species> importCalcifiedPiecesSamplings(File file,
325 TuttiProtocol protocol,
326 Map<String, Species> allSpecies) {
327
328 if (log.isInfoEnabled()) {
329 log.info("Will import protocol [" + protocol.getName() + "] cps from file: " + file);
330 }
331
332 Set<Species> result = new HashSet<>();
333
334 try (Reader reader = Files.newReader(file, StandardCharsets.UTF_8)) {
335
336 CalcifiedPiecesSamplingRowModel csvModel = CalcifiedPiecesSamplingRowModel.forImport(getCsvSeparator(), allSpecies);
337 try (Import<CalcifiedPiecesSamplingRow> importer = Import.newImport(csvModel, reader)) {
338
339 Binder<CalcifiedPiecesSamplingRow, CalcifiedPiecesSamplingDefinition> binder =
340 BinderFactory.newBinder(CalcifiedPiecesSamplingRow.class, CalcifiedPiecesSamplingDefinition.class);
341
342 Map<Integer, SpeciesProtocol> availableSpeciesProtocolMap = availableSpeciesProtocolMap(protocol);
343
344
345 ArrayListMultimap<SpeciesProtocol, CalcifiedPiecesSamplingDefinition> calcifiedPiecesSamplingDefinitionsWithoutMaturity = ArrayListMultimap.create();
346
347 ArrayListMultimap<SpeciesProtocol, CalcifiedPiecesSamplingDefinition> calcifiedPiecesSamplingDefinitionsWithTrueMaturity = ArrayListMultimap.create();
348
349 ArrayListMultimap<SpeciesProtocol, CalcifiedPiecesSamplingDefinition> calcifiedPiecesSamplingDefinitionsWithFalseMaturity = ArrayListMultimap.create();
350
351
352 Set<SpeciesProtocol> speciesProtocolsWithAndWithoutMaturity = new LinkedHashSet<>();
353
354 for (CalcifiedPiecesSamplingRow bean : importer) {
355
356 Species species = bean.getSpecies();
357
358 if (result.contains(species)) {
359
360
361 continue;
362 }
363
364 SpeciesProtocol speciesProtocol = availableSpeciesProtocolMap.get(species.getReferenceTaxonId());
365
366 if (speciesProtocol == null) {
367
368 if (log.isDebugEnabled()) {
369 log.debug("Skip species: " + species + ", not found in protocol.");
370 }
371 result.add(species);
372 continue;
373
374 }
375
376 if (speciesProtocol.getLengthStepPmfmId() == null) {
377
378 if (log.isDebugEnabled()) {
379 log.debug("Skip species: " + species + ", found in protocol, but with no length class.");
380 }
381 result.add(species);
382 continue;
383
384 }
385
386 Multimap<SpeciesProtocol, CalcifiedPiecesSamplingDefinition> calcifiedPiecesSamplingDefinitions;
387
388 CalcifiedPiecesSamplingDefinition definition = CalcifiedPiecesSamplingDefinitions.newCalcifiedPiecesSamplingDefinition(bean, binder);
389
390 Boolean maturity = definition.getMaturity();
391
392 if (maturity == null) {
393
394 if (log.isDebugEnabled()) {
395 log.debug("On species: " + species + ", add definition with no maturity defined (" + definition + ")");
396 }
397
398 calcifiedPiecesSamplingDefinitions = calcifiedPiecesSamplingDefinitionsWithoutMaturity;
399
400 if (calcifiedPiecesSamplingDefinitionsWithTrueMaturity.containsKey(speciesProtocol) ||
401 calcifiedPiecesSamplingDefinitionsWithFalseMaturity.containsKey(speciesProtocol)) {
402
403
404 speciesProtocolsWithAndWithoutMaturity.add(speciesProtocol);
405 }
406
407 } else {
408
409 if (calcifiedPiecesSamplingDefinitionsWithoutMaturity.containsKey(speciesProtocol)) {
410
411
412 speciesProtocolsWithAndWithoutMaturity.add(speciesProtocol);
413 }
414
415 if (maturity) {
416
417 if (log.isDebugEnabled()) {
418 log.debug("On species: " + species + ", add definition with maturity defined to True (" + definition + ")");
419 }
420
421 calcifiedPiecesSamplingDefinitions = calcifiedPiecesSamplingDefinitionsWithTrueMaturity;
422 } else {
423
424 if (log.isDebugEnabled()) {
425 log.debug("On species: " + species + ", add definition with maturity defined to False (" + definition + ")");
426 }
427 calcifiedPiecesSamplingDefinitions = calcifiedPiecesSamplingDefinitionsWithFalseMaturity;
428 }
429 }
430
431 calcifiedPiecesSamplingDefinitions.put(speciesProtocol, definition);
432
433 }
434
435 if (!speciesProtocolsWithAndWithoutMaturity.isEmpty()) {
436
437
438
439 String badSpecies = speciesProtocolsWithAndWithoutMaturity.stream()
440 .map(speciesProtocol -> {
441 String speciesToString = speciesProtocol.getSpeciesReferenceTaxonId() + "";
442 if (speciesProtocol.getSpeciesSurveyCode() != null) {
443 speciesToString += " (" + speciesProtocol.getSpeciesSurveyCode() + " )";
444 }
445 return speciesToString;
446 })
447 .collect(Collectors.joining("", "<li>", "</li>"));
448 throw new ImportRuntimeException(t("tutti.service.protocol.import.cps.speciesWithMaturityAndWithoutMaturity.error", badSpecies));
449
450 }
451
452 Comparator<CalcifiedPiecesSamplingDefinition> definitionsComparator = Comparator.comparing(CalcifiedPiecesSamplingDefinition::getMinSize);
453
454 checkCpfDefinitionsValidity(definitionsComparator, calcifiedPiecesSamplingDefinitionsWithoutMaturity);
455 checkCpfDefinitionsValidity(definitionsComparator, calcifiedPiecesSamplingDefinitionsWithTrueMaturity);
456 checkCpfDefinitionsValidity(definitionsComparator, calcifiedPiecesSamplingDefinitionsWithFalseMaturity);
457
458 }
459
460 } catch (ImportRuntimeException e) {
461 throw e;
462 } catch (Exception e) {
463 throw new ApplicationTechnicalException(t("tutti.service.protocol.import.cps.error", protocol.getName(), file), e);
464 }
465
466 return result;
467 }
468
469 public void exportCalcifiedPiecesSamplings(File file,
470 Multimap<SpeciesProtocol, CalcifiedPiecesSamplingDefinition> cps,
471 Map<String, Species> speciesByReferenceTaxonId) {
472
473 if (log.isInfoEnabled()) {
474 log.info("Will export cps to file: " + file);
475 }
476
477 List<CalcifiedPiecesSamplingRow> rows = new ArrayList<>();
478
479 Binder<CalcifiedPiecesSamplingDefinition, CalcifiedPiecesSamplingRow> binder =
480 BinderFactory.newBinder(CalcifiedPiecesSamplingDefinition.class, CalcifiedPiecesSamplingRow.class);
481
482 cps.keySet().forEach(speciesProtocol -> {
483
484 Collection<CalcifiedPiecesSamplingDefinition> calcifiedPiecesSamplingDefinitions = cps.get(speciesProtocol);
485
486 calcifiedPiecesSamplingDefinitions.forEach(cpsDef -> {
487
488 CalcifiedPiecesSamplingRow row = new CalcifiedPiecesSamplingRow();
489 binder.copy(cpsDef, row);
490 String refTaxId = String.valueOf(speciesProtocol.getSpeciesReferenceTaxonId());
491 Species species = speciesByReferenceTaxonId.get(refTaxId);
492 row.setSpecies(species);
493 rows.add(row);
494
495 });
496 });
497
498 try (BufferedWriter writer = Files.newWriter(file, StandardCharsets.UTF_8)) {
499
500 CalcifiedPiecesSamplingRowModel csvModel = CalcifiedPiecesSamplingRowModel.forExport(getCsvSeparator());
501 Export export = Export.newExport(csvModel, rows);
502 export.write(writer);
503
504 } catch (Exception e) {
505 throw new ApplicationTechnicalException(t("tutti.service.protocol.export.cps.error", file), e);
506 }
507 }
508
509 protected char getCsvSeparator() {
510 return context.getConfig().getCsvSeparator();
511 }
512
513 protected void cleanRptData(SpeciesProtocol sp) {
514 if (sp.withRtpMale()) {
515 Rtp rtpMale = sp.getRtpMale();
516 if (rtpMale.getA() == null && rtpMale.getB() == null) {
517 sp.setRtpMale(null);
518 }
519 }
520 if (sp.withRtpFemale()) {
521 Rtp rtpFemale = sp.getRtpFemale();
522 if (rtpFemale.getA() == null && rtpFemale.getB() == null) {
523 sp.setRtpFemale(null);
524 }
525 }
526 if (sp.withRtpUndefined()) {
527 Rtp rtpUndefined = sp.getRtpUndefined();
528 if (rtpUndefined.getA() == null && rtpUndefined.getB() == null) {
529 sp.setRtpUndefined(null);
530 }
531 }
532 }
533
534 protected void checkCpfDefinitionsValidity(Comparator<CalcifiedPiecesSamplingDefinition> definitionsComparator, ArrayListMultimap<SpeciesProtocol, CalcifiedPiecesSamplingDefinition> calcifiedPiecesSamplingDefinitions) {
535
536 calcifiedPiecesSamplingDefinitions.asMap().forEach((speciesProtocol, unsortedDefinitions) -> {
537
538 List<CalcifiedPiecesSamplingDefinition> sortedDefinitions = new ArrayList<>(unsortedDefinitions);
539 sortedDefinitions.sort(definitionsComparator);
540
541 Integer min = -1;
542 for (CalcifiedPiecesSamplingDefinition definition : sortedDefinitions) {
543 if (definition.getMinSize() != min + 1) {
544
545 throw new ImportRuntimeException(t("tutti.service.protocol.import.cps.interval.error",
546 speciesProtocol.getSpeciesReferenceTaxonId(),
547 definition.getMaturity(),
548 min,
549 definition.getMinSize()));
550 }
551 min = definition.getMaxSize();
552 }
553
554 speciesProtocol.setCalcifiedPiecesSamplingDefinition(sortedDefinitions);
555
556 });
557
558 }
559
560 private Map<Integer, SpeciesProtocol> availableSpeciesProtocolMap(TuttiProtocol protocol) {
561
562 Collection<SpeciesProtocol> availableSpeciesProtocol = new ArrayList<>();
563 if (!protocol.isSpeciesEmpty()) {
564 availableSpeciesProtocol.addAll(protocol.getSpecies());
565 }
566 if (!protocol.isBenthosEmpty()) {
567 availableSpeciesProtocol.addAll(protocol.getBenthos());
568 }
569
570 return availableSpeciesProtocol.stream()
571 .filter(SpeciesProtocol::isCalcifiedPiecesSamplingDefinitionEmpty)
572 .collect(Collectors.toMap(SpeciesProtocol::getSpeciesReferenceTaxonId, Function.identity()));
573
574
575 }
576
577 private static class SpeciesProtocolToSpeciesRowFunction implements Function<SpeciesProtocol, SpeciesRow> {
578
579 private final Map<String, Species> speciesMap;
580
581 private final Map<String, Caracteristic> caracteristicMap;
582
583 private final Binder<SpeciesProtocol, SpeciesRow> binder;
584
585 public SpeciesProtocolToSpeciesRowFunction(Map<String, Caracteristic> caracteristicMap,
586 Map<String, Species> speciesMap) {
587 this.speciesMap = speciesMap;
588 this.caracteristicMap = caracteristicMap;
589 this.binder = BinderFactory.newBinder(SpeciesProtocol.class, SpeciesRow.class);
590 }
591
592 @Override
593 public SpeciesRow apply(SpeciesProtocol input) {
594 Species species = speciesMap.get(String.valueOf(input.getSpeciesReferenceTaxonId()));
595 Preconditions.checkNotNull(species, "Could not find a species with id: " + input);
596 SpeciesRow result = new SpeciesRow();
597 binder.copy(input, result);
598 String pmfmId = input.getLengthStepPmfmId();
599 if (pmfmId != null) {
600 Caracteristic caracteristic = caracteristicMap.get(pmfmId);
601 result.setLengthStepPmfm(caracteristic);
602 }
603 result.setSpecies(species);
604
605 result.setMandatorySampleCategoryId(new ArrayList<>(input.getMandatorySampleCategoryId()));
606 return result;
607 }
608 }
609 }