View Javadoc
1   package fr.ifremer.tutti.service.psionimport;
2   
3   /*
4    * #%L
5    * Tutti :: Service
6    * %%
7    * Copyright (C) 2012 - 2014 Ifremer
8    * %%
9    * This program is free software: you can redistribute it and/or modify
10   * it under the terms of the GNU General Public License as
11   * published by the Free Software Foundation, either version 3 of the 
12   * License, or (at your option) any later version.
13   * 
14   * This program is distributed in the hope that it will be useful,
15   * but WITHOUT ANY WARRANTY; without even the implied warranty of
16   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17   * GNU General Public License for more details.
18   * 
19   * You should have received a copy of the GNU General Public 
20   * License along with this program.  If not, see
21   * <http://www.gnu.org/licenses/gpl-3.0.html>.
22   * #L%
23   */
24  
25  import com.google.common.base.Charsets;
26  import com.google.common.base.Joiner;
27  import com.google.common.base.Preconditions;
28  import com.google.common.collect.ImmutableSet;
29  import com.google.common.collect.Lists;
30  import com.google.common.collect.Maps;
31  import com.google.common.collect.Sets;
32  import com.google.common.io.Files;
33  import fr.ifremer.adagio.core.dao.referential.ObjectTypeCode;
34  import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId;
35  import fr.ifremer.adagio.core.dao.referential.pmfm.QualitativeValueId;
36  import fr.ifremer.tutti.persistence.entities.TuttiEntities;
37  import fr.ifremer.tutti.persistence.entities.data.Attachment;
38  import fr.ifremer.tutti.persistence.entities.data.Attachments;
39  import fr.ifremer.tutti.persistence.entities.data.BatchContainer;
40  import fr.ifremer.tutti.persistence.entities.data.CatchBatch;
41  import fr.ifremer.tutti.persistence.entities.data.FishingOperation;
42  import fr.ifremer.tutti.persistence.entities.data.SampleCategoryModel;
43  import fr.ifremer.tutti.persistence.entities.data.SpeciesBatch;
44  import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchFrequency;
45  import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchFrequencys;
46  import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchs;
47  import fr.ifremer.tutti.persistence.entities.protocol.SpeciesProtocol;
48  import fr.ifremer.tutti.persistence.entities.protocol.TuttiProtocol;
49  import fr.ifremer.tutti.persistence.entities.referential.Caracteristic;
50  import fr.ifremer.tutti.persistence.entities.referential.CaracteristicQualitativeValue;
51  import fr.ifremer.tutti.persistence.entities.referential.CaracteristicQualitativeValues;
52  import fr.ifremer.tutti.persistence.entities.referential.Species;
53  import fr.ifremer.tutti.persistence.entities.referential.TaxonCache;
54  import fr.ifremer.tutti.persistence.entities.referential.TaxonCaches;
55  import fr.ifremer.tutti.service.AbstractTuttiService;
56  import fr.ifremer.tutti.service.PersistenceService;
57  import fr.ifremer.tutti.service.TuttiServiceContext;
58  import fr.ifremer.tutti.type.WeightUnit;
59  import fr.ifremer.tutti.util.DateTimes;
60  import fr.ifremer.tutti.util.Weights;
61  import org.apache.commons.collections4.CollectionUtils;
62  import org.apache.commons.io.IOUtils;
63  import org.apache.commons.lang3.StringUtils;
64  import org.apache.commons.lang3.mutable.MutableInt;
65  import org.apache.commons.logging.Log;
66  import org.apache.commons.logging.LogFactory;
67  import org.nuiton.jaxx.application.ApplicationBusinessException;
68  
69  import java.io.BufferedReader;
70  import java.io.File;
71  import java.io.IOException;
72  import java.io.Serializable;
73  import java.text.DateFormat;
74  import java.text.ParseException;
75  import java.text.SimpleDateFormat;
76  import java.util.ArrayList;
77  import java.util.Date;
78  import java.util.Iterator;
79  import java.util.List;
80  import java.util.Map;
81  import java.util.Objects;
82  import java.util.Set;
83  
84  import static org.nuiton.i18n.I18n.t;
85  
86  /**
87   * To import some psion files.
88   *
89   * Created on 1/20/14.
90   *
91   * @author Tony Chemit - chemit@codelutin.com
92   * @since 3.0.1
93   */
94  public class PsionImportService extends AbstractTuttiService {
95  
96      private static final Log log = LogFactory.getLog(PsionImportService.class);
97  
98      protected static final ImmutableSet<String> NO_CATEGORY_VALUES = ImmutableSet.copyOf(new String[]{"N", "n"});
99  
100     protected static final ImmutableSet<String> SIZE_CATEGORY_VALUES = ImmutableSet.copyOf(new String[]{"G", "g", "P", "p"});
101 
102     protected static final ImmutableSet<String> SEX_CATEGORY_VALUES = ImmutableSet.copyOf(new String[]{"I", "i", "F", "f", "M", "m"});
103 
104     protected static final ImmutableSet<String> MATURITY_CATEGORY_VALUES = ImmutableSet.copyOf(new String[]{"1", "2", "3", "4", "5"});
105 
106     /**
107      * All usables keywords in a psion import.
108      *
109      * Created on 1/20/14.
110      *
111      * @author Tony Chemit - chemit@codelutin.com
112      * @since 3.0.1
113      */
114     public enum PsionImportKeyword {
115 
116         ESPE,
117         POID,
118         TAIL,
119         CATE,
120         LONG
121     }
122 
123     protected PersistenceService persistenceService;
124 
125     protected CaracteristicQualitativeValue sortedCaracteristic;
126 
127     protected CaracteristicQualitativeValue unsortedCaracteristic;
128 
129     protected Map<String, CaracteristicQualitativeValue> sizeCaracteristicValues;
130 
131     protected Map<String, CaracteristicQualitativeValue> sexCaracteristicValues;
132 
133     protected Map<String, CaracteristicQualitativeValue> maturityCaracteristicValues;
134 
135     protected final DateFormat df = new SimpleDateFormat("MM-dd-yyyy");
136 
137     @Override
138     public void setServiceContext(TuttiServiceContext context) {
139         super.setServiceContext(context);
140         persistenceService = getService(PersistenceService.class);
141 
142         { // sorted/unsorted caracteristic
143             Caracteristic caracteristic =
144                     persistenceService.getSortedUnsortedCaracteristic();
145 
146             sortedCaracteristic = CaracteristicQualitativeValues.getQualitativeValue(caracteristic, QualitativeValueId.SORTED_VRAC.getValue());
147             unsortedCaracteristic = CaracteristicQualitativeValues.getQualitativeValue(caracteristic, QualitativeValueId.SORTED_HORS_VRAC.getValue());
148         }
149 
150         { // size caracteristic
151 
152             sizeCaracteristicValues = Maps.newTreeMap();
153 
154             Caracteristic caracteristic = persistenceService.getSizeCategoryCaracteristic();
155 
156             List<CaracteristicQualitativeValue> qualitativeValues = caracteristic.getQualitativeValue();
157 
158             Map<Integer, CaracteristicQualitativeValue> sizeById = TuttiEntities.splitByIdAsInt(qualitativeValues);
159             CaracteristicQualitativeValue smallCaracteristic = sizeById.get(QualitativeValueId.SIZE_SMALL.getValue());
160             sizeCaracteristicValues.put("P", smallCaracteristic);
161             sizeCaracteristicValues.put("p", smallCaracteristic);
162             CaracteristicQualitativeValue bigCaracteristic = sizeById.get(QualitativeValueId.SIZE_BIG.getValue());
163             sizeCaracteristicValues.put("G", bigCaracteristic);
164             sizeCaracteristicValues.put("g", bigCaracteristic);
165         }
166 
167         { // sex caracteristic
168 
169             sexCaracteristicValues = Maps.newTreeMap();
170 
171             Caracteristic caracteristic = persistenceService.getSexCaracteristic();
172 
173             List<CaracteristicQualitativeValue> qualitativeValues = caracteristic.getQualitativeValue();
174 
175             Map<Integer, CaracteristicQualitativeValue> sexById = TuttiEntities.splitByIdAsInt(qualitativeValues);
176             CaracteristicQualitativeValue femaleCaracteristic = sexById.get(QualitativeValueId.SEX_FEMALE.getValue());
177             sexCaracteristicValues.put("F", femaleCaracteristic);
178             sexCaracteristicValues.put("f", femaleCaracteristic);
179             CaracteristicQualitativeValue maleCaracteristic = sexById.get(QualitativeValueId.SEX_MALE.getValue());
180             sexCaracteristicValues.put("M", maleCaracteristic);
181             sexCaracteristicValues.put("m", maleCaracteristic);
182             CaracteristicQualitativeValue unkownCaracteristic = sexById.get(QualitativeValueId.SEX_UNDEFINED.getValue());
183             sexCaracteristicValues.put("I", unkownCaracteristic);
184             sexCaracteristicValues.put("i", unkownCaracteristic);
185         }
186 
187         { // maturity caracteristic
188 
189             maturityCaracteristicValues = Maps.newTreeMap();
190 
191             Caracteristic caracteristic = persistenceService.getMaturityCaracteristic();
192 
193             List<CaracteristicQualitativeValue> qualitativeValues = caracteristic.getQualitativeValue();
194 
195             Map<Integer, CaracteristicQualitativeValue> byIds = TuttiEntities.splitByIdAsInt(qualitativeValues);
196             maturityCaracteristicValues.put("1", byIds.get(QualitativeValueId.MATURITY1.getValue()));
197             maturityCaracteristicValues.put("2", byIds.get(QualitativeValueId.MATURITY2.getValue()));
198             maturityCaracteristicValues.put("3", byIds.get(QualitativeValueId.MATURITY3.getValue()));
199             maturityCaracteristicValues.put("4", byIds.get(QualitativeValueId.MATURITY4.getValue()));
200             maturityCaracteristicValues.put("5", byIds.get(QualitativeValueId.MATURITY5.getValue()));
201         }
202     }
203 
204     public PsionImportResult importFile(File psionFile, FishingOperation operation, CatchBatch catchBatch) {
205 
206         Preconditions.checkNotNull(psionFile);
207         Preconditions.checkArgument(psionFile.exists(), "Psion file " + psionFile + " does not exist.");
208 
209         TuttiProtocol protocol = persistenceService.getProtocol();
210 
211         if (protocol == null) {
212             throw new ApplicationBusinessException(t("tutti.service.psionimport.error.no.protocol"));
213         }
214 
215         List<Species> allSpeciesWithSurveyCode = new ArrayList<>(persistenceService.getAllReferentSpecies());
216 
217         TaxonCache speciesMap = TaxonCaches.createSpeciesCacheWithoutVernacularCode(persistenceService, protocol);
218         speciesMap.load(allSpeciesWithSurveyCode);
219 
220         Map<String, Species> speciesBySurveyCode = Maps.newTreeMap();
221         for (Species species : allSpeciesWithSurveyCode) {
222             if (species.getSurveyCode() != null) {
223                 speciesBySurveyCode.put(species.getSurveyCode(), species);
224             }
225         }
226         Map<String, SpeciesProtocol> speciesProtocolBySurveyCode = Maps.newTreeMap();
227 
228         for (SpeciesProtocol speciesProtocol : protocol.getSpecies()) {
229             if (speciesProtocol.getSpeciesSurveyCode() == null) {
230                 continue;
231             }
232 
233             speciesProtocolBySurveyCode.put(speciesProtocol.getSpeciesSurveyCode(), speciesProtocol);
234         }
235 
236         BatchContainer<SpeciesBatch> rootSpeciesBatch =
237                 persistenceService.getRootSpeciesBatch(operation.getIdAsInt(), false);
238 
239         Set<Species> alreadyUsedSpecies = Sets.newHashSet();
240         for (SpeciesBatch speciesBatch : rootSpeciesBatch.getChildren()) {
241             alreadyUsedSpecies.add(speciesBatch.getSpecies());
242         }
243 
244         // load model
245         PsionImportModel importModel = new PsionImportModel();
246         try {
247             readImportFile(importModel,
248                            psionFile,
249                            operation,
250                            speciesBySurveyCode,
251                            speciesProtocolBySurveyCode,
252                            alreadyUsedSpecies);
253         } catch (IOException e) {
254             importModel.addError(e.getMessage());
255         }
256 
257         PsionImportResult result = new PsionImportResult(psionFile, importModel.getErrors());
258         if (importModel.withErrors()) {
259             if (log.isWarnEnabled()) {
260                 log.warn("Won't import psion file, errors detected.");
261             }
262 
263             return result;
264         }
265 
266         // --- Check sample category id used --- //
267         SampleCategoryModel sampleCategoryModel = context.getSampleCategoryModel();
268         Set<Integer> sampleCategoryIdUsed = importModel.getSampleCategoryIdUsed();
269         List<String> missingCategories = Lists.newArrayList();
270         for (Integer categoryId : sampleCategoryIdUsed) {
271             if (!sampleCategoryModel.containsCategoryId(categoryId)) {
272                 missingCategories.add("<li>" + categoryId + "</li>");
273             }
274         }
275 
276         if (!missingCategories.isEmpty()) {
277 
278             result.addError(
279                     t("tutti.service.psionimport.error.invalidSampleCategoryModel.message",
280                       Joiner.on("").join(missingCategories))
281             );
282 
283             return result;
284         }
285 
286         // --- Check sorted batches --- //
287         try {
288             importModel.checkSortedBatches(context.getSampleCategoryModel());
289         } catch (IOException e) {
290             result.addError(e.getMessage());
291             return result;
292         }
293 
294         // --- clean sorted batches --- //
295         importModel.cleanSortedBatches();
296 
297         // --- clean unsorted batches --- //
298         importModel.cleanUnsortedBatches();
299 
300         // --- Ok no error, can persist --- //
301 
302         persist(result, importModel, operation, catchBatch);
303 
304         return result;
305     }
306 
307     protected void readImportFile(PsionImportModel importModel,
308                                   File importFile,
309                                   FishingOperation operation,
310                                   Map<String, Species> speciesBySurveyCode,
311                                   Map<String, SpeciesProtocol> speciesProtocolBySurveyCode,
312                                   Set<Species> alreadyUsedSpecies) throws IOException {
313 
314         String operationNumber = String.valueOf(operation.getFishingOperationNumber());
315         Date operationStartDate = DateTimes.getDay(operation.getGearShootingStartDate());
316 
317         BufferedReader reader = Files.newReader(importFile, Charsets.UTF_8);
318 
319         try {
320 
321             reader.readLine(); // initiales saisisseurs
322             String operationCode = reader.readLine(); // Id du trait
323             String operationDateStr = reader.readLine(); // Date du trait
324 
325             Date operationDate;
326             try {
327                 operationDate = df.parse(operationDateStr);
328             } catch (ParseException e) {
329 
330                 throw new IOException(t("tutti.service.psionimport.error.invalid.date.format"));
331             }
332 
333             boolean correctOperation = Objects.equals(operationCode, operationNumber) &&
334                                        Objects.equals(operationDate, operationStartDate);
335 
336             if (!correctOperation) {
337                 throw new IOException(t("tutti.service.psionimport.error.invalid.operation"));
338             }
339 
340             reader.readLine(); // Heure de création du fichier
341             reader.readLine(); // Ligne blanche
342 
343             int lineNumber = 5;
344 
345             PsionImportBatchModel batch = null;
346 
347             String line;
348             String badSpecies = null;
349             while ((line = reader.readLine()) != null) {
350                 lineNumber++;
351                 if (!line.contains(":")) {
352                     throw new IOException(t("tutti.service.psionimport.error.invalid.line.syntax", lineNumber, line));
353                 }
354                 int endIndex = line.indexOf(':');
355                 String commandStr = StringUtils.trim(line.substring(0, endIndex));
356 
357 
358                 PsionImportKeyword command;
359 
360                 try {
361                     command = PsionImportKeyword.valueOf(commandStr);
362                 } catch (IllegalArgumentException e) {
363                     throw new IOException(t("tutti.service.psionimport.error.invalid.command.syntax", commandStr, lineNumber));
364                 }
365 
366                 String value = StringUtils.trim(line.substring(endIndex + 1));
367 
368                 if (PsionImportKeyword.ESPE.equals(command)) {
369 
370                     // start a new species
371 
372                     // register previous batch
373                     if (batch != null) {
374                         importModel.addBatch(batch);
375                     }
376 
377                     Species species = speciesBySurveyCode.get(value);
378 
379                     if (species == null) {
380 
381                         // could not load this species
382 
383                         badSpecies = value;
384                         batch = null;
385                         String error = t("tutti.service.psionimport.error.species.not.found", lineNumber, value);
386                         if (log.isWarnEnabled()) {
387                             log.warn(error);
388                         }
389                         importModel.addError(error);
390                         continue;
391                     }
392 
393                     if (alreadyUsedSpecies.contains(species)) {
394 
395                         // can't use an already used species
396                         badSpecies = value;
397                         batch = null;
398                         String error = t("tutti.service.psionimport.error.species.already.used", lineNumber, value);
399                         if (log.isWarnEnabled()) {
400                             log.warn(error);
401                         }
402                         importModel.addError(error);
403                         continue;
404                     }
405 
406                     badSpecies = null;
407 
408                     SpeciesProtocol speciesProtocol = speciesProtocolBySurveyCode.get(value);
409 
410                     String lengthStepCaracteristicId = speciesProtocol.getLengthStepPmfmId();
411 
412                     if (StringUtils.isBlank(lengthStepCaracteristicId)) {
413                         badSpecies = value;
414                         batch = null;
415                         String error = t("tutti.service.psionimport.error.no.lengthClass.caracteristic", lineNumber, value);
416                         if (log.isWarnEnabled()) {
417                             log.warn(error);
418                         }
419                         importModel.addError(error);
420                         continue;
421                     }
422                     batch = new PsionImportBatchModel(species, Integer.valueOf(lengthStepCaracteristicId));
423                 } else {
424 
425                     if (badSpecies != null) {
426                         // ignore this line due to bad species
427                         if (log.isDebugEnabled()) {
428                             log.debug("Ligne " + lineNumber
429                                       + " ignorée car l'espèce " + badSpecies + " n'était pas reconnue");
430                         }
431                         continue;
432                     }
433 
434                     // check batch exists
435                     if (batch == null) {
436                         throw new IOException(
437                                 t("tutti.service.psionimport.error.invalid.firstLine", line, lineNumber));
438                     }
439 
440                     switch (command) {
441 
442                         case POID:
443                             // add weight
444                             Float weight = toFloat(value, lineNumber);
445                             batch.setWeight(weight);
446                             break;
447 
448                         case TAIL:
449                             // add sample weight
450                             Float sampleWeight = toFloat(value, lineNumber);
451                             batch.setSampleWeight(sampleWeight);
452                             break;
453 
454                         case CATE:
455                             // add category
456 
457                             if (StringUtils.isBlank(value)) {
458                                 badSpecies = batch.getSpecies().getSurveyCode();
459                                 batch = null;
460                                 String error = t("tutti.service.psionimport.error.invalid.category.syntax", lineNumber, value, badSpecies);
461 
462                                 if (log.isWarnEnabled()) {
463                                     log.warn(error);
464                                 }
465                                 importModel.addError(error);
466                                 continue;
467                             }
468 
469                             if (NO_CATEGORY_VALUES.contains(value)) {
470 
471                                 // special case, no category
472 
473                             } else {
474 
475                                 // guess all categories
476 
477                                 for (int i = 0, nbCategory = value.length(); i < nbCategory; i++) {
478                                     String categoryCode = value.substring(i, i + 1);
479 
480                                     PsionImportBatchModel.SampleCategory category = guessCategory(categoryCode);
481 
482                                     if (category == null) {
483                                         badSpecies = batch.getSpecies().getSurveyCode();
484                                         batch = null;
485                                         String error = t("tutti.service.psionimport.error.invalid.category.syntax", lineNumber, categoryCode, badSpecies);
486                                         if (log.isWarnEnabled()) {
487                                             log.warn(error);
488                                         }
489                                         importModel.addError(error);
490                                         break;
491                                     }
492 
493                                     batch.setCategory(category.getCategoryId(), category.getCategoryValue());
494                                 }
495 
496                                 if (batch == null) {
497 
498                                     // at least one category was not ok
499                                     continue;
500                                 }
501                             }
502 
503                             batch.setCategoryCode(value);
504 
505                             break;
506 
507                         case LONG:
508                             // add frequency
509                             Float size = toFloat(value, lineNumber);
510                             batch.addFrequency(size, 1);
511                             break;
512                     }
513                 }
514             }
515 
516             if (batch != null) {
517 
518                 // save it
519                 importModel.addBatch(batch);
520             }
521 
522             reader.close();
523         } finally {
524             IOUtils.closeQuietly(reader);
525         }
526     }
527 
528     protected void persist(PsionImportResult result,
529                            PsionImportModel importModel,
530                            FishingOperation operation,
531                            CatchBatch catchBatch) {
532 
533         if (catchBatch != null) {
534             addFileAsAttachment(result.getImportFile(), catchBatch);
535         }
536 
537         // insert all imported species batches
538 
539         Set<Species> species = importModel.getSpecies();
540 
541         for (Species specy : species) {
542 
543             List<PsionImportBatchModel> sortedBatchs = importModel.getSortedBatches(specy);
544 
545             if (CollectionUtils.isNotEmpty(sortedBatchs)) {
546                 persistSortedBatches(operation, specy, sortedBatchs);
547 
548                 result.incrementNbSortedImported();
549             }
550 
551             List<PsionImportBatchModel> unsortedBatchs = importModel.getUnsortedBatches(specy);
552 
553             if (CollectionUtils.isNotEmpty(unsortedBatchs)) {
554                 persistUnsortedBatches(operation, specy, unsortedBatchs);
555 
556                 result.incrementNbUnsortedImported();
557             }
558 
559         }
560 
561         persistenceService.saveCatchBatch(catchBatch);
562     }
563 
564     protected void persistSortedBatches(FishingOperation operation,
565                                         Species specy,
566                                         List<PsionImportBatchModel> batchs) {
567 
568         if (batchs.size() == 1 && !batchs.get(0).withCategories()) {
569 
570             PsionImportBatchModel batchModel = batchs.get(0);
571 
572             // simple batch with no category
573             SpeciesBatch batch = createSpeciesBatch(operation,
574                                                     batchModel.getSpecies(),
575                                                     batchModel.getWeight(),
576                                                     batchModel.getSampleWeight(),
577                                                     PmfmId.SORTED_UNSORTED.getValue(),
578                                                     sortedCaracteristic);
579 
580 
581             batch = persistenceService.createSpeciesBatch(batch, null, true);
582 
583             persistFrequencies(batch, batchModel);
584 
585         } else {
586 
587             // batch with categories
588 
589             // Is there two weights ?
590             // If so and applyOnLeaf is off then the weight has to be placed in the sorted batch
591             // Otherwise apply both weight on categorized batch
592 
593             Float sortedBatchWeight = null;
594 
595             Float weight = batchs.get(0).getWeight();
596             Float sampleWeight = batchs.get(0).getSampleWeight();
597             boolean applyOnLeaf = batchs.get(0).isApplyBothWeightOnCategorizedBatch();
598 
599             if (sampleWeight != null) {
600 
601                 // use the weight as sorted batch weight
602 
603                 if (!applyOnLeaf) {
604 
605                     // use the weight as sorted batch weight
606                     sortedBatchWeight = weight;
607 
608                 }
609 
610             }
611 
612             SpeciesBatch rootBatch = createSpeciesBatch(operation,
613                                                         specy,
614                                                         sortedBatchWeight,
615                                                         null,
616                                                         PmfmId.SORTED_UNSORTED.getValue(),
617                                                         sortedCaracteristic);
618 
619             rootBatch = persistenceService.createSpeciesBatch(rootBatch, null, true);
620 
621             createCategoryBatches(operation, specy, batchs, rootBatch, sortedBatchWeight != null);
622 
623         }
624 
625     }
626 
627     protected void persistUnsortedBatches(FishingOperation operation,
628                                           Species specy,
629                                           List<PsionImportBatchModel> batchs) {
630 
631         if (batchs.size() == 1 && !batchs.get(0).withCategories()) {
632 
633             PsionImportBatchModel batchModel = batchs.get(0);
634 
635             // simple batch with no category
636             SpeciesBatch batch = createSpeciesBatch(operation,
637                                                     batchModel.getSpecies(),
638                                                     batchModel.getWeight(),
639                                                     batchModel.getSampleWeight(),
640                                                     PmfmId.SORTED_UNSORTED.getValue(),
641                                                     unsortedCaracteristic);
642 
643             batch = persistenceService.createSpeciesBatch(batch, null, true);
644 
645             persistFrequencies(batch, batchModel);
646 
647         } else {
648 
649             // batch with categories
650 
651             SpeciesBatch rootBatch = createSpeciesBatch(operation,
652                                                         specy,
653                                                         null,
654                                                         null,
655                                                         PmfmId.SORTED_UNSORTED.getValue(),
656                                                         unsortedCaracteristic);
657 
658             rootBatch = persistenceService.createSpeciesBatch(rootBatch, null, true);
659 
660             createCategoryBatches(operation, specy, batchs, rootBatch, false);
661 
662         }
663 
664     }
665 
666     protected void createCategoryBatches(FishingOperation operation,
667                                          Species specy,
668                                          List<PsionImportBatchModel> batchs,
669                                          SpeciesBatch rootBatch,
670                                          boolean applyOnlySampleWeight) {
671 
672         for (PsionImportBatchModel batchModel : batchs) {
673 
674             SpeciesBatch parentBatch = rootBatch;
675 
676             SpeciesBatch childBatch = null;
677 
678             Iterator<PsionImportBatchModel.SampleCategory> categoryIterator = batchModel.getCategoryIterator();
679 
680             while (categoryIterator.hasNext()) {
681                 PsionImportBatchModel.SampleCategory sampleCategory = categoryIterator.next();
682 
683                 boolean lastCategory = !categoryIterator.hasNext();
684 
685                 Integer categoryId = sampleCategory.getCategoryId();
686                 Serializable categoryValue = sampleCategory.getCategoryValue();
687 
688                 if (lastCategory) {
689 
690                     // always create the leaf
691 
692                     Float weight;
693                     Float sampleWeight;
694 
695                     if (applyOnlySampleWeight) {
696                         weight = batchModel.getSampleWeight();
697                         sampleWeight = null;
698                     } else {
699                         weight = batchModel.getWeight();
700                         sampleWeight = batchModel.getSampleWeight();
701                     }
702                     childBatch = createSpeciesBatch(operation,
703                                                     specy,
704                                                     weight,
705                                                     sampleWeight,
706                                                     categoryId,
707                                                     categoryValue);
708                 } else {
709 
710                     // try to find child in parent children
711 
712                     childBatch = null;
713                     for (SpeciesBatch speciesBatch : parentBatch.getChildBatchs()) {
714 
715                         if (speciesBatch.getSampleCategoryId().equals(categoryId) &&
716                             speciesBatch.getSampleCategoryValue().equals(categoryValue)) {
717                             childBatch = speciesBatch;
718                             break;
719                         }
720                     }
721 
722                     if (childBatch == null) {
723 
724                         // must create it
725                         childBatch = createSpeciesBatch(operation,
726                                                         specy,
727                                                         null,
728                                                         null,
729                                                         categoryId,
730                                                         categoryValue);
731                     }
732                 }
733 
734                 if (TuttiEntities.isNew(childBatch)) {
735 
736                     // persist it
737                     childBatch = persistenceService.createSpeciesBatch(childBatch, parentBatch.getIdAsInt(), true);
738                     parentBatch.addChildBatchs(childBatch);
739                 }
740 
741                 parentBatch = childBatch;
742             }
743 
744             persistFrequencies(childBatch, batchModel);
745         }
746     }
747 
748     protected PsionImportBatchModel.SampleCategory guessCategory(String categoryCode) {
749 
750         PsionImportBatchModel.SampleCategory result = null;
751 
752         Integer caracteristicId;
753         CaracteristicQualitativeValue caracteristicQualitativeValue;
754 
755         if (SIZE_CATEGORY_VALUES.contains(categoryCode)) {
756 
757             // size caracteristic
758             caracteristicId = PmfmId.SIZE_CATEGORY.getValue();
759             caracteristicQualitativeValue = sizeCaracteristicValues.get(categoryCode);
760 
761             result = new PsionImportBatchModel.SampleCategory(caracteristicId, caracteristicQualitativeValue);
762 
763         } else if (SEX_CATEGORY_VALUES.contains(categoryCode)) {
764 
765             // sex caracteristic
766             caracteristicId = PmfmId.SEX.getValue();
767             caracteristicQualitativeValue = sexCaracteristicValues.get(categoryCode);
768 
769             result = new PsionImportBatchModel.SampleCategory(caracteristicId, caracteristicQualitativeValue);
770 
771         } else if (MATURITY_CATEGORY_VALUES.contains(categoryCode)) {
772 
773             // maturity caracteristic
774             caracteristicId = PmfmId.MATURITY.getValue();
775             caracteristicQualitativeValue = maturityCaracteristicValues.get(categoryCode);
776 
777             result = new PsionImportBatchModel.SampleCategory(caracteristicId, caracteristicQualitativeValue);
778         }
779         return result;
780     }
781 
782     protected void persistFrequencies(SpeciesBatch batch, PsionImportBatchModel batchModel) {
783 
784         Integer lengthStepCaracteristicId = batchModel.getLengthStepCaracteristicId();
785         Map<Float, MutableInt> frequencies = batchModel.getFrequencies();
786         List<SpeciesBatchFrequency> toSave = Lists.newArrayList();
787 
788         Caracteristic lengthStepCaracteristic = persistenceService.getCaracteristic(lengthStepCaracteristicId);
789 
790         for (Map.Entry<Float, MutableInt> entry : frequencies.entrySet()) {
791             Float size = entry.getKey();
792             MutableInt number = entry.getValue();
793 
794             SpeciesBatchFrequency batchFrequency = SpeciesBatchFrequencys.newSpeciesBatchFrequency();
795             batchFrequency.setBatch(batch);
796             batchFrequency.setLengthStepCaracteristic(lengthStepCaracteristic);
797             batchFrequency.setLengthStep(size);
798             batchFrequency.setNumber(number.getValue());
799             toSave.add(batchFrequency);
800         }
801 
802         persistenceService.saveSpeciesBatchFrequency(batch.getIdAsInt(), toSave);
803     }
804 
805     protected SpeciesBatch createSpeciesBatch(FishingOperation operation,
806                                               Species species,
807                                               Float catchWeight,
808                                               Float sampleWeight,
809                                               Integer categoryId,
810                                               Serializable cqv) {
811         SpeciesBatch batch = SpeciesBatchs.newSpeciesBatch();
812         batch.setFishingOperation(operation);
813         batch.setSampleCategoryId(categoryId);
814         batch.setSampleCategoryValue(cqv);
815         batch.setSpecies(species);
816         if (catchWeight != null) {
817 
818             catchWeight = WeightUnit.KG.round(WeightUnit.G.toEntity(catchWeight));
819             batch.setSampleCategoryWeight(catchWeight);
820         }
821         if (sampleWeight != null) {
822 
823             sampleWeight = WeightUnit.KG.round(WeightUnit.G.toEntity(sampleWeight));
824             batch.setWeight(sampleWeight);
825         }
826 
827         batch.setChildBatchs(Lists.<SpeciesBatch>newArrayList());
828         return batch;
829     }
830 
831     protected void addFileAsAttachment(File f, CatchBatch catchBatch) {
832         Attachment attachment = Attachments.newAttachment();
833         attachment.setObjectType(ObjectTypeCode.CATCH_BATCH);
834         attachment.setObjectId(Integer.valueOf(catchBatch.getId()));
835         attachment.setName(f.getName());
836         String date = DateFormat.getDateTimeInstance().format(context.currentDate());
837         String comment = t("tutti.service.psion.import.attachment.comment", date);
838         attachment.setComment(comment);
839         persistenceService.createAttachment(attachment, f);
840     }
841 
842     protected Float toFloat(String cell, int lineNumber) throws IOException {
843         Float result = null;
844         if (!cell.isEmpty()) {
845             try {
846                 result = Float.valueOf(cell);
847             } catch (NumberFormatException e) {
848                 throw new IOException("Format de la valeur [" + lineNumber + "] : " + cell +
849                                       " incorrect, devrait être un entier décimal");
850             }
851         }
852         if (result == null) {
853             throw new IOException("La valeur [" + lineNumber + "] est obligatoire mais n'est pas renseignée");
854         }
855         return result;
856     }
857 }