Source code

001/*
002 * This file is part of McIDAS-V
003 *
004 * Copyright 2007-2016
005 * Space Science and Engineering Center (SSEC)
006 * University of Wisconsin - Madison
007 * 1225 W. Dayton Street, Madison, WI 53706, USA
008 * https://www.ssec.wisc.edu/mcidas
009 * 
010 * All Rights Reserved
011 * 
012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and
013 * some McIDAS-V source code is based on IDV and VisAD source code.  
014 * 
015 * McIDAS-V is free software; you can redistribute it and/or modify
016 * it under the terms of the GNU Lesser Public License as published by
017 * the Free Software Foundation; either version 3 of the License, or
018 * (at your option) any later version.
019 * 
020 * McIDAS-V is distributed in the hope that it will be useful,
021 * but WITHOUT ANY WARRANTY; without even the implied warranty of
022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
023 * GNU Lesser Public License for more details.
024 * 
025 * You should have received a copy of the GNU Lesser Public License
026 * along with this program.  If not, see http://www.gnu.org/licenses.
027 */
028
029package edu.wisc.ssec.mcidasv.data.hydra;
030
031import java.util.Map;
032import java.util.Objects;
033
034import org.slf4j.Logger;
035import org.slf4j.LoggerFactory;
036
037import edu.wisc.ssec.mcidasv.data.QualityFlag;
038import visad.util.Util;
039
040public class RangeProcessor {
041
042    private static final Logger logger = LoggerFactory.getLogger(RangeProcessor.class);
043
044    static RangeProcessor createRangeProcessor(MultiDimensionReader reader,
045            Map<String, Object> metadata) throws Exception {
046        if (reader instanceof GranuleAggregation) {
047            return new AggregationRangeProcessor((GranuleAggregation) reader, metadata);
048        }
049
050        if (metadata.get("scale_name") == null) {
051            String product_name = (String) metadata.get(SwathAdapter.product_name);
052            if (Objects.equals(product_name, "IASI_L1C_xxx")) {
053                return new IASI_RangeProcessor();
054            }
055            return null;
056        } else {
057            String product_name = (String) metadata.get(ProfileAlongTrack.product_name);
058            if (Objects.equals(product_name, "2B-GEOPROF")) {
059                return new CloudSat_2B_GEOPROF_RangeProcessor(reader, metadata);
060            } else {
061                return new RangeProcessor(reader, metadata);
062            }
063        }
064    }
065
066    MultiDimensionReader reader;
067    Map<String, Object> metadata;
068
069    float[] scale = null;
070    float[] offset = null;
071    double[] missing = null;
072    double[] valid_range = null;
073    double valid_low = -Double.MAX_VALUE;
074    double valid_high = Double.MAX_VALUE;
075
076    boolean unpack = false;
077    boolean unsigned = false;
078    boolean rangeCheckBeforeScaling = true;
079
080    int scaleOffsetLen = 1;
081
082    String multiScaleDimName = SpectrumAdapter.channelIndex_name;
083    boolean hasMultiDimensionScale = false;
084
085    int multiScaleDimensionIndex = 0;
086
087    int soIndex = 0;
088
089    public RangeProcessor() {
090    }
091
092    public RangeProcessor(float scale, float offset, float valid_low, float valid_high,
093            float missing) {
094        this.scale = new float[] { scale };
095        this.offset = new float[] { offset };
096        this.missing = new double[] { missing };
097        this.valid_low = valid_low;
098        this.valid_high = valid_high;
099    }
100
101    public RangeProcessor(MultiDimensionReader reader, Map<String, Object> metadata,
102            String multiScaleDimName) throws Exception {
103        this(reader, metadata);
104        this.multiScaleDimName = multiScaleDimName;
105    }
106
107    public RangeProcessor(MultiDimensionReader reader, Map<String, Object> metadata)
108            throws Exception {
109        this.reader = reader;
110        this.metadata = metadata;
111
112        if (metadata.get("unpack") != null) {
113            unpack = true;
114        }
115
116        if (metadata.get("unsigned") != null) {
117            unsigned = true;
118        }
119
120        if (metadata.get("range_check_after_scaling") != null) {
121            String s = (String) metadata.get("range_check_after_scaling");
122            logger.debug("range_check_after_scaling: " + s);
123            rangeCheckBeforeScaling = false;
124        }
125
126        String array_name = (String) metadata.get("array_name");
127
128        scale = getAttributeAsFloatArray(array_name, (String) metadata.get("scale_name"));
129
130        offset = getAttributeAsFloatArray(array_name, (String) metadata.get("offset_name"));
131
132        if (scale != null) {
133            scaleOffsetLen = scale.length;
134
135            if (offset != null) {
136                if (scale.length != offset.length) {
137                    throw new Exception(
138                            "RangeProcessor: scale and offset array lengths must be equal");
139                }
140            } else {
141                offset = new float[scaleOffsetLen];
142                for (int i = 0; i < offset.length; i++)
143                    offset[i] = 0f;
144            }
145
146        }
147
148        missing = getAttributeAsDoubleArray(array_name, (String) metadata.get("fill_value_name"));
149
150        // if we are working with unsigned data, need to convert missing vals to
151        // unsigned too
152        if (unsigned) {
153            if (missing != null) {
154                for (int i = 0; i < missing.length; i++) {
155                    missing[i] = (float) Util.unsignedShortToInt((short) missing[i]);
156                }
157            }
158        }
159
160        String metaStr = (String) metadata.get("valid_range");
161        // attr name not supplied, so try the convention default
162        if (metaStr == null) {
163            metaStr = "valid_range";
164        }
165
166        valid_range = getAttributeAsDoubleArray(array_name, metaStr);
167        if (valid_range != null) {
168
169            valid_low = valid_range[0];
170            valid_high = valid_range[1];
171
172            if (valid_range[0] > valid_range[1]) {
173                valid_low = valid_range[1];
174                valid_high = valid_range[0];
175            }
176        } else {
177            metaStr = (String) metadata.get("valid_low");
178            if (metaStr == null) { // attr name not supplied, so try the
179                                   // convention default
180                metaStr = "valid_min";
181            }
182            double[] dblA = getAttributeAsDoubleArray(array_name, metaStr);
183            if (dblA != null) {
184                valid_low = dblA[0];
185            }
186
187            metaStr = (String) metadata.get("valid_high");
188            if (metaStr == null) { // attr name not supplied, so try the
189                                   // convention default
190                metaStr = "valid_max";
191            }
192            dblA = getAttributeAsDoubleArray(array_name, metaStr);
193            if (dblA != null) {
194                valid_high = dblA[0];
195            }
196        }
197
198        if (rangeCheckBeforeScaling) {
199            if (unsigned) {
200                if (valid_low != -Double.MAX_VALUE) {
201                    valid_low = (double) Util.unsignedShortToInt((short) valid_low);
202                }
203                if (valid_high != Double.MAX_VALUE) {
204                    valid_high = (double) Util.unsignedShortToInt((short) valid_high);
205                }
206            }
207        }
208
209        String str = (String) metadata.get("multiScaleDimensionIndex");
210        hasMultiDimensionScale = (str != null);
211        multiScaleDimensionIndex = (str != null) ? Integer.parseInt(str) : 0;
212    }
213
214    public float[] getAttributeAsFloatArray(String arrayName, String attrName) throws Exception {
215        float[] fltArray = null;
216        HDFArray arrayAttr = reader.getArrayAttribute(arrayName, attrName);
217
218        if (arrayAttr != null) {
219
220            if (arrayAttr.getType().equals(Float.TYPE)) {
221                float[] attr = (float[]) arrayAttr.getArray();
222                fltArray = new float[attr.length];
223                for (int k = 0; k < attr.length; k++)
224                    fltArray[k] = attr[k];
225            } else if (arrayAttr.getType().equals(Short.TYPE)) {
226                short[] attr = (short[]) arrayAttr.getArray();
227                fltArray = new float[attr.length];
228                for (int k = 0; k < attr.length; k++)
229                    fltArray[k] = (float) attr[k];
230            } else if (arrayAttr.getType().equals(Integer.TYPE)) {
231                int[] attr = (int[]) arrayAttr.getArray();
232                fltArray = new float[attr.length];
233                for (int k = 0; k < attr.length; k++)
234                    fltArray[k] = (float) attr[k];
235            } else if (arrayAttr.getType().equals(Double.TYPE)) {
236                double[] attr = (double[]) arrayAttr.getArray();
237                fltArray = new float[attr.length];
238                for (int k = 0; k < attr.length; k++)
239                    fltArray[k] = (float) attr[k];
240            }
241
242        }
243
244        return fltArray;
245    }
246
247    public double[] getAttributeAsDoubleArray(String arrayName, String attrName) throws Exception {
248        if (attrName == null) {
249            return null;
250        }
251
252        double[] dblArray = null;
253        HDFArray arrayAttr = null;
254        try {
255            arrayAttr = reader.getArrayAttribute(arrayName, attrName);
256        } catch (Exception e) {
257            e.printStackTrace();
258        }
259
260        if (arrayAttr != null) {
261
262            if (arrayAttr.getType().equals(Float.TYPE)) {
263                float[] attr = (float[]) arrayAttr.getArray();
264                dblArray = new double[attr.length];
265                for (int k = 0; k < attr.length; k++)
266                    dblArray[k] = attr[k];
267            } else if (arrayAttr.getType().equals(Short.TYPE)) {
268                short[] attr = (short[]) arrayAttr.getArray();
269                dblArray = new double[attr.length];
270                for (int k = 0; k < attr.length; k++) {
271                    if (unsigned) {
272                        dblArray[k] = (double) Util.unsignedShortToInt((short) attr[k]);
273                    } else {
274                        dblArray[k] = (double) attr[k];
275                    }
276                }
277            } else if (arrayAttr.getType().equals(Integer.TYPE)) {
278                int[] attr = (int[]) arrayAttr.getArray();
279                dblArray = new double[attr.length];
280                for (int k = 0; k < attr.length; k++)
281                    dblArray[k] = (double) attr[k];
282            } else if (arrayAttr.getType().equals(Double.TYPE)) {
283                double[] attr = (double[]) arrayAttr.getArray();
284                dblArray = new double[attr.length];
285                for (int k = 0; k < attr.length; k++)
286                    dblArray[k] = (double) attr[k];
287            } else if (arrayAttr.getType().equals(Byte.TYPE)) {
288                byte[] attr = (byte[]) arrayAttr.getArray();
289                dblArray = new double[attr.length];
290                for (int k = 0; k < attr.length; k++)
291                    dblArray[k] = (double) attr[k];
292            } else if (arrayAttr.getType().equals(String.class)) {
293                String[] attr = (String[]) arrayAttr.getArray();
294                dblArray = new double[attr.length];
295                for (int k = 0; k < attr.length; k++)
296                    dblArray[k] = Double.valueOf(attr[0]);
297            }
298        }
299
300        return dblArray;
301    }
302
303    /**
304     * Process a range of data from an array of {@code byte} values where bytes
305     * are packed bit or multi-bit fields of quality flags. Based on info in a
306     * {@link QualityFlag} object passed in, we extract and return values for
307     * that flag.
308     * 
309     * @param values
310     *            Input byte values. Cannot be {@code null}.
311     * @param subset
312     *            Optional subset.
313     * @param qf
314     *            Quality flag.
315     *
316     * @return Processed range.
317     */
318
319    public float[] processRangeQualityFlag(byte[] values, Map subset, QualityFlag qf) {
320
321        if (subset != null) {
322            if (subset.get(multiScaleDimName) != null) {
323                soIndex = (int) ((double[]) subset.get(multiScaleDimName))[0];
324            }
325        }
326
327        float[] newValues = new float[values.length];
328
329        float val = 0f;
330        int bitOffset = qf.getBitOffset();
331        int divisor = -1;
332
333        // map bit offset to a divisor
334        switch (bitOffset) {
335            case 1:
336                divisor = 2;
337                break;
338            case 2:
339                divisor = 4;
340                break;
341            case 3:
342                divisor = 8;
343                break;
344            case 4:
345                divisor = 16;
346                break;
347            case 5:
348                divisor = 32;
349                break;
350            case 6:
351                divisor = 64;
352                break;
353            case 7:
354                divisor = 128;
355                break;
356            default:
357                divisor = 1;
358                break;
359        }
360
361        // now map bit width to a mask
362        int numBits = qf.getNumBits();
363        int mask = -1;
364        switch (numBits) {
365            case 1:
366                mask = (int) 0x00000001;
367                break;
368            case 2:
369                mask = (int) 0x00000003;
370                break;
371            case 3:
372                mask = (int) 0x00000007;
373                break;
374            case 4:
375                mask = (int) 0x0000000F;
376                break;
377            case 5:
378                mask = (int) 0x0000001F;
379                break;
380            case 6:
381                mask = (int) 0x0000003F;
382                break;
383            case 7:
384                mask = (int) 0x0000007F;
385                break;
386            default:
387                mask = (int) 0x00000000;
388                break;
389        }
390
391        int i = 0;
392        for (int k = 0; k < values.length; k++) {
393            val = (float) values[k];
394            i = Util.unsignedByteToInt(values[k]);
395            val = (float) ((i / divisor) & mask);
396            newValues[k] = val;
397        }
398
399        return newValues;
400    }
401
402    /**
403     * Process a range of data from an array of {@code byte} values.
404     * 
405     * @param values
406     *            Input {@code byte} values. Cannot be {@code null}.
407     * @param subset
408     *            Optional subset.
409     *
410     * @return Processed range.
411     */
412    
413    public float[] processRange(byte[] values, Map<String, double[]> subset) {
414
415        int multiScaleDimLen = 1;
416
417        if (subset != null) {
418            if (subset.get(multiScaleDimName) != null) {
419                double[] coords = (double[]) subset.get(multiScaleDimName);
420                soIndex = (int) coords[0];
421                multiScaleDimLen = (int) (coords[1] - coords[0] + 1.0);
422            }
423        }
424
425        float[] new_values = new float[values.length];
426
427        float val = 0f;
428        int i = 0;
429        boolean isMissing = false;
430
431        for (int k = 0; k < values.length; k++) {
432
433            val = (float) values[k];
434            if (unsigned) {
435                i = Util.unsignedByteToInt(values[k]);
436                val = (float) i;
437            }
438
439            // first, check the (possibly multiple) missing values
440            isMissing = false;
441            if (missing != null) {
442                for (int mvIdx = 0; mvIdx < missing.length; mvIdx++) {
443                    if (val == missing[mvIdx]) {
444                        isMissing = true;
445                        break;
446                    }
447                }
448            }
449
450            if (isMissing) {
451                new_values[k] = Float.NaN;
452                continue;
453            }
454
455            if (rangeCheckBeforeScaling) {
456                if ((val < valid_low) || (val > valid_high)) {
457                    new_values[k] = Float.NaN;
458                    continue;
459                }
460            }
461
462            if (scale != null) {
463                if (unpack) {
464                    if (multiScaleDimLen == 1) {
465                        new_values[k] = (scale[soIndex] * val) + offset[soIndex];
466                    } else {
467                        new_values[k] = (scale[soIndex + k] * val) + offset[soIndex + k];
468                    }
469                } else {
470                    if (multiScaleDimLen == 1) {
471                        new_values[k] = scale[soIndex] * (val - offset[soIndex]);
472                    } else {
473                        new_values[k] = scale[soIndex + k] * (val - offset[soIndex + k]);
474                    }
475                }
476
477            } else {
478                new_values[k] = val;
479            }
480
481            // do valid range check AFTER scaling?
482            if (!rangeCheckBeforeScaling) {
483                if ((new_values[k] < valid_low) || (new_values[k] > valid_high)) {
484                    new_values[k] = Float.NaN;
485                }
486            }
487        }
488        return new_values;
489    }
490
491    /**
492     * Process a range of data from an array of {@code short} values.
493     * 
494     * @param values
495     *            Input {@code short} values. Cannot be {@code null}.
496     * @param subset
497     *            Optional subset.
498     *
499     * @return Processed range.
500     */
501    
502    public float[] processRange(short[] values, Map<String, double[]> subset) {
503
504        int multiScaleDimLen = 1;
505
506        if (subset != null) {
507            if (subset.get(multiScaleDimName) != null) {
508                double[] coords = (double[]) subset.get(multiScaleDimName);
509                soIndex = (int) coords[0];
510                multiScaleDimLen = (int) (coords[1] - coords[0] + 1.0);
511            }
512        }
513
514        float[] new_values = new float[values.length];
515
516        float val = 0f;
517        int i = 0;
518        boolean isMissing = false;
519
520        for (int k = 0; k < values.length; k++) {
521
522            val = (float) values[k];
523            if (unsigned) {
524                i = Util.unsignedShortToInt(values[k]);
525                val = (float) i;
526            }
527
528            // first, check the (possibly multiple) missing values
529            isMissing = false;
530            if (missing != null) {
531                for (int mvIdx = 0; mvIdx < missing.length; mvIdx++) {
532                    if (val == missing[mvIdx]) {
533                        isMissing = true;
534                        break;
535                    }
536                }
537            }
538
539            if (isMissing) {
540                new_values[k] = Float.NaN;
541                continue;
542            }
543
544            if (rangeCheckBeforeScaling) {
545                if ((val < valid_low) || (val > valid_high)) {
546                    new_values[k] = Float.NaN;
547                    continue;
548                }
549            }
550
551            if (scale != null) {
552                if (unpack) {
553                    if (multiScaleDimLen == 1) {
554                        new_values[k] = (scale[soIndex] * val) + offset[soIndex];
555                    } else {
556                        new_values[k] = (scale[soIndex + k] * val) + offset[soIndex + k];
557                    }
558                } else {
559                    if (multiScaleDimLen == 1) {
560                        new_values[k] = scale[soIndex] * (val - offset[soIndex]);
561                    } else {
562
563                        new_values[k] = scale[soIndex + k] * (val - offset[soIndex + k]);
564                    }
565                }
566            } else {
567                new_values[k] = val;
568            }
569
570            // do valid range check AFTER scaling?
571            if (!rangeCheckBeforeScaling) {
572                if ((new_values[k] < valid_low) || (new_values[k] > valid_high)) {
573                    new_values[k] = Float.NaN;
574                }
575            }
576
577        }
578        return new_values;
579    }
580
581    /**
582     * Process a range of data from an array of {@code float} values.
583     * 
584     * @param values
585     *            Input {@code float} values. Cannot be {@code null}.
586     * @param subset
587     *            Optional subset.
588     *
589     * @return Processed array.
590     */
591    
592    public float[] processRange(float[] values, Map<String, double[]> subset) {
593
594        float[] new_values = null;
595
596        if ((missing != null) || (valid_range != null)) {
597            new_values = new float[values.length];
598        } else {
599            return values;
600        }
601
602        float val;
603
604        for (int k = 0; k < values.length; k++) {
605            val = values[k];
606            new_values[k] = val;
607
608            // first, check the (possibly multiple) missing values
609            if (missing != null) {
610                for (int mvIdx = 0; mvIdx < missing.length; mvIdx++) {
611                    if (val == missing[mvIdx]) {
612                        new_values[k] = Float.NaN;
613                        break;
614                    }
615                }
616            }
617
618            if ((valid_range != null) && ((val < valid_low) || (val > valid_high))) {
619                new_values[k] = Float.NaN;
620            }
621
622        }
623
624        return new_values;
625    }
626
627    /**
628     * Process a range of data from an array of {@code double} value.
629     * 
630     * @param values
631     *            Input {@code double} values. Cannot be {@code null}.
632     * @param subset
633     *            Optional subset.
634     *
635     * @return Processed array.
636     */
637    
638    public double[] processRange(double[] values, Map<String, double[]> subset) {
639
640        double[] new_values = null;
641
642        if ((missing != null) || (valid_range != null)) {
643            new_values = new double[values.length];
644        } else {
645            return values;
646        }
647
648        double val;
649
650        for (int k = 0; k < values.length; k++) {
651            val = values[k];
652            new_values[k] = val;
653
654            // first, check the (possibly multiple) missing values
655            if (missing != null) {
656                for (int mvIdx = 0; mvIdx < missing.length; mvIdx++) {
657                    if (val == missing[mvIdx]) {
658                        new_values[k] = Float.NaN;
659                        break;
660                    }
661                }
662            }
663
664            if ((valid_range != null) && ((val < valid_low) || (val > valid_high))) {
665                new_values[k] = Double.NaN;
666            }
667        }
668
669        return new_values;
670    }
671
672    public void setMultiScaleDimName(String multiScaleDimName) {
673        this.multiScaleDimName = multiScaleDimName;
674    }
675
676    public int getMultiScaleDimensionIndex() {
677        return multiScaleDimensionIndex;
678    }
679
680    public boolean hasMultiDimensionScale() {
681        return hasMultiDimensionScale;
682    }
683
684    public void setHasMultiDimensionScale(boolean yesno) {
685        hasMultiDimensionScale = yesno;
686    }
687
688    public void setMultiScaleIndex(int idx) {
689        this.soIndex = idx;
690    }
691
692    /**
693     * Should be generalized. For now works for short to float conversions
694     * 
695     * @param values
696     *            the set of input values to map
697     * @param lut
698     *            the lookup table for direct mapping input to output values
699     * @return output array as primitive floats
700     */
701
702    public Object processRangeApplyLUT(short[] values, float[] lut) {
703
704        float[] newValues = new float[values.length];
705
706        int lutLen = lut.length;
707
708        for (int i = 0; i < values.length; i++) {
709            short tmpVal = values[i];
710            if (tmpVal > 0) {
711                if (tmpVal < lutLen) {
712                    newValues[i] = lut[tmpVal];
713                } else {
714                    newValues[i] = Float.NaN;
715                }
716            } else {
717                newValues[i] = Float.NaN;
718            }
719        }
720
721        return newValues;
722    }
723
724}