001/* 002 * This file is part of McIDAS-V 003 * 004 * Copyright 2007-2016 005 * Space Science and Engineering Center (SSEC) 006 * University of Wisconsin - Madison 007 * 1225 W. Dayton Street, Madison, WI 53706, USA 008 * https://www.ssec.wisc.edu/mcidas 009 * 010 * All Rights Reserved 011 * 012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and 013 * some McIDAS-V source code is based on IDV and VisAD source code. 014 * 015 * McIDAS-V is free software; you can redistribute it and/or modify 016 * it under the terms of the GNU Lesser Public License as published by 017 * the Free Software Foundation; either version 3 of the License, or 018 * (at your option) any later version. 019 * 020 * McIDAS-V is distributed in the hope that it will be useful, 021 * but WITHOUT ANY WARRANTY; without even the implied warranty of 022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 023 * GNU Lesser Public License for more details. 024 * 025 * You should have received a copy of the GNU Lesser Public License 026 * along with this program. If not, see http://www.gnu.org/licenses. 027 */ 028 029package edu.wisc.ssec.mcidasv.data.hydra; 030 031import java.util.ArrayList; 032import java.util.HashMap; 033import java.util.Iterator; 034import java.util.List; 035import java.util.Map; 036import java.util.Set; 037 038import org.slf4j.Logger; 039import org.slf4j.LoggerFactory; 040 041import edu.wisc.ssec.mcidasv.data.QualityFlag; 042 043import ucar.ma2.Array; 044import ucar.ma2.DataType; 045import ucar.ma2.Index; 046import ucar.ma2.IndexIterator; 047import ucar.ma2.Range; 048import ucar.nc2.Attribute; 049import ucar.nc2.Dimension; 050import ucar.nc2.NetcdfFile; 051import ucar.nc2.Structure; 052import ucar.nc2.Variable; 053 054/** 055 * Provides a view and operations on a set of contiguous data granules as if they 056 * were a single granule. 057 * 058 * This file needs to implement the same signatures NetCDFFile does, 059 * but for aggregations of consecutive granules. 060 * 061 * @author tommyj 062 * 063 */ 064 065public class GranuleAggregation implements MultiDimensionReader { 066 067 private static final Logger logger = LoggerFactory.getLogger(GranuleAggregation.class); 068 069 // this structure holds the NcML readers that get passed in 070 List<NetcdfFile> nclist = new ArrayList<>(); 071 072 // this holds the MultiDimensionReaders, here NetCDFFile 073 List<NetCDFFile> ncdfal = null; 074 075 // need an ArrayList for each variable hashmap structure 076 List<Map<String, Variable>> varMapList = new ArrayList<>(); 077 List<Map<String, String[]>> varDimNamesList = new ArrayList<>(); 078 List<Map<String, Class>> varDataTypeList = new ArrayList<>(); 079 080 // map of granule index and granule in-track length for each variable 081 Map<String, Map<Integer, Integer>> varGranInTrackLengths = new HashMap<>(); 082 Map<String, int[]> varAggrDimLengths = new HashMap<>(); 083 084 // this object is used to handle granules like VIIRS Imagery EDRs, where scan 085 // gaps of varying sizes and locations in the granule must be removed. If 086 // present, an initial read with these "cut" ranges will be done before subsetting 087 Map<Integer, List<Range>> granCutRanges = new HashMap<>(); 088 Map<Integer, Integer> granCutScans = new HashMap<>(); 089 090 // except quality flags - only need one hashmap per aggregation 091 // it maps the broken out variable name back to the original packed variable name 092 Map<String, QualityFlag> qfMap = null; 093 094 // For those variables which are assembled from other variables based on LUTs 095 Map<String, float[]> lutMap = null; 096 097 // variable can have bulk array processor set by the application 098 Map<String, RangeProcessor> varToRangeProcessor = new HashMap<>(); 099 100 private int granuleCount = -1; 101 private String inTrackDimensionName = null; 102 private String inTrackGeoDimensionName = null; 103 private String crossTrackDimensionName = null; 104 private Set<String> products; 105 private String origName = null; 106 // assume we are working with VIIRS, will toggle if not 107 private boolean isVIIRS = true; 108 109 public GranuleAggregation(List<NetCDFFile> ncdfal, Set<String> products, 110 String inTrackDimensionName, String inTrackGeoDimensionName, 111 String crossTrackDimensionName, boolean isVIIRS) throws Exception { 112 if (ncdfal == null) throw new Exception("No data: empty Suomi NPP aggregation object"); 113 this.inTrackDimensionName = inTrackDimensionName; 114 this.crossTrackDimensionName = crossTrackDimensionName; 115 this.inTrackGeoDimensionName = inTrackGeoDimensionName; 116 this.ncdfal = ncdfal; 117 this.products = products; 118 this.isVIIRS = isVIIRS; 119 init(ncdfal); 120 } 121 122 public GranuleAggregation(List<NetCDFFile> ncdfal, Set<String> products, 123 String inTrackDimensionName, String inTrackGeoDimensionName, 124 String crossTrackDimensionName) throws Exception { 125 this(ncdfal, products, inTrackDimensionName, inTrackGeoDimensionName, crossTrackDimensionName, false); 126 } 127 128 public GranuleAggregation(List<NetCDFFile> ncdfal, Set<String> products, 129 String inTrackDimensionName, String crossTrackDimensionName) throws Exception { 130 this(ncdfal, products, inTrackDimensionName, inTrackDimensionName, crossTrackDimensionName, false); 131 } 132 133 public GranuleAggregation(List<NetCDFFile> ncdfal, Set<String> products, 134 String inTrackDimensionName, String crossTrackDimensionName, boolean isEDR) throws Exception { 135 this(ncdfal, products, inTrackDimensionName, inTrackDimensionName, crossTrackDimensionName, isEDR); 136 } 137 138 public Class getArrayType(String array_name) { 139 array_name = mapNameIfQualityFlag(array_name); 140 array_name = mapNameIfLUTVar(array_name); 141 return varDataTypeList.get(0).get(array_name); 142 } 143 144 public String[] getDimensionNames(String array_name) { 145 array_name = mapNameIfQualityFlag(array_name); 146 array_name = mapNameIfLUTVar(array_name); 147 return varDimNamesList.get(0).get(array_name); 148 } 149 150 public int[] getDimensionLengths(String array_name) { 151 array_name = mapNameIfQualityFlag(array_name); 152 array_name = mapNameIfLUTVar(array_name); 153 return varAggrDimLengths.get(array_name); 154 } 155 156 private String mapNameIfQualityFlag(String array_name) { 157 // only applies if name is from a packed quality flag 158 // we pull data from the "mapped" variable name, a packed byte 159 if (qfMap != null) { 160 if (qfMap.containsKey(array_name)) { 161 origName = array_name; 162 QualityFlag qf = qfMap.get(array_name); 163 String mappedName = qf.getPackedName(); 164 logger.debug("Key: " + array_name + " mapped to: " + mappedName); 165 return mappedName; 166 } 167 } 168 return array_name; 169 } 170 171 private String mapNameIfLUTVar(String array_name) { 172 // only applies if name is from a LUT pseudo variable 173 // we pull data from a "mapped" variable name, and apply a LUT to that variable 174 origName = ""; 175 if (lutMap != null) { 176 if (lutMap.containsKey(array_name)) { 177 origName = array_name; 178 String mappedName = array_name.substring(0, array_name.length() - 3); 179 logger.debug("Key: " + array_name + " mapped to: " + mappedName); 180 return mappedName; 181 } 182 } 183 return array_name; 184 } 185 186 public float[] getFloatArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 187 return (float[]) readArray(array_name, start, count, stride); 188 } 189 190 public int[] getIntArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 191 return (int[]) readArray(array_name, start, count, stride); 192 } 193 194 public double[] getDoubleArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 195 return (double[]) readArray(array_name, start, count, stride); 196 } 197 198 public short[] getShortArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 199 return (short[]) readArray(array_name, start, count, stride); 200 } 201 202 public byte[] getByteArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 203 return (byte[]) readArray(array_name, start, count, stride); 204 } 205 206 public Object getArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 207 return readArray(array_name, start, count, stride); 208 } 209 210 public HDFArray getGlobalAttribute(String attr_name) throws Exception { 211 throw new Exception("GranuleAggregation.getGlobalAttributes: Unimplemented"); 212 } 213 214 public HDFArray getArrayAttribute(String array_name, String attr_name) throws Exception { 215 Variable var = varMapList.get(0).get(array_name); 216 if (var == null) return null; 217 218 Attribute attr = var.findAttribute(attr_name); 219 if (attr == null) return null; 220 221 Array attrVals = attr.getValues(); 222 DataType dataType = attr.getDataType(); 223 Object array = attrVals.copyTo1DJavaArray(); 224 225 HDFArray harray = null; 226 227 if (dataType.getPrimitiveClassType() == Float.TYPE) { 228 harray = HDFArray.make((float[])array); 229 } 230 else if (dataType.getPrimitiveClassType() == Double.TYPE) { 231 harray = HDFArray.make((double[])array); 232 } 233 else if (dataType == DataType.STRING) { 234 harray = HDFArray.make((String[])array); 235 } 236 else if (dataType.getPrimitiveClassType() == Short.TYPE) { 237 harray = HDFArray.make((short[])array); 238 } 239 else if (dataType.getPrimitiveClassType() == Integer.TYPE) { 240 harray = HDFArray.make((int[])array); 241 } 242 return harray; 243 } 244 245 public void close() throws Exception { 246 // close each NetCDF file 247 for (NetcdfFile n : nclist) { 248 n.close(); 249 } 250 } 251 252 private void init(List<NetCDFFile> ncdfal) throws Exception { 253 254 logger.debug("init in..."); 255 // make a NetCDFFile object from the NcML for each granule 256 for (NetCDFFile n : ncdfal) { 257 logger.debug("loading another NetCDF file from NcML..."); 258 NetcdfFile ncfile = n.getNetCDFFile(); 259 nclist.add(ncfile); 260 } 261 262 granuleCount = nclist.size(); 263 logger.debug("Granule count: " + granuleCount); 264 265 // All files do NOT have the same structure, so need to look at each ncfile 266 // For ex, some MODIS granules have slightly different in-track and along-track 267 // lengths 268 269 NetcdfFile ncfile = null; 270 for (int ncIdx = 0; ncIdx < nclist.size(); ncIdx++) { 271 272 // good place to initialize the cut Range ArrayList for each granule 273 Integer granuleIndex = new Integer(ncIdx); 274 List<Range> al = new ArrayList<>(); 275 276 int cutScanCount = 0; 277 278 ncfile = nclist.get(ncIdx); 279 280 Iterator<Variable> varIter = ncfile.getVariables().iterator(); 281 while (varIter.hasNext()) { 282 Variable var = varIter.next(); 283 logger.trace("Variable " + var.getShortName() + ", Rank: " + var.getRank()); 284 varAggrDimLengths.put(var.getFullName(), new int[var.getRank()]); 285 varGranInTrackLengths.put(var.getFullName(), new HashMap<>()); 286 287 // Here, let's try to check the data for EDR fill lines 288 // and if found, try to handle it by simply adjusting the dimensions 289 // for this granule. Sound like a plan? We'll see... 290 291 // TJJ May 2016 292 // "simply adjusting the dimensions" he says 293 // Anyway, we now do this check for EDRs and SDRs, it can manifest for both 294 295 if (isVIIRS) { 296 297 // look through lat grid, look for missing scans 298 String varName = var.getShortName(); 299 if (varName.endsWith("Latitude")) { 300 // iterate through the scan lines, looking for fill lines 301 // NOTE: we only need to check the first column! so set 302 // up an appropriate Range to cut the read down significantly 303 int[] shape = var.getShape(); 304 List<Range> alr = new ArrayList<>(); 305 alr.add(new Range(0, shape[0] - 1, 1)); 306 alr.add(new Range(0, 1, 1)); 307 Array a = var.read(alr); 308 int granLength = shape[0]; 309 int scanLength = shape[1]; 310 Index index = a.getIndex(); 311 float fVal = 0.0f; 312 313 int rangeOffset = 0; 314 boolean prvScanWasCut = false; 315 boolean needClosingRange = false; 316 boolean hadCutRanges = false; 317 boolean someMissing = false; 318 319 for (int i = 0; i < shape[0]; i++) { 320 321 someMissing = false; 322 fVal = a.getFloat(index.set(i, 0)); 323 if (fVal < -90.0f) { 324 someMissing = true; 325 } 326 327 if (someMissing) { 328 hadCutRanges = true; 329 cutScanCount++; 330 logger.trace("Found a cut scan " + i + ", last val: " + fVal); 331 if ((prvScanWasCut) || (i == 0)) { 332 if (i == 0) { 333 rangeOffset = 0; 334 } else { 335 rangeOffset = i + 1; 336 } 337 } else { 338 try { 339 // We are using 2D ranges 340 logger.trace("Adding Range: " + rangeOffset 341 + ", " + (i - 1) + ", 1"); 342 al.add(new Range(rangeOffset, i - 1, 1)); 343 logger.trace("Adding Range: " + 0 + ", " 344 + (scanLength - 1) + ", 1"); 345 al.add(new Range(0, scanLength - 1, 1)); 346 } catch (Exception e) { 347 e.printStackTrace(); 348 } 349 rangeOffset = i; 350 } 351 prvScanWasCut = true; 352 } else { 353 prvScanWasCut = false; 354 } 355 356 // check to see if closing Range needed, good data at end 357 if ((! prvScanWasCut) && (i == (granLength - 1))) { 358 if (hadCutRanges) { 359 needClosingRange = true; 360 } 361 } 362 } 363 364 if (needClosingRange) { 365 // We are using 2D ranges 366 logger.trace("Adding closing cut range: " + rangeOffset + ", " + (shape[0] - 1) + ", 1"); 367 al.add(new Range(rangeOffset, shape[0] - 1, 1)); 368 al.add(new Range(0, scanLength - 1, 1)); 369 } 370 371 // if only one contiguous range, process as a normal clean granule 372 if (! hadCutRanges) { 373 al.clear(); 374 } 375 376 logger.debug("Total scans cut this granule: " + cutScanCount); 377 378 } 379 } 380 } 381 granCutScans.put(granuleIndex, new Integer(cutScanCount)); 382 granCutRanges.put(granuleIndex, al); 383 } 384 385 for (int ncIdx = 0; ncIdx < nclist.size(); ncIdx++) { 386 387 ncfile = nclist.get(ncIdx); 388 389 Map<String, Variable> varMap = new HashMap<>(); 390 Map<String, String[]> varDimNames = new HashMap<>(); 391 Map<String, Class> varDataType = new HashMap<>(); 392 393 Iterator<Variable> varIter = ncfile.getVariables().iterator(); 394 int varInTrackIndex = -1; 395 while (varIter.hasNext()) { 396 Variable var = varIter.next(); 397 398 boolean foundProduct = false; 399 for (String s : products) { 400 if (s.contains(var.getFullName())) { 401 logger.trace("Valid product: " + var.getFullName()); 402 foundProduct = true; 403 break; 404 } 405 } 406 407 if (! foundProduct) { 408 logger.trace("Skipping variable: " + var.getFullName()); 409 continue; 410 } 411 412 if (var instanceof Structure) { 413 // simply skip these, applicable only to IASI far as I know 414 continue; 415 } 416 417 int rank = var.getRank(); 418 419 // bypass any less-than-2D variables for now... 420 if (rank < 2) { 421 continue; 422 } 423 424 String varName = var.getFullName(); 425 varMap.put(varName, var); 426 Iterator<Dimension> dimIter = var.getDimensions().iterator(); 427 String[] dimNames = new String[rank]; 428 int[] dimLengths = new int[rank]; 429 int cnt = 0; 430 boolean notDisplayable = false; 431 varInTrackIndex = getInTrackIndex(var); 432 433 while (dimIter.hasNext()) { 434 Dimension dim = dimIter.next(); 435 String s = dim.getShortName(); 436 if ((s != null) && (!s.isEmpty())) { 437 if ((! s.equals(inTrackDimensionName)) && 438 ((! s.startsWith("Band")) && (cnt == 0)) && 439 (! varName.endsWith("Latitude")) && 440 (! varName.endsWith("latitude")) && 441 (! varName.endsWith("Latitude_TC")) && 442 (! varName.endsWith("Longitude")) && 443 (! varName.endsWith("longitude")) && 444 (! varName.endsWith("Longitude_TC")) && 445 (! s.equals(crossTrackDimensionName))) { 446 notDisplayable = true; 447 break; 448 } 449 } 450 String dimName = dim.getShortName(); 451 logger.debug("GranuleAggregation init, variable: " + varName + ", dimension name: " + dimName + ", length: " + dim.getLength()); 452 if (dimName == null) dimName = "dim" + cnt; 453 if (dimName.isEmpty()) { 454 dimName = "dim" + cnt; 455 } 456 dimNames[cnt] = dimName; 457 dimLengths[cnt] = dim.getLength(); 458 cnt++; 459 } 460 461 // skip to next variable if it's not displayable data 462 if (notDisplayable) continue; 463 464 // adjust in-track dimension if needed (scans were cut) 465 int cutScans = 0; 466 if (! granCutScans.isEmpty() && granCutScans.containsKey(ncIdx)) { 467 cutScans = granCutScans.get(new Integer(ncIdx)); 468 } else { 469 granCutScans.put(new Integer(ncIdx), new Integer(0)); 470 } 471 dimLengths[varInTrackIndex] = dimLengths[varInTrackIndex] - cutScans; 472 473 // XXX TJJ - can below block go away? Think so... 474 int[] aggrDimLengths = varAggrDimLengths.get(varName); 475 for (int i = 0; i < rank; i++) { 476 if (i == varInTrackIndex) { 477 aggrDimLengths[i] += dimLengths[i]; 478 } else { 479 aggrDimLengths[i] = dimLengths[i]; 480 } 481 } 482 483 varDimNames.put(varName, dimNames); 484 varDataType.put(varName, var.getDataType().getPrimitiveClassType()); 485 486 if (varInTrackIndex < 0) { 487 logger.debug("Skipping variable with unknown dimension: " + var.getFullName()); 488 continue; 489 } 490 491 Map<Integer, Integer> granIdxToInTrackLen = varGranInTrackLengths.get(varName); 492 granIdxToInTrackLen.put(ncIdx, new Integer(dimLengths[varInTrackIndex])); 493 494 dimLengths[varInTrackIndex] = dimLengths[varInTrackIndex] * granuleCount; 495 varDataType.put(varName, var.getDataType().getPrimitiveClassType()); 496 } 497 498 // add the new hashmaps to our enclosing lists 499 varMapList.add(varMap); 500 varDimNamesList.add(varDimNames); 501 varDataTypeList.add(varDataType); 502 503 } 504 } 505 506 /** 507 * Based on the names of the variable dimensions, determine the in-track index. 508 * 509 * @param v {@code Variable} that {@literal "contains"} dimension names that 510 * allow for inference of the in-track index. {@code null} is allowed. 511 * 512 * @return correct index (0 or greater), or -1 if error. 513 */ 514 515 private int getInTrackIndex(Variable v) { 516 517 int index = -1; 518 boolean is2D = false; 519 boolean is3D = false; 520 521 String inTrackName = null; 522 523 // typical sanity check 524 if (v == null) return index; 525 526 // lat/lon vars have different dimension names 527 if ((v.getFullName().endsWith("Latitude")) || 528 (v.getFullName().endsWith("Latitude_TC")) || 529 (v.getFullName().endsWith("Longitude")) || 530 (v.getFullName().endsWith("latitude")) || 531 (v.getFullName().endsWith("longitude")) || 532 (v.getFullName().endsWith("Longitude_TC"))) { 533 if ((v.getFullName().startsWith("All_Data")) || 534 (v.getFullName().startsWith("observation_data"))) { 535 inTrackName = inTrackDimensionName; 536 } else { 537 inTrackName = inTrackGeoDimensionName; 538 } 539 } else { 540 inTrackName = inTrackDimensionName; 541 } 542 // pull out the dimensions 543 List<Dimension> dList = v.getDimensions(); 544 545 // right now, we only handle 2D and 3D variables. 546 // TJJ XXX it does get trickier, and we will have to expand this 547 // to deal with for example CrIS data... 548 int numDimensions = dList.size(); 549 550 // the only 4D data right now is CrIS, return 0 551 if (numDimensions == 4) return 0; 552 553 if ((numDimensions == 2) || (numDimensions == 3)) { 554 if (numDimensions == 2) is2D = true; 555 if (numDimensions == 3) is3D = true; 556 } else { 557 return index; 558 } 559 560 // if the data is 2D, we use the SwathAdapter class, 561 // if 3D, we use the SpectrumAdapter class 562 for (int i = 0; i < numDimensions; i++) { 563 if (is2D) { 564 // XXX TJJ - if empty name, in-track index is 0 565 if ((dList.get(i).getShortName() == null) || (dList.get(i).getShortName().isEmpty())) { 566 logger.trace("Empty dimension name!, assuming in-track dim is 0"); 567 return 0; 568 } 569 if (dList.get(i).getShortName().equals(inTrackName)) { 570 index = i; 571 break; 572 } 573 } 574 if (is3D) { 575 // XXX TJJ - if empty name, in-track index is 0 576 if ((dList.get(i).getShortName() == null) || (dList.get(i).getShortName().isEmpty())) { 577 logger.debug("Empty dimension name!, assuming in-track dim is 0"); 578 return 0; 579 } 580 if (dList.get(i).getShortName().equals(inTrackName)) { 581 index = i; 582 break; 583 } 584 } 585 } 586 587 // hopefully we found the right one 588 return index; 589 } 590 591 private synchronized Object readArray(String array_name, int[] start, int[] count, int[] stride) throws Exception { 592 593 String mapName = array_name; 594 array_name = mapNameIfQualityFlag(array_name); 595 array_name = mapNameIfLUTVar(array_name); 596 // how many dimensions are we dealing with 597 int dimensionCount = start.length; 598 599 // pull out a representative variable so we can determine which index is in-track 600 Variable vTmp = varMapList.get(0).get(array_name); 601 logger.trace(""); 602 logger.trace("Working on var: " + array_name); 603 int vInTrackIndex = getInTrackIndex(vTmp); 604 605 int loGranuleId = 0; 606 int hiGranuleId = 0; 607 608 Map<Integer, Integer> granIdxToInTrackLen = varGranInTrackLengths.get(array_name); 609 int numGrans = granIdxToInTrackLen.size(); 610 611 int[] vGranuleLengths = new int[numGrans]; 612 for (int k = 0; k < numGrans; k++) { 613 vGranuleLengths[k] = granIdxToInTrackLen.get(k); 614 logger.debug("readArray, gran len: " + vGranuleLengths[k] + ", scans cut: " + granCutScans.get(k)); 615 } 616 617 int strt = start[vInTrackIndex]; 618 int stp = strt + (count[vInTrackIndex] - 1) * stride[vInTrackIndex]; 619 int cnt = 0; 620 for (int k = 0; k < numGrans; k++) { 621 int granLen = granIdxToInTrackLen.get(k); 622 cnt += granLen; 623 if (strt < cnt) { 624 loGranuleId = k; 625 break; 626 } 627 } 628 629 cnt = 0; 630 for (int k = 0; k < numGrans; k++) { 631 int granLen = granIdxToInTrackLen.get(k); 632 cnt += granLen; 633 if (stp < cnt) { 634 hiGranuleId = k; 635 break; 636 } 637 } 638 639 int totalScansCut = 0; 640 for (int k = loGranuleId; k <= hiGranuleId; k++) { 641 totalScansCut += granCutScans.get(k); 642 } 643 logger.debug("Scans cut for this selection: " + totalScansCut); 644 645 // next, we break out the offsets, counts, and strides for each granule 646 int granuleSpan = hiGranuleId - loGranuleId + 1; 647 648 logger.debug("readArray req, loGran: " + loGranuleId + ", hiGran: " + 649 hiGranuleId + ", granule span: " + granuleSpan + ", dimCount: " + dimensionCount); 650 651 for (int i = 0; i < dimensionCount; i++) { 652 logger.debug("start[" + i + "]: " + start[i]); 653 logger.debug("count[" + i + "]: " + count[i]); 654 logger.debug("stride[" + i + "]: " + stride[i]); 655 } 656 657 int [][] startSet = new int [granuleSpan][dimensionCount]; 658 int [][] countSet = new int [granuleSpan][dimensionCount]; 659 int [][] strideSet = new int [granuleSpan][dimensionCount]; 660 int countSubtotal = 0; 661 662 int inTrackTotal = 0; 663 for (int i = 0; i < loGranuleId; i++) { 664 inTrackTotal += vGranuleLengths[i]; 665 } 666 667 // this part is a little tricky - set the values for each granule we need to access for this read 668 for (int i = 0; i < granuleSpan; i++) { 669 670 inTrackTotal += vGranuleLengths[loGranuleId + i]; 671 672 for (int j = 0; j < dimensionCount; j++) { 673 // for all indeces other than the in-track index, the numbers match what was passed in 674 if (j != vInTrackIndex) { 675 startSet[i][j] = start[j]; 676 countSet[i][j] = count[j] * stride[j]; 677 strideSet[i][j] = stride[j]; 678 } else { 679 // for the in-track index, it's not so easy... 680 // for first granule, start is what's passed in 681 if (i == 0) { 682 if (inTrackTotal > vGranuleLengths[loGranuleId + i]) { 683 startSet[i][j] = 684 (start[j] % (inTrackTotal - vGranuleLengths[loGranuleId + i])); 685 } else { 686 startSet[i][j] = start[j]; 687 } 688 } else { 689 startSet[i][j] = 690 ((inTrackTotal - (vGranuleLengths[loGranuleId + i])) % stride[j]); 691 // if there is a remainder, need to offset stride - remainder into next gran 692 if (startSet[i][j] != 0) startSet[i][j] = stride[j] - startSet[i][j]; 693 } 694 695 // counts may be different for start, end, and middle granules 696 if (i == 0) { 697 // is this the first and only granule? 698 if (granuleSpan == 1) { 699 countSet[i][j] = count[j] * stride[j]; 700 // TJJ May 2016 701 // This condition manifests because there are times when 702 // "fill" scans are cut from otherwise fine granules. 703 // e.g., to the chooser it may look like there are 3072 valid lines, 704 // but by the time we get here we realize there are really 3056 705 // This typically shortens the granule by one scan (16 lines) 706 if (countSet[i][j] > (vGranuleLengths[loGranuleId+i])) 707 countSet[i][j] = vGranuleLengths[loGranuleId+i]; 708 709 // or is this the first of multiple granules... 710 } else { 711 if ((inTrackTotal - start[j]) < (count[j] * stride[j])) { 712 countSet[i][j] = inTrackTotal - start[j]; 713 } else { 714 countSet[i][j] = count[j] * stride[j]; 715 } 716 countSubtotal += countSet[i][j]; 717 } 718 } else { 719 // middle granules 720 if (i < (granuleSpan - 1)) { 721 countSet[i][j] = vGranuleLengths[loGranuleId+i] - startSet[i][j]; 722 countSubtotal += countSet[i][j]; 723 } else { 724 // the end granule 725 countSet[i][j] = (count[j] * stride[j]) - countSubtotal - startSet[i][j]; 726 // TJJ May 2016 727 // This condition manifests because there are times when 728 // "fill" scans are cut from otherwise fine granules. 729 // e.g., to the chooser it may look like there are 3072 valid lines, 730 // but by the time we get here we realize there are really 3056 731 // This typically shortens the granule by one scan (16 lines) 732 if (countSet[i][j] > (vGranuleLengths[loGranuleId+i] - startSet[i][j])) 733 countSet[i][j] = vGranuleLengths[loGranuleId+i] - startSet[i][j]; 734 } 735 } 736 // luckily, stride never changes 737 strideSet[i][j] = stride[j]; 738 } 739 } 740 } 741 742 int totalLength = 0; 743 int rangeListCount = 0; 744 List<Array> arrayList = new ArrayList<>(); 745 for (int granuleIdx = 0; granuleIdx < granuleCount; granuleIdx++) { 746 if ((granuleIdx >= loGranuleId) && (granuleIdx <= hiGranuleId)) { 747 Variable var = varMapList.get(loGranuleId + (granuleIdx-loGranuleId)).get(array_name); 748 749 if (var instanceof Structure) { 750 // what to do here? 751 } else { 752 List<Range> rangeList = new ArrayList<>(); 753 for (int dimensionIdx = 0; dimensionIdx < dimensionCount; dimensionIdx++) { 754 logger.debug("Creating new Range: " + startSet[rangeListCount][dimensionIdx] + 755 ", " + (startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1) + ", " + strideSet[rangeListCount][dimensionIdx]); 756 Range range = new Range( 757 startSet[rangeListCount][dimensionIdx], 758 startSet[rangeListCount][dimensionIdx] + countSet[rangeListCount][dimensionIdx] - 1, 759 strideSet[rangeListCount][dimensionIdx] 760 ); 761 rangeList.add(dimensionIdx, range); 762 } 763 rangeListCount++; 764 765 // If there were chunks of fill data to remove... 766 List<Range> al = granCutRanges.get(new Integer(granuleIdx)); 767 if (! al.isEmpty()) { 768 List<Variable> varChunks = new ArrayList<>(); 769 for (int rangeCount = 0; rangeCount < al.size(); rangeCount+=2) { 770 List<Range> rl = new ArrayList<>(); 771 rl.add(al.get(rangeCount)); 772 rl.add(al.get(rangeCount + 1)); 773 varChunks.add(var.section(rl)); 774 } 775 776 int [] newShape = var.getShape(); 777 int cutScans = granCutScans.get(granuleIdx); 778 newShape[0] = newShape[0] - cutScans; 779 logger.trace("New Shape: " + newShape[0] + ", " + newShape[1]); 780 Array single = Array.factory(var.getDataType(), newShape); 781 782 // now read variable chunk data into single contiguous array 783 int idx = 0; 784 for (Variable v : varChunks) { 785 Array data = v.read(); 786 int [] tmpShape = v.getShape(); 787 for (int tIdx = 0; tIdx < tmpShape.length; tIdx++) { 788 logger.trace("Shape[" + tIdx + "]: " + tmpShape[tIdx]); 789 } 790 IndexIterator ii = data.getIndexIterator(); 791 while (ii.hasNext()) { 792 single.setFloat(idx, ii.getFloatNext()); 793 idx++; 794 } 795 } 796 797 // finally, apply subset ranges 798 logger.debug("Size of cut src array: " + single.getSize()); 799 Array subarray = single.section(rangeList); 800 totalLength += subarray.getSize(); 801 arrayList.add(subarray); 802 logger.debug("Size of cut sub array: " + subarray.getSize()); 803 804 } else { 805 Array subarray = var.read(rangeList); 806 totalLength += subarray.getSize(); 807 logger.debug("Size of reg sub array: " + subarray.getSize()); 808 arrayList.add(subarray); 809 } 810 811 } 812 // put in an empty ArrayList placeholder to retain a slot for each granule 813 } else { 814 Array emptyArray = null; 815 arrayList.add(emptyArray); 816 } 817 } 818 819 // last, concatenate the individual NetCDF arrays pulled out 820 821 Class outType; 822 Class arrayType = getArrayType(array_name); 823 RangeProcessor rngProcessor = varToRangeProcessor.get(array_name); 824 if (rngProcessor == null) { 825 outType = getArrayType(array_name); 826 } 827 else { 828 outType = java.lang.Float.TYPE; 829 } 830 logger.debug("Creating aggregated array, totalLength: " + totalLength); 831 832 // TJJ May 2016 833 // I'm starting to think there may be a bug in the Java NetCDF section/subarray 834 // code - I have chased this quite a bit with no solution, the total length 835 // sectioned out sometimes exceeds the total requested. It's almost as if a 836 // previous section length is retained. Anyway, as a hack for now I will just 837 // truncate if this occurs. We also need to watch for overflow in the arraycopy 838 // calls below 839 840 if (count.length < 3) { 841 if (totalLength > (count[0] * count[1])) { 842 totalLength = count[0] * count[1]; 843 } 844 } 845 846 float[] finalArray = new float[totalLength]; 847 848 int destPos = 0; 849 int granIdx = 0; 850 851 int remaining = totalLength; 852 for (Array a : arrayList) { 853 if (a != null) { 854 Object primArray = a.copyTo1DJavaArray(); 855 primArray = processArray( 856 mapName, array_name, arrayType, granIdx, primArray, rngProcessor, start, count 857 ); 858 if (a.getSize() > remaining) { 859 System.arraycopy(primArray, 0, finalArray, destPos, remaining); 860 } else { 861 System.arraycopy(primArray, 0, finalArray, destPos, (int) a.getSize()); 862 } 863 destPos += a.getSize(); 864 remaining -= (int) a.getSize(); 865 } 866 granIdx++; 867 } 868 869 return finalArray; 870 } 871 872 /** 873 * @param qfMap the qfMap to set 874 */ 875 876 public void setQfMap(Map<String, QualityFlag> qfMap) { 877 this.qfMap = qfMap; 878 } 879 880 /** 881 * @param lutMap the lutMap to set 882 */ 883 884 public void setLUTMap(Map<String, float[]> lutMap) { 885 this.lutMap = lutMap; 886 } 887 888 public Map<String, Variable> getVarMap() { 889 return varMapList.get(0); 890 } 891 892 public List<NetCDFFile> getReaders() { 893 return this.ncdfal; 894 } 895 896 /* pass individual granule pieces just read from dataset through the RangeProcessor */ 897 private Object processArray(String mapName, String array_name, Class arrayType, int granIdx, Object values, RangeProcessor rngProcessor, int[] start, int[] count) { 898 899 if (rngProcessor == null) { 900 return values; 901 } 902 else { 903 ((AggregationRangeProcessor) rngProcessor).setWhichRangeProcessor(granIdx); 904 905 Object outArray = null; 906 907 if (arrayType == Short.TYPE) { 908 // if variable is a LUT var, apply LUT 909 if ((lutMap != null) && (lutMap.containsKey(mapName))) { 910 float lut[] = lutMap.get(mapName); 911 outArray = rngProcessor.processRangeApplyLUT((short []) values, lut); 912 } else { 913 outArray = rngProcessor.processRange((short[]) values, null); 914 } 915 } else if (arrayType == Byte.TYPE) { 916 // if variable is a bit-field quality flag, apply mask 917 if ((qfMap != null) && (qfMap.containsKey(origName))) { 918 QualityFlag qf = qfMap.get(origName); 919 outArray = rngProcessor.processRangeQualityFlag((byte[]) values, null, qf); 920 } else { 921 outArray = rngProcessor.processRange((byte[]) values, null); 922 } 923 } else if (arrayType == Float.TYPE) { 924 outArray = rngProcessor.processRange((float[]) values, null); 925 } else if (arrayType == Double.TYPE) { 926 outArray = rngProcessor.processRange((double[]) values, null); 927 } 928 929 return outArray; 930 } 931 } 932 933 /* Application can supply a RangeProcessor for a variable 'arrayName' */ 934 public void addRangeProcessor(String arrayName, RangeProcessor rangeProcessor) { 935 varToRangeProcessor.put(arrayName, rangeProcessor); 936 } 937 938}