001 /*
002 * This file is part of McIDAS-V
003 *
004 * Copyright 2007-2013
005 * Space Science and Engineering Center (SSEC)
006 * University of Wisconsin - Madison
007 * 1225 W. Dayton Street, Madison, WI 53706, USA
008 * https://www.ssec.wisc.edu/mcidas
009 *
010 * All Rights Reserved
011 *
012 * McIDAS-V is built on Unidata's IDV and SSEC's VisAD libraries, and
013 * some McIDAS-V source code is based on IDV and VisAD source code.
014 *
015 * McIDAS-V is free software; you can redistribute it and/or modify
016 * it under the terms of the GNU Lesser Public License as published by
017 * the Free Software Foundation; either version 3 of the License, or
018 * (at your option) any later version.
019 *
020 * McIDAS-V is distributed in the hope that it will be useful,
021 * but WITHOUT ANY WARRANTY; without even the implied warranty of
022 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
023 * GNU Lesser Public License for more details.
024 *
025 * You should have received a copy of the GNU Lesser Public License
026 * along with this program. If not, see http://www.gnu.org/licenses.
027 */
028
029 package edu.wisc.ssec.mcidasv.data.cyclone;
030
031 import java.io.ByteArrayInputStream;
032 import java.io.ByteArrayOutputStream;
033 import java.io.File;
034 import java.io.FileNotFoundException;
035 import java.io.IOException;
036 import java.net.URL;
037 import java.text.SimpleDateFormat;
038 import java.util.ArrayList;
039 import java.util.Calendar;
040 import java.util.Date;
041 import java.util.GregorianCalendar;
042 import java.util.Hashtable;
043 import java.util.List;
044 import java.util.TimeZone;
045 import java.util.zip.GZIPInputStream;
046
047 import org.apache.commons.net.ftp.FTP;
048 import org.apache.commons.net.ftp.FTPClient;
049
050 import ucar.unidata.data.BadDataException;
051 import ucar.unidata.data.DataSourceDescriptor;
052 import ucar.unidata.util.DateUtil;
053 import ucar.unidata.util.IOUtil;
054 import ucar.unidata.util.StringUtil;
055 import visad.DateTime;
056 import visad.Real;
057 import visad.RealType;
058 import visad.VisADException;
059 import visad.georef.EarthLocation;
060 import visad.georef.EarthLocationLite;
061
062 /**
063 */
064 public class AtcfStormDataSource extends StormDataSource {
065
066 /** _more_ */
067 private int BASEIDX = 0;
068
069 /** _more_ */
070 private int IDX_BASIN = BASEIDX++;
071
072 /** _more_ */
073 private int IDX_CY = BASEIDX++;
074
075 /** _more_ */
076 private int IDX_YYYYMMDDHH = BASEIDX++;
077
078 /** _more_ */
079 private int IDX_TECHNUM = BASEIDX++;
080
081 /** _more_ */
082 private int IDX_TECH = BASEIDX++;
083
084 /** _more_ */
085 private int IDX_TAU = BASEIDX++;
086
087 /** _more_ */
088 private int IDX_LAT = BASEIDX++;
089
090 /** _more_ */
091 private int IDX_LON = BASEIDX++;
092
093 /** _more_ */
094 private int IDX_VMAX = BASEIDX++;
095
096 /** _more_ */
097 private int IDX_MSLP = BASEIDX++;
098
099 /** _more_ */
100 private int IDX_TY = BASEIDX++;
101
102 /** _more_ */
103 private int IDX_RAD = BASEIDX++;
104
105 /** _more_ */
106 private int IDX_WINDCODE = BASEIDX++;
107
108 /** _more_ */
109 private int IDX_RAD1 = BASEIDX++;
110
111 /** _more_ */
112 private int IDX_RAD2 = BASEIDX++;
113
114 /** _more_ */
115 private int IDX_RAD3 = BASEIDX++;
116
117 /** _more_ */
118 private int IDX_RAD4 = BASEIDX++;
119
120 /** _more_ */
121 private int IDX_RADP = BASEIDX++;
122
123 /** _more_ */
124 private int IDX_RRP = BASEIDX++;
125
126 /** _more_ */
127 private int IDX_MRD = BASEIDX++;
128
129 /** _more_ */
130 private int IDX_GUSTS = BASEIDX++;
131
132 /** _more_ */
133 private int IDX_EYE = BASEIDX++;
134
135 /** _more_ */
136 private int IDX_SUBREGION = BASEIDX++;
137
138 /** _more_ */
139 private int IDX_MAXSEAS = BASEIDX++;
140
141 /** _more_ */
142 private int IDX_INITIALS = BASEIDX++;
143
144 /** _more_ */
145 private int IDX_DIR = BASEIDX++;
146
147 /** _more_ */
148 private int IDX_SPEED = BASEIDX++;
149
150 /** _more_ */
151 private int IDX_STORMNAME = BASEIDX++;
152
153 /** _more_ */
154 private int IDX_DEPTH = BASEIDX++;
155
156 /** _more_ */
157 private int IDX_SEAS = BASEIDX++;
158
159 /** _more_ */
160 private int IDX_SEASCODE = BASEIDX++;
161
162 /** _more_ */
163 private int IDX_SEAS1 = BASEIDX++;
164
165 /** _more_ */
166 private int IDX_SEAS2 = BASEIDX++;
167
168 /** _more_ */
169 private int IDX_SEAS3 = BASEIDX++;
170
171 /** _more_ */
172 private int IDX_SEAS4 = BASEIDX++;
173
174 /** _more_ */
175 private static final String PREFIX_ANALYSIS = "a";
176
177 /** _more_ */
178 private static final String PREFIX_BEST = "b";
179
180 /** _more_ */
181 private static final String WAY_BEST = "BEST";
182
183 /** _more_ */
184 private static final String WAY_CARQ = "CARQ";
185
186 /** _more_ */
187 private static final String WAY_WRNG = "WRNG";
188
189 /** _more_ */
190 private static String DEFAULT_PATH = "ftp://anonymous:password@ftp.nhc.noaa.gov/atcf";
191
192 /** _more_ */
193 private String path;
194
195 /** _more_ */
196 private List<StormInfo> stormInfos;
197
198 /** _more_ */
199 private StormTrackCollection localTracks;
200
201 /**
202 * _more_
203 *
204 * @throws Exception
205 * _more_
206 */
207 public AtcfStormDataSource() throws Exception {
208 }
209
210 /**
211 * _more_
212 *
213 * @return _more_
214 */
215 public String getFullDescription() {
216 return "ATCF Data Source<br>Path:" + path;
217 }
218
219 /**
220 * _more_
221 *
222 * @param descriptor
223 * _more_
224 * @param url
225 * _more_
226 * @param properties
227 * _more_
228 */
229 public AtcfStormDataSource(DataSourceDescriptor descriptor, String url,
230 Hashtable properties) {
231 super(descriptor, "ATCF Storm Data", "ATCF Storm Data", properties);
232 if ((url == null) || (url.trim().length() == 0)
233 || url.trim().equalsIgnoreCase("default")) {
234 url = DEFAULT_PATH;
235 }
236 path = url;
237 }
238
239 /**
240 * _more_
241 *
242 * @return _more_
243 */
244 public String getId() {
245 return "atcf";
246 }
247
248 /**
249 * _more_
250 *
251 * @param suffix
252 * _more_
253 *
254 * @return _more_
255 */
256 private String getFullPath(String suffix) {
257 return path + "/" + suffix;
258 }
259
260 /**
261 * _more_
262 */
263 protected void initializeStormData() {
264 try {
265 incrOutstandingGetDataCalls();
266 stormInfos = new ArrayList<StormInfo>();
267 if (path.toLowerCase().endsWith(".atcf")
268 || path.toLowerCase().endsWith(".gz")
269 || path.toLowerCase().endsWith(".dat")) {
270 String name = IOUtil.stripExtension(IOUtil.getFileTail(path));
271 StormInfo si = new StormInfo(name, new DateTime(new Date()));
272 stormInfos.add(si);
273 localTracks = new StormTrackCollection();
274 readTracks(si, localTracks, path, null, true);
275 List<StormTrack> trackList = localTracks.getTracks();
276
277 if (trackList.size() > 0) {
278 si.setStartTime(trackList.get(0).getStartTime());
279 }
280 return;
281 }
282
283 byte[] techs = readFile(getFullPath("nhc_techlist.dat"), true);
284 if (techs != null) {
285 /*
286 * NUM TECH ERRS RETIRED COLOR DEFAULTS INT-DEFS RADII-DEFS
287 * LONG-NAME 00 CARQ 0 0 0 0 0 1 Combined ARQ Position 00 WRNG 0
288 * 0 0 0 0 1 Warning
289 */
290 int cnt = 0;
291 for (String line : StringUtil.split(new String(techs), "\n",
292 true, true)) {
293 if (cnt++ == 0) {
294 continue;
295 }
296 if (line.length() > 67) {
297 String id = line.substring(3, 10).trim();
298 String name = line.substring(67).trim();
299 // System.out.println (id + ":" +name);
300 getWay(id, name);
301 }
302 }
303 }
304
305 // byte[] bytes = readFile(getFullPath("archive/storm.table"),
306 byte[] bytes = readFile(getFullPath("index/storm_list.txt"), false);
307 String stormTable = new String(bytes);
308 List lines = StringUtil.split(stormTable, "\n", true, true);
309
310 SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMddHH");
311 fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
312 for (int i = 0; i < lines.size(); i++) {
313 String line = (String) lines.get(i);
314 List toks = StringUtil.split(line, ",", true);
315 String name = (String) toks.get(0);
316 String basin = (String) toks.get(1);
317 String number = (String) toks.get(7);
318 String year = (String) toks.get(8);
319 int y = new Integer(year).intValue();
320 String id = basin + "_" + number + "_" + year;
321 if (name.equals("UNNAMED")) {
322 name = id;
323 }
324 String dttm = (String) toks.get(11);
325 Date date = fmt.parse(dttm);
326 StormInfo si = new StormInfo(id, name, basin, number,
327 new DateTime(date));
328 stormInfos.add(si);
329
330 }
331 } catch (Exception exc) {
332 logException("Error initializing ATCF data", exc);
333 } finally {
334 decrOutstandingGetDataCalls();
335 }
336 }
337
338 /**
339 * _more_
340 *
341 * @return _more_
342 */
343 public List<StormInfo> getStormInfos() {
344 return stormInfos;
345 }
346
347 /**
348 * _more_
349 *
350 * @param s
351 * _more_
352 *
353 * @return _more_
354 */
355 private double getDouble(String s) {
356 if (s == null) {
357 return Double.NaN;
358 }
359 if (s.length() == 0) {
360 return Double.NaN;
361 }
362 return new Double(s).doubleValue();
363 }
364
365 /**
366 * _more_
367 *
368 * @throws VisADException
369 * _more_
370 */
371 protected void initParams() throws VisADException {
372 super.initParams();
373 if (obsParams == null) {
374 obsParams = new StormParam[] { PARAM_STORMCATEGORY,
375 PARAM_MINPRESSURE, PARAM_MAXWINDSPEED_KTS };
376
377 }
378 }
379
380 /**
381 * _more_
382 *
383 * @param stormInfo
384 * _more_
385 * @param tracks
386 * _more_
387 * @param trackFile
388 * _more_
389 * @param waysToUse
390 * _more_
391 * @param throwError
392 * _more_
393 *
394 *
395 * @return _more_
396 * @throws Exception
397 * _more_
398 */
399 private boolean readTracks(StormInfo stormInfo,
400 StormTrackCollection tracks, String trackFile,
401 Hashtable<String, Boolean> waysToUse, boolean throwError)
402 throws Exception {
403
404 long t1 = System.currentTimeMillis();
405 byte[] bytes = readFile(trackFile, true);
406 long t2 = System.currentTimeMillis();
407 // System.err.println("read time:" + (t2 - t1));
408 boolean isZip = trackFile.endsWith(".gz");
409 if ((bytes == null) && isZip) {
410 String withoutGZ = trackFile.substring(0, trackFile.length() - 3);
411 bytes = readFile(withoutGZ, true);
412 isZip = false;
413 }
414
415 if (bytes == null) {
416 if (throwError) {
417 throw new BadDataException("Unable to read track file:"
418 + trackFile);
419 }
420 return false;
421 }
422
423 if (isZip) {
424 GZIPInputStream zin = new GZIPInputStream(new ByteArrayInputStream(
425 bytes));
426 bytes = IOUtil.readBytes(zin);
427 zin.close();
428 }
429 GregorianCalendar convertCal = new GregorianCalendar(
430 DateUtil.TIMEZONE_GMT);
431 convertCal.clear();
432
433 String trackData = new String(bytes);
434 List lines = StringUtil.split(trackData, "\n", true, true);
435 SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMddHH");
436 fmt.setTimeZone(TimeZone.getTimeZone("UTC"));
437 Hashtable trackMap = new Hashtable();
438 Real altReal = new Real(RealType.Altitude, 0);
439 // System.err.println("obs:" + lines.size());
440 /*
441 * Hashtable okWays = new Hashtable(); okWays.put(WAY_CARQ, "");
442 * okWays.put(WAY_WRNG, ""); okWays.put(WAY_BEST, ""); okWays.put("ETA",
443 * ""); okWays.put("NGX", ""); okWays.put("BAMS", "");
444 */
445 Hashtable seenDate = new Hashtable();
446 initParams();
447 int xcnt = 0;
448 for (int i = 0; i < lines.size(); i++) {
449 String line = (String) lines.get(i);
450 if (i == 0) {
451 // System.err.println(line);
452 }
453 List toks = StringUtil.split(line, ",", true);
454 /*
455 * System.err.println(toks.size() + " " + BASEIDX);
456 * if(toks.size()<BASEIDX-1) { System.err.println("bad line:" +
457 * line); continue; } else { System.err.println("good line:" +
458 * line); }
459 */
460
461 // BASIN,CY,YYYYMMDDHH,TECHNUM,TECH,TAU,LatN/S,LonE/W,VMAX,MSLP,TY,RAD,WINDCODE,RAD1,RAD2,RAD3,RAD4,RADP,RRP,MRD,GUSTS,EYE,SUBREGION,MAXSEAS,INITIALS,DIR,SPEED,STORMNAME,DEPTH,SEAS,SEASCODE,SEAS1,SEAS2,SEAS3,SEAS4
462 // AL, 01, 2007050612, , BEST, 0, 355N, 740W, 35, 1012, EX, 34, NEQ,
463 // 0, 0, 0, 120,
464 // AL, 01, 2007050812, 01, CARQ, -24, 316N, 723W, 55, 0, DB, 34,
465 // AAA, 0, 0, 0, 0,
466
467 String dateString = (String) toks.get(IDX_YYYYMMDDHH);
468 String wayString = (String) toks.get(IDX_TECH);
469 // if (okWays.get(wayString) == null) {
470 // continue;
471 // }
472 boolean isBest = wayString.equals(WAY_BEST);
473 boolean isWarning = wayString.equals(WAY_WRNG);
474 boolean isCarq = wayString.equals(WAY_CARQ);
475
476 int category = ((IDX_TY < toks.size()) ? getCategory((String) toks
477 .get(IDX_TY)) : CATEGORY_XX);
478 if (category != CATEGORY_XX) {
479 // System.err.println("cat:" + category);
480 }
481
482 String fhour = (String) toks.get(IDX_TAU);
483 int forecastHour = new Integer(fhour).intValue();
484 // A hack - we've seen some atfc files that have a 5 character
485 // forecast hour
486 // right padded with "00", eg., 01200
487 if ((fhour.length() == 5) && (forecastHour > 100)) {
488 forecastHour = forecastHour / 100;
489 }
490
491 if (isWarning || isCarq) {
492 forecastHour = -forecastHour;
493 }
494
495 // Check for unique dates for this way
496 String dttmkey = wayString + "_" + dateString + "_" + forecastHour;
497 if (seenDate.get(dttmkey) != null) {
498 continue;
499 }
500 seenDate.put(dttmkey, dttmkey);
501
502 Date dttm = fmt.parse(dateString);
503 convertCal.setTime(dttm);
504 String key;
505 Way way = getWay(wayString, null);
506 if (!isBest && (waysToUse != null) && (waysToUse.size() > 0)
507 && (waysToUse.get(wayString) == null)) {
508 continue;
509 }
510
511 if (isBest) {
512 key = wayString;
513 } else {
514 key = wayString + "_" + dateString;
515 convertCal.add(Calendar.HOUR_OF_DAY, forecastHour);
516 }
517 dttm = convertCal.getTime();
518 StormTrack track = (StormTrack) trackMap.get(key);
519 if (track == null) {
520 way = (isBest ? Way.OBSERVATION : way);
521 track = new StormTrack(stormInfo, addWay(way), new DateTime(
522 dttm), obsParams);
523 trackMap.put(key, track);
524 tracks.addTrack(track);
525 }
526 String latString = (String) toks.get(IDX_LAT);
527 String lonString = (String) toks.get(IDX_LON);
528 String t = latString + " " + lonString;
529
530 boolean south = latString.endsWith("S");
531 boolean west = lonString.endsWith("W");
532 double latitude = Double.parseDouble(latString.substring(0,
533 latString.length() - 1)) / 10.0;
534 double longitude = Double.parseDouble(lonString.substring(0,
535 lonString.length() - 1)) / 10.0;
536 if (south) {
537 latitude = -latitude;
538 }
539 if (west) {
540 longitude = -longitude;
541 }
542
543 EarthLocation elt = new EarthLocationLite(new Real(
544 RealType.Latitude, latitude), new Real(RealType.Longitude,
545 longitude), altReal);
546
547 List<Real> attributes = new ArrayList<Real>();
548
549 double windspeed = ((IDX_VMAX < toks.size()) ? getDouble((String) toks
550 .get(IDX_VMAX))
551 : Double.NaN);
552 double pressure = ((IDX_MSLP < toks.size()) ? getDouble((String) toks
553 .get(IDX_MSLP))
554 : Double.NaN);
555 attributes.add(PARAM_STORMCATEGORY.getReal((double) category));
556 attributes.add(PARAM_MINPRESSURE.getReal(pressure));
557 attributes.add(PARAM_MAXWINDSPEED_KTS.getReal(windspeed));
558
559 StormTrackPoint stp = new StormTrackPoint(elt, new DateTime(dttm),
560 forecastHour, attributes);
561
562 track.addPoint(stp);
563 }
564 return true;
565 }
566
567 /**
568 * _more_
569 *
570 * @return _more_
571 */
572 public String getWayName() {
573 return "Tech";
574 }
575
576 /**
577 * _more_
578 *
579 * @param stormInfo
580 * _more_
581 * @param waysToUse
582 * _more_
583 * @param observationWay
584 * _more_
585 *
586 * @return _more_
587 *
588 * @throws Exception
589 * _more_
590 */
591 public StormTrackCollection getTrackCollectionInner(StormInfo stormInfo,
592 Hashtable<String, Boolean> waysToUse, Way observationWay)
593 throws Exception {
594 if (localTracks != null) {
595 return localTracks;
596 }
597
598 long t1 = System.currentTimeMillis();
599 StormTrackCollection tracks = new StormTrackCollection();
600
601 String trackFile;
602 boolean justObs = (waysToUse != null) && (waysToUse.size() == 1)
603 && (waysToUse.get(Way.OBSERVATION.toString()) != null);
604 int nowYear = new GregorianCalendar(DateUtil.TIMEZONE_GMT)
605 .get(Calendar.YEAR);
606 int stormYear = getYear(stormInfo.getStartTime());
607 // If its the current year then its in the aid_public dir
608 String aSubDir = ((stormYear == nowYear) ? "aid_public"
609 : ("archive/" + stormYear));
610 String bSubDir = ((stormYear == nowYear) ? "btk"
611 : ("archive/" + stormYear));
612 if (!justObs) {
613 trackFile = getFullPath(aSubDir + "/" + PREFIX_ANALYSIS
614 + stormInfo.getBasin().toLowerCase()
615 + stormInfo.getNumber() + stormYear + ".dat.gz");
616 // What we think might be in the archive might actually be the last
617 // year
618 // and they haven't moved it into the archive
619 try {
620 readTracks(stormInfo, tracks, trackFile, waysToUse, true);
621 } catch (BadDataException bde) {
622 if (!aSubDir.equals("aid_public")) {
623 try {
624 trackFile = getFullPath("aid_public/" + PREFIX_ANALYSIS
625 + stormInfo.getBasin().toLowerCase()
626 + stormInfo.getNumber() + stormYear + ".dat.gz");
627 readTracks(stormInfo, tracks, trackFile, waysToUse,
628 true);
629 } catch (BadDataException bde2) {
630 System.err.println("Failed reading 'A' file for storm:"
631 + stormInfo + " file:" + trackFile);
632 }
633 }
634 // System.err.println("Failed reading 'A' file for storm:" +
635 // stormInfo+" file:" + trackFile);
636 }
637 }
638 // Now read the b"est file
639 trackFile = getFullPath(bSubDir + "/" + PREFIX_BEST
640 + stormInfo.getBasin().toLowerCase() + stormInfo.getNumber()
641 + stormYear + ".dat.gz");
642 try {
643 readTracks(stormInfo, tracks, trackFile, null, true);
644 } catch (BadDataException bde) {
645 if (!bSubDir.equals("btk")) {
646 try {
647 trackFile = getFullPath("btk/" + PREFIX_BEST
648 + stormInfo.getBasin().toLowerCase()
649 + stormInfo.getNumber() + stormYear + ".dat.gz");
650 readTracks(stormInfo, tracks, trackFile, null, true);
651 } catch (BadDataException bde2) {
652 System.err.println("Failed reading 'B' file for storm:"
653 + stormInfo + " file:" + trackFile);
654 }
655
656 }
657 // System.err.println("Failed reading 'B' file for storm:" +
658 // stormInfo+" file:" + trackFile);
659 }
660 long t2 = System.currentTimeMillis();
661 // System.err.println("time: " + (t2 - t1));
662
663 return tracks;
664 }
665
666 /**
667 * Set the Directory property.
668 *
669 * @param value
670 * The new value for Directory
671 */
672 public void setPath(String value) {
673 path = value;
674 }
675
676 /**
677 * Get the Directory property.
678 *
679 * @return The Directory
680 */
681 public String getPath() {
682 return path;
683 }
684
685 /**
686 * _more_
687 *
688 * @param file
689 * _more_
690 * @param ignoreErrors
691 * _more_
692 *
693 * @return _more_
694 *
695 * @throws Exception
696 * _more_
697 */
698 private byte[] readFile(String file, boolean ignoreErrors) throws Exception {
699 if (new File(file).exists()) {
700 return IOUtil.readBytes(IOUtil.getInputStream(file, getClass()));
701 }
702 if (!file.startsWith("ftp:")) {
703 if (ignoreErrors) {
704 return null;
705 }
706 throw new FileNotFoundException("Could not read file: " + file);
707 }
708
709 URL url = new URL(file);
710 FTPClient ftp = new FTPClient();
711 try {
712 ftp.connect(url.getHost());
713 ftp.login("anonymous", "password");
714 ftp.setFileType(FTP.IMAGE_FILE_TYPE);
715 ftp.enterLocalPassiveMode();
716 ByteArrayOutputStream bos = new ByteArrayOutputStream();
717 if (ftp.retrieveFile(url.getPath(), bos)) {
718 return bos.toByteArray();
719 } else {
720 throw new IOException("Unable to retrieve file:" + url);
721 }
722 } catch (org.apache.commons.net.ftp.FTPConnectionClosedException fcce) {
723 System.err.println("ftp error:" + fcce);
724 System.err.println(ftp.getReplyString());
725 if (!ignoreErrors) {
726 throw fcce;
727 }
728 return null;
729 } catch (Exception exc) {
730 if (!ignoreErrors) {
731 throw exc;
732 }
733 return null;
734 } finally {
735 try {
736 ftp.logout();
737 } catch (Exception exc) {
738 }
739 try {
740 ftp.disconnect();
741 } catch (Exception exc) {
742 }
743
744 }
745 }
746
747 }