@Test
public void testMicrosoftBlobS3() throws IOException {
  // https://nexradsa.blob.core.windows.net/nexrad-l2/1997/07/07/KHPX/KHPX19970707_000827.gz
  String host = "nexradsa.blob.core.windows.net";
  String bucket = "nexrad-l2";
  String key = "1991/07/20/KTLX/KTLX19910720_160529.gz";
  String s3Uri = "cdms3://" + host + "/" + bucket + "?" + key;
  try (NetcdfFile ncfile = NetcdfFiles.open(s3Uri)) {

    assertThat(ncfile.findDimension("scanR")).isNotNull();
    assertThat(ncfile.findDimension("gateR")).isNotNull();
    assertThat(ncfile.findDimension("radialR")).isNotNull();

    Variable reflectivity = ncfile.findVariable("Reflectivity");
    Assert.assertNotNull(reflectivity);

    // read array
    Array array = reflectivity.read();

    assertThat(array.getRank()).isEqualTo(3);

    assertThat(array.getShape()).isEqualTo(new int[] {1, 366, 460});
  }
}
 

@Test
public void compareWithBuilder()
    throws IOException, InstantiationException, IllegalAccessException, ClassNotFoundException {
  logger.info("TestBuilders on {}%n", filename);
  SPFactory.setServiceProvider("ucar.nc2.iosp.netcdf3.N3raf");
  try (NetcdfFile org = NetcdfFile.open(filename)) {
    SPFactory.setServiceProvider("ucar.nc2.internal.iosp.netcdf3.N3iospNew");
    try (NetcdfFile withBuilder = NetcdfFiles.open(filename)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      if (!compare.compare(org, withBuilder)) {
        System.out.printf("Compare %s%n%s%n", filename, f);
        fail();
      }
    }
  } finally {
    SPFactory.setServiceProvider("ucar.nc2.iosp.netcdf3.N3raf");
  }
}
 

private String findUniqueName(Structure.Builder<?> struct, String want, String def) {
  if (want == null) {
    return def + tempNo++;
  }

  String vwant = NetcdfFiles.makeValidCdmObjectName(want);
  Optional<Variable.Builder<?>> oldV = struct.findMemberVariable(vwant);
  if (!oldV.isPresent()) {
    return vwant;
  }

  int seq = 2;
  while (true) {
    String wantSeq = vwant + "-" + seq;
    oldV = struct.findMemberVariable(wantSeq);
    if (!oldV.isPresent()) {
      return wantSeq;
    }
    seq++;
  }
}
 

/**
 * Test GridCoordSystem.getLatLon()
 *
 * @throws IOException if ...
 * @throws InvalidRangeException if ...
 */
@Test
public void checkGridCoordSystem_getLatLon() throws IOException, InvalidRangeException {
  int[] origin = new int[] {j, i};
  int[] shape = new int[] {1, 1};

  NetcdfFile ncf = NetcdfFiles.open(datasetLocation);
  Variable latVar = ncf.findVariable("lat");
  Array latArray = latVar.read(origin, shape);
  Variable lonVar = ncf.findVariable("lon");
  Array lonArray = lonVar.read(origin, shape);

  double latVal = latArray.getDouble(latArray.getIndex());
  double lonVal = lonArray.getDouble(lonArray.getIndex());

  GridDataset gd = GridDataset.open(datasetLocation);
  GeoGrid gg = gd.findGridByName("hs");
  GridCoordSystem gridCoordSys = gg.getCoordinateSystem();
  // gridCoordSys.getXHorizAxis().;
  // gridCoordSys.getYHorizAxis();
  LatLonPoint llPnt = gridCoordSys.getLatLon(170, 62);

  Assert.assertEquals(lat, llPnt.getLatitude(), 0.001);
  Assert.assertEquals(lon, llPnt.getLongitude(), 0.001);
}
 

@Test
public void testTimeAxisEval() throws IOException {
  /**
   * The following tests BugFixes.evalTimeAxes, called by ucar.nc2.dt.grid.GridCoordSys.isGridCoordSys
   */
  String testFileFullPath = TestDir.cdmUnitTestDir + "ft/grid/echoTops_runtime.nc";
  GridDataset runtimeDataset = new GridDataset(new NetcdfDataset(NetcdfFiles.open(testFileFullPath)));
  if (runtimeDataset.getGrids().isEmpty()) {
    throw new RuntimeException("Runtime data file did not generate a dataset with grids");
  }
  if (runtimeDataset.getGrids().get(0).getCoordinateSystem().getRunTimeAxis() == null) {
    throw new RuntimeException("Runtime data file did not generate a dataset with a RunTime axis");
  }

  System.out.println("BugFixesTest - completed.");
}
 

private boolean doCompare(String location, boolean showCompare, boolean showEach, boolean compareData)
    throws IOException {
  try (NetcdfFile ncfile = NetcdfFiles.open(location); NetcdfFile jni = openJni(location)) {
    jni.setLocation(location + " (jni)");
    // System.out.printf("Compare %s to %s%n", ncfile.getIosp().getClass().getName(),
    // jni.getIosp().getClass().getName());

    Formatter f = new Formatter();
    CompareNetcdf2 tc = new CompareNetcdf2(f, showCompare, showEach, compareData);
    boolean ok = tc.compare(ncfile, jni, new CompareNetcdf2.Netcdf4ObjectFilter());
    System.out.printf(" %s compare %s ok = %s%n", ok ? "" : "***", location, ok);
    if (!ok || (showCompare && showCompareResults))
      System.out.printf("%s%n=====================================%n", f);
    return ok;
  }
}
 

private boolean scanBufrFile(String filename)
    throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException {
  int count = 0;
  try (RandomAccessFile raf = new RandomAccessFile(filename, "r")) {

    MessageScanner scan = new MessageScanner(raf);
    while (scan.hasNext()) {
      Message m = scan.next();
      if (m == null)
        continue;
      byte[] mbytes = scan.getMessageBytes(m);
      try (NetcdfFile ncfile = NetcdfFiles.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp2")) {
        NetcdfDataset ncd = new NetcdfDataset(ncfile);
      }
    }
  }
  return true;
}
 

@Test
public void testThinGridAtEndofFile() throws IOException {
  final String testfile = "../grib/src/test/data/thinGrid.grib1";
  try (NetcdfFile nc = NetcdfFiles.open(testfile)) {

    Variable var = nc.findVariable("Temperature_isobaric");
    Array data = var.read();
    Assert.assertEquals(73 * 73, data.getSize());

    float first = data.getFloat(0);
    float last = data.getFloat((73 * 73) - 1);

    Assert.assertEquals(291.0, first, 1e-6);
    Assert.assertEquals(278.0, last, 1e-6);
  }
}
 

private void compareBuilder() {
  if (ds == null)
    return;
  String fileLocation = ds.getLocation();
  try (NetcdfFile org = NetcdfFile.open(fileLocation)) {
    try (NetcdfFile withBuilder = NetcdfFiles.open(fileLocation)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      boolean ok = compare.compare(org, withBuilder, new CoordsObjFilter());
      infoTA.setText(f.toString());
      infoTA.gotoTop();
      infoWindow.setTitle("Compare Old (file1) with Builder (file 2)");
      infoWindow.show();
    }
  } catch (Throwable ioe) {
    StringWriter sw = new StringWriter(10000);
    ioe.printStackTrace(new PrintWriter(sw));
    infoTA.setText(sw.toString());
    infoTA.gotoTop();
    infoWindow.show();
  }
}
 

private Map<Integer, GridMatch> getGridsOld(MFile ff, Formatter f) throws IOException {
  Map<Integer, GridMatch> grids = new HashMap<>(100);
  try (NetcdfFile ncfile = NetcdfFiles.open(ff.getPath(), "ucar.nc2.iosp.grib.GribServiceProvider", -1, null, null)) {
    NetcdfDataset ncd = new NetcdfDataset(ncfile);
    GridDataset grid = new GridDataset(ncd);
    for (GridDatatype dt : grid.getGrids()) {
      GridMatch gm = new GridMatch(grid, dt, false);
      GridMatch dup = grids.get(gm.hashCode());
      if (dup != null)
        f.format(" DUP OLD (%d == %d) = %s (%s) and DUP %s (%s)%n", gm.hashCode(), dup.hashCode(),
            gm.grid.getFullName(), gm.show(), dup.grid.getFullName(), dup.show());
      else
        grids.put(gm.hashCode(), gm);
    }
  } catch (Throwable t) {
    t.printStackTrace();
  }
  return grids;
}
 

/**
 * AWS S3
 *
 * https://aws.amazon.com/s3/
 *
 * @throws IOException Error accessing object store
 */
@Test
public void awsFullReadFile() throws IOException {
  System.setProperty(AWS_REGION_PROP_NAME, AWS_G16_REGION);
  try (NetcdfFile ncfile = NetcdfFiles.open(AWS_G16_S3_URI_FULL)) {
    testFullReadGoes16S3(ncfile);
  } finally {
    System.clearProperty(AWS_REGION_PROP_NAME);
  }
}
 

@Test
public void awsPartialReadFileDefaultRegionProp() throws IOException, InvalidRangeException {
  System.setProperty(AWS_REGION_PROP_NAME, AWS_G16_REGION);
  try (NetcdfFile ncfile = NetcdfFiles.open(AWS_G16_S3_URI_FULL)) {
    testPartialReadGoes16S3(ncfile);
  } finally {
    System.clearProperty(AWS_REGION_PROP_NAME);
  }
}
 

@Test
public void awsPartialReadFileSimple() throws IOException, InvalidRangeException {
  System.setProperty(AWS_REGION_PROP_NAME, AWS_G16_REGION);
  try (NetcdfFile ncfile = NetcdfFiles.open(AWS_G16_S3_URI_SIMPLE)) {
    testPartialReadGoes16S3(ncfile);
  } finally {
    System.clearProperty(AWS_REGION_PROP_NAME);
  }
}
 

@Test
public void awsProfileSharedCredsGoodDefault() throws IOException {
  System.setProperty(AWS_SHARED_CREDENTIALS_FILE_PROP, credentialsFileGoodDefault);
  try (NetcdfFile ncfile = NetcdfFiles.open(AWS_G16_S3_URI_FULL)) {
    assertThat(ncfile).isNotNull();
  } finally {
    System.clearProperty(AWS_SHARED_CREDENTIALS_FILE_PROP);
  }
}
 

@Test(expected = software.amazon.awssdk.services.s3.model.S3Exception.class)
public void awsProfileSharedCredsBadDefault() throws IOException {
  // point to a shared credentials file with a bad default region (i.e. GOES-16 does not exist in the Asia Pacific
  // (Mumbai) region
  System.setProperty(AWS_SHARED_CREDENTIALS_FILE_PROP, credentialsFileBadDefault);
  try (NetcdfFile ncfile = NetcdfFiles.open(AWS_G16_S3_URI_FULL)) {
    assertThat(ncfile).isNotNull();
  } finally {
    System.clearProperty(AWS_SHARED_CREDENTIALS_FILE_PROP);
  }
}
 

@Test
public void awsProfileSharedCredsBadDefaultGoodProfileName() throws IOException {
  // point to a shared credentials file with a bad default region (i.e. GOES-16 does not exist in the Asia Pacific
  // (Mumbai) region) but use a profile with a good region
  System.setProperty(AWS_SHARED_CREDENTIALS_FILE_PROP, credentialsFileBadDefault);
  String cdmS3Uri = String.format("cdms3://%[email protected]/noaa-goes16?", GOOD_PROFILE_NAME) + COMMON_G16_KEY;
  try (NetcdfFile ncfile = NetcdfFiles.open(cdmS3Uri)) {
    assertThat(ncfile).isNotNull();
  } finally {
    System.clearProperty(AWS_SHARED_CREDENTIALS_FILE_PROP);
  }
}
 

@Test(expected = software.amazon.awssdk.services.s3.model.S3Exception.class)
public void awsProfileSharedCredsGoodDefaultBadProfileName() throws IOException {
  // point to a shared credentials file with a good default region but use a profile that uses a bad region (one
  // without goes16 data)
  System.setProperty(AWS_SHARED_CREDENTIALS_FILE_PROP, credentialsFileBadDefault);
  String cdmS3Uri = String.format("cdms3://%[email protected]/noaa-goes16?", BAD_PROFILE_NAME) + COMMON_G16_KEY;
  try (NetcdfFile ncfile = NetcdfFiles.open(cdmS3Uri)) {
    assertThat(ncfile).isNotNull();
  } finally {
    System.clearProperty(AWS_SHARED_CREDENTIALS_FILE_PROP);
  }
}
 

@Test
public void awsSharedCredsPrecedence() throws IOException {
  // Test that the region set in the custom shared credentials file takes precedence
  System.setProperty(AWS_REGION_PROP_NAME, Region.AP_SOUTH_1.id());
  System.setProperty(AWS_SHARED_CREDENTIALS_FILE_PROP, credentialsFileBadDefault);
  String cdmS3Uri = String.format("cdms3://%[email protected]/noaa-goes16?", GOOD_PROFILE_NAME) + COMMON_G16_KEY;
  try (NetcdfFile ncfile = NetcdfFiles.open(cdmS3Uri)) {
    assertThat(ncfile).isNotNull();
  } finally {
    System.clearProperty(AWS_SHARED_CREDENTIALS_FILE_PROP);
    System.clearProperty(AWS_REGION_PROP_NAME);
  }
}
 

/**
 * Open data file
 *
 * @param fileName File name
 */
public void openData(String fileName) {
    try {
        boolean canOpen = NetcdfFiles.canOpen(fileName);
        if (canOpen) {
            this.openNetCDFData(fileName);
        } else if (ARLDataInfo.canOpen(fileName)) {
            this.openARLData(fileName);
        }
    } catch (IOException ex) {
        Logger.getLogger(MeteoDataInfo.class.getName()).log(Level.SEVERE, null, ex);
    }
}
 

public void shoudGetVerticalStridedSubset() throws Exception {

    mockMvc.perform(requestBuilder).andExpect(MockMvcResultMatchers.status().isOk()).andExpect(new ResultMatcher() {
      public void match(MvcResult result) throws Exception {
        // Open the binary response in memory
        NetcdfFile nf;
        ucar.nc2.dt.grid.GridDataset gdsDataset;
        if (TestDir.cdmUseBuilders) {
          nf = NetcdfFiles.openInMemory("test_data.ncs", result.getResponse().getContentAsByteArray());
          gdsDataset =
              new ucar.nc2.dt.grid.GridDataset(NetcdfDatasets.enhance(nf, NetcdfDataset.getDefaultEnhanceMode(), null));
        } else {
          nf = NetcdfFile.openInMemory("test_data.ncs", result.getResponse().getContentAsByteArray());
          gdsDataset = new ucar.nc2.dt.grid.GridDataset(new NetcdfDataset(nf));
        }

        assertTrue(gdsDataset.getCalendarDateRange().isPoint());
        List<VariableSimpleIF> vars = gdsDataset.getDataVariables();
        int[][] shapes = new int[vars.size()][];
        int cont = 0;
        for (VariableSimpleIF var : vars) {
          // int[] shape =var.getShape();
          shapes[cont] = var.getShape();
          cont++;
          // String dimensions =var.getDimensions().toString();
          // int rank =var.getRank();
        }

        assertArrayEquals(expectedShapes, shapes);
      }
    });
    // assertEquals(expectedValue, Integer.valueOf( gdsDataset.getDataVariables().size()));
  }
 

@Test
public void compareStores() throws IOException {
  System.setProperty(AWS_REGION_PROP_NAME, AWS_G16_REGION);
  try (NetcdfFile osdc = NetcdfFiles.open(OSDC_G16_S3_URI);
      NetcdfFile gcs = NetcdfFiles.open(GCS_G16_S3_URI);
      NetcdfFile aws = NetcdfFiles.open(AWS_G16_S3_URI_FULL)) {
    Formatter f = new Formatter();
    CompareNetcdf2 comparer = new CompareNetcdf2(f, false, false, true);
    Assert.assertTrue(comparer.compare(aws, gcs));
    Assert.assertTrue(comparer.compare(aws, osdc));
    Assert.assertTrue(comparer.compare(osdc, gcs));
  } finally {
    System.clearProperty(AWS_REGION_PROP_NAME);
  }
}
 

@Test
@Category(NeedsUcarNetwork.class)
public void ncarPartialReadFile() throws IOException, InvalidRangeException {
  try (NetcdfFile ncfile = NetcdfFiles.open(NCAR_G16_S3_URI)) {
    testPartialReadGoes16S3(ncfile);
  }
}
 

@Test
@Category(NeedsUcarNetwork.class)
public void testActiveScaleWithCredsS3() throws IOException {
  String host = "stratus.ucar.edu";
  String bucket = "unidata-netcdf-zarr-testing";
  String key = "netcdf-java/test/GFS_Global_0p25deg_20200326_1200_apparent_temperature.nc4";
  String s3Uri = "cdms3://" + NCAR_PROFILE_NAME + "@" + host + "/" + bucket + "?" + key;
  try (NetcdfFile ncfile = NetcdfFiles.open(s3Uri)) {
    Attribute conv = ncfile.findGlobalAttributeIgnoreCase("Conventions");
    assertThat(conv).isNotNull();
    assertThat(conv.getStringValue()).ignoringCase().isEqualTo("CF-1.6");
  }
}
 

@Test
@Category(NeedsUcarNetwork.class)
public void testActiveScaleS3() throws IOException {
  // do not need credentials to run this one, just access to the internal ucar network.
  String host = "stratus.ucar.edu";
  String bucket = "rda-data";
  String key = "ds262.0/CERFACS/uo_Omon_NEMO3-2_FRCCORE2_f_r1i1p1_199801-200712.nc";
  String s3Uri = "cdms3://" + host + "/" + bucket + "?" + key;
  try (NetcdfFile ncfile = NetcdfFiles.open(s3Uri)) {
    Attribute conv = ncfile.findGlobalAttributeIgnoreCase("Conventions");
    assertThat(conv).isNotNull();
    assertThat(conv.getStringValue()).ignoringCase().isEqualTo("CF-1.4");
  }
}
 

@Test
public void testCompressedObjectRead() throws IOException {
  String region = Region.US_EAST_1.toString();
  String bucket = "noaa-nexrad-level2";
  String key = "1991/07/20/KTLX/KTLX19910720_160529.gz";
  String s3uri = "cdms3:" + bucket + "?" + key;

  System.setProperty("aws.region", region);
  try (NetcdfFile ncfile = NetcdfFiles.open(s3uri)) {

    assertThat(ncfile.findDimension("scanR")).isNotNull();
    assertThat(ncfile.findDimension("gateR")).isNotNull();
    assertThat(ncfile.findDimension("radialR")).isNotNull();

    Variable reflectivity = ncfile.findVariable("Reflectivity");
    Assert.assertNotNull(reflectivity);

    // read array
    Array array = reflectivity.read();

    assertThat(array.getRank()).isEqualTo(3);

    assertThat(array.getShape()).isEqualTo(new int[] {1, 366, 460});
  } finally {
    System.clearProperty("aws.region");
  }
}
 

public static NetcdfFile open(String filename) throws IOException {
  logger.debug("**** Open {}", filename);
  NetcdfFile ncfile;
  if (cdmUseBuilders || isLocationObjectStore(filename)) {
    ncfile = NetcdfFiles.open(filename, null);
  } else {
    ncfile = NetcdfFile.open(filename, null);
  }
  logger.debug("open {}", ncfile);

  return ncfile;
}
 

void toStringStart(Indent indent, boolean strict, @Nullable String nameOverride) {
  String name = (nameOverride != null) ? nameOverride : ncfile.getLocation();
  if (strict) {
    if (name.endsWith(".nc"))
      name = name.substring(0, name.length() - 3);
    if (name.endsWith(".cdl"))
      name = name.substring(0, name.length() - 4);
    name = NetcdfFiles.makeValidCDLName(name);
  }
  out.format("%snetcdf %s {%n", indent, name);
  indent.incr();
  writeCDL(ncfile.getRootGroup(), indent);
  indent.decr();
}
 

private void writeCDL(Dimension dim, Indent indent) {
  String name = strict ? NetcdfFiles.makeValidCDLName(dim.getShortName()) : dim.getShortName();
  out.format("%s%s", indent, name);
  if (dim.isUnlimited())
    out.format(" = UNLIMITED;   // (%d currently)", dim.getLength());
  else if (dim.isVariableLength())
    out.format(" = UNKNOWN;");
  else
    out.format(" = %d;", dim.getLength());
}
 

private boolean compareWithBuilder(String filename) throws IOException {
  logger.info("TestBuilders on {}%n", filename);
  try (NetcdfFile org = NetcdfFile.open(filename)) {
    try (NetcdfFile withBuilder = NetcdfFiles.open(filename)) {
      Formatter f = new Formatter();
      CompareNetcdf2 compare = new CompareNetcdf2(f, false, false, true);
      if (!compare.compare(org, withBuilder, null)) {
        System.out.printf("Compare %s%n%s%n", filename, f);
        fail();
        return false;
      }
    }
  }
  return true;
}
 

private void showNew(String filename) throws IOException {
  try (NetcdfFile withBuilder = NetcdfFiles.open(filename)) {
    // Variable v = withBuilder.findVariable("catchments_x");
    // Array data = v.read();
    System.out.printf("withBuilder = %s%n", withBuilder);
  }
}