From 9319091b7b03bbe94970ff9f6c7870a7d60c085f Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Wed, 24 Jul 2024 14:19:36 -0500 Subject: [PATCH 1/3] Fix preset sharding options and add tests --- .../com/glencoesoftware/zarr/Convert.java | 57 +++++++++++++----- .../zarr/test/ConversionTest.java | 59 +++++++++++++++++++ 2 files changed, 102 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/glencoesoftware/zarr/Convert.java b/src/main/java/com/glencoesoftware/zarr/Convert.java index 966ff40..ac7d8c6 100644 --- a/src/main/java/com/glencoesoftware/zarr/Convert.java +++ b/src/main/java/com/glencoesoftware/zarr/Convert.java @@ -242,9 +242,12 @@ public void convertToV3() throws Exception { ZarrArray tile = field.openArray("/" + res); LOGGER.info("opened array {}", resolutionPath); - int[] chunkSizes = tile.getChunks(); + int[] originalChunkSizes = tile.getChunks(); int[] shape = tile.getShape(); + int[] chunkSizes = new int[originalChunkSizes.length]; + System.arraycopy(originalChunkSizes, 0, chunkSizes, 0, chunkSizes.length); + int[] gridPosition = new int[] {0, 0, 0, 0, 0}; int tileX = chunkSizes[chunkSizes.length - 2]; int tileY = chunkSizes[chunkSizes.length - 1]; @@ -257,22 +260,31 @@ public void convertToV3() throws Exception { if (shardConfig != null) { switch (shardConfig) { case SINGLE: - codecBuilder = codecBuilder.withSharding(shape); + // single shard covering the whole image + // internal chunk sizes remain the same as in input data + chunkSizes = shape; break; case CHUNK: - codecBuilder = codecBuilder.withSharding(chunkSizes); + // exactly one shard per chunk + // no changes needed break; case SUPERCHUNK: - int[] shardSize = new int[chunkSizes.length]; - System.arraycopy(chunkSizes, 0, shardSize, 0, shardSize.length); - shardSize[4] *= 2; - shardSize[3] *= 2; - codecBuilder = codecBuilder.withSharding(shardSize); + // each shard covers 2x2 chunks + chunkSizes[4] *= 2; + chunkSizes[3] *= 2; break; case CUSTOM: // TODO break; } + + if (chunkAndShardCompatible(originalChunkSizes, chunkSizes, shape)) { + codecBuilder = codecBuilder.withSharding(originalChunkSizes); + } + else { + LOGGER.warn("Skipping sharding due to incompatible sizes"); + chunkSizes = originalChunkSizes; + } } if (codecs != null) { for (String codecName : codecs) { @@ -296,15 +308,15 @@ else if (codecName.equals("blosc")) { Array.metadataBuilder() .withShape(Utils.toLongArray(shape)) .withDataType(getV3Type(type)) - .withChunkShape(chunkSizes) + .withChunkShape(chunkSizes) // if sharding is used, this will be the shard size .withFillValue(255) .withCodecs(c -> builder) .build() ); - for (int t=0; t Date: Wed, 24 Jul 2024 15:36:34 -0500 Subject: [PATCH 2/3] Use larger heap size for tests This wasn't necessary when building locally, but hopefully fixes CI builds. --- build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.gradle b/build.gradle index eb9e462..f8e2bda 100644 --- a/build.gradle +++ b/build.gradle @@ -59,6 +59,8 @@ dependencies { test { useJUnit() + + maxHeapSize = "2g" } jar { From 1c817c196891322982d9329fcb10c595d343ec73 Mon Sep 17 00:00:00 2001 From: Melissa Linkert Date: Thu, 25 Jul 2024 17:55:27 -0500 Subject: [PATCH 3/3] Add `--debug`/`--log-level` option and extra DEBUG-level logging --- .../com/glencoesoftware/zarr/Convert.java | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/glencoesoftware/zarr/Convert.java b/src/main/java/com/glencoesoftware/zarr/Convert.java index ac7d8c6..66ca616 100644 --- a/src/main/java/com/glencoesoftware/zarr/Convert.java +++ b/src/main/java/com/glencoesoftware/zarr/Convert.java @@ -37,6 +37,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.Callable; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Set; @@ -44,6 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import ch.qos.logback.classic.Level; import picocli.CommandLine; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @@ -61,6 +63,7 @@ public class Convert implements Callable { private String inputLocation; private String outputLocation; + private String logLevel = "INFO"; private boolean writeV2; private ShardConfiguration shardConfig; @@ -90,6 +93,26 @@ public void setOutput(String output) { outputLocation = output; } + /** + * Set the slf4j logging level. Defaults to "INFO". + * + * @param level logging level + */ + @Option( + names = {"--log-level", "--debug"}, + arity = "0..1", + description = "Change logging level; valid values are " + + "OFF, ERROR, WARN, INFO, DEBUG, TRACE and ALL. " + + "(default: ${DEFAULT-VALUE})", + defaultValue = "INFO", + fallbackValue = "DEBUG" + ) + public void setLogLevel(String level) { + if (level != null) { + logLevel = level; + } + } + @Option( names = "--write-v2", description = "Read v3, write v2", @@ -134,6 +157,10 @@ public void setCompression(String[] compression) { @Override public Integer call() throws Exception { + ch.qos.logback.classic.Logger root = (ch.qos.logback.classic.Logger) + LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME); + root.setLevel(Level.toLevel(logLevel)); + if (writeV2) { convertToV2(); } @@ -151,6 +178,7 @@ public void convertToV3() throws Exception { Path inputPath = Paths.get(inputLocation); // get the root-level attributes + LOGGER.debug("opening v2 root group: {}", inputPath); ZarrGroup reader = ZarrGroup.open(inputPath); Map attributes = reader.getAttributes(); @@ -163,6 +191,7 @@ public void convertToV3() throws Exception { // but this doesn't seem to actually create the group // separating the group creation and attribute writing into // two calls seems to work correctly + LOGGER.debug("opening v3 root group: {}", outputLocation); FilesystemStore outputStore = new FilesystemStore(outputLocation); Group outputRootGroup = Group.create(outputStore.resolve()); outputRootGroup.setAttributes(attributes); @@ -175,9 +204,11 @@ public void convertToV3() throws Exception { for (String seriesGroupKey : groupKeys) { if (seriesGroupKey.indexOf("/") > 0) { + LOGGER.debug("skipping v2 group key: {}", seriesGroupKey); continue; } Path seriesPath = inputPath.resolve(seriesGroupKey); + LOGGER.debug("opening v2 group: {}", seriesPath); ZarrGroup seriesGroup = ZarrGroup.open(seriesPath); LOGGER.info("opened {}", seriesPath); @@ -190,13 +221,16 @@ public void convertToV3() throws Exception { Set columnKeys = seriesGroup.getGroupKeys(); // "pass through" if this is not HCS if (columnKeys.size() == 0) { + LOGGER.debug("no column group keys (likely not HCS)"); columnKeys.add(""); } for (String columnKey : columnKeys) { if (columnKey.indexOf("/") > 0) { + LOGGER.debug("skipping v2 column group key: {}", columnKey); continue; } Path columnPath = columnKey.isEmpty() ? seriesPath : seriesPath.resolve(columnKey); + LOGGER.debug("opening v2 group: {}", columnPath); ZarrGroup column = ZarrGroup.open(columnPath); if (!columnKey.isEmpty()) { @@ -208,14 +242,15 @@ public void convertToV3() throws Exception { Set fieldKeys = column.getGroupKeys(); // "pass through" if this is not HCS if (fieldKeys.size() == 0) { + LOGGER.debug("no field group keys"); fieldKeys.add(""); } for (String fieldKey : fieldKeys) { Path fieldPath = fieldKey.isEmpty() ? columnPath : columnPath.resolve(fieldKey); + LOGGER.debug("opening v2 field group: {}", fieldPath); ZarrGroup field = ZarrGroup.open(fieldPath); - Map fieldAttributes = field.getAttributes(); if (!fieldKey.isEmpty()) { Group outputFieldGroup = Group.create(outputStore.resolve(seriesGroupKey, columnKey, fieldKey)); @@ -239,6 +274,7 @@ public void convertToV3() throws Exception { for (int res=0; res