small update testbench, tpxo9.zarr(->36th), dev files

cywhale · Jul 11, 2024 · 9fc3ee1 · 9fc3ee1
1 parent b6ecee7
commit 9fc3ee1
Show file tree

Hide file tree

Showing 6 changed files with 68 additions and 67 deletions.
diff --git a/change_log.md b/change_log.md
@@ -44,3 +44,4 @@
 
     -- Add validation with NOAA tide observation data api.tidesandcurrents.noaa.gov/
     -- Modify testbench (for CWA, NOAA) bugs e.g. local time (NOAA) to UTC
+    -- small update testbench, tpxo9.zarr(->36th), dev files
diff --git a/dev/extract_constants.ipynb b/dev/extract_constants.ipynb
diff --git a/dev/zarr_fillna.ipynb b/dev/zarr_fillna.ipynb
@@ -63,7 +63,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -72,7 +72,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -81,7 +81,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -100,7 +100,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -1683,7 +1683,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
@@ -1737,14 +1737,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Total points to process: 83369\n"
+      "Total points to process: 75339\n"
      ]
     }
    ],
@@ -1755,7 +1755,7 @@
     "#All_na_condition set False 508981 pts -> 428631 pts(09, 5x5) -> 380901(10, 2x5) -> 340137(11, 2x3) -> 308379(12, 3x2) -> 285428 -> 273804(13, 2*4)\n",
     "# -> 231480 (14) -> 212160 (15, 2x2) -> 186720(16, 1x3) -> 169656(17, 4x1) -> 152016(18, 3x1)\n",
     "#change to u_ph, v_ph 201098(18) -> 193613(19, 3x1) -> 186827(20, 1x3) 130268(30) -> 121328(31, 2x1 7000) -> 112562 (32 2x1 7000) -> 106084 (33 1x1 7000)\n",
-    "# -> 98889 (33 1x1 8000) -> 91433 (34 1x1 9000) -> 83369 (35 1x1 9000)\n",
+    "# -> 98889 (33 1x1 8000) -> 91433 (34 1x1 9000) -> 83369 (35 1x1 9000) -> 75539 (36 1x1 9000)\n",
     "#129433(u,v_amp, 20) -> 124429(21, 1x3) -> 111709(22, 3x1)\n",
     "#149838(u_amp, v_ph) -> 149703(23 3x1), 142564(u_ph, v_amp)-> 134890(24 2x1, 5000) -> 124926(25 2x1, 6000)\n",
     "#116082(26 2x1,6000), 93784(u,v_amp) -> 81787(27 1x2 6000) -> 72959(28 1x2 final 4414) -> 67663 (29 2x1 6000) -> 65523 (30 2x1 5000)\n",
@@ -2091,7 +2091,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -2164,14 +2164,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Total points to process: 83369\n"
+      "Total points to process: 75339\n"
      ]
     }
    ],
@@ -2220,14 +2220,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "83369\n"
+      "75339\n"
      ]
     }
    ],

diff --git a/dev/zarr_fillna_parallel.py b/dev/zarr_fillna_parallel.py
@@ -34,8 +34,8 @@
 
 ## Global variables
 maxWorkers = 10
-MAX_CLUSTERS = 9000 #-1 #None if need test
-START_CLUSTER = 3000
+MAX_CLUSTERS = 9000 #for 1x1, 9000 is max, greater cause crash #-1 #None if need test
+START_CLUSTER = 11000
 
 def log_elapsed_time(start_time, work=""):
     et = time.time()
@@ -230,9 +230,9 @@ def main():
     st = time.time()
     start_time = datetime.fromtimestamp(st)
     print("Fill_NA Main process start: ", start_time)
-    input_file = "../data/tpxo9_bak35.zarr"
+    input_file = "../data/tpxo9_bak37.zarr" #not yet
     output_file = "../data/tpxo9_new.zarr"
-    resave_file = "../data/tpxo9_fillna36.zarr"
+    resave_file = "../data/tpxo9_fillna38.zarr"
     All_NA_CONDITION = False #all NA or any NA in constituents should be recomputed
     #SAVE_SMALL_DATA = False #always False
     RESAVE = False

diff --git a/dev/zarr_fillna_savefile.py b/dev/zarr_fillna_savefile.py
@@ -70,7 +70,7 @@ def resave_fillna_dataset(method1, method2):
 
 
 if __name__ == "__main__":
-    method1_file = "../data/tpxo9_bak35.zarr" #"tpxo9_method1.zarr" #All_NA_CONDITION set False 
+    method1_file = "../data/tpxo9_bak36.zarr" #"tpxo9_method1.zarr" #All_NA_CONDITION set False 
     method2_file = "../data/tpxo9_new.zarr" #"tpxo9_method2.zarr"
     zarr.convenience.consolidate_metadata(method2_file)
     resave_fillna_dataset(method1_file, method2_file)
diff --git a/test/simu_cwa_tide01.py → test/compare_observ_tide01.py b/test/simu_cwa_tide01.py → test/compare_observ_tide01.py
@@ -15,12 +15,12 @@
 load_dotenv()
 
 # Global test counter
-test_sid = 12
+test_sid = 17
 target = 'NOAA' # 'CWA', 'NOAA'
 target_token = os.getenv(f"{target}_TOKEN")
 fetch_mode = 'reuse_all' #'reuse_all' #'exclude-station-list' #reuse_data_odb # 'reuse_stations_force-update-data' 
 Number_of_Testing = 30
-ext_start_date = '2024-07-08T00:00:00' #2024-07-07T00:00:00' #'2024-07-04T00:00:00'  # Specify an external start date if needed
+ext_start_date = '' #2024-07-07T00:00:00' #'2024-07-04T00:00:00'  # Specify an external start date if needed
 metadata = None
 test_dir = 'test/'