From ee1110389ed37b0f3213e2d217035938b001b1db Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Sun, 9 Jun 2024 12:37:04 +0300 Subject: [PATCH 01/30] add some files support, fixing rebase issue --- src/imagery/i.eodag/i.eodag.py | 270 +++++++++++++++++++++++---------- 1 file changed, 187 insertions(+), 83 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index e9929242eb..638bbca243 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -17,7 +17,6 @@ # ############################################################################# - # %Module # % description: Eodag interface to install imagery datasets from various providers. # % keyword: imagery @@ -28,7 +27,6 @@ # % keyword: datasets # % keyword: download # %end - # %option # % key: dataset # % type: string @@ -37,20 +35,37 @@ # % answer: S1_SAR_GRD # % guisection: Filter # %end - +# %option G_OPT_V_MAP +# % description: If not given then current computational extent is used +# % label: Name of input vector map to define Area of Interest (AOI) +# % required: no +# % guisection: Region +# %end +# %option +# % key: clouds +# % type: integer +# % description: Maximum cloud cover percentage for Sentinel scene +# % required: no +# % guisection: Filter +# %end +# %option G_OPT_V_OUTPUT +# % key: footprints +# % description: Name for output vector map with footprints +# % label: Only supported for download from ESA_Copernicus Open Access Hub +# % required: no +# % guisection: Output +# %end # %option G_OPT_M_DIR # % key: output # % description: Name for output directory where to store downloaded data OR search results # % required: no # % guisection: Output # %end - # %option G_OPT_F_INPUT # % key: config # % label: Full path to yaml config file # % required: no # %end - # %option # % key: id # % type: string @@ -58,7 +73,13 @@ # % description: List of scenes IDs to download # % guisection: Filter # %end - +# %option +# % key: file +# % type: string +# % multiple: yes +# % description: List of text files with IDs to download +# % guisection: Filter +# %end # %option # % key: provider # % type: string @@ -66,37 +87,31 @@ # % required: yes # % guisection: Filter # %end - # %option # % key: start # % type: string # % description: Start date (in any ISO 8601 format), by default it is 60 days ago # % guisection: Filter # %end - # %option # % key: end # % type: string # % description: End date (in any ISO 8601 format) # % guisection: Filter # %end - # %flag # % key: l # % description: List the search result without downloading # %end - # %flag # % key: e # % description: Extract the downloaded the datasets # %end - # %flag # % key: d # % description: Delete the product archieve after downloading # %end - import sys import os import getpass @@ -115,22 +130,19 @@ def create_dir(directory): def get_bb(vector=None): - args = {} - if vector: - args["vector"] = vector # are we in LatLong location? kv = gs.parse_command("g.proj", flags="j") if "+proj" not in kv: gs.fatal(_("Unable to get bounding box: unprojected location not supported")) if kv["+proj"] != "longlat": - info = gs.parse_command("g.region", flags="uplg", **args) + info = gs.parse_command("g.region", flags="uplg") return { "lonmin": info["nw_long"], "latmin": info["sw_lat"], "lonmax": info["ne_long"], "latmax": info["nw_lat"], } - info = gs.parse_command("g.region", flags="upg", **args) + info = gs.parse_command("g.region", flags="upg") return { "lonmin": info["w"], "latmin": info["s"], @@ -139,54 +151,91 @@ def get_bb(vector=None): } +def get_aoi(vector=None): + """Get the AOI for querying""" + # Handle empty AOI + if not vector: + return get_bb() + + # TODO: Read Vector WKB + + def download_by_id(query_id: str): - gs.verbose( - _( - "Searching for product ending with: {}".format( - query_id[-min(len(query_id), 8) :] - ) - ) - ) - product, count = dag.search(id=query_id) + gs.message(_("Attempting to download product: {}".format(query_id))) + product, count = dag.search(id=query_id, provider=options["provider"]) if count != 1: - raise ParameterError("Product couldn't be uniquely identified") + raise ParameterError("Product couldn't be uniquely identified.") if not product[0].properties["id"].startswith(query_id): - raise ParameterError("Product wasn't found") - gs.verbose( - _("Poduct ending with: {} is found.".format(query_id[-min(len(query_id), 8) :])) - ) - gs.verbose(_("Downloading...")) + raise ParameterError("Product wasn't found.") + gs.verbose(_("Poduct {} is found.".format(query_id))) + gs.verbose(_("Downloading {}".format(query_id))) dag.download(product[0]) +def ids_from_file_txt(ids_file_txt): + ids_set = set() + with open(ids_file_txt, "r") as ids_stream: + line_index = 0 + lines = ids_stream.read().split("\n") + for line in lines: + line_index += 1 + if not line: + continue + line = line.strip() + if line.find(" ") != -1: + gs.warning( + _( + 'File "{}", line {}, has space(s). Skipping line... '.format( + ids_file_txt, line_index + ) + ) + ) + continue + ids_set.add(line.strip()) + return ids_set + + def download_by_ids(products_ids): # Search in all recognized providers for product_id in products_ids: try: download_by_id(product_id) - except ParameterError: - gs.error( - _( - "Product ending with: {}, failed to download".format( - product_id[-min(len(id), 8) :] - ) - ) - ) + except ParameterError as e: + gs.error(e) + gs.error(_("Product {} failed to download".format(product_id))) + + +def parse_id_option(id_option_string): + to_download_ids = id_option_string.split(",") + ids_set = set() + for to_download_id in to_download_ids: + ids_set.add(to_download_id.strip()) + return ids_set + + +def parse_file_option(file_option_string): + ids_set = set() + files_list = file_option_string.split(",") + for file in files_list: + try: + ids_set.update(ids_from_file_txt(file)) + except FileNotFoundError: + gs.warning(_('Couldn\'t read file "{}", Skipping file...'.format(file))) + return ids_set def setup_environment_variables(): + # Setting the envirionmnets variables has to come before the eodag initialization os.environ["EODAG__{}__DOWNLOAD__EXTRACT".format(options["provider"])] = str( flags["e"] ) os.environ["EODAG__{}__DOWNLOAD__DELETE_ARCHIV".format(options["provider"])] = str( flags["d"] ) - if options["output"]: os.environ[ "EODAG__{}__DOWNLOAD__OUTPUTS_PREFIX".format(options["provider"]) ] = options["output"] - if options["config"]: os.environ["EODAG_CFG_FILE"] = options["config"] @@ -206,7 +255,7 @@ def no_fallback_search(search_parameters, provider): return SearchResult([]) except Exception as e: gs.verbose(e) - gs.fatal(_("Server error, try again.")) + gs.fatal(_("Server error, please try again.")) # https://eodag.readthedocs.io/en/stable/api_reference/core.html#eodag.api.core.EODataAccessGateway.search_iter_page # This will use the prefered provider by default @@ -215,54 +264,98 @@ def no_fallback_search(search_parameters, provider): # TODO: Would it be useful if user could iterate through # the pages manually, and look for the product themselves? try: - return list(search_result)[0] + # Merging the pages into one list with all products + return [j for i in search_result for j in i] except Exception as e: gs.verbose(e) - gs.fatal(_("Server error, try again.")) + gs.fatal(_("Server error, please try again.")) -def main(): - # products: https://github.com/CS-SI/eodag/blob/develop/eodag/resources/product_types.yml +def create_products_dataframe(eo_products): + result_dict = {"id": [], "time": [], "cloud_coverage": [], "product_type": []} + for product in eo_products: + if "id" in product.properties and product.properties["id"] is not None: + result_dict["id"].append(product.properties["id"]) + else: + result_dict["id"].append(None) + if ( + "startTimeFromAscendingNode" in product.properties + and product.properties["startTimeFromAscendingNode"] is not None + ): + try: + result_dict["time"].append( + normalize_time(product.properties["startTimeFromAscendingNode"]) + ) + except: + result_dict["time"].append( + product.properties["startTimeFromAscendingNode"] + ) + else: + result_dict["time"].append(None) + if ( + "cloudCover" in product.properties + and product.properties["cloudCover"] is not None + ): + result_dict["cloud_coverage"].append(product.properties["cloudCover"]) + else: + result_dict["cloud_coverage"].append(None) + if ( + "productType" in product.properties + and product.properties["productType"] is not None + ): + result_dict["product_type"].append(product.properties["productType"]) + else: + result_dict["product_type"].append(None) - # setting the envirionmnets variables has to come before the dag initialization - setup_environment_variables() + df = pd.DataFrame().from_dict(result_dict) + return df + + +def main(): + # Products: https://github.com/CS-SI/eodag/blob/develop/eodag/resources/product_types.yml global dag + setup_environment_variables() dag = EODataAccessGateway() - if options["provider"]: dag.set_preferred_provider(options["provider"]) else: - gs.fatal(_("Please specify a provider...")) - - # Download by ids... if ids are provided only these ids will be downloaded - if options["id"]: - ids = options["id"].split(",") - download_by_ids(ids) + # TODO: Add a way to search witout specifying a provider + gs.fatal(_("Please specify a provider.")) + + # Download by ids + # Searching for additional products won't take place + if options["id"] or options["file"]: + # Duplicates will be + ids_set = set() + if options["id"]: + ids_set.update(parse_id_option(options["id"])) + if options["file"]: + ids_set.update(parse_file_option(options["file"])) + gs.message(_("Found {} distinct product(s) IDs.".format(len(ids_set)))) + for product_id in ids_set: + gs.message(product_id) + gs.verbose(_("Attempting to download.")) + download_by_ids(ids_set) else: - - items_per_page = 20 + items_per_page = 40 # TODO: Check that the product exists, # could be handled by catching exceptions when searching... product_type = options["dataset"] - # TODO: Allow user to specify a shape file path - geom = ( - # use boudning box of current computational region - get_bb() - # { "lonmin": 1.9, "latmin": 43.9, "lonmax": 2, "latmax": 45, } # hardcoded for testing - ) + # HARDCODED VALUES FOR TESTING { "lonmin": 1.9, "latmin": 43.9, "lonmax": 2, "latmax": 45, } # hardcoded for testing + geom = get_aoi(options["map"]) gs.verbose(_("Region used for searching: {}".format(geom))) - search_parameters = { "items_per_page": items_per_page, "productType": product_type, - # TODO: Convert to a shapely object "geom": geom, } - # Assumes that the user enter time in UTC + if options["clouds"]: + search_parameters["cloudCover"] = options["clouds"] + end_date = options["end"] if not options["end"]: end_date = datetime.utcnow().isoformat() @@ -296,24 +389,30 @@ def main(): search_parameters["end"] = end_date search_results = no_fallback_search(search_parameters, options["provider"]) - num_results = len(search_results) - gs.verbose( - _("Found {} matching scenes of type {}".format(num_results, product_type)) - ) + print(num_results) if flags["l"]: - # TODO: Oragnize output format better - idx = 0 - for product in search_results: - print( - _( - "Product #{} - ID:{},provider:{}".format( - idx, product.properties["id"], product.provider - ) - ) - ) - idx += 1 + df = create_products_dataframe(search_results) + gs.message(_("{} product(s) found.").format(num_results)) + for idx in range(len(df)): + product_id = df["id"].iloc[idx] + if product_id is None: + time_string = "id_NA" + time_string = df["time"].iloc[idx] + if time_string is None: + time_string = "time_NA" + else: + time_string += "Z" + cloud_cover_string = df["cloud_coverage"].iloc[idx] + if cloud_cover_string is not None: + cloud_cover_string = f"{cloud_cover_string:2.0f}%" + else: + cloud_cover_string = "cloudcover_NA" + product_type = df["product_type"].iloc[idx] + if product_type is None: + product_type = "producttype_NA" + print(f"{product_id} {time_string} {cloud_cover_string} {product_type}") else: # TODO: Consider adding a quicklook flag # TODO: Add timeout and wait parameters for downloading offline products... @@ -334,7 +433,14 @@ def main(): from eodag import EODataAccessGateway from eodag import setup_logging from eodag.api.search_result import SearchResult + except: + gs.fatal(_("Cannot import eodag. Please intall the library first.")) + try: + import pandas as pd + except: + gs.fatal(_("Cannot import pandas. Please intall the library first.")) + if "DEBUG" in gs.read_command("g.gisenv"): debug_level = int(gs.read_command("g.gisenv", get="DEBUG")) if not debug_level: setup_logging(1) @@ -342,7 +448,5 @@ def main(): setup_logging(2) else: setup_logging(3) - except: - gs.fatal(_("Cannot import eodag. Please intall the library first.")) sys.exit(main()) From 7c26ffb73d1e228c63be4a774f8f7e3987bf9ce2 Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Sun, 9 Jun 2024 17:23:01 +0300 Subject: [PATCH 02/30] Update src/imagery/i.eodag/i.eodag.py Co-authored-by: Stefan Blumentrath --- src/imagery/i.eodag/i.eodag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 638bbca243..2523de8e71 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -129,7 +129,7 @@ def create_dir(directory): gs.fatal(_("Could not create directory {}").format(dir)) -def get_bb(vector=None): +def get_bb(): # are we in LatLong location? kv = gs.parse_command("g.proj", flags="j") if "+proj" not in kv: From 3022b85c6a0776ff3a7d25fb95a130b5bb1557aa Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Mon, 10 Jun 2024 20:57:41 +0300 Subject: [PATCH 03/30] Update src/imagery/i.eodag/i.eodag.py Co-authored-by: Stefan Blumentrath --- src/imagery/i.eodag/i.eodag.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 2523de8e71..7961117404 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -76,8 +76,8 @@ # %option # % key: file # % type: string -# % multiple: yes -# % description: List of text files with IDs to download +# % multiple: no +# % description: Text file with IDs to download, one per row # % guisection: Filter # %end # %option From 8c7da0bcd01cde1e65d6063dc1d8f41a870bf04b Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Mon, 10 Jun 2024 20:59:28 +0300 Subject: [PATCH 04/30] Update src/imagery/i.eodag/i.eodag.py Co-authored-by: Stefan Blumentrath --- src/imagery/i.eodag/i.eodag.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 7961117404..b180775e6f 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -177,8 +177,7 @@ def ids_from_file_txt(ids_file_txt): with open(ids_file_txt, "r") as ids_stream: line_index = 0 lines = ids_stream.read().split("\n") - for line in lines: - line_index += 1 + for line_index, enumerate(line in lines): if not line: continue line = line.strip() From 907dd81c96f7e54a37a604450f9beb1582ed2e31 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Mon, 10 Jun 2024 21:00:04 +0300 Subject: [PATCH 05/30] Space out the options section --- src/imagery/i.eodag/i.eodag.py | 42 +++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 7961117404..78ea75b662 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -27,6 +27,24 @@ # % keyword: datasets # % keyword: download # %end + +# FLAGS +# %flag +# % key: l +# % description: List the search result without downloading +# %end + +# %flag +# % key: e +# % description: Extract the downloaded the datasets +# %end + +# %flag +# % key: d +# % description: Delete the product archieve after downloading +# %end + +# OPTIONS # %option # % key: dataset # % type: string @@ -35,12 +53,14 @@ # % answer: S1_SAR_GRD # % guisection: Filter # %end + # %option G_OPT_V_MAP # % description: If not given then current computational extent is used # % label: Name of input vector map to define Area of Interest (AOI) # % required: no # % guisection: Region # %end + # %option # % key: clouds # % type: integer @@ -48,6 +68,7 @@ # % required: no # % guisection: Filter # %end + # %option G_OPT_V_OUTPUT # % key: footprints # % description: Name for output vector map with footprints @@ -55,17 +76,20 @@ # % required: no # % guisection: Output # %end + # %option G_OPT_M_DIR # % key: output # % description: Name for output directory where to store downloaded data OR search results # % required: no # % guisection: Output # %end + # %option G_OPT_F_INPUT # % key: config # % label: Full path to yaml config file # % required: no # %end + # %option # % key: id # % type: string @@ -73,6 +97,7 @@ # % description: List of scenes IDs to download # % guisection: Filter # %end + # %option # % key: file # % type: string @@ -80,6 +105,7 @@ # % description: Text file with IDs to download, one per row # % guisection: Filter # %end + # %option # % key: provider # % type: string @@ -87,30 +113,21 @@ # % required: yes # % guisection: Filter # %end + # %option # % key: start # % type: string # % description: Start date (in any ISO 8601 format), by default it is 60 days ago # % guisection: Filter # %end + # %option # % key: end # % type: string # % description: End date (in any ISO 8601 format) # % guisection: Filter # %end -# %flag -# % key: l -# % description: List the search result without downloading -# %end -# %flag -# % key: e -# % description: Extract the downloaded the datasets -# %end -# %flag -# % key: d -# % description: Delete the product archieve after downloading -# %end + import sys import os @@ -390,7 +407,6 @@ def main(): search_results = no_fallback_search(search_parameters, options["provider"]) num_results = len(search_results) - print(num_results) if flags["l"]: df = create_products_dataframe(search_results) From d2f9d540cdb47fe056040046f5d473a67fd91823 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Mon, 10 Jun 2024 21:18:03 +0300 Subject: [PATCH 06/30] Fix enumerate, remove footprint option --- src/imagery/i.eodag/i.eodag.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 6e76fb8aea..a3e612b5a8 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -69,14 +69,6 @@ # % guisection: Filter # %end -# %option G_OPT_V_OUTPUT -# % key: footprints -# % description: Name for output vector map with footprints -# % label: Only supported for download from ESA_Copernicus Open Access Hub -# % required: no -# % guisection: Output -# %end - # %option G_OPT_M_DIR # % key: output # % description: Name for output directory where to store downloaded data OR search results @@ -184,17 +176,15 @@ def download_by_id(query_id: str): raise ParameterError("Product couldn't be uniquely identified.") if not product[0].properties["id"].startswith(query_id): raise ParameterError("Product wasn't found.") - gs.verbose(_("Poduct {} is found.".format(query_id))) - gs.verbose(_("Downloading {}".format(query_id))) + gs.verbose(_("Poduct {} is found. Downloading...".format(query_id))) dag.download(product[0]) def ids_from_file_txt(ids_file_txt): ids_set = set() with open(ids_file_txt, "r") as ids_stream: - line_index = 0 lines = ids_stream.read().split("\n") - for line_index, enumerate(line in lines): + for line_index, line in enumerate(lines): if not line: continue line = line.strip() @@ -202,7 +192,7 @@ def ids_from_file_txt(ids_file_txt): gs.warning( _( 'File "{}", line {}, has space(s). Skipping line... '.format( - ids_file_txt, line_index + ids_file_txt, line_index + 1 ) ) ) From 4200205657277db6081c28e4563216f7c92d0243 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Mon, 10 Jun 2024 21:46:13 +0300 Subject: [PATCH 07/30] Use Path read_text function instead of ids_from_file_txt --- src/imagery/i.eodag/i.eodag.py | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index a3e612b5a8..c6106bd2e0 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -180,27 +180,6 @@ def download_by_id(query_id: str): dag.download(product[0]) -def ids_from_file_txt(ids_file_txt): - ids_set = set() - with open(ids_file_txt, "r") as ids_stream: - lines = ids_stream.read().split("\n") - for line_index, line in enumerate(lines): - if not line: - continue - line = line.strip() - if line.find(" ") != -1: - gs.warning( - _( - 'File "{}", line {}, has space(s). Skipping line... '.format( - ids_file_txt, line_index + 1 - ) - ) - ) - continue - ids_set.add(line.strip()) - return ids_set - - def download_by_ids(products_ids): # Search in all recognized providers for product_id in products_ids: @@ -337,10 +316,13 @@ def main(): if options["id"]: ids_set.update(parse_id_option(options["id"])) if options["file"]: - ids_set.update(parse_file_option(options["file"])) + ids_set = set( + Path(options["file"]).read_text(encoding="UTF8").strip().split("\n") + ) + # Remove empty lines + ids_set.discard(str()) gs.message(_("Found {} distinct product(s) IDs.".format(len(ids_set)))) - for product_id in ids_set: - gs.message(product_id) + gs.message("\n".join(ids_set)) gs.verbose(_("Attempting to download.")) download_by_ids(ids_set) else: From e9e42cc9e5b1232873bf79a7ae092f57929d6843 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Mon, 10 Jun 2024 21:48:52 +0300 Subject: [PATCH 08/30] Remove parse_file_option function --- src/imagery/i.eodag/i.eodag.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index c6106bd2e0..6c1c07ec19 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -198,17 +198,6 @@ def parse_id_option(id_option_string): return ids_set -def parse_file_option(file_option_string): - ids_set = set() - files_list = file_option_string.split(",") - for file in files_list: - try: - ids_set.update(ids_from_file_txt(file)) - except FileNotFoundError: - gs.warning(_('Couldn\'t read file "{}", Skipping file...'.format(file))) - return ids_set - - def setup_environment_variables(): # Setting the envirionmnets variables has to come before the eodag initialization os.environ["EODAG__{}__DOWNLOAD__EXTRACT".format(options["provider"])] = str( From a0943b90a76e6dfe073cd5e9ac5c77ca627b6445 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Mon, 10 Jun 2024 22:15:48 +0300 Subject: [PATCH 09/30] Merge file and id options --- src/imagery/i.eodag/i.eodag.py | 35 +++++++++------------------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 6c1c07ec19..de547e0b54 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -86,15 +86,7 @@ # % key: id # % type: string # % multiple: yes -# % description: List of scenes IDs to download -# % guisection: Filter -# %end - -# %option -# % key: file -# % type: string -# % multiple: no -# % description: Text file with IDs to download, one per row +# % description: List of scenes IDs to download | A single text file with one ID on each line # % guisection: Filter # %end @@ -190,14 +182,6 @@ def download_by_ids(products_ids): gs.error(_("Product {} failed to download".format(product_id))) -def parse_id_option(id_option_string): - to_download_ids = id_option_string.split(",") - ids_set = set() - for to_download_id in to_download_ids: - ids_set.add(to_download_id.strip()) - return ids_set - - def setup_environment_variables(): # Setting the envirionmnets variables has to come before the eodag initialization os.environ["EODAG__{}__DOWNLOAD__EXTRACT".format(options["provider"])] = str( @@ -300,19 +284,18 @@ def main(): # Download by ids # Searching for additional products won't take place if options["id"] or options["file"]: - # Duplicates will be ids_set = set() - if options["id"]: - ids_set.update(parse_id_option(options["id"])) - if options["file"]: + if Path(options["id"]).is_file(): + gs.message(_('Reading file "{}"'.format(options["id"]))) ids_set = set( - Path(options["file"]).read_text(encoding="UTF8").strip().split("\n") + Path(options["id"]).read_text(encoding="UTF8").strip().split("\n") ) - # Remove empty lines - ids_set.discard(str()) - gs.message(_("Found {} distinct product(s) IDs.".format(len(ids_set)))) + else: + ids_set = set(pid.strip() for pid in options["id"].split(",")) + # Remove empty strings + ids_set.discard(str()) + gs.message(_("Found {} distinct ID(s).".format(len(ids_set)))) gs.message("\n".join(ids_set)) - gs.verbose(_("Attempting to download.")) download_by_ids(ids_set) else: items_per_page = 40 From 1553eadc87824478aac07629bebd7a457bb29d91 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Tue, 11 Jun 2024 01:00:19 +0300 Subject: [PATCH 10/30] Add AOI from vector map --- src/imagery/i.eodag/i.eodag.py | 59 +++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index de547e0b54..d0d96e3088 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -64,7 +64,7 @@ # %option # % key: clouds # % type: integer -# % description: Maximum cloud cover percentage for Sentinel scene +# % description: Maximum cloud cover percentage for Sentinel scene [0, 100] # % required: no # % guisection: Filter # %end @@ -117,9 +117,11 @@ import os import getpass from pathlib import Path +from subprocess import PIPE from datetime import datetime, timedelta import grass.script as gs +from grass.pygrass.modules import Module from grass.exceptions import ParameterError @@ -158,7 +160,56 @@ def get_aoi(vector=None): if not vector: return get_bb() - # TODO: Read Vector WKB + args = {} + args["input"] = vector + + if gs.vector_info_topo(vector)["areas"] <= 0: + gs.fatal(_("No areas found in AOI map <{}>...").format(vector)) + elif gs.vector_info_topo(vector)["areas"] > 1: + gs.warning( + _( + "More than one area found in AOI map <{}>. \ + Using only the first area..." + ).format(vector) + ) + + # are we in LatLong location? + s = gs.read_command("g.proj", flags="j") + kv = gs.parse_key_val(s) + if "+proj" not in kv: + gs.fatal(_("Unable to get AOI: unprojected location not supported")) + + geom_dict = gs.parse_command("v.out.ascii", format="wkt", **args) + num_vertices = len(str(geom_dict.keys()).split(",")) + geom = [key for key in geom_dict][0] + if kv["+proj"] != "longlat": + gs.verbose( + _("Generating WKT from AOI map ({} vertices)...").format(num_vertices) + ) + # TODO: Might need to check for number of coordinates + # Make sure it won't cause problems like in: + # https://github.com/OSGeo/grass-addons/blob/grass8/src/imagery/i.sentinel/i.sentinel.download/i.sentinel.download.py#L273 + feature_type = geom[: geom.find("(")] + coords = geom.replace(feature_type + "((", "").replace("))", "").split(", ") + projected_geom = feature_type + "((" + coord_proj = Module( + "m.proj", + input="-", + flags="od", + stdin_="\n".join(coords), + stdout_=PIPE, + stderr_=PIPE, + ) + projected_geom += (", ").join( + [ + " ".join(poly_coords.split("|")[0:2]) + for poly_coords in coord_proj.outputs["stdout"] + .value.strip() + .split("\n") + ] + ) + "))" + return projected_geom + return geom def download_by_id(query_id: str): @@ -281,9 +332,9 @@ def main(): # TODO: Add a way to search witout specifying a provider gs.fatal(_("Please specify a provider.")) - # Download by ids + # Download by IDs # Searching for additional products won't take place - if options["id"] or options["file"]: + if options["id"]: ids_set = set() if Path(options["id"]).is_file(): gs.message(_('Reading file "{}"'.format(options["id"]))) From 7b1f013ead4d18a34555f03b9bd12630f1e89de9 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Tue, 11 Jun 2024 11:09:09 +0300 Subject: [PATCH 11/30] Seperate file and id options exclusively --- src/imagery/i.eodag/i.eodag.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index d0d96e3088..4fb7975f2e 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -86,7 +86,15 @@ # % key: id # % type: string # % multiple: yes -# % description: List of scenes IDs to download | A single text file with one ID on each line +# % description: List of scenes IDs to download +# % guisection: Filter +# %end + +# %option +# % key: file +# % type: string +# % multiple: no +# % description: Text file with a collection of IDs, one ID per line # % guisection: Filter # %end @@ -112,6 +120,10 @@ # % guisection: Filter # %end +# %rules +# % exclusive: file, id +# %end + import sys import os @@ -335,14 +347,21 @@ def main(): # Download by IDs # Searching for additional products won't take place if options["id"]: + ids_set = set(pid.strip() for pid in options["id"].split(",")) + # Remove empty strings + ids_set.discard(str()) + gs.message(_("Found {} distinct ID(s).".format(len(ids_set)))) + gs.message("\n".join(ids_set)) + download_by_ids(ids_set) + elif options["file"]: ids_set = set() - if Path(options["id"]).is_file(): - gs.message(_('Reading file "{}"'.format(options["id"]))) + if Path(options["file"]).is_file(): + gs.message(_('Reading file "{}"'.format(options["file"]))) ids_set = set( - Path(options["id"]).read_text(encoding="UTF8").strip().split("\n") + Path(options["file"]).read_text(encoding="UTF8").strip().split("\n") ) else: - ids_set = set(pid.strip() for pid in options["id"].split(",")) + gs.fatal(_('Could not open file "{}"'.format(options["file"]))) # Remove empty strings ids_set.discard(str()) gs.message(_("Found {} distinct ID(s).".format(len(ids_set)))) From c5d52e976e6e741d972fea6e062a3b6ab683939e Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Tue, 11 Jun 2024 11:36:55 +0300 Subject: [PATCH 12/30] Code reformatting --- src/imagery/i.eodag/i.eodag.py | 98 +++++++++++++++++----------------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 4fb7975f2e..1fa1b22c6f 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -224,25 +224,20 @@ def get_aoi(vector=None): return geom -def download_by_id(query_id: str): - gs.message(_("Attempting to download product: {}".format(query_id))) - product, count = dag.search(id=query_id, provider=options["provider"]) - if count != 1: - raise ParameterError("Product couldn't be uniquely identified.") - if not product[0].properties["id"].startswith(query_id): - raise ParameterError("Product wasn't found.") - gs.verbose(_("Poduct {} is found. Downloading...".format(query_id))) - dag.download(product[0]) - - -def download_by_ids(products_ids): - # Search in all recognized providers - for product_id in products_ids: - try: - download_by_id(product_id) - except ParameterError as e: - gs.error(e) - gs.error(_("Product {} failed to download".format(product_id))) +def search_by_ids(products_ids): + gs.message("Searching for products...") + search_result = [] + for query_id in products_ids: + gs.message(_("Searching for {}".format(query_id))) + product, count = dag.search(id=query_id, provider=options["provider"]) + if count > 1: + gs.message(_("Could not be uniquely identified.")) + elif count == 0 or not product[0].properties["id"].startswith(query_id): + gs.message(_("Not found.")) + else: + gs.message(_("Found.")) + search_result.append(product[0]) + return search_result def setup_environment_variables(): @@ -332,6 +327,28 @@ def create_products_dataframe(eo_products): return df +def list_products(products): + df = create_products_dataframe(products) + for idx in range(len(df)): + product_id = df["id"].iloc[idx] + if product_id is None: + time_string = "id_NA" + time_string = df["time"].iloc[idx] + if time_string is None: + time_string = "time_NA" + else: + time_string += "Z" + cloud_cover_string = df["cloud_coverage"].iloc[idx] + if cloud_cover_string is not None: + cloud_cover_string = f"{cloud_cover_string:2.0f}%" + else: + cloud_cover_string = "cloudcover_NA" + product_type = df["product_type"].iloc[idx] + if product_type is None: + product_type = "producttype_NA" + print(f"{product_id} {time_string} {cloud_cover_string} {product_type}") + + def main(): # Products: https://github.com/CS-SI/eodag/blob/develop/eodag/resources/product_types.yml @@ -352,7 +369,7 @@ def main(): ids_set.discard(str()) gs.message(_("Found {} distinct ID(s).".format(len(ids_set)))) gs.message("\n".join(ids_set)) - download_by_ids(ids_set) + search_result = search_by_ids(ids_set) elif options["file"]: ids_set = set() if Path(options["file"]).is_file(): @@ -366,7 +383,7 @@ def main(): ids_set.discard(str()) gs.message(_("Found {} distinct ID(s).".format(len(ids_set)))) gs.message("\n".join(ids_set)) - download_by_ids(ids_set) + search_result = search_by_ids(ids_set) else: items_per_page = 40 # TODO: Check that the product exists, @@ -418,35 +435,16 @@ def main(): search_parameters["start"] = start_date search_parameters["end"] = end_date - search_results = no_fallback_search(search_parameters, options["provider"]) - num_results = len(search_results) - - if flags["l"]: - df = create_products_dataframe(search_results) - gs.message(_("{} product(s) found.").format(num_results)) - for idx in range(len(df)): - product_id = df["id"].iloc[idx] - if product_id is None: - time_string = "id_NA" - time_string = df["time"].iloc[idx] - if time_string is None: - time_string = "time_NA" - else: - time_string += "Z" - cloud_cover_string = df["cloud_coverage"].iloc[idx] - if cloud_cover_string is not None: - cloud_cover_string = f"{cloud_cover_string:2.0f}%" - else: - cloud_cover_string = "cloudcover_NA" - product_type = df["product_type"].iloc[idx] - if product_type is None: - product_type = "producttype_NA" - print(f"{product_id} {time_string} {cloud_cover_string} {product_type}") - else: - # TODO: Consider adding a quicklook flag - # TODO: Add timeout and wait parameters for downloading offline products... - # https://eodag.readthedocs.io/en/stable/getting_started_guide/product_storage_status.html - dag.download_all(search_results) + search_result = no_fallback_search(search_parameters, options["provider"]) + + gs.message(_("{} product(s) found.").format(len(search_result))) + if flags["l"]: + list_products(search_result) + else: + # TODO: Consider adding a quicklook flag + # TODO: Add timeout and wait parameters for downloading offline products... + # https://eodag.readthedocs.io/en/stable/getting_started_guide/product_storage_status.html + dag.download_all(search_result) if __name__ == "__main__": From ff68e868792dd7fbd6dceb648dac0f001d12f9d8 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Tue, 11 Jun 2024 12:10:27 +0300 Subject: [PATCH 13/30] update manual --- src/imagery/i.eodag/.i.eodag.py.swp | Bin 0 -> 32768 bytes src/imagery/i.eodag/i.eodag.html | 14 ++++++++++---- src/imagery/i.eodag/i.eodag.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 src/imagery/i.eodag/.i.eodag.py.swp diff --git a/src/imagery/i.eodag/.i.eodag.py.swp b/src/imagery/i.eodag/.i.eodag.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..ea786ba8984eb40c0de2923b221a87d801e3787a GIT binary patch literal 32768 zcmeI5d2l3GdB6u7+Zb$&Z7_!+Jda{_WV9pgdVL@)+ww{)3-7MBbgUur(zIrJM!lNp z9;SPw70ZTTFh_!`!i~YHR5%KX0HJIk1PEYK4u>Ij0)ZUGVFH*yAP`Iv5{~@7_xc=- z#S#9Gu>#o>m?y@Of0Bu>9!d+H-DxN8!ik@!f7ZReAmZ zpQ^Lu+TQl%)&scvZ5`lny=@1aC{QzgthILpINwg=MLS}2Z|S{bI(~gKEHieeF^j>(3e180(}YeCD4~ZUjlsz z^d<0XA%U=UYT*UsG1o{%_OQ0`-z6AOb=u4n4fxZO#66i~y zFM%Esu-V}M#_5H^JrCz4`~UI!|DVq)6uu2_f?ME5_#IdR1-}7*^T~hK z3OcQp-{x^~a8QTp)>QGS;<%{ayPjHE9o27ER@Imy!j|0*9IN6to3>Xi73*c~dPvn8 zc0Cv?9xj@I29(`))S9CRVy_+U^+`({^;LzcQNWbHS z!PtDe;|wY1h#Q2KzoLJdl!xv0QJp{&ttFn)P$adrw%=6R2&Zh~M4W+gn<7eLZDaKl zI_=4J+iz#xw$_cGreEzeoW#RE{j(5xH%@{|+iiu)4vpbVk)4w}DKw|EgoMr=L%HRV7(bB`S*Tt*hy| z8Fl%kBRiBS-$CsqTyGIu5Von(`oW5`KB}Sw`7Md!gkzoyZHjBlxQEozx}vVyokpl! z{tMKlBY0U4lwJ3?NLe6Z%lqR`TibbVhq_uFRqKdR3x!)2%HSAw$Z4?Es4$)Hadrlo}l&eExJsutx^b%v90%FAK1u|;NL zhbNYwT=kP_foVIGXo}&}gMDT8_LD_p- ziCe1SgGk+8`T$8rP;orkC8?2awCb;U4WHp%$|6ljTspn0q-hF0NDX#!Rt1jTt}GKlKxvt(tJ7?P+K}db z*2MJerYX+!6I-WvpMAt_cABcv@HJHB7;(#gQSkV`Suz zmZ~W}*9#piCo^)E&~zSplwr0%n|12a=}6t!kj|)RwM?m2u16WDRXq%;rrpw&%c+qv zHBOt=4S;kYzfhINXQoY;mn)v>njQ%2bh@6RjeDxnY0Ef9#+!t$M|8VE*Br`2`l(KU zQF9}P z7KzMMHV=97NZ7UsPJ+%C-XZZFL~SM?rXFDOk0P66{V-QF zlVXBQrKb>Tc13p`uE)?rKNtClt4N^qOls9`yM8B-f;0nLSptowMTLQ1<)<(}(7BKF zc~vK5a1A8d+-I*#ZxS#yxpHiE=U`dQGQjG3A>)nOJ#*k$)BC4*VC!bc$e{}&%*X;Dv zwezDSu(W$n?Ht*;Ydim3A~Te0#mrnyHtO#4umT^V`I=p zQ#t15lQR?JQ)>Lcv`FE=!t8;Wxk-sksfnn^m+7Q7Gf#`pDc8%)?sYX9QgL@{60mP( zVj3|J+P=yC^E0!hxj_kUZ>M4_T5qR9YaLPtowo0}JR0|^ZO12&km;7Eu4!~6!EKqK z4yf^k`D>F9^0_rJ%uFoojgqS(BY1i7EBw%5Trk`T+QUn($JA<7wbsLB-@CXV>wj5m z-vzS%kN4MK&-(smcq!Dtfg7L)!z(tiTNHh99%ezaQ>|7lH%V!XTUhKPR4h z;cHQxpOJX`pS}e866i~yFM+-U`V#0%pf7>G1o{%_OW?#LAgevJ{cLKuVX0}mUWuKR zf;^2s=xI7FtKrw{tQkuen++3r=`;c--FcBGd8a$GPT6GQ#B--^HDqb3#>Ui+t^Dpt z{o+%$tunq-kC}(;RH1(;v7011@$6<*ogE2pwKycZPh-W2$!iv-vRhx~f$Ujr?pHRf zB!lwF?N`~Jq%GQ>X|XD{gR(l%aL{f^azp_lOMLcU+BTb5HMbpvW!J0)M_vw8GpN8p!^UF&5`7cJN7 zP(Yx#8n4L4)f5{-YxcT!tuw5@8C51b1$wJ7vm4ninHrMTlk^*DQbnzOG23pUdu@jJ1uudtVLO}$XTc-kp>RLz{;$L9;3n{(4mCIcSHm}0`+puj0`G_S z!L9HLsK6{-2hV`3VHXU**&zA?(G&ayi0!}!;hk^`ybN~35L^hq2~PmA8~6|Q|8If- zZh!@thiAak;Uf4F`~M$@KZ6g#d*F}Y_u&xifvez3xCEXA=fPv((QrEa0$YF|z?a~| z@G5vEtigULg4hfwI0b%+9l#wR_5&dV(1t2p1Lwj|C=XpGC)Nk{UH2u>mq1?veF^;f zNI>QQ^2tt@`sPyIGtR!bR5a)An@jaF)SBy?OGUfWeRHY4xm4d=N}G6T<11}e^8dqJ zDr*52S*pqYf04P;C*(YRX8r#a*7YBSH^X6g4m=sY$6Eeh;EV7F@KTWV{RMCme3Wj%i}oCY6fJ^nU00N21S_%`eBPs2N*0$0G9a59L^zO2VXcm_<_!zeDABAVeZBvUW}9#w^WKBTVdT?S!wwGidNrg4|%0=LoD$lW4@r5Sdg#Q{4Ew6 z#Y0B#OIbD17m`Cc0&(^YSTHp|-dJXkEowrj;hgQ<2n3N!VvSj1$#c!=QnYl-Q3BDfxIwYj!-t-s~n$Avd* z%*H9DP8;nqR%Dowip0@VHtiLR%sLpDiQd`8+)N*tP?-O%M8;~dQaVmQ2K{}fauB6<-HcTw` z6F0@UWD9W^@!zN=$3K&GU*eFe6emnwkx5xbwJ)9%k5sPoI11&7Mt_^m&oCe7sramP;ZoXENpRy=>L)LWe0aw34NlrrHZGSW`+66(=c z=!w{FF@o0}M&jE{(29o&QV8-P1sh&wYW=Z-b`||otUG?QILc<3-yF)ni8|_e4(8w> z`eliQcb7vqbee(1plP+}rg58pkhg4UK_dV;mP==+Z&a$ zVR648$sUXjhe?j)H-XB*GuwAdi5#%W&-L2_>e}(yiQO|3lN0LT_}+!dIkks#4)c?9 z^J4IOR23V(*K|EHv!nd9A%5DS`MK+2^PAW95nk_%Ug{saE*VnCWY}G{+trF+Wiv^J z)~Lfyi#RJxlA9!xDkg^VnZg{Pdz3?RY)CCLayJMOb70Q*VOz@(V0s+78ajJ9GmuEB z=ZL-RR91{Jvy2u|BFq7~AT6bQ>5;r$Bf5<8V|i4T(T3qz6tlIoZn@Q<1dOR++OR0x zco;9<;1$b`zZH?w6C|mjOQsBvTEdXJ!0VC<%I2 zHW1TT2C(yP(vpnhg0RAFY<_4#VtghaC5yH>WC*!m;AQ{GQ91?nr`L^gltX!8X3gGaqP_eTt4U zFjHI;lOFHlab zrvqfZ*lKWSLl@CNV)`$=3a8@IE-E`lM-~o2%c;1vb!Pg6sx@V~Sy4P+mBoh55@S7c zK8Rxzvi68VkSdo3qEnJ@C$fa1>atdFY7VEMs;XADFKLD=<7aj+Xh4Vnx{_mn3{urb@q~z-je~GG2A)DWvZZQYOGUsRO z97^phm04>H@1EN}H*F5o4lm(#I5H549xGFf3%~R%4KpQXIO)UKy+puycbyr@kbJW5sJpWKl*EWr zYyWt$pdIlK)jsDGbC&&_vrRkH+*@X`HNBKgGDauka%z9ktsY6IkP`UD6iYJnQ`RkMd2(E7BRaRbS-H!=v016}&P1h$lYB|bP(iWFsDtSf3Kp48 zRcnLM_+72xHM$`osziY-?(LfBjY{#OLES3ys-5Ir+)UbZux*dZSCG1)LvB^e(jD*7 zfuX`-0n0@Um9S){l=}OiY0%vN5Pa?3SvQpV#H`wW0F=GQ!y` zkd3KYG`&K!Hch2}oYK(|#S7cW8c&Kb$*g2L8^wc?P1NL$Qz8;|nj!fWe|m{Q>qO>h z(|f4Q<8s%4%&zyd4$Q3R;sA0bmQPh7b|ZhgH}T_ly_4mM{x4N2ooFgI270{7%xO-J zv*fpu*XGS+3ttbaX6PRBIO|1DDp`-c1TODW!P0c;%Zt5}x@i|8VPr*O#&_w#qZlPT z$4|8D=6xoa^yNn--chx92$hAdaMAa!mL5Hg!qo~N?dzEC=rUA)+m(sXAP?!bVDNIgwEZMLqfhs8^#gX7R9I1GyrQ{9;n1byyy|=}d>S z%T8NN-L(Q;^j92$_Ry@$T_rN8Mc0|g6&}%~vLG!msqOn2$WTH;>o<4PA@6gf+GDNJo7ZaK5QrcI^Mcbjy|iVH0hxz!7<%>6h~f{*h`8m!7O zCzrsyiZfh6yRY}A22z`I{jA5LEH*|@vhaUjm!ycJ`ec^Yv)Ty0w3ung;wDo`(O2#F zlS)DzM0XW0+r=rxNomUbzvZtui#ymO8dD?bJU#PLrff+`_9{P`PEw^iq^yN0|e1@D{U0S z+?7)ui#8{E2f{qH*TzWvSk_=>;+l9jdoWoC`Z_AQGFkt#EG>MJ^`@-+yQVsjKVIs7#NPfEXhRjs@O|_Fq9?cw-U_dV4mePS3S0x{!7tDYd>P&h zb0E5cpQ8_WH{1kkFb7Y8GvN$)1bmwUkh}A)hV8Ho?nW2z5fEL$t?)7!g(t#E@M+5H zQ=rRD%1oM}e0&Z-8C}m}QQwMT?cOtG?U~*?*|oKz?`ewWZnWP-H)rl@iq&rtr0n=6 zug$ecU30!DChO=i%bJ+Ee*fN?@d?YCS(xW`r#b7u?Bt&5XC{j6B&fKBW@iwYIX`KQ&+fi<`rwwylAP0;%yMKygz?Px0VAAz z=KSpV?)l@7N|c`Z(lb47>GDM7+4LMv_axdoHcj+RU)eoo8RqmL#6XXlRA$TiawR>z z*-{hO)l23^UoBQC4f*8Wl%qK%MJa2@#gUC;NUD-Idmg!o(8fh9qfW-he)iB~ONdnd zTFcJf5m&;-X z&gWV*bR`aUvIC~K4-;o;EfbHVg!P3rCPv=ru4eRxG;Z<+JFc;GYtm$PFz;V?qGLtB zTFgJ!*T0Pww|6}*Hlv$UlZ2*%qYjztuTnKmS7ygljXO~>?a?bRDd$otE$65(WUJe6 zCNr!aJ`fl3il<;wSIm=W?@*+fBroUP1iffX1IpZG$5pVU;$T|221KNM;`KmA#Xd7P z<@ovkiLutS>#I*M@A@Aa$M=!(|1^%jCiZi^p7H-Y9@x9{vWp8kcK(SIvM(`)VjX4n zyDBzIC~41ZhMOe@ok z;#6{y6czyHUN3E&b^Ueo=BO;!{pl+Kv3cT#K=g@hEez^zA>KYF5-WLAkf7v4-@-#} z4pETbbf7D%i_z7+htvhAgt1YmW0-*Tp*WD5AYBt?ySl@Zzcf~Ptd*&U7^asiWFj3@!#S;v6ACHIyQ4G-_De~}HSLO{$F3}^ z8*-jA_Hu|mXZ@;Y3c9w*)ytNw(X7yK`K2h~i$#ehU}~Hjy0az7!dJW5BrRe*8&~!y zP$LfN^l?Reg>(xS~hks{kS9_@o%JFJmJm^KvbI8D-npR>`}b4Ytm(AXl8G6~5D|XNH3`{XO%r zW=Y?ZNse(98%9Mrb+}6cacywcm95PK(?(cPV7TfEESFPBzzOBH5odSH&h>u)z+ELz literal 0 HcmV?d00001 diff --git a/src/imagery/i.eodag/i.eodag.html b/src/imagery/i.eodag/i.eodag.html index becbb2f30a..29788dea6a 100644 --- a/src/imagery/i.eodag/i.eodag.html +++ b/src/imagery/i.eodag/i.eodag.html @@ -57,18 +57,24 @@

WARNING: I.EODAG IS UNDER DEVELOPMENT. THIS IS AN EXPERIMENTAL VERSION.

EXAMPLES

-Search and list the available Sentinel 2 scenes in the Copernicus Data Space Ecosystem: +Search and list the available Sentinel 2 scenes in the Copernicus Data Space Ecosystem, using a Vector Map as an AOI:
 i.eodag -l start=2022-05-25 end=2022-06-01 \
-    dataset=S2_MSI_L2A provider=cop_dataspace
+    map=vector_map_sample dataset=S2_MSI_L2A provider=cop_dataspace
 
-Download all available scenes in the tmp directory: +Download all available scenes in the tmp directory, with Cloud Coverage not exceeding 50%:
 i.eodag start=2022-05-25 end=2022-06-01 \
-    dataset=S2_MSI_L2A provider=cop_dataspace
+    dataset=S2_MSI_L2A provider=cop_dataspace clouds=50
+
+ +Download only selected scenes from a text file of IDs, using the Copernicus Data Space Ecosystem as the provider: + +
+i.eodag file=ids_list.txt provider=cop_dataspace
 
Download and extract only selected scenes into the download_here directory, diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 1fa1b22c6f..219b3bec11 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -49,7 +49,7 @@ # % key: dataset # % type: string # % description: Imagery dataset to search for -# % required: yes +# % required: no # % answer: S1_SAR_GRD # % guisection: Filter # %end From b1b92861d5b1972ef8ee2f2931543e57de051e87 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Tue, 11 Jun 2024 12:29:17 +0300 Subject: [PATCH 14/30] Add additional filtering layer --- src/imagery/i.eodag/.i.eodag.py.swp | Bin 32768 -> 28672 bytes src/imagery/i.eodag/i.eodag.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/imagery/i.eodag/.i.eodag.py.swp b/src/imagery/i.eodag/.i.eodag.py.swp index ea786ba8984eb40c0de2923b221a87d801e3787a..5939478296415f3952aca2aa5c5577a742bfc5e6 100644 GIT binary patch delta 2551 zcmaLY3s6*59LMp4fQq_2Qi1SwSz}rqT-O)oi&02>_$Ykh00S3x7q+$qcEJKi7Dt?L z)JC`?U;2RO*PY*`EU;R9(FJP-#O=2 zR(7a78x)sonf=04Wt7<>Xpur}iYc(?EZ-!C#5TrViHHalUaohLH)(ZRRAj%P0xc~v zCO!1=C86s5A9^38_Z{`#uJ<|mjllh2{d%I_AJlJX>(k~1_ixAt;h_#r3a6AeV;xc~38MI(Ow!w~BNXAGcK|v(8cNSs;mSHif;6x^d z!H8anLu6+P62jn4Cn3(E5o@5K7S$++1L^3Fht%R8?xGdn;S0<{DiV=^rx1s(ctCv{ zumVd^jS3Wp5=ah%L5KzsiExDCdI%N3C0xY!IEE%PU@3B;;yLuiZGvdSEu6qn)L=25 zMJiG-98cgv7-7gj4t1zmKgV~(c(;@G)sz@t%Ro!0cP{0OCkh!>AlUaJgxHEjm<|IT zg!5%+#u`k*1%nWNe1cu5M=?~)#`SK@!D)O7KenL?ndsk5q>2~@N4g5J7w=#-CgVl) z!bu*^k9v5Ki;Ds#OC&xJT}#x zSCH+tc}krn1B61$_n2R^c|2;q&6I62PLR_bu6#xR3$xj5)ciB%cvUM|QJwa+PKgQ_ z%l0lMS%R04j2N^r@d!3TmdJ*5j6q)n;UZ-@gVXo{`>+nxC`A@BF%82EtlXgt1U{xr zi&2bRB%qD*v_O{cFd9&fETrNW%5x7NU>>F-5wSQx?tUyl3U;wWmLdsZxW+E{wi7FN z4}%6QL_Ve>6}=G-xiway44IgQap;E#gu{Sqto9~s#+#UeN37_9EXqLr0=5?`YXcI03ZhT;M|gV`Fgo88MG<$jar>?czdb|T-__D zkg6)D&@Pi=VDWEZT!o1c*iCx1sKE+kU^>EZm4M_XsKb2NV8$S9Cnzr_U_64mk?MZZ z@xu$b&3nn~``y|tftGI$@)WZvz-ZO~C&xF*B2CPOolF=P4 z1awg0$4aOej~@7(DE2`{(j?y~HI`6cZ0;4gbl9$nHr^AhT__&ao*j{iCML@4>~yl7 zA)^D>fVZ#^d9dIP8Er;As^Eq^zy@LfOz4fz$ZI#=$1*taGGY;qFJ*RX32_bNDK;4+ zaD(iQ;X^o(g6{Z@>|~xh;DUVd{Z4MTa2|Dd9qEWdE4lg6h?THIzRd2E+pj2-*}cIa z4F=q#JbNK`U@WPwX)D{D=Brm^h4KX)eq<~?SaeC zWqW)g|8>*M$!>F5O-1fPYiXW-n9F$%yE0F8I;^9G{4LPhFDc`cNvhLh)1vGaEj!;3 zo2)ooC8oln5=WuSlVQwr7M5B)#;gHaZ+ik05`tBiRa=yw*i(MYf2L_oj_6kcA3W1t nY7=srz0j@b55=b||9Is*ex`r@Cp=X~may9F%KyErMvFfIGbFm* delta 2543 zcmZA24{(lk0KoBQW1F?jyRjxz-X)wJPPL^}HnSot`lIzf%@}cX$2xV@4W%gV;*}^} z7dlRKwU-psHMUSFbPIiadKnwv<^s z$8>5LpCVDElOD7on+T~KX>JBMz<$1=k@YNP22~7W03A4P>VM%F+u6!&p5c1R=tmK) zX-c;~HdB)v+jCD|Y~A3}SikaavBw5w)OB2&7TZ?PD&AO7m{MDw83bDh!=20eQp7a# zEY*zQK6;SK)|LiK4PzKgKTc+uaW*oU5e%R!nVj)Cd)brt%*~lWd|~#0%y@q9qGq*~ z!-JriIbEbhIGS^d@Oz)QkG*W+Df-ZzJR-EDIVX%_A9Iq79dfsC-Z}B=Wa6dOtOdpE4h&;|GmvsJes~lwqYnaU}CNhXi2ydw-b}v1EtFEop$zFhHHFKl;##t3#z8X@&SVV(xsfft z^=1|^l)kj)2Vb<2kJ-o)Mp8`~oj7VDz9UY!mpsAsY0{NBxtMSh+0JBYC?TJ88r}O1 ztl%Y{rjRs_xyQfd3&OqNVTx%@D?W5juj4VQ=}RwCsq5rEA50fI6JDRkE;+UmW?7#! zD?b{qxb9R^YyerdO!Udnk@A7=kdZtiB0c|-f%RIv0 zgmB&7KoUQzv6q!B;Zd%kC%e>WNR@4#ljpgYi)cl?9CbX&6WmNOeYlP&$0gaua%M7y z5~7@v_nJ%1Plf9Tzc$akM9gFEyM;guS}WU?}0<(~a|6=yY1N9jqn% zeVEU9hSP&1x;0S8Vq!eP2!>NdDS7;*-!2w%J7Lo^x~*U;6Sns41aS= p=uQ_JH2Z)s>H?W7xZKknO<^*7OgWrhF% diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 219b3bec11..8b150bc65e 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -349,6 +349,22 @@ def list_products(products): print(f"{product_id} {time_string} {cloud_cover_string} {product_type}") +def apply_filters(search_result): + filtered_result = [] + for product in search_result: + valid = True + if ( + options["clouds"] + and "cloudCover" in product.properties + and product.properties["cloudCover"] is not None + and product.properties["cloudCover"] > int(options["clouds"]) + ): + valid = False + if valid: + filtered_result.append(product) + return filtered_result + + def main(): # Products: https://github.com/CS-SI/eodag/blob/develop/eodag/resources/product_types.yml @@ -437,6 +453,8 @@ def main(): search_result = no_fallback_search(search_parameters, options["provider"]) + gs.message(_("Applying filters...")) + search_result = apply_filters(search_result) gs.message(_("{} product(s) found.").format(len(search_result))) if flags["l"]: list_products(search_result) From d303ac400d3bbd1edddf651b423fd01113af7436 Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Tue, 11 Jun 2024 12:32:28 +0300 Subject: [PATCH 15/30] Delete src/imagery/i.eodag/.i.eodag.py.swp --- src/imagery/i.eodag/.i.eodag.py.swp | Bin 28672 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/imagery/i.eodag/.i.eodag.py.swp diff --git a/src/imagery/i.eodag/.i.eodag.py.swp b/src/imagery/i.eodag/.i.eodag.py.swp deleted file mode 100644 index 5939478296415f3952aca2aa5c5577a742bfc5e6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28672 zcmeI4d2l3GeaD+{hu9{VV8WH>QCvGR+L3l`j1iWDENRyYyQ_^3+eBX4t(l%tduO`G z-96IEo3%q?DyD!Gkb-bjnLi+92)7FY#3393!j*y~5J)8uu5g$mm?I%n2>JZp>vJ@c zWt++$)O784XXf=ge((2w@Aw^0YZF(^A5~AQO;|jSw5->ix8A(&g|D*?Jl3+-osQd( z^m}+3?waTL``5eoknbj4;dm=_qG-R zd_CmAkOO<;Kre3Yzw|8YqJszYR3E9Fug-nc%HGNja}7B#B%o8y?I&k`GCI92PMDPlg~fxd;j3%x08JStnd9plHbkb^SY#g zw4Dbg>0`;~hx^_q$-|E!2ZkINa$v}TAqR#W7;<39fguNm92jz7$blgTh8(yL9B^>( zKZ|l76$1eN|5X3~<~f%24e%=P3h)Qu_rV%a;7Q;g&bF*OKpi;XJn)ILEbFb{&EPs< zgB4(bZ~cm8eGYs8ya~J#+yZ>i0!{F2a20qaI0rloJQV!w5tj8s@LupP@J{d+@LJ%4 zXM+=93Cx2_!9MWgU$(5TgHM5XgFC?6z^lL#I2-)*;gnh;8t)GxB*-T_JhZO zbHLf)!QeX>cfJPx9efr1Gx#KUAGjSnAFKff90pGU-^XBcH~1d-4EO|iF?cSR1vB6Z za4~ok_zLnRGWLx7YO z*Mm;S@f+1jtEO|0sb<@0MN^gQE2BF5h;l+#ZMss$5hp$p_$}2A>Q3wh{%=_gx2alg zY_F|VM<*dl8jy$jn@;Sst9G?A6ZF~*O+`I)o!C_kFLdj15N@f{XDXw$W)OCqxJv4% zY0^Dace}BnRw_I0#Ow0f6dJ8{PV%D){jMKJQ_Eq`9aHWpFN*Epr2c8DiNmc)O`vJk zmO^I zvPAXJ>&D87;7HP50~W+y2Zow<(%)lBQX6hNc1&YYZ;c_TyHVD{@%iI(=3N(-w7s>o z4&hMNup4)YDoTF_*|%s5608w*f$KNq1D@5}y~x{eCsosH!$eP~htjE1##YaZU_RWL zRENAaJkjMFuIZjmpYt0^x}vHGisNso`K1MQ=_L~flo{VqT_oP>(pnUUp5M|5C*7?{ zl@JtkrHOmp@>1+DT)VbCrq;F;vg-8O%mw~M>XHeHY(>gx1-nodN!{{(@3e*P4LxMF zF{!p-qwu-Jr|AvHYddRgS24I7Uc(KeNwprw-Dq-rTnByCm?e#P-E9Q*sOAOZu0IYL zlo)5G#W2p+RJjeCacQ{YSv|G&xYORHGikyh%?OOt;Ov=(XEQrlmBw<#gJx#&3Ep%`M%u7Il-D_fZ|T z7mhF6hwS-d$5)njl>LQX-0f*WRNDwB4bU;xESYgC&?O3M?dH16LJn~eOD~IX!WCCf zTSqM{s>pT1`Z^6njFu5yO|z+L8_h@U+4;pCDK4xNyHkABIpuYF9aV1!y#`F%KyHv7 zhA<`;Quk$v??_VH46It67*hu(CQfLsn&I>O*d=hXz-OsV_hG;o7T2>yw$cfsg%`3QHhOh- zOQg_qH(cd3;btbOr7c7AM2;#u*F1MB4jrnKsteMtf0~{ZWLz&hyVs{z7b#tuCwd`d zhGb2Y$kfZRu69HnIx>B@%m`-JW)QMdq~vMl=Ba&Dn!KDkyo_*@)aC>iIc9mtN=M10 zDc7lYm~v7QbF33fLo+8K$Vfed&~)m0?eKi|9u~T!Oe!Lg(nqNcC-j0|l<#acsWdqk zStCEs#~PvP(j$vr!;RTpBWbqu9PwCABF-k4Pi-6=t*J$JSS=T_;i#E~RY=$lp5j5jXIVbmgHgsL{;bSZ6FxybbY(Cyw zYkPG~vb420fB4GfNeHaYjH-hZ2QS{we@~Gk%9RpkuIAb;??$4)$2dnBGBqXiZo_Q{ z-HxmbvFmjDN#iWX;e2jkcKWcIK0Ys8IKHxYd|_!$8dGXES@PvPsl&|k;d5)P8pnGr zMq?^nZw&!Q7iQ;S1GOEUJGQ*ASX~;G`i}JKj$-!q>de+Lb(I?izDLru-w54+N@C7i zzPh5_6M{oIp^m8OmE|k58p^#jEiBBg97)JkNf5le_&Gi-80U}oqHui8^Eq2>sP0z0 z9{3koTK`9vJs&+-^#3tF@c$tPh8!4jV90?X2ZkINa$v}T`^y12Lt{k6R_w*qj^p`N4Az#UrVrY3 zuWPr1R*RE(^&(@3mX~fja&v>Wq?EHBEM~=a&ppy!%WlhoUrkM^1G|+yke8*TSo;g5 z^MZv$8~|u_RgCmvYApHyS{Qo$jmnr9^`|Pcb62b!t|V4@@hF%CaRuyLR{Q|cL3v7U z0LRy!2xIl`qH8!&O&xE$*cgNyQ7c8e#;hGW_y(F@7{#@;^?si~Vmvq1h<&}=u%tz8 z^p`L4?A*LDwBF~|wYF>OaJpX27zAu%I77M8{(~iLGFjBdG)t4ie|f<)TS*Rf$%k5p zIbgaj{(!^S**Bdnom+QUe=`;$46xdRQ*iQxRU;v}oaEo2NmaB?YrGkv(nhhzSQu5d zpmQ2+(2FCr=B_&%X!@b#mO*XxaF$>vKzYT$xaML?YkOj{u4u}n5xc6tG9)Owi@O-Au)_x-inQ3{<6dOU@6uO!M%41c?82m)Zom_|BqkzJ=Qkbv5BRQ(ZwdQ> zm@N{;T-5iV*%YgJY7C3$zNITH)H#EjN<0UOo8wZ(QMerHC)+`AQV+f^P+oG{;vP#3 zY|$XLz+WvltCe=;gjqEhs}9me55miCA>~%5CA;)7DUnD+#*+T({lIQA@M})}q;1}%WaT`^MWH=S(P|}iiBwLE zYW*KI-qP{*=>O}qdz(wF7kl28CVFP$BxB;wz3&7pz{+|Y78~7-=9oz`S&p!kH6W#wu z;4NSS90g8$bH{!0W*ufg|9j_{qNxUI(rQui-H#`hrtGWf|QkFk$ z+Dxf@2}&kgBO$6##bCN-=Z?%R&)L(9Ggr=EwJTZDb3CkOecjfeyVnNiu3283o>|^| zQzO_7wSQ7?<5urXRl_Dz}?}pQLy?M+rQpSIR<#K=d`^WwI9Ti z=mN7pChV1+!OrT+Lk>dDU7?>C=97I`k3>2`rDdYV8idQank|Ya#bzip5oCpcPVEc{HT;$ORLZuNFrI{XtLLYtx+lrx``~5C6TL^8+5WRC27=mJ)S`E!WZ9;swXSB zoS%Dy6XkVBQfg7RjS^A0)=$E0Br$zz8&13DGWx`hBuBUOm3#_wx&fPLJ-~T{F&`(Q ze3R|{`kGm9kf11T@V#%%lkyc+sa+qSy%Z`*>G{3N=<3Ae!NLei3A3unw{ox5lc!V_ zWr1UWuF#S!^5g;_zo5hoLwicXSH>(!NgqSw4sST!O$mcc-KMjnin2SvMs^UuUOOlklZp=-T4`lyRrO_lU7LMLpjiq*c zVd=0Nl>gk&T02-XDVO&B>7mqf$X_2yzh|~+>S_~MP%4EKCg zp(txO=N!X_%26>MI0>3jS3h&vs9$*IwfDZtiI!l47~fzts+WayIie9edy`ep+Ct)5 z5V=M~SU^v1tR_)*C)D{UC2gYP+D?p4BJGPL$kK#MR?{w@qB@gIFj_r6!Oo*I7TcqM zK1;KKXBlghj4(E&vve$7$Fm%if;^76#gM)YSqy4URQEh&4DJzTH&xj@8T0GH3!ShP zjSg1TSA5y4Wt+wfpy1aeyGb@1vhY8>e?`5nsJm_>r3tp|De{m$v6Z8unL&$87y1nn z(y9=DZ-NpvPh|kg;*jYoCW52LpPf_85!wtPF_4akh?V9_KPyMg2?kH6 z#63wqIE`*3Bb`dxWv9*=V*>M=S9(`R zVr?hz5?)OmV^p&3{EV(d@l=?l9p_YKl3{8Fs};X;BHuw|8W&%3WG=p>q~~WP8HVCZ zN)}#jrZ1CjpVkCOlLIGgi)}%ok#d3Wbz6n)Ag6y|A2w}SdkW;3T;-wJlp; zw|1_q&_ADCeU+7ZDg@lC-nP^3A``nWu2OYT;>&B+nuxuu`fI_dy)Itj|Gxko^8KPu zqW?QupCld}eLsx>cmv;W0dv3t??KOh33xGh5m*I_;Njq#=<*T^@G@{3xS$61gSVs8 z-wdX~Gr$AE2hrp21UCcm_m6{L2VX{q{}T8jcsp>wG`JgGUgGj!3${QT><3>&r~eH2 zAb2^L0SCYn!QJTbZv+x={{moxMKA)shVK3^;8t)WkhuCwz(c@C(Ba<$PJ$Dl3ig2? zqr=|?z6rhoJ`QdJHv`f0F9nYQ-$I}NIJh0W0^A5L2j_xMqsxC7yc9U#8Q{_2Zy1LU z02z<>fjvAT6!LV5FOI9t%4xO_ZK?AbXVhsi!(pqF*kgX_ctg(6?CE)8nqN+M#HL%i zw&cI!JEpJLj-+nMY9E`{q?Usr+mw zG;F6>$k3^#8$~L%E2#sE!=vr#lLu=P%`+F2y0kr4YWyvCZa_ZM#lj0LiaU0)Fi4_a zRDL(7Vtbd0naQhv7MZ|M6tip_iQ-E`rKIeMHk$JqJB_=?_P^7p^UgG-Rs8Zu%ZA3V z%h>^YI(An%3O1UfiPqm}QZ%{EkO)y^OVnjk%*mMwJE}Lrvc)&ep6!LnI^AemB3ioU zZlfkXhC{kC&IjC%(C5^uact=Cqxn`Y{Q8is#eXa zFe8gw(Hyf!?&yY+!aFmkkn4W&CYf@34jQ^B#I z!6diRg6U^z#L85a^7i{8)AEDK@0%8I?_p6lI7i~++OKk$t<}-_c-#qQL?in z>{9A}Nrqk__b3-7t`rLn5({~sGnT0<_SV6q?zn_V9YvOy{ase@q(!OE4Kk;jd0!xt zwfwZSJBjfUtB)4A)E<5!dg5xFS#L$gu*}lEs)}sR6__+%q#l&FKq!C@$iNFC27|J!bWKt*Lp!kh0xYxOKfGd--f6 z%-9VjenP9?UCs((;=~pq`n7Co$<-sdZevbh#{QFu`$(Z6cN=@S1Cvl{;~=F?QS8%E z;(I4@+Z%55TY8qK7O1QBwM7h9VoQ(QW{;@%rhE&d^rw2^g-N#DCu#Ue+Y(;Bsr~QR$qb!tlRG61Ay`}73wPNPf>VQ^iAWF= z3I(@cA0t!&--uM^*WK|th70Yo9?1h4blQMOgEYb+>q34C`2Uh6k=sO`jA4IKM^;o< z3lmxL6oeG}A9j#bNQ@~YwNgn|hLh2h^M5xu>8>8YdZEo4@=l(@OF6P-BROdN3J#HU zoXn7Fu@pzv-aOM;;WVrv>2tP=G~~diFOb$et{s@Wv!Yp*MrF<^bMG$M^cnbVFDAABAGP1;ntW*{3g;Kl^FIV$1O6C12h_m?_yKzU>%nc{ z77zlt3veMA1y%3_@Ym@4?*VTHF92MMtZv`H> z0-Ou}8~tBw0`CAmko)`pi|+qD@MZ8aunDdKzY6X`_rDXo4ZIXIf!xXeF?#=hfh}+f z{0=w^{0BP!`+@lTZvruhfZVtLHSlQgef0l-1h;_~0|#6IE(AY7=l?$V1b73u1&GhT z0=|pR|4AS||C_)WunOkDLGWdC|95~_gBOCE!A;ljI;BWH%{gyl-4#E_Wi$B%hTzY}K=}kKqp)#XsX~V{Fks zzoai%XL&NOtyQc?z87rT@-y2SEcqo56E7u9T~?s15Hcv)vVyGOxldbrH`G4epM6@d zFP%=3f;!Bxnw(E0?Hh2_aJQ9(6bv9t!A4SzN5`Ka2j z^fiA5Wed+!o+EETD$VmyshkdVLiLhlm6a^;Q!MA%Vy5Xx!?z?uiLm}`WtrQR>Cpc3 zWqHbs?F>)fkBfo_SjPSB-n+K<#O$Oyb1R-iV7Y zGQU$-$HC;K_V8T%lreQiYF~H4Mjh3^p=nPR?7aDewZ^+&2p&Vk#%0L@@s zyQvy-Cg*RWa1#=}#jH4}LoD`k#XI8?9z;v`RCjh+$A`{n*S5ZTYEJv+4z&JzCQ&|0 z<#fe+Xwgs_SW|b5Jj!g44%GDW(s>G-jUXQ3nw}ZhdarqfLeeT*9z7>|JefhpX z(3G$L%7P+BwX!euYi435cYKkvLQ`d?NzHj@<(EA2CsnHbb#QPdUOwh_KC3yR#Ar`d zR+bN4iaS*ZCR``=(Qk|FNm!ysB`%u^3*%r{_bt+fh>^aVyw@(Vz6_?=izL|Al~_65 zFa9f2V!Ez<&VAj@OZ)=MRhb!GuaOR+wPKdDm)f|^;{zcg?D z#?H8oeo8JkBuT8ATxlQ}NdBTu=8cPLTl`TSQQ*yupFX|Zgr&m^v*tcs)1{K9?`2=y zX};#bB_d(GMn2)B0-Xeh4mn&NvXUxQVIE78Tq#J%Et!-{2b0UifR+zZdN|1m*mF%7 zr{usXJ{?O6rH_zN^f}0+*-n;~W|z-wPNn2zZ(&EiKBiodKl7z?^jTO-OOs(>AKPIe Q(V;Sen38wAdoXGJAC)El>Hq)$ From 46913ee8c8cdd3022e100907cc2c95baaaaa553d Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Tue, 11 Jun 2024 21:04:22 +0300 Subject: [PATCH 16/30] Change default provider and dataset --- src/imagery/i.eodag/i.eodag.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 8b150bc65e..1aa0c8af57 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -50,7 +50,7 @@ # % type: string # % description: Imagery dataset to search for # % required: no -# % answer: S1_SAR_GRD +# % answer: S2_MSI_L2A # % guisection: Filter # %end @@ -102,7 +102,8 @@ # % key: provider # % type: string # % description: Available providers: https://eodag.readthedocs.io/en/stable/getting_started_guide/providers.html -# % required: yes +# % required: no +# % answer: peps # % guisection: Filter # %end From 4a70d498456e1b37b4db4fd159cc8d2706bb0c29 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Wed, 12 Jun 2024 19:32:36 +0300 Subject: [PATCH 17/30] Change default dataset to S2_MSI_L1C --- src/imagery/i.eodag/i.eodag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 1aa0c8af57..af8c2c522a 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -50,7 +50,7 @@ # % type: string # % description: Imagery dataset to search for # % required: no -# % answer: S2_MSI_L2A +# % answer: S2_MSI_L1C # % guisection: Filter # %end From b1d2fec60ce49634afe388d7fb65323b1d4a6842 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Wed, 12 Jun 2024 20:01:29 +0300 Subject: [PATCH 18/30] Make provider optional --- src/imagery/i.eodag/i.eodag.py | 57 ++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index af8c2c522a..23f1f8ec2d 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -36,12 +36,12 @@ # %flag # % key: e -# % description: Extract the downloaded the datasets +# % description: Extract the downloaded the datasets, not considered unless provider is set # %end # %flag # % key: d -# % description: Delete the product archieve after downloading +# % description: Delete the product archieve after downloading, not considered unless provider is set # %end # OPTIONS @@ -101,9 +101,8 @@ # %option # % key: provider # % type: string -# % description: Available providers: https://eodag.readthedocs.io/en/stable/getting_started_guide/providers.html +# % description: The provider to search within. Available providers: https://eodag.readthedocs.io/en/stable/getting_started_guide/providers.html # % required: no -# % answer: peps # % guisection: Filter # %end @@ -230,7 +229,10 @@ def search_by_ids(products_ids): search_result = [] for query_id in products_ids: gs.message(_("Searching for {}".format(query_id))) - product, count = dag.search(id=query_id, provider=options["provider"]) + if options["provider"]: # If provider is set, then search without fallback + product, count = dag.search(id=query_id, provider=options["provider"]) + else: + product, count = dag.search(id=query_id) if count > 1: gs.message(_("Could not be uniquely identified.")) elif count == 0 or not product[0].properties["id"].startswith(query_id): @@ -243,16 +245,31 @@ def search_by_ids(products_ids): def setup_environment_variables(): # Setting the envirionmnets variables has to come before the eodag initialization - os.environ["EODAG__{}__DOWNLOAD__EXTRACT".format(options["provider"])] = str( - flags["e"] - ) - os.environ["EODAG__{}__DOWNLOAD__DELETE_ARCHIV".format(options["provider"])] = str( - flags["d"] - ) - if options["output"]: - os.environ[ - "EODAG__{}__DOWNLOAD__OUTPUTS_PREFIX".format(options["provider"]) - ] = options["output"] + if options["provider"]: + # Flags can't be taken into consideration without specifying the provider + os.environ["EODAG__{}__DOWNLOAD__EXTRACT".format(options["provider"])] = str( + flags["e"] + ) + os.environ["EODAG__{}__DOWNLOAD__DELETE_ARCHIV".format(options["provider"])] = ( + str(flags["d"]) + ) + if options["output"]: + os.environ[ + "EODAG__{}__DOWNLOAD__OUTPUTS_PREFIX".format(options["provider"]) + ] = options["output"] + else: + if flags["e"]: + gs.warning( + _( + "Ignoring 'e' flag...\n'extract' option in the config file will be used.\nIf you wish to use the 'e' flag, please set a provider." + ) + ) + if flags["d"]: + gs.warning( + _( + "Ignoring 'd' flag...\n'delete_archive' option in the config file will be used.\nIf you wish to use the 'd' flag, please set a provider" + ) + ) if options["config"]: os.environ["EODAG_CFG_FILE"] = options["config"] @@ -291,6 +308,7 @@ def no_fallback_search(search_parameters, provider): def create_products_dataframe(eo_products): result_dict = {"id": [], "time": [], "cloud_coverage": [], "product_type": []} for product in eo_products: + print(product.properties) if "id" in product.properties and product.properties["id"] is not None: result_dict["id"].append(product.properties["id"]) else: @@ -374,9 +392,6 @@ def main(): dag = EODataAccessGateway() if options["provider"]: dag.set_preferred_provider(options["provider"]) - else: - # TODO: Add a way to search witout specifying a provider - gs.fatal(_("Please specify a provider.")) # Download by IDs # Searching for additional products won't take place @@ -451,8 +466,10 @@ def main(): # TODO: Requires further testing to make sure the isoformat works with all the providers search_parameters["start"] = start_date search_parameters["end"] = end_date - - search_result = no_fallback_search(search_parameters, options["provider"]) + if options["provider"]: + search_result = no_fallback_search(search_parameters, options["provider"]) + else: + search_result = dag.search_all(**search_parameters) gs.message(_("Applying filters...")) search_result = apply_filters(search_result) From 98187ef333924979e4a52d096eceaf7c17b69bb4 Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Thu, 13 Jun 2024 00:26:11 +0300 Subject: [PATCH 19/30] Update i.eodag.py --- src/imagery/i.eodag/i.eodag.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 23f1f8ec2d..963f6149b0 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -308,7 +308,6 @@ def no_fallback_search(search_parameters, provider): def create_products_dataframe(eo_products): result_dict = {"id": [], "time": [], "cloud_coverage": [], "product_type": []} for product in eo_products: - print(product.properties) if "id" in product.properties and product.properties["id"] is not None: result_dict["id"].append(product.properties["id"]) else: From d178953d16e107198ea7f097e6c70a26357fd185 Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Fri, 14 Jun 2024 11:55:43 +0300 Subject: [PATCH 20/30] Update src/imagery/i.eodag/i.eodag.html Co-authored-by: Veronica Andreo --- src/imagery/i.eodag/i.eodag.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imagery/i.eodag/i.eodag.html b/src/imagery/i.eodag/i.eodag.html index 29788dea6a..00333b2bbe 100644 --- a/src/imagery/i.eodag/i.eodag.html +++ b/src/imagery/i.eodag/i.eodag.html @@ -61,7 +61,7 @@

EXAMPLES

 i.eodag -l start=2022-05-25 end=2022-06-01 \
-    map=vector_map_sample dataset=S2_MSI_L2A provider=cop_dataspace
+    map=durham dataset=S2_MSI_L2A provider=cop_dataspace
 
Download all available scenes in the tmp directory, with Cloud Coverage not exceeding 50%: From d4f5f12fc8bac151764c864d1fe995926a763a97 Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Fri, 14 Jun 2024 11:56:07 +0300 Subject: [PATCH 21/30] Update src/imagery/i.eodag/i.eodag.html Co-authored-by: Veronica Andreo --- src/imagery/i.eodag/i.eodag.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/imagery/i.eodag/i.eodag.html b/src/imagery/i.eodag/i.eodag.html index 00333b2bbe..04afe2d8f3 100644 --- a/src/imagery/i.eodag/i.eodag.html +++ b/src/imagery/i.eodag/i.eodag.html @@ -60,6 +60,8 @@

EXAMPLES

Search and list the available Sentinel 2 scenes in the Copernicus Data Space Ecosystem, using a Vector Map as an AOI:
+v.extract input=urbanarea where="NAME = 'Durham'" output=durham
+
 i.eodag -l start=2022-05-25 end=2022-06-01 \
     map=durham dataset=S2_MSI_L2A provider=cop_dataspace
 
From 4d9af9066586ddaabb881f033d5e3cafbd2f9bae Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Fri, 14 Jun 2024 14:51:51 +0300 Subject: [PATCH 22/30] Update src/imagery/i.eodag/i.eodag.py Co-authored-by: Stefan Blumentrath --- src/imagery/i.eodag/i.eodag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 963f6149b0..993b830490 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -41,7 +41,7 @@ # %flag # % key: d -# % description: Delete the product archieve after downloading, not considered unless provider is set +# % description: Delete the product archive after downloading, not considered unless provider is set # %end # OPTIONS From fb1c3d1c1c8e9655524a1c4bacbbd2976593d54d Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Fri, 14 Jun 2024 14:52:06 +0300 Subject: [PATCH 23/30] Update src/imagery/i.eodag/i.eodag.py Co-authored-by: Stefan Blumentrath --- src/imagery/i.eodag/i.eodag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 993b830490..8fd3a9e7eb 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -64,7 +64,7 @@ # %option # % key: clouds # % type: integer -# % description: Maximum cloud cover percentage for Sentinel scene [0, 100] +# % description: Maximum cloud cover percentage for scene [0, 100] # % required: no # % guisection: Filter # %end From 0dd43709010cd02bf5902a74ae3beaaa56fefad6 Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Fri, 14 Jun 2024 14:52:25 +0300 Subject: [PATCH 24/30] Update src/imagery/i.eodag/i.eodag.py Co-authored-by: Stefan Blumentrath --- src/imagery/i.eodag/i.eodag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 8fd3a9e7eb..9e56dceead 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -101,7 +101,7 @@ # %option # % key: provider # % type: string -# % description: The provider to search within. Available providers: https://eodag.readthedocs.io/en/stable/getting_started_guide/providers.html +# % description: The provider to search within. Providers available by default: https://eodag.readthedocs.io/en/stable/getting_started_guide/providers.html # % required: no # % guisection: Filter # %end From 39d0068dcc04987022c14189f5d01fcd0c53113f Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Fri, 14 Jun 2024 15:09:36 +0300 Subject: [PATCH 25/30] Refactor parsing of id and file options --- src/imagery/i.eodag/i.eodag.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 9e56dceead..2df3f09873 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -394,15 +394,12 @@ def main(): # Download by IDs # Searching for additional products won't take place + ids_set = set() if options["id"]: + # Parse IDs ids_set = set(pid.strip() for pid in options["id"].split(",")) - # Remove empty strings - ids_set.discard(str()) - gs.message(_("Found {} distinct ID(s).".format(len(ids_set)))) - gs.message("\n".join(ids_set)) - search_result = search_by_ids(ids_set) elif options["file"]: - ids_set = set() + # Read IDs from file if Path(options["file"]).is_file(): gs.message(_('Reading file "{}"'.format(options["file"]))) ids_set = set( @@ -410,7 +407,8 @@ def main(): ) else: gs.fatal(_('Could not open file "{}"'.format(options["file"]))) - # Remove empty strings + + if len(ids_set): ids_set.discard(str()) gs.message(_("Found {} distinct ID(s).".format(len(ids_set)))) gs.message("\n".join(ids_set)) From 179e6fd07ef0c1d619e62a3b30ed318aafa35640 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Fri, 14 Jun 2024 15:36:35 +0300 Subject: [PATCH 26/30] Use kwargs in setup_environments_variables --- src/imagery/i.eodag/i.eodag.py | 50 +++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 2df3f09873..cf0a6ad6e8 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -243,35 +243,47 @@ def search_by_ids(products_ids): return search_result -def setup_environment_variables(): +def setup_environment_variables(env, **kwargs): + provider = kwargs.get("provider") + extract = kwargs.get("e") + delete_archive = kwargs.get("d") + output = kwargs.get("output") + config = kwargs.get("config") + # Setting the envirionmnets variables has to come before the eodag initialization - if options["provider"]: + if config: + env["EODAG_CFG_FILE"] = options["config"] + if provider: # Flags can't be taken into consideration without specifying the provider - os.environ["EODAG__{}__DOWNLOAD__EXTRACT".format(options["provider"])] = str( - flags["e"] - ) - os.environ["EODAG__{}__DOWNLOAD__DELETE_ARCHIV".format(options["provider"])] = ( - str(flags["d"]) - ) - if options["output"]: - os.environ[ - "EODAG__{}__DOWNLOAD__OUTPUTS_PREFIX".format(options["provider"]) - ] = options["output"] + env[f"EODAG__{provider.upper()}__DOWNLOAD__EXTRACT"] = str(extract) + env[f"EODAG__{provider.upper()}__DOWNLOAD__DELETE_ARCHIV"] = str(delete_archive) + if output: + env[f"EODAG__{provider.upper()}__DOWNLOAD__OUTPUTS_PREFIX"] = output else: - if flags["e"]: + if extract: + gs.warning( + _( + """Ignoring 'e' flag... + 'extract' option in the config file will be used. + If you wish to use the 'e' flag, please specify a provider.""" + ) + ) + if delete_archive: gs.warning( _( - "Ignoring 'e' flag...\n'extract' option in the config file will be used.\nIf you wish to use the 'e' flag, please set a provider." + """Ignoring 'd' flag... + 'delete_archive' option in the config file will be used. + If you wish to use the 'd' flag, please specify a provider.""" ) ) - if flags["d"]: + if output: gs.warning( _( - "Ignoring 'd' flag...\n'delete_archive' option in the config file will be used.\nIf you wish to use the 'd' flag, please set a provider" + """Ignoring 'output' option... + 'output' option in the config file will be used. + If you wish to use the 'output' option, please specify a provider.""" ) ) - if options["config"]: - os.environ["EODAG_CFG_FILE"] = options["config"] def normalize_time(datetime_str: str): @@ -387,7 +399,7 @@ def main(): # Products: https://github.com/CS-SI/eodag/blob/develop/eodag/resources/product_types.yml global dag - setup_environment_variables() + setup_environment_variables(os.environ, **options, **flags) dag = EODataAccessGateway() if options["provider"]: dag.set_preferred_provider(options["provider"]) From f39a6ab6e51fadb84e56a3622359ed32cb001d64 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Fri, 14 Jun 2024 15:40:21 +0300 Subject: [PATCH 27/30] Rename dataframe keys names --- src/imagery/i.eodag/i.eodag.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index cf0a6ad6e8..61b5665ea1 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -318,7 +318,7 @@ def no_fallback_search(search_parameters, provider): def create_products_dataframe(eo_products): - result_dict = {"id": [], "time": [], "cloud_coverage": [], "product_type": []} + result_dict = {"id": [], "time": [], "cloudCover": [], "productType": []} for product in eo_products: if "id" in product.properties and product.properties["id"] is not None: result_dict["id"].append(product.properties["id"]) @@ -342,16 +342,16 @@ def create_products_dataframe(eo_products): "cloudCover" in product.properties and product.properties["cloudCover"] is not None ): - result_dict["cloud_coverage"].append(product.properties["cloudCover"]) + result_dict["cloudCover"].append(product.properties["cloudCover"]) else: - result_dict["cloud_coverage"].append(None) + result_dict["cloudCover"].append(None) if ( "productType" in product.properties and product.properties["productType"] is not None ): - result_dict["product_type"].append(product.properties["productType"]) + result_dict["productType"].append(product.properties["productType"]) else: - result_dict["product_type"].append(None) + result_dict["productType"].append(None) df = pd.DataFrame().from_dict(result_dict) return df @@ -368,15 +368,15 @@ def list_products(products): time_string = "time_NA" else: time_string += "Z" - cloud_cover_string = df["cloud_coverage"].iloc[idx] + cloud_cover_string = df["cloudCover"].iloc[idx] if cloud_cover_string is not None: cloud_cover_string = f"{cloud_cover_string:2.0f}%" else: - cloud_cover_string = "cloudcover_NA" - product_type = df["product_type"].iloc[idx] - if product_type is None: - product_type = "producttype_NA" - print(f"{product_id} {time_string} {cloud_cover_string} {product_type}") + cloud_cover_string = "cloudCover_NA" + product_type_string = df["productType"].iloc[idx] + if product_type_string is None: + product_type_string = "productType_NA" + print(f"{product_id} {time_string} {cloud_cover_string} {product_type_string}") def apply_filters(search_result): From 3e4e50b97f525d56b967cb8c17b24e29c56b3ec2 Mon Sep 17 00:00:00 2001 From: Hamed Elgizery Date: Fri, 14 Jun 2024 16:08:55 +0300 Subject: [PATCH 28/30] Update src/imagery/i.eodag/i.eodag.py Co-authored-by: Stefan Blumentrath --- src/imagery/i.eodag/i.eodag.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index 61b5665ea1..a8e3e1c84a 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -229,10 +229,7 @@ def search_by_ids(products_ids): search_result = [] for query_id in products_ids: gs.message(_("Searching for {}".format(query_id))) - if options["provider"]: # If provider is set, then search without fallback - product, count = dag.search(id=query_id, provider=options["provider"]) - else: - product, count = dag.search(id=query_id) + product, count = dag.search(id=query_id, provider=options["provider"] or None) if count > 1: gs.message(_("Could not be uniquely identified.")) elif count == 0 or not product[0].properties["id"].startswith(query_id): From d9a601f8e3248fe0fee391cefa57b472940c4b76 Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Fri, 14 Jun 2024 16:18:20 +0300 Subject: [PATCH 29/30] Refactor get_bb in get_aoi --- src/imagery/i.eodag/i.eodag.py | 78 ++++++++++++++-------------------- 1 file changed, 32 insertions(+), 46 deletions(-) diff --git a/src/imagery/i.eodag/i.eodag.py b/src/imagery/i.eodag/i.eodag.py index a8e3e1c84a..e418e2d73e 100755 --- a/src/imagery/i.eodag/i.eodag.py +++ b/src/imagery/i.eodag/i.eodag.py @@ -144,12 +144,8 @@ def create_dir(directory): gs.fatal(_("Could not create directory {}").format(dir)) -def get_bb(): - # are we in LatLong location? - kv = gs.parse_command("g.proj", flags="j") - if "+proj" not in kv: - gs.fatal(_("Unable to get bounding box: unprojected location not supported")) - if kv["+proj"] != "longlat": +def get_bb(proj): + if proj["+proj"] != "longlat": info = gs.parse_command("g.region", flags="uplg") return { "lonmin": info["nw_long"], @@ -168,9 +164,14 @@ def get_bb(): def get_aoi(vector=None): """Get the AOI for querying""" + + proj = gs.parse_command("g.proj", flags="j") + if "+proj" not in proj: + gs.fatal(_("Unable to get AOI: unprojected location not supported")) + # Handle empty AOI if not vector: - return get_bb() + return get_bb(proj) args = {} args["input"] = vector @@ -185,16 +186,10 @@ def get_aoi(vector=None): ).format(vector) ) - # are we in LatLong location? - s = gs.read_command("g.proj", flags="j") - kv = gs.parse_key_val(s) - if "+proj" not in kv: - gs.fatal(_("Unable to get AOI: unprojected location not supported")) - geom_dict = gs.parse_command("v.out.ascii", format="wkt", **args) num_vertices = len(str(geom_dict.keys()).split(",")) geom = [key for key in geom_dict][0] - if kv["+proj"] != "longlat": + if proj["+proj"] != "longlat": gs.verbose( _("Generating WKT from AOI map ({} vertices)...").format(num_vertices) ) @@ -317,38 +312,29 @@ def no_fallback_search(search_parameters, provider): def create_products_dataframe(eo_products): result_dict = {"id": [], "time": [], "cloudCover": [], "productType": []} for product in eo_products: - if "id" in product.properties and product.properties["id"] is not None: - result_dict["id"].append(product.properties["id"]) - else: - result_dict["id"].append(None) - if ( - "startTimeFromAscendingNode" in product.properties - and product.properties["startTimeFromAscendingNode"] is not None - ): - try: - result_dict["time"].append( - normalize_time(product.properties["startTimeFromAscendingNode"]) - ) - except: - result_dict["time"].append( - product.properties["startTimeFromAscendingNode"] - ) - else: - result_dict["time"].append(None) - if ( - "cloudCover" in product.properties - and product.properties["cloudCover"] is not None - ): - result_dict["cloudCover"].append(product.properties["cloudCover"]) - else: - result_dict["cloudCover"].append(None) - if ( - "productType" in product.properties - and product.properties["productType"] is not None - ): - result_dict["productType"].append(product.properties["productType"]) - else: - result_dict["productType"].append(None) + for key in result_dict: + if key == "time": + if ( + "startTimeFromAscendingNode" in product.properties + and product.properties["startTimeFromAscendingNode"] is not None + ): + try: + result_dict["time"].append( + normalize_time( + product.properties["startTimeFromAscendingNode"] + ) + ) + except: + result_dict["time"].append( + product.properties["startTimeFromAscendingNode"] + ) + else: + result_dict["time"].append(None) + else: + if key in product.properties and product.properties[key] is not None: + result_dict[key].append(product.properties[key]) + else: + result_dict[key].append(None) df = pd.DataFrame().from_dict(result_dict) return df From d1a492f4af7ed69cf1f9a4ea63b0da6e6206a6fc Mon Sep 17 00:00:00 2001 From: HamedElgizery Date: Fri, 14 Jun 2024 16:27:06 +0300 Subject: [PATCH 30/30] Add pandas to requirements --- src/imagery/i.eodag/i.eodag.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/imagery/i.eodag/i.eodag.html b/src/imagery/i.eodag/i.eodag.html index 04afe2d8f3..35c3fe859f 100644 --- a/src/imagery/i.eodag/i.eodag.html +++ b/src/imagery/i.eodag/i.eodag.html @@ -95,6 +95,8 @@

REQUIREMENTS

SEE ALSO