diff --git a/course_discovery/apps/edly_discovery_app/constants.py b/course_discovery/apps/edly_discovery_app/constants.py new file mode 100644 index 0000000000..1f071f6b16 --- /dev/null +++ b/course_discovery/apps/edly_discovery_app/constants.py @@ -0,0 +1,89 @@ +REQUIRED_MAPPING = { + "modelresult": { + "properties": { + "aggregation_key": {"type": "string", "index": "not_analyzed"}, + "announcement": {"type": "date", "format": "dateOptionalTime"}, + "authoring_organization_bodies": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "authoring_organization_uuids": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "authoring_organizations": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "authoring_organizations_autocomplete": {"type": "string", "boost": 25.0, "index_analyzer": "ngram_analyzer", "search_analyzer": "snowball_with_synonyms"}, + "authoring_organizations_exact": {"type": "string", "index": "not_analyzed"}, + "availability": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "average_rating": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "bio": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "bio_language": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "card_image_url": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "content_type": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "content_type_exact": {"type": "string", "index": "not_analyzed"}, + "course_duration_override": {"type": "long"}, + "course_key": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "course_runs": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "created": {"type": "date", "format": "dateOptionalTime"}, + "credit_backing_organizations": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "credit_backing_organizations_exact": {"type": "string", "index": "not_analyzed"}, + "designation": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "django_ct": {"type": "string", "index": "not_analyzed", "include_in_all": False}, + "django_id": {"type": "string", "index": "not_analyzed", "include_in_all": False}, + "end": {"type": "date", "format": "dateOptionalTime"}, + "enrollment_end": {"type": "date", "format": "dateOptionalTime"}, + "enrollment_start": {"type": "date", "format": "dateOptionalTime"}, + "expected_learning_items": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "featured": {"type": "boolean"}, + "first_enrollable_paid_seat_price": {"type": "long"}, + "first_enrollable_paid_seat_sku": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "full_description": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "full_name": {"type": "string", "index": "not_analyzed"}, + "get_profile_image_url": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "go_live_date": {"type": "date", "format": "dateOptionalTime"}, + "has_enrollable_paid_seats": {"type": "boolean"}, + "has_enrollable_seats": {"type": "boolean"}, + "hidden": {"type": "boolean"}, + "hidden_exact": {"type": "boolean"}, + "id": {"type": "string"}, + "image_url": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "is_current_and_still_upgradeable": {"type": "boolean"}, + "is_marketing_price_hidden": {"type": "boolean"}, + "is_marketing_price_set": {"type": "boolean"}, + "is_program_eligible_for_one_click_purchase": {"type": "boolean"}, + "key": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "language": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "language_exact": {"type": "string", "index": "not_analyzed"}, + "languages": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "level_type": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "level_type_exact": {"type": "string", "index": "not_analyzed"}, + "license": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "license_exact": {"type": "string", "index": "not_analyzed"}, + "logo_image_urls": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "marketing_id": {"type": "long"}, + "marketing_price_value": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "marketing_url": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "max_effort": {"type": "long"}, + "max_hours_effort_per_week": {"type": "long"}, + "min_effort": {"type": "long"}, + "min_hours_effort_per_week": {"type": "long"}, + "mobile_available": {"type": "boolean"}, + "mobile_available_exact": {"type": "boolean"}, + "modified": {"type": "date", "format": "dateOptionalTime"}, + "number": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "org": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "organizations": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "organizations_exact": {"type": "string", "index": "not_analyzed"}, + "outcome": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "pacing_type": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "pacing_type_exact": {"type": "string", "index": "not_analyzed"}, + "paid_seat_enrollment_end": {"type": "date", "format": "dateOptionalTime"}, + "partner": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "partner_exact": {"type": "string", "index": "not_analyzed"}, + "position": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "prerequisites": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "prerequisites_exact": {"type": "string", "index": "not_analyzed"}, + "program_types": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "published": {"type": "boolean"}, + "published_exact": {"type": "boolean"}, + "salutation": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "search_card_display": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "seat_types": {"type": "string", "analyzer": "snowball_with_synonyms"}, + "seat_types_exact": {"type": "string", "index": "not_analyzed"} + } + } +} diff --git a/course_discovery/apps/edly_discovery_app/management/commands/set_alias.py b/course_discovery/apps/edly_discovery_app/management/commands/set_alias.py new file mode 100644 index 0000000000..c95a3b88e5 --- /dev/null +++ b/course_discovery/apps/edly_discovery_app/management/commands/set_alias.py @@ -0,0 +1,78 @@ +import json +import logging + +from django.core.management import CommandError +from haystack import connections as haystack_connections + +from django.core.management.base import BaseCommand +from edly_discovery_app.constants import REQUIRED_MAPPING + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Check if all required mappings are present in the catalog file and Set alieas to latest catalog" + backends = [] + + def add_arguments(self, parser): + parser.add_argument('file_path', type=str, help="The path to the catalog JSON file (file name should match with indices name)") + + def handle(self, *args, **kwargs): + file_path = kwargs['file_path'] + + with open(file_path, 'r') as file: + catalog_data = json.load(file) + + result = self.check_mappings(catalog_data, REQUIRED_MAPPING) + + style = self.style.SUCCESS if result else self.style.ERROR + logger.info(style(f'All required mappings present: {result}')) + + if not result: + return False + + try: + self.set_alias(file_path.split('/')[-1]) + except Exception as e: + logger.info(self.style.ERROR(f'Alias are updated to given index: False')) + raise CommandError(f'ERROR exception : {e}') + + logger.info(style(f'Alias are updated to given index: True')) + + + def check_mappings(self, catalog_data, required_mappings): + catalog_key = list(catalog_data.keys())[0] + catalog_mappings = catalog_data[catalog_key]['mappings'] + error_style = self.style.ERROR + + for key, value in required_mappings.items(): + if key not in catalog_mappings: + logger.info(error_style(f'Mapping does not exist for key" {key}')) + return False + + for prop, prop_value in value['properties'].items(): + if prop not in catalog_mappings[key]['properties']: + logger.info(error_style(f'Property does not exist: "{prop}"')) + return False + + prop_name = catalog_mappings[key]['properties'][prop] + if prop_name != prop_value: + logger.info(error_style(f'Invalid value of property "{prop}": "{prop_name}"')) + return False + return True + + + def set_alias(self, index_name): + self.backends = list(haystack_connections.connections_info.keys()) + + for backend_name in self.backends: + connection = haystack_connections[backend_name] + backend = connection.get_backend() + alias = backend.index_name + body = { + 'actions': [ + {'remove': {'alias': alias, 'index': '*'}}, + {'add': {'alias': alias, 'index': index_name}}, + ] + } + backend.conn.indices.update_aliases(body) diff --git a/course_discovery/apps/edx_haystack_extensions/management/commands/update_index.py b/course_discovery/apps/edx_haystack_extensions/management/commands/update_index.py index 73003677b7..99113b6d7f 100644 --- a/course_discovery/apps/edx_haystack_extensions/management/commands/update_index.py +++ b/course_discovery/apps/edx_haystack_extensions/management/commands/update_index.py @@ -22,6 +22,14 @@ def add_arguments(self, parser): help='Disables checks limiting the number of records modified.' ) + parser.add_argument( + "-na", + "--no-update-alias", + action="store_true", + dest="update_alias_disabled", + help="Update aliases after creating new index.", + ) + def get_record_count(self, conn, index_name): return conn.count(index_name).get('count') @@ -52,17 +60,21 @@ def handle(self, **options): for backend, index, alias, record_count in alias_mappings: # Run a sanity check to ensure we aren't drastically changing the # index, which could be indicative of a bug. + update_alias = not options.get('update_alias_disabled') + if index in indexes_pending and not options.get('disable_change_limit', False): record_count_is_sane, index_info_string = self.sanity_check_new_index( backend.conn, index, record_count ) if record_count_is_sane: - self.set_alias(backend, alias, index) + if update_alias: + self.set_alias(backend, alias, index) indexes_pending.pop(index, None) else: indexes_pending[index] = index_info_string else: - self.set_alias(backend, alias, index) + if update_alias: + self.set_alias(backend, alias, index) indexes_pending.pop(index, None) if indexes_pending: