diff --git a/.dockerignore b/.dockerignore index a50ec16c2a49..10e1a2d62782 100644 --- a/.dockerignore +++ b/.dockerignore @@ -48,12 +48,7 @@ src/olympia/discovery/strings.jinja2 static-build/* static/css/node_lib/* static/js/node_lib/* -storage/files/* -storage/git-storage/* -storage/guarded-addons/* -storage/mlbf/* -storage/shared_storage/* -storage/sitemaps/* +storage tmp/* # Additionally ignore these files from the docker build that are not in .gitignore @@ -62,7 +57,6 @@ tmp/* .github docs private -storage docker-bake.hcl docker-compose*.yml Dockerfile* diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml index 98c71da8aca1..a8e6b6ce44ff 100644 --- a/.github/actions/build-docker/action.yml +++ b/.github/actions/build-docker/action.yml @@ -10,9 +10,6 @@ inputs: version: required: true description: The image version to tag with - target: - required: true - description: The stage to target in the build push: required: false description: Push the image? @@ -57,7 +54,7 @@ runs: - name: Create .env and version.json files shell: bash run: | - echo "DOCKER_TARGET=${{ inputs.target }}" >> $GITHUB_ENV + echo "DOCKER_TARGET=production" >> $GITHUB_ENV echo "DOCKER_VERSION=${{ steps.meta.outputs.version }}" >> $GITHUB_ENV echo "DOCKER_COMMIT=${{ steps.context.outputs.git_sha }}" >> $GITHUB_ENV echo "DOCKER_BUILD=${{ steps.context.outputs.git_build_url }}" >> $GITHUB_ENV diff --git a/.github/actions/run-docker/action.yml b/.github/actions/run-docker/action.yml index a1029964e295..aa0956a177bd 100644 --- a/.github/actions/run-docker/action.yml +++ b/.github/actions/run-docker/action.yml @@ -15,7 +15,7 @@ inputs: compose_file: description: 'The docker-compose file to use' required: false - default: 'docker-compose.yml:docker-compose.ci.yml' + default: 'docker-compose.yml' logs: description: 'Show logs' required: false diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 260e220e50ee..d77f71045bb4 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -41,34 +41,28 @@ jobs: - name: Needs Locale Compilation services: '' - compose_file: docker-compose.yml:docker-compose.ci.yml run: | make compile_locales make test_needs_locales_compilation - name: Static Assets services: '' - compose_file: docker-compose.yml:docker-compose.ci.yml run: make test_static_assets - name: Internal Routes services: '' - compose_file: docker-compose.yml:docker-compose.ci.yml run: make test_internal_routes_allowed - name: Elastic Search services: '' - compose_file: docker-compose.yml:docker-compose.ci.yml run: make test_es_tests - name: Codestyle services: web - compose_file: docker-compose.yml:docker-compose.ci.yml run: make lint-codestyle - name: Manage Check services: web nginx - compose_file: docker-compose.yml:docker-compose.ci.yml run: make check steps: - uses: actions/checkout@v4 @@ -78,5 +72,4 @@ jobs: version: ${{ inputs.version }} digest: ${{ inputs.digest }} services: ${{ matrix.services }} - compose_file: ${{ matrix.compose_file }} run: ${{ matrix.run }} diff --git a/.github/workflows/_test_main.yml b/.github/workflows/_test_main.yml index a58541c48a90..56576b5b4417 100644 --- a/.github/workflows/_test_main.yml +++ b/.github/workflows/_test_main.yml @@ -88,7 +88,7 @@ jobs: services: '' digest: ${{ inputs.digest }} version: ${{ inputs.version }} - compose_file: docker-compose.yml + compose_file: docker-compose.yml:docker-compose.dev.yml run: | split="--splits ${{ needs.test_config.outputs.splits }}" group="--group ${{ matrix.group }}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2304bd4d0a80..db0da68cff47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,6 @@ jobs: registry: ${{ steps.docker_hub.outputs.registry }} image: ${{ steps.docker_hub.outputs.image }} version: ci-${{ needs.context.outputs.docker_version }} - target: development push: true test_make_docker_configuration: @@ -88,7 +87,7 @@ jobs: shell: bash run: | docker compose version - npm exec jest -- ./tests/make --runInBand + make test_setup test_run_docker_action: runs-on: ubuntu-latest @@ -158,6 +157,14 @@ jobs: exit 1 fi + - name: Test setup + uses: ./.github/actions/run-docker + with: + digest: ${{ needs.build.outputs.digest }} + version: ${{ needs.build.outputs.version }} + run: | + pytest tests/make/ + docs_build: runs-on: ubuntu-latest needs: build @@ -171,7 +178,7 @@ jobs: with: digest: ${{ needs.build.outputs.digest }} version: ${{ needs.build.outputs.version }} - compose_file: docker-compose.yml + compose_file: docker-compose.yml:docker-compose.dev.yml run: | make docs @@ -222,7 +229,7 @@ jobs: with: digest: ${{ needs.build.outputs.digest }} version: ${{ needs.build.outputs.version }} - compose_file: docker-compose.yml + compose_file: docker-compose.yml:docker-compose.dev.yml run: make extract_locales - name: Push Locales @@ -289,7 +296,6 @@ jobs: registry: ${{ steps.docker_hub.outputs.registry }} image: ${{ steps.docker_hub.outputs.image }} version: ${{ needs.context.outputs.docker_version }} - target: production push: true push_gar: @@ -321,5 +327,4 @@ jobs: registry: ${{ steps.docker_gar.outputs.registry }} image: ${{ steps.docker_gar.outputs.image }} version: ${{ needs.context.outputs.docker_version }} - target: production push: true diff --git a/.gitignore b/.gitignore index ee75887a9920..18885398f5da 100644 --- a/.gitignore +++ b/.gitignore @@ -47,17 +47,12 @@ src/olympia/discovery/strings.jinja2 static-build/* static/css/node_lib/* static/js/node_lib/* -storage/files/* -storage/git-storage/* -storage/guarded-addons/* -storage/mlbf/* -storage/shared_storage/* -storage/sitemaps/* +storage tmp/* # End of .gitignore. Please keep this in sync with the top section of .dockerignore # do not ignore the following files -!docker-compose.development.yml +!docker-compose.dev.yml !docker-compose.private.yml !private/README.md diff --git a/Dockerfile b/Dockerfile index a1e58b08ab26..76572d20d488 100644 --- a/Dockerfile +++ b/Dockerfile @@ -96,6 +96,10 @@ EOF ARG DOCKER_TARGET ENV DOCKER_TARGET=${DOCKER_TARGET} +# Add our custom mime types (required for for ts/json/md files) +COPY docker/etc/mime.types /etc/mime.types + + # Define production dependencies as a single layer # let's the rest of the stages inherit prod dependencies # and makes copying the /deps dir to the final layer easy. @@ -115,7 +119,10 @@ ${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r require npm ci ${NPM_ARGS} --include=prod EOF -FROM base AS pip_development +# In the local development image, we stop with dependencies +# Everything else is about compiling production assets which is not needed +# in the typical development workflow. +FROM base AS development RUN \ # Files required to install pip dependencies @@ -127,11 +134,7 @@ RUN \ # Mounts for caching dependencies --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \ --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \ -< settings_local.py DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets EOF -FROM base AS sources +FROM base AS production ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION @@ -176,27 +179,15 @@ ENV DOCKER_BUILD=${DOCKER_BUILD} ENV DOCKER_COMMIT=${DOCKER_COMMIT} ENV DOCKER_VERSION=${DOCKER_VERSION} -# Add our custom mime types (required for for ts/json/md files) -COPY docker/etc/mime.types /etc/mime.types # Copy the rest of the source files from the host COPY --chown=olympia:olympia . ${HOME} +# Copy compiled locales from builder +COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale # Copy assets from assets COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build - -# Set shell back to sh until we can prove we can use bash at runtime -SHELL ["/bin/sh", "-c"] - -FROM sources AS development - -# Copy dependencies from `pip_development` -COPY --from=pip_development --chown=olympia:olympia /deps /deps - -FROM sources AS production - -# Copy compiled locales from builder -COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale # Copy dependencies from `pip_production` COPY --from=pip_production --chown=olympia:olympia /deps /deps - +# Set shell back to sh until we can prove we can use bash at runtime +SHELL ["/bin/sh", "-c"] diff --git a/Makefile-docker b/Makefile-docker index 05d87b8fcf85..40bec2906502 100644 --- a/Makefile-docker +++ b/Makefile-docker @@ -76,6 +76,11 @@ update_assets: # Collect static files: This MUST be run last or files will be missing $(PYTHON_COMMAND) manage.py collectstatic --noinput +.PHONY: update_deps +update_deps: + $(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt + $(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt + npm install $(NPM_ARGS) --no-save # TOOD: remove this after we migrate addons-frontned to not depend on it. .PHONY: setup-ui-tests diff --git a/Makefile-os b/Makefile-os index fa981627a307..4694c644d51f 100644 --- a/Makefile-os +++ b/Makefile-os @@ -68,6 +68,10 @@ help_submake: @echo "\nAll other commands will be passed through to the docker 'web' container make:" @make -f Makefile-docker help_submake +.PHONY: test_setup +test_setup: + npm exec jest -- ./tests/make --runInBand + .PHONY: setup setup: ## create configuration files version.json and .env required to run this project for path in $(CLEAN_PATHS); do rm -rf "$(PWD)/$$path" && echo "$$path removed"; done diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml deleted file mode 100644 index 72863e437745..000000000000 --- a/docker-compose.ci.yml +++ /dev/null @@ -1,13 +0,0 @@ -services: - worker: - environment: - - HOST_UID=9500 - volumes: - - /data/olympia - - web: - extends: - service: worker - -volumes: - data_olympia: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 000000000000..09dacd10fdba --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,17 @@ +services: + worker: + environment: + - HOST_UID + volumes: + - ./:/data/olympia + # Don't mount generated files. They only exist in the container + # and would otherwiser be deleted by mounbting data_olympia + - /data/olympia/static-build + - /data/olympia/site-static + - ./package.json:/deps/package.json + - ./package-lock.json:/deps/package-lock.json + + web: + extends: + service: worker + diff --git a/docker-compose.yml b/docker-compose.yml index 27eb9eca3eff..48bba48cffcb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ x-env-mapping: &env - HISTIGNORE=ls:exit:"cd .." - HISTCONTROL=erasedups - CIRCLECI - - HOST_UID + - HOST_UID=9500 - DEBUG - DATA_BACKUP_SKIP @@ -51,15 +51,6 @@ services: "--", "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO", ] - volumes: - - data_olympia:/data/olympia - # Don't mount generated files. They only exist in the container - # and would otherwiser be deleted by mounbting data_olympia - - /data/olympia/static-build - - /data/olympia/site-static - - storage:/data/olympia/storage - - ./package.json:/deps/package.json - - ./package-lock.json:/deps/package-lock.json extra_hosts: - "olympia.test:127.0.0.1" restart: on-failure:5 @@ -99,8 +90,7 @@ services: image: nginx volumes: - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf - - ./static:/srv/static - - storage:/srv/user-media + - ./:/srv ports: - "80:80" networks: @@ -143,13 +133,9 @@ services: - "discovery.type=single-node" - "ES_JAVA_OPTS=-Xms512m -Xmx512m" mem_limit: 2g - volumes: - - data_elasticsearch:/usr/share/elasticsearch/data redis: image: redis:6.2 - volumes: - - data_redis:/data rabbitmq: image: rabbitmq:3.12 @@ -160,8 +146,6 @@ services: - RABBITMQ_DEFAULT_USER=olympia - RABBITMQ_DEFAULT_PASS=olympia - RABBITMQ_DEFAULT_VHOST=olympia - volumes: - - data_rabbitmq:/var/lib/rabbitmq autograph: image: mozilla/autograph:3.3.2 @@ -191,23 +175,10 @@ networks: default: volumes: - data_redis: - data_elasticsearch: data_mysqld: # Keep this value in sync with Makefile-os # External volumes must be manually created/destroyed name: addons-server_data_mysqld external: true - data_rabbitmq: - data_olympia: - driver: local - driver_opts: - type: none - o: bind - device: ${PWD} - storage: - driver: local - driver_opts: - type: none - o: bind - device: ./storage + + diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a48da2312246..2e80966b2ec7 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -23,6 +23,17 @@ if [[ -n "${HOST_UID:-}" ]]; then echo "${OLYMPIA_USER} UID: ${OLYMPIA_UID} -> ${HOST_UID}" fi +# TODO: this doesn't work but we have to check if the image we built is production +# but we are for some reason trying to run it using development dependencies. +# this is pretty typical for CI... we could consider just building ci images though as a simpler solution. It's just nice to test the image you already pushed cause muh CI/CD... +if [[ "${DOCKER_VERSION:-}" != 'local' && "${DOCKER_TARGET:-}" == 'production' ]]; then + echo "Running remote image. Re-installing with development dependencies..." + rm -rf /deps + mkdir -p /deps + chown -R ${get_olympia_uid}:${get_olympia_gid} /deps + su -s /bin/bash $OLYMPIA_USER -c "make update_deps" +fi + cat <` it will only run the commands that are necessary to bring your environment up to date. If nothing has changed, nothing will happen because your environment is already in the desired state. +### Make up OPTIONS + +Make up can be run with several arguments to configure how the proejct is run. + +- **COMPOSE_FILE**: The compose file to use. default: `docker-compose.yml:docker-compose.dev.yml` + Specifies which compose files to use during docker compose commands. +- **DOCKER_VERSION**: The version of the docker image to use. default: `local` to build the image local. + Otherwise it will pull the image from dockerhub with the specified version. +- **DOCKER_DIGEST**: (overrides `DOCKER_VERSION`) The digest of the docker image to use. + This is useful to run a very specific build of addons-server, e.g. from a specific CI run. +- **DOCKER_TAG**: The tag of the docker image to use. This is useful if you already know the full tag or + want to run an image not on dockerhub. +- **DOCKER_TARGET**: The target of the docker image to use. default: `development` to run the development target. + Set to `production` to run the production target. This is the exact same image we use in production. +- **DEBUG**: The debug mode to use. default: `True` if `DOCKER_TARGET` is `development` else `False`. + This enables certain settings in django to make debugging easier. Works in development or production mode. + However, debug_toolbar only works if both `DEBUG` is true and `DOCKER_TARGET` is set to `development`. + +Here are some common use cases: + +1. True production mode: `make up DOCKER_TARGET=production DEBUG=False COMPOSE_FILE=docker-compose.yml` +2. development mode: `make up DOCKER_TARGET=development DEBUG=True COMPOSE_FILE=docker-compose.yml:docker-compose.dev.yml` + +When you run make up, many of the options are saved to your .env file and will be reused on subsequent calls to make up. +This allows you to customize your environment to your liking without having to specify the same values over and over. + +If you want to reset your .env file to the defaults, you can remove the `.env` file and run `make up` again. +This will create a new .env file with the default values or with whatever values you override with. + ## Shutting down your environment > TLDR; just run `make down` @@ -187,7 +216,7 @@ Though it is **highly recommended to use the make commands** instead of directly ### Docker Compose Files - **[docker-compose.yml][docker-compose]**: The primary Docker Compose file defining services, networks, and volumes for local and CI environments. -- **[docker-compose.ci.yml][docker-compose-ci]**: Overrides certain configurations for CI-specific needs, ensuring the environment is optimized for automated testing and builds. +- **[docker-compose.dev.yml][docker-compose-dev]**: Overrides certain configurations for local development needs, ensuring the environment is optimized for local development. - **[docker-compose.private.yml][docker-compose-private]**: Runs addons-server with the _customs_ service that is only available to Mozilla employees Our docker compose files rely on substituted values, all of which are included in our .env file for direct CLI compatibility. @@ -317,7 +346,7 @@ and docker-comose.yml file locally. To fix this error `rm -f .env` to remove your .env and `make up` to restart the containers. [docker-compose]: ../../../docker-compose.yml -[docker-compose-ci]: ../../../docker-compose.ci.yml +[docker-compose-dev]: ../../../docker-compose.dev.yml [docker-compose-private]: ../../../docker-compose.private.yml [docker-image-digest]: https://github.com/opencontainers/.github/blob/main/docs/docs/introduction/digests.md [addons-server-tags]: https://hub.docker.com/r/mozilla/addons-server/tags diff --git a/scripts/setup.py b/scripts/setup.py index 8508dfd8ba7a..b730e7acb869 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -22,39 +22,45 @@ def get_env_file(): return env -env = get_env_file() - def get_value(key, default_value): if key in os.environ: return os.environ[key] - if key in env: - return env[key] + from_file = get_env_file() + + if key in from_file: + return from_file[key] return default_value def get_docker_tag(): - image_name = 'mozilla/addons-server' - version = os.environ.get('DOCKER_VERSION') - digest = os.environ.get('DOCKER_DIGEST') - - tag = f'{image_name}:local' - - if digest: - tag = f'{image_name}@{digest}' - elif version: - tag = f'{image_name}:{version}' - else: - tag = get_value('DOCKER_TAG', tag) - # extract version or digest from existing tag - if '@' in tag: - digest = tag.split('@')[1] - elif ':' in tag: - version = tag.split(':')[1] - - print('Docker tag: ', tag) + image = 'mozilla/addons-server' + version = 'local' + + # First get the tag from the full tag variable + tag = get_value('DOCKER_TAG', f'{image}:{version}') + # extract version or digest from existing tag + if '@' in tag: + image, digest = tag.split('@') + version = None + elif ':' in tag: + image, version = tag.split(':') + digest = None + + # DOCKER_DIGEST or DOCKER_VERSION can override the extracted version or digest + # Note: it will inherit the image from the provided DOCKER_TAG if also provided + if bool(os.environ.get('DOCKER_DIGEST', False)): + digest = os.environ['DOCKER_DIGEST'] + tag = f'{image}@{digest}' + version = None + elif bool(os.environ.get('DOCKER_VERSION', False)): + version = os.environ['DOCKER_VERSION'] + tag = f'{image}:{version}' + digest = None + + print('tag: ', tag) print('version: ', version) print('digest: ', digest) @@ -74,16 +80,39 @@ def get_docker_tag(): # 3. the value defined in the environment variable # 4. the value defined in the make args. -docker_tag, docker_version, docker_digest = get_docker_tag() - -docker_target = get_value('DOCKER_TARGET', 'development') -compose_file = get_value('COMPOSE_FILE', ('docker-compose.yml')) - -set_env_file( - { - 'COMPOSE_FILE': compose_file, - 'DOCKER_TAG': docker_tag, - 'DOCKER_TARGET': docker_target, - 'HOST_UID': get_value('HOST_UID', os.getuid()), - } -) +def main(): + docker_tag, docker_version, _ = get_docker_tag() + + is_local = docker_version == 'local' + + # Local images can freely choose the target stage to build + docker_target = get_value('DOCKER_TARGET', ( + 'development' if is_local else 'production' + )) + + is_production = docker_target == 'production' + + # The default value for which compose files to use is based on the target + # but can be freely overridden by the user. E.g verifying a patch against a production image + compose_file = get_value('COMPOSE_FILE', ( + 'docker-compose.yml' if is_production else + 'docker-compose.yml:docker-compose.dev.yml' + )) + + # DEBUG is special, as we should allow the user to override it + # but we should not set a default to the previously set value but instead + # to the most sensible default. + debug = os.environ.get('DEBUG', str(False if is_production else True)) + + set_env_file( + { + 'COMPOSE_FILE': compose_file, + 'DOCKER_TAG': docker_tag, + 'DOCKER_TARGET': docker_target, + 'HOST_UID': get_value('HOST_UID', os.getuid()), + 'DEBUG': debug, + } + ) + +if __name__ == '__main__': + main() diff --git a/src/olympia/amo/management/commands/data_seed.py b/src/olympia/amo/management/commands/data_seed.py index 713504cb80d0..fdf4bfeeb2a2 100644 --- a/src/olympia/amo/management/commands/data_seed.py +++ b/src/olympia/amo/management/commands/data_seed.py @@ -1,3 +1,5 @@ +import os +import shutil from django.conf import settings from django.core.management import call_command @@ -10,6 +12,36 @@ class Command(BaseDataCommand): 'generated add-ons, and data from AMO production.' ) + def _clean_storage(self, root: str, dir_dict: dict[str, str | dict]) -> None: + for key, value in dir_dict.items(): + curr_path = os.path.join(root, key) + if isinstance(value, dict): + self._clean_storage(curr_path, value) + else: + shutil.rmtree(curr_path, ignore_errors=True) + os.makedirs(curr_path, exist_ok=True) + + def clean_storage(self): + self.logger.info('Cleaning storage...') + self._clean_storage(settings.STORAGE_ROOT, { + 'files': '', + 'shared_storage': { + 'tmp': { + 'addons': '', + 'data': '', + 'file_viewer': '', + 'guarded-addons': '', + 'icon': '', + 'log': '', + 'persona_header': '', + 'preview': '', + 'test': '', + 'uploads': '', + }, + 'uploads': '', + } + }) + def handle(self, *args, **options): num_addons = 10 num_themes = 5 @@ -18,6 +50,7 @@ def handle(self, *args, **options): self.logger.info('Resetting database...') call_command('flush', '--noinput') + self.clean_storage() # reindex --wipe will force the ES mapping to be re-installed. call_command('reindex', '--wipe', '--force', '--noinput') call_command('migrate', '--noinput') @@ -43,4 +76,4 @@ def handle(self, *args, **options): call_command('generate_default_addons_for_frontend') call_command('data_dump', '--name', self.data_backup_init) - call_command('reindex', '--wipe', '--force', '--noinput') + call_command('data_load', '--name', self.data_backup_init) diff --git a/src/olympia/amo/management/commands/initialize.py b/src/olympia/amo/management/commands/initialize.py index 4976ef9190a9..420a010d7796 100644 --- a/src/olympia/amo/management/commands/initialize.py +++ b/src/olympia/amo/management/commands/initialize.py @@ -58,4 +58,4 @@ def handle(self, *args, **options): # We should reindex even if no data is loaded/modified # because we might have a fresh instance of elasticsearch else: - call_command('reindex', '--wipe', '--force', '--noinput') + call_command('reindex', '--noinput', '--skip-if-exists') diff --git a/src/olympia/amo/tests/test_commands.py b/src/olympia/amo/tests/test_commands.py index 9a09ca44c6fc..8b105f7855b3 100644 --- a/src/olympia/amo/tests/test_commands.py +++ b/src/olympia/amo/tests/test_commands.py @@ -344,7 +344,8 @@ class Commands: data_seed = mock.call('data_seed') flush = mock.call('flush', '--noinput') - reindex = mock.call('reindex', '--wipe', '--force', '--noinput') + reindex_force_wipe = mock.call('reindex', '--wipe', '--force', '--noinput') + reindex_skip_if_exists = mock.call('reindex', '--noinput', '--skip-if-exists') load_initial_data = mock.call('loaddata', 'initial.json') import_prod_versions = mock.call('import_prod_versions') createsuperuser = mock.call( @@ -504,7 +505,7 @@ def test_handle_without_clean_or_load_with_local_admin(self): self.mocks['mock_call_command'], [ self.mock_commands.migrate, - self.mock_commands.reindex, + self.mock_commands.reindex_skip_if_exists, ], ) @@ -709,7 +710,7 @@ def test_loads_correct_path(self, mock_exists): [ self.mock_commands.db_restore(db_path), self.mock_commands.media_restore(storage_path), - self.mock_commands.reindex, + self.mock_commands.reindex_force_wipe, ], ) @@ -760,7 +761,7 @@ def test_default(self): self.mocks['mock_call_command'], [ self.mock_commands.flush, - self.mock_commands.reindex, + self.mock_commands.reindex_force_wipe, self.mock_commands.migrate, self.mock_commands.load_initial_data, self.mock_commands.import_prod_versions, @@ -771,6 +772,6 @@ def test_default(self): self.mock_commands.generate_themes(5), self.mock_commands.generate_default_addons_for_frontend, self.mock_commands.data_dump(self.base_data_command.data_backup_init), - self.mock_commands.reindex, + self.mock_commands.data_load(self.base_data_command.data_backup_init), ], ) diff --git a/src/olympia/amo/utils.py b/src/olympia/amo/utils.py index 3ddd68a969cc..c31557af8825 100644 --- a/src/olympia/amo/utils.py +++ b/src/olympia/amo/utils.py @@ -1165,7 +1165,7 @@ def extract_colors_from_image(path): def use_fake_fxa(): """Return whether or not to use a fake FxA server for authentication. Should always return False in production""" - return settings.DEV_MODE and settings.USE_FAKE_FXA_AUTH + return settings.DEV_MODE or settings.USE_FAKE_FXA_AUTH class AMOJSONEncoder(JSONEncoder): diff --git a/src/olympia/search/management/commands/reindex.py b/src/olympia/search/management/commands/reindex.py index 69b8ebcc1bd5..5e05fe0579f7 100644 --- a/src/olympia/search/management/commands/reindex.py +++ b/src/olympia/search/management/commands/reindex.py @@ -117,6 +117,12 @@ def add_arguments(self, parser): help=('Do not ask for confirmation before wiping. Default: False'), default=False, ) + parser.add_argument( + '--skip-if-exists', + action='store_true', + help=('Skip the reindex if the index already exists.'), + default=False, + ) def accepted_keys(self): return ', '.join(settings.ES_INDEXES.keys()) @@ -129,7 +135,6 @@ def handle(self, *args, **kwargs): """ force = kwargs['force'] - if Reindexing.objects.is_reindexing() and not force: raise CommandError('Indexation already occurring - use --force to bypass') @@ -141,6 +146,10 @@ def handle(self, *args, **kwargs): ) self.stdout.write('Starting the reindexation for %s.' % alias) + if kwargs['skip_if_exists'] and ES.indices.exists(index=alias): + self.stdout.write('Index %s already exists. Skipping reindex.' % alias) + return + if kwargs['wipe']: skip_confirmation = kwargs['noinput'] confirm = '' diff --git a/src/olympia/search/tests/test_commands.py b/src/olympia/search/tests/test_commands.py index 648bd00e9829..e04353141f3d 100644 --- a/src/olympia/search/tests/test_commands.py +++ b/src/olympia/search/tests/test_commands.py @@ -97,7 +97,7 @@ def get_indices_aliases(cls): items.sort() return items - def _test_reindexation(self, wipe=False): + def _test_reindexation(self, wipe=False, skip_if_exists=False): # Current indices with aliases. old_indices = self.get_indices_aliases() @@ -113,7 +113,7 @@ def run(self): # alias in setUpClass. time.sleep(1) management.call_command( - 'reindex', wipe=wipe, noinput=True, stdout=self.stdout + 'reindex', wipe=wipe, noinput=True, skip_if_exists=skip_if_exists, stdout=self.stdout ) t = ReindexThread() @@ -125,6 +125,10 @@ def run(self): while t.is_alive() and not Reindexing.objects.is_reindexing(): connection._commit() + if skip_if_exists: + assert not Reindexing.objects.is_reindexing() + return + if not wipe: # We should still be able to search in the foreground while the # reindex is being done in the background. We should also be able @@ -229,3 +233,12 @@ def test_create_workflow_addons(self): for addons. """ self._test_workflow('default') + + @mock.patch('olympia.search.management.commands.reindex.ES') + def test_reindex_skip_if_exists(self, mock_es): + """ + Test reindex command with skip_if_exists option when index already exists. + """ + mock_es.indices.exists.return_value = True + self._test_reindexation(skip_if_exists=True) + assert mock_es.indices.create.call_count == 0 diff --git a/src/olympia/urls.py b/src/olympia/urls.py index d683cd185218..fc4e967243c9 100644 --- a/src/olympia/urls.py +++ b/src/olympia/urls.py @@ -2,6 +2,7 @@ from django.contrib import admin from django.shortcuts import redirect from django.urls import include, re_path, reverse +from django.views.i18n import JavaScriptCatalog from django.views.static import serve as serve_static from olympia.amo.utils import urlparams @@ -132,6 +133,13 @@ def serve_static_files(request, path, **kwargs): serve_static, {'document_root': settings.MEDIA_ROOT}, ), + # Specific fallback for statically generated i18n js files + # Not generated in dev mode builds + re_path( + r'^static/js/i18n/.*\.js$', + JavaScriptCatalog.as_view(), + name='js-i18n-catalog', + ), # fallback for static files that are not available directly over nginx. # Mostly vendor files from python or npm dependencies that are not available # in the static files directory. diff --git a/storage/files/.gitkeep b/storage/files/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/storage/shared_storage/tmp/addons/.gitignore b/storage/shared_storage/tmp/addons/.gitignore deleted file mode 100644 index 72e8ffc0db8a..000000000000 --- a/storage/shared_storage/tmp/addons/.gitignore +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/storage/shared_storage/tmp/data/.gitkeep b/storage/shared_storage/tmp/data/.gitkeep deleted file mode 100644 index 9096300f21ac..000000000000 --- a/storage/shared_storage/tmp/data/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -stub file to create an empty dir diff --git a/storage/shared_storage/tmp/file_viewer/.gitkeep b/storage/shared_storage/tmp/file_viewer/.gitkeep deleted file mode 100644 index 9096300f21ac..000000000000 --- a/storage/shared_storage/tmp/file_viewer/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -stub file to create an empty dir diff --git a/storage/shared_storage/tmp/guarded-addons/.gitkeep b/storage/shared_storage/tmp/guarded-addons/.gitkeep deleted file mode 100644 index 9096300f21ac..000000000000 --- a/storage/shared_storage/tmp/guarded-addons/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -stub file to create an empty dir diff --git a/storage/shared_storage/tmp/icon/.gitkeep b/storage/shared_storage/tmp/icon/.gitkeep deleted file mode 100644 index 9096300f21ac..000000000000 --- a/storage/shared_storage/tmp/icon/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -stub file to create an empty dir diff --git a/storage/shared_storage/tmp/log/.gitkeep b/storage/shared_storage/tmp/log/.gitkeep deleted file mode 100644 index 9096300f21ac..000000000000 --- a/storage/shared_storage/tmp/log/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -stub file to create an empty dir diff --git a/storage/shared_storage/tmp/persona_header/.gitkeep b/storage/shared_storage/tmp/persona_header/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/storage/shared_storage/tmp/preview/.gitkeep b/storage/shared_storage/tmp/preview/.gitkeep deleted file mode 100644 index 9096300f21ac..000000000000 --- a/storage/shared_storage/tmp/preview/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -stub file to create an empty dir diff --git a/storage/shared_storage/tmp/test/.gitkeep b/storage/shared_storage/tmp/test/.gitkeep deleted file mode 100644 index 9096300f21ac..000000000000 --- a/storage/shared_storage/tmp/test/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -stub file to create an empty dir diff --git a/storage/shared_storage/tmp/uploads/.gitkeep b/storage/shared_storage/tmp/uploads/.gitkeep deleted file mode 100644 index 9096300f21ac..000000000000 --- a/storage/shared_storage/tmp/uploads/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -stub file to create an empty dir diff --git a/storage/shared_storage/uploads/.check b/storage/shared_storage/uploads/.check deleted file mode 100644 index d86bac9de59a..000000000000 --- a/storage/shared_storage/uploads/.check +++ /dev/null @@ -1 +0,0 @@ -OK diff --git a/storage/shared_storage/uploads/.gitkeep b/storage/shared_storage/uploads/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/make/make.spec.js b/tests/make/make.spec.js index 859cae69208c..a170e2e63b00 100644 --- a/tests/make/make.spec.js +++ b/tests/make/make.spec.js @@ -8,24 +8,19 @@ const rootPath = path.join(__dirname, '..', '..'); const envPath = path.join(rootPath, '.env'); function runSetup(env) { + fs.writeFileSync(envPath, ''); spawnSync('make', ['setup'], { env: { ...process.env, ...env }, encoding: 'utf-8', }); -} - -function readEnvFile(name) { - return parse(fs.readFileSync(envPath, { encoding: 'utf-8' }))[name]; + return parse(fs.readFileSync(envPath, { encoding: 'utf-8' })) } test('map docker compose config', () => { - const values = { + values = runSetup({ DOCKER_VERSION: 'version', HOST_UID: 'uid', - }; - - fs.writeFileSync(envPath, ''); - runSetup(values); + }); const { stdout: rawConfig } = spawnSync( 'docker', @@ -37,18 +32,29 @@ test('map docker compose config', () => { const { web } = config.services; expect(web.image).toStrictEqual( - `mozilla/addons-server:${values.DOCKER_VERSION}`, + `mozilla/addons-server:version`, ); expect(web.platform).toStrictEqual('linux/amd64'); - expect(web.environment.HOST_UID).toStrictEqual(values.HOST_UID); + expect(web.environment.HOST_UID).toStrictEqual('9500'); expect(config.volumes.data_mysqld.name).toStrictEqual( 'addons-server_data_mysqld', ); + + const { stdout: rawConfigDev } = spawnSync( + 'docker', + ['compose', 'config', '--format', 'json'], + { + encoding: 'utf-8', + env: { ...process.env, ...{ COMPOSE_FILE: 'docker-compose.yml:docker-compose.dev.yml' } }, + }, + ); + + const configDev = JSON.parse(rawConfigDev); + const { web: webDev } = configDev.services; }); describe('docker-bake.hcl', () => { function getBakeConfig(env = {}) { - fs.writeFileSync(envPath, ''); runSetup(env); const { stdout: output } = spawnSync( 'make', @@ -102,129 +108,3 @@ describe('docker-bake.hcl', () => { }); }); -function standardPermutations(name, defaultValue) { - return [ - { - name, - file: undefined, - env: undefined, - expected: defaultValue, - }, - { - name, - file: 'file', - env: undefined, - expected: 'file', - }, - { - name, - file: undefined, - env: 'env', - expected: 'env', - }, - { - name, - file: 'file', - env: 'env', - expected: 'env', - }, - ]; -} - -describe.each([ - { - version: undefined, - digest: undefined, - tag: undefined, - expected: 'mozilla/addons-server:local', - }, - { - version: 'version', - digest: undefined, - tag: undefined, - expected: 'mozilla/addons-server:version', - }, - { - version: undefined, - digest: 'sha256:digest', - tag: undefined, - expected: 'mozilla/addons-server@sha256:digest', - }, - { - version: 'version', - digest: 'sha256:digest', - tag: undefined, - expected: 'mozilla/addons-server@sha256:digest', - }, - { - version: 'version', - digest: 'sha256:digest', - tag: 'previous', - expected: 'mozilla/addons-server@sha256:digest', - }, - { - version: undefined, - digest: undefined, - tag: 'previous', - expected: 'previous', - }, -])('DOCKER_TAG', ({ version, digest, tag, expected }) => { - it(`version:${version}_digest:${digest}_tag:${tag}`, () => { - fs.writeFileSync(envPath, ''); - runSetup({ - DOCKER_VERSION: version, - DOCKER_DIGEST: digest, - DOCKER_TAG: tag, - }); - - const actual = readEnvFile('DOCKER_TAG'); - expect(actual).toStrictEqual(expected); - }); -}); - -const testCases = [ - ...standardPermutations('DOCKER_TAG', 'mozilla/addons-server:local'), - ...standardPermutations('DOCKER_TARGET', 'development'), - ...standardPermutations('HOST_UID', process.getuid().toString()), - ...standardPermutations('COMPOSE_FILE', 'docker-compose.yml'), -]; - -describe.each(testCases)('.env file', ({ name, file, env, expected }) => { - it(`name:${name}_file:${file}_env:${env}`, () => { - fs.writeFileSync(envPath, file ? `${name}=${file}` : ''); - - runSetup({ [name]: env }); - - const actual = readEnvFile(name); - expect(actual).toStrictEqual(expected); - }); -}); - -const testedKeys = new Set(testCases.map(({ name }) => name)); - -// Keys testsed outside the scope of testCases -const skippedKeys = ['DOCKER_COMMIT', 'DOCKER_VERSION', 'DOCKER_BUILD', 'PWD']; - -test('All dynamic properties in any docker compose file are referenced in the test', () => { - const composeFiles = globSync('docker-compose*.yml', { cwd: rootPath }); - const variableDefinitions = []; - - for (let file of composeFiles) { - const fileContent = fs.readFileSync(path.join(rootPath, file), { - encoding: 'utf-8', - }); - - for (let line of fileContent.split('\n')) { - const regex = /\${(.*?)(?::-.*)?}/g; - let match; - while ((match = regex.exec(line)) !== null) { - const variable = match[1]; - if (!skippedKeys.includes(variable)) variableDefinitions.push(variable); - } - } - } - - for (let variable of variableDefinitions) { - expect(testedKeys).toContain(variable); - } -}); diff --git a/tests/make/test_setup.py b/tests/make/test_setup.py new file mode 100644 index 000000000000..e5390bfc0ec9 --- /dev/null +++ b/tests/make/test_setup.py @@ -0,0 +1,170 @@ +import os +import unittest +from unittest import mock + +from scripts.setup import get_docker_tag, main + +def override_env(**kwargs): + return mock.patch.dict(os.environ, kwargs, clear=True) + +keys = ['COMPOSE_FILE', 'DOCKER_TAG', 'DOCKER_TARGET', 'HOST_UID', 'DEBUG'] + + +class BaseTestClass(unittest.TestCase): + def assert_set_env_file_called_with(self, **kwargs): + expected = { key: kwargs.get(key, mock.ANY) for key in keys } + assert mock.call(expected) in self.mock_set_env_file.call_args_list + + def setUp(self): + patch = mock.patch('scripts.setup.set_env_file') + self.addCleanup(patch.stop) + self.mock_set_env_file = patch.start() + + patch_two = mock.patch('scripts.setup.get_env_file', return_value={}) + self.addCleanup(patch_two.stop) + self.mock_get_env_file = patch_two.start() + + +@override_env() +class TestGetDockerTag(BaseTestClass): + def test_default_value_is_local(self): + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'mozilla/addons-server:local') + self.assertEqual(version, 'local') + self.assertEqual(digest, None) + + @override_env(DOCKER_VERSION="test") + def test_version_overrides_default(self): + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'mozilla/addons-server:test') + self.assertEqual(version, 'test') + self.assertEqual(digest, None) + + @override_env(DOCKER_DIGEST="sha256:123") + def test_digest_overrides_version_and_default(self): + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'mozilla/addons-server@sha256:123') + self.assertEqual(version, None) + self.assertEqual(digest, 'sha256:123') + + with override_env(DOCKER_VERSION="test", DOCKER_DIGEST="sha256:123"): + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'mozilla/addons-server@sha256:123') + self.assertEqual(version, None) + self.assertEqual(digest, 'sha256:123') + + @override_env(DOCKER_TAG="image:latest") + def test_tag_overrides_default_version(self): + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'image:latest') + self.assertEqual(version, 'latest') + self.assertEqual(digest, None) + + with override_env(DOCKER_TAG="image:latest", DOCKER_VERSION="test"): + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'image:test') + self.assertEqual(version, 'test') + self.assertEqual(digest, None) + + @override_env(DOCKER_TAG="image@sha256:123") + def test_tag_overrides_default_digest(self): + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'image@sha256:123') + self.assertEqual(version, None) + self.assertEqual(digest, 'sha256:123') + + with mock.patch.dict(os.environ, {"DOCKER_DIGEST": "test"}): + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'image@test') + self.assertEqual(version, None) + self.assertEqual(digest, 'test') + + def test_version_from_env_file(self): + self.mock_get_env_file.return_value = {'DOCKER_TAG': 'image:latest'} + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'image:latest') + self.assertEqual(version, 'latest') + self.assertEqual(digest, None) + + def test_digest_from_env_file(self): + self.mock_get_env_file.return_value = {'DOCKER_TAG': 'image@sha256:123'} + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'image@sha256:123') + self.assertEqual(version, None) + self.assertEqual(digest, 'sha256:123') + + @override_env(DOCKER_VERSION='') + def test_default_when_version_is_empty(self): + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'mozilla/addons-server:local') + self.assertEqual(version, 'local') + self.assertEqual(digest, None) + + @override_env(DOCKER_DIGEST='') + def test_default_when_digest_is_empty(self): + self.mock_get_env_file.return_value = {'DOCKER_TAG': 'image@sha256:123'} + tag, version, digest = get_docker_tag() + self.assertEqual(tag, 'image@sha256:123') + self.assertEqual(version, None) + self.assertEqual(digest, 'sha256:123') + + +@override_env() +class TestDockerTarget(BaseTestClass): + def test_default_development_target(self): + main() + self.assert_set_env_file_called_with(DOCKER_TARGET='development') + + @override_env(DOCKER_VERSION="test") + def test_default_production_target(self): + main() + self.assert_set_env_file_called_with(DOCKER_TARGET='production') + + def test_default_env_file(self): + self.mock_get_env_file.return_value = {'DOCKER_TAG': 'mozilla/addons-server:test'} + main() + self.assert_set_env_file_called_with(DOCKER_TARGET='production') + +@override_env() +class TestComposeFile(BaseTestClass): + def test_default_compose_file(self): + main() + self.assert_set_env_file_called_with(COMPOSE_FILE='docker-compose.yml:docker-compose.dev.yml') + + @override_env(DOCKER_TARGET='production') + def test_default_target_production(self): + main() + self.assert_set_env_file_called_with(COMPOSE_FILE='docker-compose.yml') + + @override_env(COMPOSE_FILE='test') + def test_compose_file_override(self): + main() + self.assert_set_env_file_called_with(COMPOSE_FILE='test') + +@override_env() +class TestDebug(BaseTestClass): + def test_default_debug(self): + main() + self.assert_set_env_file_called_with(DEBUG='True') + + @override_env(DOCKER_TARGET='production') + def test_production_debug(self): + main() + self.assert_set_env_file_called_with(DEBUG='False') + + @override_env(DOCKER_TARGET='production') + def test_override_env_debug_false_on_target_production(self): + self.mock_get_env_file.return_value = {'DEBUG': 'True'} + main() + self.assert_set_env_file_called_with(DEBUG='False') + + @override_env(DOCKER_TARGET='development') + def test_override_env_debug_true_on_target_development(self): + self.mock_get_env_file.return_value = {'DEBUG': 'False'} + main() + self.assert_set_env_file_called_with(DEBUG='True') + + @override_env(DEBUG='test') + def test_debug_override(self): + main() + self.assert_set_env_file_called_with(DEBUG='test')