Make "up" go Brrrrrrrr

data seed command synchronizes storage directory

Remove storage volume (reuse olympia volume)

Remove data_olympia (use anonymous volume)

Remove sources from development DOCKER_TARGET

Invert docker-compose.ci to docker-compose.dev (update defaults in setup script to support prod mode in ci)

.dockerignore

@@ -48,12 +48,7 @@ src/olympia/discovery/strings.jinja2
 static-build/*
 static/css/node_lib/*
 static/js/node_lib/*
-storage/files/*
-storage/git-storage/*
-storage/guarded-addons/*
-storage/mlbf/*
-storage/shared_storage/*
-storage/sitemaps/*
+storage
 tmp/*
 
 # Additionally ignore these files from the docker build that are not in .gitignore
@@ -62,7 +57,6 @@ tmp/*
 .github
 docs
 private
-storage
 docker-bake.hcl
 docker-compose*.yml
 Dockerfile*

.github/actions/build-docker/action.yml

@@ -10,9 +10,6 @@ inputs:
   version:
     required: true
     description: The image version to tag with
-  target:
-    required: true
-    description: The stage to target in the build
   push:
     required: false
     description: Push the image?
@@ -57,7 +54,7 @@ runs:
     - name: Create .env and version.json files
       shell: bash
       run: |
-        echo "DOCKER_TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+        echo "DOCKER_TARGET=production" >> $GITHUB_ENV
         echo "DOCKER_VERSION=${{ steps.meta.outputs.version }}" >> $GITHUB_ENV
         echo "DOCKER_COMMIT=${{ steps.context.outputs.git_sha }}" >> $GITHUB_ENV
         echo "DOCKER_BUILD=${{ steps.context.outputs.git_build_url }}" >> $GITHUB_ENV

.github/actions/run-docker/action.yml

@@ -15,7 +15,7 @@ inputs:
   compose_file:
     description: 'The docker-compose file to use'
     required: false
-    default: 'docker-compose.yml:docker-compose.ci.yml'
+    default: 'docker-compose.yml'
   logs:
     description: 'Show logs'
     required: false

.github/workflows/_test.yml

@@ -41,34 +41,28 @@ jobs:
           -
             name: Needs Locale Compilation
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: |
               make compile_locales
               make test_needs_locales_compilation
           -
             name: Static Assets
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_static_assets
           -
             name: Internal Routes
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_internal_routes_allowed
           -
             name: Elastic Search
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_es_tests
           -
             name: Codestyle
             services: web
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make lint-codestyle
           -
             name: Manage Check
             services: web nginx
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make check
     steps:
       - uses: actions/checkout@v4
@@ -78,5 +72,4 @@ jobs:
           version: ${{ inputs.version }}
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
-          compose_file: ${{ matrix.compose_file }}
           run: ${{ matrix.run }}

.github/workflows/_test_main.yml

@@ -88,7 +88,7 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          compose_file: docker-compose.yml
+          compose_file: docker-compose.yml:docker-compose.dev.yml
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"
             group="--group ${{ matrix.group }}"

.github/workflows/ci.yml

@@ -72,7 +72,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ci-${{ needs.context.outputs.docker_version }}
-          target: development
           push: true
 
   test_make_docker_configuration:
@@ -88,7 +87,7 @@ jobs:
         shell: bash
         run: |
           docker compose version
-          npm exec jest -- ./tests/make --runInBand
+          make test_setup
 
   test_run_docker_action:
     runs-on: ubuntu-latest
@@ -158,6 +157,14 @@ jobs:
               exit 1
             fi
 
+      - name: Test setup
+        uses: ./.github/actions/run-docker
+        with:
+          digest: ${{ needs.build.outputs.digest }}
+          version: ${{ needs.build.outputs.version }}
+          run: |
+            pytest tests/make/
+
   docs_build:
     runs-on: ubuntu-latest
     needs: build
@@ -171,7 +178,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          compose_file: docker-compose.yml:docker-compose.dev.yml
           run: |
             make docs
 
@@ -222,7 +229,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          compose_file: docker-compose.yml:docker-compose.dev.yml
           run: make extract_locales
 
       - name: Push Locales
@@ -289,7 +296,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true
 
   push_gar:
@@ -321,5 +327,4 @@ jobs:
           registry: ${{ steps.docker_gar.outputs.registry }}
           image: ${{ steps.docker_gar.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true

.gitignore

@@ -47,17 +47,12 @@ src/olympia/discovery/strings.jinja2
 static-build/*
 static/css/node_lib/*
 static/js/node_lib/*
-storage/files/*
-storage/git-storage/*
-storage/guarded-addons/*
-storage/mlbf/*
-storage/shared_storage/*
-storage/sitemaps/*
+storage
 tmp/*
 
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore
 
 # do not ignore the following files
-!docker-compose.development.yml
+!docker-compose.dev.yml
 !docker-compose.private.yml
 !private/README.md

Dockerfile

@@ -96,6 +96,10 @@ EOF
 ARG DOCKER_TARGET
 ENV DOCKER_TARGET=${DOCKER_TARGET}
 
+# Add our custom mime types (required for for ts/json/md files)
+COPY docker/etc/mime.types /etc/mime.types
+
+
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
 # and makes copying the /deps dir to the final layer easy.
@@ -115,7 +119,10 @@ ${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r require
 npm ci ${NPM_ARGS} --include=prod
 EOF
 
-FROM base AS pip_development
+# In the local development image, we stop with dependencies
+# Everything else is about compiling production assets which is not needed
+# in the typical development workflow.
+FROM base AS development
 
 RUN \
     # Files required to install pip dependencies
@@ -127,11 +134,7 @@ RUN \
     # Mounts for caching dependencies
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
-<<EOF
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
-npm install ${NPM_ARGS} --no-save
-EOF
+    make update_deps
 
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
@@ -168,35 +171,23 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
-FROM base AS sources
+FROM base AS production
 
 ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
 
 ENV DOCKER_BUILD=${DOCKER_BUILD}
 ENV DOCKER_COMMIT=${DOCKER_COMMIT}
 ENV DOCKER_VERSION=${DOCKER_VERSION}
 
-# Add our custom mime types (required for for ts/json/md files)
-COPY docker/etc/mime.types /etc/mime.types
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
+# Copy compiled locales from builder
+COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy assets from assets
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
-
-# Set shell back to sh until we can prove we can use bash at runtime
-SHELL ["/bin/sh", "-c"]
-
-FROM sources AS development
-
-# Copy dependencies from `pip_development`
-COPY --from=pip_development --chown=olympia:olympia /deps /deps
-
-FROM sources AS production
-
-# Copy compiled locales from builder
-COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
 
-
+# Set shell back to sh until we can prove we can use bash at runtime
+SHELL ["/bin/sh", "-c"]

Makefile-docker

@@ -76,6 +76,11 @@ update_assets:
 	# Collect static files: This MUST be run last or files will be missing
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput
 
+.PHONY: update_deps
+update_deps:
+	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
+	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
+	npm install $(NPM_ARGS) --no-save
 
 # TOOD: remove this after we migrate addons-frontned to not depend on it.
 .PHONY: setup-ui-tests

Makefile-os

@@ -68,6 +68,10 @@ help_submake:
 	@echo "\nAll other commands will be passed through to the docker 'web' container make:"
 	@make -f Makefile-docker help_submake
 
+.PHONY: test_setup
+test_setup:
+	npm exec jest -- ./tests/make --runInBand
+
 .PHONY: setup
 setup: ## create configuration files version.json and .env required to run this project
 	for path in $(CLEAN_PATHS); do rm -rf "$(PWD)/$$path" && echo "$$path removed"; done

docker-compose.dev.yml

@@ -0,0 +1,17 @@
+services:
+  worker:
+    environment:
+      - HOST_UID
+    volumes:
+      - ./:/data/olympia
+      # Don't mount generated files. They only exist in the container
+      # and would otherwiser be deleted by mounbting data_olympia
+      - /data/olympia/static-build
+      - /data/olympia/site-static
+      - ./package.json:/deps/package.json
+      - ./package-lock.json:/deps/package-lock.json
+
+  web:
+    extends:
+      service: worker
+

docker-compose.yml

@@ -20,7 +20,7 @@ x-env-mapping: &env
     - HISTIGNORE=ls:exit:"cd .."
     - HISTCONTROL=erasedups
     - CIRCLECI
-    - HOST_UID
+    - HOST_UID=9500
     - DEBUG
     - DATA_BACKUP_SKIP
 
@@ -51,15 +51,6 @@ services:
       "--",
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
-    volumes:
-      - data_olympia:/data/olympia
-      # Don't mount generated files. They only exist in the container
-      # and would otherwiser be deleted by mounbting data_olympia
-      - /data/olympia/static-build
-      - /data/olympia/site-static
-      - storage:/data/olympia/storage
-      - ./package.json:/deps/package.json
-      - ./package-lock.json:/deps/package-lock.json
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5
@@ -99,8 +90,7 @@ services:
     image: nginx
     volumes:
       - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
-      - ./static:/srv/static
-      - storage:/srv/user-media
+      - ./:/srv
     ports:
       - "80:80"
     networks:
@@ -143,13 +133,9 @@ services:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     mem_limit: 2g
-    volumes:
-      - data_elasticsearch:/usr/share/elasticsearch/data
 
   redis:
     image: redis:6.2
-    volumes:
-      - data_redis:/data
 
   rabbitmq:
     image: rabbitmq:3.12
@@ -160,8 +146,6 @@ services:
       - RABBITMQ_DEFAULT_USER=olympia
       - RABBITMQ_DEFAULT_PASS=olympia
       - RABBITMQ_DEFAULT_VHOST=olympia
-    volumes:
-      - data_rabbitmq:/var/lib/rabbitmq
 
   autograph:
     image: mozilla/autograph:3.3.2
@@ -191,23 +175,10 @@ networks:
   default:
 
 volumes:
-  data_redis:
-  data_elasticsearch:
   data_mysqld:
     # Keep this value in sync with Makefile-os
     # External volumes must be manually created/destroyed
     name: addons-server_data_mysqld
     external: true
-  data_rabbitmq:
-  data_olympia:
-    driver: local
-    driver_opts:
-      type: none
-      o: bind
-      device: ${PWD}
-  storage:
-    driver: local
-    driver_opts:
-      type: none
-      o: bind
-      device: ./storage
+
+