[upgrade][l] Upgrade to 2.10 and fix extensions and new features (#170)

.env.example

@@ -1,6 +1,9 @@
 # CKAN Dockerfile Related
-CKAN_BASE_IMAGE=ckan-base-2.9.3:base
-CKAN_SERVICE_IMAGE=ckan:latest
+CKAN_BASE_IMAGE=ckan-base-2.10.1:base
+CKAN_SERVICE_IMAGE=ckan:2.10
+
+SOLR_CONFIG_CKAN_DIR=/opt/solr/server/solr/configsets/ckan/conf
+
 
 # DB image settings
 POSTGRES_PASSWORD=ckan
@@ -48,7 +51,7 @@ TEST_CKAN_REDIS_URL=redis://redis:6379/1
 # Core settings
 CKAN__STORAGE_PATH=/tmp
 
-CKAN__PLUGINS=envvars resource_proxy geo_view geojson_view spatial_metadata spatial_query image_view text_view recline_view data_catalog_510 scheming_datasets developerpage saml2auth
+CKAN__PLUGINS=resource_proxy recline_view text_view image_view spatial_query spatial_metadata geojson_view geo_view developerpage datastore datapusher saml2auth scheming_datasets data_catalog_510 envvars
 CKAN__VIEWS__DEFAULT_VIEWS=image_view recline_view pdf_view geojson_view geo_view
 CKANEXT__SPATIAL__SEARCH_BACKEND=solr
 CKAN___SCHEMING__DATASET_SCHEMAS=ckanext.data_catalog_510:dataset-schema.json

Makefile

@@ -1,12 +1,12 @@
 include .env
-.PHONY: build-base-images dev-2.9 base-2.9
+.PHONY: build-base-images dev-2.10 base-2.10
 
 MAKEFLAGS += -j2
 
-build-base-images: dev-2.9
+build-base-images: dev-2.10
 
-dev-2.9: base-2.9
-		docker build -t $(CKAN_SERVICE_IMAGE) -f ckan-build/ckan-service/Dockerfile . --build-arg baseImage=${CKAN_BASE_IMAGE}
+dev-2.10: base-2.10
+		docker build -t $(CKAN_SERVICE_IMAGE) -f ckan-build/ckan-service/Dockerfile . --build-arg baseImage=${CKAN_BASE_IMAGE} --progress=plain
 
-base-2.9:
+base-2.10:
 		docker build -t $(CKAN_BASE_IMAGE) -f ckan-build/ckan/Dockerfile .

ckan-build/ckan-service/Dockerfile

@@ -46,8 +46,10 @@ WORKDIR $CKAN_HOME
 
 # Generate CKAN Config
 RUN ckan generate config $CKAN_CONFIG/production.ini
-RUN ckan-pip3 uninstall cryptography 
 RUN ckan-pip3 install --upgrade cryptography==36.0.2 pyOpenSSL==22.0.0
 
+# Install Extensions
+RUN /bin/sh ${CKAN_HOME}/scripts/install_extensions.sh
+
 ENTRYPOINT [ ]
-CMD ["/bin/sh", "scripts/start_ckan.sh"]
+CMD ["/bin/sh", "scripts/start_ckan.sh"]

ckan-build/ckan-service/patches/ckan/link_header_fix.patch

@@ -0,0 +1,13 @@
+diff --git a/ckanext/resourceproxy/blueprint.py b/ckanext/resourceproxy/blueprint.py
+index 035b144c5..e968d66a7 100644
+--- a/ckanext/resourceproxy/blueprint.py
++++ b/ckanext/resourceproxy/blueprint.py
+@@ -68,7 +68,7 @@ def proxy_resource(context: Context, data_dict: DataDict):
+         if not did_get:
+             r = requests.get(url, timeout=timeout, stream=True)
+
+-        response.headers[u'content-type'] = r.headers[u'content-type']
++        response.headers[u'content-type'] = r.headers.get(u'content-type')
+         response.charset = r.encoding or "utf-8"
+
+         length = 0

ckan-build/ckan-service/scripts/start_ckan.sh

@@ -1,8 +1,5 @@
 # /bin/bash
 
-# Install the Extensions
-/bin/sh scripts/install_extensions.sh
-
 # Update the Plugins from the ENV
 ckan config-tool $CKAN_CONFIG/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
 ckan config-tool $CKAN_CONFIG/production.ini "ckan.views.default_views = ${CKAN__VIEWS__DEFAULT_VIEWS}"

ckan-build/ckan-service/scripts/start_ckan_development.sh

@@ -2,9 +2,6 @@
 # Initialize the CKAN DB
 ckan db init
 
-# Install the Extensions
-/bin/sh scripts/install_extensions.sh
-
 # Update the Plugins from the ENV
 ckan config-tool $CKAN_CONFIG/production.ini "ckan.plugins = ${CKAN__PLUGINS}"
 

ckan-build/ckan/Dockerfile

@@ -1,6 +1,6 @@
+
 # See CKAN docs on installation from Docker Compose on usage
 FROM ubuntu:focal-20210119
-MAINTAINER Open Knowledge
 
 # Set timezone
 ENV TZ=UTC
@@ -85,22 +85,17 @@ WORKDIR $CKAN_VENV/src/
 RUN git clone https://github.com/ckan/ckan.git
 WORKDIR $CKAN_VENV/src/ckan
 RUN git reset --hard 
-RUN git checkout tags/ckan-2.9.3
+RUN git checkout tags/ckan-2.10.1
 WORKDIR $CKAN_VENV/src/
 RUN ckan-pip3 install -U pip && \
     ckan-pip3 install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirement-setuptools.txt && \
-    ckan-pip3 install zope.interface>=4.3.2 && \
-    sed -i "/$zope.interface==4.3.2/d" $CKAN_VENV/src/ckan/requirements.txt && \
     ckan-pip3 install --upgrade --no-cache-dir -r $CKAN_VENV/src/ckan/requirements.txt && \
     ckan-pip3 install -e $CKAN_VENV/src/ckan/ && \
     ckan generate config $CKAN_CONFIG/production.ini && \
-    ckan-pip3 install pyopenssl --upgrade && \
     ln -s $CKAN_VENV/src/ckan/ckan/config/who.ini $CKAN_CONFIG/who.ini && \
-    cp -v $CKAN_VENV/src/ckan/contrib/docker/ckan-entrypoint.sh /ckan-entrypoint.sh && \
-    chmod +x /ckan-entrypoint.sh && \
     chown -R ckan:ckan $CKAN_HOME $CKAN_VENV $CKAN_CONFIG $CKAN_STORAGE_PATH
 
 USER ckan
 EXPOSE 5000
 
-CMD ["ckan","-c","/etc/ckan/production.ini", "run", "--host", "0.0.0.0"]
+CMD ["ckan","-c","/etc/ckan/production.ini", "run", "--host", "0.0.0.0"]

docker-compose-dev.yml

@@ -67,10 +67,14 @@ services:
 
   solr:
     container_name: solr
-    build:
-      context: solr/
+    image: ckan/ckan-solr:2.10-solr8
     volumes:
-      - solr_data:/opt/solr/server/solr/ckan/data/index
+      - solr_data:/var/solr
+      - ./solr/schema.xml:${SOLR_CONFIG_CKAN_DIR}/managed-schema.xml
+      - ./solr/solrconfig.xml:${SOLR_CONFIG_CKAN_DIR}/solrconfig.xml
+
+      # - ./solr/schema.xml:/opt/solr/server/solr/configsets/ckan/conf/managed-schema
+
     ports:
       - "8983:8983"
     networks:
@@ -95,7 +99,10 @@ services:
         - DATAPUSHER_SQLALCHEMY_DATABASE_URI=${DATAPUSHER_SQLALCHEMY_DATABASE_URI}
     ports:
       - "8800:8800"
-
+    networks:
+      main:
+        aliases:
+          - solr
 volumes:
   ckan_storage:
   pg_data_510:

solr/schema.xml

@@ -22,44 +22,46 @@
 -->
 
 <!-- We update the version when there is a backward-incompatible change to this
-schema. In this case the version should be set to the next CKAN version number.
-(x.y but not x.y.z since it needs to be a float) -->
-<schema name="ckan" version="2.9">
+schema. We used to use the `version` attribute for this but this is an internal
+attribute that should not be used so starting from CKAN 2.10 we use the `name`
+attribute with the form `ckan-X.Y` -->
+<schema name="ckan-2.10" version="1.6">
 
 <types>
     <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
     <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
     <fieldtype name="binary" class="solr.BinaryField"/>
-    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
-    <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
-    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
-    <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
-    <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
-    <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
-    <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
-    <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
-    <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>
-    <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0"/>
-
-    <fieldType name="tdates" class="solr.TrieDateField" precisionStep="7" positionIncrementGap="0" multiValued="true"/>
+    <fieldType name="int" class="solr.IntPointField" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="float" class="solr.FloatPointField" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="long" class="solr.LongPointField" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="double" class="solr.DoublePointField" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="pint" class="solr.IntPointField" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="pfloat" class="solr.FloatPointField" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="plong" class="solr.LongPointField" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="pdouble" class="solr.DoublePointField" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="date" class="solr.DatePointField" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="pdate" class="solr.DatePointField" omitNorms="true" positionIncrementGap="0"/>
+
+    <fieldType name="pdates" class="solr.DatePointField" positionIncrementGap="0" multiValued="true"/>
     <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
-    <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
-    <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
-    <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
-    <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
+    <fieldType name="pints" class="solr.IntPointField" positionIncrementGap="0" multiValued="true"/>
+    <fieldType name="pfloats" class="solr.FloatPointField" positionIncrementGap="0" multiValued="true"/>
+    <fieldType name="plongs" class="solr.LongPointField" positionIncrementGap="0" multiValued="true"/>
+    <fieldType name="pdoubles" class="solr.DoublePointField" positionIncrementGap="0" multiValued="true"/>
 
     <fieldType name="text" class="solr.TextField" positionIncrementGap="100">
         <analyzer type="index">
             <tokenizer class="solr.WhitespaceTokenizerFactory"/>
-            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+            <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
+            <filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
             <filter class="solr.LowerCaseFilterFactory"/>
             <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
             <filter class="solr.ASCIIFoldingFilterFactory"/>
         </analyzer>
         <analyzer type="query">
             <tokenizer class="solr.WhitespaceTokenizerFactory"/>
-            <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
-            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+            <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+            <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
             <filter class="solr.LowerCaseFilterFactory"/>
             <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
             <filter class="solr.ASCIIFoldingFilterFactory"/>
@@ -68,16 +70,17 @@ schema. In this case the version should be set to the next CKAN version number.
 
 
     <!-- A general unstemmed text field - good if one does not know the language of the field -->
-    <fieldType name="textgen" class="solr.TextField" positionIncrementGap="100">
+    <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
         <analyzer type="index">
             <tokenizer class="solr.WhitespaceTokenizerFactory"/>
-            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+            <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+            <filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
             <filter class="solr.LowerCaseFilterFactory"/>
         </analyzer>
         <analyzer type="query">
             <tokenizer class="solr.WhitespaceTokenizerFactory"/>
-            <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
-            <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+            <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+            <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
             <filter class="solr.LowerCaseFilterFactory"/>
         </analyzer>
     </fieldType>
@@ -113,10 +116,10 @@ schema. In this case the version should be set to the next CKAN version number.
     <field name="ckan_url" type="string" indexed="true" stored="true" omitNorms="true" />
     <field name="download_url" type="string" indexed="true" stored="true" omitNorms="true" />
     <field name="notes" type="text" indexed="true" stored="true"/>
-    <field name="author" type="textgen" indexed="true" stored="true" />
-    <field name="author_email" type="textgen" indexed="true" stored="true" />
-    <field name="maintainer" type="textgen" indexed="true" stored="true" />
-    <field name="maintainer_email" type="textgen" indexed="true" stored="true" />
+    <field name="author" type="text_general" indexed="true" stored="true" />
+    <field name="author_email" type="text_general" indexed="true" stored="true" />
+    <field name="maintainer" type="text_general" indexed="true" stored="true" />
+    <field name="maintainer_email" type="text_general" indexed="true" stored="true" />
     <field name="license" type="string" indexed="true" stored="true" />
     <field name="license_id" type="string" indexed="true" stored="true" />
     <field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
@@ -126,8 +129,8 @@ schema. In this case the version should be set to the next CKAN version number.
     <field name="capacity" type="string" indexed="true" stored="true" multiValued="false"/>
     <field name="permission_labels" type="string" indexed="true" stored="false" multiValued="true"/>
 
-    <field name="res_name" type="textgen" indexed="true" stored="true" multiValued="true" />
-    <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>
+    <field name="res_name" type="text_general" indexed="true" stored="true" multiValued="true" />
+    <field name="res_description" type="text_general" indexed="true" stored="true" multiValued="true"/>
     <field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="res_type" type="string" indexed="true" stored="true" multiValued="true"/>
@@ -183,8 +186,6 @@ schema. In this case the version should be set to the next CKAN version number.
 </fields>
 
 <uniqueKey>index_id</uniqueKey>
-<defaultSearchField>text</defaultSearchField>
-<solrQueryParser defaultOperator="AND"/>
 
 <copyField source="url" dest="urls"/>
 <copyField source="title" dest="title_ngram"/>