Ogc 508 replace elastic search by postgres v3

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         exclude: .pre-commit-config.yaml
       - id: pt_structure
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.0
+    rev: v0.7.1
     hooks:
       - id: ruff
         args: [ "--fix" ]

src/onegov/agency/views/search.py

@@ -1,9 +1,8 @@
 from onegov.agency import AgencyApp
 from onegov.agency.layout import AgencySearchLayout
 from onegov.core.security import Public
-from onegov.org.models import Search
-from onegov.org.views.search import search as search_view
-
+from onegov.org.models import Search, SearchPostgres
+from onegov.org.views.search import search as search_view, search_postgres
 
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
@@ -23,3 +22,15 @@ def search(
     if isinstance(data, dict):
         data['layout'] = AgencySearchLayout(self, request)
     return data
+
+
+@AgencyApp.html(model=SearchPostgres, template='search_postgres.pt',
+                permission=Public)
+def agency_search_postgres(
+    self: SearchPostgres['Base'],
+    request: 'AgencyRequest'
+) -> 'RenderData | Response':
+    data = search_postgres(self, request)
+    if isinstance(data, dict):
+        data['layout'] = AgencySearchLayout(self, request)
+    return data

src/onegov/directory/models/directory_entry.py

@@ -30,9 +30,9 @@ class DirectoryEntry(Base, ContentMixin, CoordinatesMixin, TimestampMixin,
     __tablename__ = 'directory_entries'
 
     es_properties = {
-        'keywords': {'type': 'keyword'},
         'title': {'type': 'localized'},
         'lead': {'type': 'localized'},
+        'keywords': {'type': 'keyword'},
         'directory_id': {'type': 'keyword'},
 
         # since the searchable text might include html, we remove it

src/onegov/fsi/views/search.py

@@ -1,9 +1,9 @@
 from onegov.core.security import Personal
 from onegov.fsi import FsiApp
-from onegov.org.models import Search
-from onegov.town6.views.search import town_search as search_view
-from onegov.org.views.search import suggestions as suggestions_view
-
+from onegov.fsi.layout import DefaultLayout
+from onegov.org.models import Search, SearchPostgres
+from onegov.org.views.search import (search, search_postgres,
+                                     suggestions, suggestions_postgres)
 
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
@@ -13,17 +13,41 @@
     from webob import Response
 
 
-@FsiApp.html(model=Search, template='search.pt', permission=Personal)
-def search(
+@FsiApp.html(
+    model=Search,
+    template='search.pt',
+    permission=Personal
+)
+def fsi_search(
     self: Search['Base'],
     request: 'FsiRequest'
 ) -> 'RenderData | Response':
-    return search_view(self, request)
+    return search(self, request, DefaultLayout(self, request))
+
+
+@FsiApp.html(
+    model=SearchPostgres,
+    template='search_postgres.pt',
+    permission=Personal
+)
+def fsi_search_postgres(
+    self: SearchPostgres['Base'],
+    request: 'FsiRequest'
+) -> 'RenderData | Response':
+    return search_postgres(self, request, DefaultLayout(self, request))
 
 
 @FsiApp.json(model=Search, name='suggest', permission=Personal)
-def suggestions(
+def fsi_suggestions(
     self: Search['Base'],
     request: 'FsiRequest'
 ) -> 'JSON_ro':
-    return suggestions_view(self, request)
+    return suggestions(self, request)
+
+
+@FsiApp.json(model=SearchPostgres, name='suggest', permission=Personal)
+def fsi_suggestions_postgres(
+    self: SearchPostgres['Base'],
+    request: 'FsiRequest'
+) -> 'JSON_ro':
+    return suggestions_postgres(self, request)

src/onegov/landsgemeinde/views/search.py

@@ -1,9 +1,8 @@
 from onegov.core.security import Public
 from onegov.landsgemeinde import LandsgemeindeApp
 from onegov.landsgemeinde.layouts import DefaultLayout
-from onegov.org.models import Search
-from onegov.org.views.search import search
-
+from onegov.org.models import Search, SearchPostgres
+from onegov.org.views.search import search, search_postgres
 
 from typing import TYPE_CHECKING
 if TYPE_CHECKING:
@@ -19,3 +18,12 @@ def landsgemeinde_search(
     request: 'LandsgemeindeRequest'
 ) -> 'RenderData | Response':
     return search(self, request, DefaultLayout(self, request))
+
+
+@LandsgemeindeApp.html(model=SearchPostgres, template='search_postgres.pt',
+                       permission=Public)
+def landsgemeinde_search_postgres(
+    self: SearchPostgres['Base'],
+    request: 'LandsgemeindeRequest'
+) -> 'RenderData | Response':
+    return search_postgres(self, request, DefaultLayout(self, request))

src/onegov/onboarding/app.py

@@ -2,15 +2,15 @@
 from onegov.file import DepotApp
 from onegov.onboarding.theme import OnboardingTheme
 from onegov.reservation import LibresIntegration
-from onegov.search import ElasticsearchApp
+from onegov.search import SearchApp
 
 
 from typing import Any, TYPE_CHECKING
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
 
-class OnboardingApp(Framework, LibresIntegration, DepotApp, ElasticsearchApp):
+class OnboardingApp(Framework, LibresIntegration, DepotApp, SearchApp):
 
     serve_static_files = True
 

src/onegov/onboarding/models/town_assistant.py

@@ -202,7 +202,7 @@ def add_town(
                 'org': name
             })
 
-            self.app.es_perform_reindex()
+            self.app.perform_reindex()
             self.app.send_transactional_email(
                 subject=title,
                 receivers=(user, ),

src/onegov/org/app.py

@@ -29,7 +29,7 @@
 from onegov.org.theme import OrgTheme
 from onegov.pay import PayApp
 from onegov.reservation import LibresIntegration
-from onegov.search import ElasticsearchApp
+from onegov.search import SearchApp
 from onegov.ticket import TicketCollection
 from onegov.ticket import TicketPermission
 from onegov.user import UserApp
@@ -54,7 +54,7 @@
     from webob import Response
 
 
-class OrgApp(Framework, LibresIntegration, ElasticsearchApp, MapboxApp,
+class OrgApp(Framework, LibresIntegration, SearchApp, MapboxApp,
              DepotApp, PayApp, FormApp, UserApp, WebsocketsApp):
 
     serve_static_files = True

src/onegov/org/cronjobs.py

@@ -91,7 +91,7 @@ def publish_files(request: 'OrgRequest') -> None:
 def reindex_published_models(request: 'OrgRequest') -> None:
     """
     Reindexes all recently published/unpublished objects
-    in the elasticsearch database.
+    in the elasticsearch and postgres database.
 
     For pages it also updates the propagated access to any
     associated files.

src/onegov/org/layout.py

@@ -43,6 +43,7 @@
 from onegov.org.models.extensions import PersonLinkExtension
 from onegov.org.models.external_link import ExternalLinkCollection
 from onegov.org.models.form import submission_deletable
+from onegov.org.models.search import SearchPostgres
 from onegov.org.open_graph import OpenGraphMixin
 from onegov.org.theme.org_theme import user_options
 from onegov.org.utils import IMG_URLS
@@ -333,11 +334,19 @@ def homepage_url(self) -> str:
     @cached_property
     def search_url(self) -> str:
         """ Returns the url to the search page. """
+        # Allows using postgres search while es search remains default
+        if (self.request.path_info
+                and 'search-postgres' in self.request.path_info):
+            return self.request.class_link(SearchPostgres)
         return self.request.class_link(Search)
 
     @cached_property
     def suggestions_url(self) -> str:
         """ Returns the url to the suggestions json view. """
+        # Allows using postgres search while es search remains default
+        if (self.request.path_info
+                and 'search-postgres' in self.request.path_info):
+            return self.request.class_link(SearchPostgres, name='suggest')
         return self.request.class_link(Search, name='suggest')
 
     @cached_property

src/onegov/org/models/__init__.py

@@ -46,7 +46,7 @@
 from onegov.org.models.recipient import ResourceRecipient
 from onegov.org.models.recipient import ResourceRecipientCollection
 from onegov.org.models.resource import DaypassResource
-from onegov.org.models.search import Search
+from onegov.org.models.search import Search, SearchPostgres
 from onegov.org.models.sitecollection import SiteCollection
 from onegov.org.models.swiss_holidays import SwissHolidays
 from onegov.org.models.tan import TANAccess
@@ -101,6 +101,7 @@
     'ResourceRecipient',
     'ResourceRecipientCollection',
     'Search',
+    'SearchPostgres',
     'SiteCollection',
     'SubmissionMessage',
     'SwissHolidays',