Change to ElasticSearch for FTS (#2541)

2025-10-26 11:27:32 +00:00 · 2023-08-14 13:57:06 +02:00 · 2023-08-14 13:57:06 +02:00 · 9e081a9f4c
commit 9e081a9f4c
parent b50a04c0a8
3 changed files with 61 additions and 176 deletions
--- a/apps/fulltextsearch.sh
+++ b/apps/fulltextsearch.sh
@ -5,15 +5,13 @@

 true
 SCRIPT_NAME="Full Text Search"
-SCRIPT_EXPLAINER="Full Text Search provides OpenSearch for Nextcloud, which makes it possible to search for text inside files."
+SCRIPT_EXPLAINER="Full Text Search provides ElastichSearch for Nextcloud, which makes it possible to search for text inside files."
 # shellcheck source=lib.sh
 source /var/scripts/fetch_lib.sh

 # Get all needed variables from the library
-ncdb
 nc_update
-opensearch_install
-ncdomain
+fulltextsearch_install

 # Check for errors + debug code and abort if something isn't right
 # 1 = ON
@ -28,14 +26,9 @@ root_check
 lowest_compatible_nc 21

 # Check if Full Text Search is already installed
-if ! does_this_docker_exist "$nc_fts" && ! does_this_docker_exist "$opens_fts" && ! is_app_installed fulltextsearch
+if ! does_this_docker_exist docker.elastic.co/elasticsearch/elasticsearch && ! is_app_installed fulltextsearch
 then
    # Ask for installing
-    if [ "${CURRENTVERSION%%.*}" -ge "25" ]
-    then
-        msg_box "Sorry, it's not possible to install FTS anymore since Nextcloud decided to remove support for OpenSearch. Read more in this issue: https://github.com/nextcloud/fulltextsearch_elasticsearch/issues/271"
-        exit 1
-    fi
    install_popup "$SCRIPT_NAME"
 else
    # Ask for removal or reinstallation
@ -56,16 +49,11 @@ else
            nextcloud_occ app:remove "$app"
        fi
    done
-    # Removal Docker image
-    docker_prune_this "$nc_fts"
-    docker_prune_volume "esdata"
-    docker-compose_down "$OPNSDIR/docker-compose.yml"
-    # Remove configuration files
-    rm -rf "$RORDIR"
-    rm -rf "$OPNSDIR"
+    # Removal Elastichsearch Docker image
+    docker_prune_this "docker.elastic.co/elasticsearch/elasticsearch"
+    rm -rf "$FULLTEXTSEARCH_DIR"
    # Show successful uninstall if applicable
    removal_popup "$SCRIPT_NAME"
-    apt-get purge docker-compose -y
 fi

 # Test RAM size (4GB min) + CPUs (min 2)
@ -95,6 +83,18 @@ then
    deluser --group solr
 fi

+# Removal Opensearch Docker image
+if does_this_docker_exist "$nc_fts" && does_this_docker_exist "$opens_fts"
+then
+    docker_prune_this "$nc_fts"
+    docker_prune_volume "esdata"
+    docker-compose_down "$OPNSDIR/docker-compose.yml"
+    # Remove configuration files
+    rm -rf "$RORDIR"
+    rm -rf "$OPNSDIR"
+    apt-get purge docker-compose -y
+fi
+
 # Check if the app is compatible with the current Nextcloud version
 if ! install_and_enable_app fulltextsearch
 then
@ -103,98 +103,23 @@ fi

 # Check & install docker
 install_docker
-install_if_not docker-compose
 set_max_count
-mkdir -p "$OPNSDIR"
-docker pull "$opens_fts"
-BCRYPT_HASH="$(docker run --rm -it $opens_fts \
-               bash -c "plugins/opensearch-security/tools/hash.sh -p $OPNSREST | tr -d ':\n' ")"

-# Create configurations YML
-# opensearch.yml
-cat << YML_OPENSEARCH > $OPNSDIR/opensearch.yml
-cluster.name: docker-cluster
-# Avoid Docker assigning IP.
-network.host: 0.0.0.0
-
-# Declaring single node cluster.
-discovery.type: single-node
-
-######## Start Security Configuration ########
-plugins.security.ssl.transport.pemcert_filepath: node.pem
-plugins.security.ssl.transport.pemkey_filepath: node-key.pem
-plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
-plugins.security.ssl.transport.enforce_hostname_verification: false
-
-# Disable ssl at REST as Fulltextsearch can't accept self-signed CA certs.
-plugins.security.ssl.http.enabled: false
-#plugins.security.ssl.http.pemcert_filepath: node.pem
-#plugins.security.ssl.http.pemkey_filepath: node-key.pem
-#plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
-plugins.security.allow_unsafe_democertificates: false
-plugins.security.allow_default_init_securityindex: true
-plugins.security.authcz.admin_dn:
-  - 'CN=admin,OU=FTS,O=OPENSEARCH,L=VM,ST=NEXTCLOUD,C=CA'
-plugins.security.nodes_dn:
-  - 'CN=${NCDOMAIN},OU=FTS,O=OPENSEARCH,L=VM,ST=NEXTCLOUD,C=CA'
-
-plugins.security.audit.type: internal_opensearch
-plugins.security.enable_snapshot_restore_privilege: true
-plugins.security.check_snapshot_restore_write_privileges: true
-plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
-plugins.security.system_indices.enabled: true
-plugins.security.system_indices.indices: [".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opendistro-notifications-*", ".opendistro-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"]
-node.max_local_storage_nodes: 1
-######## End Security Configuration ########
-YML_OPENSEARCH
-
-# internal_users.yml
-cat << YML_INTERNAL_USERS > $OPNSDIR/internal_users.yml
-_meta:
-  type: "internalusers"
-  config_version: 2
-
-${INDEX_USER}:
-  hash: "${BCRYPT_HASH}"
-  reserved: true
-  backend_roles:
-  - "admin"
-  description: "admin user for fts at opensearch."
-YML_INTERNAL_USERS
-
-# roles_mapping.yml
-cat << YML_ROLES_MAPPING > $OPNSDIR/roles_mapping.yml
-_meta:
-  type: "rolesmapping"
-  config_version: 2
-
-# Roles mapping
-all_access:
-  reserved: false
-  backend_roles:
-  - "admin"
-  description: "Maps admin to all_access"
-YML_ROLES_MAPPING
-
-# docker-compose.yml
-cat << YML_DOCKER_COMPOSE > $OPNSDIR/docker-compose.yml
+mkdir -p "$FULLTEXTSEARCH_DIR"
+cat << YML_DOCKER_COMPOSE > "$FULLTEXTSEARCH_DIR/docker-compose.yaml"
 version: '3'
 services:
-  fts_os-node:
-    image: opensearchproject/opensearch:1
-    container_name: fts_os-node
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1
+    container_name: $DOCKER_IMAGE_NAME
    restart: always
-    command:
-      - sh
-      - -c
-      - "/usr/share/opensearch/bin/opensearch-plugin list | grep -q ingest-attachment \
-         || /usr/share/opensearch/bin/opensearch-plugin install --batch ingest-attachment ;
-         ./opensearch-docker-entrypoint.sh"
+    ports:
+      - 127.0.0.1:9200:9200
    environment:
-      - cluster.name=fts_os-cluster
-      - node.name=fts_os-node
-      - bootstrap.memory_lock=true
-      - "OPENSEARCH_JAVA_OPTS=-Xms1024M -Xmx1024M"
+      - discovery.type=single-node
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=false
+      - ELASTIC_PASSWORD=$ELASTIC_USER_PASSWORD
    ulimits:
      memlock:
        soft: -1
@ -202,74 +127,40 @@ services:
      nofile:
        soft: 65536
        hard: 65536
-    volumes:
-      - fts_os-data:/usr/share/opensearch/data
-      - $OPNSDIR/root-ca.pem:/usr/share/opensearch/config/root-ca.pem
-      - $OPNSDIR/node.pem:/usr/share/opensearch/config/node.pem
-      - $OPNSDIR/node-key.pem:/usr/share/opensearch/config/node-key.pem
-      - $OPNSDIR/admin.pem:/usr/share/opensearch/config/admin.pem
-      - $OPNSDIR/admin-key.pem:/usr/share/opensearch/config/admin-key.pem
-      - $OPNSDIR/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
-      - $OPNSDIR/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
-      - $OPNSDIR/roles_mapping.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml
-    ports:
-      - 127.0.0.1:9200:9200
-      - 127.0.0.1:9600:9600 # Performance Analyzer [1]
    networks:
-      - fts_os-net
+      - $DOCKER_IMAGE_NAME-network

 volumes:
-  fts_os-data:
-
+  $DOCKER_IMAGE_NAME-data:
 networks:
-  fts_os-net:
-
-#[1] https://github.com/opensearch-project/performance-analyzer
+  $DOCKER_IMAGE_NAME-network:
 YML_DOCKER_COMPOSE

-# Prepare certs
-create_certs "$NCDOMAIN"
+# Start the docker image
+cd "$FULLTEXTSEARCH_DIR"
+docker compose up -d

-# Set permissions
-chmod 744 -R  $OPNSDIR
+# Check if online
+until curl -sS "http://elastic:$ELASTIC_USER_PASSWORD@localhost:9200/_cat/health?h=status" | grep -q "green\|yellow"
+do
+    countdown "Waiting for ElasticSearch to come online..." "3"
+done

-# Launch docker-compose
-cd $OPNSDIR
-docker-compose up -d
+# Check logs
+print_text_in_color "$ICyan" "Checking logs..."
+docker logs "$DOCKER_IMAGE_NAME"

-# Wait for bootstrapping
-if [ "$(nproc)" -gt 2 ]
-then
-    countdown "Waiting for Docker bootstrapping..." "60"
-else
-    countdown "Waiting for Docker bootstrapping..." "120"
-fi
-
-# Make sure password setup is enforced.
-docker-compose exec fts_os-node \
-    bash -c "cd \
-             plugins/opensearch-security/tools/ && \
-             bash securityadmin.sh -f \
-             ../securityconfig/internal_users.yml \
-             -t internalusers \
-             -icl \
-             -nhnv \
-             -cacert ../../../config/root-ca.pem \
-             -cert ../../../config/admin.pem \
-             -key ../../../config/admin-key.pem && \
-             chmod 0600 ../../../config/root-ca.pem ../../../config/admin.pem ../../../config/admin-key.pem"
-
-docker logs $fts_node
+countdown "Waiting a bit more before testing..." "10"

 # Get Full Text Search app for nextcloud
 install_and_enable_app fulltextsearch
 install_and_enable_app fulltextsearch_elasticsearch
 install_and_enable_app files_fulltextsearch
-chown -R www-data:www-data $NC_APPS_PATH
+chown -R www-data:www-data "$NC_APPS_PATH"

 # Final setup
 nextcloud_occ fulltextsearch:configure '{"search_platform":"OCA\\FullTextSearch_Elasticsearch\\Platform\\ElasticSearchPlatform"}'
-nextcloud_occ fulltextsearch_elasticsearch:configure "{\"elastic_host\":\"http://${INDEX_USER}:${OPNSREST}@localhost:9200\",\"elastic_index\":\"${INDEX_USER}-index\"}"
+nextcloud_occ fulltextsearch_elasticsearch:configure "{\"elastic_host\":\"http://elastic:$ELASTIC_USER_PASSWORD@localhost:9200\",\"elastic_index\":\"${NEXTCLOUD_INDEX}\"}"
 nextcloud_occ files_fulltextsearch:configure "{\"files_pdf\":\"1\",\"files_office\":\"1\"}"

 # Wait further for cache for index to work
--- a/lib.sh
+++ b/lib.sh
@ -173,20 +173,18 @@ SPAMHAUS=/etc/spamhaus.wl
 ENVASIVE=/etc/apache2/mods-available/mod-evasive.load
 APACHE2=/etc/apache2/apache2.conf
 # Full text Search
-opensearch_install() {
-    INDEX_USER=$(gen_passwd "$SHUF" '[:lower:]')
-    OPNSREST=$(gen_passwd "$SHUF" "A-Za-z0-9")
+fulltextsearch_install() {
+    NEXTCLOUD_INDEX=$(gen_passwd "$SHUF" '[:lower:]')
+    ELASTIC_USER_PASSWORD=$(gen_passwd "$SHUF" '[:lower:]')
+    DOCKER_IMAGE_NAME=es01
+    FULLTEXTSEARCH_DIR="$SCRIPTS"/fulltextsearch
+    # Legacy
+    nc_fts="ark74/nc_fts"
    nc_fts="ark74/nc_fts"
    opens_fts="opensearchproject/opensearch:1"
+    opens_fts="opensearchproject/opensearch:1"
+    fts_node="fts_os-node"
    fts_node="fts_os-node"
-}
-create_certs(){
-    download_script APP opensearch_certs
-    check_command sed -i "s|__NCDOMAIN__|$1|" "$SCRIPTS"/opensearch_certs.sh
-    check_command mv "$SCRIPTS"/opensearch_certs.sh "$OPNSDIR"
-    check_command cd "$OPNSDIR"
-    check_command bash opensearch_certs.sh
-    rm -f "$OPNSDIR"/opensearch_certs.sh
 }
 # Name in trusted_config
 ncdomain() {
@ -1798,14 +1796,14 @@ then
    echo "Docker image just got updated! We just updated $2 docker image automatically! $(date +%Y%m%d)" >> "$VMLOGS"/update.log
 fi
 }
-# docker-compose_update 'fts_os-node' 'Full Text Search' "$OPNSDIR"
+# docker-compose_update 'fulltextsearch-elasticsearch' 'Full Text Search' "$FTSDIR"
 # (docker conainter name = $1, the name in text = $2 , docker-compose directory = $3)
 docker-compose_update() {
 if is_docker_running && docker ps -a --format "{{.Names}}" | grep -q "^$1$"
 then
    cd "$3"
-    docker-compose pull
-    docker-compose up -d --remove-orphans
+    docker compose pull
+    docker compose up -d --remove-orphans
    docker image prune -a -f
    print_text_in_color "$IGreen" "$2 docker image just got updated!"
    echo "Docker image just got updated! We just updated $2 docker image automatically! $(date +%Y%m%d)" >> "$VMLOGS"/update.log
--- a/nextcloud_update.sh
+++ b/nextcloud_update.sh
@ -651,16 +651,12 @@ If you need help, please get support here: https://shop.hanssonit.se/product/pre
    # Full Text Search
    if [ "${CURRENTVERSION%%.*}" -ge "25" ]
    then
-        opensearch_install
+        fulltextsearch_install
        if does_this_docker_exist "$nc_fts" && does_this_docker_exist "$opens_fts"
        then
-            msg_box "Sorry, it's not possible to install or upgrade FTS anymore since Nextcloud decided to remove support for OpenSearchh
-Read more in this issue: https://github.com/nextcloud/fulltextsearch_elasticsearch/issues/271
-
-Please consider removing it by issuing the uninstall script: sudo bash $SCRIPTS/menu.sh --> Additional Apps --> FullTextSearch"
+            msg_box "Please consider reinstalling FUllTextSearch since you seem to have the old (and not working) implemantation by issuing the uninstall script: sudo bash $SCRIPTS/menu.sh --> Additional Apps --> FullTextSearch"
        else
-            docker_update_specific 'fts_esror' 'Full Text Search'
-            docker-compose_update 'fts_os-node' 'Full Text Search' "$OPNSDIR"
+            docker-compose_update "$DOCKER_IMAGE_NAME" 'Full Text Search' "$FULLTEXTSEARCH_DIR"
        fi
    fi
    # Plex