#!/bin/bash # Opensearch 1.x Engine via Docker # # SwITNet Ltd © - 2023, https://switnet.net/ # T&M Hansson IT AB © - 2022, https://www.hanssonit.se/ # GPLv3 or later. # # Reset Color_Off='\e[0m' # Text Reset # Regular Colors Black='\e[0;30m' # Black Red='\e[0;31m' # Red Green='\e[0;32m' # Green Yellow='\e[0;33m' # Yellow Blue='\e[0;34m' # Blue Purple='\e[0;35m' # Purple Cyan='\e[0;36m' # Cyan MEM_AVAILABLE="$(grep MemTotal /proc/meminfo| grep -o '[0-9]\+')" OPNSDIR="/opt/opensearch" INDEX_USER="$(tr -dc '[:lower:]' < /dev/urandom | fold -w 24 | head -n1)" OPNSREST="$(tr -dc "a-zA-Z0-9" < /dev/urandom | fold -w 32 | head -n1)" PUBLIC_IP="$(dig -4 +short myip.opendns.com @resolver1.opendns.com)" opens_fts="opensearchproject/opensearch:1" fts_node="fts_os-node" max_map_count="512000" NL="$(printf '\n ')" printwc() { printf "%b$2%b" "$1" "${Color_Off}" } print_title() { printwc "${Blue}" "\n#--------------------------------------------------" printwc "${Blue}" "\n# $1" printwc "${Blue}" "\n#--------------------------------------------------\n" } install_if_not() { if [ "$(dpkg-query -W -f='${Status}' "$1" 2>/dev/null | \ grep -c "ok installed")" == "1" ]; then echo " $1 is installed, skipping..." else printf "\n---- Installing %s ----\n" "$1" apt-get -yq2 install "$1" fi } docker-compose_down() { if [ -f "$1" ] then cd "$(dirname "$1")" docker-compose down --volume --rmi all else echo "Non-existing docker-compose file path, skipping..." fi } set_once_hash_comment() { if ! awk '!/^ *#/ && NF {print}' "$2"| \ grep -q "$(awk -F '=' '{print$1}' <<< "$1")" ; then echo "Setting $1 on $2..." echo "$1" | tee -a "$2" else printf "%s\"$(awk -F '=' '{print$1}' <<< "$1")\"" "\n" printf " seems already present, skipping setting this variable\n" fi } dig_dns_ip() { dig -4 +short "$1"||awk -v RS='([0-9]+\\.){3}[0-9]+' 'RT{print RT}' } create_certs() { cp static/opensearch_certs.sh "$OPNSDIR" sed -i "s|__FTSDOMAIN__|$1|" "$OPNSDIR"/opensearch_certs.sh cd "$OPNSDIR" bash opensearch_certs.sh rm -f "$OPNSDIR"/opensearch_certs.sh } countdown() { printwc "$Cyan" "$1" secs="$(($2))" while [ $secs -gt 0 ]; do echo -ne "$secs\033[0K\r" sleep 1 : $((secs--)) done } #Check if user is root if ! [ "$(id -u)" = 0 ]; then echo "You need to be root or have sudo privileges!" exit 0 fi # Check update apt-get update -q2 # Test RAM size (4GB min) + CPUs (min 2) print_title "Check system resources" printf "\n\nVerifying System Resources:" if [ "$(nproc --all)" -lt 2 ];then printf "\nWarning!: The system do not meet the minimum CPU" printf " requirements for Opensearch to run." printf "\n>> We recommend 2 cores/threads for Opensearch!\n" CPU_MIN="N" else printf "\nCPU Cores/Threads: OK (%s)\n" "$(nproc --all)" CPU_MIN="Y" fi sleep .1 ### Test RAM size (4GB min) ### if [ "$MEM_AVAILABLE" -lt 3700000 ]; then printf "\nWarning!: The system do not meet the minimum RAM" printf " requirements for Jibri to run." printf "\n>> We recommend at least 4GB RAM for Opensearch!\n\n" MEM_MIN="N" else printf "\nMemory: OK (%s) MiB\n\n" "$((MEM_AVAILABLE/1024))" MEM_MIN="Y" fi sleep .1 if [ "$CPU_MIN" = "Y" ] && [ "$MEM_MIN" = "Y" ];then echo "All requirements seems meet!" else printf "CPU (%s)/RAM (%s MiB)" "$(nproc --all)" "$((MEM_AVAILABLE/1024))" printf " does NOT meet minimum recommended requirements!" sleep .1 while [ "$CONTINUE_LR" != "yes" ] && [ "$CONTINUE_LR" != "no" ] do read -p "> Do you want to continue?: (yes or no)$NL" -r CONTINUE_LR if [ "$CONTINUE_LR" = "no" ]; then printf "\n - See you next time with more resources!..." exit elif [ "$CONTINUE_LR" = "yes" ]; then printf "\n - We highly recommend to increase the server resources." fi done fi sleep .1 if [ "$CONTINUE_LR" = "yes" ]; then printf '\nThis server will likely have issues due the lack of resources.\n' printf '>>> We highly recommend to increase resources of this server. <<<\n' fi sleep .1 print_title "Set CA domain" while [ "$ANS_DMN" != "yes" ] do read -p "> Set your CA domain (or subdomain) here: $NL" -r DOMAIN read -p " > Did you mean?: $DOMAIN (yes or no)$NL" -r ANS_DMN if [ "$ANS_DMN" = "yes" ] then echo " - Alright, let's use $DOMAIN." else echo " - Please try again." fi done sleep .1 # Simple DNS test if [ "$PUBLIC_IP" = "$(dig_dns_ip "$DOMAIN")" ]; then printf "\nServer public IP & DNS record for" printf " %s seems to match, continuing..." "$DOMAIN" else echo -n "Server public IP ($PUBLIC_IP) & DNS record for $DOMAIN" echo " don't seem to match." echo -n " > Please check your dns records are applied and updated," echo " otherwise components may fail." read -p " > Do you want to continue?: (yes or no)$NL" -r DNS_CONTINUE if [ "$DNS_CONTINUE" = "yes" ]; then echo " - We'll continue anyway..." else echo " - Exiting for now..." exit fi fi sleep .1 print_title "Set system vm.max_map_count." sysctl -w vm.max_map_count="$max_map_count" set_once_hash_comment "vm.max_map_count=$max_map_count" "/etc/sysctl.conf" print_title "Install and setup Docker" install_if_not docker.io install_if_not docker-compose mkdir -p "$OPNSDIR" docker pull "$opens_fts" BCRYPT_HASH="$(docker run --rm -it $opens_fts \ bash -c "plugins/opensearch-security/tools/hash.sh \ -p $OPNSREST | tr -d ':\n' ")" # Create configurations YML # opensearch.yml cat << YML_OPENSEARCH > $OPNSDIR/opensearch.yml cluster.name: docker-cluster # Avoid Docker assigning IP. network.host: 0.0.0.0 # Declaring single node cluster. discovery.type: single-node ######## Start Security Configuration ######## plugins.security.ssl.transport.pemcert_filepath: node.pem plugins.security.ssl.transport.pemkey_filepath: node-key.pem plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem plugins.security.ssl.transport.enforce_hostname_verification: false # Disable ssl at REST as Fulltextsearch can't accept self-signed CA certs. plugins.security.ssl.http.enabled: false #plugins.security.ssl.http.pemcert_filepath: node.pem #plugins.security.ssl.http.pemkey_filepath: node-key.pem #plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem plugins.security.allow_unsafe_democertificates: false plugins.security.allow_default_init_securityindex: true plugins.security.authcz.admin_dn: - 'CN=admin,OU=FTS,O=OPENSEARCH,L=FTS,ST=OPENSEARCH,,C=CA' plugins.security.nodes_dn: - 'CN=${DOMAIN},OU=FTS,O=OPENSEARCH,L=FTS,ST=OPENSEARCH,C=CA' plugins.security.audit.type: internal_opensearch plugins.security.enable_snapshot_restore_privilege: true plugins.security.check_snapshot_restore_write_privileges: true plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] plugins.security.system_indices.enabled: true plugins.security.system_indices.indices: [".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opendistro-notifications-*", ".opendistro-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"] node.max_local_storage_nodes: 1 ######## End Security Configuration ######## YML_OPENSEARCH # internal_users.yml cat << YML_INTERNAL_USERS > $OPNSDIR/internal_users.yml _meta: type: "internalusers" config_version: 2 ${INDEX_USER}: hash: "${BCRYPT_HASH}" reserved: true backend_roles: - "admin" description: "admin user for fts at opensearch." YML_INTERNAL_USERS # roles_mapping.yml cat << YML_ROLES_MAPPING > $OPNSDIR/roles_mapping.yml _meta: type: "rolesmapping" config_version: 2 # Roles mapping all_access: reserved: false backend_roles: - "admin" description: "Maps admin to all_access" YML_ROLES_MAPPING # docker-compose.yml cat << YML_DOCKER_COMPOSE > $OPNSDIR/docker-compose.yml version: '3' services: fts_os-node: image: $opens_fts container_name: fts_os-node restart: always command: - sh - -c - "/usr/share/opensearch/bin/opensearch-plugin list | grep -q ingest-attachment \ || /usr/share/opensearch/bin/opensearch-plugin install --batch ingest-attachment ; ./opensearch-docker-entrypoint.sh" environment: - cluster.name=fts_os-cluster - node.name=fts_os-node - bootstrap.memory_lock=true - "OPENSEARCH_JAVA_OPTS=-Xms1024M -Xmx1024M" ulimits: memlock: soft: -1 hard: -1 nofile: soft: 65536 hard: 65536 volumes: - fts_os-data:/usr/share/opensearch/data - $OPNSDIR/root-ca.pem:/usr/share/opensearch/config/root-ca.pem - $OPNSDIR/node.pem:/usr/share/opensearch/config/node.pem - $OPNSDIR/node-key.pem:/usr/share/opensearch/config/node-key.pem - $OPNSDIR/admin.pem:/usr/share/opensearch/config/admin.pem - $OPNSDIR/admin-key.pem:/usr/share/opensearch/config/admin-key.pem - $OPNSDIR/opensearch.yml:/usr/share/opensearch/config/opensearch.yml - $OPNSDIR/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml - $OPNSDIR/roles_mapping.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml ports: - 127.0.0.1:9200:9200 - 127.0.0.1:9600:9600 # Performance Analyzer [1] networks: - fts_os-net volumes: fts_os-data: networks: fts_os-net: #[1] https://github.com/opensearch-project/performance-analyzer YML_DOCKER_COMPOSE print_title "Prepare certs" create_certs "$DOMAIN" # Set permissions chmod 744 -R $OPNSDIR print_title "Launch opensearch via docker-compose" cd $OPNSDIR docker-compose up -d # Wait for bootstrapping if [ "$(nproc)" -gt 2 ] then countdown "Waiting for Docker bootstrapping..." "30" else countdown "Waiting for Docker bootstrapping..." "60" fi # Make sure password setup is enforced. docker-compose exec fts_os-node \ bash -c "cd \ plugins/opensearch-security/tools/ && \ bash securityadmin.sh -f \ ../securityconfig/internal_users.yml \ -t internalusers \ -icl \ -nhnv \ -cacert ../../../config/root-ca.pem \ -cert ../../../config/admin.pem \ -key ../../../config/admin-key.pem && \ chmod 0600 ../../../config/root-ca.pem \ ../../../config/admin.pem \ ../../../config/admin-key.pem" docker logs $fts_node printwc "$Green" "\n\nYou can now use: " printwc "$Cyan" "'http://${INDEX_USER}:${OPNSREST}@localhost:9200'\n"