diff --git a/opensearch-1.x-node-installer.sh b/opensearch-1.x-node-installer.sh new file mode 100644 index 0000000..faa93db --- /dev/null +++ b/opensearch-1.x-node-installer.sh @@ -0,0 +1,320 @@ +#!/bin/bash +# Opensearch 1.x Engine via Docker +# +# SwITNet Ltd © - 2023, https://switnet.net/ +# T&M Hansson IT AB © - 2022, https://www.hanssonit.se/ +# GPLv3 or later. +# + +# Reset +Color_Off='\e[0m' # Text Reset +# Regular Colors +Black='\e[0;30m' # Black +Red='\e[0;31m' # Red +Green='\e[0;32m' # Green +Yellow='\e[0;33m' # Yellow +Blue='\e[0;34m' # Blue +Purple='\e[0;35m' # Purple +Cyan='\e[0;36m' # Cyan + +MEM_AVAILABLE="$(grep MemTotal /proc/meminfo| grep -o '[0-9]\+')" +OPNSDIR="/opt/opensearch" +INDEX_USER="$(tr -dc '[:lower:]' < /dev/urandom | fold -w 24 | head -n1)" +OPNSREST="$(tr -dc "a-zA-Z0-9" < /dev/urandom | fold -w 32 | head -n1)" +opens_fts="opensearchproject/opensearch" +fts_node="fts_os-node" +max_map_count="512000" + +printwc() { + printf "%b$2%b" "$1" "${Color_Off}" +} +install_ifnot() { + if [ "$(dpkg-query -W -f='${Status}' "$1" 2>/dev/null | \ + grep -c "ok installed")" == "1" ]; then + echo " $1 is installed, skipping..." + else + printf "\n---- Installing %s ----" "$1" + apt-get -yq2 install "$1" + fi +} +docker-compose_down() { +if [ -f "$1" ] +then + cd "$(dirname "$1")" + docker-compose down --volume --rmi all +else + echo "Non-existing docker-compose file path, skipping..." +fi +} +set_once_hash_comment() { +if ! awk '!/^ *#/ && NF {print}' "$2"| \ + grep -q "$(awk -F '=' '{print$1}' <<< "$1")" ; then + echo "Setting $1 on $2..." + echo "$1" | tee -a "$2" +else + printf "%s\"$(awk -F '=' '{print$1}' <<< "$1")\"" "\n" + printf " seems already present, skipping setting this variable\n" +fi +} +dig_dns_ip() { + dig -4 +short "$1"||awk -v RS='([0-9]+\\.){3}[0-9]+' 'RT{print RT}' +} +create_certs(){ + sed -i "s|__FTSDOMAIN__|$1|" static/opensearch_certs.sh + bash static/opensearch_certs.sh +} +countdown() { +printwc "$Cyan" "$1" +secs="$(($2))" +while [ $secs -gt 0 ]; do + echo -ne "$secs\033[0K\r" + sleep 1 + : $((secs--)) +done +} + +# Test RAM size (4GB min) + CPUs (min 2) +#Check system resources +printf "\n\nVerifying System Resources:" +if [ "$(nproc --all)" -lt 2 ];then + printf "\nWarning!: The system do not meet the minimum CPU" + printf " requirements for Opensearch to run." + printf "\n>> We recommend 2 cores/threads for Opensearch!\n" + CPU_MIN="N" +else + printf "\nCPU Cores/Threads: OK (%s)\n" "$(nproc --all)" + CPU_MIN="Y" +fi +sleep .1 +### Test RAM size (4GB min) ### +if [ "$MEM_AVAILABLE" -lt 3700000 ]; then + printf "\nWarning!: The system do not meet the minimum RAM" + printf " requirements for Jibri to run." + printf "\n>> We recommend at least 4GB RAM for Opensearch!\n\n" + MEM_MIN="N" +else + printf "\nMemory: OK (%s) MiB\n\n" "$((MEM_AVAILABLE/1024))" + MEM_MIN="Y" +fi +sleep .1 +if [ "$CPU_MIN" = "Y" ] && [ "$MEM_MIN" = "Y" ];then + echo "All requirements seems meet!" +else + printf "CPU (%s)/RAM (%s MiB)" "$(nproc --all)" "$((MEM_AVAILABLE/1024))" + printf " does NOT meet minimum recommended requirements!" +sleep .1 + while [ "$CONTINUE_LR" != "yes" ] && [ "$CONTINUE_LR" != "no" ] + do + read -p "> Do you want to continue?: (yes or no)$NL" -r CONTINUE_LR + if [ "$CONTINUE_LR" = "no" ]; then + printf "\n - See you next time with more resources!..." + exit + elif [ "$CONTINUE_LR" = "yes" ]; then + printf "\n - We highly recommend to increase the server resources." + fi + done +fi +sleep .1 +if [ "$CONTINUE_LR" = "yes" ]; then +printf '\nThis server will likely have issues due the lack of resources.\n' +printf '>>> We highly recommend to increase resources of this server. <<<\n' +fi +sleep .1 +# Set domain +while [ "$ANS_DMN" != "yes" ] +do + read -p "> Set your CA domain (or subdomain) here: $NL" -r DOMAIN + read -p " > Did you mean?: $DOMAIN (yes or no)$NL" -r ANS_DMN + if [ "$ANS_DMN" = "yes" ] + then + echo " - Alright, let's use $DOMAIN." + else + echo " - Please try again." + fi +done +sleep .1 +# Simple DNS test +if [ "$PUBLIC_IP" = "$(dig_dns_ip "$DOMAIN")" ]; then + printf "\nServer public IP & DNS record for" + printf " %s seems to match, continuing..." "$DOMAIN" +else + echo -n "Server public IP ($PUBLIC_IP) & DNS record for $DOMAIN" + echo " don't seem to match." + echo -n " > Please check your dns records are applied and updated," + echo " otherwise components may fail." + read -p " > Do you want to continue?: (yes or no)$NL" -r DNS_CONTINUE + if [ "$DNS_CONTINUE" = "yes" ]; then + echo " - We'll continue anyway..." + else + echo " - Exiting for now..." + exit + fi +fi +sleep .1 +# Check & install docker +install_if_not docker +install_if_not docker-compose + +sysctl -w vm.max_map_count="$max_map_count" +set_once_hash_comment "vm.max_map_count=$max_map_count" "/etc/sysctl.conf" + +mkdir -p "$OPNSDIR" +docker pull "$opens_fts" +BCRYPT_HASH="$(docker run --rm -it $opens_fts \ + bash -c "plugins/opensearch-security/tools/hash.sh \ + -p $OPNSREST | tr -d ':\n' ")" + +# Create configurations YML +# opensearch.yml +cat << YML_OPENSEARCH > $OPNSDIR/opensearch.yml +cluster.name: docker-cluster +# Avoid Docker assigning IP. +network.host: 0.0.0.0 + +# Declaring single node cluster. +discovery.type: single-node + +######## Start Security Configuration ######## +plugins.security.ssl.transport.pemcert_filepath: node.pem +plugins.security.ssl.transport.pemkey_filepath: node-key.pem +plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem +plugins.security.ssl.transport.enforce_hostname_verification: false + +# Disable ssl at REST as Fulltextsearch can't accept self-signed CA certs. +plugins.security.ssl.http.enabled: false +#plugins.security.ssl.http.pemcert_filepath: node.pem +#plugins.security.ssl.http.pemkey_filepath: node-key.pem +#plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem +plugins.security.allow_unsafe_democertificates: false +plugins.security.allow_default_init_securityindex: true +plugins.security.authcz.admin_dn: + - 'CN=admin,OU=FTS,O=OPENSEARCH,L=FTS,ST=OPENSEARCH,,C=CA' +plugins.security.nodes_dn: + - 'CN=${DOMAIN},OU=FTS,O=OPENSEARCH,L=FTS,ST=OPENSEARCH,C=CA' + +plugins.security.audit.type: internal_opensearch +plugins.security.enable_snapshot_restore_privilege: true +plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +plugins.security.system_indices.enabled: true +plugins.security.system_indices.indices: [".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opendistro-notifications-*", ".opendistro-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"] +node.max_local_storage_nodes: 1 +######## End Security Configuration ######## +YML_OPENSEARCH + +# internal_users.yml +cat << YML_INTERNAL_USERS > $OPNSDIR/internal_users.yml +_meta: + type: "internalusers" + config_version: 2 + +${INDEX_USER}: + hash: "${BCRYPT_HASH}" + reserved: true + backend_roles: + - "admin" + description: "admin user for fts at opensearch." +YML_INTERNAL_USERS + +# roles_mapping.yml +cat << YML_ROLES_MAPPING > $OPNSDIR/roles_mapping.yml +_meta: + type: "rolesmapping" + config_version: 2 + +# Roles mapping +all_access: + reserved: false + backend_roles: + - "admin" + description: "Maps admin to all_access" +YML_ROLES_MAPPING + +# docker-compose.yml +cat << YML_DOCKER_COMPOSE > $OPNSDIR/docker-compose.yml +version: '3' +services: + fts_os-node: + image: opensearchproject/opensearch:1 + container_name: fts_os-node + restart: always + command: + - sh + - -c + - "/usr/share/opensearch/bin/opensearch-plugin list | grep -q ingest-attachment \ + || /usr/share/opensearch/bin/opensearch-plugin install --batch ingest-attachment ; + ./opensearch-docker-entrypoint.sh" + environment: + - cluster.name=fts_os-cluster + - node.name=fts_os-node + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms1024M -Xmx1024M" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - fts_os-data:/usr/share/opensearch/data + - $OPNSDIR/root-ca.pem:/usr/share/opensearch/config/root-ca.pem + - $OPNSDIR/node.pem:/usr/share/opensearch/config/node.pem + - $OPNSDIR/node-key.pem:/usr/share/opensearch/config/node-key.pem + - $OPNSDIR/admin.pem:/usr/share/opensearch/config/admin.pem + - $OPNSDIR/admin-key.pem:/usr/share/opensearch/config/admin-key.pem + - $OPNSDIR/opensearch.yml:/usr/share/opensearch/config/opensearch.yml + - $OPNSDIR/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml + - $OPNSDIR/roles_mapping.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml + ports: + - 127.0.0.1:9200:9200 + - 127.0.0.1:9600:9600 # Performance Analyzer [1] + networks: + - fts_os-net + +volumes: + fts_os-data: + +networks: + fts_os-net: + +#[1] https://github.com/opensearch-project/performance-analyzer +YML_DOCKER_COMPOSE + +# Prepare certs +create_certs "$DOMAIN" + +# Set permissions +chmod 744 -R $OPNSDIR + +# Launch docker-compose +cd $OPNSDIR +docker-compose up -d + +# Wait for bootstrapping +if [ "$(nproc)" -gt 2 ] +then + countdown "Waiting for Docker bootstrapping..." "30" +else + countdown "Waiting for Docker bootstrapping..." "60" +fi + +# Make sure password setup is enforced. +docker-compose exec fts_os-node \ + bash -c "cd \ + plugins/opensearch-security/tools/ && \ + bash securityadmin.sh -f \ + ../securityconfig/internal_users.yml \ + -t internalusers \ + -icl \ + -nhnv \ + -cacert ../../../config/root-ca.pem \ + -cert ../../../config/admin.pem \ + -key ../../../config/admin-key.pem && \ + chmod 0600 ../../../config/root-ca.pem \ + ../../../config/admin.pem \ + ../../../config/admin-key.pem" + +docker logs $fts_node + +echo "You can now use: \"http://${INDEX_USER}:${OPNSREST}@localhost:9200\"" diff --git a/static/opensearch_certs.sh b/static/opensearch_certs.sh new file mode 100644 index 0000000..af2d3da --- /dev/null +++ b/static/opensearch_certs.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Create TLS self-signed CA certificates for 5 years required to comply +# with transport security layer requirement. +# Source: +# https://opensearch.org/docs/latest/security-plugin/configuration/generate-certificates/#sample-script + +mkdir tls_store +TLS_DN="/C=CA/ST=OPENSEARCH/L=NODE/O=OPENSEARCH/OU=FTS" + +# Root CA +openssl genrsa -out root-ca-key.pem 4096 +openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "${TLS_DN}/CN=ROOT" -out root-ca.pem -days 1825 +# Admin cert +openssl genrsa -out admin-key-temp.pem 4096 +openssl pkcs8 -inform PEM -outform PEM -in admin-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out admin-key.pem +openssl req -new -key admin-key.pem -subj "${TLS_DN}/CN=ADMIN" -out admin.csr +openssl x509 -req -in admin.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out admin.pem -days 1825 +# Node cert +openssl genrsa -out node-key-temp.pem 4096 +openssl pkcs8 -inform PEM -outform PEM -in node-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out node-key.pem +openssl req -new -key node-key.pem -subj "${TLS_DN}/CN=__FTSDOMAIN__" -out node.csr +openssl x509 -req -in node.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out node.pem -days 1825 +# Client cert +openssl genrsa -out client-key-temp.pem 4096 +openssl pkcs8 -inform PEM -outform PEM -in client-key-temp.pem -topk8 -nocrypt -v1 PBE-SHA1-3DES -out client-key.pem +openssl req -new -key client-key.pem -subj "${TLS_DN}/CN=CLIENT" -out client.csr +openssl x509 -req -in client.csr -CA root-ca.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out client.pem -days 1825 + +# Cleanup +rm admin-key-temp.pem \ + admin.csr \ + node-key-temp.pem \ + node.csr \ + client-key-temp.pem \ + client.csr + +# Store +mv client.pem \ + client-key.pem \ + root-ca.srl \ + root-ca-key.pem -t tls_store