Swarm Storage HowTo!

KEY=$(od -vN 32 -An -tx1 /dev/urandom | tr -d ' \n' ; echo)
SECRET=$(od -vN 32 -An -tx1 /dev/urandom | tr -d ' \n' ; echo)
echo $KEY > key
echo $SECRET > secret
echo $KEY | docker secret create access_key -
echo $SECRET | docker secret create secret_key -

docker node update --label-add minio1=true [node-name] (1)
docker node update --label-add minio2=true [node-name]
docker node update --label-add minio3=true [node-name]
docker node update --label-add minio4=true [node-name]

docker node update --label-add group=minio [node-name] (2)
docker node update --label-add group=minio [node-name]

[req]
distinguished_name = req_distinguished_name
x509_extensions = v3_req
prompt = no

[req_distinguished_name]
C = US (1)
ST = VA (1)
L = Somewhere (1)
O = MyOrg (1)
OU = MyOU (1)
CN = MyServerName (1)

[v3_req]
subjectAltName = @alt_names

[alt_names]
IP.1 = 127.0.0.1 (2)

openssl req -x509 -nodes -days 730 -newkey rsa:2048 -keyout private.key -out public.crt -config openssl.conf

services:
  minio1: (1)
    image: minio/minio:RELEASE.2020-04-10T03-34-42Z (2)
    hostname: minio1
    volumes:
      - minio1-data:/export (3)
    ports:
      - "9001:9000" (4)
    networks:
      - minio_distributed (5)
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:
        constraints:
          - node.labels.minio1==true (6)
    command: server http://minio{1...4}/export (7)
    secrets: (8)
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] (9)
      interval: 30s
      timeout: 20s
      retries: 3

  minio2: (10)
    image: minio/minio:RELEASE.2020-04-10T03-34-42Z
    hostname: minio2 (10)
    volumes:
      - minio2-data:/export (11)
    ports:
      - "9002:9000" (12)
    networks:
      - minio_distributed (5)
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:
        constraints:
          - node.labels.minio2==true (13)
    command: server http://minio{1...4}/export
    secrets:
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  minio3:
    image: minio/minio:RELEASE.2020-04-10T03-34-42Z
    hostname: minio3
    volumes:
      - minio3-data:/export
    ports:
      - "9003:9000"
    networks:
      - minio_distributed (5)
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:
        constraints:
          - node.labels.minio3==true
    command: server http://minio{1...4}/export
    secrets:
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  minio4:
    image: minio/minio:RELEASE.2020-04-10T03-34-42Z
    hostname: minio4
    volumes:
      - minio4-data:/export
    ports:
      - "9004:9000"
    networks:
      - minio_distributed (5)
    deploy:
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      placement:
        constraints:
          - node.labels.minio4==true
    command: server http://minio{1...4}/export
    secrets:
      - secret_key
      - access_key
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  web:
    image: nginx:1.17.9-alpine
    deploy:
      mode: replicated
      restart_policy:
        delay: 10s
        max_attempts: 10
        window: 60s
      replicas: 2
      placement:
        max_replicas_per_node: 1
        constraints:
          - node.labels.group==minio (14)
    ports:
      - "8080:80"
      - "9443:443"
    volumes: (15)
      - /PATH_to_FILE/minio.conf:/etc/nginx/conf.d/default.conf (16)
      - /PATH_to_FILE/public.crt:/etc/nginx/public.crt (17)
      - /PATH_to_FILE/private.key:/etc/nginx/private.key (17)
    networks:
      - minio_distributed (5)


volumes:
  minio1-data:

  minio2-data:

  minio3-data:

  minio4-data:


networks:
  minio_distributed: (5)
    driver: overlay

secrets:
  secret_key:
    external: true
  access_key:
    external: true

upstream minio_servers {
    server minio1:9000;  (1)
    server minio2:9000;  (1)
    server minio3:9000;  (1)
    server minio4:9000;  (1)
}
proxy_cache_path /var/tmp levels=1:2 keys_zone=my_cache:10m max_size=10g inactive=60m use_temp_path=off;
server {
    listen 80;
    server_name name.example.org; (2)
    return 301 https://name.example.org$request_uri;  (3)
}
server {
 listen  443 ssl;
 server_name name.example.org;

 # To allow special characters in headers
 ignore_invalid_headers off;
 # Allow any size file to be uploaded.
 # Set to a value such as 1000m; to restrict file size to a specific value
 client_max_body_size 0;
 # To disable buffering
 proxy_buffering off;

  ssl_certificate    /etc/nginx/public.crt; (4)
    ssl_certificate_key /etc/nginx/private.key; (4)
    ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
    ssl_ciphers         HIGH:!aNULL:!MD5;

 location / {
 proxy_cache      my_cache;
   proxy_set_header X-Real-IP $remote_addr;
   proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
   proxy_set_header X-Forwarded-Proto $scheme;
   proxy_set_header Host $http_host;

    proxy_set_header X-NginX-Proxy true;
    proxy_ssl_session_reuse off;
    proxy_redirect off;


   proxy_connect_timeout 300;
   # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
   proxy_http_version 1.1;
   proxy_set_header Connection "";
   chunked_transfer_encoding off;

   #proxy_pass http://minio1:9000; # If you are using docker-compose this would be the hostname i.e. minio
  proxy_pass http://minio_servers; (5)
   # Health Check endpoint might go here. See https://www.nginx.com/resources/wiki/modules/healthcheck/
   # /minio/health/live;
 }
}

# copy files to proxy server
scp minio.conf user@IP:/PATH_to_FILE/minio.conf (1)
scp private.key user@IP:/PATH_to_FILE/private.key (1)
scp public.crt user@IP:/PATH_to_FILE/public.crt (1)

docker stack deploy --compose-file=docker-compose.yaml minio_stack

curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
unzip awscliv2.zip
sudo ./aws/install

[default]
aws_secret_access_key = key (1)
aws_access_key_id = secret  (1)

[default]
s3 =
    signature_version = s3v4
region = us-east-1

wget https://dl.min.io/client/mc/release/linux-amd64/mc
chmod +x mc
./mc --help