Redis 3.2 Cluster on Debian 8 (9 Nodes on 3 Servers)

Aus Carl-Christian Sautter - Wiki
Wechseln zu: Navigation, Suche

Package Installation

apt-get install -t jessie-backports redis-server

Create File /etc/apt/preferences.d/redis-server

Package: redis-server
Pin: release a=jessie-backports
Pin-Priority: 200

Create File /etc/apt/preferences.d/redis-tools

Package: redis-tools
Pin: release a=jessie-backports
Pin-Priority: 200

Create 3 Redis-Nodes on Each Server

init.d (systemd link)

Copy /etc/init.d/redis-server to /etc/init.d/redis2 and /etc/init.d/redis3 and edit the files as follows.

/etc/init.d/redis2

#! /bin/sh
### BEGIN INIT INFO
# Provides:		redis2
# Required-Start:	$syslog $remote_fs
# Required-Stop:	$syslog $remote_fs
# Should-Start:		$local_fs
# Should-Stop:		$local_fs
# Default-Start:	2 3 4 5
# Default-Stop:		0 1 6
# Short-Description:	redis-server - Persistent key-value db
# Description:		redis-server - Persistent key-value db
### END INIT INFO


PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
DAEMON=/usr/bin/redis-server
DAEMON_ARGS=/etc/redis/redis2.conf
NAME=redis2
DESC=redis2

RUNDIR=/var/run/redis
PIDFILE=$RUNDIR/redis-server2.pid

test -x $DAEMON || exit 0

if [ -r /etc/default/$NAME ]
then
	. /etc/default/$NAME
fi

. /lib/lsb/init-functions

set -e

if [ "$(id -u)" != "0" ]
then
	log_failure_msg "Must be run as root."
	exit 1
fi

Run_parts () {
	if [ -d /etc/redis/${NAME}.${1}.d ]
	then
		su redis -s /bin/sh -c "run-parts --exit-on-error /etc/redis/${NAME}.${1}.d"
	fi
}

case "$1" in
  start)
	echo -n "Starting $DESC: "
	mkdir -p $RUNDIR
	touch $PIDFILE
	chown redis:redis $RUNDIR $PIDFILE
	chmod 755 $RUNDIR

	if [ -n "$ULIMIT" ]
	then
		ulimit -n $ULIMIT
	fi

	Run_parts pre-up

	if start-stop-daemon --start --quiet --oknodo --umask 007 --pidfile $PIDFILE --chuid redis:redis --exec $DAEMON -- $DAEMON_ARGS
	then
		Run_parts post-up
		echo "$NAME."
	else
		echo "failed"
	fi
	;;
  stop)
	echo -n "Stopping $DESC: "

	Run_parts pre-down

	if start-stop-daemon --stop --retry forever/TERM/1 --quiet --oknodo --pidfile $PIDFILE --exec $DAEMON
	then
		Run_parts post-down
		echo "$NAME."
	else
		echo "failed"
	fi
	rm -f $PIDFILE
	sleep 1
	;;

  restart|force-reload)
	${0} stop
	${0} start
	;;

  status)
	status_of_proc -p ${PIDFILE} ${DAEMON} ${NAME}
	;;

  *)
	echo "Usage: /etc/init.d/$NAME {start|stop|restart|force-reload|status}" >&2
	exit 1
	;;
esac

exit 0

/etc/init.d/redis3

#! /bin/sh
### BEGIN INIT INFO
# Provides:		redis3
# Required-Start:	$syslog $remote_fs
# Required-Stop:	$syslog $remote_fs
# Should-Start:		$local_fs
# Should-Stop:		$local_fs
# Default-Start:	2 3 4 5
# Default-Stop:		0 1 6
# Short-Description:	redis-server - Persistent key-value db
# Description:		redis-server - Persistent key-value db
### END INIT INFO


PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
DAEMON=/usr/bin/redis-server
DAEMON_ARGS=/etc/redis/redis3.conf
NAME=redis3
DESC=redis3

RUNDIR=/var/run/redis
PIDFILE=$RUNDIR/redis-server3.pid

test -x $DAEMON || exit 0

if [ -r /etc/default/$NAME ]
then
	. /etc/default/$NAME
fi

. /lib/lsb/init-functions

set -e

if [ "$(id -u)" != "0" ]
then
	log_failure_msg "Must be run as root."
	exit 1
fi

Run_parts () {
	if [ -d /etc/redis/${NAME}.${1}.d ]
	then
		su redis -s /bin/sh -c "run-parts --exit-on-error /etc/redis/${NAME}.${1}.d"
	fi
}

case "$1" in
  start)
	echo -n "Starting $DESC: "
	mkdir -p $RUNDIR
	touch $PIDFILE
	chown redis:redis $RUNDIR $PIDFILE
	chmod 755 $RUNDIR

	if [ -n "$ULIMIT" ]
	then
		ulimit -n $ULIMIT
	fi

	Run_parts pre-up

	if start-stop-daemon --start --quiet --oknodo --umask 007 --pidfile $PIDFILE --chuid redis:redis --exec $DAEMON -- $DAEMON_ARGS
	then
		Run_parts post-up
		echo "$NAME."
	else
		echo "failed"
	fi
	;;
  stop)
	echo -n "Stopping $DESC: "

	Run_parts pre-down

	if start-stop-daemon --stop --retry forever/TERM/1 --quiet --oknodo --pidfile $PIDFILE --exec $DAEMON
	then
		Run_parts post-down
		echo "$NAME."
	else
		echo "failed"
	fi
	rm -f $PIDFILE
	sleep 1
	;;

  restart|force-reload)
	${0} stop
	${0} start
	;;

  status)
	status_of_proc -p ${PIDFILE} ${DAEMON} ${NAME}
	;;

  *)
	echo "Usage: /etc/init.d/$NAME {start|stop|restart|force-reload|status}" >&2
	exit 1
	;;
esac

exit 0

Systemd

Copy /lib/systemd/system/redis-server.service to /lib/systemd/system/redis2.service and /lib/systemd/system/redis3.service and edit the files as follows.

/lib/systemd/system/redis2.service

[Unit]
Description=Advanced key-value store
After=network.target
Documentation=http://redis.io/documentation, man:redis-server(1)

[Service]
Type=forking
ExecStart=/usr/bin/redis-server /etc/redis/redis2.conf
PIDFile=/var/run/redis/redis-server2.pid
TimeoutStopSec=0
Restart=always
User=redis
Group=redis
RunTimeDirectory=redis

ExecStartPre=-/bin/run-parts --verbose /etc/redis/redis-server.pre-up.d
ExecStartPost=-/bin/run-parts --verbose /etc/redis/redis-server.post-up.d
ExecStop=-/bin/run-parts --verbose /etc/redis/redis-server.pre-down.d
ExecStop=/bin/kill -s TERM $MAINPID
ExecStopPost=-/bin/run-parts --verbose /etc/redis/redis-server.post-down.d

UMask=007
PrivateTmp=yes
LimitNOFILE=65535
PrivateDevices=yes
ProtectHome=yes
ReadOnlyDirectories=/
ReadWriteDirectories=-/var/lib/redis
ReadWriteDirectories=-/var/log/redis
ReadWriteDirectories=-/var/run/redis
CapabilityBoundingSet=~CAP_SYS_PTRACE

# redis-server writes its own config file when in cluster mode so we allow
# writing there (NB. ProtectSystem=true over ProtectSystem=full)
ProtectSystem=true
ReadWriteDirectories=-/etc/redis

[Install]
WantedBy=multi-user.target
Alias=redis2.service

/lib/systemd/system/redis3.service

[Unit]
Description=Advanced key-value store
After=network.target
Documentation=http://redis.io/documentation, man:redis-server(1)

[Service]
Type=forking
ExecStart=/usr/bin/redis-server /etc/redis/redis3.conf
PIDFile=/var/run/redis/redis-server3.pid
TimeoutStopSec=0
Restart=always
User=redis
Group=redis
RunTimeDirectory=redis

ExecStartPre=-/bin/run-parts --verbose /etc/redis/redis-server.pre-up.d
ExecStartPost=-/bin/run-parts --verbose /etc/redis/redis-server.post-up.d
ExecStop=-/bin/run-parts --verbose /etc/redis/redis-server.pre-down.d
ExecStop=/bin/kill -s TERM $MAINPID
ExecStopPost=-/bin/run-parts --verbose /etc/redis/redis-server.post-down.d

UMask=007
PrivateTmp=yes
LimitNOFILE=65535
PrivateDevices=yes
ProtectHome=yes
ReadOnlyDirectories=/
ReadWriteDirectories=-/var/lib/redis
ReadWriteDirectories=-/var/log/redis
ReadWriteDirectories=-/var/run/redis
CapabilityBoundingSet=~CAP_SYS_PTRACE

# redis-server writes its own config file when in cluster mode so we allow
# writing there (NB. ProtectSystem=true over ProtectSystem=full)
ProtectSystem=true
ReadWriteDirectories=-/etc/redis

[Install]
WantedBy=multi-user.target
Alias=redis3.service

Enable systemd Services

systemctl enable redis2.service
systemctl enable redis3.service

Configure Redis Nodes on Server 1

Create and/or edit following Files

/etc/redis/redis.conf

protected-mode no
port 7000
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis/redis-server.pid
loglevel notice
logfile /var/log/redis/redis-server.log
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /var/lib/redis
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
cluster-enabled yes
cluster-config-file nodes-7000.conf
cluster-node-timeout 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes

/etc/redis/redis2.conf

protected-mode no
port 7001
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis/redis-server2.pid
loglevel notice
logfile /var/log/redis/redis-server2.log
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump2.rdb
dir /var/lib/redis
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "appendonly2.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
cluster-enabled yes
cluster-config-file nodes-7001.conf
cluster-node-timeout 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes

/etc/redis/redis3.conf

protected-mode no
port 7002
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis/redis-server3.pid
loglevel notice
logfile /var/log/redis/redis-server3.log
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump3.rdb
dir /var/lib/redis
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "appendonly3.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
cluster-enabled yes
cluster-config-file nodes-7002.conf
cluster-node-timeout 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes

Start Redis Instances

service redis stop
service redis start
service redis2 start
service redis3 start

Check if Servers are running

# ps ax | grep redis
  999 ?        Ssl    0:01 /usr/bin/redis-server *:7002 [cluster]      
 1004 ?        Ssl    0:01 /usr/bin/redis-server *:7000 [cluster]     
 1020 ?        Ssl    0:01 /usr/bin/redis-server *:7001 [cluster]      

Configure Redis Nodes on Server 2&3

Copy scripts from server 1 and repeat steps

Create Cluster from Nodes

Install Gem and Ruby redis Package

gem install redis
gem install redis -v 3.3.3 # Debian Jessie 8

Add main nodes to Cluster

/usr/share/doc/redis-tools/examples/redis-trib.rb \
create \
10.0.0.1:7000 \
10.0.0.2:7000 \
10.0.0.3:7000

Add additinal Nodes as unused Masters

./redis-trib.rb add-node 10.0.0.1:7001 10.0.0.1:7000
./redis-trib.rb add-node 10.0.0.2:7001 10.0.0.1:7000
./redis-trib.rb add-node 10.0.0.3:7001 10.0.0.1:7000
./redis-trib.rb add-node 10.0.0.1:7002 10.0.0.1:7000
./redis-trib.rb add-node 10.0.0.2:7002 10.0.0.1:7000
./redis-trib.rb add-node 10.0.0.3:7002 10.0.0.1:7000

Get Overview:

#redis-cli -p 7000 -h 10.0.0.2 cluster nodes
65efd98cd770883f4b7b1499b7c5a69364372140 10.0.0.1:7002 master - 0 1483024348155 7 connected
8b19529803e3e0728a6d513b80a0a606a25c2dea 10.0.0.3:7002 master - 0 1483024348155 8 connected
aaea0e7653ec7266c5d9dc257c859ed5fea1795f 10.0.0.2:7001 master - 0 1483024347653 6 connected
5a9975342f52dd14cf562c40e1162b7a7d43e7e6 10.0.0.1:7000 master - 0 1483024347153 1 connected 0-5460
ad87a54352d95a942954fecc4159aa53ece4c186 10.0.0.2:7000 myself,master - 0 0 2 connected 5461-10922
fe88146cb72dd9d3de225b66367b52bd498c763f 10.0.0.3:7001 master - 0 1483024348656 0 connected
d6b5492a996870a419d6d11731bbdb6ec423a2dd 10.0.0.2:7002 master - 0 1483024348655 5 connected
a58dff2e021c8ba8d1bb65c46bdda74ad0c1b8a0 10.0.0.3:7000 master - 0 1483024348155 3 connected 10923-16383
741e95e2c9ba6b732cdc40a0e8c4d4b6f7e54a10 10.0.0.1:7001 master - 0 1483024348656 4 connected

Connect Replicas:

  • for host 1 use 2&3 as replica
  • for host 2 use 1&3 as replica
  • for host 3 use 1&2 as replica
redis-cli -p 7001 -h 10.0.0.2 cluster replicate 5a9975342f52dd14cf562c40e1162b7a7d43e7e6
redis-cli -p 7001 -h 10.0.0.3 cluster replicate 5a9975342f52dd14cf562c40e1162b7a7d43e7e6
redis-cli -p 7001 -h 10.0.0.1 cluster replicate ad87a54352d95a942954fecc4159aa53ece4c186
redis-cli -p 7002 -h 10.0.0.3 cluster replicate ad87a54352d95a942954fecc4159aa53ece4c186
redis-cli -p 7002 -h 10.0.0.1 cluster replicate a58dff2e021c8ba8d1bb65c46bdda74ad0c1b8a0
redis-cli -p 7002 -h 10.0.0.2 cluster replicate a58dff2e021c8ba8d1bb65c46bdda74ad0c1b8a0

Check Results:

#redis-cli -p 7000 -h 10.0.0.2 cluster nodes