Upgrade to redispy 4.1.3 (min python 3.6)

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         platform: ['ubuntu-20.04', 'ubuntu-18.04', 'ubuntu-16.04']
-        python: ['2.7', '3.6', '3.7', '3.8', '3.9']
+        python: ['3.6', '3.7', '3.8', '3.9']
 
     steps:
     - uses: actions/checkout@v2

RLTest/__main__.py

@@ -283,6 +283,9 @@ def do_normal_conn(self, line):
 parser.add_argument(
     '--tls-ca-cert-file', default=None, help='/path/to/ca.crt')
 
+parser.add_argument(
+    '--tls-passphrase', default=None, help='passphrase to use on decript key file')
+
 class EnvScopeGuard:
     def __init__(self, runner):
         self.runner = runner
@@ -402,6 +405,7 @@ def __init__(self):
         Defaults.tls_cert_file = self.args.tls_cert_file
         Defaults.tls_key_file = self.args.tls_key_file
         Defaults.tls_ca_cert_file = self.args.tls_ca_cert_file
+        Defaults.tls_passphrase = self.args.tls_passphrase
         Defaults.oss_password = self.args.oss_password
         Defaults.cluster_node_timeout = self.args.cluster_node_timeout
         if Defaults.use_unix and Defaults.use_slaves:

RLTest/env.py

@@ -8,7 +8,6 @@
 import unittest
 import warnings
 
-from .Enterprise import EnterpriseClusterEnv
 from .exists_redis import ExistsRedisEnv
 from .redis_cluster import ClusterEnv
 from .redis_enterprise_cluster import EnterpriseRedisClusterEnv
@@ -115,6 +114,7 @@ class Defaults:
     tls_cert_file = None
     tls_key_file = None
     tls_ca_cert_file = None
+    tls_passphrase = None
     debugger = None
     debug_print = False
     debug_pause = False
@@ -146,6 +146,7 @@ def getKwargs(self):
             'tlsCertFile': self.tls_cert_file,
             'tlsKeyFile': self.tls_key_file,
             'tlsCaCertFile': self.tls_ca_cert_file,
+            'tlsPassphrase': self.tls_passphrase,
             'password': self.oss_password
         }
         return kwargs
@@ -167,7 +168,7 @@ def compareEnvs(self, env):
     def __init__(self, testName=None, testDescription=None, module=None,
                  moduleArgs=None, env=None, useSlaves=None, shardsCount=None, decodeResponses=None,
                  useAof=None, useRdbPreamble=None, forceTcp=False, useTLS=False, tlsCertFile=None, tlsKeyFile=None,
-                 tlsCaCertFile=None, logDir=None, redisBinaryPath=None, dmcBinaryPath=None,
+                 tlsCaCertFile=None, tlsPassphrase=None, logDir=None, redisBinaryPath=None, dmcBinaryPath=None,
                  redisEnterpriseBinaryPath=None, noDefaultModuleArgs=False, clusterNodeTimeout = None,
                  freshEnv=False):
 
@@ -198,6 +199,7 @@ def __init__(self, testName=None, testDescription=None, module=None,
         self.tlsCertFile = tlsCertFile if tlsCertFile else Defaults.tls_cert_file
         self.tlsKeyFile = tlsKeyFile if tlsKeyFile else Defaults.tls_key_file
         self.tlsCaCertFile = tlsCaCertFile if tlsCaCertFile else Defaults.tls_ca_cert_file
+        self.tlsPassphrase = tlsPassphrase if tlsPassphrase else Defaults.tls_passphrase
 
         self.redisBinaryPath = expandBinary(redisBinaryPath) if redisBinaryPath else Defaults.binary
         self.dmcBinaryPath = expandBinary(dmcBinaryPath) if dmcBinaryPath else Defaults.proxy_binary
@@ -298,7 +300,8 @@ def getEnvKwargs(self):
             'tlsCertFile': self.tlsCertFile,
             'tlsKeyFile': self.tlsKeyFile,
             'tlsCaCertFile': self.tlsCaCertFile,
-            'clusterNodeTimeout': self.clusterNodeTimeout
+            'clusterNodeTimeout': self.clusterNodeTimeout,
+            'tlsPassphrase': self.tlsPassphrase
         }
         return kwargs
 

RLTest/redis_cluster.py

@@ -1,9 +1,7 @@
 from __future__ import print_function
 
-from rediscluster.connection import SSLClusterConnection, ClusterConnectionPool
-
 from .redis_std import StandardEnv
-import rediscluster
+from redis.cluster import ClusterNode
 import redis
 import time
 from RLTest.utils import Colors
@@ -21,6 +19,7 @@ def __init__(self, **kwargs):
         useSlaves = kwargs.get('useSlaves', False)
         self.useTLS = kwargs['useTLS']
         self.decodeResponses = kwargs.get('decodeResponses', False)
+        self.tlsPassphrase = kwargs.get('tlsPassphrase', None)
         startPort = 20000
         totalRedises = self.shardsCount * (2 if useSlaves else 1)
         randomizePorts = kwargs.pop('randomizePorts', False)
@@ -51,7 +50,11 @@ def waitCluster(self, timeout_sec=40):
             ok = 0
             for shard in self.shards:
                 con = shard.getConnection()
-                status = con.execute_command('CLUSTER', 'INFO')
+                try:
+                    status = con.execute_command('CLUSTER', 'INFO')
+                except Exception as e:
+                    print('got error on cluster info, will try again, %s' % str(e))
+                    continue
                 if 'cluster_state:ok' in str(status):
                     ok += 1
             if ok == len(self.shards):
@@ -115,28 +118,21 @@ def getConnection(self, shardId=1):
         return self.shards[shardId - 1].getConnection()
 
     def getClusterConnection(self):
+        statupNode = [ClusterNode(a['host'], a['port']) for a in self.getMasterNodesList()]
         if self.useTLS:
-            # workaround for error on
-            # got an unexpected keyword argument 'ssl'
-            # we enforce the connection_class instead of setting ssl=True
-            pool = ClusterConnectionPool(
-                startup_nodes=self.getMasterNodesList(),
-                connection_class=SSLClusterConnection,
-                ssl_cert_reqs=None,
+            return redis.RedisCluster(
+                ssl=True,
                 ssl_keyfile=self.shards[0].getTLSKeyFile(),
                 ssl_certfile=self.shards[0].getTLSCertFile(),
+                ssl_cert_reqs=None,
                 ssl_ca_certs=self.shards[0].getTLSCACertFile(),
-            )
-            if pool.connection_kwargs:
-                pool.connection_kwargs.pop('ssl', None)
-            return rediscluster.RedisCluster(
-                startup_nodes=self.getMasterNodesList(),
-                connection_pool=pool,
+                ssl_password=self.tlsPassphrase,
+                startup_nodes=statupNode,
                 decode_responses=self.decodeResponses
             )
         else:
-            return rediscluster.RedisCluster(
-                startup_nodes=self.getMasterNodesList(),
+            return redis.RedisCluster(
+                startup_nodes=statupNode,
                 decode_responses=self.decodeResponses, password=self.password)
 
     def getSlaveConnection(self):
@@ -159,26 +155,9 @@ def getOSSMasterNodesConnectionList(self):
 
     # Gets a cluster connection by key. On std redis the default connection is returned.
     def getConnectionByKey(self, key, command):
-        if self.useTLS:
-            # workaround for error on
-            # got an unexpected keyword argument 'ssl'
-            # we enforce the connection_class instead of setting ssl=True
-            pool = ClusterConnectionPool(
-                startup_nodes=self.getMasterNodesList(),
-                connection_class=SSLClusterConnection,
-                ssl_cert_reqs=None,
-                ssl_keyfile=self.shards[0].getTLSKeyFile(),
-                ssl_certfile=self.shards[0].getTLSCertFile(),
-                ssl_ca_certs=self.shards[0].getTLSCACertFile(),
-            )
-            if pool.connection_kwargs:
-                pool.connection_kwargs.pop('ssl', None)
-        else:
-            pool = ClusterConnectionPool(
-                startup_nodes=self.getMasterNodesList()
-            )
-        con = pool.get_connection_by_key(key, command)
-        return redis.StrictRedis(host=con.host, port=con.port,  decode_responses=self.decodeResponses, password=self.password)
+        clusterConn = self.getClusterConnection()
+        target_node = clusterConn._determine_nodes(command, key) # we will always which will give us the node responsible for the key
+        return clusterConn.get_redis_connection(target_node[0])
 
     def flush(self):
         self.getClusterConnection().flushall()

RLTest/redis_enterprise_cluster.py

@@ -1,4 +1,3 @@
-import rediscluster
 from redis import StrictRedis
 from .exists_redis import ExistsRedisEnv
 

RLTest/redis_std.py

@@ -21,7 +21,7 @@ class StandardEnv(object):
     def __init__(self, redisBinaryPath, port=6379, modulePath=None, moduleArgs=None, outputFilesFormat=None,
                  dbDirPath=None, useSlaves=False, serverId=1, password=None, libPath=None, clusterEnabled=False, decodeResponses=False,
                  useAof=False, useRdbPreamble=True, debugger=None, noCatch=False, unix=False, verbose=False, useTLS=False, tlsCertFile=None,
-                 tlsKeyFile=None, tlsCaCertFile=None, clusterNodeTimeout = None):
+                 tlsKeyFile=None, tlsCaCertFile=None, clusterNodeTimeout = None, tlsPassphrase = None):
         self.uuid = uuid.uuid4().hex
         self.redisBinaryPath = os.path.expanduser(redisBinaryPath) if redisBinaryPath.startswith(
             '~/') else redisBinaryPath
@@ -53,6 +53,7 @@ def __init__(self, redisBinaryPath, port=6379, modulePath=None, moduleArgs=None,
         self.tlsKeyFile = tlsKeyFile
         self.tlsCaCertFile = tlsCaCertFile
         self.clusterNodeTimeout = clusterNodeTimeout
+        self.tlsPassphrase = tlsPassphrase
 
         if port > 0:
             self.port = port
@@ -197,6 +198,8 @@ def createCmdArgs(self, role):
             cmdArgs += ['--tls-cert-file', self.getTLSCertFile()]
             cmdArgs += ['--tls-key-file', self.getTLSKeyFile()]
             cmdArgs += ['--tls-ca-cert-file', self.getTLSCACertFile()]
+            if self.tlsPassphrase:
+                cmdArgs += ['--tls-key-file-pass', self.tlsPassphrase]
 
         return cmdArgs
 
@@ -362,6 +365,7 @@ def _getConnection(self, role):
             return redis.StrictRedis('localhost', self.getPort(role),
                                      password=self.password,
                                      ssl=True,
+                                     ssl_password=self.tlsPassphrase,
                                      ssl_keyfile=self.getTLSKeyFile(),
                                      ssl_certfile=self.getTLSCertFile(),
                                      ssl_cert_reqs=None,

pyproject.toml

@@ -24,10 +24,9 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^2.7,<2.8 || >= 3.5.0"
+python = ">= 3.6.0 || < 4.0.0"
 distro = "^1.5.0"
-redis = "^3.5.3"
-redis-py-cluster = "*"
+redis = "^4.1.2"
 psutil = "5.8.0" # 5.9.0 currently broken on macOS
 pytest-cov = "2.5"