Upgrade to redispy 4.1.3 (min python 3.6) (#154)

Co-authored-by: Guy Korland <gkorland@gmail.com>
Co-authored-by: Chayim I. Kirshen <c@kirshen.com>

.github/workflows/ci.yml

@@ -15,8 +15,8 @@ jobs:
     timeout-minutes: 40
     strategy:
       matrix:
-        platform: ['ubuntu-20.04', 'ubuntu-18.04', 'ubuntu-16.04']
-        python: ['2.7', '3.6', '3.7', '3.8', '3.9']
+        platform: ['ubuntu-20.04']
+        python: ['3.6', '3.7', '3.8', '3.9']
 
     steps:
     - uses: actions/checkout@v2
@@ -51,19 +51,19 @@ jobs:
         poetry export --dev --without-hashes -o requirements-${{matrix.platform}}-${{matrix.python}}.txt
         pip install -r requirements-${{matrix.platform}}-${{matrix.python}}.txt
 
-    - name: Cache Redis
-      id: cache-redis
-      uses: actions/cache@v1
-      with:
-        path: redis
-        key: ${{ matrix.platform }}-${{ matrix.python }}-redis
-        restore-keys: |
-          ${{ matrix.platform }}-${{ matrix.python }}-redis
+    # - name: Cache Redis
+    #   id: cache-redis
+    #   uses: actions/cache@v1
+    #   with:
+    #     path: redis
+    #     key: ${{ matrix.platform }}-${{ matrix.python }}-redis
+    #     restore-keys: |
+    #       ${{ matrix.platform }}-${{ matrix.python }}-redis
 
     - name: Install Redis Server test dependencies
-      if: steps.cache-redis.outputs.cache-hit != 'true'
+      # if: steps.cache-redis.outputs.cache-hit != 'true'
       run: |
-        git clone https://github.com/redis/redis.git --branch unstable --depth 1
+        git clone https://github.com/redis/redis.git --branch 6.2.6 --depth 1
         cd redis
         make BUILD_TLS=yes -j
         ./src/redis-server --version
@@ -75,13 +75,13 @@ jobs:
         ./utils/gen-test-certs.sh
 
     - name: Unit Test with pytest
-      timeout-minutes: 10
+      timeout-minutes: 30
       run: |
         TLS_CERT=./redis/tests/tls/redis.crt \
         TLS_KEY=./redis/tests/tls/redis.key \
         TLS_CACERT=./redis/tests/tls/ca.crt \
         REDIS_BINARY=./redis/src/redis-server \
-        pytest --ignore=tests/flow --ignore=test_example.py
+        pytest --ignore=tests/flow --ignore=test_example.py -v
 
     - name: Install RLTest
       run: |
@@ -187,7 +187,7 @@ jobs:
         cd ../..
 
     - name: Generate coverage report
-      if: matrix.platform == 'ubuntu-18.04' && matrix.python == '3.6'
+      if: matrix.python == '3.6'
       run: |
         TLS_CERT=./redis/tests/tls/redis.crt \
         TLS_KEY=./redis/tests/tls/redis.key \
@@ -197,7 +197,7 @@ jobs:
 
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v1
-      if: matrix.platform == 'ubuntu-18.04' && matrix.python == '3.6'
+      if: matrix.python == '3.6'
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: ./coverage.xml

RLTest/__main__.py

@@ -283,6 +283,9 @@ def do_normal_conn(self, line):
 parser.add_argument(
     '--tls-ca-cert-file', default=None, help='/path/to/ca.crt')
 
+parser.add_argument(
+    '--tls-passphrase', default=None, help='passphrase to use on decript key file')
+
 class EnvScopeGuard:
     def __init__(self, runner):
         self.runner = runner
@@ -402,6 +405,7 @@ def __init__(self):
         Defaults.tls_cert_file = self.args.tls_cert_file
         Defaults.tls_key_file = self.args.tls_key_file
         Defaults.tls_ca_cert_file = self.args.tls_ca_cert_file
+        Defaults.tls_passphrase = self.args.tls_passphrase
         Defaults.oss_password = self.args.oss_password
         Defaults.cluster_node_timeout = self.args.cluster_node_timeout
         if Defaults.use_unix and Defaults.use_slaves:

RLTest/env.py

@@ -8,7 +8,6 @@
 import unittest
 import warnings
 
-from .Enterprise import EnterpriseClusterEnv
 from .exists_redis import ExistsRedisEnv
 from .redis_cluster import ClusterEnv
 from .redis_enterprise_cluster import EnterpriseRedisClusterEnv
@@ -115,6 +114,7 @@ class Defaults:
     tls_cert_file = None
     tls_key_file = None
     tls_ca_cert_file = None
+    tls_passphrase = None
     debugger = None
     debug_print = False
     debug_pause = False
@@ -147,6 +147,7 @@ def getKwargs(self):
             'tlsCertFile': self.tls_cert_file,
             'tlsKeyFile': self.tls_key_file,
             'tlsCaCertFile': self.tls_ca_cert_file,
+            'tlsPassphrase': self.tls_passphrase,
             'password': self.oss_password
         }
         return kwargs
@@ -168,7 +169,7 @@ def compareEnvs(self, env):
     def __init__(self, testName=None, testDescription=None, module=None,
                  moduleArgs=None, env=None, useSlaves=None, shardsCount=None, decodeResponses=None,
                  useAof=None, useRdbPreamble=None, forceTcp=False, useTLS=False, tlsCertFile=None, tlsKeyFile=None,
-                 tlsCaCertFile=None, logDir=None, redisBinaryPath=None, dmcBinaryPath=None,
+                 tlsCaCertFile=None, tlsPassphrase=None, logDir=None, redisBinaryPath=None, dmcBinaryPath=None,
                  redisEnterpriseBinaryPath=None, noDefaultModuleArgs=False, clusterNodeTimeout = None,
                  freshEnv=False):
 
@@ -199,6 +200,7 @@ def __init__(self, testName=None, testDescription=None, module=None,
         self.tlsCertFile = tlsCertFile if tlsCertFile else Defaults.tls_cert_file
         self.tlsKeyFile = tlsKeyFile if tlsKeyFile else Defaults.tls_key_file
         self.tlsCaCertFile = tlsCaCertFile if tlsCaCertFile else Defaults.tls_ca_cert_file
+        self.tlsPassphrase = tlsPassphrase if tlsPassphrase else Defaults.tls_passphrase
 
         self.redisBinaryPath = expandBinary(redisBinaryPath) if redisBinaryPath else Defaults.binary
         self.dmcBinaryPath = expandBinary(dmcBinaryPath) if dmcBinaryPath else Defaults.proxy_binary
@@ -301,6 +303,7 @@ def getEnvKwargs(self):
             'tlsKeyFile': self.tlsKeyFile,
             'tlsCaCertFile': self.tlsCaCertFile,
             'clusterNodeTimeout': self.clusterNodeTimeout,
+            'tlsPassphrase': self.tlsPassphrase,
             'port': self.port
         }
         return kwargs

RLTest/redis_cluster.py

@@ -1,9 +1,7 @@
 from __future__ import print_function
 
-from rediscluster.connection import SSLClusterConnection, ClusterConnectionPool
-
 from .redis_std import StandardEnv
-import rediscluster
+from redis.cluster import ClusterNode
 import redis
 import time
 from RLTest.utils import Colors
@@ -21,6 +19,7 @@ def __init__(self, **kwargs):
         useSlaves = kwargs.get('useSlaves', False)
         self.useTLS = kwargs['useTLS']
         self.decodeResponses = kwargs.get('decodeResponses', False)
+        self.tlsPassphrase = kwargs.get('tlsPassphrase', None)
         startPort = kwargs.pop('port', 10000)
         totalRedises = self.shardsCount * (2 if useSlaves else 1)
         randomizePorts = kwargs.pop('randomizePorts', False)
@@ -51,7 +50,11 @@ def waitCluster(self, timeout_sec=40):
             ok = 0
             for shard in self.shards:
                 con = shard.getConnection()
-                status = con.execute_command('CLUSTER', 'INFO')
+                try:
+                    status = con.execute_command('CLUSTER', 'INFO')
+                except Exception as e:
+                    print('got error on cluster info, will try again, %s' % str(e))
+                    continue
                 if 'cluster_state:ok' in str(status):
                     ok += 1
             if ok == len(self.shards):
@@ -115,28 +118,21 @@ def getConnection(self, shardId=1):
         return self.shards[shardId - 1].getConnection()
 
     def getClusterConnection(self):
+        statupNode = [ClusterNode(a['host'], a['port']) for a in self.getMasterNodesList()]
         if self.useTLS:
-            # workaround for error on
-            # got an unexpected keyword argument 'ssl'
-            # we enforce the connection_class instead of setting ssl=True
-            pool = ClusterConnectionPool(
-                startup_nodes=self.getMasterNodesList(),
-                connection_class=SSLClusterConnection,
-                ssl_cert_reqs=None,
+            return redis.RedisCluster(
+                ssl=True,
                 ssl_keyfile=self.shards[0].getTLSKeyFile(),
                 ssl_certfile=self.shards[0].getTLSCertFile(),
+                ssl_cert_reqs=None,
                 ssl_ca_certs=self.shards[0].getTLSCACertFile(),
-            )
-            if pool.connection_kwargs:
-                pool.connection_kwargs.pop('ssl', None)
-            return rediscluster.RedisCluster(
-                startup_nodes=self.getMasterNodesList(),
-                connection_pool=pool,
+                ssl_password=self.tlsPassphrase,
+                startup_nodes=statupNode,
                 decode_responses=self.decodeResponses
             )
         else:
-            return rediscluster.RedisCluster(
-                startup_nodes=self.getMasterNodesList(),
+            return redis.RedisCluster(
+                startup_nodes=statupNode,
                 decode_responses=self.decodeResponses, password=self.password)
 
     def getSlaveConnection(self):
@@ -159,26 +155,9 @@ def getOSSMasterNodesConnectionList(self):
 
     # Gets a cluster connection by key. On std redis the default connection is returned.
     def getConnectionByKey(self, key, command):
-        if self.useTLS:
-            # workaround for error on
-            # got an unexpected keyword argument 'ssl'
-            # we enforce the connection_class instead of setting ssl=True
-            pool = ClusterConnectionPool(
-                startup_nodes=self.getMasterNodesList(),
-                connection_class=SSLClusterConnection,
-                ssl_cert_reqs=None,
-                ssl_keyfile=self.shards[0].getTLSKeyFile(),
-                ssl_certfile=self.shards[0].getTLSCertFile(),
-                ssl_ca_certs=self.shards[0].getTLSCACertFile(),
-            )
-            if pool.connection_kwargs:
-                pool.connection_kwargs.pop('ssl', None)
-        else:
-            pool = ClusterConnectionPool(
-                startup_nodes=self.getMasterNodesList()
-            )
-        con = pool.get_connection_by_key(key, command)
-        return redis.StrictRedis(host=con.host, port=con.port,  decode_responses=self.decodeResponses, password=self.password)
+        clusterConn = self.getClusterConnection()
+        target_node = clusterConn._determine_nodes(command, key) # we will always which will give us the node responsible for the key
+        return clusterConn.get_redis_connection(target_node[0])
 
     def flush(self):
         self.getClusterConnection().flushall()

RLTest/redis_enterprise_cluster.py

@@ -1,4 +1,3 @@
-import rediscluster
 from redis import StrictRedis
 from .exists_redis import ExistsRedisEnv
 

RLTest/redis_std.py

@@ -21,7 +21,7 @@ class StandardEnv(object):
     def __init__(self, redisBinaryPath, port=6379, modulePath=None, moduleArgs=None, outputFilesFormat=None,
                  dbDirPath=None, useSlaves=False, serverId=1, password=None, libPath=None, clusterEnabled=False, decodeResponses=False,
                  useAof=False, useRdbPreamble=True, debugger=None, noCatch=False, unix=False, verbose=False, useTLS=False, tlsCertFile=None,
-                 tlsKeyFile=None, tlsCaCertFile=None, clusterNodeTimeout = None):
+                 tlsKeyFile=None, tlsCaCertFile=None, clusterNodeTimeout = None, tlsPassphrase = None):
         self.uuid = uuid.uuid4().hex
         self.redisBinaryPath = os.path.expanduser(redisBinaryPath) if redisBinaryPath.startswith(
             '~/') else redisBinaryPath
@@ -53,6 +53,7 @@ def __init__(self, redisBinaryPath, port=6379, modulePath=None, moduleArgs=None,
         self.tlsKeyFile = tlsKeyFile
         self.tlsCaCertFile = tlsCaCertFile
         self.clusterNodeTimeout = clusterNodeTimeout
+        self.tlsPassphrase = tlsPassphrase
 
         if port > 0:
             self.port = port
@@ -197,6 +198,8 @@ def createCmdArgs(self, role):
             cmdArgs += ['--tls-cert-file', self.getTLSCertFile()]
             cmdArgs += ['--tls-key-file', self.getTLSKeyFile()]
             cmdArgs += ['--tls-ca-cert-file', self.getTLSCACertFile()]
+            if self.tlsPassphrase:
+                cmdArgs += ['--tls-key-file-pass', self.tlsPassphrase]
 
         return cmdArgs
 
@@ -362,6 +365,7 @@ def _getConnection(self, role):
             return redis.StrictRedis('localhost', self.getPort(role),
                                      password=self.password,
                                      ssl=True,
+                                     ssl_password=self.tlsPassphrase,
                                      ssl_keyfile=self.getTLSKeyFile(),
                                      ssl_certfile=self.getTLSCertFile(),
                                      ssl_cert_reqs=None,

pyproject.toml

@@ -24,10 +24,9 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^2.7,<2.8 || >= 3.5.0"
+python = ">= 3.6.0"
 distro = "^1.5.0"
-redis = "^3.5.3"
-redis-py-cluster = "*"
+redis = "^4.1.2"
 psutil = "5.8.0" # 5.9.0 currently broken on macOS
 pytest-cov = "2.5"
 
@@ -43,7 +42,7 @@ flake8 = "*"
 rmtest = "^0.7.0"
 nose = "^1.3.7"
 ml2rt = "^0.2.0"
-pytest = "4.6"
+pytest = "^6.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]