Redis(單機&叢集)Pipeline工具類

筆者語錄： 我只想讓一切變得簡單。
提示： 本文會先給出測試程式碼及測試效果(使用示例)，然後再貼工具類程式碼。

---

效能對比(簡單)測試(含使用示例)：

測試單機redis使用進行普通操作與pipeline操作：

• 測試程式碼
  

• 測試結果
  

測試叢集redis使用進行普通操作與pipeline操作value：

• 測試程式碼
  

• 測試結果
  

測試叢集redis使用進行普通操作與pipeline操作hash：

• 測試程式碼
  

• 測試結果
  

---

Pipeline工具類：

• 相關(核心)依賴：

  <dependency>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-starter-data-redis</artifactId>
      <exclusions>
          <exclusion>
              <groupId>io.lettuce</groupId>
              <artifactId>lettuce-core</artifactId>
          </exclusion>
      </exclusions>
  </dependency>
  <dependency>
      <groupId>redis.clients</groupId>
      <artifactId>jedis</artifactId>
  </dependency>
  

• Pipeline工具類

  import com.niantou.redispipeline.author.JustryDeng;
  import lombok.extern.slf4j.Slf4j;
  import net.jcip.annotations.ThreadSafe;
  import org.springframework.beans.BeansException;
  import org.springframework.context.ApplicationContext;
  import org.springframework.context.ApplicationContextAware;
  import org.springframework.data.redis.connection.RedisClusterConnection;
  import org.springframework.data.redis.connection.RedisCommands;
  import org.springframework.data.redis.connection.RedisConnectionFactory;
  import org.springframework.data.redis.connection.jedis.JedisClusterConnection;
  import org.springframework.data.redis.core.RedisCallback;
  import org.springframework.data.redis.core.RedisTemplate;
  import org.springframework.data.redis.serializer.RedisSerializer;
  import org.springframework.lang.NonNull;
  import org.springframework.stereotype.Component;
  import org.springframework.util.Assert;
  import redis.clients.jedis.BinaryJedisCluster;
  import redis.clients.jedis.Jedis;
  import redis.clients.jedis.JedisCluster;
  import redis.clients.jedis.JedisClusterConnectionHandler;
  import redis.clients.jedis.JedisClusterInfoCache;
  import redis.clients.jedis.JedisPool;
  import redis.clients.jedis.Pipeline;
  import redis.clients.jedis.Response;
  import redis.clients.jedis.exceptions.JedisMovedDataException;
  import redis.clients.jedis.exceptions.JedisNoReachableClusterNodeException;
  import redis.clients.jedis.util.JedisClusterCRC16;
  
  import java.lang.reflect.Field;
  import java.util.ArrayList;
  import java.util.HashMap;
  import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.function.BiConsumer;
  import java.util.function.BiFunction;
  import java.util.function.Function;
  import java.util.stream.Collectors;
  
  /**
   * redis pipeline 工具類
   *
   * @author {@link JustryDeng}
   * @since 2020/11/13 2:41:40
   */
  @Slf4j
  @Component
  @ThreadSafe
  @SuppressWarnings("unused")
  public final class RedisPipelineUtil implements ApplicationContextAware {
      
      private static RedisTemplate<Object, Object> defaultRedisTemplate;
  
      /**
       * 獲取key序列化器
       *
       * @return  key序列化器
       */
      @NonNull
      public static RedisSerializer<Object> getKeySerializer() {
          //noinspection unchecked
          return (RedisSerializer<Object>) defaultRedisTemplate.getKeySerializer();
      }
  
      /**
       * 獲取value序列化器
       *
       * @return  value序列化器
       */
      @NonNull
      public static RedisSerializer<Object> getValueSerializer() {
          //noinspection unchecked
          return (RedisSerializer<Object>) defaultRedisTemplate.getValueSerializer();
      }
      
      /**
       * 獲取hash-key序列化器
       *
       * @return  hash-key序列化器
       */
      @NonNull
      public static RedisSerializer<Object> getHashKeySerializer() {
          //noinspection unchecked
          return (RedisSerializer<Object>)defaultRedisTemplate.getHashKeySerializer();
      }
      
      /**
       * 獲取hash-value序列化器
       *
       * @return  hash-value序列化器
       */
      @NonNull
      public static RedisSerializer<Object> getHashValueSerializer() {
          //noinspection unchecked
          return (RedisSerializer<Object>)defaultRedisTemplate.getHashValueSerializer();
      }
      
      /**
       * 流水線批量操作(單節點)
       * <p>
       * 注: SessionCallback是對RedisCallback的進一步封裝， 不過我們都已經使用pipeline了， 那乾脆直接用RedisCallback好了。
       *
       * @param biConsumer
       *            批量操作邏輯
       * @param paramList
       *            biConsumer用到的引數
       * @return  結果集
       */
      public static <R, P> List<R> pipeline4Standalone(BiConsumer<RedisCommands, P> biConsumer, final List<P> paramList) {
          //noinspection unchecked
          return (List<R>) defaultRedisTemplate.executePipelined((RedisCallback<R>) connection -> {
              for (P p : paramList) {
                  biConsumer.accept(connection, p);
              }
              return null;
          }, defaultRedisTemplate.getValueSerializer());
      }
  
      /**
       * 流水線批量操作(單節點)
       * <p>
       * 注: SessionCallback是對RedisCallback的進一步封裝， 不過我們都已經使用pipeline了， 那乾脆直接用RedisCallback好了。
       *
       * @param redisTemplate
       *            操作模板
       * @param biConsumer
       *            批量操作邏輯
       * @param paramList
       *            biConsumer用到的引數集合
       * @return  結果集
       */
      public static <R, P> List<R> pipeline4Standalone(RedisTemplate<?, ?> redisTemplate, BiConsumer<RedisCommands, P> biConsumer, final List<P> paramList) {
          //noinspection unchecked
          return (List<R>) redisTemplate.executePipelined((RedisCallback<R>) connection -> {
              for (P p : paramList) {
                  biConsumer.accept(connection, p);
              }
              return null;
          }, redisTemplate.getValueSerializer());
      }
      
      /**
       * 由於字串使用的相對較多， 這裡官(本)方(人)直接對字串提供出來一個操作
       * <p>
       * @see RedisPipelineUtil#pipeline4ClusterSimpleStr(RedisTemplate, BiFunction, List)
       */
      public static <R> List<R> pipeline4ClusterSimpleStr(BiFunction<Pipeline, PipelineParamSupplier<String>, Response<R>> biFunction,
                                                          List<String> paramList)
                                                          throws JedisMovedDataException {
          return RedisPipelineUtil.pipeline4ClusterSimpleStr(defaultRedisTemplate, biFunction, paramList);
      }
      
      /**
       * @see RedisPipelineUtil#pipeline4Cluster(JedisCluster, BiFunction, List)
       */
      @SuppressWarnings("rawtypes")
      public static <R> List<R>  pipeline4ClusterSimpleStr(@NonNull RedisTemplate redisTemplate,
                                                           BiFunction<Pipeline, PipelineParamSupplier<String>,
                                                           Response<R>> biFunction, List<String> paramList)
                                                           throws JedisMovedDataException {
          RedisConnectionFactory redisConnectionFactory = redisTemplate.getConnectionFactory();
          Assert.notNull(redisConnectionFactory, "redisConnectionFactory cannot be null");
          RedisClusterConnection clusterConnection = redisConnectionFactory.getClusterConnection();
          if (!(clusterConnection instanceof JedisClusterConnection)) {
              throw new UnsupportedOperationException("cannot support RedisClusterConnection [" +  clusterConnection.getClass().getName() + "]");
          }
          JedisCluster jedisCluster = ((JedisClusterConnection) clusterConnection).getNativeConnection();
          @SuppressWarnings("unchecked") RedisSerializer<Object> keySerializer = (RedisSerializer<Object>)redisTemplate.getKeySerializer();
          return RedisPipelineUtil.pipeline4ClusterSimpleStr(jedisCluster, keySerializer, biFunction, paramList);
      }
      
      /**
       * 為保證keySerializer與jedisCluster是配套的，這裡將此方法私有化，不對外提供
       * <p>
       * @see RedisPipelineUtil#pipeline4Cluster(JedisCluster, BiFunction, List)
       */
      private static <R> List<R> pipeline4ClusterSimpleStr(@NonNull JedisCluster jedisCluster, RedisSerializer<Object> keySerializer,
                                                           BiFunction<Pipeline, PipelineParamSupplier<String>, Response<R>> biFunction,
                                                           List<String> paramList)
              throws JedisMovedDataException {
          List<StringSelfSupplier> supplierParamList = paramList.stream().map(x -> new StringSelfSupplier(x, keySerializer)).collect(Collectors.toList());
          return RedisPipelineUtil.pipeline4Cluster(jedisCluster, biFunction, supplierParamList);
      }
      
      /**
       * @see RedisPipelineUtil#pipeline4Cluster(RedisTemplate, BiFunction, List)
       */
      public static <P extends PipelineParamSupplier<T>, T, R> List<R> pipeline4Cluster(BiFunction<Pipeline, PipelineParamSupplier<T>,
                                                                                        Response<R>> biFunction, List<P> paramList)
                                                                                        throws JedisMovedDataException {
          return RedisPipelineUtil.pipeline4Cluster(defaultRedisTemplate, biFunction, paramList);
      }
      
      /**
       * @see RedisPipelineUtil#pipeline4Cluster(JedisCluster, BiFunction, List)
       */
      @SuppressWarnings("rawtypes")
      public static <P extends PipelineParamSupplier<T>, T, R> List<R> pipeline4Cluster(@NonNull RedisTemplate redisTemplate,
                                                                                        BiFunction<Pipeline, PipelineParamSupplier<T>, Response<R>> biFunction,
                                                                                        List<P> paramList)
                                                                                        throws JedisMovedDataException {
          RedisConnectionFactory redisConnectionFactory = redisTemplate.getConnectionFactory();
          Assert.notNull(redisConnectionFactory, "redisConnectionFactory cannot be null");
          RedisClusterConnection clusterConnection = redisConnectionFactory.getClusterConnection();
          if (!(clusterConnection instanceof JedisClusterConnection)) {
              throw new UnsupportedOperationException("cannot support RedisClusterConnection [" +  clusterConnection.getClass().getName() + "]");
          }
          return RedisPipelineUtil.pipeline4Cluster(((JedisClusterConnection) clusterConnection).getNativeConnection(), biFunction, paramList);
      }
      
      /**
       * (使用JedisCluster，實現)流水線批量操作(叢集)
       *
       * @param jedisCluster
       *            JedisCluster例項
       * 其餘引數
       * @see JedisClusterPipeline#pipeline4Cluster(BiFunction, List)
       */
      public static <P extends PipelineParamSupplier<T>, T, R> List<R> pipeline4Cluster(@NonNull JedisCluster jedisCluster,
                                                                                        BiFunction<Pipeline, PipelineParamSupplier<T>,
                                                                                        Response<R>> biFunction, List<P> paramList)
                                                                                        throws JedisMovedDataException {
          JedisClusterPipeline jedisClusterPipeline = new JedisClusterPipeline(jedisCluster);
          return jedisClusterPipeline.pipeline(biFunction, paramList);
      }
      
      @Override
      @SuppressWarnings("rawtypes")
      public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
          // 初始化
          Map<String, RedisTemplate> beansOfType = applicationContext.getBeansOfType(RedisTemplate.class);
          Map.Entry<String, RedisTemplate> redisTemplateEntry = beansOfType.entrySet().stream()
                  .findFirst().orElseThrow(() -> new IllegalArgumentException(" cannot find any RedisTemplate"));
          //noinspection unchecked
          RedisPipelineUtil.defaultRedisTemplate = redisTemplateEntry.getValue();
          log.info(" use [{}] as the default RedisPipelineUtil's RedisTemplate", redisTemplateEntry.getKey());
      }
      
      /**
       * jedis使用pipeline操作redis-cluster輔助類
       *
       * 參考並整理自
       *     <a href="https://blog.csdn.net/youaremoon/article/details/51751991?utm_medium=distribute.pc_relevant.none-task-blog-searchFromBaidu-2.control&depth_1-utm_source=distribute.pc_relevant.none-task-blog-searchFromBaidu-2.control"/>
       *     <a href="https://www.cnblogs.com/xiaodf/p/11002184.html"/>
       *     <a href="https://blog.csdn.net/xiaoliu598906167/article/details/82218525?utm_medium=distribute.pc_aggpage_search_result.none-task-blog-2~all~first_rank_v2~rank_v25-10-82218525.nonecase&utm_term=pipeline%20redis%20%E8%BF%94%E5%9B%9E%E5%80%BC&spm=1000.2123.3001.4430"/>
       *
       * @author {@link JustryDeng}
       * @since 2020/11/27 16:29:59
       */
      public static class JedisClusterPipeline {
          
          private final JedisClusterConnectionHandler connectionHandler;
          
          private final JedisClusterInfoCache infoCache;
      
          public JedisClusterPipeline(JedisCluster jedisCluster) {
              try {
                  Field connectionHandlerField = BinaryJedisCluster.class.getDeclaredField("connectionHandler");
                  boolean accessible = connectionHandlerField.isAccessible();
                  connectionHandlerField.setAccessible(true);
                  this.connectionHandler = (JedisClusterConnectionHandler) connectionHandlerField.get(jedisCluster);
                  connectionHandlerField.setAccessible(accessible);
                  
                  Field cacheField = JedisClusterConnectionHandler.class.getDeclaredField("cache");
                  accessible = cacheField.isAccessible();
                  cacheField.setAccessible(true);
                  this.infoCache = (JedisClusterInfoCache) cacheField.get(connectionHandler);
                  cacheField.setAccessible(accessible);
              } catch (Exception e) {
                  throw new RuntimeException(e);
              }
          }
      
          /**
           * (使用JedisCluster，實現)流水線批量操作(叢集)
           * <p>
           * 注: 【據說】對叢集redis進行pipeline, Jedis比Lettuce快。
           * <p>
           * 泛型說明
           * <ul>
           *     <li>P：操作引數， 其需要實現{@link PipelineParamSupplier<T>}, 以獲得 1.redis-key  2.最終進行pipeline的操作引數</li>
           *     <li>T：最終的pipeline操作引數</li>
           *     <li>R: 為返回的資料集合泛型</li>
           * </ul>
           *
           * @param biFunction
           *            批量操作邏輯
           * @param paramList
           *            biFunction會用到的引數
           * @throws JedisMovedDataException
           *            key對應的slot槽點變化時丟擲
           * @return  結果集
           */
          public <P extends PipelineParamSupplier<T>, T, R> List<R> pipeline(BiFunction<Pipeline, PipelineParamSupplier<T>, Response<R>> biFunction,
                                                                             List<P> paramList) throws JedisMovedDataException {
              // 從paramList中抽取到對應的redis-key集合
              Map<byte[], P> redisKeyParamMap = paramList.stream().collect(Collectors.toMap(P::getRedisKey, Function.identity()));
              Set<byte[]> allKeys = redisKeyParamMap.keySet();
              Map<JedisPool, List<byte[]>> poolKeys = new HashMap<>(8);
              // 重新整理叢集資訊
              connectionHandler.renewSlotCache();
              for (byte[] key : allKeys) {
                  int slot = JedisClusterCRC16.getSlot(key);
                  JedisPool jedisPool = getJedisPoolFromSlot(slot);
                  if (poolKeys.containsKey(jedisPool)) {
                      List<byte[]> keys = poolKeys.get(jedisPool);
                      keys.add(key);
                  } else {
                      List<byte[]> keys = new ArrayList<>();
                      keys.add(key);
                      poolKeys.put(jedisPool, keys);
                  }
              }
              int size = allKeys.size();
              List<R> result = new ArrayList<>(size);
              List<Response<R>> responseList = new ArrayList<>(size);
              poolKeys.forEach((JedisPool jedisPool, List<byte[]> keys) -> {
                  Jedis jedis = jedisPool.getResource();
                  Pipeline pipeline = jedis.pipelined();
                  try {
                      keys.forEach(key -> responseList.add(biFunction.apply(pipeline, redisKeyParamMap.get(key))));
                  } finally {
                      try {
                          pipeline.close();
                      } catch (Exception e) {
                          log.error(e.getMessage());
                      }
                      try {
                          jedis.close();
                      } catch (Exception e) {
                          log.error(e.getMessage());
                      }
                  }
              });
              responseList.forEach(response -> result.add(response.get()));
              return result;
          }
      
          /**
           * 根據槽點獲取要對應使用的JedisPool
           */
          private JedisPool getJedisPoolFromSlot(int slot) {
              JedisPool connectionPool = infoCache.getSlotPool(slot);
              if (connectionPool != null) {
                  // It can't guaranteed to get valid connection because of node assignment
                  return connectionPool;
              } else {
                  // It's abnormal situation for cluster mode, that we have just nothing for slot, try to rediscover state
                  // 重新整理叢集資訊
                  connectionHandler.renewSlotCache();
                  connectionPool = infoCache.getSlotPool(slot);
                  if (connectionPool != null) {
                      return connectionPool;
                  } else {
                      throw new JedisNoReachableClusterNodeException("No reachable node in cluster for slot " + slot);
                  }
              }
          }
      }
      
      /**
       * Jedis操作redis-cluster時， pipeline操作引數提供器
       *
       * @author {@link JustryDeng}
       * @since 2020/11/28 16:45:40
       */
      public interface PipelineParamSupplier<T> {
          
          /**
           * 獲取(序列化後的)redis key
           * <p>
           * P.S. 在使用Pipeline操作叢集時，redis key使用這個方法獲取。
           * <p>
           * 注: 這裡之所以要將【獲取redis key】抽取為一個方法，是因為相關邏輯中有多個地方會用到。 如果這些地方在將key物件序列化為byte[]時，
           *     採用了不同的序列化方式， 那麼可能存在資料槽slot定位不一致的問題， 進而(因程式碼不當)引起JedisMovedDataException異常。
           *     為了避免上述問題，這裡將獲取redis-key的操作，抽取統一。
           *
           * @return  redis key
           */
          byte[] getRedisKey();
      
          /**
           * 獲取pipeline操作需要的引數
           * <p>
           *  P.S. 在使用Pipeline操作叢集時，redis key使用這個方法獲取。
           * @return pipeline操作引數
           */
          T getParam();
      }
      
      
      /**
       * 官(本)方(人)對常用的字串提供PipelineParamSupplier實現
       *
       * @author {@link JustryDeng}
       * @since 2020/11/25 22:06:23
       */
      public static class StringSelfSupplier implements PipelineParamSupplier<String> {
          
          private final String str;
          
          private final RedisSerializer<Object> keySerializer;
          
          public StringSelfSupplier(String str, RedisSerializer<Object> keySerializer) {
              this.str = str;
              this.keySerializer = keySerializer;
          }
          
          @Override
          public byte[] getRedisKey() {
              return keySerializer.serialize(getParam());
          }
          
          @Override
          public String getParam() {
              return this.str;
          }
      }
  }
  

---

pipeline工具類，簡單編寫完畢！

^_^ 如有不當之處，歡迎指正

^_^ 參考連結
         https://blog.csdn.net/youaremoon…
         https://www.cnblogs.com/xiaodf…
         https://blog.csdn.net/xiaoliu598906167…

^_^ 測試程式碼託管連結
         https://github.com/JustryDeng…redis-pipeline

^_^ 本文已經被收錄進《Spring原始碼梳理&實戰》，筆者JustryDeng