本文主要介紹使用 Java 來操作 Pulsar,文中所使用到的軟體版本:Java 17.0.7(Pulsar 服務使用)、Java 1.8.0_341(客戶端使用)、Pulsar 3.3.0、pulsar-client 3.3.0。
1、引入依賴
<dependency> <groupId>org.apache.pulsar</groupId> <artifactId>pulsar-client</artifactId> <version>3.3.0</version> </dependency> <dependency> <groupId>org.apache.pulsar</groupId> <artifactId>pulsar-client-admin</artifactId> <version>3.3.0</version> </dependency>
2、初始化 PulsarClient 和 PulsarAdmin
PulsarClient:
@Before public void before() throws PulsarClientException { client = PulsarClient.builder() .serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650") .build(); }
PulsarAdmin:
@Before public void before() throws PulsarClientException { admin = PulsarAdmin.builder() .serviceHttpUrl("http://10.49.196.30:8080,10.49.196.31:8080,10.49.196.32:8080") .build(); }
3、消費者
3.1、同步消費
@Test public void sync() throws PulsarClientException { Consumer<String> consumer = client.newConsumer(Schema.STRING) .topic("my-topic") //主題 .subscriptionName("my-subscription") //訂閱名稱 .subscriptionType(SubscriptionType.Shared) //訂閱模式 .subscribe(); while (true) { Message<String> message = consumer.receive(); try { log.info("topicName={},value={}", message.getTopicName(), message.getValue()); consumer.acknowledge(message); } catch (Exception e) { consumer.negativeAcknowledge(message); } } }
3.2、非同步消費
public void async() throws InterruptedException { client.newConsumer(Schema.STRING) .topic("my-topic2") .subscriptionName("my-subscription") .subscriptionType(SubscriptionType.Shared) .subscribeAsync() .thenAccept(this::receiveAsync); Thread.sleep(1000 * 500); } private void receiveAsync(Consumer<String> consumer) { consumer.receiveAsync().thenAccept(message -> { try { log.info("messageId={},value={}", message.getMessageId(), message.getValue()); consumer.acknowledge(message); } catch (Exception e) { consumer.negativeAcknowledge(message); } receiveAsync(consumer); }); }
3.3、使用 MessageListener
@Test public void messageListener() throws Exception { PulsarClient client = PulsarClient.builder() .serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650") .listenerThreads(3) .build(); MessageListener<String> messageListener = (consumer, msg) -> { try { log.info("messageId={},value={}", msg.getMessageId(), msg.getValue()); consumer.acknowledge(msg); } catch (Exception e) { consumer.negativeAcknowledge(msg); } }; client.newConsumer(Schema.STRING) .topic("my-topic") .subscriptionName("my-subscription") .subscriptionType(SubscriptionType.Shared) .messageListener(messageListener) .subscribe(); Thread.sleep(1000 * 500); }
3.4、重試信主題
/** * 重試信主題 * 處理失敗的訊息會進入重試信主題,達到最大重試次數後進入死信主題 */ @Test public void retryTopic() throws Exception { Consumer<String> consumer = client.newConsumer(Schema.STRING) .topic("my-topic-r") .subscriptionName("my-subscription") .subscriptionType(SubscriptionType.Shared) .enableRetry(true) .deadLetterPolicy(DeadLetterPolicy.builder() .maxRedeliverCount(5) .build()) .subscribe(); while (true) { Message<String> message = consumer.receive(); try { log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue()); //TODO:業務處理,可能發生異常 //throw new RuntimeException(); consumer.acknowledge(message); } catch (Exception e) { log.error("", e); consumer.reconsumeLater(message, 5L, TimeUnit.SECONDS); } } }
3.5、死信主題
/** * 死信主題 * 由確認超時、負面確認或重試信主題 三種情況訊息處理失敗後,重試最大次數後訊息會進入死信主題 */ @Test public void deadTopic() throws Exception { Consumer<String> consumer = client.newConsumer(Schema.STRING) .topic("my-topic-d") .subscriptionName("my-subscription") .subscriptionType(SubscriptionType.Shared) .deadLetterPolicy(DeadLetterPolicy.builder() .maxRedeliverCount(5) .initialSubscriptionName("init-sub") .build()) .subscribe(); while (true) { Message<String> message = consumer.receive(); try { log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue()); //TODO:業務處理,可能發生異常 //throw new RuntimeException(); consumer.acknowledge(message); } catch (Exception e) { log.error("", e); consumer.negativeAcknowledge(message); } } } /** * 訂閱死信佇列,處理其中的訊息 */ @Test public void consumerDeadTopic() throws Exception { Consumer<String> consumer = client.newConsumer(Schema.STRING) .topic("my-topic-d-my-subscription-DLQ") .subscriptionName("init-sub") .subscriptionType(SubscriptionType.Shared) .subscribe(); while (true) { Message<String> message = consumer.receive(); try { log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue()); } catch (Exception e) { log.error("", e); } consumer.acknowledge(message); } }
3.6、完整程式碼
package com.abc.demo.pulsar; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.*; import org.junit.Before; import org.junit.Test; import java.util.concurrent.TimeUnit; @Slf4j public class ConsumerCase { private PulsarClient client; @Before public void before() throws PulsarClientException { client = PulsarClient.builder() .serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650") .build(); } @Test public void sync() throws PulsarClientException { Consumer<String> consumer = client.newConsumer(Schema.STRING) .topic("my-topic") //主題 .subscriptionName("my-subscription") //訂閱名稱 .subscriptionType(SubscriptionType.Shared) //訂閱模式 .subscribe(); while (true) { Message<String> message = consumer.receive(); try { log.info("topicName={},value={}", message.getTopicName(), message.getValue()); consumer.acknowledge(message); } catch (Exception e) { consumer.negativeAcknowledge(message); } } } @Test public void async() throws InterruptedException { client.newConsumer(Schema.STRING) .topic("my-topic2") .subscriptionName("my-subscription") .subscriptionType(SubscriptionType.Shared) .subscribeAsync() .thenAccept(this::receiveAsync); Thread.sleep(1000 * 500); } private void receiveAsync(Consumer<String> consumer) { consumer.receiveAsync().thenAccept(message -> { try { log.info("messageId={},value={}", message.getMessageId(), message.getValue()); consumer.acknowledge(message); } catch (Exception e) { consumer.negativeAcknowledge(message); } receiveAsync(consumer); }); } @Test public void messageListener() throws Exception { PulsarClient client = PulsarClient.builder() .serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650") .listenerThreads(3) .build(); MessageListener<String> messageListener = (consumer, msg) -> { try { log.info("messageId={},value={}", msg.getMessageId(), msg.getValue()); consumer.acknowledge(msg); } catch (Exception e) { consumer.negativeAcknowledge(msg); } }; client.newConsumer(Schema.STRING) .topic("my-topic") .subscriptionName("my-subscription") .subscriptionType(SubscriptionType.Shared) .messageListener(messageListener) .subscribe(); Thread.sleep(1000 * 500); } /** * 重試信主題 * 處理失敗的訊息會進入重試信主題,達到最大重試次數後進入死信主題 */ @Test public void retryTopic() throws Exception { Consumer<String> consumer = client.newConsumer(Schema.STRING) .topic("my-topic-r") .subscriptionName("my-subscription") .subscriptionType(SubscriptionType.Shared) .enableRetry(true) .deadLetterPolicy(DeadLetterPolicy.builder() .maxRedeliverCount(5) .build()) .subscribe(); while (true) { Message<String> message = consumer.receive(); try { log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue()); //TODO:業務處理,可能發生異常 //throw new RuntimeException(); consumer.acknowledge(message); } catch (Exception e) { log.error("", e); consumer.reconsumeLater(message, 5L, TimeUnit.SECONDS); } } } /** * 死信主題 * 由確認超時、負面確認或重試信主題 三種情況訊息處理失敗後,重試最大次數後訊息會進入死信主題 */ @Test public void deadTopic() throws Exception { Consumer<String> consumer = client.newConsumer(Schema.STRING) .topic("my-topic-d") .subscriptionName("my-subscription") .subscriptionType(SubscriptionType.Shared) .deadLetterPolicy(DeadLetterPolicy.builder() .maxRedeliverCount(5) .initialSubscriptionName("init-sub") .build()) .subscribe(); while (true) { Message<String> message = consumer.receive(); try { log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue()); //TODO:業務處理,可能發生異常 //throw new RuntimeException(); consumer.acknowledge(message); } catch (Exception e) { log.error("", e); consumer.negativeAcknowledge(message); } } } /** * 訂閱死信佇列,處理其中的訊息 */ @Test public void consumerDeadTopic() throws Exception { Consumer<String> consumer = client.newConsumer(Schema.STRING) .topic("my-topic-d-my-subscription-DLQ") .subscriptionName("init-sub") .subscriptionType(SubscriptionType.Shared) .subscribe(); while (true) { Message<String> message = consumer.receive(); try { log.info("messageId={},topicName={},value={}", message.getMessageId(), message.getTopicName(), message.getValue()); } catch (Exception e) { log.error("", e); } consumer.acknowledge(message); } } }
4、Reader
@Test public void reader() throws Exception { //MessageId messageId = MessageId.fromByteArray(Base64.getDecoder().decode("CBcQBTAA")); MessageId messageId = MessageId.earliest; Reader reader = client.newReader(Schema.STRING) .topic("my-topic") .startMessageId(messageId) .create(); while (true) { Message msg = reader.readNext(); log.info("messageId={},messageIdBase64={},value={}", msg.getMessageId(), Base64.getEncoder().encodeToString(msg.getMessageId().toByteArray()), msg.getValue()); } }
完整程式碼:
package com.abc.demo.pulsar; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.*; import org.junit.Before; import org.junit.Test; import java.util.Base64; @Slf4j public class ReaderCase { private PulsarClient client; @Before public void before() throws PulsarClientException { client = PulsarClient.builder() .serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650") .build(); } @Test public void reader() throws Exception { //MessageId messageId = MessageId.fromByteArray(Base64.getDecoder().decode("CBcQBTAA")); MessageId messageId = MessageId.earliest; Reader reader = client.newReader(Schema.STRING) .topic("my-topic") .startMessageId(messageId) .create(); while (true) { Message msg = reader.readNext(); log.info("messageId={},messageIdBase64={},value={}", msg.getMessageId(), Base64.getEncoder().encodeToString(msg.getMessageId().toByteArray()), msg.getValue()); } } }
5、生產者
5.1、同步傳送
@Test public void sync() throws PulsarClientException { Producer<String> producer = client.newProducer(Schema.STRING) .topic("my-topic-d") .create(); for (int i = 0; i < 3; i++) { MessageId messageId = producer.send(("hello" + i)); log.info("messageId={}", messageId); } producer.close(); }
5.2、非同步傳送
@Test public void async() throws InterruptedException { client.newProducer(Schema.STRING) .topic("my-topic2") .createAsync() .thenAccept(producer -> { for (int i = 0; i < 10; i++) { producer.sendAsync("hello" + i).thenAccept(messageId -> { log.info("messageId={}", messageId); }); } }); Thread.sleep(1000 * 5); }
5.3、詳細設定訊息
@Test public void configMessage() throws PulsarClientException { Producer<byte[]> producer = client.newProducer() .topic("my-topic") .create(); MessageId messageId = producer.newMessage(Schema.STRING) .key("my-key") //設定訊息key .eventTime(System.currentTimeMillis()) //設定事件事件 .sequenceId(123) //設定 sequenceId .deliverAfter(1, TimeUnit.MINUTES) //延遲投遞訊息 .property("my-key", "my-value") //自定義屬性 .value("content") .send(); log.info("messageId={}", messageId); producer.close(); }
5.4、完整程式碼
package com.abc.demo.pulsar; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.api.*; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @Slf4j public class ProducerCase { private PulsarClient client; @Before public void before() throws PulsarClientException { client = PulsarClient.builder() //.serviceUrl("pulsar://10.49.196.33:6650") .serviceUrl("pulsar://10.49.196.30:6650,10.49.196.31:6650,10.49.196.32:6650") .build(); } @After public void after() throws PulsarClientException { client.close(); } @Test public void sync() throws PulsarClientException { Producer<String> producer = client.newProducer(Schema.STRING) .topic("my-topic-d") .create(); for (int i = 0; i < 3; i++) { MessageId messageId = producer.send(("hello" + i)); log.info("messageId={}", messageId); } producer.close(); } @Test public void async() throws InterruptedException { client.newProducer(Schema.STRING) .topic("my-topic2") .createAsync() .thenAccept(producer -> { for (int i = 0; i < 10; i++) { producer.sendAsync("hello" + i).thenAccept(messageId -> { log.info("messageId={}", messageId); }); } }); Thread.sleep(1000 * 5); } @Test public void configMessage() throws PulsarClientException { Producer<byte[]> producer = client.newProducer() .topic("my-topic") .create(); MessageId messageId = producer.newMessage(Schema.STRING) .key("my-key") //設定訊息key .eventTime(System.currentTimeMillis()) //設定事件事件 .sequenceId(123) //設定 sequenceId .deliverAfter(1, TimeUnit.MINUTES) //延遲投遞訊息 .property("my-key", "my-value") //自定義屬性 .value("content") .send(); log.info("messageId={}", messageId); producer.close(); } }
6、Admin
6.1、Brokers
6.1.1、列出活動 broker
admin.brokers().getActiveBrokers(clusterName)
6.1.2、列出 broker 的名稱空間
admin.brokers().getOwnedNamespaces(cluster,brokerUrl);
6.1.3、獲取動態配置名稱
admin.brokers().getDynamicConfigurationNames();
6.1.4、更新動態配置
admin.brokers().updateDynamicConfiguration(configName, configValue);
6.1.5、獲取已經更新過的動態配置
admin.brokers().getAllDynamicConfigurations();
6.1.6、獲取 leader broker
admin.brokers().getLeaderBroker()
6.2、Clusters
6.2.1、獲取叢集配置資訊
admin.clusters().getCluster(clusterName);
6.2.2、獲取叢集列表
admin.clusters().getClusters();
6.3、Tenant
6.3.1、列出租戶
admin.tenants().getTenants();
6.3.2、建立租戶
admin.tenants().createTenant(tenantName, tenantInfo);
6.3.3、獲取租戶配置資訊
admin.tenants().getTenantInfo(tenantName);
6.3.4、刪除租戶
admin.Tenants().deleteTenant(tenantName);
6.3.5、更新租戶
admin.tenants().updateTenant(tenantName, tenantInfo);
6.4、Namespaces
6.4.1、建立名稱空間
admin.namespaces().createNamespace(namespace);
6.4.2、獲取名稱空間策略
admin.namespaces().getPolicies(namespace);
6.4.3、列出名稱空間
admin.namespaces().getNamespaces(tenant);
6.4.4、刪除名稱空間
admin.namespaces().deleteNamespace(namespace);
6.5、Topics
6.5.1、檢視訂閱的待消費訊息(不會消費訊息)
admin.topics().peekMessages(topic, subName, numMessages);
6.5.2、透過訊息 ID 獲取訊息
admin.topics().getMessageById(topic, ledgerId, entryId);
6.5.3、透過相對於最早或最新訊息的位置來查詢訊息
admin.topics().examineMessage(topic, "latest", 1);
6.5.4、透過時間獲取訊息的 ID
admin.topics().getMessageIdByTimestamp(topic, timestamp);
6.5.5、跳過特定主題的某個訂閱中的若干條未消費訊息
admin.topics().skipMessages(topic, subName, numMessages);
6.5.6、跳過特定主題的某個訂閱中的所有未消費訊息
admin.topics().skipAllMessages(topic, subName);
6.5.7、根據時間重置遊標
admin.topics().resetCursor(topic, subName, timestamp);
6.5.8、獲取主題所屬的 broker
admin.lookups().lookupTopic(topic);
6.5.9、獲取分割槽主題所屬的 broker
admin.lookups().lookupPartitionedTopic(topic);
6.5.10、獲取主題的訂閱資訊
admin.topics().getSubscriptions(topic);
6.5.11、獲取最新訊息的 ID
admin.topics().getLastMessage(topic);
6.5.12、建立非分割槽主題
admin.topics().createNonPartitionedTopic(topicName);
6.5.13、刪除非分割槽主題
admin.topics().delete(topic);
6.5.14、列出非分割槽主題
admin.topics().getList(namespace);
6.5.15、獲取非分割槽主題狀態
admin.topics().getStats(topic, false /* is precise backlog */);
6.5.16、獲取非分割槽主題內部狀態
admin.topics().getInternalStats(topic);
6.5.17、建立分割槽主題
admin.topics().createPartitionedTopic(topicName, numPartitions);
6.5.18、獲取分割槽主題後設資料資訊
admin.topics().getPartitionedTopicMetadata(topicName);
6.5.19、更新分割槽主題的分割槽數(只能比原來大)
admin.topics().updatePartitionedTopic(topic, numPartitions);
6.5.20、刪除分割槽主題
admin.topics().deletePartitionedTopic(topic);
6.5.21、列出分割槽主題
admin.topics().getPartitionedTopicList(namespace);
6.5.22、獲取分割槽主題狀態
admin.topics().getPartitionedStats(topic, true /* per partition */, false /* is precise backlog */);
6.5.23、獲取分割槽主題內部狀態
admin.topics().getPartitionedInternalStats(topic);
6.5.24、建立訂閱
admin.topics().createSubscription(topic, subscriptionName, MessageId.latest);
6.5.25、獲取訂閱
admin.topics().getSubscriptions(topic);
6.5.26、刪除訂閱
admin.topics().deleteSubscription(topic, subscriptionName);
6.6、完整程式碼
package com.abc.demo.pulsar; import lombok.extern.slf4j.Slf4j; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; import org.apache.pulsar.client.api.PulsarClientException; import org.apache.pulsar.common.policies.data.TenantInfoImpl; import org.junit.After; import org.junit.Before; import org.junit.Test; import java.util.Collections; import java.util.HashSet; @Slf4j public class AdminCase { private PulsarAdmin admin; @Before public void before() throws PulsarClientException { admin = PulsarAdmin.builder() .serviceHttpUrl("http://10.49.196.30:8080,10.49.196.31:8080,10.49.196.32:8080") .build(); } @After public void after() { admin.close(); } @Test public void broker() throws PulsarAdminException { log.info("getActiveBrokers={}", admin.brokers().getActiveBrokers()); log.info("getOwnedNamespaces={}", admin.brokers().getOwnedNamespaces("pulsar-cluster-1", "app1:8080")); log.info("getDynamicConfigurationNames={}", admin.brokers().getDynamicConfigurationNames()); admin.brokers().updateDynamicConfiguration("allowAutoTopicCreation", "true"); log.info("getAllDynamicConfigurations={}", admin.brokers().getAllDynamicConfigurations()); log.info("getLeaderBroker={}", admin.brokers().getLeaderBroker()); } @Test public void clusters() throws PulsarAdminException { log.info("getCluster={}", admin.clusters().getCluster("pulsar-cluster-1")); log.info("getClusters={}", admin.clusters().getClusters()); } @Test public void tenants() throws PulsarAdminException { log.info("getTenants={}", admin.tenants().getTenants()); TenantInfoImpl tenantInfo = new TenantInfoImpl(); tenantInfo.setAdminRoles(new HashSet<>()); tenantInfo.setAllowedClusters(Collections.singleton("pulsar-cluster-1")); admin.tenants().createTenant("test-tenant", tenantInfo); log.info("getTenantInfo={}", admin.tenants().getTenantInfo("test-tenant")); admin.tenants().updateTenant("test-tenant", tenantInfo); admin.tenants().deleteTenant("test-tenant"); } @Test public void namespaces() throws PulsarAdminException { admin.namespaces().createNamespace("public/test-ns"); log.info("getPolicies={}", admin.namespaces().getPolicies("public/default")); log.info("getNamespaces={}", admin.namespaces().getNamespaces("public")); admin.namespaces().deleteNamespace("public/test-ns"); } @Test public void topics() throws PulsarAdminException { log.info("peekMessages={}", admin.topics().peekMessages("persistent://public/default/my-topic", "my-subscription", 3)); Message<byte[]> message = admin.topics().getMessageById("persistent://public/default/my-topic", 171, 16); log.info("getMessageById={}", new String(message.getData())); message = admin.topics().examineMessage("persistent://public/default/my-topic", "latest", 1); log.info("examineMessage={}", new String(message.getData())); log.info("getMessageIdByTimestamp={}", admin.topics().getMessageIdByTimestamp("persistent://public/default/my-topic", System.currentTimeMillis())); admin.topics().skipMessages("persistent://public/default/my-topic", "my-subscription", 1); admin.topics().skipAllMessages("persistent://public/default/my-topic", "my-subscription"); admin.topics().resetCursor("persistent://public/default/my-topic", "my-subscription", System.currentTimeMillis() - 1000 * 60 * 15); log.info("lookupTopic={}", admin.lookups().lookupTopic("persistent://public/default/my-topic")); log.info("lookupPartitionedTopic={}", admin.lookups().lookupPartitionedTopic("persistent://public/default/my-topic2")); log.info("getSubscriptions={}", admin.topics().getSubscriptions("persistent://public/default/my-topic")); log.info("getLastMessageId={}", admin.topics().getLastMessageId("persistent://public/default/my-topic")); admin.topics().createNonPartitionedTopic("persistent://public/default/test-topic"); admin.topics().delete("persistent://public/default/test-topic"); log.info("getList={}", admin.topics().getList("public/default")); log.info("getStats={}", admin.topics().getStats("persistent://public/default/my-topic", false)); log.info("getInternalStats={}", admin.topics().getInternalStats("persistent://public/default/my-topic")); admin.topics().createPartitionedTopic("persistent://public/default/test-topic-p", 2); log.info("getPartitionedTopicMetadata={}", admin.topics().getPartitionedTopicMetadata("persistent://public/default/test-topic-p")); admin.topics().updatePartitionedTopic("persistent://public/default/test-topic-p", 3); admin.topics().deletePartitionedTopic("persistent://public/default/test-topic-p"); log.info("getStats={}", admin.topics().getPartitionedStats("persistent://public/default/my-topic2", false)); log.info("getInternalStats={}", admin.topics().getPartitionedInternalStats("persistent://public/default/my-topic2")); admin.topics().createSubscription("persistent://public/default/my-topic", "test-subscription", MessageId.latest); log.info("getStats={}", admin.topics().getSubscriptions("persistent://public/default/my-topic")); admin.topics().deleteSubscription("persistent://public/default/my-topic", "test-subscription"); } }
參考:
https://pulsar.apache.org/docs/3.3.x/client-libraries-java/
https://pulsar.apache.org/docs/3.3.x/admin-api-overview/