使用 Debezium 實現真正的原子微服務以確保資料一致性 – brainDOSE

<dependency>
   <groupId>io.debezium</groupId>
   <artifactId>debezium-quarkus-outbox</artifactId>
   <version>1.7.0.Alpha1</version>
</dependency>

/**
 * Provide the event data model for Casa outbox event implementation.
 * Provides the implementation for Debezium Outbox Pattern.
 */
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(JsonInclude.Include.NON_NULL)
public class CasaEventData {
    /**
     * Casa unique transaction id.
     */
    private String id;
    private String recipientAccountNo;
    private String sourceAccountNo;
    private double amount;
    /**
     * Casa created timestamp to indicates the timestamp when the Casa create event is being fired.
     */
    private Instant createdTimestamp;
    /**
     * Core processed timestamp to indicates the timestamp when the core processed the transaction.
     */
    private Instant coreProcessedTimestamp;
    /**
     * Audit timestamp to indicates the timestamp when the audit event is being fired.
     */
    private Instant auditTimestamp;
    /**
     * Response received timestamp to indicates the timestamp when the response from core is received by origination.
     */
    private Instant responseReceivedTimestamp;
    /**
     * Referecense information for recipient.
     */
    private String recipientReference;
    /**
     * Payment type. @see blog.braindose.paygate.model.PaymentTypes
     */
    private PaymentTypes paymentType;
    /**
     * Status of casa processing. @see blog.braindose.paygate.model.Status
     */
    private Status status;
    /**
     * Messages response from core backend if any. This could be error message when processing failed.
     */
    private String responseMessages;
 
    /**
     * Kafka header id
     * This is an unique id for each kafka message. Can be used to perform deduplication in the event of duplicated message sent by producer in the event of message resent due to unforeseen failure.
     */
    private String messageId;
 
    /**
     * Event source. This should be unique identifiable value for auditing purpose. Suggest to use Class.getName()
     */
    private String eventSources;
    /**
     * Event timestamp to provides timestamp when the event is being fired.
     */
    private Instant eventTimestamp;
 
   /// ... more omitted codes

/**
 * Casa Event for Outbox pattern implementation using Debezium.
 */
@Immutable
public class CasaEvent implements ExportedEvent<String, JsonNode> {
 
    private static ObjectMapper mapper = new ObjectMapper();
 
    /**
     * Unique Casa transaction id. @see blog.braindose.opay.casa.Casaid
     */
    private final String id;
    /**
     * Payload to be sent to Kafka in JSON format.
     */
    private final JsonNode casa;
    /**
     * Timestamp for outbox pattern implementation. Defaulted to Casa transactionTimestamp. @see blog.braindose.opay.casa.Casa#transactionTimestamp
     */
    private final Instant timestamp;
 
    public CasaEvent(CasaEventData casa) {
        this.id = casa.getId();
        this.timestamp = casa.getEventTimestamp();
        this.casa = convertToJson(casa);
    }
 
    private JsonNode convertToJson(CasaEventData casa) {
        ObjectNode asJson = mapper.createObjectNode()
                .put("id", casa.getId())
                .put("recipientAccountNo", casa.getRecipientAccountNo())
                .put("sourceAccountNo", casa.getSourceAccountNo())
                .put("amount", casa.getAmount())
                .put("recipientReference", casa.getRecipientReference())
                .put("paymentType", casa.getPaymentType().toString())
                .put("createdTimestamp", casa.getCreatedTimestamp().toString())
                .put("eventSources", casa.getEventSources().toString())
                .put("eventTimestamp", casa.getEventTimestamp().toString())
                .put("status", casa.getStatus().toString());
        return asJson;
    }
 
    @Override
    public String getAggregateId() {
        return id;
    }
 
    @Override
    public String getAggregateType() {
        return "casa";
    }
 
    @Override
    public JsonNode getPayload() {
        return casa;
    }
 
    @Override
    public String getType() {
        return "payment";
    }
 
    @Override
    public Instant getTimestamp() {
        return timestamp;
    }
 
}

Column     |          Type   | Modifiers
--------------+------------------------+-----------
id        | uuid            | not null
aggregatetype | character varying(255) | not null
aggregateid  | character varying(255) | not null
type       | character varying(255) | not null
payload     | jsonb                  |

/**
 * Provides REST interfaces for Casa services.
 */
@Path("casa")
public class CasaResource {
 
    @Inject
    Event<ExportedEvent<?, ?>> event;
     
    private static final Logger LOGGER = Logger.getLogger(CasaResource.class);
    private CasaEventData casaEventData = null;
    private boolean failed = false;
 
    /**
     * Create a new Casa transaction
     * @param casa
     * @return
     */
    @POST
    @Produces(MediaType.APPLICATION_JSON)
    @Consumes(MediaType.APPLICATION_JSON)
    @Transactional
    public Casa add(Casa casa) {
        try{
            casa.id = GenTxnId.id(TxnTypes.CASA);
            casa.createdTimestamp = Instant.now();
            casa.status = Status.SUBMITTED;
            casaEventData = new CasaEventData(casa.id, casa.recipientAccountNo, casa.sourceAccountNo, casa.amount, casa.createdTimestamp, casa.recipientReference, casa.paymentType, casa.status);
            casaEventData.setEventTimestamp(casa.createdTimestamp);
            casaEventData.setEventSources(Casa.class.getName());
            casa.persistAndFlush();
            event.fire(new CasaEvent(casaEventData));
        }
        catch(PersistenceException e){
            failed = true;
            if (casaEventData != null){
                casaEventData.setStatus(Status.FAILED);
                casaEventData.setResponseMessages("Error creating the Casa record in database.");
            }
            LOGGER.error("Error creating the Casa record in database.", e);
            throw e;
        }
        finally{
            if (casaEventData != null){
                event.fire(new CasaAuditEvent(casaEventData, EventTypes.PAYMENT, AggregateTypes.AUDIT_CASA));
                if (failed) event.fire(new CasaFailedEvent(casaEventData));
            }
        }
        return casa;
    }
    /// more codes omitted
    ...
    ...
    ...
 
}

@Inject
Event<ExportedEvent<?, ?>> event;

event.fire(new CasaEvent(casaEventData));

# Debezium outbox
quarkus.debezium-outbox.table-name=CasaOutboxEvent
%dev.quarkus.debezium-outbox.remove-after-insert=false
%prod.quarkus.debezium-outbox.remove-after-insert=true

{
    "name": "outbox-connector-casa",
    "config" : {
        "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
        "tasks.max": "1",
        "database.hostname": "casa-postgres",
        "database.port": "5432",
        "database.user": "casa",
        "database.password": "casa",
        "database.dbname": "casa",
        "database.server.name": "casa-event",
        "schema.include.list": "payment",
        "table.include.list": "payment.CasaOutboxEvent",
        "tombstones.on.delete": "false",
        "transforms": "outbox",
        "transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter",
        "transforms.outbox.route.topic.replacement": "${routedByValue}.events",
        "transforms.outbox.table.field.event.timestamp": "timestamp",
        "transforms.outbox.table.field.event.id": "aggregateid",
        "transforms.outbox.table.fields.additional.placement": "type:header:eventType"
    }
}

if (casaEventData != null){
   event.fire(new CasaAuditEvent(casaEventData, EventTypes.PAYMENT, AggregateTypes.AUDIT_CASA));
   if (failed) event.fire(new CasaFailedEvent(casaEventData));
}

builder.stream(
            KAFKA_TOPIC_CASA_AUDIT,
            Consumed.with(Serdes.String(), Serdes.String()))        // Serialized JSON string from JsonNode creates extra double quotes, causing it is not possible to use Jackson to deserialize into Java object
            .join(
                coreAuditStream,
                (casaAudit, coreAudit) -> {
                    // Multiple audit entries since Casa service received the response from core service.
                    List<AuditEntry> auditEntries = new ArrayList<>();
                    try {
                        LOGGER.debug("Processing casa audit trail...");
                        CasaEventData casaAuditObj = createCasaEventData(casaAudit);
                        auditEntries.add(createAuditEntry(casaAuditObj));
                         
                        LOGGER.debug("Processing core audit trail for casa transaction ...");
 
                        auditEntries.add(createAuditEntry(createCasaEventData(coreAudit)));
                         
                        AuditData<Casa> auditData = new AuditData<>(
                            casaAuditObj.getId(), 
                            auditEntries, 
                            new Casa(casaAuditObj.getRecipientAccountNo(), casaAuditObj.getSourceAccountNo(), casaAuditObj.getAmount(), casaAuditObj.getRecipientReference()), 
                            Instant.now().toString(), 
                            casaAuditObj.getStatus().toString());
 
                        String jsonInString = mapper.writeValueAsString(auditData);
                        LOGGER.debug("Joined result = " + jsonInString);
                        return jsonInString;
                         
                    } catch (JsonProcessingException e) {
                        LOGGER.error("Problem parsing Kafka message into JSON.");
                        throw new RuntimeException("Problem parsing Kafka message into JSON", e);
                    }
                },
                JoinWindows.of(Duration.ofMinutes(KAFKA_STREAMS_JOINWINDOW_DURATION)),
                Joined.with(Serdes.String(), Serdes.String(), Serdes.String())
            )
            .groupByKey()
            .reduce(            // deduplication of audit trail ... 
                (value1, value2) -> AuditData.reduce(value1, value2),
                Materialized.with(Serdes.String(), Serdes.String())
            )
            .toStream()
            //.print(org.apache.kafka.streams.kstream.Printed.toSysOut())
            .to(KAFKA_TOPIC_PAYMENT_AUDIT, Produced.with(Serdes.String(), Serdes.String()))
        ;

docker compose up --build

{
    "name": "outbox-connector-casa",
    "config" : {
        "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
        "tasks.max": "1",
        "database.hostname": "casa-postgres",
        "database.port": "5432",
        "database.user": "casa",
        "database.password": "casa",
        "database.dbname": "casa",
        "database.server.name": "casa-event",
        "schema.include.list": "payment",
        "table.include.list": "payment.CasaOutboxEvent",
        "tombstones.on.delete": "false",
        "transforms": "outbox",
        "transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter",
        "transforms.outbox.route.topic.replacement": "${routedByValue}.events",
        "transforms.outbox.table.field.event.timestamp": "timestamp",
        "transforms.outbox.table.field.event.id": "aggregateid",
        "transforms.outbox.table.fields.additional.placement": "type:header:eventType"
    }
}
{
    "name": "outbox-connector-core",
    "config" : {
        "connector.class": "io.debezium.connector.postgresql.PostgresConnector",
        "tasks.max": "1",
        "database.hostname": "core-postgres",
        "database.port": "5432",
        "database.user": "core",
        "database.password": "core",
        "database.dbname": "core",
        "database.server.name": "core-event",
        "schema.include.list": "core",
        "table.include.list": "core.ResponseOutboxEvent",
        "tombstones.on.delete": "false",
        "transforms": "outbox",
        "transforms.outbox.type": "io.debezium.transforms.outbox.EventRouter",
        "transforms.outbox.route.topic.replacement": "${routedByValue}.events",
        "transforms.outbox.table.field.event.timestamp": "timestamp",
        "transforms.outbox.table.field.event.id": "aggregateid",
        "transforms.outbox.table.fields.additional.placement": "type:header:eventType"
    }
}
{
    "name": "mongodb-sink",
    "config": {
        "connector.class": "com.mongodb.kafka.connect.MongoSinkConnector",
        "tasks.max": 1,
        "topics": "payment.audit.events",
        "connection.uri": "mongodb://audit:audit@audit-mongodb:27017",
        "database": "audit",
        "collection": "payment",
        "key.converter": "org.apache.kafka.connect.json.JsonConverter",
        "key.converter.schemas.enable": false,
        "value.converter": "org.apache.kafka.connect.json.JsonConverter",
        "value.converter.schemas.enable": false,
        "max.num.retries": 3
    }
}

rs0:PRIMARY> use audit
switched to db audit
rs0:PRIMARY> db.payment.find().pretty();
{
    "_id" : ObjectId("613981a793f9aa4f64c33eb6"),
    "payload" : {
        "amount" : 50.58,
        "recipientAccountNo" : "1-987654-1234-4569",
        "recipientReference" : "Payment for lunch",
        "sourceAccountNo" : "1-234567-4321-9876"
    },
    "id" : "1-20210909-033739893-17839",
    "lastStatus" : "COMPLETED",
    "auditEntries" : [
        {
            "eventSource" : "blog.braindose.opay.casa.Casa",
            "eventTimestamp" : "2021-09-09T03:37:39.905947Z",
            "responseMessages" : null,
            "status" : "SUBMITTED"
        },
        {
            "eventSource" : "blog.braindose.opay.core.casa.ConsumeCasa",
            "eventTimestamp" : "2021-09-09T03:37:42.180891Z",
            "responseMessages" : null,
            "status" : "COMPLETED"
        },
        {
            "eventSource" : "blog.braindose.opay.casa.ConsumeCasaResponse",
            "eventTimestamp" : "2021-09-09T03:37:43.271790Z",
            "responseMessages" : null,
            "status" : "COMPLETED"
        }
    ],
    "eventTimestamp" : "2021-09-09T03:37:45.720514Z"
}