image-20250507222944766

依赖模块

1. CanalServerWithNetty

1.1 源码

public class CanalServerWithNetty extends AbstractCanalLifeCycle implements CanalServer {

private CanalServerWithEmbedded embeddedServer; // 嵌入式server
private String ip;
private int port;
private Channel serverChannel = null;
private ServerBootstrap bootstrap = null;
private ChannelGroup childGroups = null; // socket channel
// container, used to
// close sockets
// explicitly.

private static class SingletonHolder {

private static final CanalServerWithNetty CANAL_SERVER_WITH_NETTY = new CanalServerWithNetty();
}

private CanalServerWithNetty(){
this.embeddedServer = CanalServerWithEmbedded.instance();
this.childGroups = new DefaultChannelGroup();
}

public static CanalServerWithNetty instance() {
return SingletonHolder.CANAL_SERVER_WITH_NETTY;
}

public void start() {
super.start();

if (!embeddedServer.isStart()) {
embeddedServer.start();
}

this.bootstrap = new ServerBootstrap(new NioServerSocketChannelFactory(Executors.newCachedThreadPool(),
Executors.newCachedThreadPool()));
/*
* enable keep-alive mechanism, handle abnormal network connection
* scenarios on OS level. the threshold parameters are depended on OS.
* e.g. On Linux: net.ipv4.tcp_keepalive_time = 300
* net.ipv4.tcp_keepalive_probes = 2 net.ipv4.tcp_keepalive_intvl = 30
*/
bootstrap.setOption("child.keepAlive", true);
/*
* optional parameter.
*/
bootstrap.setOption("child.tcpNoDelay", true);

// 构造对应的pipeline
bootstrap.setPipelineFactory(() -> {
ChannelPipeline pipelines = Channels.pipeline();
pipelines.addLast(FixedHeaderFrameDecoder.class.getName(), new FixedHeaderFrameDecoder());
// support to maintain child socket channel.
pipelines.addLast(HandshakeInitializationHandler.class.getName(),
new HandshakeInitializationHandler(childGroups));
pipelines.addLast(ClientAuthenticationHandler.class.getName(),
new ClientAuthenticationHandler(embeddedServer));

SessionHandler sessionHandler = new SessionHandler(embeddedServer);
pipelines.addLast(SessionHandler.class.getName(), sessionHandler);
return pipelines;
});

// 启动
if (StringUtils.isNotEmpty(ip)) {
this.serverChannel = bootstrap.bind(new InetSocketAddress(this.ip, this.port));
} else {
this.serverChannel = bootstrap.bind(new InetSocketAddress(this.port));
}
}

public void stop() {
super.stop();

if (this.serverChannel != null) {
this.serverChannel.close().awaitUninterruptibly(1000);
}

// close sockets explicitly to reduce socket channel hung in complicated
// network environment.
if (this.childGroups != null) {
this.childGroups.close().awaitUninterruptibly(5000);
}

if (this.bootstrap != null) {
this.bootstrap.releaseExternalResources();
}

if (embeddedServer.isStart()) {
embeddedServer.stop();
}
}

public void setIp(String ip) {
this.ip = ip;
}

public void setPort(int port) {
this.port = port;
}

public void setEmbeddedServer(CanalServerWithEmbedded embeddedServer) {
this.embeddedServer = embeddedServer;
}

}

1.2 Handler

1.2.1 FixedHeaderFrameDecoder

public class FixedHeaderFrameDecoder extends ReplayingDecoder<VoidEnum> {

protected Object decode(ChannelHandlerContext ctx, Channel channel, ChannelBuffer buffer, VoidEnum state)
throws Exception {
return buffer.readBytes(buffer.readInt());
}
}

1.2.2 ClientAuthenticationHandler

public class ClientAuthenticationHandler extends SimpleChannelHandler {

private static final Logger logger = LoggerFactory.getLogger(ClientAuthenticationHandler.class);
private final int SUPPORTED_VERSION = 3;
private final int defaultSubscriptorDisconnectIdleTimeout = 60 * 60 * 1000;
private CanalServerWithEmbedded embeddedServer;
private byte[] seed;

public ClientAuthenticationHandler(){

}

public ClientAuthenticationHandler(CanalServerWithEmbedded embeddedServer){
this.embeddedServer = embeddedServer;
}

public void messageReceived(final ChannelHandlerContext ctx, MessageEvent e) throws Exception {
ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
final Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
switch (packet.getVersion()) {
case SUPPORTED_VERSION:
default:
final ClientAuth clientAuth = ClientAuth.parseFrom(packet.getBody());
if (seed == null) {
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("auth failed for seed is null", clientAuth.getUsername()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, null);
break;
}

if (!embeddedServer.auth(clientAuth.getUsername(), clientAuth.getPassword().toStringUtf8(), seed)) {
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("auth failed for user:{}", clientAuth.getUsername()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, null);
break;
}

// 如果存在订阅信息
if (StringUtils.isNotEmpty(clientAuth.getDestination())
&& StringUtils.isNotEmpty(clientAuth.getClientId())) {
ClientIdentity clientIdentity = new ClientIdentity(clientAuth.getDestination(),
Short.valueOf(clientAuth.getClientId()),
clientAuth.getFilter());
try {
MDC.put("destination", clientIdentity.getDestination());
embeddedServer.subscribe(clientIdentity);
// 尝试启动,如果已经启动,忽略
if (!embeddedServer.isStart(clientIdentity.getDestination())) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
if (!runningMonitor.isStart()) {
runningMonitor.start();
}
}
} finally {
MDC.remove("destination");
}
}
// 鉴权一次性,暂不统计
NettyUtils.ack(ctx.getChannel(), future -> {
logger.info("remove unused channel handlers after authentication is done successfully.");
ctx.getPipeline().remove(HandshakeInitializationHandler.class.getName());
ctx.getPipeline().remove(ClientAuthenticationHandler.class.getName());

int readTimeout = defaultSubscriptorDisconnectIdleTimeout;
int writeTimeout = defaultSubscriptorDisconnectIdleTimeout;
if (clientAuth.getNetReadTimeout() > 0) {
readTimeout = clientAuth.getNetReadTimeout();
}
if (clientAuth.getNetWriteTimeout() > 0) {
writeTimeout = clientAuth.getNetWriteTimeout();
}
// fix bug: soTimeout parameter's unit from connector is
// millseconds.
IdleStateHandler idleStateHandler = new IdleStateHandler(NettyUtils.hashedWheelTimer,
readTimeout,
writeTimeout,
0,
TimeUnit.MILLISECONDS);
ctx.getPipeline().addBefore(SessionHandler.class.getName(),
IdleStateHandler.class.getName(),
idleStateHandler);

IdleStateAwareChannelHandler idleStateAwareChannelHandler = new IdleStateAwareChannelHandler() {

public void channelIdle(ChannelHandlerContext ctx1, IdleStateEvent e1) throws Exception {
logger.warn("channel:{} idle timeout exceeds, close channel to save server resources...",
ctx1.getChannel());
ctx1.getChannel().close();
}

};
ctx.getPipeline().addBefore(SessionHandler.class.getName(),
IdleStateAwareChannelHandler.class.getName(),
idleStateAwareChannelHandler);
});
break;
}
}

public void setEmbeddedServer(CanalServerWithEmbedded embeddedServer) {
this.embeddedServer = embeddedServer;
}

public void setSeed(byte[] seed) {
this.seed = seed;
}

}

1.2.3 HandshakeInitializationHandler

public class HandshakeInitializationHandler extends SimpleChannelHandler {

// support to maintain socket channel.
private ChannelGroup childGroups;

public HandshakeInitializationHandler(ChannelGroup childGroups){
this.childGroups = childGroups;
}

private static final Logger logger = LoggerFactory.getLogger(HandshakeInitializationHandler.class);

public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
// add new socket channel in channel container, used to manage sockets.
if (childGroups != null) {
childGroups.add(ctx.getChannel());
}

final byte[] seed = org.apache.commons.lang3.RandomUtils.nextBytes(8);
byte[] body = Packet.newBuilder()
.setType(CanalPacket.PacketType.HANDSHAKE)
.setVersion(NettyUtils.VERSION)
.setBody(Handshake.newBuilder().setSeeds(ByteString.copyFrom(seed)).build().toByteString())
.build()
.toByteArray();

NettyUtils.write(ctx.getChannel(), body, future -> {
ctx.getPipeline().get(HandshakeInitializationHandler.class.getName());
ClientAuthenticationHandler handler = (ClientAuthenticationHandler) ctx.getPipeline()
.get(ClientAuthenticationHandler.class.getName());
handler.setSeed(seed);
});
logger.info("send handshake initialization packet to : {}", ctx.getChannel());
}
}

1.2.4 SessionHandler

public class SessionHandler extends SimpleChannelHandler {

private static final Logger logger = LoggerFactory.getLogger(SessionHandler.class);
private CanalServerWithEmbedded embeddedServer;

public SessionHandler(){
}

public SessionHandler(CanalServerWithEmbedded embeddedServer){
this.embeddedServer = embeddedServer;
}

@SuppressWarnings({ "deprecation" })
public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
logger.info("message receives in session handler...");
long start = System.nanoTime();
ChannelBuffer buffer = (ChannelBuffer) e.getMessage();
Packet packet = Packet.parseFrom(buffer.readBytes(buffer.readableBytes()).array());
ClientIdentity clientIdentity = null;
try {
switch (packet.getType()) {
case SUBSCRIPTION:
Sub sub = Sub.parseFrom(packet.getBody());
if (StringUtils.isNotEmpty(sub.getDestination()) && StringUtils.isNotEmpty(sub.getClientId())) {
clientIdentity = new ClientIdentity(sub.getDestination(),
Short.valueOf(sub.getClientId()),
sub.getFilter());
MDC.put("destination", clientIdentity.getDestination());

// 尝试启动,如果已经启动,忽略
if (!embeddedServer.isStart(clientIdentity.getDestination())) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
if (!runningMonitor.isStart()) {
runningMonitor.start();
}
}

embeddedServer.subscribe(clientIdentity);
// ctx.setAttachment(clientIdentity);// 设置状态数据
byte[] ackBytes = NettyUtils.ackPacket();
NettyUtils.write(ctx.getChannel(), ackBytes, new ChannelFutureAggregator(sub.getDestination(),
sub,
packet.getType(),
ackBytes.length,
System.nanoTime() - start));
} else {
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", sub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(sub.getDestination(),
sub,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case UNSUBSCRIPTION:
Unsub unsub = Unsub.parseFrom(packet.getBody());
if (StringUtils.isNotEmpty(unsub.getDestination()) && StringUtils.isNotEmpty(unsub.getClientId())) {
clientIdentity = new ClientIdentity(unsub.getDestination(),
Short.valueOf(unsub.getClientId()),
unsub.getFilter());
MDC.put("destination", clientIdentity.getDestination());
embeddedServer.unsubscribe(clientIdentity);
stopCanalInstanceIfNecessary(clientIdentity);// 尝试关闭
byte[] ackBytes = NettyUtils.ackPacket();
NettyUtils.write(ctx.getChannel(),
ackBytes,
new ChannelFutureAggregator(unsub.getDestination(),
unsub,
packet.getType(),
ackBytes.length,
System.nanoTime() - start));
} else {
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", unsub.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(unsub.getDestination(),
unsub,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case GET:
Get get = CanalPacket.Get.parseFrom(packet.getBody());
if (StringUtils.isNotEmpty(get.getDestination()) && StringUtils.isNotEmpty(get.getClientId())) {
clientIdentity = new ClientIdentity(get.getDestination(), Short.valueOf(get.getClientId()));
MDC.put("destination", clientIdentity.getDestination());
Message message = null;

// if (get.getAutoAck()) {
// if (get.getTimeout() == -1) {//是否是初始值
// message = embeddedServer.get(clientIdentity,
// get.getFetchSize());
// } else {
// TimeUnit unit = convertTimeUnit(get.getUnit());
// message = embeddedServer.get(clientIdentity,
// get.getFetchSize(), get.getTimeout(), unit);
// }
// } else {
if (get.getTimeout() == -1) {// 是否是初始值
message = embeddedServer.getWithoutAck(clientIdentity, get.getFetchSize());
} else {
TimeUnit unit = convertTimeUnit(get.getUnit());
message = embeddedServer.getWithoutAck(clientIdentity,
get.getFetchSize(),
get.getTimeout(),
unit);
}
// }

if (message.getId() != -1 && message.isRaw()) {
List<ByteString> rowEntries = message.getRawEntries();
// message size
int messageSize = 0;
messageSize += com.google.protobuf.CodedOutputStream.computeInt64Size(1, message.getId());

int dataSize = 0;
for (ByteString rowEntry : rowEntries) {
dataSize += CodedOutputStream.computeBytesSizeNoTag(rowEntry);
}
messageSize += dataSize;
messageSize += 1 * rowEntries.size();
// packet size
int size = 0;
size += com.google.protobuf.CodedOutputStream.computeEnumSize(3,
PacketType.MESSAGES.getNumber());
size += com.google.protobuf.CodedOutputStream.computeTagSize(5)
+ com.google.protobuf.CodedOutputStream.computeRawVarint32Size(messageSize)
+ messageSize;
// recyle bytes
// ByteBuffer byteBuffer = (ByteBuffer)
// ctx.getAttachment();
// if (byteBuffer != null && size <=
// byteBuffer.capacity()) {
// byteBuffer.clear();
// } else {
// byteBuffer =
// ByteBuffer.allocate(size).order(ByteOrder.BIG_ENDIAN);
// ctx.setAttachment(byteBuffer);
// }
// CodedOutputStream output =
// CodedOutputStream.newInstance(byteBuffer);
byte[] body = new byte[size];
CodedOutputStream output = CodedOutputStream.newInstance(body);
output.writeEnum(3, PacketType.MESSAGES.getNumber());

output.writeTag(5, WireFormat.WIRETYPE_LENGTH_DELIMITED);
output.writeRawVarint32(messageSize);
// message
output.writeInt64(1, message.getId());
for (ByteString rowEntry : rowEntries) {
output.writeBytes(2, rowEntry);
}
output.checkNoSpaceLeft();
NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(),
get,
packet.getType(),
body.length,
System.nanoTime() - start,
message.getId() == -1));

// output.flush();
// byteBuffer.flip();
// NettyUtils.write(ctx.getChannel(), byteBuffer,
// null);
} else {
Packet.Builder packetBuilder = CanalPacket.Packet.newBuilder();
packetBuilder.setType(PacketType.MESSAGES).setVersion(NettyUtils.VERSION);

Messages.Builder messageBuilder = CanalPacket.Messages.newBuilder();
messageBuilder.setBatchId(message.getId());
if (message.getId() != -1) {
if (message.isRaw() && !CollectionUtils.isEmpty(message.getRawEntries())) {
messageBuilder.addAllMessages(message.getRawEntries());
} else if (!CollectionUtils.isEmpty(message.getEntries())) {
for (Entry entry : message.getEntries()) {
messageBuilder.addMessages(entry.toByteString());
}
}
}
byte[] body = packetBuilder.setBody(messageBuilder.build().toByteString())
.build()
.toByteArray();
NettyUtils.write(ctx.getChannel(), body, new ChannelFutureAggregator(get.getDestination(),
get,
packet.getType(),
body.length,
System.nanoTime() - start,
message.getId() == -1));// 输出数据
}
} else {
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", get.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(get.getDestination(),
get,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case CLIENTACK:
ClientAck ack = CanalPacket.ClientAck.parseFrom(packet.getBody());
MDC.put("destination", ack.getDestination());
if (StringUtils.isNotEmpty(ack.getDestination()) && StringUtils.isNotEmpty(ack.getClientId())) {
if (ack.getBatchId() == 0L) {
byte[] errorBytes = NettyUtils.errorPacket(402,
MessageFormatter.format("batchId should assign value", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(ack.getDestination(),
ack,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 402));
} else if (ack.getBatchId() == -1L) { // -1代表上一次get没有数据,直接忽略之
// donothing
} else {
clientIdentity = new ClientIdentity(ack.getDestination(), Short.valueOf(ack.getClientId()));
embeddedServer.ack(clientIdentity, ack.getBatchId());
new ChannelFutureAggregator(ack.getDestination(),
ack,
packet.getType(),
0,
System.nanoTime() - start).operationComplete(null);
}
} else {
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", ack.toString()).getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(ack.getDestination(),
ack,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
case CLIENTROLLBACK:
ClientRollback rollback = CanalPacket.ClientRollback.parseFrom(packet.getBody());
MDC.put("destination", rollback.getDestination());
if (StringUtils.isNotEmpty(rollback.getDestination())
&& StringUtils.isNotEmpty(rollback.getClientId())) {
clientIdentity = new ClientIdentity(rollback.getDestination(),
Short.valueOf(rollback.getClientId()));
if (rollback.getBatchId() == 0L) {
embeddedServer.rollback(clientIdentity);// 回滚所有批次
} else {
embeddedServer.rollback(clientIdentity, rollback.getBatchId()); // 只回滚单个批次
}
new ChannelFutureAggregator(rollback.getDestination(),
rollback,
packet.getType(),
0,
System.nanoTime() - start).operationComplete(null);
} else {
byte[] errorBytes = NettyUtils.errorPacket(401,
MessageFormatter.format("destination or clientId is null", rollback.toString())
.getMessage());
NettyUtils.write(ctx.getChannel(),
errorBytes,
new ChannelFutureAggregator(rollback.getDestination(),
rollback,
packet.getType(),
errorBytes.length,
System.nanoTime() - start,
(short) 401));
}
break;
default:
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("packet type={} is NOT supported!", packet.getType()).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel()
.getRemoteAddress()
.toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
break;
}
} catch (Throwable exception) {
byte[] errorBytes = NettyUtils.errorPacket(400,
MessageFormatter.format("something goes wrong with channel:{}, exception={}",
ctx.getChannel(),
ExceptionUtils.getStackTrace(exception)).getMessage());
NettyUtils.write(ctx.getChannel(), errorBytes, new ChannelFutureAggregator(ctx.getChannel()
.getRemoteAddress()
.toString(), null, packet.getType(), errorBytes.length, System.nanoTime() - start, (short) 400));
} finally {
MDC.remove("destination");
}
}

public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
logger.error("something goes wrong with channel:{}, exception={}",
ctx.getChannel(),
ExceptionUtils.getStackTrace(e.getCause()));

ctx.getChannel().close();
}

public void channelClosed(ChannelHandlerContext ctx, ChannelStateEvent e) throws Exception {
// logger.info("remove binding subscription value object if any...");
// ClientIdentity clientIdentity = (ClientIdentity) ctx.getAttachment();
// // 如果唯一的订阅者都取消了订阅,直接关闭服务,针对内部版本模式下可以减少资源浪费
// if (clientIdentity != null) {
// stopCanalInstanceIfNecessary(clientIdentity);
// }
}

private void stopCanalInstanceIfNecessary(ClientIdentity clientIdentity) {
List<ClientIdentity> clientIdentitys = embeddedServer.listAllSubscribe(clientIdentity.getDestination());
if (clientIdentitys != null && clientIdentitys.size() == 1 && clientIdentitys.contains(clientIdentity)) {
ServerRunningMonitor runningMonitor = ServerRunningMonitors.getRunningMonitor(clientIdentity.getDestination());
if (runningMonitor.isStart()) {
runningMonitor.release();
}
}
}

private TimeUnit convertTimeUnit(int unit) {
switch (unit) {
case 0:
return TimeUnit.NANOSECONDS;
case 1:
return TimeUnit.MICROSECONDS;
case 2:
return TimeUnit.MILLISECONDS;
case 3:
return TimeUnit.SECONDS;
case 4:
return TimeUnit.MINUTES;
case 5:
return TimeUnit.HOURS;
case 6:
return TimeUnit.DAYS;
default:
return TimeUnit.MILLISECONDS;
}
}

public void setEmbeddedServer(CanalServerWithEmbedded embeddedServer) {
this.embeddedServer = embeddedServer;
}

}

1.3 Listener

1.3.1 ChannelFutureAggregator

public class ChannelFutureAggregator implements ChannelFutureListener {

private ClientRequestResult result;

public ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency, boolean empty) {
this(destination, request, type, amount, latency, empty, (short) 0);
}

public ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency) {
this(destination, request, type, amount, latency, false, (short) 0);
}

public ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency, short errorCode) {
this(destination, request, type, amount, latency, false, errorCode);
}

private ChannelFutureAggregator(String destination, GeneratedMessageV3 request, CanalPacket.PacketType type, int amount, long latency, boolean empty, short errorCode) {
this.result = new ClientRequestResult.Builder()
.destination(destination)
.type(type)
.request(request)
.amount(amount + HEADER_LENGTH)
.latency(latency)
.errorCode(errorCode)
.empty(empty)
.build();
}

@Override
public void operationComplete(ChannelFuture future) {
// profiling after I/O operation
if (future != null && future.getCause() != null) {
result.channelError = future.getCause();
}
profiler().profiling(result);
}

/**
* Client request result pojo
*/
public static class ClientRequestResult {

private String destination;
private CanalPacket.PacketType type;
private GeneratedMessageV3 request;
private int amount;
private long latency;
private short errorCode;
private boolean empty;
private Throwable channelError;

private ClientRequestResult() {}

private ClientRequestResult(Builder builder) {
this.destination = Preconditions.checkNotNull(builder.destination);
this.type = Preconditions.checkNotNull(builder.type);
this.request = builder.request;
this.amount = builder.amount;
this.latency = builder.latency;
this.errorCode = builder.errorCode;
this.empty = builder.empty;
this.channelError = builder.channelError;
}

// auto-generated
public static class Builder {

private String destination;
private CanalPacket.PacketType type;
private GeneratedMessageV3 request;
private int amount;
private long latency;
private short errorCode;
private boolean empty;
private Throwable channelError;

Builder destination(String destination) {
this.destination = destination;
return this;
}

Builder type(CanalPacket.PacketType type) {
this.type = type;
return this;
}

Builder request(GeneratedMessageV3 request) {
this.request = request;
return this;
}

Builder amount(int amount) {
this.amount = amount;
return this;
}

Builder latency(long latency) {
this.latency = latency;
return this;
}

Builder errorCode(short errorCode) {
this.errorCode = errorCode;
return this;
}

Builder empty(boolean empty) {
this.empty = empty;
return this;
}

public Builder channelError(Throwable channelError) {
this.channelError = channelError;
return this;
}

public Builder fromPrototype(ClientRequestResult prototype) {
destination = prototype.destination;
type = prototype.type;
request = prototype.request;
amount = prototype.amount;
latency = prototype.latency;
errorCode = prototype.errorCode;
empty = prototype.empty;
channelError = prototype.channelError;
return this;
}

ClientRequestResult build() {
return new ClientRequestResult(this);
}
}
// getters
public String getDestination() {
return destination;
}

public CanalPacket.PacketType getType() {
return type;
}

public GeneratedMessageV3 getRequest() {
return request;
}

public int getAmount() {
return amount;
}

public long getLatency() {
return latency;
}

public short getErrorCode() {
return errorCode;
}

public boolean getEmpty() {
return empty;
}

public Throwable getChannelError() {
return channelError;
}
}
}

2. CanalServerWithEmbedded

2.1 源码

/**
* 嵌入式版本实现
*
* @author jianghang 2012-7-12 下午01:34:00
* @author zebin.xuzb
* @version 1.0.0
*/
public class CanalServerWithEmbedded extends AbstractCanalLifeCycle implements CanalServer, CanalService {

private static final Logger logger = LoggerFactory.getLogger(CanalServerWithEmbedded.class);
private Map<String, CanalInstance> canalInstances;
// private Map<ClientIdentity, Position> lastRollbackPostions;
private CanalInstanceGenerator canalInstanceGenerator;
private int metricsPort;
private CanalMetricsService metrics = NopCanalMetricsService.NOP;
private String user;
private String passwd;

private static class SingletonHolder {

private static final CanalServerWithEmbedded CANAL_SERVER_WITH_EMBEDDED = new CanalServerWithEmbedded();
}

public CanalServerWithEmbedded(){
// 希望也保留用户new单独实例的需求,兼容历史
}

public static CanalServerWithEmbedded instance() {
return SingletonHolder.CANAL_SERVER_WITH_EMBEDDED;
}

public void start() {
if (!isStart()) {
super.start();
// 如果存在provider,则启动metrics service
if(metricsPort > 0) {
loadCanalMetrics();
metrics.setServerPort(metricsPort);
metrics.initialize();
}
canalInstances = MigrateMap.makeComputingMap(destination -> canalInstanceGenerator.generate(destination));
// lastRollbackPostions = new MapMaker().makeMap();
}
}

public void stop() {
super.stop();
for (Map.Entry<String, CanalInstance> entry : canalInstances.entrySet()) {
try {
CanalInstance instance = entry.getValue();
if (instance.isStart()) {
try {
String destination = entry.getKey();
MDC.put("destination", destination);
entry.getValue().stop();
logger.info("stop CanalInstances[{}] successfully", destination);
} finally {
MDC.remove("destination");
}
}
} catch (Exception e) {
logger.error(String.format("stop CanalInstance[%s] has an error", entry.getKey()), e);
}
}
metrics.terminate();
}

public boolean auth(String user, String passwd, byte[] seed) {
// 如果user/passwd密码为空,则任何用户账户都能登录
if ((StringUtils.isEmpty(this.user) || StringUtils.equals(this.user, user))) {
if (StringUtils.isEmpty(this.passwd)) {
return true;
} else if (StringUtils.isEmpty(passwd)) {
// 如果server密码有配置,客户端密码为空,则拒绝
return false;
}

try {
byte[] passForClient = SecurityUtil.hexStr2Bytes(passwd);
return SecurityUtil.scrambleServerAuth(passForClient, SecurityUtil.hexStr2Bytes(this.passwd), seed);
} catch (NoSuchAlgorithmException e) {
return false;
}
}

return false;
}

public void start(final String destination) {
final CanalInstance canalInstance = canalInstances.get(destination);
if (!canalInstance.isStart()) {
try {
MDC.put("destination", destination);
if (metrics.isRunning()) {
metrics.register(canalInstance);
}
canalInstance.start();
logger.info("start CanalInstances[{}] successfully", destination);
} finally {
MDC.remove("destination");
}
}
}

public void stop(String destination) {
CanalInstance canalInstance = canalInstances.remove(destination);
if (canalInstance != null) {
if (canalInstance.isStart()) {
try {
MDC.put("destination", destination);
canalInstance.stop();
if (metrics.isRunning()) {
metrics.unregister(canalInstance);
}
logger.info("stop CanalInstances[{}] successfully", destination);
} finally {
MDC.remove("destination");
}
}
}
}

public boolean isStart(String destination) {
return canalInstances.containsKey(destination) && canalInstances.get(destination).isStart();
}

/**
* 客户端订阅,重复订阅时会更新对应的filter信息
*/
@Override
public void subscribe(ClientIdentity clientIdentity) throws CanalServerException {
checkStart(clientIdentity.getDestination());

CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
if (!canalInstance.getMetaManager().isStart()) {
canalInstance.getMetaManager().start();
}

canalInstance.getMetaManager().subscribe(clientIdentity); // 执行一下meta订阅

Position position = canalInstance.getMetaManager().getCursor(clientIdentity);
if (position == null) {
position = canalInstance.getEventStore().getFirstPosition();// 获取一下store中的第一条
if (position != null) {
canalInstance.getMetaManager().updateCursor(clientIdentity, position); // 更新一下cursor
}
logger.info("subscribe successfully, {} with first position:{} ", clientIdentity, position);
} else {
logger.info("subscribe successfully, {} use last cursor position:{} ", clientIdentity, position);
}

// 通知下订阅关系变化
canalInstance.subscribeChange(clientIdentity);
}

/**
* 取消订阅
*/
@Override
public void unsubscribe(ClientIdentity clientIdentity) throws CanalServerException {
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
canalInstance.getMetaManager().unsubscribe(clientIdentity); // 执行一下meta订阅

logger.info("unsubscribe successfully, {}", clientIdentity);
}

/**
* 查询所有的订阅信息
*/
public List<ClientIdentity> listAllSubscribe(String destination) throws CanalServerException {
CanalInstance canalInstance = canalInstances.get(destination);
return canalInstance.getMetaManager().listAllSubscribeInfo(destination);
}

/**
* 获取数据
*
* <pre>
* 注意: meta获取和数据的获取需要保证顺序性,优先拿到meta的,一定也会是优先拿到数据,所以需要加同步. (不能出现先拿到meta,拿到第二批数据,这样就会导致数据顺序性出现问题)
* </pre>
*/
@Override
public Message get(ClientIdentity clientIdentity, int batchSize) throws CanalServerException {
return get(clientIdentity, batchSize, null, null);
}

/**
* 获取数据,可以指定超时时间.
*
* <pre>
* 几种case:
* a. 如果timeout为null,则采用tryGet方式,即时获取
* b. 如果timeout不为null
* 1. timeout为0,则采用get阻塞方式,获取数据,不设置超时,直到有足够的batchSize数据才返回
* 2. timeout不为0,则采用get+timeout方式,获取数据,超时还没有batchSize足够的数据,有多少返回多少
*
* 注意: meta获取和数据的获取需要保证顺序性,优先拿到meta的,一定也会是优先拿到数据,所以需要加同步. (不能出现先拿到meta,拿到第二批数据,这样就会导致数据顺序性出现问题)
* </pre>
*/
@Override
public Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit)
throws CanalServerException {
checkStart(clientIdentity.getDestination());
checkSubscribe(clientIdentity);
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
synchronized (canalInstance) {
// 获取到流式数据中的最后一批获取的位置
PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);

if (positionRanges != null) {
throw new CanalServerException(String.format("clientId:%s has last batch:[%s] isn't ack , maybe loss data",
clientIdentity.getClientId(),
positionRanges));
}

Events<Event> events = null;
Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);

if (CollectionUtils.isEmpty(events.getEvents())) {
logger.debug("get successfully, clientId:{} batchSize:{} but result is null",
clientIdentity.getClientId(),
batchSize);
return new Message(-1, true, new ArrayList()); // 返回空包,避免生成batchId,浪费性能
} else {
// 记录到流式信息
Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
boolean raw = isRaw(canalInstance.getEventStore());
List entrys = null;
if (raw) {
// new list
entrys = events.getEvents().stream().map(Event::getRawEntry).collect(Collectors.toList());
} else {
entrys = events.getEvents().stream().map(Event::getEntry).collect(Collectors.toList());
}
if (logger.isInfoEnabled()) {
logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]",
clientIdentity.getClientId(),
batchSize,
entrys.size(),
batchId,
events.getPositionRange());
}
// 直接提交ack
ack(clientIdentity, batchId);
return new Message(batchId, raw, entrys);
}
}
}

/**
* 不指定 position 获取事件。canal 会记住此 client 最新的 position。 <br/>
* 如果是第一次 fetch,则会从 canal 中保存的最老一条数据开始输出。
*
* <pre>
* 注意: meta获取和数据的获取需要保证顺序性,优先拿到meta的,一定也会是优先拿到数据,所以需要加同步. (不能出现先拿到meta,拿到第二批数据,这样就会导致数据顺序性出现问题)
* </pre>
*/
@Override
public Message getWithoutAck(ClientIdentity clientIdentity, int batchSize) throws CanalServerException {
return getWithoutAck(clientIdentity, batchSize, null, null);
}

/**
* 不指定 position 获取事件。canal 会记住此 client 最新的 position。 <br/>
* 如果是第一次 fetch,则会从 canal 中保存的最老一条数据开始输出。
*
* <pre>
* 几种case:
* a. 如果timeout为null,则采用tryGet方式,即时获取
* b. 如果timeout不为null
* 1. timeout为0,则采用get阻塞方式,获取数据,不设置超时,直到有足够的batchSize数据才返回
* 2. timeout不为0,则采用get+timeout方式,获取数据,超时还没有batchSize足够的数据,有多少返回多少
*
* 注意: meta获取和数据的获取需要保证顺序性,优先拿到meta的,一定也会是优先拿到数据,所以需要加同步. (不能出现先拿到meta,拿到第二批数据,这样就会导致数据顺序性出现问题)
* </pre>
*/
@Override
public Message getWithoutAck(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit)
throws CanalServerException {
checkStart(clientIdentity.getDestination());
checkSubscribe(clientIdentity);

CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
synchronized (canalInstance) {
// 获取到流式数据中的最后一批获取的位置
PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);

Events<Event> events = null;
if (positionRanges != null) { // 存在流数据
events = getEvents(canalInstance.getEventStore(), positionRanges.getStart(), batchSize, timeout, unit);
} else {// ack后第一次获取
Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
if (start == null) { // 第一次,还没有过ack记录,则获取当前store中的第一条
start = canalInstance.getEventStore().getFirstPosition();
}

events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
}

if (CollectionUtils.isEmpty(events.getEvents())) {
// logger.debug("getWithoutAck successfully, clientId:{}
// batchSize:{} but result
// is null",
// clientIdentity.getClientId(),
// batchSize);
return new Message(-1, true, new ArrayList()); // 返回空包,避免生成batchId,浪费性能
} else {
// 记录到流式信息
Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
boolean raw = isRaw(canalInstance.getEventStore());
List entrys = null;
if (raw) {
// new list
entrys = events.getEvents().stream().map(Event::getRawEntry).collect(Collectors.toList());
} else {
entrys = events.getEvents().stream().map(Event::getEntry).collect(Collectors.toList());
}
if (logger.isInfoEnabled()) {
logger.info("getWithoutAck successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]",
clientIdentity.getClientId(),
batchSize,
entrys.size(),
batchId,
events.getPositionRange());
}
return new Message(batchId, raw, entrys);
}

}
}

/**
* 查询当前未被ack的batch列表,batchId会按照从小到大进行返回
*/
public List<Long> listBatchIds(ClientIdentity clientIdentity) throws CanalServerException {
checkStart(clientIdentity.getDestination());
checkSubscribe(clientIdentity);

CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
Map<Long, PositionRange> batchs = canalInstance.getMetaManager().listAllBatchs(clientIdentity);
List<Long> result = new ArrayList<>(batchs.keySet());
Collections.sort(result);
return result;
}

/**
* 进行 batch id 的确认。确认之后,小于等于此 batchId 的 Message 都会被确认。
*
* <pre>
* 注意:进行反馈时必须按照batchId的顺序进行ack(需有客户端保证)
* </pre>
*/
@Override
public void ack(ClientIdentity clientIdentity, long batchId) throws CanalServerException {
checkStart(clientIdentity.getDestination());
checkSubscribe(clientIdentity);

CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
PositionRange<LogPosition> positionRanges = null;
positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity, batchId); // 更新位置
if (positionRanges == null) { // 说明是重复的ack/rollback
throw new CanalServerException(String.format("ack error , clientId:%s batchId:%d is not exist , please check",
clientIdentity.getClientId(),
batchId));
}

// 更新cursor最好严格判断下位置是否有跳跃更新
// Position position = lastRollbackPostions.get(clientIdentity);
// if (position != null) {
// // Position position =
// canalInstance.getMetaManager().getCursor(clientIdentity);
// LogPosition minPosition =
// CanalEventUtils.min(positionRanges.getStart(), (LogPosition)
// position);
// if (minPosition == position) {// ack的position要晚于该最后ack的位置,可能有丢数据
// throw new CanalServerException(
// String.format(
// "ack error , clientId:%s batchId:%d %s is jump ack , last ack:%s",
// clientIdentity.getClientId(), batchId, positionRanges,
// position));
// }
// }

// 更新cursor
if (positionRanges.getAck() != null) {
canalInstance.getMetaManager().updateCursor(clientIdentity, positionRanges.getAck());
if (logger.isInfoEnabled()) {
logger.info("ack successfully, clientId:{} batchId:{} position:{}",
clientIdentity.getClientId(),
batchId,
positionRanges);
}
}

// 可定时清理数据
canalInstance.getEventStore().ack(positionRanges.getEnd(), positionRanges.getEndSeq());
}

/**
* 回滚到未进行 {@link #ack} 的地方,下次fetch的时候,可以从最后一个没有 {@link #ack} 的地方开始拿
*/
@Override
public void rollback(ClientIdentity clientIdentity) throws CanalServerException {
checkStart(clientIdentity.getDestination());
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
// 因为存在第一次链接时自动rollback的情况,所以需要忽略未订阅
boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
if (!hasSubscribe) {
return;
}

synchronized (canalInstance) {
// 清除batch信息
canalInstance.getMetaManager().clearAllBatchs(clientIdentity);
// rollback eventStore中的状态信息
canalInstance.getEventStore().rollback();
logger.info("rollback successfully, clientId:{}", new Object[] { clientIdentity.getClientId() });
}
}

/**
* 回滚到未进行 {@link #ack} 的地方,下次fetch的时候,可以从最后一个没有 {@link #ack} 的地方开始拿
*/
@Override
public void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException {
checkStart(clientIdentity.getDestination());
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());

// 因为存在第一次链接时自动rollback的情况,所以需要忽略未订阅
boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
if (!hasSubscribe) {
return;
}
synchronized (canalInstance) {
// 清除batch信息
PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity,
batchId);
if (positionRanges == null) { // 说明是重复的ack/rollback
throw new CanalServerException(String.format("rollback error, clientId:%s batchId:%d is not exist , please check",
clientIdentity.getClientId(),
batchId));
}

// lastRollbackPostions.put(clientIdentity,
// positionRanges.getEnd());// 记录一下最后rollback的位置
// TODO 后续rollback到指定的batchId位置
canalInstance.getEventStore().rollback();// rollback
// eventStore中的状态信息
logger.info("rollback successfully, clientId:{} batchId:{} position:{}",
clientIdentity.getClientId(),
batchId,
positionRanges);
}
}

public Map<String, CanalInstance> getCanalInstances() {
return Maps.newHashMap(canalInstances);
}

// ======================== helper method =======================

/**
* 根据不同的参数,选择不同的方式获取数据
*/
private Events<Event> getEvents(CanalEventStore eventStore, Position start, int batchSize, Long timeout,
TimeUnit unit) {
if (timeout == null) {
return eventStore.tryGet(start, batchSize);
} else {
try {
if (timeout <= 0) {
return eventStore.get(start, batchSize);
} else {
return eventStore.get(start, batchSize, timeout, unit);
}
} catch (Exception e) {
throw new CanalServerException(e);
}
}
}

private void checkSubscribe(ClientIdentity clientIdentity) {
CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
if (!hasSubscribe) {
throw new CanalServerException(String.format("ClientIdentity:%s should subscribe first",
clientIdentity.toString()));
}
}

private void checkStart(String destination) {
if (!isStart(destination)) {
throw new CanalServerException(String.format("destination:%s should start first", destination));
}
}

private void loadCanalMetrics() {
ServiceLoader<CanalMetricsProvider> providers = ServiceLoader.load(CanalMetricsProvider.class);
List<CanalMetricsProvider> list = new ArrayList<>();
for (CanalMetricsProvider provider : providers) {
list.add(provider);
}

if (list.isEmpty()) {
return;
}

// only allow ONE provider
if (list.size() > 1) {
logger.warn("Found more than one CanalMetricsProvider, use the first one.");
// 报告冲突
for (CanalMetricsProvider p : list) {
logger.warn("Found CanalMetricsProvider: {}.", p.getClass().getName());
}
}

CanalMetricsProvider provider = list.get(0);
this.metrics = provider.getService();
}

private boolean isRaw(CanalEventStore eventStore) {
if (eventStore instanceof MemoryEventStoreWithBuffer) {
return ((MemoryEventStoreWithBuffer) eventStore).isRaw();
}

return true;
}

// ========= setter ==========

public void setCanalInstanceGenerator(CanalInstanceGenerator canalInstanceGenerator) {
this.canalInstanceGenerator = canalInstanceGenerator;
}

public void setMetricsPort(int metricsPort) {
this.metricsPort = metricsPort;
}

public void setUser(String user) {
this.user = user;
}

public void setPasswd(String passwd) {
this.passwd = passwd;
}

}

2.2 使用

@Ignore
public abstract class BaseCanalServerWithEmbededTest {

protected static final String cluster1 = "127.0.0.1:2188";
protected static final String DESTINATION = "example";
protected static final String DETECTING_SQL = "insert into retl.xdual values(1,now()) on duplicate key update x=now()";
protected static final String MYSQL_ADDRESS = "127.0.0.1";
protected static final String USERNAME = "canal";
protected static final String PASSWORD = "canal";
protected static final String FILTER = ".*\\\\..*";

private CanalServerWithEmbedded server;
private ClientIdentity clientIdentity = new ClientIdentity(DESTINATION, (short) 1); ;

@Before
public void setUp() {
server = CanalServerWithEmbedded.instance();
server.setCanalInstanceGenerator(destination -> {
Canal canal = buildCanal();
return new CanalInstanceWithManager(canal, FILTER);
});
server.start();
server.start(DESTINATION);
}

@After
public void tearDown() {
server.stop();
}

@Test
public void testGetWithoutAck() {
int maxEmptyCount = 10;
int emptyCount = 0;
int totalCount = 0;
server.subscribe(clientIdentity);
while (emptyCount < maxEmptyCount) {
Message message = server.getWithoutAck(clientIdentity, 11);
if (CollectionUtils.isEmpty(message.getEntries())) {
emptyCount++;
try {
Thread.sleep(emptyCount * 300L);
} catch (InterruptedException e) {
Assert.fail();
}

System.out.println("empty count : " + emptyCount);
} else {
emptyCount = 0;
totalCount += message.getEntries().size();
server.ack(clientIdentity, message.getId());
}
}

System.out.println("!!!!!! testGetWithoutAck totalCount : " + totalCount);
server.unsubscribe(clientIdentity);
}

@Test
public void testGet() {
int maxEmptyCount = 10;
int emptyCount = 0;
int totalCount = 0;
server.subscribe(clientIdentity);
while (emptyCount < maxEmptyCount) {
Message message = server.get(clientIdentity, 11);
if (CollectionUtils.isEmpty(message.getEntries())) {
emptyCount++;
try {
Thread.sleep(emptyCount * 300L);
} catch (InterruptedException e) {
Assert.fail();
}

System.out.println("empty count : " + emptyCount);
} else {
emptyCount = 0;
totalCount += message.getEntries().size();
}
}

System.out.println("!!!!!! testGet totalCount : " + totalCount);
server.unsubscribe(clientIdentity);
}
}

3. CanalMetricsService

public interface CanalMetricsService {

/**
* Initialization on canal server startup.
*/
void initialize();

/**
* Clean-up at canal server stop phase.
*/
void terminate();

/**
* @return {@code true} if the metrics service is running, otherwise {@code false}.
*/
boolean isRunning();

/**
* Register instance level metrics for specified instance.
* @param instance {@link CanalInstance}
*/
void register(CanalInstance instance);

/**
* Unregister instance level metrics for specified instance.
* @param instance {@link CanalInstance}
*/
void unregister(CanalInstance instance);

/**
* @param port server port for pull
*/
void setServerPort(int port);

}

3.1 NopCanalMetricsService

public class NopCanalMetricsService implements CanalMetricsService {

public static final NopCanalMetricsService NOP = new NopCanalMetricsService();

private NopCanalMetricsService() {}

@Override
public void initialize() {

}

@Override
public void terminate() {

}

@Override
public boolean isRunning() {
return false;
}

@Override
public void register(CanalInstance instance) {

}

@Override
public void unregister(CanalInstance instance) {

}

@Override
public void setServerPort(int port) {

}
}

3.2 CanalMetricsProvider

public interface CanalMetricsProvider {

/**
* @return Impl of {@link CanalMetricsService}
*/
CanalMetricsService getService();

}

4.

1. CanalInstance

public interface CanalInstance extends CanalLifeCycle {

String getDestination();

CanalEventParser getEventParser();

CanalEventSink getEventSink();

CanalEventStore getEventStore();

CanalMetaManager getMetaManager();

CanalAlarmHandler getAlarmHandler();

/**
* 客户端发生订阅/取消订阅行为
*/
boolean subscribeChange(ClientIdentity identity);

CanalMQConfig getMqConfig();
}

1.1 AbstractCanalInstance

/**
* Created with Intellig IDEA. Author: yinxiu Date: 2016-01-07 Time: 22:26
*/
public class AbstractCanalInstance extends AbstractCanalLifeCycle implements CanalInstance {

private static final Logger logger = LoggerFactory.getLogger(AbstractCanalInstance.class);

protected Long canalId; // 和manager交互唯一标示
protected String destination; // 队列名字
protected CanalEventStore<Event> eventStore; // 有序队列

protected CanalEventParser eventParser; // 解析对应的数据信息
protected CanalEventSink<List<CanalEntry.Entry>> eventSink; // 链接parse和store的桥接器
protected CanalMetaManager metaManager; // 消费信息管理器
protected CanalAlarmHandler alarmHandler; // alarm报警机制
protected CanalMQConfig mqConfig; // mq的配置



@Override
public boolean subscribeChange(ClientIdentity identity) {
if (StringUtils.isNotEmpty(identity.getFilter())) {
logger.info("subscribe filter change to " + identity.getFilter());
AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());

boolean isGroup = (eventParser instanceof GroupEventParser);
if (isGroup) {
// 处理group的模式
List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
if(singleEventParser instanceof AbstractEventParser) {
((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
}
}
} else {
if(eventParser instanceof AbstractEventParser) {
((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
}
}

}

// filter的处理规则
// a. parser处理数据过滤处理
// b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份,每份的数据可以根据自己的过滤规则不同而有不同的数据
// 后续内存版的一对多分发,可以考虑
return true;
}

@Override
public void start() {
super.start();
if (!metaManager.isStart()) {
metaManager.start();
}

if (!alarmHandler.isStart()) {
alarmHandler.start();
}

if (!eventStore.isStart()) {
eventStore.start();
}

if (!eventSink.isStart()) {
eventSink.start();
}

if (!eventParser.isStart()) {
beforeStartEventParser(eventParser);
eventParser.start();
afterStartEventParser(eventParser);
}
logger.info("start successful....");
}

@Override
public void stop() {
super.stop();
logger.info("stop CannalInstance for {}-{} ", new Object[] { canalId, destination });

if (eventParser.isStart()) {
beforeStopEventParser(eventParser);
eventParser.stop();
afterStopEventParser(eventParser);
}

if (eventSink.isStart()) {
eventSink.stop();
}

if (eventStore.isStart()) {
eventStore.stop();
}

if (metaManager.isStart()) {
metaManager.stop();
}

if (alarmHandler.isStart()) {
alarmHandler.stop();
}

logger.info("stop successful....");
}

protected void beforeStartEventParser(CanalEventParser eventParser) {

boolean isGroup = (eventParser instanceof GroupEventParser);
if (isGroup) {
// 处理group的模式
List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
startEventParserInternal(singleEventParser, true);
}
} else {
startEventParserInternal(eventParser, false);
}
}

// around event parser, default impl
protected void afterStartEventParser(CanalEventParser eventParser) {
// 读取一下历史订阅的filter信息
List<ClientIdentity> clientIdentitys = metaManager.listAllSubscribeInfo(destination);
for (ClientIdentity clientIdentity : clientIdentitys) {
subscribeChange(clientIdentity);
}
}

// around event parser
protected void beforeStopEventParser(CanalEventParser eventParser) {
// noop
}

protected void afterStopEventParser(CanalEventParser eventParser) {

boolean isGroup = (eventParser instanceof GroupEventParser);
if (isGroup) {
// 处理group的模式
List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
stopEventParserInternal(singleEventParser);
}
} else {
stopEventParserInternal(eventParser);
}
}

/**
* 初始化单个eventParser,不需要考虑group
*/
protected void startEventParserInternal(CanalEventParser eventParser, boolean isGroup) {
if (eventParser instanceof AbstractEventParser) {
AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
// 首先启动log position管理器
CanalLogPositionManager logPositionManager = abstractEventParser.getLogPositionManager();
if (!logPositionManager.isStart()) {
logPositionManager.start();
}
}

if (eventParser instanceof MysqlEventParser) {
MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
CanalHAController haController = mysqlEventParser.getHaController();

if (haController instanceof HeartBeatHAController) {
((HeartBeatHAController) haController).setCanalHASwitchable(mysqlEventParser);
}

if (!haController.isStart()) {
haController.start();
}

}
}

protected void stopEventParserInternal(CanalEventParser eventParser) {
if (eventParser instanceof AbstractEventParser) {
AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
// 首先启动log position管理器
CanalLogPositionManager logPositionManager = abstractEventParser.getLogPositionManager();
if (logPositionManager.isStart()) {
logPositionManager.stop();
}
}

if (eventParser instanceof MysqlEventParser) {
MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
CanalHAController haController = mysqlEventParser.getHaController();
if (haController.isStart()) {
haController.stop();
}
}
}

// ==================getter==================================
@Override
public String getDestination() {
return destination;
}

@Override
public CanalEventParser getEventParser() {
return eventParser;
}

@Override
public CanalEventSink getEventSink() {
return eventSink;
}

@Override
public CanalEventStore getEventStore() {
return eventStore;
}

@Override
public CanalMetaManager getMetaManager() {
return metaManager;
}

@Override
public CanalAlarmHandler getAlarmHandler() {
return alarmHandler;
}

@Override
public CanalMQConfig getMqConfig() {
return mqConfig;
}
}

1.2 CanalInstanceWithManager

/**
* 单个canal实例,比如一个destination会独立一个实例
*
* @author jianghang 2012-7-11 下午09:26:51
* @version 1.0.0
*/
public class CanalInstanceWithManager extends AbstractCanalInstance {

private static final Logger logger = LoggerFactory.getLogger(CanalInstanceWithManager.class);
protected String filter; // 过滤表达式
protected CanalParameter parameters; // 对应参数

public CanalInstanceWithManager(Canal canal, String filter){
this.parameters = canal.getCanalParameter();
this.canalId = canal.getId();
this.destination = canal.getName();
this.filter = filter;

logger.info("init CanalInstance for {}-{} with parameters:{}", canalId, destination, parameters);
// 初始化报警机制
initAlarmHandler();
// 初始化metaManager
initMetaManager();
// 初始化eventStore
initEventStore();
// 初始化eventSink
initEventSink();
// 初始化eventParser;
initEventParser();

// 基础工具,需要提前start,会有先订阅再根据filter条件启动parse的需求
if (!alarmHandler.isStart()) {
alarmHandler.start();
}

if (!metaManager.isStart()) {
metaManager.start();
}
logger.info("init successful....");
}

public void start() {
// 初始化metaManager
logger.info("start CannalInstance for {}-{} with parameters:{}", canalId, destination, parameters);
super.start();
}

@SuppressWarnings("resource")
protected void initAlarmHandler() {
logger.info("init alarmHandler begin...");
String alarmHandlerClass = parameters.getAlarmHandlerClass();
String alarmHandlerPluginDir = parameters.getAlarmHandlerPluginDir();
if (alarmHandlerClass == null || alarmHandlerPluginDir == null) {
alarmHandler = new LogAlarmHandler();
} else {
try {
File externalLibDir = new File(alarmHandlerPluginDir);
File[] jarFiles = externalLibDir.listFiles((dir, name) -> name.endsWith(".jar"));
if (jarFiles == null || jarFiles.length == 0) {
throw new IllegalStateException(String.format("alarmHandlerPluginDir [%s] can't find any name endswith \".jar\" file.",
alarmHandlerPluginDir));
}
URL[] urls = new URL[jarFiles.length];
for (int i = 0; i < jarFiles.length; i++) {
urls[i] = jarFiles[i].toURI().toURL();
}
ClassLoader currentClassLoader = new URLClassLoader(urls,
CanalInstanceWithManager.class.getClassLoader());
Class<CanalAlarmHandler> _alarmClass = (Class<CanalAlarmHandler>) currentClassLoader.loadClass(alarmHandlerClass);
alarmHandler = _alarmClass.newInstance();
logger.info("init [{}] alarm handler success.", alarmHandlerClass);
} catch (Throwable e) {
String errorMsg = String.format("init alarmHandlerPluginDir [%s] alarm handler [%s] error: %s",
alarmHandlerPluginDir,
alarmHandlerClass,
ExceptionUtils.getFullStackTrace(e));
logger.error(errorMsg);
throw new CanalException(errorMsg, e);
}
}
logger.info("init alarmHandler end! \n\t load CanalAlarmHandler:{} ", alarmHandler.getClass().getName());
}

protected void initMetaManager() {
logger.info("init metaManager begin...");
MetaMode mode = parameters.getMetaMode();
if (mode.isMemory()) {
metaManager = new MemoryMetaManager();
} else if (mode.isZookeeper()) {
metaManager = new ZooKeeperMetaManager();
((ZooKeeperMetaManager) metaManager).setZkClientx(getZkclientx());
} else if (mode.isMixed()) {
// metaManager = new MixedMetaManager();
metaManager = new PeriodMixedMetaManager();// 换用优化过的mixed, at
// 2012-09-11
// 设置内嵌的zk metaManager
ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
zooKeeperMetaManager.setZkClientx(getZkclientx());
((PeriodMixedMetaManager) metaManager).setZooKeeperMetaManager(zooKeeperMetaManager);
} else if (mode.isLocalFile()) {
FileMixedMetaManager fileMixedMetaManager = new FileMixedMetaManager();
fileMixedMetaManager.setDataDir(parameters.getDataDir());
fileMixedMetaManager.setPeriod(parameters.getMetaFileFlushPeriod());
metaManager = fileMixedMetaManager;
} else {
throw new CanalException("unsupport MetaMode for " + mode);
}

logger.info("init metaManager end! \n\t load CanalMetaManager:{} ", metaManager.getClass().getName());
}

protected void initEventStore() {
logger.info("init eventStore begin...");
StorageMode mode = parameters.getStorageMode();
if (mode.isMemory()) {
MemoryEventStoreWithBuffer memoryEventStore = new MemoryEventStoreWithBuffer();
memoryEventStore.setBufferSize(parameters.getMemoryStorageBufferSize());
memoryEventStore.setBufferMemUnit(parameters.getMemoryStorageBufferMemUnit());
memoryEventStore.setBatchMode(BatchMode.valueOf(parameters.getStorageBatchMode().name()));
memoryEventStore.setDdlIsolation(parameters.getDdlIsolation());
memoryEventStore.setRaw(parameters.getMemoryStorageRawEntry());
eventStore = memoryEventStore;
} else if (mode.isFile()) {
// 后续版本支持
throw new CanalException("unsupport MetaMode for " + mode);
} else if (mode.isMixed()) {
// 后续版本支持
throw new CanalException("unsupport MetaMode for " + mode);
} else {
throw new CanalException("unsupport MetaMode for " + mode);
}

if (eventStore instanceof AbstractCanalStoreScavenge) {
StorageScavengeMode scavengeMode = parameters.getStorageScavengeMode();
AbstractCanalStoreScavenge eventScavengeStore = (AbstractCanalStoreScavenge) eventStore;
eventScavengeStore.setDestination(destination);
eventScavengeStore.setCanalMetaManager(metaManager);
eventScavengeStore.setOnAck(scavengeMode.isOnAck());
eventScavengeStore.setOnFull(scavengeMode.isOnFull());
eventScavengeStore.setOnSchedule(scavengeMode.isOnSchedule());
if (scavengeMode.isOnSchedule()) {
eventScavengeStore.setScavengeSchedule(parameters.getScavengeSchdule());
}
}
logger.info("init eventStore end! \n\t load CanalEventStore:{}", eventStore.getClass().getName());
}

protected void initEventSink() {
logger.info("init eventSink begin...");

int groupSize = getGroupSize();
if (groupSize <= 1) {
eventSink = new EntryEventSink();
} else {
eventSink = new GroupEventSink(groupSize);
}

if (eventSink instanceof EntryEventSink) {
((EntryEventSink) eventSink).setFilterTransactionEntry(false);
((EntryEventSink) eventSink).setEventStore(getEventStore());
}
// if (StringUtils.isNotEmpty(filter)) {
// AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(filter);
// ((AbstractCanalEventSink) eventSink).setFilter(aviaterFilter);
// }
logger.info("init eventSink end! \n\t load CanalEventSink:{}", eventSink.getClass().getName());
}

protected void initEventParser() {
logger.info("init eventParser begin...");
SourcingType type = parameters.getSourcingType();

List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
if (!CollectionUtils.isEmpty(groupDbAddresses)) {
int size = groupDbAddresses.get(0).size();// 取第一个分组的数量,主备分组的数量必须一致
List<CanalEventParser> eventParsers = new ArrayList<>();
for (int i = 0; i < size; i++) {
List<InetSocketAddress> dbAddress = new ArrayList<>();
SourcingType lastType = null;
for (List<DataSourcing> groupDbAddress : groupDbAddresses) {
if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
throw new CanalException(String.format("master/slave Sourcing type is unmatch. %s vs %s",
lastType,
groupDbAddress.get(i).getType()));
}

lastType = groupDbAddress.get(i).getType();
dbAddress.add(groupDbAddress.get(i).getDbAddress());
}

// 初始化其中的一个分组parser
eventParsers.add(doInitEventParser(lastType, dbAddress));
}

if (eventParsers.size() > 1) { // 如果存在分组,构造分组的parser
GroupEventParser groupEventParser = new GroupEventParser();
groupEventParser.setEventParsers(eventParsers);
this.eventParser = groupEventParser;
} else {
this.eventParser = eventParsers.get(0);
}
} else {
// 创建一个空数据库地址的parser,可能使用了tddl指定地址,启动的时候才会从tddl获取地址
this.eventParser = doInitEventParser(type, new ArrayList<>());
}

logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
}

private CanalEventParser doInitEventParser(SourcingType type, List<InetSocketAddress> dbAddresses) {
CanalEventParser eventParser;
if (type.isMysql()) {
MysqlEventParser mysqlEventParser = null;
if (StringUtils.isNotEmpty(parameters.getRdsAccesskey())
&& StringUtils.isNotEmpty(parameters.getRdsSecretkey())
&& StringUtils.isNotEmpty(parameters.getRdsInstanceId())) {

mysqlEventParser = new RdsBinlogEventParserProxy();
((RdsBinlogEventParserProxy) mysqlEventParser).setAccesskey(parameters.getRdsAccesskey());
((RdsBinlogEventParserProxy) mysqlEventParser).setSecretkey(parameters.getRdsSecretkey());
((RdsBinlogEventParserProxy) mysqlEventParser).setInstanceId(parameters.getRdsInstanceId());
} else {
mysqlEventParser = new MysqlEventParser();
}
mysqlEventParser.setDestination(destination);
// 编码参数
mysqlEventParser.setConnectionCharset(parameters.getConnectionCharset());
// 网络相关参数1
mysqlEventParser.setDefaultConnectionTimeoutInSeconds(parameters.getDefaultConnectionTimeoutInSeconds());
mysqlEventParser.setSendBufferSize(parameters.getSendBufferSize());
mysqlEventParser.setReceiveBufferSize(parameters.getReceiveBufferSize());
// 心跳检查参数
mysqlEventParser.setDetectingEnable(parameters.getDetectingEnable());
mysqlEventParser.setDetectingSQL(parameters.getDetectingSQL());
mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
// 数据库信息参数
mysqlEventParser.setSlaveId(parameters.getSlaveId());
if (!CollectionUtils.isEmpty(dbAddresses)) {
mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
parameters.getDbUsername(),
parameters.getDbPassword(),
parameters.getDefaultDatabaseName(),
parameters.getSslInfo()));

if (dbAddresses.size() > 1) {
mysqlEventParser.setStandbyInfo(new AuthenticationInfo(dbAddresses.get(1),
parameters.getDbUsername(),
parameters.getDbPassword(),
parameters.getDefaultDatabaseName(),
parameters.getSslInfo()));
}
}

if (!CollectionUtils.isEmpty(parameters.getPositions())) {
EntryPosition masterPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0),
EntryPosition.class);
// binlog位置参数
mysqlEventParser.setMasterPosition(masterPosition);

if (parameters.getPositions().size() > 1) {
EntryPosition standbyPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(1),
EntryPosition.class);
mysqlEventParser.setStandbyPosition(standbyPosition);
}
}
mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds());
mysqlEventParser.setProfilingEnabled(false);
mysqlEventParser.setFilterTableError(parameters.getFilterTableError());
mysqlEventParser.setParallel(parameters.getParallel());
mysqlEventParser.setIsGTIDMode(BooleanUtils.toBoolean(parameters.getGtidEnable()));
mysqlEventParser.setMultiStreamEnable(parameters.getMultiStreamEnable());
// tsdb
if (parameters.getTsdbSnapshotInterval() != null) {
mysqlEventParser.setTsdbSnapshotInterval(parameters.getTsdbSnapshotInterval());
}
if (parameters.getTsdbSnapshotExpire() != null) {
mysqlEventParser.setTsdbSnapshotExpire(parameters.getTsdbSnapshotExpire());
}
boolean tsdbEnable = BooleanUtils.toBoolean(parameters.getTsdbEnable());
// manager启动模式默认使用mysql tsdb机制
final String tsdbSpringXml = "classpath:spring/tsdb/mysql-tsdb.xml";
if (tsdbEnable) {
mysqlEventParser.setTableMetaTSDBFactory(new DefaultTableMetaTSDBFactory() {

@Override
public void destory(String destination) {
TableMetaTSDBBuilder.destory(destination);
}

@Override
public TableMetaTSDB build(String destination, String springXml) {
synchronized (CanalInstanceWithManager.class) {
try {
System.setProperty("canal.instance.tsdb.url", parameters.getTsdbJdbcUrl());
System.setProperty("canal.instance.tsdb.dbUsername", parameters.getTsdbJdbcUserName());
System.setProperty("canal.instance.tsdb.dbPassword", parameters.getTsdbJdbcPassword());
System.setProperty("canal.instance.destination", destination);

return TableMetaTSDBBuilder.build(destination, tsdbSpringXml);
} finally {
// reset
System.setProperty("canal.instance.destination", "");
System.setProperty("canal.instance.tsdb.url", "");
System.setProperty("canal.instance.tsdb.dbUsername", "");
System.setProperty("canal.instance.tsdb.dbPassword", "");
}
}
}
});
mysqlEventParser.setTsdbJdbcUrl(parameters.getTsdbJdbcUrl());
mysqlEventParser.setTsdbJdbcUserName(parameters.getTsdbJdbcUserName());
mysqlEventParser.setTsdbJdbcPassword(parameters.getTsdbJdbcPassword());
mysqlEventParser.setTsdbSpringXml(tsdbSpringXml);
mysqlEventParser.setEnableTsdb(tsdbEnable);
}
eventParser = mysqlEventParser;
} else if (type.isLocalBinlog()) {
LocalBinlogEventParser localBinlogEventParser = new LocalBinlogEventParser();
localBinlogEventParser.setDestination(destination);
localBinlogEventParser.setBufferSize(parameters.getReceiveBufferSize());
localBinlogEventParser.setConnectionCharset(parameters.getConnectionCharset());
localBinlogEventParser.setDirectory(parameters.getLocalBinlogDirectory());
localBinlogEventParser.setProfilingEnabled(false);
localBinlogEventParser.setDetectingEnable(parameters.getDetectingEnable());
localBinlogEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
localBinlogEventParser.setFilterTableError(parameters.getFilterTableError());
localBinlogEventParser.setParallel(parameters.getParallel());
// 数据库信息,反查表结构时需要
if (!CollectionUtils.isEmpty(dbAddresses)) {
localBinlogEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0),
parameters.getDbUsername(),
parameters.getDbPassword(),
parameters.getDefaultDatabaseName(),
parameters.getSslInfo()));
}

eventParser = localBinlogEventParser;
} else if (type.isOracle()) {
throw new CanalException("unsupport SourcingType for " + type);
} else {
throw new CanalException("unsupport SourcingType for " + type);
}

// add transaction support at 2012-12-06
if (eventParser instanceof AbstractEventParser) {
AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
abstractEventParser.setTransactionSize(parameters.getTransactionSize());
abstractEventParser.setLogPositionManager(initLogPositionManager());
abstractEventParser.setAlarmHandler(getAlarmHandler());
abstractEventParser.setEventSink(getEventSink());

if (StringUtils.isNotEmpty(filter)) {
AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(filter);
abstractEventParser.setEventFilter(aviaterFilter);
}

// 设置黑名单
if (StringUtils.isNotEmpty(parameters.getBlackFilter())) {
AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(parameters.getBlackFilter());
abstractEventParser.setEventBlackFilter(aviaterFilter);
}
}
if (eventParser instanceof MysqlEventParser) {
MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;

// 初始化haController,绑定与eventParser的关系,haController会控制eventParser
CanalHAController haController = initHaController();
mysqlEventParser.setHaController(haController);
}
return eventParser;
}

protected CanalHAController initHaController() {
logger.info("init haController begin...");
HAMode haMode = parameters.getHaMode();
CanalHAController haController = null;
if (haMode.isHeartBeat()) {
haController = new HeartBeatHAController();
((HeartBeatHAController) haController).setDetectingRetryTimes(parameters.getDetectingRetryTimes());
((HeartBeatHAController) haController).setSwitchEnable(parameters.getHeartbeatHaEnable());
} else {
throw new CanalException("unsupport HAMode for " + haMode);
}
logger.info("init haController end! \n\t load CanalHAController:{}", haController.getClass().getName());

return haController;
}

protected CanalLogPositionManager initLogPositionManager() {
logger.info("init logPositionPersistManager begin...");
IndexMode indexMode = parameters.getIndexMode();
CanalLogPositionManager logPositionManager;
if (indexMode.isMemory()) {
logPositionManager = new MemoryLogPositionManager();
} else if (indexMode.isZookeeper()) {
logPositionManager = new ZooKeeperLogPositionManager(getZkclientx());
} else if (indexMode.isMixed()) {
MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
ZooKeeperLogPositionManager zooKeeperLogPositionManager = new ZooKeeperLogPositionManager(getZkclientx());
logPositionManager = new PeriodMixedLogPositionManager(memoryLogPositionManager,
zooKeeperLogPositionManager,
1000L);
} else if (indexMode.isMeta()) {
logPositionManager = new MetaLogPositionManager(metaManager);
} else if (indexMode.isMemoryMetaFailback()) {
MemoryLogPositionManager primary = new MemoryLogPositionManager();
MetaLogPositionManager secondary = new MetaLogPositionManager(metaManager);

logPositionManager = new FailbackLogPositionManager(primary, secondary);
} else {
throw new CanalException("unsupport indexMode for " + indexMode);
}

logger.info("init logPositionManager end! \n\t load CanalLogPositionManager:{}", logPositionManager.getClass()
.getName());

return logPositionManager;
}

protected void startEventParserInternal(CanalEventParser eventParser, boolean isGroup) {
if (eventParser instanceof AbstractEventParser) {
AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
abstractEventParser.setAlarmHandler(getAlarmHandler());
}

super.startEventParserInternal(eventParser, isGroup);
}

private int getGroupSize() {
List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
if (!CollectionUtils.isEmpty(groupDbAddresses)) {
return groupDbAddresses.get(0).size();
} else {
// 可能是基于tddl的启动
return 1;
}
}

private synchronized ZkClientx getZkclientx() {
// 做一下排序,保证相同的机器只使用同一个链接
List<String> zkClusters = new ArrayList<>(parameters.getZkClusters());
Collections.sort(zkClusters);

return ZkClientx.getZkClient(StringUtils.join(zkClusters, ";"));
}

public void setAlarmHandler(CanalAlarmHandler alarmHandler) {
this.alarmHandler = alarmHandler;
}

}

1.3 CanalInstanceWithSpring

/**
* 基于spring容器启动canal实例,方便独立于manager启动
*
* @author jianghang 2012-7-12 下午01:21:26
* @author zebin.xuzb
* @version 1.0.0
*/
public class CanalInstanceWithSpring extends AbstractCanalInstance {

private static final Logger logger = LoggerFactory.getLogger(CanalInstanceWithSpring.class);

public void start() {
logger.info("start CannalInstance for {}-{} ", new Object[] { 1, destination });
super.start();
}

// ======== setter ========

public void setDestination(String destination) {
this.destination = destination;
}

public void setEventParser(CanalEventParser eventParser) {
this.eventParser = eventParser;
}

public void setEventSink(CanalEventSink<List<CanalEntry.Entry>> eventSink) {
this.eventSink = eventSink;
}

public void setEventStore(CanalEventStore<Event> eventStore) {
this.eventStore = eventStore;
}

public void setMetaManager(CanalMetaManager metaManager) {
this.metaManager = metaManager;
}

public void setAlarmHandler(CanalAlarmHandler alarmHandler) {
this.alarmHandler = alarmHandler;
}

public void setMqConfig(CanalMQConfig mqConfig) {
this.mqConfig = mqConfig;
}

}

2. CanalInstanceGenerator

public interface CanalInstanceGenerator {

/**
* 通过 destination 产生特定的 {@link CanalInstance}
*
* @param destination
* @return
*/
CanalInstance generate(String destination);
}

2.1 ManagerCanalInstanceGenerator

/**
* 基于manager生成对应的{@linkplain CanalInstance}
*
* @author jianghang 2012-7-12 下午05:37:09
* @version 1.0.0
*/
public class ManagerCanalInstanceGenerator implements CanalInstanceGenerator {

private CanalConfigClient canalConfigClient;

public CanalInstance generate(String destination) {
Canal canal = canalConfigClient.findCanal(destination);
String filter = canalConfigClient.findFilter(destination);
return new CanalInstanceWithManager(canal, filter);
}

// ================ setter / getter ================

public void setCanalConfigClient(CanalConfigClient canalConfigClient) {
this.canalConfigClient = canalConfigClient;
}

}

2.1.1 CanalConfigClient

/**
* 对应canal的配置
*
* @author jianghang 2012-7-4 下午03:09:17
* @version 1.0.0
*/
public class CanalConfigClient {

/**
* 根据对应的destinantion查询Canal信息
*/
public Canal findCanal(String destination) {
// TODO 根据自己的业务实现
throw new UnsupportedOperationException();
}

/**
* 根据对应的destinantion查询filter信息
*/
public String findFilter(String destination) {
// TODO 根据自己的业务实现
throw new UnsupportedOperationException();
}

}

2.2 SpringCanalInstanceGenerator

/**
* @author zebin.xuzb @ 2012-7-12
* @version 1.0.0
*/
public class SpringCanalInstanceGenerator implements CanalInstanceGenerator {

private static final Logger logger = LoggerFactory.getLogger(SpringCanalInstanceGenerator.class);
private String springXml;
private String defaultName = "instance";
private BeanFactory beanFactory;

public CanalInstance generate(String destination) {
synchronized (CanalEventParser.class) {
try {
// 设置当前正在加载的通道,加载spring查找文件时会用到该变量
System.setProperty("canal.instance.destination", destination);
this.beanFactory = getBeanFactory(springXml);
String beanName = destination;
if (!beanFactory.containsBean(beanName)) {
beanName = defaultName;
}

return (CanalInstance) beanFactory.getBean(beanName);
} catch (Throwable e) {
logger.error("generator instance failed.", e);
throw new CanalException(e);
} finally {
System.setProperty("canal.instance.destination", "");
}
}
}

private BeanFactory getBeanFactory(String springXml) {
if (!StringUtils.startsWithIgnoreCase(springXml, "classpath:")) {
springXml = "classpath:" + springXml;
}
ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml);
return applicationContext;
}

public void setSpringXml(String springXml) {
this.springXml = springXml;
}
}

2.3 PlainCanalInstanceGenerator

/**
* 基于manager生成对应的{@linkplain CanalInstance}
*
* @author jianghang 2012-7-12 下午05:37:09
* @version 1.0.0
*/
public class PlainCanalInstanceGenerator implements CanalInstanceGenerator {

private static final Logger logger = LoggerFactory.getLogger(PlainCanalInstanceGenerator.class);
private String springXml;
private PlainCanalConfigClient canalConfigClient;
private String defaultName = "instance";
private BeanFactory beanFactory;
private Properties canalConfig;

public PlainCanalInstanceGenerator(Properties canalConfig){
this.canalConfig = canalConfig;
}

public CanalInstance generate(String destination) {
synchronized (CanalEventParser.class) {
try {
PlainCanal canal = canalConfigClient.findInstance(destination, null);
if (canal == null) {
throw new CanalException("instance : " + destination + " config is not found");
}
Properties properties = canal.getProperties();
// merge local
properties.putAll(canalConfig);

// 设置动态properties,替换掉本地properties
com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer.propertiesLocal.set(properties);
// 设置当前正在加载的通道,加载spring查找文件时会用到该变量
System.setProperty("canal.instance.destination", destination);
this.beanFactory = getBeanFactory(springXml);
String beanName = destination;
if (!beanFactory.containsBean(beanName)) {
beanName = defaultName;
}

return (CanalInstance) beanFactory.getBean(beanName);
} catch (Throwable e) {
logger.error("generator instance failed.", e);
throw new CanalException(e);
} finally {
System.setProperty("canal.instance.destination", "");
com.alibaba.otter.canal.instance.spring.support.PropertyPlaceholderConfigurer.propertiesLocal.remove();
}
}
}

// ================ setter / getter ================

private BeanFactory getBeanFactory(String springXml) {
if (!StringUtils.startsWithIgnoreCase(springXml, "classpath:")) {
springXml = "classpath:" + springXml;
}
ApplicationContext applicationContext = new ClassPathXmlApplicationContext(springXml);
return applicationContext;
}

public void setCanalConfigClient(PlainCanalConfigClient canalConfigClient) {
this.canalConfigClient = canalConfigClient;
}

public void setSpringXml(String springXml) {
this.springXml = springXml;
}

}

2.3.1 PlainCanalConfigClient

/**
* 远程配置获取
*
* @author rewerma 2019-01-25 下午05:20:16
* @author agapple 2019年8月26日 下午7:52:06
* @since 1.1.4
*/
public class PlainCanalConfigClient extends AbstractCanalLifeCycle implements CanalLifeCycle {

private final static Integer REQUEST_TIMEOUT = 5000;
private String configURL;
private String user;
private String passwd;
private HttpHelper httpHelper;
private String localIp;
private int adminPort;
private boolean autoRegister;
private String autoCluster;
private String name;

public PlainCanalConfigClient(String configURL, String user, String passwd, String localIp, int adminPort,
boolean autoRegister, String autoCluster, String name){
this(configURL, user, passwd, localIp, adminPort);
this.autoCluster = autoCluster;
this.autoRegister = autoRegister;
this.name = name;
}

public PlainCanalConfigClient(String configURL, String user, String passwd, String localIp, int adminPort){
this.configURL = configURL;
if (!StringUtils.startsWithIgnoreCase(configURL, "http")) {
this.configURL = "http://" + configURL;
} else {
this.configURL = configURL;
}
this.user = user;
this.passwd = passwd;
this.httpHelper = new HttpHelper();
if (StringUtils.isEmpty(localIp)) {
this.localIp = "127.0.0.1";// 本地测试用
} else {
this.localIp = localIp;
}
this.adminPort = adminPort;
}

/**
* 加载canal.properties文件
*
* @return 远程配置的properties
*/
public PlainCanal findServer(String md5) {
if (StringUtils.isEmpty(md5)) {
md5 = "";
}
String url = configURL + "/api/v1/config/server_polling?ip=" + localIp + "&port=" + adminPort + "&md5=" + md5
+ "&register=" + (autoRegister ? 1 : 0) + "&cluster=" + StringUtils.stripToEmpty(autoCluster) + "&name=" + StringUtils.stripToEmpty(name);
return queryConfig(url);
}

/**
* 加载远程的instance.properties
*/
public PlainCanal findInstance(String destination, String md5) {
if (StringUtils.isEmpty(md5)) {
md5 = "";
}
String url = configURL + "/api/v1/config/instance_polling/" + UrlEscapers.urlPathSegmentEscaper().escape(destination) + "?md5=" + md5;
return queryConfig(url);
}

/**
* 返回需要运行的instance列表
*/
public String findInstances(String md5) {
if (StringUtils.isEmpty(md5)) {
md5 = "";
}
String url = configURL + "/api/v1/config/instances_polling?md5=" + md5 + "&ip=" + localIp + "&port="
+ adminPort;
ResponseModel<CanalConfig> config = doQuery(url);
if (config.data != null) {
return config.data.content;
} else {
return null;
}
}

private PlainCanal queryConfig(String url) {
try {
ResponseModel<CanalConfig> config = doQuery(url);
return processData(config.data);
} catch (Throwable e) {
throw new CanalException("load manager config failed.", e);
}
}

private ResponseModel<CanalConfig> doQuery(String url) {
Map<String, String> heads = new HashMap<>();
heads.put("user", user);
heads.put("passwd", passwd);
String response = httpHelper.get(url, heads, REQUEST_TIMEOUT);
ResponseModel<CanalConfig> resp = JSON.parseObject(response,
new TypeReference<ResponseModel<CanalConfig>>() {
});

if (!HttpHelper.REST_STATE_OK.equals(resp.code)) {
throw new CanalException("requestGet for canal config error: " + resp.message);
}

return resp;
}

private PlainCanal processData(CanalConfig config) throws IOException, NoSuchAlgorithmException {
Properties properties = new Properties();
String md5 = null;
String status = null;
if (config != null && StringUtils.isNotEmpty(config.content)) {
md5 = SecurityUtil.md5String(config.content);
status = config.status;
properties.load(new ByteArrayInputStream(config.content.getBytes(StandardCharsets.UTF_8)));
} else {
// null代表没有新配置变更
return null;
}

return new PlainCanal(properties, status, md5);
}

private static class ResponseModel<T> {

public Integer code;
public String message;
public T data;
}

private static class CanalConfig {

public String content;
public String status;

}
}

3. PropertyPlaceholderConfigurer

3.1 PropertyPlaceholderConfigurer

/**
* 扩展Spring的
* {@linkplain org.springframework.beans.factory.config.PropertyPlaceholderConfigurer}
* ,增加默认值的功能。 例如:${placeholder:defaultValue},假如placeholder的值不存在,则默认取得
* defaultValue。
*
* @author jianghang 2013-1-24 下午03:37:56
* @version 1.0.0
*/
public class PropertyPlaceholderConfigurer extends org.springframework.beans.factory.config.PropertyPlaceholderConfigurer implements ResourceLoaderAware, InitializingBean {

private static final String PLACEHOLDER_PREFIX = "${";
private static final String PLACEHOLDER_SUFFIX = "}";
public static ThreadLocal<Properties> propertiesLocal = ThreadLocal.withInitial(Properties::new);

private ResourceLoader loader;
private String[] locationNames;

public PropertyPlaceholderConfigurer(){
setIgnoreUnresolvablePlaceholders(true);
}

public void setResourceLoader(ResourceLoader loader) {
this.loader = loader;
}

public void setLocationNames(String[] locations) {
this.locationNames = locations;
}

public void afterPropertiesSet() throws Exception {
Assert.notNull(loader, "no resourceLoader");

if (locationNames != null) {
for (int i = 0; i < locationNames.length; i++) {
locationNames[i] = resolveSystemPropertyPlaceholders(locationNames[i]);
}
}

if (locationNames != null) {
List<Resource> resources = new ArrayList<>(locationNames.length);

for (String location : locationNames) {
location = trimToNull(location);

if (location != null) {
resources.add(loader.getResource(location));
}
}

super.setLocations(resources.toArray(new Resource[resources.size()]));
}
}

private String resolveSystemPropertyPlaceholders(String text) {
StringBuilder buf = new StringBuilder(text);

for (int startIndex = buf.indexOf(PLACEHOLDER_PREFIX); startIndex >= 0;) {
int endIndex = buf.indexOf(PLACEHOLDER_SUFFIX, startIndex + PLACEHOLDER_PREFIX.length());

if (endIndex != -1) {
String placeholder = buf.substring(startIndex + PLACEHOLDER_PREFIX.length(), endIndex);
int nextIndex = endIndex + PLACEHOLDER_SUFFIX.length();

try {
String value = resolveSystemPropertyPlaceholder(placeholder);

if (value != null) {
buf.replace(startIndex, endIndex + PLACEHOLDER_SUFFIX.length(), value);
nextIndex = startIndex + value.length();
} else {
System.err.println("Could not resolve placeholder '"
+ placeholder
+ "' in ["
+ text
+ "] as system property: neither system property nor environment variable found");
}
} catch (Throwable ex) {
System.err.println("Could not resolve placeholder '" + placeholder + "' in [" + text
+ "] as system property: " + ex);
}

startIndex = buf.indexOf(PLACEHOLDER_PREFIX, nextIndex);
} else {
startIndex = -1;
}
}

return buf.toString();
}

private String resolveSystemPropertyPlaceholder(String placeholder) {
DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder);
String value = System.getProperty(dp.placeholder);

if (value == null) {
value = System.getenv(dp.placeholder);
}

if (value == null) {
value = dp.defaultValue;
}

return value;
}

@Override
protected String resolvePlaceholder(String placeholder, Properties props, int systemPropertiesMode) {
DefaultablePlaceholder dp = new DefaultablePlaceholder(placeholder);
String propVal = null;
// 以system为准覆盖本地配置, 适用于docker
if (systemPropertiesMode == SYSTEM_PROPERTIES_MODE_OVERRIDE) {
propVal = resolveSystemProperty(dp.placeholder);
}

// 以threadlocal的为准覆盖file properties
if (propVal == null) {
Properties localProperties = propertiesLocal.get();
propVal = resolvePlaceholder(dp.placeholder, localProperties);
}

if (propVal == null) {
propVal = resolvePlaceholder(dp.placeholder, props);
}

if (propVal == null && systemPropertiesMode == SYSTEM_PROPERTIES_MODE_FALLBACK) {
propVal = resolveSystemProperty(dp.placeholder);
}

if (propVal == null) {
propVal = dp.defaultValue;
}

return trimToEmpty(propVal);
}

private static class DefaultablePlaceholder {

private final String defaultValue;
private final String placeholder;

public DefaultablePlaceholder(String placeholder){
int commaIndex = placeholder.indexOf(":");
String defaultValue = null;

if (commaIndex >= 0) {
defaultValue = trimToEmpty(placeholder.substring(commaIndex + 1));
placeholder = trimToEmpty(placeholder.substring(0, commaIndex));
}

this.placeholder = placeholder;
this.defaultValue = defaultValue;
}
}

private String trimToNull(String str) {
if (str == null) {
return null;
}

String result = str.trim();

if (result == null || result.length() == 0) {
return null;
}

return result;
}

public static String trimToEmpty(String str) {
if (str == null) {
return "";
}

return str.trim();
}
}

3.2 SocketAddressEditor

public class SocketAddressEditor extends PropertyEditorSupport implements PropertyEditorRegistrar {

public void registerCustomEditors(PropertyEditorRegistry registry) {
registry.registerCustomEditor(InetSocketAddress.class, this);
}

public void setAsText(String text) throws IllegalArgumentException {
String[] addresses = AddressUtils.splitIPAndPort(text);
if (addresses.length > 0) {
if (addresses.length != 2) {
throw new RuntimeException("address[" + text + "] is illegal, eg.127.0.0.1:3306");
} else {
setValue(new InetSocketAddress(addresses[0], Integer.valueOf(addresses[1])));
}
} else {
setValue(null);
}
}
}

本站由 卡卡龙 使用 Stellar 1.33.1主题创建

本站访问量 次. 本文阅读量 次.