zookeeper Unreasonable length错误导致无法启动
现象: 启动报错Exception in thread "main" java.io.IOException: Unreasonable length = 1860320at org.apache.jute.BinaryInputArchive.checkLength(BinaryInputArchive.java:127)at org.apache.jute.BinaryInputArchive
·
现象: 启动报错
Exception in thread "main" java.io.IOException: Unreasonable length = 1860320
at org.apache.jute.BinaryInputArchive.checkLength(BinaryInputArchive.java:127)
at org.apache.jute.BinaryInputArchive.readBuffer(BinaryInputArchive.java:92)
at com.sf.boot.code.Zk4lTest.testLog(Zk4lTest.java:241)
at com.sf.boot.code.Zk4lTest.main(Zk4lTest.java:392)
原因:
- zookeeper启动的时候会加载datalog(快照日志)进行内存数据恢复,zookeeper的快照日志是按照行记录的,在读取的时候会判断record的大小是否大于jute.maxbuffer(默认1M).
- 在写入数据的时候也是会判断节点数据是否大于jute.maxbuffer,那么这样的数据是如何写入快照日志的?答案是closeSession的时候
- 当一个客户端建立了很多的临时节点时,在进行closeSession的时候会把所有的临时节点的path集中在一起写到快照日志的record(因为closeSession时是需要删除所有临时节点的)。
这时候如果path很长很多,那么就有可能超过jute.maxbuffer。例如:dubbo的场景,dubbo是把所有注册数据都当成path,这就导致path很长
解决方案:
- zookeeper启动的时候设置 jvm参数大小 -Djute.maxbuffer
- 删除zookeeper有问题的快照日志文件,不过这样做会导致还没有持久化的数据丢失。zookeeper是写快照日志,当快照日志达到一定条数才会落盘持久化
解析快照日志文件
可以使用org.apache.zookeeper.server.LogFormatter 解析快照日志。
下面的方法是copy LogFormatter的代码修改了一下
/**
* 测试快照日志是否正常
* @param filePath
* @param errorFilePath
* @param byteLength
* @throws Exception
*/
public static void testLog(String filePath,String errorFilePath,long byteLength) throws Exception {
FileInputStream fis = new FileInputStream(filePath);
BinaryInputArchive logStream = BinaryInputArchive.getArchive(fis);
FileHeader fhdr = new FileHeader();
fhdr.deserialize(logStream, "fileheader");
int count = 0;
int length = 0;
while (true) {
long crcValue = 0;
byte[] bytes;
try {
crcValue = logStream.readLong("crcvalue");
bytes = logStream.readBuffer("txnEntry");
length += bytes.length;
} catch (Exception e) {
System.out.println("11EOF reached after " + count + " txns. crcValue:"+crcValue);
throw e;
}
if (bytes.length > byteLength){
File errorFile = new File(errorFilePath+"."+count);
FileUtils.write(errorFile,new String(bytes));
}
// System.out.println(crcValue);
if (bytes.length == 0) {
// Since we preallocate, we define EOF to be an
// empty transaction
System.out.println("22EOF reached after " + count + " txns.");
System.out.println(length);
return;
}
Checksum crc = new Adler32();
crc.update(bytes, 0, bytes.length);
if (crcValue != crc.getValue()) {
throw new IOException("CRC doesn't match " + crcValue +
" vs " + crc.getValue());
}
TxnHeader hdr = new TxnHeader();
Record txn = SerializeUtils.deserializeTxn(bytes, hdr);
if (hdr.getType() == -11){
System.out.println();
System.out.println("count:"+count+"byte:"+bytes.length);
System.out.println("原始数据:"+new String(bytes));
System.out.println(DateFormat.getDateTimeInstance(DateFormat.SHORT,
DateFormat.LONG).format(new Date(hdr.getTime()))
+ " session 0x"
+ Long.toHexString(hdr.getClientId())
+ " cxid 0x"
+ Long.toHexString(hdr.getCxid())
+ " zxid 0x"
+ Long.toHexString(hdr.getZxid())
+ " type:" + hdr.getType() + " " + JSON.toJSONString(txn));
}
if (logStream.readByte("EOR") != 'B') {
// LOG.error("Last transaction was partial.");
throw new EOFException("Last transaction was partial.");
}
count++;
}
}
测试该错误的代码
public static String getPath(String path){
Random random = new Random();
return path+random.nextLong();
}
public static CuratorFramework testLocalZk() throws Exception {
CuratorFramework curatorFramework = getInstance("127.0.0.1:2181");
// createNode(curatorFramework,CreateMode.PERSISTENT,"/a","a");
String path = "/dubbo/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService/consumers/consumer192.168.255.10/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService?application=sgs-kafka-gw&category=consumers&check=false&dubbo=sf.2.0.4&group=ENV3_2&interface=com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService&methods=deliveryQuickRepairDataProcess,fvpUnpackMatchRespMsgProcess,deliverySdsDynamicReimburseProcess,deliveryTaskMsgProcess,deliveryOrderMsgProcess,wantedDeliveryTask,deliveryOrderScstmsMsgProcess,saveWaybillRemark2Sisp,imageGatewayUploadInfoMsgProcess,deliveryMatchEmployeeNotify,barCountDownProcess,deliveryCXSignConfirm,deliveryExtraTaskStatusProcess,storeOneKeyAcceptProcess,queryWaybillInfoMsgProcess,deliveryDetainPredictionProcess,deliveryChangeOmsMsgProcess,deliveryCusChgResProcess,paySuccessCallback,batchDeliveryExpressDeliveredNotify,deliveryOrderOmsOperationMsgProcess,deliverySpecialConfigMsgProcess,processPisLimitTimeMsg,deliveryDestDeptUnloadMsgProcess,deliveryAppointRespProcess,deliveryOrderOmsMsgProcess,processBigDataWaybillLabel,deliveryDdsHandoverNotify,deliveryAppointSelfPickupProcess,deliverySdsSortDynamicTypeProcess,hhtDeliveryBarRecordMsgProcess,deliveryExtraTaskInfoProcess,deliveryOrderEosfmstmsMsgProcess,expressStatusMsgProcess,handoverAsyncOperation,deliveryWiInstructionsMsgProcess,uploadImageInfoToWbepAsyncMsgProcess,deliveryChangeMsgProcess,hwDeliveryTimeMsgProcess,deliveryPackageHandoverAsyncMsgProcess,deliveryOrderUniPayStatusMsgProcess,deliveryOrderPayStatusMsgProcess,deliveryDelayWaybillDetailMsgProcess,deliveryPackage,transferSorterProcess,sfNewCardPayStatusMsgProcess,deliveryInsCallOutInfoMsgProcess,deliveryUploadImageAsyncMsgProcess,waybillExceptionMsgProcess&owner=morly&pid=11844&revision=19.2-20210407.102748-18&side=consumer&timeout=10000×tamp=1617898909739";
// String getPath(path) = "/dubbo/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService/consumers/consumer192.168.255.10/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService?application=sgs-kafka-gw&category=consumers&check=false&dubbo=sf.2.0.4&group=ENV3_2&interface=com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService&methods=deliveryQuickRepairDataProcess,fvpUnpackMatchRespMsgProcess,deliverySdsDynamicReimburseProcess,deliveryTaskMsgProcess,deliveryOrderMsgProcess,wantedDeliveryTask,deliveryOrderScstmsMsgProcess,saveWaybillRemark2Sisp,imageGatewayUploadInfoMsgProcess,deliveryMatchEmployeeNotify,barCountDownProcess,deliveryCXSignConfirm,deliveryExtraTaskStatusProcess,storeOneKeyAcceptProcess,queryWaybillInfoMsgProcess,deliveryDetainPredictionProcess,deliveryChangeOmsMsgProcess,deliveryCusChgResProcess,paySuccessCallback,batchDeliveryExpressDeliveredNotify,deliveryOrderOmsOperationMsgProcess,deliverySpecialConfigMsgProcess,processPisLimitTimeMsg,deliveryDestDeptUnloadMsgProcess,deliveryAppointRespProcess,deliveryOrderOmsMsgProcess,processBigDataWaybillLabel,deliveryDdsHandoverNotify,deliveryAppointSelfPickupProcess,deliverySdsSortDynamicTypeProcess,hhtDeliveryBarRecordMsgProcess,deliveryExtraTaskInfoProcess,deliveryOrderEosfmstmsMsgProcess,expressStatusMsgProcess,handoverAsyncOperation,deliveryWiInstructionsMsgProcess,uploadImageInfoToWbepAsyncMsgProcess,deliveryChangeMsgProcess,hwDeliveryTimeMsgProcess,deliveryPackageHandoverAsyncMsgProcess,deliveryOrderUniPayStatusMsgProcess,deliveryOrderPayStatusMsgProcess,deliveryDelayWaybillDetailMsgProcess,deliveryPackage,transferSorterProcess,sfNewCardPayStatusMsgProcess,deliveryInsCallOutInfoMsgProcess,deliveryUploadImageAsyncMsgProcess,waybillExceptionMsgProcess&owner=morly&pid=11844&revision=19.2-20210407.102748-18&side=consumer&timeout=10000×tamp=1617900833708";
// createNode(curatorFramework,CreateMode.PERSISTENT,"/dubbo/com.sf.sgs.delivery.client.rpc.IDeliveryKafkaGatewayService/consumers/consumer127.0.0.1");
for (int i=0;i<1000;i++){
createNode(curatorFramework,CreateMode.EPHEMERAL,getPath(path)+"-"+i,path);
}
return curatorFramework;
}
更多推荐
已为社区贡献1条内容
所有评论(0)