mongo副本集 叢集中只存在一個SECONDARY節點處理
由於各種奇葩的原因 導致一個叢集中只剩下一個SECONDARY節點，此時該如何快速處理 讓DB能儘快對外提供服務？
如下是一次採用強制修改repletset vote屬性 提升SECONDARY為PRIMARY的過程
###############
目前架構 1主2從
###############
repset:SECONDARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:36:09.833Z"),
"myState" : 2,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:36:09.337Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:09.378Z"),
"pingMs" : 0,
"electionTime" : Timestamp(1499499337, 1),
"electionDate" : ISODate("2017-07-08T07:35:37Z"),
"configVersion" : 1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:36:09.337Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:09.587Z"),
"pingMs" : 0,
"configVersion" : 1
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"configVersion" : 1,
"self" : true
}
],
"ok" : 1
}
repset:SECONDARY>
mongodb 9895 1 0 12:59 ? 00:01:01 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10002/data --logpath=/comm/mg10002/log/mg10002.log --port 10002 --replSet repset
mongodb 10842 1 0 14:35 ? 00:00:10 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10000/data --logpath=/comm/mg10000/log/mg10000.log --port 10000 --replSet repset
mongodb 10897 1 0 14:35 ? 00:00:10 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10001/data --logpath=/comm/mg10001/log/mg10001.log --port 10001 --replSet repset
####################################
模擬故障現象 先停掉一個從mg10002，再停掉主mg10000
####################################
repset:SECONDARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:38:02.359Z"),
"myState" : 2,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:38:01.690Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:37:03.428Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "Failed attempt to connect to 192.168.16.130:10000; couldn't connect to server 192.168.16.130:10000 (192.168.16.130), connection attempt failed",
"configVersion" : -1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 147,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"configVersion" : 1,
"self" : true
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:38:01.690Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:49.362Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "Failed attempt to connect to 192.168.16.130:10002; couldn't connect to server 192.168.16.130:10002 (192.168.16.130), connection attempt failed",
"configVersion" : -1
}
],
"ok" : 1
}
repset:SECONDARY>
########################
現在只剩下一個為secondary的10001
########################
--修改故障節點10000 10001的vote屬性為0 不參與選舉 保留存活的secondary節點vote屬性
################################
use local
db.system.replset.find()
db.system.replset.remove({"_id":"repset"});
db.system.replset.insert({ "_id" : "repset", "version" : 1, "members" : [
{ "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 },
{ "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 },
{ "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 } ],
"settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } });
###############################
repset:SECONDARY> use localuse local
switched to db local
repset:SECONDARY> db.system.replset.find()db.system.replset.find()
{ "_id" : "repset", "version" : 1, "members" : [ { "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } }
repset:SECONDARY>
repset:SECONDARY> db.system.replset.remove({"_id":"repset"});db.system.replset.remove({"_id":"repset"});
WriteResult({ "nRemoved" : 1 })
repset:SECONDARY>
.......
m.replset.insert({ "_id" : "repset", "version" :db.system.replset.insert({ "_id" : "repset", "version" : db.system.replset.insert({ "_id" : "repset", "version" : 1db.system.replset.insert({ "_id" : "repset", "version" : 1, "members" : [ { "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 }, { "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } });
WriteResult({ "nInserted" : 1 })
repset:SECONDARY> exitexit
bye
######################
重啟存活的節點 已經轉換為primary
######################
[mongodb@node1 ~]$ mongod --port ${PORT} --shutdown --dbpath=/comm/mg${PORT}/data
killing process with pid: 12349
[mongodb@node1 ~]$ $MONGODB_HOME/bin/mongod --fork --dbpath /comm/mg${PORT}/data --logpath=/comm/mg${PORT}/log/mg${PORT}.log --port ${PORT} --replSet repset
about to fork child process, waiting until server is ready for connections.
forked process: 12544
child process started successfully, parent exiting
[mongodb@node1 ~]$ $MONGODB_HOME/bin/mongo --port ${PORT}
MongoDB shell version: 3.0.15-9-g0513806
connecting to: 127.0.0.1:10001/test
Server has startup warnings:
repset:PRIMARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:41:50.053Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:41:49.137Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"configVersion" : -1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 14,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"electionTime" : Timestamp(1499499699, 1),
"electionDate" : ISODate("2017-07-08T07:41:39Z"),
"configVersion" : 1,
"self" : true
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:41:49.137Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"configVersion" : -1
}
],
"ok" : 1
}
repset:PRIMARY>
#############
後續將有問題的節點重新搭建即可
#############
如下是一次採用強制修改repletset vote屬性 提升SECONDARY為PRIMARY的過程
###############
目前架構 1主2從
###############
repset:SECONDARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:36:09.833Z"),
"myState" : 2,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:36:09.337Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:09.378Z"),
"pingMs" : 0,
"electionTime" : Timestamp(1499499337, 1),
"electionDate" : ISODate("2017-07-08T07:35:37Z"),
"configVersion" : 1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:36:09.337Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:09.587Z"),
"pingMs" : 0,
"configVersion" : 1
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"configVersion" : 1,
"self" : true
}
],
"ok" : 1
}
repset:SECONDARY>
mongodb 9895 1 0 12:59 ? 00:01:01 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10002/data --logpath=/comm/mg10002/log/mg10002.log --port 10002 --replSet repset
mongodb 10842 1 0 14:35 ? 00:00:10 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10000/data --logpath=/comm/mg10000/log/mg10000.log --port 10000 --replSet repset
mongodb 10897 1 0 14:35 ? 00:00:10 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10001/data --logpath=/comm/mg10001/log/mg10001.log --port 10001 --replSet repset
####################################
模擬故障現象 先停掉一個從mg10002，再停掉主mg10000
####################################
repset:SECONDARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:38:02.359Z"),
"myState" : 2,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:38:01.690Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:37:03.428Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "Failed attempt to connect to 192.168.16.130:10000; couldn't connect to server 192.168.16.130:10000 (192.168.16.130), connection attempt failed",
"configVersion" : -1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 147,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"configVersion" : 1,
"self" : true
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:38:01.690Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:49.362Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "Failed attempt to connect to 192.168.16.130:10002; couldn't connect to server 192.168.16.130:10002 (192.168.16.130), connection attempt failed",
"configVersion" : -1
}
],
"ok" : 1
}
repset:SECONDARY>
########################
現在只剩下一個為secondary的10001
########################
--修改故障節點10000 10001的vote屬性為0 不參與選舉 保留存活的secondary節點vote屬性
################################
use local
db.system.replset.find()
db.system.replset.remove({"_id":"repset"});
db.system.replset.insert({ "_id" : "repset", "version" : 1, "members" : [
{ "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 },
{ "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 },
{ "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 } ],
"settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } });
###############################
repset:SECONDARY> use localuse local
switched to db local
repset:SECONDARY> db.system.replset.find()db.system.replset.find()
{ "_id" : "repset", "version" : 1, "members" : [ { "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } }
repset:SECONDARY>
repset:SECONDARY> db.system.replset.remove({"_id":"repset"});db.system.replset.remove({"_id":"repset"});
WriteResult({ "nRemoved" : 1 })
repset:SECONDARY>
.......
m.replset.insert({ "_id" : "repset", "version" :db.system.replset.insert({ "_id" : "repset", "version" : db.system.replset.insert({ "_id" : "repset", "version" : 1db.system.replset.insert({ "_id" : "repset", "version" : 1, "members" : [ { "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 }, { "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } });
WriteResult({ "nInserted" : 1 })
repset:SECONDARY> exitexit
bye
######################
重啟存活的節點 已經轉換為primary
######################
[mongodb@node1 ~]$ mongod --port ${PORT} --shutdown --dbpath=/comm/mg${PORT}/data
killing process with pid: 12349
[mongodb@node1 ~]$ $MONGODB_HOME/bin/mongod --fork --dbpath /comm/mg${PORT}/data --logpath=/comm/mg${PORT}/log/mg${PORT}.log --port ${PORT} --replSet repset
about to fork child process, waiting until server is ready for connections.
forked process: 12544
child process started successfully, parent exiting
[mongodb@node1 ~]$ $MONGODB_HOME/bin/mongo --port ${PORT}
MongoDB shell version: 3.0.15-9-g0513806
connecting to: 127.0.0.1:10001/test
Server has startup warnings:
repset:PRIMARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:41:50.053Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:41:49.137Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"configVersion" : -1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 14,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"electionTime" : Timestamp(1499499699, 1),
"electionDate" : ISODate("2017-07-08T07:41:39Z"),
"configVersion" : 1,
"self" : true
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:41:49.137Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"configVersion" : -1
}
],
"ok" : 1
}
repset:PRIMARY>
#############
後續將有問題的節點重新搭建即可
#############
來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/26526320/viewspace-2141768/，如需轉載，請註明出處，否則將追究法律責任。
相關文章
- ElasticSearch之叢集中的節點Elasticsearch
- 【Mongodb】mongo複製集只剩一個secondery節點的解決辦法MongoDB
- 處理尚不存在的 DOM 節點
- mongo副本集搭建Go
- 如何處理redis叢集中hot key和big keyRedis
- 關於叢集節點timeline不一致的處理方式
- 採用secondary的冷備份的方式來新增新的secondary節點
- MongoDB 4.2副本集新增/刪除副本（一主一副一仲裁）MongoDB
- mongodb副本集新增刪除節點MongoDB
- MongoDB 4.2副本集自動故障轉移（一主一副一仲裁）MongoDB
- mongodb 4.0副本集搭建MongoDB
- MongoDB 6.0.3副本集搭建MongoDB
- mongo資料庫單節點搭建Go資料庫
- 一個篩選mongo存在某個欄位的資料的技巧Go
- consul 多節點/單節點叢集搭建
- 單機Linux下搭建MongoDB副本集-三節點LinuxMongoDB
- java處理流 和節點流（在位元組流和字元流中，又分為處理流和節點流）Java字元
- 數字影象處理-第一節
- 將 master 節點伺服器從 k8s 叢集中移除並重新加入AST伺服器K8S
- 關於Oracle 11G RAC雙節點之間存在防火牆導致只能一個節點執行Oracle防火牆
- MongoDB日常運維-04副本集搭建MongoDB運維
- 汙水處理自動化控制與汙水處理廠集中控制
- ent M2M模型在pxc叢集中的一個大坑模型
- Oracle RAC命中ORA-7445只能開啟一個節點故障案例分析Oracle
- MongoDB叢集搭建(包括隱藏節點，仲裁節點)MongoDB
- 三、實戰系列問題處理集中營
- js判斷dom節點是否存在JS
- 2.MongoDB 4.2副本集環境基於時間點的恢復MongoDB
- Jedis操作單節點redis，叢集及redisTemplate操作redis叢集（一）Redis
- 填充每個節點的下一個右側節點指標指標
- 銳龍處理器和酷睿處理器哪個好 電腦處理器銳龍和酷睿哪個好一點
- ClusterShell：一個在叢集節點上並行執行命令的好工具並行
- 4.2 叢集節點初步搭建
- Solaris叢集節點重啟
- HAC叢集新增新節點
- HAC叢集更改IP（單節點更改、全部節點更改）
- 副本集要點
- mongodb副本集用一致性快照方法新增從節點步驟MongoDB
- 填充每個節點的下一個右側節點指標 II指標