mongo副本集 叢集中只存在一個SECONDARY節點處理
由於各種奇葩的原因 導致一個叢集中只剩下一個SECONDARY節點，此時該如何快速處理 讓DB能儘快對外提供服務？
如下是一次採用強制修改repletset vote屬性 提升SECONDARY為PRIMARY的過程
###############
目前架構 1主2從
###############
repset:SECONDARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:36:09.833Z"),
"myState" : 2,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:36:09.337Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:09.378Z"),
"pingMs" : 0,
"electionTime" : Timestamp(1499499337, 1),
"electionDate" : ISODate("2017-07-08T07:35:37Z"),
"configVersion" : 1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:36:09.337Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:09.587Z"),
"pingMs" : 0,
"configVersion" : 1
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"configVersion" : 1,
"self" : true
}
],
"ok" : 1
}
repset:SECONDARY>
mongodb 9895 1 0 12:59 ? 00:01:01 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10002/data --logpath=/comm/mg10002/log/mg10002.log --port 10002 --replSet repset
mongodb 10842 1 0 14:35 ? 00:00:10 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10000/data --logpath=/comm/mg10000/log/mg10000.log --port 10000 --replSet repset
mongodb 10897 1 0 14:35 ? 00:00:10 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10001/data --logpath=/comm/mg10001/log/mg10001.log --port 10001 --replSet repset
####################################
模擬故障現象 先停掉一個從mg10002，再停掉主mg10000
####################################
repset:SECONDARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:38:02.359Z"),
"myState" : 2,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:38:01.690Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:37:03.428Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "Failed attempt to connect to 192.168.16.130:10000; couldn't connect to server 192.168.16.130:10000 (192.168.16.130), connection attempt failed",
"configVersion" : -1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 147,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"configVersion" : 1,
"self" : true
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:38:01.690Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:49.362Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "Failed attempt to connect to 192.168.16.130:10002; couldn't connect to server 192.168.16.130:10002 (192.168.16.130), connection attempt failed",
"configVersion" : -1
}
],
"ok" : 1
}
repset:SECONDARY>
########################
現在只剩下一個為secondary的10001
########################
--修改故障節點10000 10001的vote屬性為0 不參與選舉 保留存活的secondary節點vote屬性
################################
use local
db.system.replset.find()
db.system.replset.remove({"_id":"repset"});
db.system.replset.insert({ "_id" : "repset", "version" : 1, "members" : [
{ "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 },
{ "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 },
{ "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 } ],
"settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } });
###############################
repset:SECONDARY> use localuse local
switched to db local
repset:SECONDARY> db.system.replset.find()db.system.replset.find()
{ "_id" : "repset", "version" : 1, "members" : [ { "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } }
repset:SECONDARY>
repset:SECONDARY> db.system.replset.remove({"_id":"repset"});db.system.replset.remove({"_id":"repset"});
WriteResult({ "nRemoved" : 1 })
repset:SECONDARY>
.......
m.replset.insert({ "_id" : "repset", "version" :db.system.replset.insert({ "_id" : "repset", "version" : db.system.replset.insert({ "_id" : "repset", "version" : 1db.system.replset.insert({ "_id" : "repset", "version" : 1, "members" : [ { "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 }, { "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } });
WriteResult({ "nInserted" : 1 })
repset:SECONDARY> exitexit
bye
######################
重啟存活的節點 已經轉換為primary
######################
[mongodb@node1 ~]$ mongod --port ${PORT} --shutdown --dbpath=/comm/mg${PORT}/data
killing process with pid: 12349
[mongodb@node1 ~]$ $MONGODB_HOME/bin/mongod --fork --dbpath /comm/mg${PORT}/data --logpath=/comm/mg${PORT}/log/mg${PORT}.log --port ${PORT} --replSet repset
about to fork child process, waiting until server is ready for connections.
forked process: 12544
child process started successfully, parent exiting
[mongodb@node1 ~]$ $MONGODB_HOME/bin/mongo --port ${PORT}
MongoDB shell version: 3.0.15-9-g0513806
connecting to: 127.0.0.1:10001/test
Server has startup warnings:
repset:PRIMARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:41:50.053Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:41:49.137Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"configVersion" : -1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 14,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"electionTime" : Timestamp(1499499699, 1),
"electionDate" : ISODate("2017-07-08T07:41:39Z"),
"configVersion" : 1,
"self" : true
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:41:49.137Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"configVersion" : -1
}
],
"ok" : 1
}
repset:PRIMARY>
#############
後續將有問題的節點重新搭建即可
#############
如下是一次採用強制修改repletset vote屬性 提升SECONDARY為PRIMARY的過程
###############
目前架構 1主2從
###############
repset:SECONDARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:36:09.833Z"),
"myState" : 2,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:36:09.337Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:09.378Z"),
"pingMs" : 0,
"electionTime" : Timestamp(1499499337, 1),
"electionDate" : ISODate("2017-07-08T07:35:37Z"),
"configVersion" : 1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:36:09.337Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:09.587Z"),
"pingMs" : 0,
"configVersion" : 1
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 22,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"configVersion" : 1,
"self" : true
}
],
"ok" : 1
}
repset:SECONDARY>
mongodb 9895 1 0 12:59 ? 00:01:01 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10002/data --logpath=/comm/mg10002/log/mg10002.log --port 10002 --replSet repset
mongodb 10842 1 0 14:35 ? 00:00:10 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10000/data --logpath=/comm/mg10000/log/mg10000.log --port 10000 --replSet repset
mongodb 10897 1 0 14:35 ? 00:00:10 /comm/mongodb/3.0/bin/mongod --fork --dbpath /comm/mg10001/data --logpath=/comm/mg10001/log/mg10001.log --port 10001 --replSet repset
####################################
模擬故障現象 先停掉一個從mg10002，再停掉主mg10000
####################################
repset:SECONDARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:38:02.359Z"),
"myState" : 2,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:38:01.690Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:37:03.428Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "Failed attempt to connect to 192.168.16.130:10000; couldn't connect to server 192.168.16.130:10000 (192.168.16.130), connection attempt failed",
"configVersion" : -1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 147,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"configVersion" : 1,
"self" : true
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:38:01.690Z"),
"lastHeartbeatRecv" : ISODate("2017-07-08T07:36:49.362Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "Failed attempt to connect to 192.168.16.130:10002; couldn't connect to server 192.168.16.130:10002 (192.168.16.130), connection attempt failed",
"configVersion" : -1
}
],
"ok" : 1
}
repset:SECONDARY>
########################
現在只剩下一個為secondary的10001
########################
--修改故障節點10000 10001的vote屬性為0 不參與選舉 保留存活的secondary節點vote屬性
################################
use local
db.system.replset.find()
db.system.replset.remove({"_id":"repset"});
db.system.replset.insert({ "_id" : "repset", "version" : 1, "members" : [
{ "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 },
{ "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 },
{ "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 } ],
"settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } });
###############################
repset:SECONDARY> use localuse local
switched to db local
repset:SECONDARY> db.system.replset.find()db.system.replset.find()
{ "_id" : "repset", "version" : 1, "members" : [ { "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } }
repset:SECONDARY>
repset:SECONDARY> db.system.replset.remove({"_id":"repset"});db.system.replset.remove({"_id":"repset"});
WriteResult({ "nRemoved" : 1 })
repset:SECONDARY>
.......
m.replset.insert({ "_id" : "repset", "version" :db.system.replset.insert({ "_id" : "repset", "version" : db.system.replset.insert({ "_id" : "repset", "version" : 1db.system.replset.insert({ "_id" : "repset", "version" : 1, "members" : [ { "_id" : 0, "host" : "192.168.16.130:10000", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 }, { "_id" : 1, "host" : "192.168.16.130:10001", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 1 }, { "_id" : 2, "host" : "192.168.16.130:10002", "arbiterOnly" : false, "buildIndexes" : true, "hidden" : false, "priority" : 1, "tags" : { }, "slaveDelay" : 0, "votes" : 0 } ], "settings" : { "chainingAllowed" : true, "heartbeatTimeoutSecs" : 10, "getLastErrorModes" : { }, "getLastErrorDefaults" : { "w" : 1, "wtimeout" : 0 } } });
WriteResult({ "nInserted" : 1 })
repset:SECONDARY> exitexit
bye
######################
重啟存活的節點 已經轉換為primary
######################
[mongodb@node1 ~]$ mongod --port ${PORT} --shutdown --dbpath=/comm/mg${PORT}/data
killing process with pid: 12349
[mongodb@node1 ~]$ $MONGODB_HOME/bin/mongod --fork --dbpath /comm/mg${PORT}/data --logpath=/comm/mg${PORT}/log/mg${PORT}.log --port ${PORT} --replSet repset
about to fork child process, waiting until server is ready for connections.
forked process: 12544
child process started successfully, parent exiting
[mongodb@node1 ~]$ $MONGODB_HOME/bin/mongo --port ${PORT}
MongoDB shell version: 3.0.15-9-g0513806
connecting to: 127.0.0.1:10001/test
Server has startup warnings:
repset:PRIMARY> rs.status()rs.status()
{
"set" : "repset",
"date" : ISODate("2017-07-08T07:41:50.053Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "192.168.16.130:10000",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:41:49.137Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"configVersion" : -1
},
{
"_id" : 1,
"name" : "192.168.16.130:10001",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 14,
"optime" : Timestamp(1499488823, 2),
"optimeDate" : ISODate("2017-07-08T04:40:23Z"),
"electionTime" : Timestamp(1499499699, 1),
"electionDate" : ISODate("2017-07-08T07:41:39Z"),
"configVersion" : 1,
"self" : true
},
{
"_id" : 2,
"name" : "192.168.16.130:10002",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(0, 0),
"optimeDate" : ISODate("1970-01-01T00:00:00Z"),
"lastHeartbeat" : ISODate("2017-07-08T07:41:49.137Z"),
"lastHeartbeatRecv" : ISODate("1970-01-01T00:00:00Z"),
"configVersion" : -1
}
],
"ok" : 1
}
repset:PRIMARY>
#############
後續將有問題的節點重新搭建即可
#############
來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/26526320/viewspace-2141768/，如需轉載，請註明出處，否則將追究法律責任。
相關文章
- mongo副本集叢集安裝配置Go
- 處理尚不存在的 DOM 節點
- 【Mongodb】mongo複製集只剩一個secondery節點的解決辦法MongoDB
- mongo副本集搭建Go
- 新增和刪除hadoop叢集中的節點Hadoop
- 如何處理redis叢集中hot key和big keyRedis
- 處理rac資料庫一個節點監聽異常資料庫
- oracle 11gR2 rac 兩節點有一個節點down掉問題處理Oracle
- MongoDB 4.2副本集新增/刪除副本（一主一副一仲裁）MongoDB
- 關於叢集節點timeline不一致的處理方式
- MongoDB 4.2副本集自動故障轉移（一主一副一仲裁）MongoDB
- mongodb 4.0副本集搭建MongoDB
- MongoDB 6.0.3副本集搭建MongoDB
- mongodb副本集新增刪除節點MongoDB
- oracle 11gR2 crs 其中一個節點grid叢集啟動不成功處理案例Oracle
- nginx叢集中圖片指定一個地址中Nginx
- mongo資料庫單節點搭建Go資料庫
- RAC第一個節點被剔除叢集故障分析
- 一個4節點Hadoop叢集的配置示例Hadoop
- 一個篩選mongo存在某個欄位的資料的技巧Go
- MongoDB副本集新增和刪除仲裁節點一例MongoDB
- MongoDB副本集節點的優先值修改一例MongoDB
- oracle rac 其中第一個節點監聽偶爾中斷處理案例Oracle
- 從greenplum叢集中移除節點，並用序列恢復的方法將刪除節點的資料重分佈到剩餘節點
- mongo 叢集配置Go
- 將 master 節點伺服器從 k8s 叢集中移除並重新加入AST伺服器K8S
- MongoDB日常運維-04副本集搭建MongoDB運維
- 索引節點(inode)爆滿問題處理索引
- 關於Oracle 11G RAC雙節點之間存在防火牆導致只能一個節點執行Oracle防火牆
- 叢集系統與事務處理需要注意的一點
- consul 多節點/單節點叢集搭建
- RAC修改叢集兩個節點public ip地址
- 數字影象處理-第一節
- 處理tns不存在
- Silverlight表格繫結中的一點細節處理
- ent M2M模型在pxc叢集中的一個大坑模型
- 單機Linux下搭建MongoDB副本集-三節點LinuxMongoDB
- Oracle RAC命中ORA-7445只能開啟一個節點故障案例分析Oracle