1. 叢集環境
ambari-version：2.7.5
HDP-version：3.0
2.問題描述
hadoop-yarn的啟動之後，執行一段時間，莫名其妙的出現新的任務無法提交上去，檢視yarn的狀態之後，發現yarn的狀態都是正常的，並且所有的資源都是充足的，但是提交任務之後就會一直處於accept狀態
3.問題表現
4.問題排查
4.1 首先檢視了日誌，發現了有報錯資訊：
2023-02-02 15:56:57,934 INFO capacity.LeafQueue (LeafQueue.java:activateApplications(911)) - Application application_1673397736150_0694 from user: hdfs activated in queue: default
2023-02-02 15:56:57,937 INFO capacity.LeafQueue (LeafQueue.java:addApplicationAttempt(941)) - Application added - appId: application_1673397736150_0694 user: hdfs, leaf-queue: default #user-pending-applications: 0 #user-active-applications: 1 #queue-pending-applications: 0 #queue-active-applications: 1
2023-02-02 15:56:57,938 INFO capacity.CapacityScheduler (CapacityScheduler.java:addApplicationAttempt(1036)) - Added Application Attempt appattempt_1673397736150_0694_000001 to scheduler from user hdfs in queue default
2023-02-02 15:56:57,938 INFO attempt.RMAppAttemptImpl (RMAppAttemptImpl.java:handle(925)) - appattempt_1673397736150_0694_000001 State change from SUBMITTED to SCHEDULED on event = ATTEMPT_ADDED
2023-02-02 15:57:28,011 ERROR webapp.Dispatcher (Dispatcher.java:service(171)) - error handling URI: /cluster
org.eclipse.jetty.io.RuntimeIOException: org.eclipse.jetty.io.EofException
at org.eclipse.jetty.server.ResponseWriter.isOpen(ResponseWriter.java:133)
at org.eclipse.jetty.server.ResponseWriter.println(ResponseWriter.java:314)
at org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl$EImp._p(HamletImpl.java:110)
at org.apache.hadoop.yarn.webapp.hamlet2.Hamlet$SCRIPT.__(Hamlet.java:457)
at org.apache.hadoop.yarn.server.resourcemanager.webapp.RMAppsBlock.renderData(RMAppsBlock.java:199)
at org.apache.hadoop.yarn.server.webapp.AppsBlock.render(AppsBlock.java:145)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock.render(HtmlBlock.java:69)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock.renderPartial(HtmlBlock.java:79)
at org.apache.hadoop.yarn.webapp.View.render(View.java:243)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock$Block.subView(HtmlBlock.java:43)
at org.apache.hadoop.yarn.webapp.hamlet2.Hamlet.__(Hamlet.java:30354)
at org.apache.hadoop.yarn.server.resourcemanager.webapp.AppsBlockWithMetrics.render(AppsBlockWithMetrics.java:29)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock.render(HtmlBlock.java:69)
at org.apache.hadoop.yarn.webapp.view.HtmlBlock.renderPartial(HtmlBlock.java:79)
at org.apache.hadoop.yarn.webapp.View.render(View.java:243)
at org.apache.hadoop.yarn.webapp.view.HtmlPage$Page.subView(HtmlPage.java:49)
at org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl$EImp._v(HamletImpl.java:117)
at org.apache.hadoop.yarn.webapp.hamlet2.Hamlet$TD.__(Hamlet.java:848)
at org.apache.hadoop.yarn.webapp.view.TwoColumnLayout.render(TwoColumnLayout.java:71)
at org.apache.hadoop.yarn.webapp.view.HtmlPage.render(HtmlPage.java:82)
at org.apache.hadoop.yarn.webapp.Dispatcher.render(Dispatcher.java:206)
at org.apache.hadoop.yarn.webapp.Dispatcher.service(Dispatcher.java:165)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at com.google.inject.servlet.ServletDefinition.doServiceImpl(ServletDefinition.java:287)
at com.google.inject.servlet.ServletDefinition.doService(ServletDefinition.java:277)
at com.google.inject.servlet.ServletDefinition.service(ServletDefinition.java:182)
at com.google.inject.servlet.ManagedServletPipeline.service(ManagedServletPipeline.java:91)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:85)
at com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:941)
at com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:875)
at org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppFilter.doFilter(RMWebAppFilter.java:178)
at com.sun.jersey.spi.container.servlet.ServletContainer.doFilter(ServletContainer.java:829)
at com.google.inject.servlet.FilterChainInvocation.doFilter(FilterChainInvocation.java:82)
at com.google.inject.servlet.ManagedFilterPipeline.dispatch(ManagedFilterPipeline.java:119)
at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:133)
at com.google.inject.servlet.GuiceFilter$1.call(GuiceFilter.java:130)
at com.google.inject.servlet.GuiceFilter$Context.call(GuiceFilter.java:203)
at com.google.inject.servlet.GuiceFilter.doFilter(GuiceFilter.java:130)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.security.http.XFrameOptionsFilter.doFilter(XFrameOptionsFilter.java:57)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:644)
at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:592)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.security.http.CrossOriginFilter.doFilter(CrossOriginFilter.java:98)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.http.HttpServer2$QuotingInputFilter.doFilter(HttpServer2.java:1619)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.apache.hadoop.http.NoCacheFilter.doFilter(NoCacheFilter.java:45)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:548)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:226)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:119)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
at org.eclipse.jetty.server.Server.handle(Server.java:539)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:333)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.eclipse.jetty.io.EofException
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:199)
at org.eclipse.jetty.io.WriteFlusher.flush(WriteFlusher.java:420)
at org.eclipse.jetty.io.WriteFlusher.write(WriteFlusher.java:313)
at org.eclipse.jetty.io.AbstractEndPoint.write(AbstractEndPoint.java:147)
at org.eclipse.jetty.server.HttpConnection$SendCallback.process(HttpConnection.java:745)
at org.eclipse.jetty.util.IteratingCallback.processing(IteratingCallback.java:241)
at org.eclipse.jetty.util.IteratingCallback.iterate(IteratingCallback.java:224)
at org.eclipse.jetty.server.HttpConnection.send(HttpConnection.java:519)
at org.eclipse.jetty.server.HttpChannel.sendResponse(HttpChannel.java:753)
at org.eclipse.jetty.server.HttpChannel.write(HttpChannel.java:804)
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:235)
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:219)
at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:470)
at java.io.ByteArrayOutputStream.writeTo(ByteArrayOutputStream.java:167)
at org.eclipse.jetty.server.Utf8HttpWriter.write(Utf8HttpWriter.java:183)
at org.eclipse.jetty.server.HttpWriter.write(HttpWriter.java:71)
at org.eclipse.jetty.server.HttpWriter.write(HttpWriter.java:65)
at org.eclipse.jetty.server.ResponseWriter.write(ResponseWriter.java:231)
at org.eclipse.jetty.server.ResponseWriter.write(ResponseWriter.java:248)
at org.eclipse.jetty.server.ResponseWriter.print(ResponseWriter.java:298)
at org.apache.hadoop.yarn.webapp.hamlet2.HamletImpl$EImp._p(HamletImpl.java:107)
... 70 more
Caused by: java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:51)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.eclipse.jetty.io.ChannelEndPoint.flush(ChannelEndPoint.java:177)
... 90 more
2023-02-02 15:57:36,903 INFO zookeeper.ReadOnlyZKClient (ReadOnlyZKClient.java:run(315)) - 0x0769d513 no activities for 60000 ms, close active connection. Will reconnect next time when there are new requests
從上面的報錯日誌中可以看出，報錯資訊跟yarn貌似沒有半毛錢的關係，所以即系排查，由於是因為任務提交不上去，也就是說任務無法正常申請到資源，所以故而聯想到可能會是resourceManager的記憶體是不是有什麼問題，所以獲取了resourceManager的日誌以及jvm的對記憶體資訊
4.2 resourceManager的日誌排查以及堆記憶體資訊的排查
4.2.1 檢視日誌
從日誌中並未查出有用的資訊，然後接著檢視了jvm堆資訊
4.2.2 檢視程序堆資訊
[yarn@node3 ~]$ jmap -heap 29691
Attaching to process ID 29691, please wait...
Debugger attached successfully.
Server compiler detected.
JVM version is 25.161-b12
using thread-local object allocation.
Parallel GC with 8 thread(s)
Heap Configuration:
MinHeapFreeRatio = 0
MaxHeapFreeRatio = 100
MaxHeapSize = 2147483648 (2048.0MB)
NewSize = 175112192 (167.0MB)
MaxNewSize = 715653120 (682.5MB)
OldSize = 351272960 (335.0MB)
NewRatio = 2
SurvivorRatio = 8
MetaspaceSize = 21807104 (20.796875MB)
CompressedClassSpaceSize = 1073741824 (1024.0MB)
MaxMetaspaceSize = 17592186044415 MB
G1HeapRegionSize = 0 (0.0MB)
Heap Usage:
PS Young Generation
Eden Space:
capacity = 49807360 (47.5MB)
used = 23949536 (22.840057373046875MB)
free = 25857824 (24.659942626953125MB)
48.08433131167763% used
From Space:
capacity = 524288 (0.5MB)
used = 425984 (0.40625MB)
free = 98304 (0.09375MB)
81.25% used
To Space:
capacity = 524288 (0.5MB)
used = 0 (0.0MB)
free = 524288 (0.5MB)
0.0% used
PS Old Generation
capacity = 284164096 (271.0MB)
used = 173603056 (165.56077575683594MB)
free = 110561040 (105.43922424316406MB)
61.09253717964426% used
27626 interned Strings occupying 2802968 bytes.
透過jvm的堆資訊檢視，記憶體使用情況也是正常的，這就奇怪了，於是接著往下排查。
4.3 引發的思考
既然堆記憶體的使用情況也都正常，那會是什麼原因呢，難道是yarn的資源排程的問題，導致無法申請資源？帶著這個問題，找了一下yarn的原始碼中關於資源計算這方面的資訊，果然是大有所收穫。
5.原始碼中尋找問題
5.1 追蹤原始碼
找到原始碼中對應的類：【ResourceCalculator.class】，在這個class有兩個實現,分別是：
- DefaultResourceCalculator.class
- DominantResourceCalculator.class
來分別看一下兩個類中對於資源計算的不同方式：
1. DefaultResourceCalculator.class
...
@Private
@Unstable
public class DefaultResourceCalculator extends ResourceCalculator {
private static final Log LOG =
LogFactory.getLog(DefaultResourceCalculator.class);
@Override
public int compare(Resource unused, Resource lhs, Resource rhs,
boolean singleType) {
// Only consider memory 這裡只考慮記憶體
return Long.compare(lhs.getMemorySize(), rhs.getMemorySize());
}
@Override
public long computeAvailableContainers(Resource available, Resource required) {
// Only consider memory
return available.getMemorySize() / required.getMemorySize();
}
...
@Override
public long computeAvailableContainers(Resource available, Resource required) {
// Only consider memory
return available.getMemorySize() / required.getMemorySize();
}
...
2. DominantResourceCalculator.class
...
/** 這裡會比較兩種資源，記憶體和vcore
* Compare two resources - if the value for every resource type for the lhs
* is greater than that of the rhs, return 1. If the value for every resource
* type in the lhs is less than the rhs, return -1. Otherwise, return 0
*
* @param lhs resource to be compared
* @param rhs resource to be compared
* @return 0, 1, or -1
*/
private int compare(Resource lhs, Resource rhs) {
boolean lhsGreater = false;
boolean rhsGreater = false;
int ret = 0;
int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
ResourceInformation lhsResourceInformation = lhs
.getResourceInformation(i);
ResourceInformation rhsResourceInformation = rhs
.getResourceInformation(i);
int diff = lhsResourceInformation.compareTo(rhsResourceInformation);
if (diff >= 1) {
lhsGreater = true;
} else if (diff <= -1) {
rhsGreater = true;
}
}
if (lhsGreater && rhsGreater) {
ret = 0;
} else if (lhsGreater) {
ret = 1;
} else if (rhsGreater) {
ret = -1;
}
return ret;
}
...
@Override
public long computeAvailableContainers(Resource available,
Resource required) {
long min = Long.MAX_VALUE;
int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
ResourceInformation availableResource = available
.getResourceInformation(i);
ResourceInformation requiredResource = required.getResourceInformation(i);
if (requiredResource.getValue() != 0) {
long tmp = availableResource.getValue() / requiredResource.getValue();
min = min < tmp ? min : tmp;
}
}
return min > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) min;
}
...
從上面的程式碼中可以看出，對於預設的資源計算方式，只計算記憶體的資源，不會去考慮cpu的數量，但是對於第二種資源計算方式來說的話，它的記憶體計算是結合了記憶體以及cpu一起來計算的。
到此，我們找到了問題的原因,因為在ambari安裝之後，預設的情況下是使用的第一種資源排程方式，這個時候需要將該資源排程方式進行修改：
修改前：
修改完成之後，在此重啟yarn，問題得以解決
6. 思考
日常叢集的維護的過程中，真的是會出現各種各樣的意想不到的問題，當我們遇到問題的時候，
- 一定要冷靜的先思考，找準方向，然後在朝著我們的方向順藤摸瓜的去找問題的答案。
- 原始碼的重要性。我們在尋找很多方案之後都探尋無果的情況下，這個時候我們不妨可以從原始碼中著手，問題一定在其中。
- 對於問題我們需要報有一種追根問題的心態，就是一定要包某個問題吃透，瞭解這個問題為什麼會產生，原因是什麼，然後才能從根本上解決問題