Flink -- Failover

 

JobManager failover

 

LeaderLatch

private synchronized void setLeadership(boolean newValue)
{
    boolean oldValue = hasLeadership.getAndSet(newValue);

    if ( oldValue && !newValue ) //原來是leader，當前不是leader，所以是lost leadership
    { // Lost leadership, was true, now false
        listeners.forEach(new Function<LeaderLatchListener, Void>()
            {
                @Override
                public Void apply(LeaderLatchListener listener)
                {
                    listener.notLeader();
                    return null;
                }
            });
    }
    else if ( !oldValue && newValue )
    { // Gained leadership, was false, now true
        listeners.forEach(new Function<LeaderLatchListener, Void>()
            {
                @Override
                public Void apply(LeaderLatchListener input)
                {
                    input.isLeader();
                    return null;
                }
            });
    }

    notifyAll();
}

ZooKeeperLeaderElectionService

@Override
public void isLeader() {
   synchronized (lock) {
      issuedLeaderSessionID = UUID.randomUUID();


      leaderContender.grantLeadership(issuedLeaderSessionID);
   }
}

@Override
public void notLeader() {
   synchronized (lock) {
      issuedLeaderSessionID = null;
      confirmedLeaderSessionID = null;



      leaderContender.revokeLeadership();
   }
}

可以看到，只是分別呼叫leaderContender.grantLeadership，leaderContender.revokeLeadership

 

而JobManager繼承了leaderContender介面，

revokeLeadership

val newFuturesToComplete = cancelAndClearEverything(
  new Exception("JobManager is no longer the leader."))

 

在cancelAndClearEverything中，關鍵的是suspend executionGraph；停止執行，但是並不會job刪除，這樣其他的JobManager還能重新提交

* The SUSPENDED state is a local terminal state which stops the execution of the job but does
* not remove the job from the HA job store so that it can be recovered by another JobManager.

private def cancelAndClearEverything(cause: Throwable)
  : Seq[Future[Unit]] = {
  val futures = for ((jobID, (eg, jobInfo)) <- currentJobs) yield {
    future {
      eg.suspend(cause) //suspend Execution Graph

      if (jobInfo.listeningBehaviour != ListeningBehaviour.DETACHED) {
        jobInfo.client ! decorateMessage(
          Failure(new JobExecutionException(jobID, "All jobs are cancelled and cleared.", cause)))
      }
    }(context.dispatcher)
  }

  currentJobs.clear()

  futures.toSeq
}

 

grantLeadership

context.system.scheduler.scheduleOnce(
  jobRecoveryTimeout,
  self,
  decorateMessage(RecoverAllJobs))(
  context.dispatcher)

主要是要恢復所有的job，RecoverAllJobs

case RecoverAllJobs =>
  future {
    try {
      // The ActorRef, which is part of the submitted job graph can only be
      // de-serialized in the scope of an actor system.
      akka.serialization.JavaSerializer.currentSystem.withValue(
        context.system.asInstanceOf[ExtendedActorSystem]) {

        log.info(s"Attempting to recover all jobs.")

        val jobGraphs = submittedJobGraphs.recoverJobGraphs().asScala //從submittedJobGraphs store裡面讀出所有submitted的job，也是從zk裡面讀出

        if (!leaderElectionService.hasLeadership()) {
          // we've lost leadership. mission: abort.
          log.warn(s"Lost leadership during recovery. Aborting recovery of ${jobGraphs.size} " +
            s"jobs.")
        } else {
          log.info(s"Re-submitting ${jobGraphs.size} job graphs.")

          jobGraphs.foreach{
            submittedJobGraph =>
              self ! decorateMessage(RecoverSubmittedJob(submittedJobGraph)) //recover job
          }
        }
      }
    } catch {
      case t: Throwable => log.error("Fatal error: Failed to recover jobs.", t)
    }
  }(context.dispatcher)

 

在recover job，

case RecoverSubmittedJob(submittedJobGraph) =>
  if (!currentJobs.contains(submittedJobGraph.getJobId)) {
    submitJob(
      submittedJobGraph.getJobGraph(),
      submittedJobGraph.getJobInfo(),
      isRecovery = true)
  }
  else {
    log.info(s"Ignoring job recovery for ${submittedJobGraph.getJobId}, " +
      s"because it is already submitted.")
  }

其實就是重新的submit job，注意這裡的，isRecovery = true

在submit job時，如果isRecovery = true，會做下面的操作，然後後續具體的操作參考Checkpoint篇

if (isRecovery) {
  executionGraph.restoreLatestCheckpointedState()
}

 

TaskManager Failover

在job manager內部通過death watch發現task manager dead，

/**
    * Handler to be executed when a task manager terminates.
    * (Akka Deathwatch or notifiction from ResourceManager)
    *
    * @param taskManager The ActorRef of the taskManager
    */
  private def handleTaskManagerTerminated(taskManager: ActorRef): Unit = {
    if (instanceManager.isRegistered(taskManager)) {
      log.info(s"Task manager ${taskManager.path} terminated.")

      instanceManager.unregisterTaskManager(taskManager, true)
      context.unwatch(taskManager)
    }
  }

instanceManager.unregisterTaskManager，

/**
* Unregisters the TaskManager with the given {@link ActorRef}. Unregistering means to mark
* the given instance as dead and notify {@link InstanceListener} about the dead instance.
*
* @param instanceID TaskManager which is about to be marked dead.
*/
public void unregisterTaskManager(ActorRef instanceID, boolean terminated){
    Instance instance = registeredHostsByConnection.get(instanceID);
    
    if (instance != null){
        ActorRef host = instance.getActorGateway().actor();
        
        registeredHostsByConnection.remove(host);
        registeredHostsById.remove(instance.getId());
        registeredHostsByResource.remove(instance.getResourceId());
        
        if (terminated) {
            deadHosts.add(instance.getActorGateway().actor());
        }
        
        instance.markDead();
        
        totalNumberOfAliveTaskSlots -= instance.getTotalNumberOfSlots();
        
        notifyDeadInstance(instance);
    }
}

 

instance.markDead()

public void markDead() {

    // create a copy of the slots to avoid concurrent modification exceptions
    List<Slot> slots;
    
    synchronized (instanceLock) {
    if (isDead) {
        return;
    }
    isDead = true;
    
    // no more notifications for the slot releasing
    this.slotAvailabilityListener = null;
    
    slots = new ArrayList<Slot>(allocatedSlots);
    
    allocatedSlots.clear();
        availableSlots.clear();
    }
    
    /*
    * releaseSlot must not own the instanceLock in order to avoid dead locks where a slot
    * owning the assignment group lock wants to give itself back to the instance which requires
    * the instance lock
    */
    for (Slot slot : slots) {
        slot.releaseSlot();
    }
}

 

SimpleSolt.releaseSlot

@Override 
public void releaseSlot() { 

    if (!isCanceled()) { 

        // kill all tasks currently running in this slot 
        Execution exec = this.executedTask; 
        if (exec != null && !exec.isFinished()) { 
            exec.fail(new Exception( 
                    "The slot in which the task was executed has been released. Probably loss of TaskManager " 
                            + getInstance())); 
        } 

        // release directly (if we are directly allocated), 
        // otherwise release through the parent shared slot 
        if (getParent() == null) { 
            // we have to give back the slot to the owning instance 
            if (markCancelled()) { 
                getInstance().returnAllocatedSlot(this); 
            } 
        } else { 
            // we have to ask our parent to dispose us 
            getParent().releaseChild(this); 
        }

}

 

Execution.fail

public void fail(Throwable t) {
   processFail(t, false);
}

 

Execution.processFail

先將Execution的狀態設為failed

transitionState(current, FAILED, t)

private boolean transitionState(ExecutionState currentState, ExecutionState targetState, Throwable error) { 

    if (STATE_UPDATER.compareAndSet(this, currentState, targetState)) {
        markTimestamp(targetState); 

        try {
            vertex.notifyStateTransition(attemptId, targetState, error);
        }
        catch (Throwable t) {
            LOG.error("Error while notifying execution graph of execution state transition.", t);
        }
        return true;
    } else {
        return false;
    }
}

設定完後，需要notifyStateTransition

getExecutionGraph().notifyExecutionChange(getJobvertexId(), subTaskIndex, executionId, newState, error);

void notifyExecutionChange(JobVertexID vertexId, int subtask, ExecutionAttemptID executionID, ExecutionState
                        newExecutionState, Throwable error)
{
    ExecutionJobVertex vertex = getJobVertex(vertexId);

    if (executionListenerActors.size() > 0) {
        String message = error == null ? null : ExceptionUtils.stringifyException(error);
        ExecutionGraphMessages.ExecutionStateChanged actorMessage =
                new ExecutionGraphMessages.ExecutionStateChanged(jobID, vertexId,  vertex.getJobVertex().getName(),
                                                                vertex.getParallelism(), subtask,
                                                                executionID, newExecutionState,
                                                                System.currentTimeMillis(), message);

        for (ActorGateway listener : executionListenerActors) {
            listener.tell(actorMessage);
        }
    }

    // see what this means for us. currently, the first FAILED state means -> FAILED
    if (newExecutionState == ExecutionState.FAILED) {
        fail(error);
    }
}

主要就是將ExecutionGraphMessages.ExecutionStateChanged，傳送給所有的listeners

listener是在JobManager裡面在提交job的時候加上的，

     if (jobInfo.listeningBehaviour == ListeningBehaviour.EXECUTION_RESULT_AND_STATE_CHANGES) {
          // the sender wants to be notified about state changes
          val gateway = new AkkaActorGateway(jobInfo.client, leaderSessionID.orNull)

          executionGraph.registerExecutionListener(gateway)
          executionGraph.registerJobStatusListener(gateway)
      }

而在client，

JobClientActor，只是log和print這些資訊

if (message instanceof ExecutionGraphMessages.ExecutionStateChanged) {
    logAndPrintMessage((ExecutionGraphMessages.ExecutionStateChanged) message);
} else if (message instanceof ExecutionGraphMessages.JobStatusChanged) {
    logAndPrintMessage((ExecutionGraphMessages.JobStatusChanged) message);
}

 

注意，這裡如果newExecutionState == ExecutionState.FAILED，會呼叫ExecutionGraph.fail
就像註釋說的，第一個failed，就意味著整個jobfailed

public void fail(Throwable t) {
    while (true) {
        JobStatus current = state;
        // stay in these states
        if (current == JobStatus.FAILING ||
            current == JobStatus.SUSPENDED ||
            current.isGloballyTerminalState()) {
            return;
        } else if (current == JobStatus.RESTARTING && transitionState(current, JobStatus.FAILED, t)) {
            synchronized (progressLock) {
                postRunCleanup();
                progressLock.notifyAll();
                return;
            }
        } else if (transitionState(current, JobStatus.FAILING, t)) { //將job的狀態設為JobStatus.FAILING
            this.failureCause = t;

            if (!verticesInCreationOrder.isEmpty()) {
                // cancel all. what is failed will not cancel but stay failed
                for (ExecutionJobVertex ejv : verticesInCreationOrder) {
                    ejv.cancel();
                }
            } else {
                // set the state of the job to failed
                transitionState(JobStatus.FAILING, JobStatus.FAILED, t); //
            }

            return;
        }

    }
}

可以看到，這裡直接把job狀態設為Failing，並且呼叫所有的ExecutionJobVertex.cancel

 

接著，從ExecutionGraph中deregister這個execution，

vertex.getExecutionGraph().deregisterExecution(this);

Execution contained = currentExecutions.remove(exec.getAttemptId());

 

最終，呼叫

vertex.executionFailed(t);

void executionFailed(Throwable t) {
    jobVertex.vertexFailed(subTaskIndex, t);
}

 

ExecutionJobVertex

void vertexFailed(int subtask, Throwable error) {
    subtaskInFinalState(subtask);
}

private void subtaskInFinalState(int subtask) {
    synchronized (stateMonitor) {
        if (!finishedSubtasks[subtask]) {
            finishedSubtasks[subtask] = true;
            
            if (numSubtasksInFinalState+1 == parallelism) { //看看對於Vertex而言，是否所有的subTask都已經finished
                
                // call finalizeOnMaster hook
                try {
                    getJobVertex().finalizeOnMaster(getGraph().getUserClassLoader());
                }
                catch (Throwable t) {
                    getGraph().fail(t);
                }

                numSubtasksInFinalState++;
                
                // we are in our final state
                stateMonitor.notifyAll();
                
                // tell the graph
                graph.jobVertexInFinalState();
            } else {
                numSubtasksInFinalState++;
            }
        }
    }
}

graph.jobVertexInFinalState()

void jobVertexInFinalState() {
        numFinishedJobVertices++;

        if (numFinishedJobVertices == verticesInCreationOrder.size()) { //是否所有JobVertices都已經finished

            // we are done, transition to the final state
            JobStatus current;
            while (true) {
                current = this.state;

                if (current == JobStatus.RUNNING) {
                    if (transitionState(current, JobStatus.FINISHED)) {
                        postRunCleanup();
                        break;
                    }
                }
                else if (current == JobStatus.CANCELLING) {
                    if (transitionState(current, JobStatus.CANCELED)) {
                        postRunCleanup();
                        break;
                    }
                }
                else if (current == JobStatus.FAILING) {
                    boolean allowRestart = !(failureCause instanceof SuppressRestartsException);

                    if (allowRestart && restartStrategy.canRestart() && transitionState(current, JobStatus.RESTARTING)) {
                        restartStrategy.restart(this);
                        break;
                    } else if ((!allowRestart || !restartStrategy.canRestart()) && transitionState(current, JobStatus.FAILED, failureCause)) {
                        postRunCleanup();
                        break;
                    }
                }
                else if (current == JobStatus.SUSPENDED) {
                    // we've already cleaned up when entering the SUSPENDED state
                    break;
                }
                else if (current.isGloballyTerminalState()) {
                    LOG.warn("Job has entered globally terminal state without waiting for all " +
                        "job vertices to reach final state.");
                    break;
                }
                else {
                    fail(new Exception("ExecutionGraph went into final state from state " + current));
                    break;
                }
            }
            // done transitioning the state

            // also, notify waiters
            progressLock.notifyAll();
        }
    }
}

如果Job狀態是JobStatus.FAILING，並且滿足restart的條件，transitionState(current, JobStatus.RESTARTING)

restartStrategy.restart(this);

這個restart策略是可以配置的，但無論什麼策略最終呼叫到，

executionGraph.restart();

public void restart() {
    try {
        synchronized (progressLock) {
            JobStatus current = state;

            if (current == JobStatus.CANCELED) {
                LOG.info("Canceled job during restart. Aborting restart.");
                return;
            } else if (current == JobStatus.FAILED) {
                LOG.info("Failed job during restart. Aborting restart.");
                return;
            } else if (current == JobStatus.SUSPENDED) {
                LOG.info("Suspended job during restart. Aborting restart.");
                return;
            } else if (current != JobStatus.RESTARTING) {
                throw new IllegalStateException("Can only restart job from state restarting.");
            }

            if (scheduler == null) {
                throw new IllegalStateException("The execution graph has not been scheduled before - scheduler is null.");
            }

            this.currentExecutions.clear();

            Collection<CoLocationGroup> colGroups = new HashSet<>();

            for (ExecutionJobVertex jv : this.verticesInCreationOrder) {

                CoLocationGroup cgroup = jv.getCoLocationGroup();
                if(cgroup != null && !colGroups.contains(cgroup)){
                    cgroup.resetConstraints();
                    colGroups.add(cgroup);
                }

                jv.resetForNewExecution();
            }

            for (int i = 0; i < stateTimestamps.length; i++) {
                if (i != JobStatus.RESTARTING.ordinal()) {
                    // Only clear the non restarting state in order to preserve when the job was
                    // restarted. This is needed for the restarting time gauge
                    stateTimestamps[i] = 0;
                }
            }
            numFinishedJobVertices = 0;
            transitionState(JobStatus.RESTARTING, JobStatus.CREATED);

            // if we have checkpointed state, reload it into the executions
            if (checkpointCoordinator != null) {
                boolean restored = checkpointCoordinator
                        .restoreLatestCheckpointedState(getAllVertices(), false, false); //重新載入checkpoint和狀態

                // TODO(uce) Temporary work around to restore initial state on
                // failure during recovery. Will be superseded by FLINK-3397.
                if (!restored && savepointCoordinator != null) {
                    String savepointPath = savepointCoordinator.getSavepointRestorePath();
                    if (savepointPath != null) {
                        savepointCoordinator.restoreSavepoint(getAllVertices(), savepointPath);
                    }
                }
            }
        }

        scheduleForExecution(scheduler); //把ExecuteGraph加入排程，重新提交
    }
    catch (Throwable t) {
        fail(t);
    }
}