@InterfaceAudience.LimitedPrivate(value="yarn") @InterfaceStability.Unstable public class FairScheduler extends AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>
| Modifier and Type | Field and Description |
|---|---|
protected boolean |
assignMultiple |
static org.apache.hadoop.yarn.api.records.Resource |
CONTAINER_RESERVED |
protected boolean |
continuousSchedulingEnabled |
protected int |
continuousSchedulingSleepMs |
protected int |
maxAssign |
protected long |
nodeLocalityDelayMs |
protected double |
nodeLocalityThreshold |
protected long |
rackLocalityDelayMs |
protected double |
rackLocalityThreshold |
protected boolean |
sizeBasedWeight |
activitiesManager, applications, EMPTY_ALLOCATION, lastNodeUpdateTime, minimumAllocation, nmExpireInterval, nmHeartbeatInterval, nodeTracker, readLock, rmContext, schedulerHealth, THREAD_JOIN_TIMEOUT_MS, updateInterval, writeLock| Constructor and Description |
|---|
FairScheduler() |
| Modifier and Type | Method and Description |
|---|---|
protected void |
addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String queueName,
String user,
boolean isAppRecovering)
Add a new application to the scheduler, with a given id, queue name, and
user.
|
protected void |
addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId,
boolean transferStateFromPreviousAttempt,
boolean isAttemptRecovering)
Add a new application attempt to the scheduler.
|
Allocation |
allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId,
List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask,
List<org.apache.hadoop.yarn.api.records.ContainerId> release,
List<String> blacklistAdditions,
List<String> blacklistRemovals,
ContainerUpdates updateRequests)
The main api between the ApplicationMaster and the Scheduler.
|
boolean |
checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI,
org.apache.hadoop.yarn.api.records.QueueACL acl,
String queueName)
Check if the user has permission to perform the operation.
|
long |
checkAndGetApplicationLifetime(String queueName,
long lifetime)
Verify whether a submitted application lifetime is valid as per configured
Queue lifetime.
|
protected void |
completedContainerInternal(RMContainer rmContainer,
org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus,
RMContainerEventType event)
Clean up a completed container.
|
protected void |
createPreemptionThread() |
AllocationConfiguration |
getAllocationConfiguration() |
List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> |
getAppsInQueue(String queueName)
Gets the apps under a given queue
|
ResourceWeights |
getAppWeight(FSAppAttempt app) |
FairSchedulerConfiguration |
getConf() |
RMContainerTokenSecretManager |
getContainerTokenSecretManager() |
FSContext |
getContext() |
int |
getContinuousSchedulingSleepMs() |
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog |
getEventLog() |
org.apache.hadoop.yarn.api.records.Resource |
getIncrementResourceCapability() |
long |
getNodeLocalityDelayMs() |
double |
getNodeLocalityThreshold() |
org.apache.hadoop.yarn.api.records.Resource |
getNormalizedResource(org.apache.hadoop.yarn.api.records.Resource requestedResource)
Normalize a resource request.
|
int |
getNumClusterNodes()
Get the number of nodes available in the cluster.
|
int |
getNumNodesInRack(String rackName) |
Set<String> |
getPlanQueues()
Gets the list of names for queues managed by the Reservation System
|
org.apache.hadoop.yarn.api.records.QueueInfo |
getQueueInfo(String queueName,
boolean includeChildQueues,
boolean recursive)
Get queue information
|
QueueManager |
getQueueManager() |
List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> |
getQueueUserAclInfo()
Get acls for queues for current user.
|
long |
getRackLocalityDelayMs() |
double |
getRackLocalityThreshold() |
float |
getReservableNodesRatio() |
org.apache.hadoop.yarn.util.resource.ResourceCalculator |
getResourceCalculator() |
QueueMetrics |
getRootQueueMetrics()
Get the root queue for the scheduler.
|
FSAppAttempt |
getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId) |
EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> |
getSchedulingResourceTypes()
Return a collection of the resource types that are considered when
scheduling
|
void |
handle(SchedulerEvent event) |
boolean |
isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator,
org.apache.hadoop.yarn.api.records.Resource resource) |
boolean |
isContinuousSchedulingEnabled() |
void |
killContainer(RMContainer container)
Kill a RMContainer.
|
String |
moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String queueName)
Moves the given application to the given queue
|
protected void |
nodeUpdate(RMNode nm)
Process a heartbeat update from a node.
|
void |
preValidateMoveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String newQueue) |
void |
recover(RMStateStore.RMState state) |
void |
reinitialize(org.apache.hadoop.conf.Configuration conf,
RMContext rmContext)
Re-initialize the
ResourceScheduler. |
void |
removeQueue(String queueName)
Only supports removing empty leaf queues
|
void |
serviceInit(org.apache.hadoop.conf.Configuration conf) |
void |
serviceStart() |
void |
serviceStop() |
void |
setEntitlement(String queueName,
QueueEntitlement entitlement)
This method increase the entitlement for current queue (must respect
invariants, e.g., no overcommit of parents, non negative, etc.).
|
void |
setRMContext(RMContext rmContext)
Set RMContext for
ResourceScheduler. |
void |
update()
Recompute the internal variables used by the scheduler - per-job weights,
fair shares, deficits, minimum slot allocations, and amount of used and
required resources per job.
|
void |
updateNodeResource(RMNode nm,
org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
Process resource update on a node and update Queue.
|
addQueue, asyncContainerRelease, checkAndGetApplicationPriority, clearPendingContainerCache, completedContainer, containerIncreasedOnNode, containerLaunchedOnNode, createReleaseCache, createSchedContainerChangeRequests, getActivitiesManager, getApplicationAttempt, getAppResourceUsageReport, getBlacklistedNodes, getClock, getClusterResource, getCurrentAttemptForContainer, getLastNodeUpdateTime, getMaxClusterLevelAppPriority, getMaximumAllocation, getMaximumApplicationLifetime, getMaximumResourceCapability, getMaximumResourceCapability, getMinimumAllocation, getMinimumResourceCapability, getNode, getNodeIds, getNodeReport, getNodeTracker, getPendingResourceRequestsForAttempt, getRMContainer, getSchedulerAppInfo, getSchedulerApplications, getSchedulerHealth, getSchedulerNode, getTransferredContainers, handleContainerUpdates, initMaximumResourceCapability, killAllAppsInQueue, moveAllApps, normalizeRequests, recoverContainersOnNode, refreshMaximumAllocation, releaseContainers, resetSchedulerMetrics, rollbackContainerUpdate, setClock, setClusterMaxPriority, setLastNodeUpdateTime, shouldContainersBeAutoUpdated, triggerUpdate, updateApplicationPriority, updateCompletedContainers, updateNewContainerInfo, updateNodeResourceUtilization, updateSchedulerHealthInformationclose, getBlockers, getConfig, getFailureCause, getFailureState, getLifecycleHistory, getName, getServiceState, getStartTime, init, isInState, noteFailure, putBlocker, registerGlobalListener, registerServiceListener, removeBlocker, setConfig, start, stop, toString, unregisterGlobalListener, unregisterServiceListener, waitForServiceToStoppublic static final org.apache.hadoop.yarn.api.records.Resource CONTAINER_RESERVED
protected boolean sizeBasedWeight
protected boolean continuousSchedulingEnabled
protected volatile int continuousSchedulingSleepMs
protected double nodeLocalityThreshold
protected double rackLocalityThreshold
protected long nodeLocalityDelayMs
protected long rackLocalityDelayMs
protected boolean assignMultiple
protected int maxAssign
public FSContext getContext()
public boolean isAtLeastReservationThreshold(org.apache.hadoop.yarn.util.resource.ResourceCalculator resourceCalculator,
org.apache.hadoop.yarn.api.records.Resource resource)
public FairSchedulerConfiguration getConf()
public int getNumNodesInRack(String rackName)
public QueueManager getQueueManager()
public void update()
update in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>public RMContainerTokenSecretManager getContainerTokenSecretManager()
public ResourceWeights getAppWeight(FSAppAttempt app)
public org.apache.hadoop.yarn.api.records.Resource getIncrementResourceCapability()
public double getNodeLocalityThreshold()
public double getRackLocalityThreshold()
public long getNodeLocalityDelayMs()
public long getRackLocalityDelayMs()
public boolean isContinuousSchedulingEnabled()
public int getContinuousSchedulingSleepMs()
public org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerEventLog getEventLog()
protected void addApplication(org.apache.hadoop.yarn.api.records.ApplicationId applicationId,
String queueName,
String user,
boolean isAppRecovering)
protected void addApplicationAttempt(org.apache.hadoop.yarn.api.records.ApplicationAttemptId applicationAttemptId,
boolean transferStateFromPreviousAttempt,
boolean isAttemptRecovering)
protected void completedContainerInternal(RMContainer rmContainer, org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus, RMContainerEventType event)
completedContainerInternal in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>public org.apache.hadoop.yarn.api.records.Resource getNormalizedResource(org.apache.hadoop.yarn.api.records.Resource requestedResource)
YarnSchedulergetNormalizedResource in interface YarnSchedulergetNormalizedResource in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>requestedResource - the resource to be normalizedpublic void killContainer(RMContainer container)
AbstractYarnSchedulerkillContainer in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>container - the container to killpublic Allocation allocate(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId, List<org.apache.hadoop.yarn.api.records.ResourceRequest> ask, List<org.apache.hadoop.yarn.api.records.ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals, ContainerUpdates updateRequests)
YarnSchedulerAllocation for the applicationprotected void nodeUpdate(RMNode nm)
AbstractYarnSchedulernodeUpdate in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>nm - The RMNode corresponding to the NodeManagerpublic FSAppAttempt getSchedulerApp(org.apache.hadoop.yarn.api.records.ApplicationAttemptId appAttemptId)
public org.apache.hadoop.yarn.util.resource.ResourceCalculator getResourceCalculator()
public QueueMetrics getRootQueueMetrics()
YarnSchedulerpublic void handle(SchedulerEvent event)
public void recover(RMStateStore.RMState state) throws Exception
Exceptionpublic void setRMContext(RMContext rmContext)
ResourceSchedulerResourceScheduler.
This method should be called immediately after instantiating
a scheduler once.rmContext - created by ResourceManagerprotected void createPreemptionThread()
public void serviceInit(org.apache.hadoop.conf.Configuration conf)
throws Exception
serviceInit in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>Exceptionpublic void serviceStart()
throws Exception
serviceStart in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>Exceptionpublic void serviceStop()
throws Exception
serviceStop in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>Exceptionpublic void reinitialize(org.apache.hadoop.conf.Configuration conf,
RMContext rmContext)
throws IOException
ResourceSchedulerResourceScheduler.conf - configurationIOExceptionpublic org.apache.hadoop.yarn.api.records.QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, boolean recursive) throws IOException
YarnSchedulerqueueName - queue nameincludeChildQueues - include child queues?recursive - get children queues?IOExceptionpublic List<org.apache.hadoop.yarn.api.records.QueueUserACLInfo> getQueueUserAclInfo()
YarnSchedulerpublic int getNumClusterNodes()
YarnSchedulerpublic boolean checkAccess(org.apache.hadoop.security.UserGroupInformation callerUGI,
org.apache.hadoop.yarn.api.records.QueueACL acl,
String queueName)
YarnSchedulerQueueACL.ADMINISTER_QUEUE permission,
this user can view/modify the applications in this queuetrue if the user has the permission,
false otherwisepublic AllocationConfiguration getAllocationConfiguration()
public List<org.apache.hadoop.yarn.api.records.ApplicationAttemptId> getAppsInQueue(String queueName)
YarnSchedulerqueueName - the name of the queue.public String moveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId, String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnSchedulermoveApplication in interface YarnSchedulermoveApplication in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>org.apache.hadoop.yarn.exceptions.YarnException - if the move cannot be carried outpublic void preValidateMoveApplication(org.apache.hadoop.yarn.api.records.ApplicationId appId,
String newQueue)
throws org.apache.hadoop.yarn.exceptions.YarnException
preValidateMoveApplication in interface YarnSchedulerpreValidateMoveApplication in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>appId - Application IDnewQueue - Target QueueNameorg.apache.hadoop.yarn.exceptions.YarnException - if the pre-validation for move cannot be carried outpublic void updateNodeResource(RMNode nm, org.apache.hadoop.yarn.api.records.ResourceOption resourceOption)
updateNodeResource in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>public EnumSet<org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes> getSchedulingResourceTypes()
getSchedulingResourceTypes in interface YarnSchedulergetSchedulingResourceTypes in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>public Set<String> getPlanQueues() throws org.apache.hadoop.yarn.exceptions.YarnException
YarnSchedulergetPlanQueues in interface YarnSchedulergetPlanQueues in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>org.apache.hadoop.yarn.exceptions.YarnExceptionpublic void setEntitlement(String queueName, QueueEntitlement entitlement) throws org.apache.hadoop.yarn.exceptions.YarnException
YarnSchedulersetEntitlement in interface YarnSchedulersetEntitlement in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>queueName - the queue for which we change entitlemententitlement - the new entitlement for the queue (capacity,
maxCapacity, etc..)org.apache.hadoop.yarn.exceptions.YarnExceptionpublic void removeQueue(String queueName) throws org.apache.hadoop.yarn.exceptions.YarnException
removeQueue in interface YarnSchedulerremoveQueue in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>queueName - name of queue to removeorg.apache.hadoop.yarn.exceptions.YarnException - if queue to remove is either not a leaf or if its
not emptypublic float getReservableNodesRatio()
public long checkAndGetApplicationLifetime(String queueName, long lifetime)
YarnSchedulercheckAndGetApplicationLifetime in interface YarnSchedulercheckAndGetApplicationLifetime in class AbstractYarnScheduler<FSAppAttempt,FSSchedulerNode>queueName - Name of the Queuelifetime - configured application lifetimeCopyright © 2022 Apache Software Foundation. All rights reserved.