Java源码示例：org.apache.hadoop.yarn.api.records.QueueInfo
示例1
@Test
public void testDefaultNodeLabelExpressionQueueConfig() throws Exception {
CapacityScheduler cs = new CapacityScheduler();
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
conf.setDefaultNodeLabelExpression("root.a", " x");
conf.setDefaultNodeLabelExpression("root.b", " y ");
cs.setConf(new YarnConfiguration());
cs.setRMContext(resourceManager.getRMContext());
cs.init(conf);
cs.start();
QueueInfo queueInfoA = cs.getQueueInfo("a", true, false);
Assert.assertEquals(queueInfoA.getQueueName(), "a");
Assert.assertEquals(queueInfoA.getDefaultNodeLabelExpression(), "x");
QueueInfo queueInfoB = cs.getQueueInfo("b", true, false);
Assert.assertEquals(queueInfoB.getQueueName(), "b");
Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y");
}
示例2
/**
* Utility method to validate a list resource requests, by insuring that the
* requested memory/vcore is non-negative and not greater than max
*/
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
Resource maximumResource, String queueName, YarnScheduler scheduler,
RMContext rmContext)
throws InvalidResourceRequestException {
QueueInfo queueInfo = null;
try {
queueInfo = scheduler.getQueueInfo(queueName, false, false);
} catch (IOException e) {
}
for (ResourceRequest resReq : ask) {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
queueName, scheduler, rmContext, queueInfo);
}
}
示例3
@Override
public QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
queueInfo.setQueueName(DEFAULT_QUEUE.getQueueName());
queueInfo.setCapacity(1.0f);
if (clusterResource.getMemory() == 0) {
queueInfo.setCurrentCapacity(0.0f);
} else {
queueInfo.setCurrentCapacity((float) usedResource.getMemory()
/ clusterResource.getMemory());
}
queueInfo.setMaximumCapacity(1.0f);
queueInfo.setChildQueues(new ArrayList<QueueInfo>());
queueInfo.setQueueState(QueueState.RUNNING);
return queueInfo;
}
示例4
/**
* Lists the Queue Information matching the given queue name
*
* @param queueName
* @throws YarnException
* @throws IOException
*/
private int listQueue(String queueName) throws YarnException, IOException {
int rc;
PrintWriter writer = new PrintWriter(
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
QueueInfo queueInfo = client.getQueueInfo(queueName);
if (queueInfo != null) {
writer.println("Queue Information : ");
printQueueInfo(writer, queueInfo);
rc = 0;
} else {
writer.println("Cannot get queue from RM by queueName = " + queueName
+ ", please check.");
rc = -1;
}
writer.flush();
return rc;
}
示例5
@Test
public void testGetQueueInfo() throws Exception {
QueueCLI cli = createAndGetQueueCLI();
Set<String> nodeLabels = new HashSet<String>();
nodeLabels.add("GPU");
nodeLabels.add("JDK_7");
QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
null, null, QueueState.RUNNING, nodeLabels, "GPU");
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
int result = cli.run(new String[] { "-status", "queueA" });
assertEquals(0, result);
verify(client).getQueueInfo("queueA");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Queue Information : ");
pw.println("Queue Name : " + "queueA");
pw.println("\tState : " + "RUNNING");
pw.println("\tCapacity : " + "40.0%");
pw.println("\tCurrent Capacity : " + "50.0%");
pw.println("\tMaximum Capacity : " + "80.0%");
pw.println("\tDefault Node Label expression : " + "GPU");
pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
pw.close();
String queueInfoStr = baos.toString("UTF-8");
Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}
示例6
@Override
public void trackQueue(String queueName) {
trackedQueues.add(queueName);
FifoScheduler fifo = (FifoScheduler) scheduler;
// for FifoScheduler, only DEFAULT_QUEUE
// here the three parameters doesn't affect results
final QueueInfo queue = fifo.getQueueInfo(queueName, false, false);
// track currentCapacity, maximumCapacity (always 1.0f)
metrics.register("variable.queue." + queueName + ".currentcapacity",
new Gauge<Float>() {
@Override
public Float getValue() {
return queue.getCurrentCapacity();
}
}
);
metrics.register("variable.queue." + queueName + ".",
new Gauge<Float>() {
@Override
public Float getValue() {
return queue.getCurrentCapacity();
}
}
);
}
示例7
@Test
public void testDefaultNodeLabelExpressionQueueConfig() throws Exception {
CapacityScheduler cs = new CapacityScheduler();
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupQueueConfiguration(conf);
conf.setDefaultNodeLabelExpression("root.a", " x");
conf.setDefaultNodeLabelExpression("root.b", " y ");
cs.setConf(new YarnConfiguration());
cs.setRMContext(resourceManager.getRMContext());
cs.init(conf);
cs.start();
QueueInfo queueInfoA = cs.getQueueInfo("a", true, false);
Assert.assertEquals(queueInfoA.getQueueName(), "a");
Assert.assertEquals(queueInfoA.getDefaultNodeLabelExpression(), "x");
QueueInfo queueInfoB = cs.getQueueInfo("b", true, false);
Assert.assertEquals(queueInfoB.getQueueName(), "b");
Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y");
}
示例8
/**
* Lists the Queue Information matching the given queue name
*
* @param queueName
* @throws YarnException
* @throws IOException
*/
private int listQueue(String queueName) throws YarnException, IOException {
int rc;
PrintWriter writer = new PrintWriter(
new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
QueueInfo queueInfo = client.getQueueInfo(queueName);
if (queueInfo != null) {
writer.println("Queue Information : ");
printQueueInfo(writer, queueInfo);
rc = 0;
} else {
writer.println("Cannot get queue from RM by queueName = " + queueName
+ ", please check.");
rc = -1;
}
writer.flush();
return rc;
}
示例9
@Test
public void testGetQueueInfo() throws Exception {
QueueCLI cli = createAndGetQueueCLI();
Set<String> nodeLabels = new HashSet<String>();
nodeLabels.add("GPU");
nodeLabels.add("JDK_7");
QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
null, null, QueueState.RUNNING, nodeLabels, "GPU");
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
int result = cli.run(new String[] { "-status", "queueA" });
assertEquals(0, result);
verify(client).getQueueInfo("queueA");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Queue Information : ");
pw.println("Queue Name : " + "queueA");
pw.println("\tState : " + "RUNNING");
pw.println("\tCapacity : " + "40.0%");
pw.println("\tCurrent Capacity : " + "50.0%");
pw.println("\tMaximum Capacity : " + "80.0%");
pw.println("\tDefault Node Label expression : " + "GPU");
pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
pw.close();
String queueInfoStr = baos.toString("UTF-8");
Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}
示例10
@Test
public void testGetQueueInfoWithEmptyNodeLabel() throws Exception {
QueueCLI cli = createAndGetQueueCLI();
QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
null, null, QueueState.RUNNING, null, null);
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
int result = cli.run(new String[] { "-status", "queueA" });
assertEquals(0, result);
verify(client).getQueueInfo("queueA");
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Queue Information : ");
pw.println("Queue Name : " + "queueA");
pw.println("\tState : " + "RUNNING");
pw.println("\tCapacity : " + "40.0%");
pw.println("\tCurrent Capacity : " + "50.0%");
pw.println("\tMaximum Capacity : " + "80.0%");
pw.println("\tDefault Node Label expression : ");
pw.println("\tAccessible Node Labels : ");
pw.close();
String queueInfoStr = baos.toString("UTF-8");
Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}
示例11
private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext)
throws IOException {
Dispatcher dispatcher = mock(Dispatcher.class);
when(rmContext.getDispatcher()).thenReturn(dispatcher);
EventHandler eventHandler = mock(EventHandler.class);
when(dispatcher.getEventHandler()).thenReturn(eventHandler);
QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class);
queInfo.setQueueName("testqueue");
when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean()))
.thenReturn(queInfo);
when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean()))
.thenThrow(new IOException("queue does not exist"));
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
yarnScheduler);
when(rmContext.getRMApps()).thenReturn(apps);
when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn(
getSchedulerApps(apps));
ResourceScheduler rs = mock(ResourceScheduler.class);
when(rmContext.getScheduler()).thenReturn(rs);
}
示例12
@Override
public void trackQueue(String queueName) {
trackedQueues.add(queueName);
FifoScheduler fifo = (FifoScheduler) scheduler;
// for FifoScheduler, only DEFAULT_QUEUE
// here the three parameters doesn't affect results
final QueueInfo queue = fifo.getQueueInfo(queueName, false, false);
// track currentCapacity, maximumCapacity (always 1.0f)
metrics.register("variable.queue." + queueName + ".currentcapacity",
new Gauge<Float>() {
@Override
public Float getValue() {
return queue.getCurrentCapacity();
}
}
);
metrics.register("variable.queue." + queueName + ".",
new Gauge<Float>() {
@Override
public Float getValue() {
return queue.getCurrentCapacity();
}
}
);
}
示例13
/**
* Traverse queues bf.
*
* @param queueName the queue name
* @param queueAlert the queue alert
* @param rmCommunicator
* @throws YarnException the yarn exception
* @throws IOException Signals that an I/O exception has occurred.
*/
private void traverseQueuesBF(String queueName, List<AlertInfo> queueAlert, RMCommunicator rmCommunicator) throws YarnException, IOException {
final String parentQueue = queueName;
QueueInfo qi = rmCommunicator.getQueueInfo(queueName);
List<String> queueNames = new ArrayList<String>(5);
float childrenCapacity = 0.0f;
for (QueueInfo info : qi.getChildQueues()) {
queueNames.add(info.getQueueName());
childrenCapacity += info.getCapacity();
}
if(childrenCapacity > 1.0){
AlertInfo alertInfo = new AlertInfo (ExtendedConstants.WARNING_LEVEL,ExtendedConstants.HYPHEN,"Queue "+parentQueue+":"+"child capacity exceeded 100 percent", getDate());
queueAlert.add(alertInfo);
}
for(String name : queueNames) {
traverseQueuesBF(name, queueAlert, rmCommunicator);
}
}
示例14
/**
* Utility method to validate a list resource requests, by insuring that the
* requested memory/vcore is non-negative and not greater than max
*/
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
Resource maximumResource, String queueName, YarnScheduler scheduler,
RMContext rmContext)
throws InvalidResourceRequestException {
QueueInfo queueInfo = null;
try {
queueInfo = scheduler.getQueueInfo(queueName, false, false);
} catch (IOException e) {
}
for (ResourceRequest resReq : ask) {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
queueName, scheduler, rmContext, queueInfo);
}
}
示例15
@Override
public QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
queueInfo.setQueueName(DEFAULT_QUEUE.getQueueName());
queueInfo.setCapacity(1.0f);
if (clusterResource.getMemory() == 0) {
queueInfo.setCurrentCapacity(0.0f);
} else {
queueInfo.setCurrentCapacity((float) usedResource.getMemory()
/ clusterResource.getMemory());
}
queueInfo.setMaximumCapacity(1.0f);
queueInfo.setChildQueues(new ArrayList<QueueInfo>());
queueInfo.setQueueState(QueueState.RUNNING);
return queueInfo;
}
示例16
private static void normalizeNodeLabelExpressionInRequest(
ResourceRequest resReq, QueueInfo queueInfo) {
String labelExp = resReq.getNodeLabelExpression();
// if queue has default label expression, and RR doesn't have, use the
// default label expression of queue
if (labelExp == null && queueInfo != null && ResourceRequest.ANY
.equals(resReq.getResourceName())) {
labelExp = queueInfo.getDefaultNodeLabelExpression();
}
// If labelExp still equals to null, set it to be NO_LABEL
if (labelExp == null) {
labelExp = RMNodeLabelsManager.NO_LABEL;
}
resReq.setNodeLabelExpression(labelExp);
}
示例17
@Override
public synchronized QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
QueueInfo queueInfo = getQueueInfo();
List<QueueInfo> childQueuesInfo = new ArrayList<QueueInfo>();
if (includeChildQueues) {
for (CSQueue child : childQueues) {
// Get queue information recursively?
childQueuesInfo.add(
child.getQueueInfo(recursive, recursive));
}
}
queueInfo.setChildQueues(childQueuesInfo);
return queueInfo;
}
示例18
@Override
public String getClusterDescription() {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
final String format = "|%-16s |%-16s %n";
ps.printf("|Property |Value %n");
ps.println("+---------------------------------------+");
int totalMemory = 0;
int totalCores = 0;
for (NodeReport rep : nodes) {
final Resource res = rep.getCapability();
totalMemory += res.getMemory();
totalCores += res.getVirtualCores();
ps.format(format, "NodeID", rep.getNodeId());
ps.format(format, "Memory", res.getMemory() + " MB");
ps.format(format, "vCores", res.getVirtualCores());
ps.format(format, "HealthReport", rep.getHealthReport());
ps.format(format, "Containers", rep.getNumContainers());
ps.println("+---------------------------------------+");
}
ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
List<QueueInfo> qInfo = yarnClient.getAllQueues();
for (QueueInfo q : qInfo) {
ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
}
return baos.toString();
} catch (Exception e) {
throw new RuntimeException("Couldn't get cluster description", e);
}
}
示例19
@Override
public QueueInfo getQueueInfo() {
if (this.queueInfo != null) {
return this.queueInfo;
}
GetQueueInfoResponseProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasQueueInfo()) {
return null;
}
this.queueInfo = convertFromProtoFormat(p.getQueueInfo());
return this.queueInfo;
}
示例20
@Override
public void setQueueInfo(QueueInfo queueInfo) {
maybeInitBuilder();
if(queueInfo == null) {
builder.clearQueueInfo();
}
this.queueInfo = queueInfo;
}
示例21
@Test(timeout = 15000)
public void testGetQueueInfoOnHA() throws Exception {
QueueInfo queueInfo = client.getQueueInfo("root");
Assert.assertTrue(queueInfo != null);
Assert.assertEquals(cluster.createFakeQueueInfo(),
queueInfo);
}
示例22
private void addChildQueuesInfoToProto() {
maybeInitBuilder();
builder.clearChildQueues();
if (childQueuesList == null)
return;
Iterable<QueueInfoProto> iterable = new Iterable<QueueInfoProto>() {
@Override
public Iterator<QueueInfoProto> iterator() {
return new Iterator<QueueInfoProto>() {
Iterator<QueueInfo> iter = childQueuesList.iterator();
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public QueueInfoProto next() {
return convertToProtoFormat(iter.next());
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
};
builder.addAllChildQueues(iterable);
}
示例23
@Override
public List<QueueInfo> getAllQueues() throws YarnException,
IOException {
List<QueueInfo> queues = new ArrayList<QueueInfo>();
QueueInfo rootQueue =
rmClient.getQueueInfo(getQueueInfoRequest(ROOT, false, true, true))
.getQueueInfo();
getChildQueues(rootQueue, queues, true);
return queues;
}
示例24
@Override
public QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive) {
QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
queueInfo.setQueueName(getQueueName());
if (scheduler.getClusterResource().getMemory() == 0) {
queueInfo.setCapacity(0.0f);
} else {
queueInfo.setCapacity((float) getFairShare().getMemory() /
scheduler.getClusterResource().getMemory());
}
if (getFairShare().getMemory() == 0) {
queueInfo.setCurrentCapacity(0.0f);
} else {
queueInfo.setCurrentCapacity((float) getResourceUsage().getMemory() /
getFairShare().getMemory());
}
ArrayList<QueueInfo> childQueueInfos = new ArrayList<QueueInfo>();
if (includeChildQueues) {
Collection<FSQueue> childQueues = getChildQueues();
for (FSQueue child : childQueues) {
childQueueInfos.add(child.getQueueInfo(recursive, recursive));
}
}
queueInfo.setChildQueues(childQueueInfos);
queueInfo.setQueueState(QueueState.RUNNING);
return queueInfo;
}
示例25
@Private
@Unstable
public static GetQueueInfoResponse newInstance(QueueInfo queueInfo) {
GetQueueInfoResponse response = Records.newRecord(GetQueueInfoResponse.class);
response.setQueueInfo(queueInfo);
return response;
}
示例26
public static void normalizeAndvalidateRequest(ResourceRequest resReq,
Resource maximumResource, String queueName, YarnScheduler scheduler,
RMContext rmContext, QueueInfo queueInfo)
throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
false, rmContext, queueInfo);
}
示例27
private void addQueue(QueueInfo queueInfo, List<QueueInfo> list) {
list.add(queueInfo);
if (queueInfo.getChildQueues() != null) {
for (QueueInfo childQueue : queueInfo.getChildQueues()) {
addQueue(childQueue, list);
}
}
}
示例28
@Override
public String getClusterDescription() {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(baos);
YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
final String format = "|%-16s |%-16s %n";
ps.printf("|Property |Value %n");
ps.println("+---------------------------------------+");
int totalMemory = 0;
int totalCores = 0;
for (NodeReport rep : nodes) {
final Resource res = rep.getCapability();
totalMemory += res.getMemory();
totalCores += res.getVirtualCores();
ps.format(format, "NodeID", rep.getNodeId());
ps.format(format, "Memory", res.getMemory() + " MB");
ps.format(format, "vCores", res.getVirtualCores());
ps.format(format, "HealthReport", rep.getHealthReport());
ps.format(format, "Containers", rep.getNumContainers());
ps.println("+---------------------------------------+");
}
ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
List<QueueInfo> qInfo = yarnClient.getAllQueues();
for (QueueInfo q : qInfo) {
ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
}
return baos.toString();
} catch (Exception e) {
throw new RuntimeException("Couldn't get cluster description", e);
}
}
示例29
@Test
public void testCapacitySchedulerInfo() throws Exception {
QueueInfo queueInfo = resourceManager.getResourceScheduler().getQueueInfo("a", true, true);
Assert.assertEquals(queueInfo.getQueueName(), "a");
Assert.assertEquals(queueInfo.getChildQueues().size(), 2);
List<QueueUserACLInfo> userACLInfo = resourceManager.getResourceScheduler().getQueueUserAclInfo();
Assert.assertNotNull(userACLInfo);
for (QueueUserACLInfo queueUserACLInfo : userACLInfo) {
Assert.assertEquals(getQueueCount(userACLInfo, queueUserACLInfo.getQueueName()), 1);
}
}
示例30
protected QueueInfo getQueueInfo() {
QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
queueInfo.setQueueName(queueName);
queueInfo.setAccessibleNodeLabels(accessibleLabels);
queueInfo.setCapacity(queueCapacities.getCapacity());
queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity());
queueInfo.setQueueState(state);
queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
queueInfo.setCurrentCapacity(getUsedCapacity());
return queueInfo;
}
