001/** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019package org.apache.hadoop.mapred; 020 021import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString; 022import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString; 023 024import java.io.DataInput; 025import java.io.DataOutput; 026import java.io.IOException; 027import java.text.ParseException; 028import java.util.Collection; 029import java.util.HashMap; 030import java.util.Iterator; 031 032import org.apache.commons.collections.IteratorUtils; 033import org.apache.commons.logging.Log; 034import org.apache.hadoop.classification.InterfaceAudience; 035import org.apache.hadoop.classification.InterfaceStability; 036import org.apache.hadoop.mapreduce.FileSystemCounter; 037import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup; 038import org.apache.hadoop.mapreduce.counters.AbstractCounters; 039import org.apache.hadoop.mapreduce.counters.CounterGroupBase; 040import org.apache.hadoop.mapreduce.counters.CounterGroupFactory; 041import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup; 042import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup; 043import org.apache.hadoop.mapreduce.counters.GenericCounter; 044import org.apache.hadoop.mapreduce.counters.Limits; 045import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter; 046import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter; 047 048import com.google.common.collect.Iterators; 049 050/** 051 * A set of named counters. 052 * 053 * <p><code>Counters</code> represent global counters, defined either by the 054 * Map-Reduce framework or applications. Each <code>Counter</code> can be of 055 * any {@link Enum} type.</p> 056 * 057 * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of 058 * counters from a particular <code>Enum</code> class. 059 */ 060@InterfaceAudience.Public 061@InterfaceStability.Stable 062public class Counters 063 extends AbstractCounters<Counters.Counter, Counters.Group> { 064 065 public static int MAX_COUNTER_LIMIT = Limits.getCountersMax(); 066 public static int MAX_GROUP_LIMIT = Limits.getGroupsMax(); 067 private static HashMap<String, String> depricatedCounterMap = 068 new HashMap<String, String>(); 069 070 static { 071 initDepricatedMap(); 072 } 073 074 public Counters() { 075 super(groupFactory); 076 } 077 078 public Counters(org.apache.hadoop.mapreduce.Counters newCounters) { 079 super(newCounters, groupFactory); 080 } 081 082 @SuppressWarnings({ "deprecation" }) 083 private static void initDepricatedMap() { 084 depricatedCounterMap.put(FileInputFormat.Counter.class.getName(), 085 FileInputFormatCounter.class.getName()); 086 depricatedCounterMap.put(FileOutputFormat.Counter.class.getName(), 087 FileOutputFormatCounter.class.getName()); 088 depricatedCounterMap.put( 089 org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.class 090 .getName(), FileInputFormatCounter.class.getName()); 091 depricatedCounterMap.put( 092 org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.class 093 .getName(), FileOutputFormatCounter.class.getName()); 094 } 095 096 private static String getNewGroupKey(String oldGroup) { 097 if (depricatedCounterMap.containsKey(oldGroup)) { 098 return depricatedCounterMap.get(oldGroup); 099 } 100 return null; 101 } 102 103 /** 104 * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters 105 * @param newCounters new Counters 106 * @return old Counters instance corresponding to newCounters 107 */ 108 static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) { 109 return new Counters(newCounters); 110 } 111 112 public synchronized Group getGroup(String groupName) { 113 return super.getGroup(groupName); 114 } 115 116 @SuppressWarnings("unchecked") 117 public synchronized Collection<String> getGroupNames() { 118 return IteratorUtils.toList(super.getGroupNames().iterator()); 119 } 120 121 public synchronized String makeCompactString() { 122 StringBuilder builder = new StringBuilder(); 123 boolean first = true; 124 for(Group group: this){ 125 for(Counter counter: group) { 126 if (first) { 127 first = false; 128 } else { 129 builder.append(','); 130 } 131 builder.append(group.getDisplayName()); 132 builder.append('.'); 133 builder.append(counter.getDisplayName()); 134 builder.append(':'); 135 builder.append(counter.getCounter()); 136 } 137 } 138 return builder.toString(); 139 } 140 141 /** 142 * A counter record, comprising its name and value. 143 */ 144 @InterfaceAudience.Public 145 @InterfaceStability.Stable 146 public static class Counter implements org.apache.hadoop.mapreduce.Counter { 147 org.apache.hadoop.mapreduce.Counter realCounter; 148 149 Counter(org.apache.hadoop.mapreduce.Counter counter) { 150 this.realCounter = counter; 151 } 152 153 public Counter() { 154 this(new GenericCounter()); 155 } 156 157 @SuppressWarnings("deprecation") 158 @Override 159 public void setDisplayName(String displayName) { 160 realCounter.setDisplayName(displayName); 161 } 162 163 @Override 164 public String getName() { 165 return realCounter.getName(); 166 } 167 168 @Override 169 public String getDisplayName() { 170 return realCounter.getDisplayName(); 171 } 172 173 @Override 174 public long getValue() { 175 return realCounter.getValue(); 176 } 177 178 @Override 179 public void setValue(long value) { 180 realCounter.setValue(value); 181 } 182 183 @Override 184 public void increment(long incr) { 185 realCounter.increment(incr); 186 } 187 188 @Override 189 public void write(DataOutput out) throws IOException { 190 realCounter.write(out); 191 } 192 193 @Override 194 public void readFields(DataInput in) throws IOException { 195 realCounter.readFields(in); 196 } 197 198 /** 199 * Returns the compact stringified version of the counter in the format 200 * [(actual-name)(display-name)(value)] 201 * @return the stringified result 202 */ 203 public String makeEscapedCompactString() { 204 return toEscapedCompactString(realCounter); 205 } 206 207 /** 208 * Checks for (content) equality of two (basic) counters 209 * @param counter to compare 210 * @return true if content equals 211 * @deprecated 212 */ 213 @Deprecated 214 public boolean contentEquals(Counter counter) { 215 return realCounter.equals(counter.getUnderlyingCounter()); 216 } 217 218 /** 219 * @return the value of the counter 220 */ 221 public long getCounter() { 222 return realCounter.getValue(); 223 } 224 225 @Override 226 public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() { 227 return realCounter; 228 } 229 230 @Override 231 public synchronized boolean equals(Object genericRight) { 232 if (genericRight instanceof Counter) { 233 synchronized (genericRight) { 234 Counter right = (Counter) genericRight; 235 return getName().equals(right.getName()) && 236 getDisplayName().equals(right.getDisplayName()) && 237 getValue() == right.getValue(); 238 } 239 } 240 return false; 241 } 242 243 @Override 244 public int hashCode() { 245 return realCounter.hashCode(); 246 } 247 } 248 249 250 /** 251 * <code>Group</code> of counters, comprising of counters from a particular 252 * counter {@link Enum} class. 253 * 254 * <p><code>Group</code>handles localization of the class name and the 255 * counter names.</p> 256 */ 257 @InterfaceAudience.Public 258 @InterfaceStability.Stable 259 public static class Group implements CounterGroupBase<Counter> { 260 private CounterGroupBase<Counter> realGroup; 261 262 protected Group() { 263 realGroup = null; 264 } 265 266 Group(GenericGroup group) { 267 this.realGroup = group; 268 } 269 Group(FSGroupImpl group) { 270 this.realGroup = group; 271 } 272 273 @SuppressWarnings({ "unchecked", "rawtypes" }) 274 Group(FrameworkGroupImpl group) { 275 this.realGroup = group; 276 } 277 278 /** 279 * @param counterName the name of the counter 280 * @return the value of the specified counter, or 0 if the counter does 281 * not exist. 282 */ 283 public long getCounter(String counterName) { 284 return getCounterValue(realGroup, counterName); 285 } 286 287 /** 288 * @return the compact stringified version of the group in the format 289 * {(actual-name)(display-name)(value)[][][]} where [] are compact strings 290 * for the counters within. 291 */ 292 public String makeEscapedCompactString() { 293 return toEscapedCompactString(realGroup); 294 } 295 296 /** 297 * Get the counter for the given id and create it if it doesn't exist. 298 * @param id the numeric id of the counter within the group 299 * @param name the internal counter name 300 * @return the counter 301 * @deprecated use {@link #findCounter(String)} instead 302 */ 303 @Deprecated 304 public Counter getCounter(int id, String name) { 305 return findCounter(name); 306 } 307 308 /** 309 * Get the counter for the given name and create it if it doesn't exist. 310 * @param name the internal counter name 311 * @return the counter 312 */ 313 public Counter getCounterForName(String name) { 314 return findCounter(name); 315 } 316 317 @Override 318 public void write(DataOutput out) throws IOException { 319 realGroup.write(out); 320 } 321 322 @Override 323 public void readFields(DataInput in) throws IOException { 324 realGroup.readFields(in); 325 } 326 327 @Override 328 public Iterator<Counter> iterator() { 329 return realGroup.iterator(); 330 } 331 332 @Override 333 public String getName() { 334 return realGroup.getName(); 335 } 336 337 @Override 338 public String getDisplayName() { 339 return realGroup.getDisplayName(); 340 } 341 342 @Override 343 public void setDisplayName(String displayName) { 344 realGroup.setDisplayName(displayName); 345 } 346 347 @Override 348 public void addCounter(Counter counter) { 349 realGroup.addCounter(counter); 350 } 351 352 @Override 353 public Counter addCounter(String name, String displayName, long value) { 354 return realGroup.addCounter(name, displayName, value); 355 } 356 357 @Override 358 public Counter findCounter(String counterName, String displayName) { 359 return realGroup.findCounter(counterName, displayName); 360 } 361 362 @Override 363 public Counter findCounter(String counterName, boolean create) { 364 return realGroup.findCounter(counterName, create); 365 } 366 367 @Override 368 public Counter findCounter(String counterName) { 369 return realGroup.findCounter(counterName); 370 } 371 372 @Override 373 public int size() { 374 return realGroup.size(); 375 } 376 377 @Override 378 public void incrAllCounters(CounterGroupBase<Counter> rightGroup) { 379 realGroup.incrAllCounters(rightGroup); 380 } 381 382 @Override 383 public CounterGroupBase<Counter> getUnderlyingGroup() { 384 return realGroup; 385 } 386 387 @Override 388 public synchronized boolean equals(Object genericRight) { 389 if (genericRight instanceof CounterGroupBase<?>) { 390 @SuppressWarnings("unchecked") 391 CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>) 392 genericRight).getUnderlyingGroup(); 393 return Iterators.elementsEqual(iterator(), right.iterator()); 394 } 395 return false; 396 } 397 398 @Override 399 public int hashCode() { 400 return realGroup.hashCode(); 401 } 402 } 403 404 // All the group impls need this for legacy group interface 405 static long getCounterValue(CounterGroupBase<Counter> group, String counterName) { 406 Counter counter = group.findCounter(counterName, false); 407 if (counter != null) return counter.getValue(); 408 return 0L; 409 } 410 411 // Mix the generic group implementation into the Group interface 412 private static class GenericGroup extends AbstractCounterGroup<Counter> { 413 414 GenericGroup(String name, String displayName, Limits limits) { 415 super(name, displayName, limits); 416 } 417 418 @Override 419 protected Counter newCounter(String counterName, String displayName, 420 long value) { 421 return new Counter(new GenericCounter(counterName, displayName, value)); 422 } 423 424 @Override 425 protected Counter newCounter() { 426 return new Counter(); 427 } 428 429 @Override 430 public CounterGroupBase<Counter> getUnderlyingGroup() { 431 return this; 432 } 433 } 434 435 // Mix the framework group implementation into the Group interface 436 private static class FrameworkGroupImpl<T extends Enum<T>> 437 extends FrameworkCounterGroup<T, Counter> { 438 439 FrameworkGroupImpl(Class<T> cls) { 440 super(cls); 441 } 442 443 @Override 444 protected Counter newCounter(T key) { 445 return new Counter(new FrameworkCounter<T>(key, getName())); 446 } 447 448 @Override 449 public CounterGroupBase<Counter> getUnderlyingGroup() { 450 return this; 451 } 452 } 453 454 // Mix the file system counter group implementation into the Group interface 455 private static class FSGroupImpl extends FileSystemCounterGroup<Counter> { 456 457 @Override 458 protected Counter newCounter(String scheme, FileSystemCounter key) { 459 return new Counter(new FSCounter(scheme, key)); 460 } 461 462 @Override 463 public CounterGroupBase<Counter> getUnderlyingGroup() { 464 return this; 465 } 466 } 467 468 public synchronized Counter findCounter(String group, String name) { 469 if (name.equals("MAP_INPUT_BYTES")) { 470 LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + 471 "Use FileInputFormatCounters as group name and " + 472 " BYTES_READ as counter name instead"); 473 return findCounter(FileInputFormatCounter.BYTES_READ); 474 } 475 String newGroupKey = getNewGroupKey(group); 476 if (newGroupKey != null) { 477 group = newGroupKey; 478 } 479 return getGroup(group).getCounterForName(name); 480 } 481 482 /** 483 * Provide factory methods for counter group factory implementation. 484 * See also the GroupFactory in 485 * {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters} 486 */ 487 static class GroupFactory extends CounterGroupFactory<Counter, Group> { 488 489 @Override 490 protected <T extends Enum<T>> 491 FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) { 492 return new FrameworkGroupFactory<Group>() { 493 @Override public Group newGroup(String name) { 494 return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package 495 } 496 }; 497 } 498 499 @Override 500 protected Group newGenericGroup(String name, String displayName, 501 Limits limits) { 502 return new Group(new GenericGroup(name, displayName, limits)); 503 } 504 505 @Override 506 protected Group newFileSystemGroup() { 507 return new Group(new FSGroupImpl()); 508 } 509 } 510 511 private static final GroupFactory groupFactory = new GroupFactory(); 512 513 /** 514 * Find a counter by using strings 515 * @param group the name of the group 516 * @param id the id of the counter within the group (0 to N-1) 517 * @param name the internal name of the counter 518 * @return the counter for that name 519 * @deprecated use {@link #findCounter(String, String)} instead 520 */ 521 @Deprecated 522 public Counter findCounter(String group, int id, String name) { 523 return findCounter(group, name); 524 } 525 526 /** 527 * Increments the specified counter by the specified amount, creating it if 528 * it didn't already exist. 529 * @param key identifies a counter 530 * @param amount amount by which counter is to be incremented 531 */ 532 public void incrCounter(Enum<?> key, long amount) { 533 findCounter(key).increment(amount); 534 } 535 536 /** 537 * Increments the specified counter by the specified amount, creating it if 538 * it didn't already exist. 539 * @param group the name of the group 540 * @param counter the internal name of the counter 541 * @param amount amount by which counter is to be incremented 542 */ 543 public void incrCounter(String group, String counter, long amount) { 544 findCounter(group, counter).increment(amount); 545 } 546 547 /** 548 * Returns current value of the specified counter, or 0 if the counter 549 * does not exist. 550 * @param key the counter enum to lookup 551 * @return the counter value or 0 if counter not found 552 */ 553 public synchronized long getCounter(Enum<?> key) { 554 return findCounter(key).getValue(); 555 } 556 557 /** 558 * Increments multiple counters by their amounts in another Counters 559 * instance. 560 * @param other the other Counters instance 561 */ 562 public synchronized void incrAllCounters(Counters other) { 563 for (Group otherGroup: other) { 564 Group group = getGroup(otherGroup.getName()); 565 group.setDisplayName(otherGroup.getDisplayName()); 566 for (Counter otherCounter : otherGroup) { 567 Counter counter = group.getCounterForName(otherCounter.getName()); 568 counter.setDisplayName(otherCounter.getDisplayName()); 569 counter.increment(otherCounter.getValue()); 570 } 571 } 572 } 573 574 /** 575 * @return the total number of counters 576 * @deprecated use {@link #countCounters()} instead 577 */ 578 public int size() { 579 return countCounters(); 580 } 581 582 /** 583 * Convenience method for computing the sum of two sets of counters. 584 * @param a the first counters 585 * @param b the second counters 586 * @return a new summed counters object 587 */ 588 public static Counters sum(Counters a, Counters b) { 589 Counters counters = new Counters(); 590 counters.incrAllCounters(a); 591 counters.incrAllCounters(b); 592 return counters; 593 } 594 595 /** 596 * Logs the current counter values. 597 * @param log The log to use. 598 */ 599 public void log(Log log) { 600 log.info("Counters: " + size()); 601 for(Group group: this) { 602 log.info(" " + group.getDisplayName()); 603 for (Counter counter: group) { 604 log.info(" " + counter.getDisplayName() + "=" + 605 counter.getCounter()); 606 } 607 } 608 } 609 610 /** 611 * Represent the counter in a textual format that can be converted back to 612 * its object form 613 * @return the string in the following format 614 * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}* 615 */ 616 public String makeEscapedCompactString() { 617 return toEscapedCompactString(this); 618 } 619 620 /** 621 * Convert a stringified (by {@link #makeEscapedCompactString()} counter 622 * representation into a counter object. 623 * @param compactString to parse 624 * @return a new counters object 625 * @throws ParseException 626 */ 627 public static Counters fromEscapedCompactString(String compactString) 628 throws ParseException { 629 return parseEscapedCompactString(compactString, new Counters()); 630 } 631 632 /** 633 * Counter exception thrown when the number of counters exceed the limit 634 */ 635 public static class CountersExceededException extends RuntimeException { 636 637 private static final long serialVersionUID = 1L; 638 639 public CountersExceededException(String msg) { 640 super(msg); 641 } 642 643 // Only allows chaining of related exceptions 644 public CountersExceededException(CountersExceededException cause) { 645 super(cause); 646 } 647 } 648}