001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019package org.apache.hadoop.mapred;
020
021import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString;
022import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString;
023
024import java.io.DataInput;
025import java.io.DataOutput;
026import java.io.IOException;
027import java.text.ParseException;
028import java.util.Collection;
029import java.util.HashMap;
030import java.util.Iterator;
031
032import org.apache.commons.collections.IteratorUtils;
033import org.apache.commons.logging.Log;
034import org.apache.hadoop.classification.InterfaceAudience;
035import org.apache.hadoop.classification.InterfaceStability;
036import org.apache.hadoop.mapreduce.FileSystemCounter;
037import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup;
038import org.apache.hadoop.mapreduce.counters.AbstractCounters;
039import org.apache.hadoop.mapreduce.counters.CounterGroupBase;
040import org.apache.hadoop.mapreduce.counters.CounterGroupFactory;
041import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup;
042import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup;
043import org.apache.hadoop.mapreduce.counters.GenericCounter;
044import org.apache.hadoop.mapreduce.counters.Limits;
045import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
046import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter;
047
048import com.google.common.collect.Iterators;
049
050/**
051 * A set of named counters.
052 *
053 * <p><code>Counters</code> represent global counters, defined either by the
054 * Map-Reduce framework or applications. Each <code>Counter</code> can be of
055 * any {@link Enum} type.</p>
056 *
057 * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
058 * counters from a particular <code>Enum</code> class.
059 */
060@InterfaceAudience.Public
061@InterfaceStability.Stable
062public class Counters
063    extends AbstractCounters<Counters.Counter, Counters.Group> {
064  
065  public static int MAX_COUNTER_LIMIT = Limits.getCountersMax();
066  public static int MAX_GROUP_LIMIT = Limits.getGroupsMax();
067  private static HashMap<String, String> depricatedCounterMap =
068      new HashMap<String, String>();
069  
070  static {
071    initDepricatedMap();
072  }
073  
074  public Counters() {
075    super(groupFactory);
076  }
077
078  public Counters(org.apache.hadoop.mapreduce.Counters newCounters) {
079    super(newCounters, groupFactory);
080  }
081
082  @SuppressWarnings({ "deprecation" })
083  private static void initDepricatedMap() {
084    depricatedCounterMap.put(FileInputFormat.Counter.class.getName(),
085      FileInputFormatCounter.class.getName());
086    depricatedCounterMap.put(FileOutputFormat.Counter.class.getName(),
087      FileOutputFormatCounter.class.getName());
088    depricatedCounterMap.put(
089      org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.class
090        .getName(), FileInputFormatCounter.class.getName());
091    depricatedCounterMap.put(
092      org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.class
093        .getName(), FileOutputFormatCounter.class.getName());
094  }
095
096  private static String getNewGroupKey(String oldGroup) {
097    if (depricatedCounterMap.containsKey(oldGroup)) {
098      return depricatedCounterMap.get(oldGroup);
099    }
100    return null;
101  }
102  
103  /**
104   * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters
105   * @param newCounters new Counters
106   * @return old Counters instance corresponding to newCounters
107   */
108  static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) {
109    return new Counters(newCounters);
110  }
111
112  public synchronized Group getGroup(String groupName) {
113    return super.getGroup(groupName);
114  }
115
116  @SuppressWarnings("unchecked")
117  public synchronized Collection<String> getGroupNames() {
118    return IteratorUtils.toList(super.getGroupNames().iterator());
119  }
120
121  public synchronized String makeCompactString() {
122    StringBuilder builder = new StringBuilder();
123    boolean first = true;
124    for(Group group: this){
125      for(Counter counter: group) {
126        if (first) {
127          first = false;
128        } else {
129          builder.append(',');
130        }
131        builder.append(group.getDisplayName());
132        builder.append('.');
133        builder.append(counter.getDisplayName());
134        builder.append(':');
135        builder.append(counter.getCounter());
136      }
137    }
138    return builder.toString();
139  }
140  
141  /**
142   * A counter record, comprising its name and value.
143   */
144  @InterfaceAudience.Public
145  @InterfaceStability.Stable
146  public static class Counter implements org.apache.hadoop.mapreduce.Counter {
147    org.apache.hadoop.mapreduce.Counter realCounter;
148
149    Counter(org.apache.hadoop.mapreduce.Counter counter) {
150      this.realCounter = counter;
151    }
152
153    public Counter() {
154      this(new GenericCounter());
155    }
156
157    @SuppressWarnings("deprecation")
158    @Override
159    public void setDisplayName(String displayName) {
160      realCounter.setDisplayName(displayName);
161    }
162
163    @Override
164    public String getName() {
165      return realCounter.getName();
166    }
167
168    @Override
169    public String getDisplayName() {
170      return realCounter.getDisplayName();
171    }
172
173    @Override
174    public long getValue() {
175      return realCounter.getValue();
176    }
177
178    @Override
179    public void setValue(long value) {
180      realCounter.setValue(value);
181    }
182
183    @Override
184    public void increment(long incr) {
185      realCounter.increment(incr);
186    }
187
188    @Override
189    public void write(DataOutput out) throws IOException {
190      realCounter.write(out);
191    }
192
193    @Override
194    public void readFields(DataInput in) throws IOException {
195      realCounter.readFields(in);
196    }
197
198    /**
199     * Returns the compact stringified version of the counter in the format
200     * [(actual-name)(display-name)(value)]
201     * @return the stringified result
202     */
203    public String makeEscapedCompactString() {
204      return toEscapedCompactString(realCounter);
205    }
206
207    /**
208     * Checks for (content) equality of two (basic) counters
209     * @param counter to compare
210     * @return true if content equals
211     * @deprecated
212     */
213    @Deprecated
214    public boolean contentEquals(Counter counter) {
215      return realCounter.equals(counter.getUnderlyingCounter());
216    }
217
218    /**
219     * @return the value of the counter
220     */
221    public long getCounter() {
222      return realCounter.getValue();
223    }
224
225    @Override
226    public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() {
227      return realCounter;
228    }
229    
230    @Override
231    public synchronized boolean equals(Object genericRight) {
232      if (genericRight instanceof Counter) {
233        synchronized (genericRight) {
234          Counter right = (Counter) genericRight;
235          return getName().equals(right.getName()) &&
236                 getDisplayName().equals(right.getDisplayName()) &&
237                 getValue() == right.getValue();
238        }
239      }
240      return false;
241    }
242    
243    @Override
244    public int hashCode() {
245      return realCounter.hashCode();
246    }
247  }
248
249
250  /**
251   *  <code>Group</code> of counters, comprising of counters from a particular
252   *  counter {@link Enum} class.
253   *
254   *  <p><code>Group</code>handles localization of the class name and the
255   *  counter names.</p>
256   */
257  @InterfaceAudience.Public
258  @InterfaceStability.Stable
259  public static class Group implements CounterGroupBase<Counter> {
260    private CounterGroupBase<Counter> realGroup;
261    
262    protected Group() {
263      realGroup = null;
264    }
265    
266    Group(GenericGroup group) {
267      this.realGroup = group;
268    }
269    Group(FSGroupImpl group) {
270      this.realGroup = group;
271    }
272    
273    @SuppressWarnings({ "unchecked", "rawtypes" })
274    Group(FrameworkGroupImpl group) {
275      this.realGroup = group;
276    }
277    
278    /**
279     * @param counterName the name of the counter
280     * @return the value of the specified counter, or 0 if the counter does
281     * not exist.
282     */
283    public long getCounter(String counterName)  {
284      return getCounterValue(realGroup, counterName);
285    }
286
287    /**
288     * @return the compact stringified version of the group in the format
289     * {(actual-name)(display-name)(value)[][][]} where [] are compact strings
290     * for the counters within.
291     */
292    public String makeEscapedCompactString() {
293      return toEscapedCompactString(realGroup);
294    }
295
296    /**
297     * Get the counter for the given id and create it if it doesn't exist.
298     * @param id the numeric id of the counter within the group
299     * @param name the internal counter name
300     * @return the counter
301     * @deprecated use {@link #findCounter(String)} instead
302     */
303    @Deprecated
304    public Counter getCounter(int id, String name) {
305      return findCounter(name);
306    }
307
308    /**
309     * Get the counter for the given name and create it if it doesn't exist.
310     * @param name the internal counter name
311     * @return the counter
312     */
313    public Counter getCounterForName(String name) {
314      return findCounter(name);
315    }
316
317    @Override
318    public void write(DataOutput out) throws IOException {
319     realGroup.write(out); 
320    }
321
322    @Override
323    public void readFields(DataInput in) throws IOException {
324      realGroup.readFields(in);
325    }
326
327    @Override
328    public Iterator<Counter> iterator() {
329      return realGroup.iterator();
330    }
331
332    @Override
333    public String getName() {
334      return realGroup.getName();
335    }
336
337    @Override
338    public String getDisplayName() {
339      return realGroup.getDisplayName();
340    }
341
342    @Override
343    public void setDisplayName(String displayName) {
344      realGroup.setDisplayName(displayName);
345    }
346
347    @Override
348    public void addCounter(Counter counter) {
349      realGroup.addCounter(counter);
350    }
351
352    @Override
353    public Counter addCounter(String name, String displayName, long value) {
354      return realGroup.addCounter(name, displayName, value);
355    }
356
357    @Override
358    public Counter findCounter(String counterName, String displayName) {
359      return realGroup.findCounter(counterName, displayName);
360    }
361
362    @Override
363    public Counter findCounter(String counterName, boolean create) {
364      return realGroup.findCounter(counterName, create);
365    }
366
367    @Override
368    public Counter findCounter(String counterName) {
369      return realGroup.findCounter(counterName);
370    }
371
372    @Override
373    public int size() {
374      return realGroup.size();
375    }
376
377    @Override
378    public void incrAllCounters(CounterGroupBase<Counter> rightGroup) {
379      realGroup.incrAllCounters(rightGroup);
380    }
381    
382    @Override
383    public CounterGroupBase<Counter> getUnderlyingGroup() {
384      return realGroup;
385    }
386
387    @Override
388    public synchronized boolean equals(Object genericRight) {
389      if (genericRight instanceof CounterGroupBase<?>) {
390        @SuppressWarnings("unchecked")
391        CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>) 
392        genericRight).getUnderlyingGroup();
393        return Iterators.elementsEqual(iterator(), right.iterator());
394      }
395      return false;
396    }
397
398    @Override
399    public int hashCode() {
400      return realGroup.hashCode();
401    }
402  }
403
404  // All the group impls need this for legacy group interface
405  static long getCounterValue(CounterGroupBase<Counter> group, String counterName) {
406    Counter counter = group.findCounter(counterName, false);
407    if (counter != null) return counter.getValue();
408    return 0L;
409  }
410
411  // Mix the generic group implementation into the Group interface
412  private static class GenericGroup extends AbstractCounterGroup<Counter> {
413
414    GenericGroup(String name, String displayName, Limits limits) {
415      super(name, displayName, limits);
416    }
417
418    @Override
419    protected Counter newCounter(String counterName, String displayName,
420                                 long value) {
421      return new Counter(new GenericCounter(counterName, displayName, value));
422    }
423
424    @Override
425    protected Counter newCounter() {
426      return new Counter();
427    }
428    
429    @Override
430    public CounterGroupBase<Counter> getUnderlyingGroup() {
431     return this;
432    }
433  }
434
435  // Mix the framework group implementation into the Group interface
436  private static class FrameworkGroupImpl<T extends Enum<T>>
437      extends FrameworkCounterGroup<T, Counter> {
438
439    FrameworkGroupImpl(Class<T> cls) {
440      super(cls);
441    }
442
443    @Override
444    protected Counter newCounter(T key) {
445      return new Counter(new FrameworkCounter<T>(key, getName()));
446    }
447
448    @Override
449    public CounterGroupBase<Counter> getUnderlyingGroup() {
450      return this;
451    }
452  }
453
454  // Mix the file system counter group implementation into the Group interface
455  private static class FSGroupImpl extends FileSystemCounterGroup<Counter> {
456
457    @Override
458    protected Counter newCounter(String scheme, FileSystemCounter key) {
459      return new Counter(new FSCounter(scheme, key));
460    }
461
462    @Override
463    public CounterGroupBase<Counter> getUnderlyingGroup() {
464      return this;
465    }
466  }
467
468  public synchronized Counter findCounter(String group, String name) {
469    if (name.equals("MAP_INPUT_BYTES")) {
470      LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
471               "Use FileInputFormatCounters as group name and " +
472               " BYTES_READ as counter name instead");
473      return findCounter(FileInputFormatCounter.BYTES_READ);
474    }
475    String newGroupKey = getNewGroupKey(group);
476    if (newGroupKey != null) {
477      group = newGroupKey;
478    }
479    return getGroup(group).getCounterForName(name);
480  }
481
482  /**
483   * Provide factory methods for counter group factory implementation.
484   * See also the GroupFactory in
485   *  {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters}
486   */
487  static class GroupFactory extends CounterGroupFactory<Counter, Group> {
488
489    @Override
490    protected <T extends Enum<T>>
491    FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) {
492      return new FrameworkGroupFactory<Group>() {
493        @Override public Group newGroup(String name) {
494          return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package
495        }
496      };
497    }
498
499    @Override
500    protected Group newGenericGroup(String name, String displayName,
501                                    Limits limits) {
502      return new Group(new GenericGroup(name, displayName, limits));
503    }
504
505    @Override
506    protected Group newFileSystemGroup() {
507      return new Group(new FSGroupImpl());
508    }
509  }
510
511  private static final GroupFactory groupFactory = new GroupFactory();
512
513  /**
514   * Find a counter by using strings
515   * @param group the name of the group
516   * @param id the id of the counter within the group (0 to N-1)
517   * @param name the internal name of the counter
518   * @return the counter for that name
519   * @deprecated use {@link #findCounter(String, String)} instead
520   */
521  @Deprecated
522  public Counter findCounter(String group, int id, String name) {
523    return findCounter(group, name);
524  }
525
526  /**
527   * Increments the specified counter by the specified amount, creating it if
528   * it didn't already exist.
529   * @param key identifies a counter
530   * @param amount amount by which counter is to be incremented
531   */
532  public void incrCounter(Enum<?> key, long amount) {
533    findCounter(key).increment(amount);
534  }
535
536  /**
537   * Increments the specified counter by the specified amount, creating it if
538   * it didn't already exist.
539   * @param group the name of the group
540   * @param counter the internal name of the counter
541   * @param amount amount by which counter is to be incremented
542   */
543  public void incrCounter(String group, String counter, long amount) {
544    findCounter(group, counter).increment(amount);
545  }
546
547  /**
548   * Returns current value of the specified counter, or 0 if the counter
549   * does not exist.
550   * @param key the counter enum to lookup
551   * @return the counter value or 0 if counter not found
552   */
553  public synchronized long getCounter(Enum<?> key) {
554    return findCounter(key).getValue();
555  }
556
557  /**
558   * Increments multiple counters by their amounts in another Counters
559   * instance.
560   * @param other the other Counters instance
561   */
562  public synchronized void incrAllCounters(Counters other) {
563    for (Group otherGroup: other) {
564      Group group = getGroup(otherGroup.getName());
565      group.setDisplayName(otherGroup.getDisplayName());
566      for (Counter otherCounter : otherGroup) {
567        Counter counter = group.getCounterForName(otherCounter.getName());
568        counter.setDisplayName(otherCounter.getDisplayName());
569        counter.increment(otherCounter.getValue());
570      }
571    }
572  }
573
574  /**
575   * @return the total number of counters
576   * @deprecated use {@link #countCounters()} instead
577   */
578  public int size() {
579    return countCounters();
580  }
581
582  /**
583   * Convenience method for computing the sum of two sets of counters.
584   * @param a the first counters
585   * @param b the second counters
586   * @return a new summed counters object
587   */
588  public static Counters sum(Counters a, Counters b) {
589    Counters counters = new Counters();
590    counters.incrAllCounters(a);
591    counters.incrAllCounters(b);
592    return counters;
593  }
594
595  /**
596   * Logs the current counter values.
597   * @param log The log to use.
598   */
599  public void log(Log log) {
600    log.info("Counters: " + size());
601    for(Group group: this) {
602      log.info("  " + group.getDisplayName());
603      for (Counter counter: group) {
604        log.info("    " + counter.getDisplayName() + "=" +
605                 counter.getCounter());
606      }
607    }
608  }
609
610  /**
611   * Represent the counter in a textual format that can be converted back to
612   * its object form
613   * @return the string in the following format
614   * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*
615   */
616  public String makeEscapedCompactString() {
617    return toEscapedCompactString(this);
618  }
619
620  /**
621   * Convert a stringified (by {@link #makeEscapedCompactString()} counter
622   * representation into a counter object.
623   * @param compactString to parse
624   * @return a new counters object
625   * @throws ParseException
626   */
627  public static Counters fromEscapedCompactString(String compactString)
628      throws ParseException {
629    return parseEscapedCompactString(compactString, new Counters());
630  }
631
632  /**
633   * Counter exception thrown when the number of counters exceed the limit
634   */
635  public static class CountersExceededException extends RuntimeException {
636
637    private static final long serialVersionUID = 1L;
638
639    public CountersExceededException(String msg) {
640      super(msg);
641    }
642
643    // Only allows chaining of related exceptions
644    public CountersExceededException(CountersExceededException cause) {
645      super(cause);
646    }
647  }
648}